From eb5ca80b74f84b0a30a9e367bdd735c5f60a51d6 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 13 Jun 2026 17:28:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: terasut/gkd-qwen-2.5-0.5b-base_v5_from1.5b_eff32 Source: Original Platform --- .gitattributes | 39 + README.md | 60 + chat_template.jinja | 56 + checkpoint-1750/chat_template.jinja | 56 + checkpoint-1750/config.json | 58 + checkpoint-1750/generation_config.json | 10 + checkpoint-1750/model.safetensors | 3 + checkpoint-1750/optimizer.pt | 3 + checkpoint-1750/rng_state.pth | 3 + checkpoint-1750/scheduler.pt | 3 + checkpoint-1750/tokenizer.json | 3 + checkpoint-1750/tokenizer_config.json | 30 + checkpoint-1750/trainer_state.json | 1259 ++++++++++++++++ checkpoint-1750/training_args.bin | 3 + checkpoint-2625/chat_template.jinja | 56 + checkpoint-2625/config.json | 58 + checkpoint-2625/generation_config.json | 10 + checkpoint-2625/model.safetensors | 3 + checkpoint-2625/optimizer.pt | 3 + checkpoint-2625/rng_state.pth | 3 + checkpoint-2625/scheduler.pt | 3 + checkpoint-2625/tokenizer.json | 3 + checkpoint-2625/tokenizer_config.json | 30 + checkpoint-2625/trainer_state.json | 1868 ++++++++++++++++++++++++ checkpoint-2625/training_args.bin | 3 + checkpoint-875/chat_template.jinja | 56 + checkpoint-875/config.json | 58 + checkpoint-875/generation_config.json | 10 + checkpoint-875/model.safetensors | 3 + checkpoint-875/optimizer.pt | 3 + checkpoint-875/rng_state.pth | 3 + checkpoint-875/scheduler.pt | 3 + checkpoint-875/tokenizer.json | 3 + checkpoint-875/tokenizer_config.json | 30 + checkpoint-875/trainer_state.json | 643 ++++++++ checkpoint-875/training_args.bin | 3 + config.json | 58 + generation_config.json | 10 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 30 + training_args.bin | 3 + 42 files changed, 4548 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 checkpoint-1750/chat_template.jinja create mode 100644 checkpoint-1750/config.json create mode 100644 checkpoint-1750/generation_config.json create mode 100644 checkpoint-1750/model.safetensors create mode 100644 checkpoint-1750/optimizer.pt create mode 100644 checkpoint-1750/rng_state.pth create mode 100644 checkpoint-1750/scheduler.pt create mode 100644 checkpoint-1750/tokenizer.json create mode 100644 checkpoint-1750/tokenizer_config.json create mode 100644 checkpoint-1750/trainer_state.json create mode 100644 checkpoint-1750/training_args.bin create mode 100644 checkpoint-2625/chat_template.jinja create mode 100644 checkpoint-2625/config.json create mode 100644 checkpoint-2625/generation_config.json create mode 100644 checkpoint-2625/model.safetensors create mode 100644 checkpoint-2625/optimizer.pt create mode 100644 checkpoint-2625/rng_state.pth create mode 100644 checkpoint-2625/scheduler.pt create mode 100644 checkpoint-2625/tokenizer.json create mode 100644 checkpoint-2625/tokenizer_config.json create mode 100644 checkpoint-2625/trainer_state.json create mode 100644 checkpoint-2625/training_args.bin create mode 100644 checkpoint-875/chat_template.jinja create mode 100644 checkpoint-875/config.json create mode 100644 checkpoint-875/generation_config.json create mode 100644 checkpoint-875/model.safetensors create mode 100644 checkpoint-875/optimizer.pt create mode 100644 checkpoint-875/rng_state.pth create mode 100644 checkpoint-875/scheduler.pt create mode 100644 checkpoint-875/tokenizer.json create mode 100644 checkpoint-875/tokenizer_config.json create mode 100644 checkpoint-875/trainer_state.json create mode 100644 checkpoint-875/training_args.bin create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3b6fd86 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,39 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-1750/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-2625/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-875/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..b42bc18 --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +--- +library_name: transformers +model_name: gkd-qwen-2.5-0.5b-base_v5_from1.5b_eff32 +tags: +- generated_from_trainer +- gkd +- trl +licence: license +--- + +# Model Card for gkd-qwen-2.5-0.5b-base_v5_from1.5b_eff32 + +This model is a fine-tuned version of [None](https://huggingface.co/None). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/terasut-workspace/ai-patternrecog/runs/tcq3qf7x) + + + +This model was trained with GKD, a method introduced in [On-Policy Distillation of Language Models: Learning from Self-Generated Mistakes](https://huggingface.co/papers/2306.13649). + +## Citations + +Cite GKD as: + +```bibtex +@inproceedings{agarwal2024on-policy, + title = {{On-Policy Distillation of Language Models: Learning from Self-Generated Mistakes}}, + author = {Rishabh Agarwal and Nino Vieillard and Yongchao Zhou and Piotr Stanczyk and Sabela Ramos Garea and Matthieu Geist and Olivier Bachem}, + year = 2024, + booktitle = {The Twelfth International Conference on Learning Representations, {ICLR} 2024, Vienna, Austria, May 7-11, 2024}, + publisher = {OpenReview.net}, + url = {https://openreview.net/forum?id=3zKtaqxLhW}, +} +``` + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..64983ee --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,56 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" and not message.tool_calls %} + {{- '<|im_start|>assistant\n' }}{% generation %}{{- message.content + '<|im_end|>\n' }}{% endgeneration %} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>assistant' }}{% generation %} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }}{% endgeneration %} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-1750/chat_template.jinja b/checkpoint-1750/chat_template.jinja new file mode 100644 index 0000000..64983ee --- /dev/null +++ b/checkpoint-1750/chat_template.jinja @@ -0,0 +1,56 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" and not message.tool_calls %} + {{- '<|im_start|>assistant\n' }}{% generation %}{{- message.content + '<|im_end|>\n' }}{% endgeneration %} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>assistant' }}{% generation %} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }}{% endgeneration %} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-1750/config.json b/checkpoint-1750/config.json new file mode 100644 index 0000000..c097f0c --- /dev/null +++ b/checkpoint-1750/config.json @@ -0,0 +1,58 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.5.3", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-1750/generation_config.json b/checkpoint-1750/generation_config.json new file mode 100644 index 0000000..61f88e1 --- /dev/null +++ b/checkpoint-1750/generation_config.json @@ -0,0 +1,10 @@ +{ + "do_sample": false, + "eos_token_id": [ + 151645 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.5.3", + "use_cache": false +} diff --git a/checkpoint-1750/model.safetensors b/checkpoint-1750/model.safetensors new file mode 100644 index 0000000..4aa0621 --- /dev/null +++ b/checkpoint-1750/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:408294005c9b105e91cc612417aa08e1a6ea22f32437ba8ba9bd43c828a33c49 +size 988097824 diff --git a/checkpoint-1750/optimizer.pt b/checkpoint-1750/optimizer.pt new file mode 100644 index 0000000..488cb9e --- /dev/null +++ b/checkpoint-1750/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acacf31674c6e809b2dda67ed66a796f016426232045ee0ec4436a30a765aba0 +size 1976378699 diff --git a/checkpoint-1750/rng_state.pth b/checkpoint-1750/rng_state.pth new file mode 100644 index 0000000..a5f8ba5 --- /dev/null +++ b/checkpoint-1750/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a759efdc3644a7047be4e1cf0e339eb0c35a93c8f494a96325adfe2e6bd10e7 +size 14645 diff --git a/checkpoint-1750/scheduler.pt b/checkpoint-1750/scheduler.pt new file mode 100644 index 0000000..9885f56 --- /dev/null +++ b/checkpoint-1750/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc53ebd3c8ca395477da766ae7fa14a1cd87577954defcd87cfec2f1d6fd424a +size 1465 diff --git a/checkpoint-1750/tokenizer.json b/checkpoint-1750/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-1750/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-1750/tokenizer_config.json b/checkpoint-1750/tokenizer_config.json new file mode 100644 index 0000000..0385b12 --- /dev/null +++ b/checkpoint-1750/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-1750/trainer_state.json b/checkpoint-1750/trainer_state.json new file mode 100644 index 0000000..881aa6e --- /dev/null +++ b/checkpoint-1750/trainer_state.json @@ -0,0 +1,1259 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 1750, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.011428571428571429, + "grad_norm": 3.078125, + "learning_rate": 1.9931428571428572e-05, + "loss": 7.941626739501953, + "step": 10 + }, + { + "epoch": 0.022857142857142857, + "grad_norm": 2.9375, + "learning_rate": 1.9855238095238097e-05, + "loss": 8.19613037109375, + "step": 20 + }, + { + "epoch": 0.03428571428571429, + "grad_norm": 3.5625, + "learning_rate": 1.977904761904762e-05, + "loss": 7.826921081542968, + "step": 30 + }, + { + "epoch": 0.045714285714285714, + "grad_norm": 3.15625, + "learning_rate": 1.9702857142857144e-05, + "loss": 7.580943298339844, + "step": 40 + }, + { + "epoch": 0.05714285714285714, + "grad_norm": 2.515625, + "learning_rate": 1.9626666666666666e-05, + "loss": 7.483664703369141, + "step": 50 + }, + { + "epoch": 0.06857142857142857, + "grad_norm": 3.859375, + "learning_rate": 1.955047619047619e-05, + "loss": 7.480722808837891, + "step": 60 + }, + { + "epoch": 0.08, + "grad_norm": 3.46875, + "learning_rate": 1.9474285714285717e-05, + "loss": 7.279670715332031, + "step": 70 + }, + { + "epoch": 0.09142857142857143, + "grad_norm": 2.5625, + "learning_rate": 1.9398095238095242e-05, + "loss": 7.584724426269531, + "step": 80 + }, + { + "epoch": 0.10285714285714286, + "grad_norm": 2.515625, + "learning_rate": 1.9321904761904764e-05, + "loss": 7.565158081054688, + "step": 90 + }, + { + "epoch": 0.11428571428571428, + "grad_norm": 3.5625, + "learning_rate": 1.924571428571429e-05, + "loss": 7.366996002197266, + "step": 100 + }, + { + "epoch": 0.12571428571428572, + "grad_norm": 3.109375, + "learning_rate": 1.916952380952381e-05, + "loss": 7.434650421142578, + "step": 110 + }, + { + "epoch": 0.13714285714285715, + "grad_norm": 3.328125, + "learning_rate": 1.9093333333333336e-05, + "loss": 6.9210762023925785, + "step": 120 + }, + { + "epoch": 0.14857142857142858, + "grad_norm": 3.015625, + "learning_rate": 1.9017142857142858e-05, + "loss": 7.027304840087891, + "step": 130 + }, + { + "epoch": 0.16, + "grad_norm": 3.09375, + "learning_rate": 1.8940952380952383e-05, + "loss": 7.241950225830078, + "step": 140 + }, + { + "epoch": 0.17142857142857143, + "grad_norm": 3.0, + "learning_rate": 1.8864761904761905e-05, + "loss": 7.158929443359375, + "step": 150 + }, + { + "epoch": 0.18285714285714286, + "grad_norm": 2.890625, + "learning_rate": 1.878857142857143e-05, + "loss": 6.856580352783203, + "step": 160 + }, + { + "epoch": 0.19428571428571428, + "grad_norm": 4.09375, + "learning_rate": 1.871238095238095e-05, + "loss": 6.740969848632813, + "step": 170 + }, + { + "epoch": 0.2057142857142857, + "grad_norm": 3.40625, + "learning_rate": 1.8636190476190477e-05, + "loss": 7.115393829345703, + "step": 180 + }, + { + "epoch": 0.21714285714285714, + "grad_norm": 2.9375, + "learning_rate": 1.8560000000000002e-05, + "loss": 7.437998199462891, + "step": 190 + }, + { + "epoch": 0.22857142857142856, + "grad_norm": 2.96875, + "learning_rate": 1.8483809523809527e-05, + "loss": 6.879551696777344, + "step": 200 + }, + { + "epoch": 0.24, + "grad_norm": 2.890625, + "learning_rate": 1.840761904761905e-05, + "loss": 6.8273162841796875, + "step": 210 + }, + { + "epoch": 0.25142857142857145, + "grad_norm": 3.0, + "learning_rate": 1.8331428571428574e-05, + "loss": 6.558704376220703, + "step": 220 + }, + { + "epoch": 0.26285714285714284, + "grad_norm": 4.40625, + "learning_rate": 1.8255238095238096e-05, + "loss": 7.156157684326172, + "step": 230 + }, + { + "epoch": 0.2742857142857143, + "grad_norm": 2.671875, + "learning_rate": 1.817904761904762e-05, + "loss": 6.985882568359375, + "step": 240 + }, + { + "epoch": 0.2857142857142857, + "grad_norm": 2.46875, + "learning_rate": 1.8102857142857143e-05, + "loss": 7.106211090087891, + "step": 250 + }, + { + "epoch": 0.29714285714285715, + "grad_norm": 2.984375, + "learning_rate": 1.8026666666666668e-05, + "loss": 7.014789581298828, + "step": 260 + }, + { + "epoch": 0.30857142857142855, + "grad_norm": 2.5, + "learning_rate": 1.7950476190476193e-05, + "loss": 6.632127380371093, + "step": 270 + }, + { + "epoch": 0.32, + "grad_norm": 2.84375, + "learning_rate": 1.7874285714285715e-05, + "loss": 6.882701110839844, + "step": 280 + }, + { + "epoch": 0.3314285714285714, + "grad_norm": 3.78125, + "learning_rate": 1.779809523809524e-05, + "loss": 6.729277801513672, + "step": 290 + }, + { + "epoch": 0.34285714285714286, + "grad_norm": 3.578125, + "learning_rate": 1.7721904761904762e-05, + "loss": 6.794551849365234, + "step": 300 + }, + { + "epoch": 0.35428571428571426, + "grad_norm": 2.828125, + "learning_rate": 1.7645714285714287e-05, + "loss": 6.83813247680664, + "step": 310 + }, + { + "epoch": 0.3657142857142857, + "grad_norm": 2.875, + "learning_rate": 1.7569523809523812e-05, + "loss": 6.796338653564453, + "step": 320 + }, + { + "epoch": 0.37714285714285717, + "grad_norm": 2.90625, + "learning_rate": 1.7493333333333334e-05, + "loss": 6.7025390625, + "step": 330 + }, + { + "epoch": 0.38857142857142857, + "grad_norm": 2.953125, + "learning_rate": 1.741714285714286e-05, + "loss": 6.541476440429688, + "step": 340 + }, + { + "epoch": 0.4, + "grad_norm": 2.59375, + "learning_rate": 1.734095238095238e-05, + "loss": 6.812630462646484, + "step": 350 + }, + { + "epoch": 0.4114285714285714, + "grad_norm": 3.609375, + "learning_rate": 1.7264761904761906e-05, + "loss": 6.889384460449219, + "step": 360 + }, + { + "epoch": 0.4228571428571429, + "grad_norm": 4.03125, + "learning_rate": 1.718857142857143e-05, + "loss": 7.129203796386719, + "step": 370 + }, + { + "epoch": 0.4342857142857143, + "grad_norm": 2.40625, + "learning_rate": 1.7112380952380953e-05, + "loss": 6.382967376708985, + "step": 380 + }, + { + "epoch": 0.44571428571428573, + "grad_norm": 3.515625, + "learning_rate": 1.703619047619048e-05, + "loss": 6.663212585449219, + "step": 390 + }, + { + "epoch": 0.45714285714285713, + "grad_norm": 2.90625, + "learning_rate": 1.696e-05, + "loss": 6.636187744140625, + "step": 400 + }, + { + "epoch": 0.4685714285714286, + "grad_norm": 3.1875, + "learning_rate": 1.6883809523809525e-05, + "loss": 6.64324722290039, + "step": 410 + }, + { + "epoch": 0.48, + "grad_norm": 2.828125, + "learning_rate": 1.6807619047619047e-05, + "loss": 6.663475036621094, + "step": 420 + }, + { + "epoch": 0.49142857142857144, + "grad_norm": 3.328125, + "learning_rate": 1.6731428571428572e-05, + "loss": 6.452519226074219, + "step": 430 + }, + { + "epoch": 0.5028571428571429, + "grad_norm": 2.421875, + "learning_rate": 1.6655238095238098e-05, + "loss": 6.683634948730469, + "step": 440 + }, + { + "epoch": 0.5142857142857142, + "grad_norm": 3.046875, + "learning_rate": 1.6579047619047623e-05, + "loss": 6.780387878417969, + "step": 450 + }, + { + "epoch": 0.5257142857142857, + "grad_norm": 2.53125, + "learning_rate": 1.6502857142857145e-05, + "loss": 7.1892578125, + "step": 460 + }, + { + "epoch": 0.5371428571428571, + "grad_norm": 2.71875, + "learning_rate": 1.642666666666667e-05, + "loss": 6.890229034423828, + "step": 470 + }, + { + "epoch": 0.5485714285714286, + "grad_norm": 3.140625, + "learning_rate": 1.635047619047619e-05, + "loss": 6.747575378417968, + "step": 480 + }, + { + "epoch": 0.56, + "grad_norm": 3.3125, + "learning_rate": 1.6274285714285717e-05, + "loss": 6.98467788696289, + "step": 490 + }, + { + "epoch": 0.5714285714285714, + "grad_norm": 2.890625, + "learning_rate": 1.619809523809524e-05, + "loss": 6.455801391601563, + "step": 500 + }, + { + "epoch": 0.5828571428571429, + "grad_norm": 2.578125, + "learning_rate": 1.6121904761904764e-05, + "loss": 6.525889587402344, + "step": 510 + }, + { + "epoch": 0.5942857142857143, + "grad_norm": 3.375, + "learning_rate": 1.6045714285714286e-05, + "loss": 6.479692840576172, + "step": 520 + }, + { + "epoch": 0.6057142857142858, + "grad_norm": 2.875, + "learning_rate": 1.596952380952381e-05, + "loss": 6.6360313415527346, + "step": 530 + }, + { + "epoch": 0.6171428571428571, + "grad_norm": 2.703125, + "learning_rate": 1.5893333333333333e-05, + "loss": 6.547400665283203, + "step": 540 + }, + { + "epoch": 0.6285714285714286, + "grad_norm": 2.9375, + "learning_rate": 1.5817142857142858e-05, + "loss": 6.748792266845703, + "step": 550 + }, + { + "epoch": 0.64, + "grad_norm": 3.28125, + "learning_rate": 1.5740952380952383e-05, + "loss": 6.727365112304687, + "step": 560 + }, + { + "epoch": 0.6514285714285715, + "grad_norm": 2.984375, + "learning_rate": 1.5664761904761908e-05, + "loss": 6.611011505126953, + "step": 570 + }, + { + "epoch": 0.6628571428571428, + "grad_norm": 2.984375, + "learning_rate": 1.558857142857143e-05, + "loss": 6.471088409423828, + "step": 580 + }, + { + "epoch": 0.6742857142857143, + "grad_norm": 2.84375, + "learning_rate": 1.5512380952380955e-05, + "loss": 6.472038269042969, + "step": 590 + }, + { + "epoch": 0.6857142857142857, + "grad_norm": 3.15625, + "learning_rate": 1.5436190476190477e-05, + "loss": 6.747550201416016, + "step": 600 + }, + { + "epoch": 0.6971428571428572, + "grad_norm": 2.75, + "learning_rate": 1.5360000000000002e-05, + "loss": 6.638180541992187, + "step": 610 + }, + { + "epoch": 0.7085714285714285, + "grad_norm": 3.078125, + "learning_rate": 1.5283809523809524e-05, + "loss": 6.535469818115234, + "step": 620 + }, + { + "epoch": 0.72, + "grad_norm": 3.109375, + "learning_rate": 1.5207619047619049e-05, + "loss": 6.981074523925781, + "step": 630 + }, + { + "epoch": 0.7314285714285714, + "grad_norm": 2.78125, + "learning_rate": 1.5131428571428572e-05, + "loss": 6.307097625732422, + "step": 640 + }, + { + "epoch": 0.7428571428571429, + "grad_norm": 2.8125, + "learning_rate": 1.5055238095238096e-05, + "loss": 6.617960357666016, + "step": 650 + }, + { + "epoch": 0.7542857142857143, + "grad_norm": 2.9375, + "learning_rate": 1.497904761904762e-05, + "loss": 6.845116424560547, + "step": 660 + }, + { + "epoch": 0.7657142857142857, + "grad_norm": 2.140625, + "learning_rate": 1.4902857142857143e-05, + "loss": 6.583855438232422, + "step": 670 + }, + { + "epoch": 0.7771428571428571, + "grad_norm": 2.703125, + "learning_rate": 1.4826666666666666e-05, + "loss": 6.485343933105469, + "step": 680 + }, + { + "epoch": 0.7885714285714286, + "grad_norm": 2.96875, + "learning_rate": 1.475047619047619e-05, + "loss": 6.277442169189453, + "step": 690 + }, + { + "epoch": 0.8, + "grad_norm": 3.078125, + "learning_rate": 1.4674285714285717e-05, + "loss": 6.420030212402343, + "step": 700 + }, + { + "epoch": 0.8114285714285714, + "grad_norm": 2.625, + "learning_rate": 1.459809523809524e-05, + "loss": 6.779734802246094, + "step": 710 + }, + { + "epoch": 0.8228571428571428, + "grad_norm": 3.28125, + "learning_rate": 1.4521904761904764e-05, + "loss": 6.647151947021484, + "step": 720 + }, + { + "epoch": 0.8342857142857143, + "grad_norm": 2.5625, + "learning_rate": 1.4445714285714287e-05, + "loss": 6.453370666503906, + "step": 730 + }, + { + "epoch": 0.8457142857142858, + "grad_norm": 2.5625, + "learning_rate": 1.436952380952381e-05, + "loss": 6.6205802917480465, + "step": 740 + }, + { + "epoch": 0.8571428571428571, + "grad_norm": 2.265625, + "learning_rate": 1.4293333333333334e-05, + "loss": 6.353859329223633, + "step": 750 + }, + { + "epoch": 0.8685714285714285, + "grad_norm": 3.015625, + "learning_rate": 1.4217142857142858e-05, + "loss": 6.357454299926758, + "step": 760 + }, + { + "epoch": 0.88, + "grad_norm": 3.03125, + "learning_rate": 1.4140952380952381e-05, + "loss": 6.682913208007813, + "step": 770 + }, + { + "epoch": 0.8914285714285715, + "grad_norm": 2.625, + "learning_rate": 1.4064761904761905e-05, + "loss": 6.127779769897461, + "step": 780 + }, + { + "epoch": 0.9028571428571428, + "grad_norm": 4.375, + "learning_rate": 1.398857142857143e-05, + "loss": 6.364295578002929, + "step": 790 + }, + { + "epoch": 0.9142857142857143, + "grad_norm": 2.8125, + "learning_rate": 1.3912380952380953e-05, + "loss": 6.2481742858886715, + "step": 800 + }, + { + "epoch": 0.9257142857142857, + "grad_norm": 2.875, + "learning_rate": 1.3836190476190477e-05, + "loss": 6.126508712768555, + "step": 810 + }, + { + "epoch": 0.9371428571428572, + "grad_norm": 3.0, + "learning_rate": 1.376e-05, + "loss": 6.082854080200195, + "step": 820 + }, + { + "epoch": 0.9485714285714286, + "grad_norm": 3.09375, + "learning_rate": 1.3683809523809526e-05, + "loss": 6.128392028808594, + "step": 830 + }, + { + "epoch": 0.96, + "grad_norm": 3.40625, + "learning_rate": 1.3607619047619049e-05, + "loss": 6.288749313354492, + "step": 840 + }, + { + "epoch": 0.9714285714285714, + "grad_norm": 2.78125, + "learning_rate": 1.3531428571428573e-05, + "loss": 6.770208740234375, + "step": 850 + }, + { + "epoch": 0.9828571428571429, + "grad_norm": 2.96875, + "learning_rate": 1.3455238095238096e-05, + "loss": 6.299668502807617, + "step": 860 + }, + { + "epoch": 0.9942857142857143, + "grad_norm": 2.765625, + "learning_rate": 1.3379047619047621e-05, + "loss": 6.4255424499511715, + "step": 870 + }, + { + "epoch": 1.0057142857142858, + "grad_norm": 2.546875, + "learning_rate": 1.3302857142857145e-05, + "loss": 6.330821228027344, + "step": 880 + }, + { + "epoch": 1.0171428571428571, + "grad_norm": 2.515625, + "learning_rate": 1.3226666666666668e-05, + "loss": 6.2308086395263675, + "step": 890 + }, + { + "epoch": 1.0285714285714285, + "grad_norm": 2.34375, + "learning_rate": 1.3150476190476192e-05, + "loss": 6.077074432373047, + "step": 900 + }, + { + "epoch": 1.04, + "grad_norm": 3.28125, + "learning_rate": 1.3074285714285715e-05, + "loss": 6.353485488891602, + "step": 910 + }, + { + "epoch": 1.0514285714285714, + "grad_norm": 3.28125, + "learning_rate": 1.2998095238095239e-05, + "loss": 6.212256622314453, + "step": 920 + }, + { + "epoch": 1.062857142857143, + "grad_norm": 2.46875, + "learning_rate": 1.2921904761904762e-05, + "loss": 6.344943618774414, + "step": 930 + }, + { + "epoch": 1.0742857142857143, + "grad_norm": 3.3125, + "learning_rate": 1.2845714285714286e-05, + "loss": 6.49066390991211, + "step": 940 + }, + { + "epoch": 1.0857142857142856, + "grad_norm": 2.703125, + "learning_rate": 1.2769523809523811e-05, + "loss": 6.266908645629883, + "step": 950 + }, + { + "epoch": 1.0971428571428572, + "grad_norm": 3.421875, + "learning_rate": 1.2693333333333336e-05, + "loss": 6.110383987426758, + "step": 960 + }, + { + "epoch": 1.1085714285714285, + "grad_norm": 2.328125, + "learning_rate": 1.261714285714286e-05, + "loss": 6.216197204589844, + "step": 970 + }, + { + "epoch": 1.12, + "grad_norm": 3.09375, + "learning_rate": 1.2540952380952383e-05, + "loss": 6.426862335205078, + "step": 980 + }, + { + "epoch": 1.1314285714285715, + "grad_norm": 2.890625, + "learning_rate": 1.2464761904761907e-05, + "loss": 6.450955200195312, + "step": 990 + }, + { + "epoch": 1.1428571428571428, + "grad_norm": 3.78125, + "learning_rate": 1.238857142857143e-05, + "loss": 6.348379135131836, + "step": 1000 + }, + { + "epoch": 1.1542857142857144, + "grad_norm": 3.25, + "learning_rate": 1.2312380952380953e-05, + "loss": 6.226791763305664, + "step": 1010 + }, + { + "epoch": 1.1657142857142857, + "grad_norm": 2.734375, + "learning_rate": 1.2236190476190477e-05, + "loss": 6.333163070678711, + "step": 1020 + }, + { + "epoch": 1.177142857142857, + "grad_norm": 2.796875, + "learning_rate": 1.216e-05, + "loss": 6.321551895141601, + "step": 1030 + }, + { + "epoch": 1.1885714285714286, + "grad_norm": 4.09375, + "learning_rate": 1.2083809523809524e-05, + "loss": 6.385534286499023, + "step": 1040 + }, + { + "epoch": 1.2, + "grad_norm": 3.203125, + "learning_rate": 1.2007619047619047e-05, + "loss": 6.097745895385742, + "step": 1050 + }, + { + "epoch": 1.2114285714285715, + "grad_norm": 2.953125, + "learning_rate": 1.1931428571428571e-05, + "loss": 6.247037124633789, + "step": 1060 + }, + { + "epoch": 1.2228571428571429, + "grad_norm": 2.40625, + "learning_rate": 1.1855238095238094e-05, + "loss": 6.404273223876953, + "step": 1070 + }, + { + "epoch": 1.2342857142857142, + "grad_norm": 3.4375, + "learning_rate": 1.1779047619047621e-05, + "loss": 6.110296630859375, + "step": 1080 + }, + { + "epoch": 1.2457142857142858, + "grad_norm": 2.953125, + "learning_rate": 1.1702857142857145e-05, + "loss": 6.178505325317383, + "step": 1090 + }, + { + "epoch": 1.2571428571428571, + "grad_norm": 2.84375, + "learning_rate": 1.1626666666666668e-05, + "loss": 6.133485412597656, + "step": 1100 + }, + { + "epoch": 1.2685714285714287, + "grad_norm": 2.59375, + "learning_rate": 1.1550476190476192e-05, + "loss": 6.316691589355469, + "step": 1110 + }, + { + "epoch": 1.28, + "grad_norm": 3.03125, + "learning_rate": 1.1474285714285715e-05, + "loss": 6.122632598876953, + "step": 1120 + }, + { + "epoch": 1.2914285714285714, + "grad_norm": 3.0, + "learning_rate": 1.1398095238095239e-05, + "loss": 6.018258666992187, + "step": 1130 + }, + { + "epoch": 1.302857142857143, + "grad_norm": 2.875, + "learning_rate": 1.1321904761904762e-05, + "loss": 6.35387077331543, + "step": 1140 + }, + { + "epoch": 1.3142857142857143, + "grad_norm": 3.28125, + "learning_rate": 1.1245714285714286e-05, + "loss": 6.480564880371094, + "step": 1150 + }, + { + "epoch": 1.3257142857142856, + "grad_norm": 2.96875, + "learning_rate": 1.1169523809523811e-05, + "loss": 6.322550582885742, + "step": 1160 + }, + { + "epoch": 1.3371428571428572, + "grad_norm": 2.578125, + "learning_rate": 1.1093333333333334e-05, + "loss": 6.099396133422852, + "step": 1170 + }, + { + "epoch": 1.3485714285714285, + "grad_norm": 2.3125, + "learning_rate": 1.1017142857142858e-05, + "loss": 6.204093170166016, + "step": 1180 + }, + { + "epoch": 1.3599999999999999, + "grad_norm": 3.015625, + "learning_rate": 1.0940952380952381e-05, + "loss": 6.10711441040039, + "step": 1190 + }, + { + "epoch": 1.3714285714285714, + "grad_norm": 2.609375, + "learning_rate": 1.0864761904761905e-05, + "loss": 6.543480682373047, + "step": 1200 + }, + { + "epoch": 1.3828571428571428, + "grad_norm": 3.34375, + "learning_rate": 1.078857142857143e-05, + "loss": 6.186014175415039, + "step": 1210 + }, + { + "epoch": 1.3942857142857144, + "grad_norm": 2.265625, + "learning_rate": 1.0712380952380954e-05, + "loss": 6.085746765136719, + "step": 1220 + }, + { + "epoch": 1.4057142857142857, + "grad_norm": 3.265625, + "learning_rate": 1.0636190476190477e-05, + "loss": 6.401334381103515, + "step": 1230 + }, + { + "epoch": 1.4171428571428573, + "grad_norm": 3.046875, + "learning_rate": 1.056e-05, + "loss": 6.243403625488281, + "step": 1240 + }, + { + "epoch": 1.4285714285714286, + "grad_norm": 2.78125, + "learning_rate": 1.0483809523809526e-05, + "loss": 6.028950881958008, + "step": 1250 + }, + { + "epoch": 1.44, + "grad_norm": 2.6875, + "learning_rate": 1.040761904761905e-05, + "loss": 6.03259162902832, + "step": 1260 + }, + { + "epoch": 1.4514285714285715, + "grad_norm": 3.03125, + "learning_rate": 1.0331428571428573e-05, + "loss": 6.112017440795898, + "step": 1270 + }, + { + "epoch": 1.4628571428571429, + "grad_norm": 4.03125, + "learning_rate": 1.0255238095238096e-05, + "loss": 6.302680206298828, + "step": 1280 + }, + { + "epoch": 1.4742857142857142, + "grad_norm": 3.25, + "learning_rate": 1.017904761904762e-05, + "loss": 5.820198059082031, + "step": 1290 + }, + { + "epoch": 1.4857142857142858, + "grad_norm": 2.703125, + "learning_rate": 1.0102857142857143e-05, + "loss": 6.439159393310547, + "step": 1300 + }, + { + "epoch": 1.497142857142857, + "grad_norm": 2.25, + "learning_rate": 1.0026666666666667e-05, + "loss": 6.070474243164062, + "step": 1310 + }, + { + "epoch": 1.5085714285714285, + "grad_norm": 2.203125, + "learning_rate": 9.950476190476192e-06, + "loss": 6.100412750244141, + "step": 1320 + }, + { + "epoch": 1.52, + "grad_norm": 3.28125, + "learning_rate": 9.874285714285715e-06, + "loss": 6.042630004882812, + "step": 1330 + }, + { + "epoch": 1.5314285714285716, + "grad_norm": 3.140625, + "learning_rate": 9.798095238095239e-06, + "loss": 5.907506942749023, + "step": 1340 + }, + { + "epoch": 1.5428571428571427, + "grad_norm": 3.375, + "learning_rate": 9.721904761904762e-06, + "loss": 6.336385345458984, + "step": 1350 + }, + { + "epoch": 1.5542857142857143, + "grad_norm": 2.703125, + "learning_rate": 9.645714285714286e-06, + "loss": 6.24947509765625, + "step": 1360 + }, + { + "epoch": 1.5657142857142858, + "grad_norm": 3.015625, + "learning_rate": 9.569523809523811e-06, + "loss": 6.249856948852539, + "step": 1370 + }, + { + "epoch": 1.5771428571428572, + "grad_norm": 3.078125, + "learning_rate": 9.493333333333334e-06, + "loss": 6.1782371520996096, + "step": 1380 + }, + { + "epoch": 1.5885714285714285, + "grad_norm": 3.015625, + "learning_rate": 9.417142857142858e-06, + "loss": 6.15069808959961, + "step": 1390 + }, + { + "epoch": 1.6, + "grad_norm": 3.171875, + "learning_rate": 9.340952380952381e-06, + "loss": 6.212250518798828, + "step": 1400 + }, + { + "epoch": 1.6114285714285714, + "grad_norm": 3.375, + "learning_rate": 9.264761904761905e-06, + "loss": 6.391638565063476, + "step": 1410 + }, + { + "epoch": 1.6228571428571428, + "grad_norm": 2.5, + "learning_rate": 9.188571428571428e-06, + "loss": 6.229971313476563, + "step": 1420 + }, + { + "epoch": 1.6342857142857143, + "grad_norm": 2.984375, + "learning_rate": 9.112380952380954e-06, + "loss": 6.20762825012207, + "step": 1430 + }, + { + "epoch": 1.6457142857142857, + "grad_norm": 2.875, + "learning_rate": 9.036190476190477e-06, + "loss": 6.2814586639404295, + "step": 1440 + }, + { + "epoch": 1.657142857142857, + "grad_norm": 2.828125, + "learning_rate": 8.96e-06, + "loss": 6.375495147705078, + "step": 1450 + }, + { + "epoch": 1.6685714285714286, + "grad_norm": 2.640625, + "learning_rate": 8.883809523809524e-06, + "loss": 6.348464584350586, + "step": 1460 + }, + { + "epoch": 1.6800000000000002, + "grad_norm": 2.84375, + "learning_rate": 8.807619047619048e-06, + "loss": 5.998020935058594, + "step": 1470 + }, + { + "epoch": 1.6914285714285713, + "grad_norm": 3.453125, + "learning_rate": 8.731428571428571e-06, + "loss": 6.345339584350586, + "step": 1480 + }, + { + "epoch": 1.7028571428571428, + "grad_norm": 2.78125, + "learning_rate": 8.655238095238096e-06, + "loss": 6.198200988769531, + "step": 1490 + }, + { + "epoch": 1.7142857142857144, + "grad_norm": 3.125, + "learning_rate": 8.57904761904762e-06, + "loss": 6.145425796508789, + "step": 1500 + }, + { + "epoch": 1.7257142857142858, + "grad_norm": 2.640625, + "learning_rate": 8.502857142857143e-06, + "loss": 6.78307876586914, + "step": 1510 + }, + { + "epoch": 1.737142857142857, + "grad_norm": 2.609375, + "learning_rate": 8.426666666666667e-06, + "loss": 6.41483154296875, + "step": 1520 + }, + { + "epoch": 1.7485714285714287, + "grad_norm": 2.625, + "learning_rate": 8.35047619047619e-06, + "loss": 6.226511383056641, + "step": 1530 + }, + { + "epoch": 1.76, + "grad_norm": 2.8125, + "learning_rate": 8.274285714285715e-06, + "loss": 5.991352462768555, + "step": 1540 + }, + { + "epoch": 1.7714285714285714, + "grad_norm": 2.75, + "learning_rate": 8.198095238095239e-06, + "loss": 5.888279724121094, + "step": 1550 + }, + { + "epoch": 1.782857142857143, + "grad_norm": 3.21875, + "learning_rate": 8.121904761904762e-06, + "loss": 6.307299423217773, + "step": 1560 + }, + { + "epoch": 1.7942857142857143, + "grad_norm": 2.84375, + "learning_rate": 8.045714285714286e-06, + "loss": 6.326828384399414, + "step": 1570 + }, + { + "epoch": 1.8057142857142856, + "grad_norm": 2.5, + "learning_rate": 7.969523809523811e-06, + "loss": 6.229778289794922, + "step": 1580 + }, + { + "epoch": 1.8171428571428572, + "grad_norm": 3.140625, + "learning_rate": 7.893333333333335e-06, + "loss": 5.9689685821533205, + "step": 1590 + }, + { + "epoch": 1.8285714285714287, + "grad_norm": 2.265625, + "learning_rate": 7.817142857142858e-06, + "loss": 6.273366165161133, + "step": 1600 + }, + { + "epoch": 1.8399999999999999, + "grad_norm": 2.875, + "learning_rate": 7.740952380952382e-06, + "loss": 5.9848285675048825, + "step": 1610 + }, + { + "epoch": 1.8514285714285714, + "grad_norm": 3.375, + "learning_rate": 7.664761904761905e-06, + "loss": 6.275168609619141, + "step": 1620 + }, + { + "epoch": 1.862857142857143, + "grad_norm": 2.96875, + "learning_rate": 7.588571428571429e-06, + "loss": 6.111376953125, + "step": 1630 + }, + { + "epoch": 1.8742857142857143, + "grad_norm": 2.3125, + "learning_rate": 7.512380952380953e-06, + "loss": 6.3579669952392575, + "step": 1640 + }, + { + "epoch": 1.8857142857142857, + "grad_norm": 3.296875, + "learning_rate": 7.436190476190477e-06, + "loss": 6.263930130004883, + "step": 1650 + }, + { + "epoch": 1.8971428571428572, + "grad_norm": 2.78125, + "learning_rate": 7.360000000000001e-06, + "loss": 6.2892204284667965, + "step": 1660 + }, + { + "epoch": 1.9085714285714286, + "grad_norm": 2.859375, + "learning_rate": 7.283809523809524e-06, + "loss": 6.76464614868164, + "step": 1670 + }, + { + "epoch": 1.92, + "grad_norm": 2.921875, + "learning_rate": 7.207619047619048e-06, + "loss": 6.188260650634765, + "step": 1680 + }, + { + "epoch": 1.9314285714285715, + "grad_norm": 3.1875, + "learning_rate": 7.131428571428573e-06, + "loss": 6.480364227294922, + "step": 1690 + }, + { + "epoch": 1.9428571428571428, + "grad_norm": 2.296875, + "learning_rate": 7.055238095238096e-06, + "loss": 6.121232986450195, + "step": 1700 + }, + { + "epoch": 1.9542857142857142, + "grad_norm": 2.40625, + "learning_rate": 6.97904761904762e-06, + "loss": 6.384135437011719, + "step": 1710 + }, + { + "epoch": 1.9657142857142857, + "grad_norm": 2.734375, + "learning_rate": 6.902857142857143e-06, + "loss": 6.264633560180664, + "step": 1720 + }, + { + "epoch": 1.977142857142857, + "grad_norm": 3.203125, + "learning_rate": 6.826666666666667e-06, + "loss": 6.220773315429687, + "step": 1730 + }, + { + "epoch": 1.9885714285714284, + "grad_norm": 3.3125, + "learning_rate": 6.75047619047619e-06, + "loss": 6.006364822387695, + "step": 1740 + }, + { + "epoch": 2.0, + "grad_norm": 3.046875, + "learning_rate": 6.6742857142857155e-06, + "loss": 5.799431228637696, + "step": 1750 + } + ], + "logging_steps": 10, + "max_steps": 2625, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.36307930482862e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1750/training_args.bin b/checkpoint-1750/training_args.bin new file mode 100644 index 0000000..38338f4 --- /dev/null +++ b/checkpoint-1750/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8a2ef5b8d79835ff6ba94d10403139a3a9af53e51adfe2987ac1e7ff4f8e1ec +size 6289 diff --git a/checkpoint-2625/chat_template.jinja b/checkpoint-2625/chat_template.jinja new file mode 100644 index 0000000..64983ee --- /dev/null +++ b/checkpoint-2625/chat_template.jinja @@ -0,0 +1,56 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" and not message.tool_calls %} + {{- '<|im_start|>assistant\n' }}{% generation %}{{- message.content + '<|im_end|>\n' }}{% endgeneration %} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>assistant' }}{% generation %} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }}{% endgeneration %} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-2625/config.json b/checkpoint-2625/config.json new file mode 100644 index 0000000..c097f0c --- /dev/null +++ b/checkpoint-2625/config.json @@ -0,0 +1,58 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.5.3", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-2625/generation_config.json b/checkpoint-2625/generation_config.json new file mode 100644 index 0000000..61f88e1 --- /dev/null +++ b/checkpoint-2625/generation_config.json @@ -0,0 +1,10 @@ +{ + "do_sample": false, + "eos_token_id": [ + 151645 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.5.3", + "use_cache": false +} diff --git a/checkpoint-2625/model.safetensors b/checkpoint-2625/model.safetensors new file mode 100644 index 0000000..b0ae7da --- /dev/null +++ b/checkpoint-2625/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10cbf78c24b74c3578136d377abf51eeb08995479f597d62e92ca7c6298a5d05 +size 988097824 diff --git a/checkpoint-2625/optimizer.pt b/checkpoint-2625/optimizer.pt new file mode 100644 index 0000000..dcb571c --- /dev/null +++ b/checkpoint-2625/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efe47928026abf0fda933a387e0c21813634b64b568a52b39247f08d5a1646fa +size 1976378699 diff --git a/checkpoint-2625/rng_state.pth b/checkpoint-2625/rng_state.pth new file mode 100644 index 0000000..9ae11db --- /dev/null +++ b/checkpoint-2625/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bb7e3e0894f95e6b09893239ea7465e0b0f84960c938750037c83d683a7c132 +size 14645 diff --git a/checkpoint-2625/scheduler.pt b/checkpoint-2625/scheduler.pt new file mode 100644 index 0000000..08b60c6 --- /dev/null +++ b/checkpoint-2625/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a90cf32bcaa40d2760ac49ca5a2167553d01347c3172274302ea44fcf542f183 +size 1465 diff --git a/checkpoint-2625/tokenizer.json b/checkpoint-2625/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-2625/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-2625/tokenizer_config.json b/checkpoint-2625/tokenizer_config.json new file mode 100644 index 0000000..0385b12 --- /dev/null +++ b/checkpoint-2625/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-2625/trainer_state.json b/checkpoint-2625/trainer_state.json new file mode 100644 index 0000000..1d09b57 --- /dev/null +++ b/checkpoint-2625/trainer_state.json @@ -0,0 +1,1868 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 2625, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.011428571428571429, + "grad_norm": 3.078125, + "learning_rate": 1.9931428571428572e-05, + "loss": 7.941626739501953, + "step": 10 + }, + { + "epoch": 0.022857142857142857, + "grad_norm": 2.9375, + "learning_rate": 1.9855238095238097e-05, + "loss": 8.19613037109375, + "step": 20 + }, + { + "epoch": 0.03428571428571429, + "grad_norm": 3.5625, + "learning_rate": 1.977904761904762e-05, + "loss": 7.826921081542968, + "step": 30 + }, + { + "epoch": 0.045714285714285714, + "grad_norm": 3.15625, + "learning_rate": 1.9702857142857144e-05, + "loss": 7.580943298339844, + "step": 40 + }, + { + "epoch": 0.05714285714285714, + "grad_norm": 2.515625, + "learning_rate": 1.9626666666666666e-05, + "loss": 7.483664703369141, + "step": 50 + }, + { + "epoch": 0.06857142857142857, + "grad_norm": 3.859375, + "learning_rate": 1.955047619047619e-05, + "loss": 7.480722808837891, + "step": 60 + }, + { + "epoch": 0.08, + "grad_norm": 3.46875, + "learning_rate": 1.9474285714285717e-05, + "loss": 7.279670715332031, + "step": 70 + }, + { + "epoch": 0.09142857142857143, + "grad_norm": 2.5625, + "learning_rate": 1.9398095238095242e-05, + "loss": 7.584724426269531, + "step": 80 + }, + { + "epoch": 0.10285714285714286, + "grad_norm": 2.515625, + "learning_rate": 1.9321904761904764e-05, + "loss": 7.565158081054688, + "step": 90 + }, + { + "epoch": 0.11428571428571428, + "grad_norm": 3.5625, + "learning_rate": 1.924571428571429e-05, + "loss": 7.366996002197266, + "step": 100 + }, + { + "epoch": 0.12571428571428572, + "grad_norm": 3.109375, + "learning_rate": 1.916952380952381e-05, + "loss": 7.434650421142578, + "step": 110 + }, + { + "epoch": 0.13714285714285715, + "grad_norm": 3.328125, + "learning_rate": 1.9093333333333336e-05, + "loss": 6.9210762023925785, + "step": 120 + }, + { + "epoch": 0.14857142857142858, + "grad_norm": 3.015625, + "learning_rate": 1.9017142857142858e-05, + "loss": 7.027304840087891, + "step": 130 + }, + { + "epoch": 0.16, + "grad_norm": 3.09375, + "learning_rate": 1.8940952380952383e-05, + "loss": 7.241950225830078, + "step": 140 + }, + { + "epoch": 0.17142857142857143, + "grad_norm": 3.0, + "learning_rate": 1.8864761904761905e-05, + "loss": 7.158929443359375, + "step": 150 + }, + { + "epoch": 0.18285714285714286, + "grad_norm": 2.890625, + "learning_rate": 1.878857142857143e-05, + "loss": 6.856580352783203, + "step": 160 + }, + { + "epoch": 0.19428571428571428, + "grad_norm": 4.09375, + "learning_rate": 1.871238095238095e-05, + "loss": 6.740969848632813, + "step": 170 + }, + { + "epoch": 0.2057142857142857, + "grad_norm": 3.40625, + "learning_rate": 1.8636190476190477e-05, + "loss": 7.115393829345703, + "step": 180 + }, + { + "epoch": 0.21714285714285714, + "grad_norm": 2.9375, + "learning_rate": 1.8560000000000002e-05, + "loss": 7.437998199462891, + "step": 190 + }, + { + "epoch": 0.22857142857142856, + "grad_norm": 2.96875, + "learning_rate": 1.8483809523809527e-05, + "loss": 6.879551696777344, + "step": 200 + }, + { + "epoch": 0.24, + "grad_norm": 2.890625, + "learning_rate": 1.840761904761905e-05, + "loss": 6.8273162841796875, + "step": 210 + }, + { + "epoch": 0.25142857142857145, + "grad_norm": 3.0, + "learning_rate": 1.8331428571428574e-05, + "loss": 6.558704376220703, + "step": 220 + }, + { + "epoch": 0.26285714285714284, + "grad_norm": 4.40625, + "learning_rate": 1.8255238095238096e-05, + "loss": 7.156157684326172, + "step": 230 + }, + { + "epoch": 0.2742857142857143, + "grad_norm": 2.671875, + "learning_rate": 1.817904761904762e-05, + "loss": 6.985882568359375, + "step": 240 + }, + { + "epoch": 0.2857142857142857, + "grad_norm": 2.46875, + "learning_rate": 1.8102857142857143e-05, + "loss": 7.106211090087891, + "step": 250 + }, + { + "epoch": 0.29714285714285715, + "grad_norm": 2.984375, + "learning_rate": 1.8026666666666668e-05, + "loss": 7.014789581298828, + "step": 260 + }, + { + "epoch": 0.30857142857142855, + "grad_norm": 2.5, + "learning_rate": 1.7950476190476193e-05, + "loss": 6.632127380371093, + "step": 270 + }, + { + "epoch": 0.32, + "grad_norm": 2.84375, + "learning_rate": 1.7874285714285715e-05, + "loss": 6.882701110839844, + "step": 280 + }, + { + "epoch": 0.3314285714285714, + "grad_norm": 3.78125, + "learning_rate": 1.779809523809524e-05, + "loss": 6.729277801513672, + "step": 290 + }, + { + "epoch": 0.34285714285714286, + "grad_norm": 3.578125, + "learning_rate": 1.7721904761904762e-05, + "loss": 6.794551849365234, + "step": 300 + }, + { + "epoch": 0.35428571428571426, + "grad_norm": 2.828125, + "learning_rate": 1.7645714285714287e-05, + "loss": 6.83813247680664, + "step": 310 + }, + { + "epoch": 0.3657142857142857, + "grad_norm": 2.875, + "learning_rate": 1.7569523809523812e-05, + "loss": 6.796338653564453, + "step": 320 + }, + { + "epoch": 0.37714285714285717, + "grad_norm": 2.90625, + "learning_rate": 1.7493333333333334e-05, + "loss": 6.7025390625, + "step": 330 + }, + { + "epoch": 0.38857142857142857, + "grad_norm": 2.953125, + "learning_rate": 1.741714285714286e-05, + "loss": 6.541476440429688, + "step": 340 + }, + { + "epoch": 0.4, + "grad_norm": 2.59375, + "learning_rate": 1.734095238095238e-05, + "loss": 6.812630462646484, + "step": 350 + }, + { + "epoch": 0.4114285714285714, + "grad_norm": 3.609375, + "learning_rate": 1.7264761904761906e-05, + "loss": 6.889384460449219, + "step": 360 + }, + { + "epoch": 0.4228571428571429, + "grad_norm": 4.03125, + "learning_rate": 1.718857142857143e-05, + "loss": 7.129203796386719, + "step": 370 + }, + { + "epoch": 0.4342857142857143, + "grad_norm": 2.40625, + "learning_rate": 1.7112380952380953e-05, + "loss": 6.382967376708985, + "step": 380 + }, + { + "epoch": 0.44571428571428573, + "grad_norm": 3.515625, + "learning_rate": 1.703619047619048e-05, + "loss": 6.663212585449219, + "step": 390 + }, + { + "epoch": 0.45714285714285713, + "grad_norm": 2.90625, + "learning_rate": 1.696e-05, + "loss": 6.636187744140625, + "step": 400 + }, + { + "epoch": 0.4685714285714286, + "grad_norm": 3.1875, + "learning_rate": 1.6883809523809525e-05, + "loss": 6.64324722290039, + "step": 410 + }, + { + "epoch": 0.48, + "grad_norm": 2.828125, + "learning_rate": 1.6807619047619047e-05, + "loss": 6.663475036621094, + "step": 420 + }, + { + "epoch": 0.49142857142857144, + "grad_norm": 3.328125, + "learning_rate": 1.6731428571428572e-05, + "loss": 6.452519226074219, + "step": 430 + }, + { + "epoch": 0.5028571428571429, + "grad_norm": 2.421875, + "learning_rate": 1.6655238095238098e-05, + "loss": 6.683634948730469, + "step": 440 + }, + { + "epoch": 0.5142857142857142, + "grad_norm": 3.046875, + "learning_rate": 1.6579047619047623e-05, + "loss": 6.780387878417969, + "step": 450 + }, + { + "epoch": 0.5257142857142857, + "grad_norm": 2.53125, + "learning_rate": 1.6502857142857145e-05, + "loss": 7.1892578125, + "step": 460 + }, + { + "epoch": 0.5371428571428571, + "grad_norm": 2.71875, + "learning_rate": 1.642666666666667e-05, + "loss": 6.890229034423828, + "step": 470 + }, + { + "epoch": 0.5485714285714286, + "grad_norm": 3.140625, + "learning_rate": 1.635047619047619e-05, + "loss": 6.747575378417968, + "step": 480 + }, + { + "epoch": 0.56, + "grad_norm": 3.3125, + "learning_rate": 1.6274285714285717e-05, + "loss": 6.98467788696289, + "step": 490 + }, + { + "epoch": 0.5714285714285714, + "grad_norm": 2.890625, + "learning_rate": 1.619809523809524e-05, + "loss": 6.455801391601563, + "step": 500 + }, + { + "epoch": 0.5828571428571429, + "grad_norm": 2.578125, + "learning_rate": 1.6121904761904764e-05, + "loss": 6.525889587402344, + "step": 510 + }, + { + "epoch": 0.5942857142857143, + "grad_norm": 3.375, + "learning_rate": 1.6045714285714286e-05, + "loss": 6.479692840576172, + "step": 520 + }, + { + "epoch": 0.6057142857142858, + "grad_norm": 2.875, + "learning_rate": 1.596952380952381e-05, + "loss": 6.6360313415527346, + "step": 530 + }, + { + "epoch": 0.6171428571428571, + "grad_norm": 2.703125, + "learning_rate": 1.5893333333333333e-05, + "loss": 6.547400665283203, + "step": 540 + }, + { + "epoch": 0.6285714285714286, + "grad_norm": 2.9375, + "learning_rate": 1.5817142857142858e-05, + "loss": 6.748792266845703, + "step": 550 + }, + { + "epoch": 0.64, + "grad_norm": 3.28125, + "learning_rate": 1.5740952380952383e-05, + "loss": 6.727365112304687, + "step": 560 + }, + { + "epoch": 0.6514285714285715, + "grad_norm": 2.984375, + "learning_rate": 1.5664761904761908e-05, + "loss": 6.611011505126953, + "step": 570 + }, + { + "epoch": 0.6628571428571428, + "grad_norm": 2.984375, + "learning_rate": 1.558857142857143e-05, + "loss": 6.471088409423828, + "step": 580 + }, + { + "epoch": 0.6742857142857143, + "grad_norm": 2.84375, + "learning_rate": 1.5512380952380955e-05, + "loss": 6.472038269042969, + "step": 590 + }, + { + "epoch": 0.6857142857142857, + "grad_norm": 3.15625, + "learning_rate": 1.5436190476190477e-05, + "loss": 6.747550201416016, + "step": 600 + }, + { + "epoch": 0.6971428571428572, + "grad_norm": 2.75, + "learning_rate": 1.5360000000000002e-05, + "loss": 6.638180541992187, + "step": 610 + }, + { + "epoch": 0.7085714285714285, + "grad_norm": 3.078125, + "learning_rate": 1.5283809523809524e-05, + "loss": 6.535469818115234, + "step": 620 + }, + { + "epoch": 0.72, + "grad_norm": 3.109375, + "learning_rate": 1.5207619047619049e-05, + "loss": 6.981074523925781, + "step": 630 + }, + { + "epoch": 0.7314285714285714, + "grad_norm": 2.78125, + "learning_rate": 1.5131428571428572e-05, + "loss": 6.307097625732422, + "step": 640 + }, + { + "epoch": 0.7428571428571429, + "grad_norm": 2.8125, + "learning_rate": 1.5055238095238096e-05, + "loss": 6.617960357666016, + "step": 650 + }, + { + "epoch": 0.7542857142857143, + "grad_norm": 2.9375, + "learning_rate": 1.497904761904762e-05, + "loss": 6.845116424560547, + "step": 660 + }, + { + "epoch": 0.7657142857142857, + "grad_norm": 2.140625, + "learning_rate": 1.4902857142857143e-05, + "loss": 6.583855438232422, + "step": 670 + }, + { + "epoch": 0.7771428571428571, + "grad_norm": 2.703125, + "learning_rate": 1.4826666666666666e-05, + "loss": 6.485343933105469, + "step": 680 + }, + { + "epoch": 0.7885714285714286, + "grad_norm": 2.96875, + "learning_rate": 1.475047619047619e-05, + "loss": 6.277442169189453, + "step": 690 + }, + { + "epoch": 0.8, + "grad_norm": 3.078125, + "learning_rate": 1.4674285714285717e-05, + "loss": 6.420030212402343, + "step": 700 + }, + { + "epoch": 0.8114285714285714, + "grad_norm": 2.625, + "learning_rate": 1.459809523809524e-05, + "loss": 6.779734802246094, + "step": 710 + }, + { + "epoch": 0.8228571428571428, + "grad_norm": 3.28125, + "learning_rate": 1.4521904761904764e-05, + "loss": 6.647151947021484, + "step": 720 + }, + { + "epoch": 0.8342857142857143, + "grad_norm": 2.5625, + "learning_rate": 1.4445714285714287e-05, + "loss": 6.453370666503906, + "step": 730 + }, + { + "epoch": 0.8457142857142858, + "grad_norm": 2.5625, + "learning_rate": 1.436952380952381e-05, + "loss": 6.6205802917480465, + "step": 740 + }, + { + "epoch": 0.8571428571428571, + "grad_norm": 2.265625, + "learning_rate": 1.4293333333333334e-05, + "loss": 6.353859329223633, + "step": 750 + }, + { + "epoch": 0.8685714285714285, + "grad_norm": 3.015625, + "learning_rate": 1.4217142857142858e-05, + "loss": 6.357454299926758, + "step": 760 + }, + { + "epoch": 0.88, + "grad_norm": 3.03125, + "learning_rate": 1.4140952380952381e-05, + "loss": 6.682913208007813, + "step": 770 + }, + { + "epoch": 0.8914285714285715, + "grad_norm": 2.625, + "learning_rate": 1.4064761904761905e-05, + "loss": 6.127779769897461, + "step": 780 + }, + { + "epoch": 0.9028571428571428, + "grad_norm": 4.375, + "learning_rate": 1.398857142857143e-05, + "loss": 6.364295578002929, + "step": 790 + }, + { + "epoch": 0.9142857142857143, + "grad_norm": 2.8125, + "learning_rate": 1.3912380952380953e-05, + "loss": 6.2481742858886715, + "step": 800 + }, + { + "epoch": 0.9257142857142857, + "grad_norm": 2.875, + "learning_rate": 1.3836190476190477e-05, + "loss": 6.126508712768555, + "step": 810 + }, + { + "epoch": 0.9371428571428572, + "grad_norm": 3.0, + "learning_rate": 1.376e-05, + "loss": 6.082854080200195, + "step": 820 + }, + { + "epoch": 0.9485714285714286, + "grad_norm": 3.09375, + "learning_rate": 1.3683809523809526e-05, + "loss": 6.128392028808594, + "step": 830 + }, + { + "epoch": 0.96, + "grad_norm": 3.40625, + "learning_rate": 1.3607619047619049e-05, + "loss": 6.288749313354492, + "step": 840 + }, + { + "epoch": 0.9714285714285714, + "grad_norm": 2.78125, + "learning_rate": 1.3531428571428573e-05, + "loss": 6.770208740234375, + "step": 850 + }, + { + "epoch": 0.9828571428571429, + "grad_norm": 2.96875, + "learning_rate": 1.3455238095238096e-05, + "loss": 6.299668502807617, + "step": 860 + }, + { + "epoch": 0.9942857142857143, + "grad_norm": 2.765625, + "learning_rate": 1.3379047619047621e-05, + "loss": 6.4255424499511715, + "step": 870 + }, + { + "epoch": 1.0057142857142858, + "grad_norm": 2.546875, + "learning_rate": 1.3302857142857145e-05, + "loss": 6.330821228027344, + "step": 880 + }, + { + "epoch": 1.0171428571428571, + "grad_norm": 2.515625, + "learning_rate": 1.3226666666666668e-05, + "loss": 6.2308086395263675, + "step": 890 + }, + { + "epoch": 1.0285714285714285, + "grad_norm": 2.34375, + "learning_rate": 1.3150476190476192e-05, + "loss": 6.077074432373047, + "step": 900 + }, + { + "epoch": 1.04, + "grad_norm": 3.28125, + "learning_rate": 1.3074285714285715e-05, + "loss": 6.353485488891602, + "step": 910 + }, + { + "epoch": 1.0514285714285714, + "grad_norm": 3.28125, + "learning_rate": 1.2998095238095239e-05, + "loss": 6.212256622314453, + "step": 920 + }, + { + "epoch": 1.062857142857143, + "grad_norm": 2.46875, + "learning_rate": 1.2921904761904762e-05, + "loss": 6.344943618774414, + "step": 930 + }, + { + "epoch": 1.0742857142857143, + "grad_norm": 3.3125, + "learning_rate": 1.2845714285714286e-05, + "loss": 6.49066390991211, + "step": 940 + }, + { + "epoch": 1.0857142857142856, + "grad_norm": 2.703125, + "learning_rate": 1.2769523809523811e-05, + "loss": 6.266908645629883, + "step": 950 + }, + { + "epoch": 1.0971428571428572, + "grad_norm": 3.421875, + "learning_rate": 1.2693333333333336e-05, + "loss": 6.110383987426758, + "step": 960 + }, + { + "epoch": 1.1085714285714285, + "grad_norm": 2.328125, + "learning_rate": 1.261714285714286e-05, + "loss": 6.216197204589844, + "step": 970 + }, + { + "epoch": 1.12, + "grad_norm": 3.09375, + "learning_rate": 1.2540952380952383e-05, + "loss": 6.426862335205078, + "step": 980 + }, + { + "epoch": 1.1314285714285715, + "grad_norm": 2.890625, + "learning_rate": 1.2464761904761907e-05, + "loss": 6.450955200195312, + "step": 990 + }, + { + "epoch": 1.1428571428571428, + "grad_norm": 3.78125, + "learning_rate": 1.238857142857143e-05, + "loss": 6.348379135131836, + "step": 1000 + }, + { + "epoch": 1.1542857142857144, + "grad_norm": 3.25, + "learning_rate": 1.2312380952380953e-05, + "loss": 6.226791763305664, + "step": 1010 + }, + { + "epoch": 1.1657142857142857, + "grad_norm": 2.734375, + "learning_rate": 1.2236190476190477e-05, + "loss": 6.333163070678711, + "step": 1020 + }, + { + "epoch": 1.177142857142857, + "grad_norm": 2.796875, + "learning_rate": 1.216e-05, + "loss": 6.321551895141601, + "step": 1030 + }, + { + "epoch": 1.1885714285714286, + "grad_norm": 4.09375, + "learning_rate": 1.2083809523809524e-05, + "loss": 6.385534286499023, + "step": 1040 + }, + { + "epoch": 1.2, + "grad_norm": 3.203125, + "learning_rate": 1.2007619047619047e-05, + "loss": 6.097745895385742, + "step": 1050 + }, + { + "epoch": 1.2114285714285715, + "grad_norm": 2.953125, + "learning_rate": 1.1931428571428571e-05, + "loss": 6.247037124633789, + "step": 1060 + }, + { + "epoch": 1.2228571428571429, + "grad_norm": 2.40625, + "learning_rate": 1.1855238095238094e-05, + "loss": 6.404273223876953, + "step": 1070 + }, + { + "epoch": 1.2342857142857142, + "grad_norm": 3.4375, + "learning_rate": 1.1779047619047621e-05, + "loss": 6.110296630859375, + "step": 1080 + }, + { + "epoch": 1.2457142857142858, + "grad_norm": 2.953125, + "learning_rate": 1.1702857142857145e-05, + "loss": 6.178505325317383, + "step": 1090 + }, + { + "epoch": 1.2571428571428571, + "grad_norm": 2.84375, + "learning_rate": 1.1626666666666668e-05, + "loss": 6.133485412597656, + "step": 1100 + }, + { + "epoch": 1.2685714285714287, + "grad_norm": 2.59375, + "learning_rate": 1.1550476190476192e-05, + "loss": 6.316691589355469, + "step": 1110 + }, + { + "epoch": 1.28, + "grad_norm": 3.03125, + "learning_rate": 1.1474285714285715e-05, + "loss": 6.122632598876953, + "step": 1120 + }, + { + "epoch": 1.2914285714285714, + "grad_norm": 3.0, + "learning_rate": 1.1398095238095239e-05, + "loss": 6.018258666992187, + "step": 1130 + }, + { + "epoch": 1.302857142857143, + "grad_norm": 2.875, + "learning_rate": 1.1321904761904762e-05, + "loss": 6.35387077331543, + "step": 1140 + }, + { + "epoch": 1.3142857142857143, + "grad_norm": 3.28125, + "learning_rate": 1.1245714285714286e-05, + "loss": 6.480564880371094, + "step": 1150 + }, + { + "epoch": 1.3257142857142856, + "grad_norm": 2.96875, + "learning_rate": 1.1169523809523811e-05, + "loss": 6.322550582885742, + "step": 1160 + }, + { + "epoch": 1.3371428571428572, + "grad_norm": 2.578125, + "learning_rate": 1.1093333333333334e-05, + "loss": 6.099396133422852, + "step": 1170 + }, + { + "epoch": 1.3485714285714285, + "grad_norm": 2.3125, + "learning_rate": 1.1017142857142858e-05, + "loss": 6.204093170166016, + "step": 1180 + }, + { + "epoch": 1.3599999999999999, + "grad_norm": 3.015625, + "learning_rate": 1.0940952380952381e-05, + "loss": 6.10711441040039, + "step": 1190 + }, + { + "epoch": 1.3714285714285714, + "grad_norm": 2.609375, + "learning_rate": 1.0864761904761905e-05, + "loss": 6.543480682373047, + "step": 1200 + }, + { + "epoch": 1.3828571428571428, + "grad_norm": 3.34375, + "learning_rate": 1.078857142857143e-05, + "loss": 6.186014175415039, + "step": 1210 + }, + { + "epoch": 1.3942857142857144, + "grad_norm": 2.265625, + "learning_rate": 1.0712380952380954e-05, + "loss": 6.085746765136719, + "step": 1220 + }, + { + "epoch": 1.4057142857142857, + "grad_norm": 3.265625, + "learning_rate": 1.0636190476190477e-05, + "loss": 6.401334381103515, + "step": 1230 + }, + { + "epoch": 1.4171428571428573, + "grad_norm": 3.046875, + "learning_rate": 1.056e-05, + "loss": 6.243403625488281, + "step": 1240 + }, + { + "epoch": 1.4285714285714286, + "grad_norm": 2.78125, + "learning_rate": 1.0483809523809526e-05, + "loss": 6.028950881958008, + "step": 1250 + }, + { + "epoch": 1.44, + "grad_norm": 2.6875, + "learning_rate": 1.040761904761905e-05, + "loss": 6.03259162902832, + "step": 1260 + }, + { + "epoch": 1.4514285714285715, + "grad_norm": 3.03125, + "learning_rate": 1.0331428571428573e-05, + "loss": 6.112017440795898, + "step": 1270 + }, + { + "epoch": 1.4628571428571429, + "grad_norm": 4.03125, + "learning_rate": 1.0255238095238096e-05, + "loss": 6.302680206298828, + "step": 1280 + }, + { + "epoch": 1.4742857142857142, + "grad_norm": 3.25, + "learning_rate": 1.017904761904762e-05, + "loss": 5.820198059082031, + "step": 1290 + }, + { + "epoch": 1.4857142857142858, + "grad_norm": 2.703125, + "learning_rate": 1.0102857142857143e-05, + "loss": 6.439159393310547, + "step": 1300 + }, + { + "epoch": 1.497142857142857, + "grad_norm": 2.25, + "learning_rate": 1.0026666666666667e-05, + "loss": 6.070474243164062, + "step": 1310 + }, + { + "epoch": 1.5085714285714285, + "grad_norm": 2.203125, + "learning_rate": 9.950476190476192e-06, + "loss": 6.100412750244141, + "step": 1320 + }, + { + "epoch": 1.52, + "grad_norm": 3.28125, + "learning_rate": 9.874285714285715e-06, + "loss": 6.042630004882812, + "step": 1330 + }, + { + "epoch": 1.5314285714285716, + "grad_norm": 3.140625, + "learning_rate": 9.798095238095239e-06, + "loss": 5.907506942749023, + "step": 1340 + }, + { + "epoch": 1.5428571428571427, + "grad_norm": 3.375, + "learning_rate": 9.721904761904762e-06, + "loss": 6.336385345458984, + "step": 1350 + }, + { + "epoch": 1.5542857142857143, + "grad_norm": 2.703125, + "learning_rate": 9.645714285714286e-06, + "loss": 6.24947509765625, + "step": 1360 + }, + { + "epoch": 1.5657142857142858, + "grad_norm": 3.015625, + "learning_rate": 9.569523809523811e-06, + "loss": 6.249856948852539, + "step": 1370 + }, + { + "epoch": 1.5771428571428572, + "grad_norm": 3.078125, + "learning_rate": 9.493333333333334e-06, + "loss": 6.1782371520996096, + "step": 1380 + }, + { + "epoch": 1.5885714285714285, + "grad_norm": 3.015625, + "learning_rate": 9.417142857142858e-06, + "loss": 6.15069808959961, + "step": 1390 + }, + { + "epoch": 1.6, + "grad_norm": 3.171875, + "learning_rate": 9.340952380952381e-06, + "loss": 6.212250518798828, + "step": 1400 + }, + { + "epoch": 1.6114285714285714, + "grad_norm": 3.375, + "learning_rate": 9.264761904761905e-06, + "loss": 6.391638565063476, + "step": 1410 + }, + { + "epoch": 1.6228571428571428, + "grad_norm": 2.5, + "learning_rate": 9.188571428571428e-06, + "loss": 6.229971313476563, + "step": 1420 + }, + { + "epoch": 1.6342857142857143, + "grad_norm": 2.984375, + "learning_rate": 9.112380952380954e-06, + "loss": 6.20762825012207, + "step": 1430 + }, + { + "epoch": 1.6457142857142857, + "grad_norm": 2.875, + "learning_rate": 9.036190476190477e-06, + "loss": 6.2814586639404295, + "step": 1440 + }, + { + "epoch": 1.657142857142857, + "grad_norm": 2.828125, + "learning_rate": 8.96e-06, + "loss": 6.375495147705078, + "step": 1450 + }, + { + "epoch": 1.6685714285714286, + "grad_norm": 2.640625, + "learning_rate": 8.883809523809524e-06, + "loss": 6.348464584350586, + "step": 1460 + }, + { + "epoch": 1.6800000000000002, + "grad_norm": 2.84375, + "learning_rate": 8.807619047619048e-06, + "loss": 5.998020935058594, + "step": 1470 + }, + { + "epoch": 1.6914285714285713, + "grad_norm": 3.453125, + "learning_rate": 8.731428571428571e-06, + "loss": 6.345339584350586, + "step": 1480 + }, + { + "epoch": 1.7028571428571428, + "grad_norm": 2.78125, + "learning_rate": 8.655238095238096e-06, + "loss": 6.198200988769531, + "step": 1490 + }, + { + "epoch": 1.7142857142857144, + "grad_norm": 3.125, + "learning_rate": 8.57904761904762e-06, + "loss": 6.145425796508789, + "step": 1500 + }, + { + "epoch": 1.7257142857142858, + "grad_norm": 2.640625, + "learning_rate": 8.502857142857143e-06, + "loss": 6.78307876586914, + "step": 1510 + }, + { + "epoch": 1.737142857142857, + "grad_norm": 2.609375, + "learning_rate": 8.426666666666667e-06, + "loss": 6.41483154296875, + "step": 1520 + }, + { + "epoch": 1.7485714285714287, + "grad_norm": 2.625, + "learning_rate": 8.35047619047619e-06, + "loss": 6.226511383056641, + "step": 1530 + }, + { + "epoch": 1.76, + "grad_norm": 2.8125, + "learning_rate": 8.274285714285715e-06, + "loss": 5.991352462768555, + "step": 1540 + }, + { + "epoch": 1.7714285714285714, + "grad_norm": 2.75, + "learning_rate": 8.198095238095239e-06, + "loss": 5.888279724121094, + "step": 1550 + }, + { + "epoch": 1.782857142857143, + "grad_norm": 3.21875, + "learning_rate": 8.121904761904762e-06, + "loss": 6.307299423217773, + "step": 1560 + }, + { + "epoch": 1.7942857142857143, + "grad_norm": 2.84375, + "learning_rate": 8.045714285714286e-06, + "loss": 6.326828384399414, + "step": 1570 + }, + { + "epoch": 1.8057142857142856, + "grad_norm": 2.5, + "learning_rate": 7.969523809523811e-06, + "loss": 6.229778289794922, + "step": 1580 + }, + { + "epoch": 1.8171428571428572, + "grad_norm": 3.140625, + "learning_rate": 7.893333333333335e-06, + "loss": 5.9689685821533205, + "step": 1590 + }, + { + "epoch": 1.8285714285714287, + "grad_norm": 2.265625, + "learning_rate": 7.817142857142858e-06, + "loss": 6.273366165161133, + "step": 1600 + }, + { + "epoch": 1.8399999999999999, + "grad_norm": 2.875, + "learning_rate": 7.740952380952382e-06, + "loss": 5.9848285675048825, + "step": 1610 + }, + { + "epoch": 1.8514285714285714, + "grad_norm": 3.375, + "learning_rate": 7.664761904761905e-06, + "loss": 6.275168609619141, + "step": 1620 + }, + { + "epoch": 1.862857142857143, + "grad_norm": 2.96875, + "learning_rate": 7.588571428571429e-06, + "loss": 6.111376953125, + "step": 1630 + }, + { + "epoch": 1.8742857142857143, + "grad_norm": 2.3125, + "learning_rate": 7.512380952380953e-06, + "loss": 6.3579669952392575, + "step": 1640 + }, + { + "epoch": 1.8857142857142857, + "grad_norm": 3.296875, + "learning_rate": 7.436190476190477e-06, + "loss": 6.263930130004883, + "step": 1650 + }, + { + "epoch": 1.8971428571428572, + "grad_norm": 2.78125, + "learning_rate": 7.360000000000001e-06, + "loss": 6.2892204284667965, + "step": 1660 + }, + { + "epoch": 1.9085714285714286, + "grad_norm": 2.859375, + "learning_rate": 7.283809523809524e-06, + "loss": 6.76464614868164, + "step": 1670 + }, + { + "epoch": 1.92, + "grad_norm": 2.921875, + "learning_rate": 7.207619047619048e-06, + "loss": 6.188260650634765, + "step": 1680 + }, + { + "epoch": 1.9314285714285715, + "grad_norm": 3.1875, + "learning_rate": 7.131428571428573e-06, + "loss": 6.480364227294922, + "step": 1690 + }, + { + "epoch": 1.9428571428571428, + "grad_norm": 2.296875, + "learning_rate": 7.055238095238096e-06, + "loss": 6.121232986450195, + "step": 1700 + }, + { + "epoch": 1.9542857142857142, + "grad_norm": 2.40625, + "learning_rate": 6.97904761904762e-06, + "loss": 6.384135437011719, + "step": 1710 + }, + { + "epoch": 1.9657142857142857, + "grad_norm": 2.734375, + "learning_rate": 6.902857142857143e-06, + "loss": 6.264633560180664, + "step": 1720 + }, + { + "epoch": 1.977142857142857, + "grad_norm": 3.203125, + "learning_rate": 6.826666666666667e-06, + "loss": 6.220773315429687, + "step": 1730 + }, + { + "epoch": 1.9885714285714284, + "grad_norm": 3.3125, + "learning_rate": 6.75047619047619e-06, + "loss": 6.006364822387695, + "step": 1740 + }, + { + "epoch": 2.0, + "grad_norm": 3.046875, + "learning_rate": 6.6742857142857155e-06, + "loss": 5.799431228637696, + "step": 1750 + }, + { + "epoch": 2.0114285714285716, + "grad_norm": 2.625, + "learning_rate": 6.598095238095239e-06, + "loss": 6.092126846313477, + "step": 1760 + }, + { + "epoch": 2.0228571428571427, + "grad_norm": 2.609375, + "learning_rate": 6.5219047619047625e-06, + "loss": 5.852609634399414, + "step": 1770 + }, + { + "epoch": 2.0342857142857143, + "grad_norm": 2.859375, + "learning_rate": 6.445714285714286e-06, + "loss": 6.128126907348633, + "step": 1780 + }, + { + "epoch": 2.045714285714286, + "grad_norm": 2.75, + "learning_rate": 6.36952380952381e-06, + "loss": 6.174140930175781, + "step": 1790 + }, + { + "epoch": 2.057142857142857, + "grad_norm": 3.046875, + "learning_rate": 6.293333333333334e-06, + "loss": 6.050814819335938, + "step": 1800 + }, + { + "epoch": 2.0685714285714285, + "grad_norm": 3.171875, + "learning_rate": 6.217142857142857e-06, + "loss": 6.278412628173828, + "step": 1810 + }, + { + "epoch": 2.08, + "grad_norm": 2.78125, + "learning_rate": 6.140952380952382e-06, + "loss": 6.133785247802734, + "step": 1820 + }, + { + "epoch": 2.0914285714285716, + "grad_norm": 2.171875, + "learning_rate": 6.064761904761905e-06, + "loss": 6.1286766052246096, + "step": 1830 + }, + { + "epoch": 2.1028571428571428, + "grad_norm": 2.734375, + "learning_rate": 5.9885714285714294e-06, + "loss": 5.963103485107422, + "step": 1840 + }, + { + "epoch": 2.1142857142857143, + "grad_norm": 3.359375, + "learning_rate": 5.912380952380953e-06, + "loss": 6.345986557006836, + "step": 1850 + }, + { + "epoch": 2.125714285714286, + "grad_norm": 2.734375, + "learning_rate": 5.836190476190476e-06, + "loss": 6.234952545166015, + "step": 1860 + }, + { + "epoch": 2.137142857142857, + "grad_norm": 3.640625, + "learning_rate": 5.76e-06, + "loss": 6.143169784545899, + "step": 1870 + }, + { + "epoch": 2.1485714285714286, + "grad_norm": 3.140625, + "learning_rate": 5.683809523809525e-06, + "loss": 6.084795761108398, + "step": 1880 + }, + { + "epoch": 2.16, + "grad_norm": 3.140625, + "learning_rate": 5.607619047619049e-06, + "loss": 6.041054534912109, + "step": 1890 + }, + { + "epoch": 2.1714285714285713, + "grad_norm": 3.0625, + "learning_rate": 5.531428571428572e-06, + "loss": 6.132917022705078, + "step": 1900 + }, + { + "epoch": 2.182857142857143, + "grad_norm": 3.078125, + "learning_rate": 5.4552380952380956e-06, + "loss": 6.442701721191407, + "step": 1910 + }, + { + "epoch": 2.1942857142857144, + "grad_norm": 2.59375, + "learning_rate": 5.379047619047619e-06, + "loss": 5.945763015747071, + "step": 1920 + }, + { + "epoch": 2.2057142857142855, + "grad_norm": 2.640625, + "learning_rate": 5.3028571428571425e-06, + "loss": 6.073029708862305, + "step": 1930 + }, + { + "epoch": 2.217142857142857, + "grad_norm": 3.03125, + "learning_rate": 5.226666666666667e-06, + "loss": 6.124402236938477, + "step": 1940 + }, + { + "epoch": 2.2285714285714286, + "grad_norm": 2.59375, + "learning_rate": 5.150476190476191e-06, + "loss": 6.0411834716796875, + "step": 1950 + }, + { + "epoch": 2.24, + "grad_norm": 2.203125, + "learning_rate": 5.074285714285715e-06, + "loss": 6.078707122802735, + "step": 1960 + }, + { + "epoch": 2.2514285714285713, + "grad_norm": 2.546875, + "learning_rate": 4.998095238095238e-06, + "loss": 6.299936294555664, + "step": 1970 + }, + { + "epoch": 2.262857142857143, + "grad_norm": 2.171875, + "learning_rate": 4.9219047619047625e-06, + "loss": 5.964184188842774, + "step": 1980 + }, + { + "epoch": 2.2742857142857145, + "grad_norm": 3.828125, + "learning_rate": 4.845714285714286e-06, + "loss": 5.9020545959472654, + "step": 1990 + }, + { + "epoch": 2.2857142857142856, + "grad_norm": 2.640625, + "learning_rate": 4.76952380952381e-06, + "loss": 6.332792282104492, + "step": 2000 + }, + { + "epoch": 2.297142857142857, + "grad_norm": 3.96875, + "learning_rate": 4.693333333333334e-06, + "loss": 5.962831115722656, + "step": 2010 + }, + { + "epoch": 2.3085714285714287, + "grad_norm": 3.359375, + "learning_rate": 4.617142857142857e-06, + "loss": 6.346956634521485, + "step": 2020 + }, + { + "epoch": 2.32, + "grad_norm": 2.0625, + "learning_rate": 4.540952380952382e-06, + "loss": 6.284796905517578, + "step": 2030 + }, + { + "epoch": 2.3314285714285714, + "grad_norm": 2.625, + "learning_rate": 4.464761904761905e-06, + "loss": 6.00302734375, + "step": 2040 + }, + { + "epoch": 2.342857142857143, + "grad_norm": 2.953125, + "learning_rate": 4.388571428571429e-06, + "loss": 6.02760124206543, + "step": 2050 + }, + { + "epoch": 2.354285714285714, + "grad_norm": 2.90625, + "learning_rate": 4.312380952380953e-06, + "loss": 6.153439712524414, + "step": 2060 + }, + { + "epoch": 2.3657142857142857, + "grad_norm": 3.15625, + "learning_rate": 4.2361904761904765e-06, + "loss": 6.210586929321289, + "step": 2070 + }, + { + "epoch": 2.3771428571428572, + "grad_norm": 3.203125, + "learning_rate": 4.16e-06, + "loss": 6.0012462615966795, + "step": 2080 + }, + { + "epoch": 2.388571428571429, + "grad_norm": 2.765625, + "learning_rate": 4.083809523809524e-06, + "loss": 6.083547210693359, + "step": 2090 + }, + { + "epoch": 2.4, + "grad_norm": 2.53125, + "learning_rate": 4.007619047619048e-06, + "loss": 6.2686107635498045, + "step": 2100 + }, + { + "epoch": 2.4114285714285715, + "grad_norm": 2.828125, + "learning_rate": 3.931428571428571e-06, + "loss": 6.177051162719726, + "step": 2110 + }, + { + "epoch": 2.422857142857143, + "grad_norm": 2.3125, + "learning_rate": 3.855238095238096e-06, + "loss": 5.786938095092774, + "step": 2120 + }, + { + "epoch": 2.434285714285714, + "grad_norm": 3.15625, + "learning_rate": 3.7790476190476196e-06, + "loss": 5.815621185302734, + "step": 2130 + }, + { + "epoch": 2.4457142857142857, + "grad_norm": 2.875, + "learning_rate": 3.702857142857143e-06, + "loss": 5.926832580566407, + "step": 2140 + }, + { + "epoch": 2.4571428571428573, + "grad_norm": 2.90625, + "learning_rate": 3.6266666666666674e-06, + "loss": 5.957284927368164, + "step": 2150 + }, + { + "epoch": 2.4685714285714284, + "grad_norm": 2.921875, + "learning_rate": 3.550476190476191e-06, + "loss": 6.309428024291992, + "step": 2160 + }, + { + "epoch": 2.48, + "grad_norm": 2.5625, + "learning_rate": 3.4742857142857144e-06, + "loss": 6.313393020629883, + "step": 2170 + }, + { + "epoch": 2.4914285714285715, + "grad_norm": 2.5625, + "learning_rate": 3.3980952380952383e-06, + "loss": 6.104763031005859, + "step": 2180 + }, + { + "epoch": 2.5028571428571427, + "grad_norm": 2.828125, + "learning_rate": 3.321904761904762e-06, + "loss": 6.1094825744628904, + "step": 2190 + }, + { + "epoch": 2.5142857142857142, + "grad_norm": 2.625, + "learning_rate": 3.245714285714286e-06, + "loss": 6.11797981262207, + "step": 2200 + }, + { + "epoch": 2.525714285714286, + "grad_norm": 2.875, + "learning_rate": 3.1695238095238096e-06, + "loss": 6.259257125854492, + "step": 2210 + }, + { + "epoch": 2.5371428571428574, + "grad_norm": 2.8125, + "learning_rate": 3.093333333333334e-06, + "loss": 6.102438354492188, + "step": 2220 + }, + { + "epoch": 2.5485714285714285, + "grad_norm": 3.140625, + "learning_rate": 3.0171428571428574e-06, + "loss": 5.919002914428711, + "step": 2230 + }, + { + "epoch": 2.56, + "grad_norm": 2.515625, + "learning_rate": 2.940952380952381e-06, + "loss": 5.951727294921875, + "step": 2240 + }, + { + "epoch": 2.571428571428571, + "grad_norm": 3.53125, + "learning_rate": 2.8647619047619052e-06, + "loss": 5.871749496459961, + "step": 2250 + }, + { + "epoch": 2.5828571428571427, + "grad_norm": 2.4375, + "learning_rate": 2.7885714285714287e-06, + "loss": 6.050479507446289, + "step": 2260 + }, + { + "epoch": 2.5942857142857143, + "grad_norm": 3.296875, + "learning_rate": 2.7123809523809526e-06, + "loss": 6.234370422363281, + "step": 2270 + }, + { + "epoch": 2.605714285714286, + "grad_norm": 2.546875, + "learning_rate": 2.6361904761904766e-06, + "loss": 6.30694465637207, + "step": 2280 + }, + { + "epoch": 2.617142857142857, + "grad_norm": 3.328125, + "learning_rate": 2.56e-06, + "loss": 6.2887931823730465, + "step": 2290 + }, + { + "epoch": 2.6285714285714286, + "grad_norm": 2.78125, + "learning_rate": 2.483809523809524e-06, + "loss": 6.163116455078125, + "step": 2300 + }, + { + "epoch": 2.64, + "grad_norm": 3.265625, + "learning_rate": 2.407619047619048e-06, + "loss": 6.079030990600586, + "step": 2310 + }, + { + "epoch": 2.6514285714285712, + "grad_norm": 2.265625, + "learning_rate": 2.331428571428572e-06, + "loss": 6.324145889282226, + "step": 2320 + }, + { + "epoch": 2.662857142857143, + "grad_norm": 3.484375, + "learning_rate": 2.2552380952380953e-06, + "loss": 6.187547302246093, + "step": 2330 + }, + { + "epoch": 2.6742857142857144, + "grad_norm": 2.484375, + "learning_rate": 2.179047619047619e-06, + "loss": 6.41395263671875, + "step": 2340 + }, + { + "epoch": 2.685714285714286, + "grad_norm": 3.0625, + "learning_rate": 2.102857142857143e-06, + "loss": 6.167239761352539, + "step": 2350 + }, + { + "epoch": 2.697142857142857, + "grad_norm": 2.5, + "learning_rate": 2.0266666666666666e-06, + "loss": 6.092725372314453, + "step": 2360 + }, + { + "epoch": 2.7085714285714286, + "grad_norm": 2.328125, + "learning_rate": 1.9504761904761905e-06, + "loss": 5.933970260620117, + "step": 2370 + }, + { + "epoch": 2.7199999999999998, + "grad_norm": 2.703125, + "learning_rate": 1.8742857142857142e-06, + "loss": 6.400347900390625, + "step": 2380 + }, + { + "epoch": 2.7314285714285713, + "grad_norm": 3.53125, + "learning_rate": 1.7980952380952381e-06, + "loss": 5.9839427947998045, + "step": 2390 + }, + { + "epoch": 2.742857142857143, + "grad_norm": 2.890625, + "learning_rate": 1.721904761904762e-06, + "loss": 5.9085124969482425, + "step": 2400 + }, + { + "epoch": 2.7542857142857144, + "grad_norm": 2.4375, + "learning_rate": 1.6457142857142857e-06, + "loss": 6.027331161499023, + "step": 2410 + }, + { + "epoch": 2.7657142857142856, + "grad_norm": 2.296875, + "learning_rate": 1.5695238095238097e-06, + "loss": 6.011427307128907, + "step": 2420 + }, + { + "epoch": 2.777142857142857, + "grad_norm": 2.90625, + "learning_rate": 1.4933333333333336e-06, + "loss": 6.09801139831543, + "step": 2430 + }, + { + "epoch": 2.7885714285714287, + "grad_norm": 2.46875, + "learning_rate": 1.4171428571428573e-06, + "loss": 6.347718048095703, + "step": 2440 + }, + { + "epoch": 2.8, + "grad_norm": 2.9375, + "learning_rate": 1.3409523809523812e-06, + "loss": 6.512370300292969, + "step": 2450 + }, + { + "epoch": 2.8114285714285714, + "grad_norm": 3.046875, + "learning_rate": 1.2647619047619047e-06, + "loss": 6.259450912475586, + "step": 2460 + }, + { + "epoch": 2.822857142857143, + "grad_norm": 3.171875, + "learning_rate": 1.1885714285714286e-06, + "loss": 6.226011276245117, + "step": 2470 + }, + { + "epoch": 2.8342857142857145, + "grad_norm": 3.359375, + "learning_rate": 1.1123809523809525e-06, + "loss": 6.207767486572266, + "step": 2480 + }, + { + "epoch": 2.8457142857142856, + "grad_norm": 3.4375, + "learning_rate": 1.0361904761904764e-06, + "loss": 6.071306610107422, + "step": 2490 + }, + { + "epoch": 2.857142857142857, + "grad_norm": 3.71875, + "learning_rate": 9.600000000000001e-07, + "loss": 6.418291473388672, + "step": 2500 + }, + { + "epoch": 2.8685714285714283, + "grad_norm": 2.671875, + "learning_rate": 8.838095238095238e-07, + "loss": 5.796173858642578, + "step": 2510 + }, + { + "epoch": 2.88, + "grad_norm": 3.15625, + "learning_rate": 8.076190476190476e-07, + "loss": 5.797344589233399, + "step": 2520 + }, + { + "epoch": 2.8914285714285715, + "grad_norm": 2.046875, + "learning_rate": 7.314285714285715e-07, + "loss": 6.1136119842529295, + "step": 2530 + }, + { + "epoch": 2.902857142857143, + "grad_norm": 2.703125, + "learning_rate": 6.552380952380954e-07, + "loss": 6.141954040527343, + "step": 2540 + }, + { + "epoch": 2.914285714285714, + "grad_norm": 2.15625, + "learning_rate": 5.790476190476191e-07, + "loss": 6.122183609008789, + "step": 2550 + }, + { + "epoch": 2.9257142857142857, + "grad_norm": 3.109375, + "learning_rate": 5.028571428571429e-07, + "loss": 6.6786651611328125, + "step": 2560 + }, + { + "epoch": 2.9371428571428573, + "grad_norm": 2.96875, + "learning_rate": 4.266666666666667e-07, + "loss": 6.168092727661133, + "step": 2570 + }, + { + "epoch": 2.9485714285714284, + "grad_norm": 2.1875, + "learning_rate": 3.5047619047619054e-07, + "loss": 6.07899284362793, + "step": 2580 + }, + { + "epoch": 2.96, + "grad_norm": 3.359375, + "learning_rate": 2.742857142857143e-07, + "loss": 5.709892272949219, + "step": 2590 + }, + { + "epoch": 2.9714285714285715, + "grad_norm": 3.015625, + "learning_rate": 1.980952380952381e-07, + "loss": 5.968305206298828, + "step": 2600 + }, + { + "epoch": 2.982857142857143, + "grad_norm": 2.609375, + "learning_rate": 1.219047619047619e-07, + "loss": 5.968548965454102, + "step": 2610 + }, + { + "epoch": 2.994285714285714, + "grad_norm": 3.421875, + "learning_rate": 4.571428571428572e-08, + "loss": 6.037314987182617, + "step": 2620 + } + ], + "logging_steps": 10, + "max_steps": 2625, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.4008456502528717e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-2625/training_args.bin b/checkpoint-2625/training_args.bin new file mode 100644 index 0000000..38338f4 --- /dev/null +++ b/checkpoint-2625/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8a2ef5b8d79835ff6ba94d10403139a3a9af53e51adfe2987ac1e7ff4f8e1ec +size 6289 diff --git a/checkpoint-875/chat_template.jinja b/checkpoint-875/chat_template.jinja new file mode 100644 index 0000000..64983ee --- /dev/null +++ b/checkpoint-875/chat_template.jinja @@ -0,0 +1,56 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" and not message.tool_calls %} + {{- '<|im_start|>assistant\n' }}{% generation %}{{- message.content + '<|im_end|>\n' }}{% endgeneration %} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>assistant' }}{% generation %} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }}{% endgeneration %} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-875/config.json b/checkpoint-875/config.json new file mode 100644 index 0000000..c097f0c --- /dev/null +++ b/checkpoint-875/config.json @@ -0,0 +1,58 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.5.3", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-875/generation_config.json b/checkpoint-875/generation_config.json new file mode 100644 index 0000000..61f88e1 --- /dev/null +++ b/checkpoint-875/generation_config.json @@ -0,0 +1,10 @@ +{ + "do_sample": false, + "eos_token_id": [ + 151645 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.5.3", + "use_cache": false +} diff --git a/checkpoint-875/model.safetensors b/checkpoint-875/model.safetensors new file mode 100644 index 0000000..ca4e609 --- /dev/null +++ b/checkpoint-875/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9233c5c85ca8499a6c3666385fe2fa4e906eb311454b8df8528ea867d093ab5d +size 988097824 diff --git a/checkpoint-875/optimizer.pt b/checkpoint-875/optimizer.pt new file mode 100644 index 0000000..a05a150 --- /dev/null +++ b/checkpoint-875/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa2de7f228893361233d2483c32e1ea56e9fca3f0a4a38c90d35aba8f6833ac5 +size 1976378699 diff --git a/checkpoint-875/rng_state.pth b/checkpoint-875/rng_state.pth new file mode 100644 index 0000000..47c232f --- /dev/null +++ b/checkpoint-875/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ce952e06ca433c9f1a93c4fbb2e94e7dc9c5160b247b0f2a5c3358ba3092fb3 +size 14645 diff --git a/checkpoint-875/scheduler.pt b/checkpoint-875/scheduler.pt new file mode 100644 index 0000000..fb70995 --- /dev/null +++ b/checkpoint-875/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:977dc1268f934bf99945ed0df8810e94a86fb8f53f0a4e7c78e4ae26e606231a +size 1465 diff --git a/checkpoint-875/tokenizer.json b/checkpoint-875/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-875/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-875/tokenizer_config.json b/checkpoint-875/tokenizer_config.json new file mode 100644 index 0000000..0385b12 --- /dev/null +++ b/checkpoint-875/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-875/trainer_state.json b/checkpoint-875/trainer_state.json new file mode 100644 index 0000000..253e083 --- /dev/null +++ b/checkpoint-875/trainer_state.json @@ -0,0 +1,643 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 875, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.011428571428571429, + "grad_norm": 3.078125, + "learning_rate": 1.9931428571428572e-05, + "loss": 7.941626739501953, + "step": 10 + }, + { + "epoch": 0.022857142857142857, + "grad_norm": 2.9375, + "learning_rate": 1.9855238095238097e-05, + "loss": 8.19613037109375, + "step": 20 + }, + { + "epoch": 0.03428571428571429, + "grad_norm": 3.5625, + "learning_rate": 1.977904761904762e-05, + "loss": 7.826921081542968, + "step": 30 + }, + { + "epoch": 0.045714285714285714, + "grad_norm": 3.15625, + "learning_rate": 1.9702857142857144e-05, + "loss": 7.580943298339844, + "step": 40 + }, + { + "epoch": 0.05714285714285714, + "grad_norm": 2.515625, + "learning_rate": 1.9626666666666666e-05, + "loss": 7.483664703369141, + "step": 50 + }, + { + "epoch": 0.06857142857142857, + "grad_norm": 3.859375, + "learning_rate": 1.955047619047619e-05, + "loss": 7.480722808837891, + "step": 60 + }, + { + "epoch": 0.08, + "grad_norm": 3.46875, + "learning_rate": 1.9474285714285717e-05, + "loss": 7.279670715332031, + "step": 70 + }, + { + "epoch": 0.09142857142857143, + "grad_norm": 2.5625, + "learning_rate": 1.9398095238095242e-05, + "loss": 7.584724426269531, + "step": 80 + }, + { + "epoch": 0.10285714285714286, + "grad_norm": 2.515625, + "learning_rate": 1.9321904761904764e-05, + "loss": 7.565158081054688, + "step": 90 + }, + { + "epoch": 0.11428571428571428, + "grad_norm": 3.5625, + "learning_rate": 1.924571428571429e-05, + "loss": 7.366996002197266, + "step": 100 + }, + { + "epoch": 0.12571428571428572, + "grad_norm": 3.109375, + "learning_rate": 1.916952380952381e-05, + "loss": 7.434650421142578, + "step": 110 + }, + { + "epoch": 0.13714285714285715, + "grad_norm": 3.328125, + "learning_rate": 1.9093333333333336e-05, + "loss": 6.9210762023925785, + "step": 120 + }, + { + "epoch": 0.14857142857142858, + "grad_norm": 3.015625, + "learning_rate": 1.9017142857142858e-05, + "loss": 7.027304840087891, + "step": 130 + }, + { + "epoch": 0.16, + "grad_norm": 3.09375, + "learning_rate": 1.8940952380952383e-05, + "loss": 7.241950225830078, + "step": 140 + }, + { + "epoch": 0.17142857142857143, + "grad_norm": 3.0, + "learning_rate": 1.8864761904761905e-05, + "loss": 7.158929443359375, + "step": 150 + }, + { + "epoch": 0.18285714285714286, + "grad_norm": 2.890625, + "learning_rate": 1.878857142857143e-05, + "loss": 6.856580352783203, + "step": 160 + }, + { + "epoch": 0.19428571428571428, + "grad_norm": 4.09375, + "learning_rate": 1.871238095238095e-05, + "loss": 6.740969848632813, + "step": 170 + }, + { + "epoch": 0.2057142857142857, + "grad_norm": 3.40625, + "learning_rate": 1.8636190476190477e-05, + "loss": 7.115393829345703, + "step": 180 + }, + { + "epoch": 0.21714285714285714, + "grad_norm": 2.9375, + "learning_rate": 1.8560000000000002e-05, + "loss": 7.437998199462891, + "step": 190 + }, + { + "epoch": 0.22857142857142856, + "grad_norm": 2.96875, + "learning_rate": 1.8483809523809527e-05, + "loss": 6.879551696777344, + "step": 200 + }, + { + "epoch": 0.24, + "grad_norm": 2.890625, + "learning_rate": 1.840761904761905e-05, + "loss": 6.8273162841796875, + "step": 210 + }, + { + "epoch": 0.25142857142857145, + "grad_norm": 3.0, + "learning_rate": 1.8331428571428574e-05, + "loss": 6.558704376220703, + "step": 220 + }, + { + "epoch": 0.26285714285714284, + "grad_norm": 4.40625, + "learning_rate": 1.8255238095238096e-05, + "loss": 7.156157684326172, + "step": 230 + }, + { + "epoch": 0.2742857142857143, + "grad_norm": 2.671875, + "learning_rate": 1.817904761904762e-05, + "loss": 6.985882568359375, + "step": 240 + }, + { + "epoch": 0.2857142857142857, + "grad_norm": 2.46875, + "learning_rate": 1.8102857142857143e-05, + "loss": 7.106211090087891, + "step": 250 + }, + { + "epoch": 0.29714285714285715, + "grad_norm": 2.984375, + "learning_rate": 1.8026666666666668e-05, + "loss": 7.014789581298828, + "step": 260 + }, + { + "epoch": 0.30857142857142855, + "grad_norm": 2.5, + "learning_rate": 1.7950476190476193e-05, + "loss": 6.632127380371093, + "step": 270 + }, + { + "epoch": 0.32, + "grad_norm": 2.84375, + "learning_rate": 1.7874285714285715e-05, + "loss": 6.882701110839844, + "step": 280 + }, + { + "epoch": 0.3314285714285714, + "grad_norm": 3.78125, + "learning_rate": 1.779809523809524e-05, + "loss": 6.729277801513672, + "step": 290 + }, + { + "epoch": 0.34285714285714286, + "grad_norm": 3.578125, + "learning_rate": 1.7721904761904762e-05, + "loss": 6.794551849365234, + "step": 300 + }, + { + "epoch": 0.35428571428571426, + "grad_norm": 2.828125, + "learning_rate": 1.7645714285714287e-05, + "loss": 6.83813247680664, + "step": 310 + }, + { + "epoch": 0.3657142857142857, + "grad_norm": 2.875, + "learning_rate": 1.7569523809523812e-05, + "loss": 6.796338653564453, + "step": 320 + }, + { + "epoch": 0.37714285714285717, + "grad_norm": 2.90625, + "learning_rate": 1.7493333333333334e-05, + "loss": 6.7025390625, + "step": 330 + }, + { + "epoch": 0.38857142857142857, + "grad_norm": 2.953125, + "learning_rate": 1.741714285714286e-05, + "loss": 6.541476440429688, + "step": 340 + }, + { + "epoch": 0.4, + "grad_norm": 2.59375, + "learning_rate": 1.734095238095238e-05, + "loss": 6.812630462646484, + "step": 350 + }, + { + "epoch": 0.4114285714285714, + "grad_norm": 3.609375, + "learning_rate": 1.7264761904761906e-05, + "loss": 6.889384460449219, + "step": 360 + }, + { + "epoch": 0.4228571428571429, + "grad_norm": 4.03125, + "learning_rate": 1.718857142857143e-05, + "loss": 7.129203796386719, + "step": 370 + }, + { + "epoch": 0.4342857142857143, + "grad_norm": 2.40625, + "learning_rate": 1.7112380952380953e-05, + "loss": 6.382967376708985, + "step": 380 + }, + { + "epoch": 0.44571428571428573, + "grad_norm": 3.515625, + "learning_rate": 1.703619047619048e-05, + "loss": 6.663212585449219, + "step": 390 + }, + { + "epoch": 0.45714285714285713, + "grad_norm": 2.90625, + "learning_rate": 1.696e-05, + "loss": 6.636187744140625, + "step": 400 + }, + { + "epoch": 0.4685714285714286, + "grad_norm": 3.1875, + "learning_rate": 1.6883809523809525e-05, + "loss": 6.64324722290039, + "step": 410 + }, + { + "epoch": 0.48, + "grad_norm": 2.828125, + "learning_rate": 1.6807619047619047e-05, + "loss": 6.663475036621094, + "step": 420 + }, + { + "epoch": 0.49142857142857144, + "grad_norm": 3.328125, + "learning_rate": 1.6731428571428572e-05, + "loss": 6.452519226074219, + "step": 430 + }, + { + "epoch": 0.5028571428571429, + "grad_norm": 2.421875, + "learning_rate": 1.6655238095238098e-05, + "loss": 6.683634948730469, + "step": 440 + }, + { + "epoch": 0.5142857142857142, + "grad_norm": 3.046875, + "learning_rate": 1.6579047619047623e-05, + "loss": 6.780387878417969, + "step": 450 + }, + { + "epoch": 0.5257142857142857, + "grad_norm": 2.53125, + "learning_rate": 1.6502857142857145e-05, + "loss": 7.1892578125, + "step": 460 + }, + { + "epoch": 0.5371428571428571, + "grad_norm": 2.71875, + "learning_rate": 1.642666666666667e-05, + "loss": 6.890229034423828, + "step": 470 + }, + { + "epoch": 0.5485714285714286, + "grad_norm": 3.140625, + "learning_rate": 1.635047619047619e-05, + "loss": 6.747575378417968, + "step": 480 + }, + { + "epoch": 0.56, + "grad_norm": 3.3125, + "learning_rate": 1.6274285714285717e-05, + "loss": 6.98467788696289, + "step": 490 + }, + { + "epoch": 0.5714285714285714, + "grad_norm": 2.890625, + "learning_rate": 1.619809523809524e-05, + "loss": 6.455801391601563, + "step": 500 + }, + { + "epoch": 0.5828571428571429, + "grad_norm": 2.578125, + "learning_rate": 1.6121904761904764e-05, + "loss": 6.525889587402344, + "step": 510 + }, + { + "epoch": 0.5942857142857143, + "grad_norm": 3.375, + "learning_rate": 1.6045714285714286e-05, + "loss": 6.479692840576172, + "step": 520 + }, + { + "epoch": 0.6057142857142858, + "grad_norm": 2.875, + "learning_rate": 1.596952380952381e-05, + "loss": 6.6360313415527346, + "step": 530 + }, + { + "epoch": 0.6171428571428571, + "grad_norm": 2.703125, + "learning_rate": 1.5893333333333333e-05, + "loss": 6.547400665283203, + "step": 540 + }, + { + "epoch": 0.6285714285714286, + "grad_norm": 2.9375, + "learning_rate": 1.5817142857142858e-05, + "loss": 6.748792266845703, + "step": 550 + }, + { + "epoch": 0.64, + "grad_norm": 3.28125, + "learning_rate": 1.5740952380952383e-05, + "loss": 6.727365112304687, + "step": 560 + }, + { + "epoch": 0.6514285714285715, + "grad_norm": 2.984375, + "learning_rate": 1.5664761904761908e-05, + "loss": 6.611011505126953, + "step": 570 + }, + { + "epoch": 0.6628571428571428, + "grad_norm": 2.984375, + "learning_rate": 1.558857142857143e-05, + "loss": 6.471088409423828, + "step": 580 + }, + { + "epoch": 0.6742857142857143, + "grad_norm": 2.84375, + "learning_rate": 1.5512380952380955e-05, + "loss": 6.472038269042969, + "step": 590 + }, + { + "epoch": 0.6857142857142857, + "grad_norm": 3.15625, + "learning_rate": 1.5436190476190477e-05, + "loss": 6.747550201416016, + "step": 600 + }, + { + "epoch": 0.6971428571428572, + "grad_norm": 2.75, + "learning_rate": 1.5360000000000002e-05, + "loss": 6.638180541992187, + "step": 610 + }, + { + "epoch": 0.7085714285714285, + "grad_norm": 3.078125, + "learning_rate": 1.5283809523809524e-05, + "loss": 6.535469818115234, + "step": 620 + }, + { + "epoch": 0.72, + "grad_norm": 3.109375, + "learning_rate": 1.5207619047619049e-05, + "loss": 6.981074523925781, + "step": 630 + }, + { + "epoch": 0.7314285714285714, + "grad_norm": 2.78125, + "learning_rate": 1.5131428571428572e-05, + "loss": 6.307097625732422, + "step": 640 + }, + { + "epoch": 0.7428571428571429, + "grad_norm": 2.8125, + "learning_rate": 1.5055238095238096e-05, + "loss": 6.617960357666016, + "step": 650 + }, + { + "epoch": 0.7542857142857143, + "grad_norm": 2.9375, + "learning_rate": 1.497904761904762e-05, + "loss": 6.845116424560547, + "step": 660 + }, + { + "epoch": 0.7657142857142857, + "grad_norm": 2.140625, + "learning_rate": 1.4902857142857143e-05, + "loss": 6.583855438232422, + "step": 670 + }, + { + "epoch": 0.7771428571428571, + "grad_norm": 2.703125, + "learning_rate": 1.4826666666666666e-05, + "loss": 6.485343933105469, + "step": 680 + }, + { + "epoch": 0.7885714285714286, + "grad_norm": 2.96875, + "learning_rate": 1.475047619047619e-05, + "loss": 6.277442169189453, + "step": 690 + }, + { + "epoch": 0.8, + "grad_norm": 3.078125, + "learning_rate": 1.4674285714285717e-05, + "loss": 6.420030212402343, + "step": 700 + }, + { + "epoch": 0.8114285714285714, + "grad_norm": 2.625, + "learning_rate": 1.459809523809524e-05, + "loss": 6.779734802246094, + "step": 710 + }, + { + "epoch": 0.8228571428571428, + "grad_norm": 3.28125, + "learning_rate": 1.4521904761904764e-05, + "loss": 6.647151947021484, + "step": 720 + }, + { + "epoch": 0.8342857142857143, + "grad_norm": 2.5625, + "learning_rate": 1.4445714285714287e-05, + "loss": 6.453370666503906, + "step": 730 + }, + { + "epoch": 0.8457142857142858, + "grad_norm": 2.5625, + "learning_rate": 1.436952380952381e-05, + "loss": 6.6205802917480465, + "step": 740 + }, + { + "epoch": 0.8571428571428571, + "grad_norm": 2.265625, + "learning_rate": 1.4293333333333334e-05, + "loss": 6.353859329223633, + "step": 750 + }, + { + "epoch": 0.8685714285714285, + "grad_norm": 3.015625, + "learning_rate": 1.4217142857142858e-05, + "loss": 6.357454299926758, + "step": 760 + }, + { + "epoch": 0.88, + "grad_norm": 3.03125, + "learning_rate": 1.4140952380952381e-05, + "loss": 6.682913208007813, + "step": 770 + }, + { + "epoch": 0.8914285714285715, + "grad_norm": 2.625, + "learning_rate": 1.4064761904761905e-05, + "loss": 6.127779769897461, + "step": 780 + }, + { + "epoch": 0.9028571428571428, + "grad_norm": 4.375, + "learning_rate": 1.398857142857143e-05, + "loss": 6.364295578002929, + "step": 790 + }, + { + "epoch": 0.9142857142857143, + "grad_norm": 2.8125, + "learning_rate": 1.3912380952380953e-05, + "loss": 6.2481742858886715, + "step": 800 + }, + { + "epoch": 0.9257142857142857, + "grad_norm": 2.875, + "learning_rate": 1.3836190476190477e-05, + "loss": 6.126508712768555, + "step": 810 + }, + { + "epoch": 0.9371428571428572, + "grad_norm": 3.0, + "learning_rate": 1.376e-05, + "loss": 6.082854080200195, + "step": 820 + }, + { + "epoch": 0.9485714285714286, + "grad_norm": 3.09375, + "learning_rate": 1.3683809523809526e-05, + "loss": 6.128392028808594, + "step": 830 + }, + { + "epoch": 0.96, + "grad_norm": 3.40625, + "learning_rate": 1.3607619047619049e-05, + "loss": 6.288749313354492, + "step": 840 + }, + { + "epoch": 0.9714285714285714, + "grad_norm": 2.78125, + "learning_rate": 1.3531428571428573e-05, + "loss": 6.770208740234375, + "step": 850 + }, + { + "epoch": 0.9828571428571429, + "grad_norm": 2.96875, + "learning_rate": 1.3455238095238096e-05, + "loss": 6.299668502807617, + "step": 860 + }, + { + "epoch": 0.9942857142857143, + "grad_norm": 2.765625, + "learning_rate": 1.3379047619047621e-05, + "loss": 6.4255424499511715, + "step": 870 + } + ], + "logging_steps": 10, + "max_steps": 2625, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.697082881099981e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-875/training_args.bin b/checkpoint-875/training_args.bin new file mode 100644 index 0000000..38338f4 --- /dev/null +++ b/checkpoint-875/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8a2ef5b8d79835ff6ba94d10403139a3a9af53e51adfe2987ac1e7ff4f8e1ec +size 6289 diff --git a/config.json b/config.json new file mode 100644 index 0000000..c097f0c --- /dev/null +++ b/config.json @@ -0,0 +1,58 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.5.3", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..61f88e1 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,10 @@ +{ + "do_sample": false, + "eos_token_id": [ + 151645 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.5.3", + "use_cache": false +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..b0ae7da --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10cbf78c24b74c3578136d377abf51eeb08995479f597d62e92ca7c6298a5d05 +size 988097824 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..0385b12 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..80dca5a --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2c5a96fca09ac848cb9a7b8f3e374248bbdb8129e9aec130a23e0eeaaa5fff8 +size 6353