From 9bde458e6a9c9374120a2d0683826e5b65502a8e Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 20 Jun 2026 18:04:20 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: heyalexchoi/qwen3-1.7b-math-sft Source: Original Platform --- .gitattributes | 37 + README.md | 58 + chat_template.jinja | 85 ++ config.json | 63 ++ generation_config.json | 7 + last-checkpoint/chat_template.jinja | 85 ++ last-checkpoint/config.json | 63 ++ last-checkpoint/generation_config.json | 9 + last-checkpoint/model.safetensors | 3 + last-checkpoint/optimizer.pt | 3 + last-checkpoint/rng_state.pth | 3 + last-checkpoint/scheduler.pt | 3 + last-checkpoint/tokenizer.json | 3 + last-checkpoint/tokenizer_config.json | 29 + last-checkpoint/trainer_state.json | 1447 ++++++++++++++++++++++++ last-checkpoint/training_args.bin | 3 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 29 + training_args.bin | 3 + 20 files changed, 1939 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 last-checkpoint/chat_template.jinja create mode 100644 last-checkpoint/config.json create mode 100644 last-checkpoint/generation_config.json create mode 100644 last-checkpoint/model.safetensors create mode 100644 last-checkpoint/optimizer.pt create mode 100644 last-checkpoint/rng_state.pth create mode 100644 last-checkpoint/scheduler.pt create mode 100644 last-checkpoint/tokenizer.json create mode 100644 last-checkpoint/tokenizer_config.json create mode 100644 last-checkpoint/trainer_state.json create mode 100644 last-checkpoint/training_args.bin create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b6153b4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +last-checkpoint/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..eb581d3 --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-1.7B-Base +library_name: transformers +model_name: qwen3-1.7b-math-sft +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for qwen3-1.7b-math-sft + +This model is a fine-tuned version of [Qwen/Qwen3-1.7B-Base](https://huggingface.co/Qwen/Qwen3-1.7B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="heyalexchoi/qwen3-1.7b-math-sft", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/heyalexchoi/qwen3-math-rlvr/runs/vrgi85mj) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 1.0.0 +- Transformers: 5.5.3 +- Pytorch: 2.4.1+cu124 +- Datasets: 4.8.4 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..699ff8d --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..26c7757 --- /dev/null +++ b/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151643, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.5.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..7f4b279 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "do_sample": false, + "eos_token_id": 151645, + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.5.3" +} \ No newline at end of file diff --git a/last-checkpoint/chat_template.jinja b/last-checkpoint/chat_template.jinja new file mode 100644 index 0000000..699ff8d --- /dev/null +++ b/last-checkpoint/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/last-checkpoint/config.json b/last-checkpoint/config.json new file mode 100644 index 0000000..26c7757 --- /dev/null +++ b/last-checkpoint/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151643, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.5.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/last-checkpoint/generation_config.json b/last-checkpoint/generation_config.json new file mode 100644 index 0000000..481d96d --- /dev/null +++ b/last-checkpoint/generation_config.json @@ -0,0 +1,9 @@ +{ + "do_sample": false, + "eos_token_id": [ + 151643 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.5.3" +} diff --git a/last-checkpoint/model.safetensors b/last-checkpoint/model.safetensors new file mode 100644 index 0000000..5b07aba --- /dev/null +++ b/last-checkpoint/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9822507c09f03b5474cec78cc118ee618cd3069c353f5144d8d0083adfb886af +size 3441185608 diff --git a/last-checkpoint/optimizer.pt b/last-checkpoint/optimizer.pt new file mode 100644 index 0000000..11f7a26 --- /dev/null +++ b/last-checkpoint/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c47209943226117d946620a9186dafc2554bb241f1faec5b3d3f7a0acb3e0f9b +size 6882567586 diff --git a/last-checkpoint/rng_state.pth b/last-checkpoint/rng_state.pth new file mode 100644 index 0000000..9402f70 --- /dev/null +++ b/last-checkpoint/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52cca5856c568bc52c683b690919168fa27bfbdfefc6e0a62355afa6011157c3 +size 14244 diff --git a/last-checkpoint/scheduler.pt b/last-checkpoint/scheduler.pt new file mode 100644 index 0000000..db7051d --- /dev/null +++ b/last-checkpoint/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df92808a756148e8069513f992e97c54f3dfaa54d0714f0739c5bc226fb3fe83 +size 1064 diff --git a/last-checkpoint/tokenizer.json b/last-checkpoint/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/last-checkpoint/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/last-checkpoint/tokenizer_config.json b/last-checkpoint/tokenizer_config.json new file mode 100644 index 0000000..c960ecf --- /dev/null +++ b/last-checkpoint/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/last-checkpoint/trainer_state.json b/last-checkpoint/trainer_state.json new file mode 100644 index 0000000..29ba751 --- /dev/null +++ b/last-checkpoint/trainer_state.json @@ -0,0 +1,1447 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 100, + "global_step": 1275, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.47261681938543915, + "epoch": 0.023543260741612712, + "grad_norm": 0.53125, + "learning_rate": 1.40625e-06, + "loss": 0.7165036201477051, + "mean_token_accuracy": 0.8025671623647213, + "num_tokens": 796538.0, + "step": 10 + }, + { + "entropy": 0.5139306713826954, + "epoch": 0.047086521483225424, + "grad_norm": 0.388671875, + "learning_rate": 2.96875e-06, + "loss": 0.7095730304718018, + "mean_token_accuracy": 0.7971717294305563, + "num_tokens": 1544899.0, + "step": 20 + }, + { + "entropy": 0.5956933442503214, + "epoch": 0.07062978222483814, + "grad_norm": 0.2099609375, + "learning_rate": 4.53125e-06, + "loss": 0.6996610641479493, + "mean_token_accuracy": 0.7971747264266014, + "num_tokens": 2317683.0, + "step": 30 + }, + { + "entropy": 0.6512754812836647, + "epoch": 0.09417304296645085, + "grad_norm": 0.1474609375, + "learning_rate": 6.093750000000001e-06, + "loss": 0.6695662021636963, + "mean_token_accuracy": 0.8040148165076971, + "num_tokens": 3111422.0, + "step": 40 + }, + { + "entropy": 0.6480279197916389, + "epoch": 0.11771630370806356, + "grad_norm": 0.1083984375, + "learning_rate": 7.656250000000001e-06, + "loss": 0.6375884056091309, + "mean_token_accuracy": 0.8059590008109808, + "num_tokens": 3973704.0, + "step": 50 + }, + { + "entropy": 0.6220262819901109, + "epoch": 0.14125956444967627, + "grad_norm": 0.0986328125, + "learning_rate": 9.21875e-06, + "loss": 0.623339033126831, + "mean_token_accuracy": 0.8087143875658512, + "num_tokens": 4730669.0, + "step": 60 + }, + { + "entropy": 0.6261076383292675, + "epoch": 0.164802825191289, + "grad_norm": 0.08056640625, + "learning_rate": 1.0781250000000001e-05, + "loss": 0.6200876712799073, + "mean_token_accuracy": 0.8125950843095779, + "num_tokens": 5502078.0, + "step": 70 + }, + { + "entropy": 0.6319901453331113, + "epoch": 0.1883460859329017, + "grad_norm": 0.06591796875, + "learning_rate": 1.234375e-05, + "loss": 0.6182626247406006, + "mean_token_accuracy": 0.8144623003900051, + "num_tokens": 6266614.0, + "step": 80 + }, + { + "entropy": 0.602002534456551, + "epoch": 0.21188934667451442, + "grad_norm": 0.0625, + "learning_rate": 1.3906250000000001e-05, + "loss": 0.5683388710021973, + "mean_token_accuracy": 0.8221714589744806, + "num_tokens": 7059460.0, + "step": 90 + }, + { + "entropy": 0.5908293653279543, + "epoch": 0.23543260741612712, + "grad_norm": 0.06689453125, + "learning_rate": 1.546875e-05, + "loss": 0.5792154312133789, + "mean_token_accuracy": 0.8240513067692519, + "num_tokens": 7862593.0, + "step": 100 + }, + { + "epoch": 0.23543260741612712, + "eval_entropy": 0.5910338105935624, + "eval_loss": 0.5686901211738586, + "eval_mean_token_accuracy": 0.823962179309163, + "eval_num_tokens": 7862593.0, + "eval_runtime": 139.6331, + "eval_samples_per_second": 2.564, + "eval_steps_per_second": 2.564, + "step": 100 + }, + { + "entropy": 0.5830874053761363, + "epoch": 0.2589758681577399, + "grad_norm": 0.058837890625, + "learning_rate": 1.703125e-05, + "loss": 0.5834993839263916, + "mean_token_accuracy": 0.8253755442798137, + "num_tokens": 8702391.0, + "step": 110 + }, + { + "entropy": 0.5565910406410695, + "epoch": 0.28251912889935255, + "grad_norm": 0.05859375, + "learning_rate": 1.859375e-05, + "loss": 0.5459747314453125, + "mean_token_accuracy": 0.8344478290528059, + "num_tokens": 9393136.0, + "step": 120 + }, + { + "entropy": 0.5650864312425256, + "epoch": 0.30606238964096527, + "grad_norm": 0.06103515625, + "learning_rate": 1.9999962490381906e-05, + "loss": 0.5637893676757812, + "mean_token_accuracy": 0.829344104602933, + "num_tokens": 10137523.0, + "step": 130 + }, + { + "entropy": 0.5473398096859455, + "epoch": 0.329605650382578, + "grad_norm": 0.0625, + "learning_rate": 1.999546167668739e-05, + "loss": 0.5404181003570556, + "mean_token_accuracy": 0.8342546194791793, + "num_tokens": 10946950.0, + "step": 140 + }, + { + "entropy": 0.5475172070786357, + "epoch": 0.3531489111241907, + "grad_norm": 0.0634765625, + "learning_rate": 1.998346280806768e-05, + "loss": 0.5369014263153076, + "mean_token_accuracy": 0.8347291376441717, + "num_tokens": 11724348.0, + "step": 150 + }, + { + "entropy": 0.5577760996297002, + "epoch": 0.3766921718658034, + "grad_norm": 0.06591796875, + "learning_rate": 1.9963974885425267e-05, + "loss": 0.5580552577972412, + "mean_token_accuracy": 0.8297208815813064, + "num_tokens": 12528416.0, + "step": 160 + }, + { + "entropy": 0.5457047978416085, + "epoch": 0.4002354326074161, + "grad_norm": 0.06103515625, + "learning_rate": 1.9937012527546064e-05, + "loss": 0.5416774749755859, + "mean_token_accuracy": 0.8351736020296812, + "num_tokens": 13254942.0, + "step": 170 + }, + { + "entropy": 0.5473680015653372, + "epoch": 0.42377869334902885, + "grad_norm": 0.058837890625, + "learning_rate": 1.9902595960133182e-05, + "loss": 0.5401034355163574, + "mean_token_accuracy": 0.8332272503525019, + "num_tokens": 14035743.0, + "step": 180 + }, + { + "entropy": 0.5565412795171142, + "epoch": 0.44732195409064157, + "grad_norm": 0.052734375, + "learning_rate": 1.986075100063469e-05, + "loss": 0.5520064353942871, + "mean_token_accuracy": 0.8305680822581053, + "num_tokens": 14777254.0, + "step": 190 + }, + { + "entropy": 0.5201416643336415, + "epoch": 0.47086521483225424, + "grad_norm": 0.055908203125, + "learning_rate": 1.9811509038876772e-05, + "loss": 0.5139257431030273, + "mean_token_accuracy": 0.8422017730772495, + "num_tokens": 15582880.0, + "step": 200 + }, + { + "epoch": 0.47086521483225424, + "eval_entropy": 0.5415721843695508, + "eval_loss": 0.5158530473709106, + "eval_mean_token_accuracy": 0.8355566641138918, + "eval_num_tokens": 15582880.0, + "eval_runtime": 142.3554, + "eval_samples_per_second": 2.515, + "eval_steps_per_second": 2.515, + "step": 200 + }, + { + "entropy": 0.5265132559463381, + "epoch": 0.49440847557386697, + "grad_norm": 0.05615234375, + "learning_rate": 1.9754907013516774e-05, + "loss": 0.5199352264404297, + "mean_token_accuracy": 0.8380576658993959, + "num_tokens": 16419990.0, + "step": 210 + }, + { + "entropy": 0.5467520544305444, + "epoch": 0.5179517363154797, + "grad_norm": 0.05859375, + "learning_rate": 1.9690987384333823e-05, + "loss": 0.5457062244415283, + "mean_token_accuracy": 0.8330623578280211, + "num_tokens": 17220133.0, + "step": 220 + }, + { + "entropy": 0.5202922957018018, + "epoch": 0.5414949970570924, + "grad_norm": 0.0732421875, + "learning_rate": 1.961979810037778e-05, + "loss": 0.5149586200714111, + "mean_token_accuracy": 0.8408578157424926, + "num_tokens": 17988703.0, + "step": 230 + }, + { + "entropy": 0.5157044745981694, + "epoch": 0.5650382577987051, + "grad_norm": 0.05859375, + "learning_rate": 1.954139256400049e-05, + "loss": 0.5135462760925293, + "mean_token_accuracy": 0.8410524364560843, + "num_tokens": 18760495.0, + "step": 240 + }, + { + "entropy": 0.5239256421104074, + "epoch": 0.5885815185403178, + "grad_norm": 0.0634765625, + "learning_rate": 1.9455829590796184e-05, + "loss": 0.5215154647827148, + "mean_token_accuracy": 0.8391639325767756, + "num_tokens": 19551514.0, + "step": 250 + }, + { + "entropy": 0.5295072626322508, + "epoch": 0.6121247792819305, + "grad_norm": 0.054443359375, + "learning_rate": 1.9363173365481223e-05, + "loss": 0.5391250133514405, + "mean_token_accuracy": 0.8365894909948111, + "num_tokens": 20367505.0, + "step": 260 + }, + { + "entropy": 0.513121047988534, + "epoch": 0.6356680400235433, + "grad_norm": 0.057861328125, + "learning_rate": 1.92634933937462e-05, + "loss": 0.5058225154876709, + "mean_token_accuracy": 0.842748760432005, + "num_tokens": 21071323.0, + "step": 270 + }, + { + "entropy": 0.5175997691228986, + "epoch": 0.659211300765156, + "grad_norm": 0.056396484375, + "learning_rate": 1.915686445011651e-05, + "loss": 0.5235993385314941, + "mean_token_accuracy": 0.8399815000593662, + "num_tokens": 21895697.0, + "step": 280 + }, + { + "entropy": 0.5231689540669322, + "epoch": 0.6827545615067687, + "grad_norm": 0.05126953125, + "learning_rate": 1.9043366521860558e-05, + "loss": 0.5182478904724122, + "mean_token_accuracy": 0.8393962673842907, + "num_tokens": 22679040.0, + "step": 290 + }, + { + "entropy": 0.5049041209742426, + "epoch": 0.7062978222483814, + "grad_norm": 0.062255859375, + "learning_rate": 1.892308474898763e-05, + "loss": 0.4974985599517822, + "mean_token_accuracy": 0.8447154741734266, + "num_tokens": 23386138.0, + "step": 300 + }, + { + "epoch": 0.7062978222483814, + "eval_entropy": 0.5194896821702659, + "eval_loss": 0.49936261773109436, + "eval_mean_token_accuracy": 0.8389908150254681, + "eval_num_tokens": 23386138.0, + "eval_runtime": 136.5483, + "eval_samples_per_second": 2.622, + "eval_steps_per_second": 2.622, + "step": 300 + }, + { + "entropy": 0.5262435658834874, + "epoch": 0.7298410829899941, + "grad_norm": 0.0546875, + "learning_rate": 1.879610936038046e-05, + "loss": 0.5304554462432861, + "mean_token_accuracy": 0.836079678311944, + "num_tokens": 24170172.0, + "step": 310 + }, + { + "entropy": 0.519707421399653, + "epoch": 0.7533843437316068, + "grad_norm": 0.05419921875, + "learning_rate": 1.86625356061104e-05, + "loss": 0.525363826751709, + "mean_token_accuracy": 0.8391638267785311, + "num_tokens": 24962487.0, + "step": 320 + }, + { + "entropy": 0.5116234797984361, + "epoch": 0.7769276044732195, + "grad_norm": 0.06591796875, + "learning_rate": 1.852246368598597e-05, + "loss": 0.5102458953857422, + "mean_token_accuracy": 0.8417690549045801, + "num_tokens": 25733914.0, + "step": 330 + }, + { + "entropy": 0.5263846382498741, + "epoch": 0.8004708652148322, + "grad_norm": 0.0546875, + "learning_rate": 1.837599867438838e-05, + "loss": 0.5220338821411132, + "mean_token_accuracy": 0.8377181738615036, + "num_tokens": 26579132.0, + "step": 340 + }, + { + "entropy": 0.5115921972319484, + "epoch": 0.824014125956445, + "grad_norm": 0.0615234375, + "learning_rate": 1.822325044145041e-05, + "loss": 0.49653100967407227, + "mean_token_accuracy": 0.8414182372391223, + "num_tokens": 27202803.0, + "step": 350 + }, + { + "entropy": 0.5089998429641127, + "epoch": 0.8475573866980577, + "grad_norm": 0.0673828125, + "learning_rate": 1.8064333570637768e-05, + "loss": 0.49803886413574217, + "mean_token_accuracy": 0.842063357308507, + "num_tokens": 27874601.0, + "step": 360 + }, + { + "entropy": 0.5171531289815903, + "epoch": 0.8711006474396704, + "grad_norm": 0.059814453125, + "learning_rate": 1.7899367272794777e-05, + "loss": 0.5236594676971436, + "mean_token_accuracy": 0.8396157670766116, + "num_tokens": 28758269.0, + "step": 370 + }, + { + "entropy": 0.5161923897452653, + "epoch": 0.8946439081812831, + "grad_norm": 0.047607421875, + "learning_rate": 1.7728475296718846e-05, + "loss": 0.5221555709838868, + "mean_token_accuracy": 0.8399753466248512, + "num_tokens": 29660317.0, + "step": 380 + }, + { + "entropy": 0.4955171117559075, + "epoch": 0.9181871689228959, + "grad_norm": 0.05517578125, + "learning_rate": 1.7551785836330814e-05, + "loss": 0.49817495346069335, + "mean_token_accuracy": 0.846830865368247, + "num_tokens": 30349463.0, + "step": 390 + }, + { + "entropy": 0.505057031661272, + "epoch": 0.9417304296645085, + "grad_norm": 0.064453125, + "learning_rate": 1.7369431434510793e-05, + "loss": 0.509951400756836, + "mean_token_accuracy": 0.8432859953492879, + "num_tokens": 31065185.0, + "step": 400 + }, + { + "epoch": 0.9417304296645085, + "eval_entropy": 0.5111477355883774, + "eval_loss": 0.4909190237522125, + "eval_mean_token_accuracy": 0.8413028815272134, + "eval_num_tokens": 31065185.0, + "eval_runtime": 131.4779, + "eval_samples_per_second": 2.723, + "eval_steps_per_second": 2.723, + "step": 400 + }, + { + "entropy": 0.4994705172255635, + "epoch": 0.9652736904061212, + "grad_norm": 0.059814453125, + "learning_rate": 1.718154888367167e-05, + "loss": 0.5129815101623535, + "mean_token_accuracy": 0.8450050104409457, + "num_tokens": 31796395.0, + "step": 410 + }, + { + "entropy": 0.5047325070947408, + "epoch": 0.9888169511477339, + "grad_norm": 0.054443359375, + "learning_rate": 1.698827912314484e-05, + "loss": 0.4937889099121094, + "mean_token_accuracy": 0.8428907435387373, + "num_tokens": 32560011.0, + "step": 420 + }, + { + "entropy": 0.5052787409378932, + "epoch": 1.0117716303708064, + "grad_norm": 0.05908203125, + "learning_rate": 1.678976713345513e-05, + "loss": 0.5136929988861084, + "mean_token_accuracy": 0.8436831063949145, + "num_tokens": 33280194.0, + "step": 430 + }, + { + "entropy": 0.49020985178649423, + "epoch": 1.035314891112419, + "grad_norm": 0.05908203125, + "learning_rate": 1.658616182756426e-05, + "loss": 0.4995232582092285, + "mean_token_accuracy": 0.847404608130455, + "num_tokens": 34034683.0, + "step": 440 + }, + { + "entropy": 0.513733028434217, + "epoch": 1.0588581518540319, + "grad_norm": 0.0673828125, + "learning_rate": 1.6377615939164387e-05, + "loss": 0.524590253829956, + "mean_token_accuracy": 0.8403749600052833, + "num_tokens": 34873128.0, + "step": 450 + }, + { + "entropy": 0.5072762167081237, + "epoch": 1.0824014125956445, + "grad_norm": 0.06298828125, + "learning_rate": 1.6164285908105526e-05, + "loss": 0.520505142211914, + "mean_token_accuracy": 0.8410047568380833, + "num_tokens": 35734174.0, + "step": 460 + }, + { + "entropy": 0.49719911720603704, + "epoch": 1.105944673337257, + "grad_norm": 0.06298828125, + "learning_rate": 1.594633176304287e-05, + "loss": 0.494351863861084, + "mean_token_accuracy": 0.8454386904835701, + "num_tokens": 36393777.0, + "step": 470 + }, + { + "entropy": 0.49072135630995034, + "epoch": 1.12948793407887, + "grad_norm": 0.05517578125, + "learning_rate": 1.5723917001391912e-05, + "loss": 0.4863902568817139, + "mean_token_accuracy": 0.8458908643573523, + "num_tokens": 37251068.0, + "step": 480 + }, + { + "entropy": 0.4939659545198083, + "epoch": 1.1530311948204826, + "grad_norm": 0.06494140625, + "learning_rate": 1.5497208466681547e-05, + "loss": 0.4998356342315674, + "mean_token_accuracy": 0.8443102762103081, + "num_tokens": 38031375.0, + "step": 490 + }, + { + "entropy": 0.4939275814220309, + "epoch": 1.1765744555620954, + "grad_norm": 0.055419921875, + "learning_rate": 1.526637622339706e-05, + "loss": 0.4955411911010742, + "mean_token_accuracy": 0.8455841716378927, + "num_tokens": 38860255.0, + "step": 500 + }, + { + "epoch": 1.1765744555620954, + "eval_entropy": 0.5045043116008769, + "eval_loss": 0.4862433075904846, + "eval_mean_token_accuracy": 0.8424804836678106, + "eval_num_tokens": 38860255.0, + "eval_runtime": 135.2782, + "eval_samples_per_second": 2.646, + "eval_steps_per_second": 2.646, + "step": 500 + }, + { + "entropy": 0.4797388563863933, + "epoch": 1.200117716303708, + "grad_norm": 0.056396484375, + "learning_rate": 1.5031593429406964e-05, + "loss": 0.484775447845459, + "mean_token_accuracy": 0.8496320709586144, + "num_tokens": 39714269.0, + "step": 510 + }, + { + "entropy": 0.5136959448456764, + "epoch": 1.2236609770453208, + "grad_norm": 0.05419921875, + "learning_rate": 1.4793036206069298e-05, + "loss": 0.5073910236358643, + "mean_token_accuracy": 0.839500718563795, + "num_tokens": 40429331.0, + "step": 520 + }, + { + "entropy": 0.5001886915415525, + "epoch": 1.2472042377869335, + "grad_norm": 0.059814453125, + "learning_rate": 1.455088350611494e-05, + "loss": 0.4849071502685547, + "mean_token_accuracy": 0.8452773235738278, + "num_tokens": 41120160.0, + "step": 530 + }, + { + "entropy": 0.49588965652510525, + "epoch": 1.270747498528546, + "grad_norm": 0.05322265625, + "learning_rate": 1.430531697940693e-05, + "loss": 0.5028933525085449, + "mean_token_accuracy": 0.8444558009505272, + "num_tokens": 41875187.0, + "step": 540 + }, + { + "entropy": 0.4997696656733751, + "epoch": 1.294290759270159, + "grad_norm": 0.056884765625, + "learning_rate": 1.4056520836676551e-05, + "loss": 0.5174314498901367, + "mean_token_accuracy": 0.8428490672260522, + "num_tokens": 42701302.0, + "step": 550 + }, + { + "entropy": 0.49855835027992723, + "epoch": 1.3178340200117717, + "grad_norm": 0.059814453125, + "learning_rate": 1.380468171133842e-05, + "loss": 0.5048851013183594, + "mean_token_accuracy": 0.8445437770336867, + "num_tokens": 43490481.0, + "step": 560 + }, + { + "entropy": 0.4793679520487785, + "epoch": 1.3413772807533844, + "grad_norm": 0.058837890625, + "learning_rate": 1.354998851948818e-05, + "loss": 0.48621335029602053, + "mean_token_accuracy": 0.8496838890016079, + "num_tokens": 44257286.0, + "step": 570 + }, + { + "entropy": 0.5044341992586852, + "epoch": 1.364920541494997, + "grad_norm": 0.06591796875, + "learning_rate": 1.3292632318187857e-05, + "loss": 0.5028728485107422, + "mean_token_accuracy": 0.8416947480291128, + "num_tokens": 45031884.0, + "step": 580 + }, + { + "entropy": 0.4860644720494747, + "epoch": 1.3884638022366098, + "grad_norm": 0.054443359375, + "learning_rate": 1.3032806162145193e-05, + "loss": 0.48934264183044435, + "mean_token_accuracy": 0.848303421586752, + "num_tokens": 45805257.0, + "step": 590 + }, + { + "entropy": 0.49228332079946996, + "epoch": 1.4120070629782224, + "grad_norm": 0.051025390625, + "learning_rate": 1.2770704958894424e-05, + "loss": 0.48878173828125, + "mean_token_accuracy": 0.8455546554178, + "num_tokens": 46529684.0, + "step": 600 + }, + { + "epoch": 1.4120070629782224, + "eval_entropy": 0.49637800954573646, + "eval_loss": 0.4830160439014435, + "eval_mean_token_accuracy": 0.8431555377371485, + "eval_num_tokens": 46529684.0, + "eval_runtime": 128.7956, + "eval_samples_per_second": 2.78, + "eval_steps_per_second": 2.78, + "step": 600 + }, + { + "entropy": 0.4935727553442121, + "epoch": 1.4355503237198353, + "grad_norm": 0.05712890625, + "learning_rate": 1.2506525322587207e-05, + "loss": 0.5051933765411377, + "mean_token_accuracy": 0.8451346360147, + "num_tokens": 47351703.0, + "step": 610 + }, + { + "entropy": 0.4920971740037203, + "epoch": 1.4590935844614479, + "grad_norm": 0.053466796875, + "learning_rate": 1.2240465426503296e-05, + "loss": 0.5081326484680175, + "mean_token_accuracy": 0.8450745042413473, + "num_tokens": 48111074.0, + "step": 620 + }, + { + "entropy": 0.4823517481796443, + "epoch": 1.4826368452030607, + "grad_norm": 0.052978515625, + "learning_rate": 1.197272485439167e-05, + "loss": 0.4746738910675049, + "mean_token_accuracy": 0.8489425461739302, + "num_tokens": 48899473.0, + "step": 630 + }, + { + "entropy": 0.4855606323108077, + "epoch": 1.5061801059446733, + "grad_norm": 0.0673828125, + "learning_rate": 1.1703504450753604e-05, + "loss": 0.4840294361114502, + "mean_token_accuracy": 0.8471518792212009, + "num_tokens": 49689664.0, + "step": 640 + }, + { + "entropy": 0.4964427137747407, + "epoch": 1.529723366686286, + "grad_norm": 0.055908203125, + "learning_rate": 1.1433006170179961e-05, + "loss": 0.499052095413208, + "mean_token_accuracy": 0.8452402614057064, + "num_tokens": 50432542.0, + "step": 650 + }, + { + "entropy": 0.48346656998619436, + "epoch": 1.5532666274278988, + "grad_norm": 0.059814453125, + "learning_rate": 1.1161432925855792e-05, + "loss": 0.49048147201538084, + "mean_token_accuracy": 0.8482994694262743, + "num_tokens": 51235911.0, + "step": 660 + }, + { + "entropy": 0.4915179075673223, + "epoch": 1.5768098881695116, + "grad_norm": 0.0673828125, + "learning_rate": 1.0888988437345851e-05, + "loss": 0.48571577072143557, + "mean_token_accuracy": 0.8463525585830212, + "num_tokens": 51914843.0, + "step": 670 + }, + { + "entropy": 0.5079520621336997, + "epoch": 1.6003531489111242, + "grad_norm": 0.0625, + "learning_rate": 1.061587707777515e-05, + "loss": 0.5094398975372314, + "mean_token_accuracy": 0.8409215942025184, + "num_tokens": 52788907.0, + "step": 680 + }, + { + "entropy": 0.4922261985018849, + "epoch": 1.6238964096527368, + "grad_norm": 0.06640625, + "learning_rate": 1.0342303720519331e-05, + "loss": 0.5027109146118164, + "mean_token_accuracy": 0.8461885400116443, + "num_tokens": 53610651.0, + "step": 690 + }, + { + "entropy": 0.4926283808425069, + "epoch": 1.6474396703943497, + "grad_norm": 0.052978515625, + "learning_rate": 1.0068473585519714e-05, + "loss": 0.48818025588989256, + "mean_token_accuracy": 0.8448698211461305, + "num_tokens": 54306359.0, + "step": 700 + }, + { + "epoch": 1.6474396703943497, + "eval_entropy": 0.49725391719927337, + "eval_loss": 0.48121583461761475, + "eval_mean_token_accuracy": 0.8434386894356605, + "eval_num_tokens": 54306359.0, + "eval_runtime": 129.9385, + "eval_samples_per_second": 2.755, + "eval_steps_per_second": 2.755, + "step": 700 + }, + { + "entropy": 0.50206049811095, + "epoch": 1.6709829311359623, + "grad_norm": 0.05126953125, + "learning_rate": 9.794592085338401e-06, + "loss": 0.5090510845184326, + "mean_token_accuracy": 0.8430146727710962, + "num_tokens": 55162088.0, + "step": 710 + }, + { + "entropy": 0.4935040036216378, + "epoch": 1.694526191877575, + "grad_norm": 0.051513671875, + "learning_rate": 9.520864671068875e-06, + "loss": 0.5052507400512696, + "mean_token_accuracy": 0.8460971619933844, + "num_tokens": 55957773.0, + "step": 720 + }, + { + "entropy": 0.4884185861796141, + "epoch": 1.7180694526191878, + "grad_norm": 0.06005859375, + "learning_rate": 9.247496678217704e-06, + "loss": 0.4981977939605713, + "mean_token_accuracy": 0.8484526939690114, + "num_tokens": 56705800.0, + "step": 730 + }, + { + "entropy": 0.47191233560442924, + "epoch": 1.7416127133608006, + "grad_norm": 0.06396484375, + "learning_rate": 8.974693172672936e-06, + "loss": 0.48612565994262696, + "mean_token_accuracy": 0.8515177499502897, + "num_tokens": 57410830.0, + "step": 740 + }, + { + "entropy": 0.4871264295652509, + "epoch": 1.7651559741024132, + "grad_norm": 0.05615234375, + "learning_rate": 8.70265879687476e-06, + "loss": 0.4872490882873535, + "mean_token_accuracy": 0.8473320409655571, + "num_tokens": 58099041.0, + "step": 750 + }, + { + "entropy": 0.4919566001743078, + "epoch": 1.7886992348440258, + "grad_norm": 0.0625, + "learning_rate": 8.431597616303818e-06, + "loss": 0.5038920402526855, + "mean_token_accuracy": 0.8457822546362876, + "num_tokens": 58932723.0, + "step": 760 + }, + { + "entropy": 0.4820005618967116, + "epoch": 1.8122424955856387, + "grad_norm": 0.05810546875, + "learning_rate": 8.16171296640231e-06, + "loss": 0.4845709323883057, + "mean_token_accuracy": 0.848111093044281, + "num_tokens": 59693185.0, + "step": 770 + }, + { + "entropy": 0.4959249753504992, + "epoch": 1.8357857563272513, + "grad_norm": 0.048828125, + "learning_rate": 7.89320730004274e-06, + "loss": 0.5166537761688232, + "mean_token_accuracy": 0.8443613056093454, + "num_tokens": 60491010.0, + "step": 780 + }, + { + "entropy": 0.49144728798419235, + "epoch": 1.8593290170688639, + "grad_norm": 0.05908203125, + "learning_rate": 7.626282035658703e-06, + "loss": 0.49805240631103515, + "mean_token_accuracy": 0.8461739920079708, + "num_tokens": 61319991.0, + "step": 790 + }, + { + "entropy": 0.4859143828973174, + "epoch": 1.8828722778104767, + "grad_norm": 0.054931640625, + "learning_rate": 7.361137406151706e-06, + "loss": 0.4953606128692627, + "mean_token_accuracy": 0.8474723279476166, + "num_tokens": 62135256.0, + "step": 800 + }, + { + "epoch": 1.8828722778104767, + "eval_entropy": 0.49684396469393255, + "eval_loss": 0.48009854555130005, + "eval_mean_token_accuracy": 0.8439133836570398, + "eval_num_tokens": 62135256.0, + "eval_runtime": 140.0838, + "eval_samples_per_second": 2.556, + "eval_steps_per_second": 2.556, + "step": 800 + }, + { + "entropy": 0.4975118497386575, + "epoch": 1.9064155385520896, + "grad_norm": 0.055908203125, + "learning_rate": 7.0979723086872245e-06, + "loss": 0.5027891159057617, + "mean_token_accuracy": 0.8441127471625804, + "num_tokens": 62886746.0, + "step": 810 + }, + { + "entropy": 0.48534935507923366, + "epoch": 1.9299587992937022, + "grad_norm": 0.060546875, + "learning_rate": 6.8369841554928226e-06, + "loss": 0.48329763412475585, + "mean_token_accuracy": 0.8473416466265917, + "num_tokens": 63579402.0, + "step": 820 + }, + { + "entropy": 0.484352495148778, + "epoch": 1.9535020600353148, + "grad_norm": 0.05029296875, + "learning_rate": 6.578368725770182e-06, + "loss": 0.48952536582946776, + "mean_token_accuracy": 0.8478636931627989, + "num_tokens": 64391578.0, + "step": 830 + }, + { + "entropy": 0.48771984670311214, + "epoch": 1.9770453207769276, + "grad_norm": 0.0498046875, + "learning_rate": 6.322320018832125e-06, + "loss": 0.4877884864807129, + "mean_token_accuracy": 0.8477956593036652, + "num_tokens": 65101582.0, + "step": 840 + }, + { + "entropy": 0.4792339194279451, + "epoch": 2.0, + "grad_norm": 0.0693359375, + "learning_rate": 6.069030108574806e-06, + "loss": 0.4753533363342285, + "mean_token_accuracy": 0.8492753834296496, + "num_tokens": 65847956.0, + "step": 850 + }, + { + "entropy": 0.4897763991728425, + "epoch": 2.023543260741613, + "grad_norm": 0.0595703125, + "learning_rate": 5.8186889993942704e-06, + "loss": 0.49657087326049804, + "mean_token_accuracy": 0.8467563409358263, + "num_tokens": 66639763.0, + "step": 860 + }, + { + "entropy": 0.4955088282003999, + "epoch": 2.0470865214832252, + "grad_norm": 0.064453125, + "learning_rate": 5.571484483655443e-06, + "loss": 0.5085082530975342, + "mean_token_accuracy": 0.8449712213128805, + "num_tokens": 67385166.0, + "step": 870 + }, + { + "entropy": 0.4888547468930483, + "epoch": 2.070629782224838, + "grad_norm": 0.05810546875, + "learning_rate": 5.327602000820413e-06, + "loss": 0.4888143539428711, + "mean_token_accuracy": 0.846753703430295, + "num_tokens": 68099723.0, + "step": 880 + }, + { + "entropy": 0.49544552695006133, + "epoch": 2.094173042966451, + "grad_norm": 0.054443359375, + "learning_rate": 5.087224498341815e-06, + "loss": 0.49994626045227053, + "mean_token_accuracy": 0.843446821346879, + "num_tokens": 68788696.0, + "step": 890 + }, + { + "entropy": 0.4897686781361699, + "epoch": 2.1177163037080637, + "grad_norm": 0.062255859375, + "learning_rate": 4.8505322944255096e-06, + "loss": 0.49230360984802246, + "mean_token_accuracy": 0.8458703383803368, + "num_tokens": 69602614.0, + "step": 900 + }, + { + "epoch": 2.1177163037080637, + "eval_entropy": 0.494421407497129, + "eval_loss": 0.47960278391838074, + "eval_mean_token_accuracy": 0.8439072530362859, + "eval_num_tokens": 69602614.0, + "eval_runtime": 136.7014, + "eval_samples_per_second": 2.619, + "eval_steps_per_second": 2.619, + "step": 900 + }, + { + "entropy": 0.4872464815154672, + "epoch": 2.141259564449676, + "grad_norm": 0.059326171875, + "learning_rate": 4.617702942765641e-06, + "loss": 0.49840707778930665, + "mean_token_accuracy": 0.847221752628684, + "num_tokens": 70400747.0, + "step": 910 + }, + { + "entropy": 0.4936610346660018, + "epoch": 2.164802825191289, + "grad_norm": 0.064453125, + "learning_rate": 4.388911099353446e-06, + "loss": 0.5067010402679444, + "mean_token_accuracy": 0.8448445748537778, + "num_tokens": 71161256.0, + "step": 920 + }, + { + "entropy": 0.4865023369900882, + "epoch": 2.188346085932902, + "grad_norm": 0.05712890625, + "learning_rate": 4.164328391459752e-06, + "loss": 0.48999724388122556, + "mean_token_accuracy": 0.8471139241009951, + "num_tokens": 71912573.0, + "step": 930 + }, + { + "entropy": 0.48061597514897586, + "epoch": 2.211889346674514, + "grad_norm": 0.05810546875, + "learning_rate": 3.944123288889516e-06, + "loss": 0.4725522518157959, + "mean_token_accuracy": 0.8494672421365976, + "num_tokens": 72707432.0, + "step": 940 + }, + { + "entropy": 0.5059708185493946, + "epoch": 2.235432607416127, + "grad_norm": 0.05419921875, + "learning_rate": 3.728460977604852e-06, + "loss": 0.5131529331207275, + "mean_token_accuracy": 0.8409805115312338, + "num_tokens": 73449621.0, + "step": 950 + }, + { + "entropy": 0.4863934010267258, + "epoch": 2.25897586815774, + "grad_norm": 0.055419921875, + "learning_rate": 3.517503235811468e-06, + "loss": 0.5088976860046387, + "mean_token_accuracy": 0.846961434558034, + "num_tokens": 74347033.0, + "step": 960 + }, + { + "entropy": 0.4784310838207603, + "epoch": 2.2825191288993527, + "grad_norm": 0.0634765625, + "learning_rate": 3.3114083126014097e-06, + "loss": 0.47273902893066405, + "mean_token_accuracy": 0.8500492256134748, + "num_tokens": 75011148.0, + "step": 970 + }, + { + "entropy": 0.48275542035698893, + "epoch": 2.306062389640965, + "grad_norm": 0.06982421875, + "learning_rate": 3.110330809243134e-06, + "loss": 0.4815514087677002, + "mean_token_accuracy": 0.8492911938577891, + "num_tokens": 75701952.0, + "step": 980 + }, + { + "entropy": 0.4942383628338575, + "epoch": 2.329605650382578, + "grad_norm": 0.05029296875, + "learning_rate": 2.914421563208003e-06, + "loss": 0.5011926174163819, + "mean_token_accuracy": 0.8445728730410338, + "num_tokens": 76495004.0, + "step": 990 + }, + { + "entropy": 0.4953793972730637, + "epoch": 2.353148911124191, + "grad_norm": 0.050537109375, + "learning_rate": 2.7238275350201603e-06, + "loss": 0.49687824249267576, + "mean_token_accuracy": 0.8450685154646635, + "num_tokens": 77212538.0, + "step": 1000 + }, + { + "epoch": 2.353148911124191, + "eval_entropy": 0.49301343573538287, + "eval_loss": 0.4794597625732422, + "eval_mean_token_accuracy": 0.8439149758336264, + "eval_num_tokens": 77212538.0, + "eval_runtime": 126.3492, + "eval_samples_per_second": 2.833, + "eval_steps_per_second": 2.833, + "step": 1000 + }, + { + "entropy": 0.4894041134044528, + "epoch": 2.3766921718658036, + "grad_norm": 0.04931640625, + "learning_rate": 2.5386916980147126e-06, + "loss": 0.49138665199279785, + "mean_token_accuracy": 0.8464279536157846, + "num_tokens": 78026138.0, + "step": 1010 + }, + { + "entropy": 0.4856866804882884, + "epoch": 2.400235432607416, + "grad_norm": 0.057861328125, + "learning_rate": 2.3591529310868376e-06, + "loss": 0.4937007904052734, + "mean_token_accuracy": 0.8472504287958145, + "num_tokens": 78780085.0, + "step": 1020 + }, + { + "entropy": 0.4839438125491142, + "epoch": 2.423778693349029, + "grad_norm": 0.05126953125, + "learning_rate": 2.18534591451236e-06, + "loss": 0.4821192264556885, + "mean_token_accuracy": 0.8487556364387274, + "num_tokens": 79570654.0, + "step": 1030 + }, + { + "entropy": 0.4880734058097005, + "epoch": 2.4473219540906417, + "grad_norm": 0.06298828125, + "learning_rate": 2.0174010289178914e-06, + "loss": 0.49988112449645994, + "mean_token_accuracy": 0.8469519291073084, + "num_tokens": 80250823.0, + "step": 1040 + }, + { + "entropy": 0.480747524369508, + "epoch": 2.470865214832254, + "grad_norm": 0.05859375, + "learning_rate": 1.8554442574763298e-06, + "loss": 0.47515578269958497, + "mean_token_accuracy": 0.8491792384535074, + "num_tokens": 81007390.0, + "step": 1050 + }, + { + "entropy": 0.4821536695584655, + "epoch": 2.494408475573867, + "grad_norm": 0.05712890625, + "learning_rate": 1.699597091401093e-06, + "loss": 0.4729485511779785, + "mean_token_accuracy": 0.8483801148831844, + "num_tokens": 81740993.0, + "step": 1060 + }, + { + "entropy": 0.472045193053782, + "epoch": 2.5179517363154797, + "grad_norm": 0.060546875, + "learning_rate": 1.549976438809978e-06, + "loss": 0.4712677955627441, + "mean_token_accuracy": 0.8525977376848459, + "num_tokens": 82419756.0, + "step": 1070 + }, + { + "entropy": 0.4656744490377605, + "epoch": 2.541494997057092, + "grad_norm": 0.058349609375, + "learning_rate": 1.4066945370270225e-06, + "loss": 0.47236061096191406, + "mean_token_accuracy": 0.8537530150264502, + "num_tokens": 83196283.0, + "step": 1080 + }, + { + "entropy": 0.478969038836658, + "epoch": 2.565038257798705, + "grad_norm": 0.0556640625, + "learning_rate": 1.2698588683881185e-06, + "loss": 0.48878769874572753, + "mean_token_accuracy": 0.8492098618298769, + "num_tokens": 84036439.0, + "step": 1090 + }, + { + "entropy": 0.4828123481944203, + "epoch": 2.588581518540318, + "grad_norm": 0.0556640625, + "learning_rate": 1.1395720796135868e-06, + "loss": 0.4844477653503418, + "mean_token_accuracy": 0.8475308798253536, + "num_tokens": 84820180.0, + "step": 1100 + }, + { + "epoch": 2.588581518540318, + "eval_entropy": 0.4935321879453499, + "eval_loss": 0.4793815016746521, + "eval_mean_token_accuracy": 0.8439810825459784, + "eval_num_tokens": 84820180.0, + "eval_runtime": 126.7867, + "eval_samples_per_second": 2.824, + "eval_steps_per_second": 2.824, + "step": 1100 + }, + { + "entropy": 0.49042358435690403, + "epoch": 2.6121247792819307, + "grad_norm": 0.06005859375, + "learning_rate": 1.0159319048081463e-06, + "loss": 0.49592008590698244, + "mean_token_accuracy": 0.8456685792654752, + "num_tokens": 85635100.0, + "step": 1110 + }, + { + "entropy": 0.4801861563697457, + "epoch": 2.6356680400235435, + "grad_norm": 0.0498046875, + "learning_rate": 8.990310921460865e-07, + "loss": 0.48504209518432617, + "mean_token_accuracy": 0.8492361098527909, + "num_tokens": 86360259.0, + "step": 1120 + }, + { + "entropy": 0.47704076319932937, + "epoch": 2.659211300765156, + "grad_norm": 0.056640625, + "learning_rate": 7.889573342965884e-07, + "loss": 0.4823050022125244, + "mean_token_accuracy": 0.8498645894229412, + "num_tokens": 87089379.0, + "step": 1130 + }, + { + "entropy": 0.491375151835382, + "epoch": 2.6827545615067687, + "grad_norm": 0.055419921875, + "learning_rate": 6.857932026414305e-07, + "loss": 0.4903556823730469, + "mean_token_accuracy": 0.8461613107472659, + "num_tokens": 87800946.0, + "step": 1140 + }, + { + "entropy": 0.49147001672536134, + "epoch": 2.7062978222483816, + "grad_norm": 0.05029296875, + "learning_rate": 5.896160853344079e-07, + "loss": 0.49661922454833984, + "mean_token_accuracy": 0.8448879312723875, + "num_tokens": 88653378.0, + "step": 1150 + }, + { + "entropy": 0.48449703082442286, + "epoch": 2.729841082989994, + "grad_norm": 0.0537109375, + "learning_rate": 5.00498129248913e-07, + "loss": 0.49818434715271, + "mean_token_accuracy": 0.8479541372507811, + "num_tokens": 89456233.0, + "step": 1160 + }, + { + "entropy": 0.5064624701626599, + "epoch": 2.753384343731607, + "grad_norm": 0.061279296875, + "learning_rate": 4.1850618585726477e-07, + "loss": 0.5106511592864991, + "mean_token_accuracy": 0.8413057509809733, + "num_tokens": 90273983.0, + "step": 1170 + }, + { + "entropy": 0.47172180134803054, + "epoch": 2.7769276044732196, + "grad_norm": 0.046875, + "learning_rate": 3.437017610823323e-07, + "loss": 0.4803746223449707, + "mean_token_accuracy": 0.8520814351737499, + "num_tokens": 91088834.0, + "step": 1180 + }, + { + "entropy": 0.48386119250208137, + "epoch": 2.800470865214832, + "grad_norm": 0.05908203125, + "learning_rate": 2.7614096915913325e-07, + "loss": 0.5014448642730713, + "mean_token_accuracy": 0.848398182913661, + "num_tokens": 91850372.0, + "step": 1190 + }, + { + "entropy": 0.473122557811439, + "epoch": 2.824014125956445, + "grad_norm": 0.0537109375, + "learning_rate": 2.1587449054095933e-07, + "loss": 0.46098814010620115, + "mean_token_accuracy": 0.8519114807248116, + "num_tokens": 92604266.0, + "step": 1200 + }, + { + "epoch": 2.824014125956445, + "eval_entropy": 0.49339639424611736, + "eval_loss": 0.4793107807636261, + "eval_mean_token_accuracy": 0.8440321680553798, + "eval_num_tokens": 92604266.0, + "eval_runtime": 129.0476, + "eval_samples_per_second": 2.774, + "eval_steps_per_second": 2.774, + "step": 1200 + }, + { + "entropy": 0.4882160659879446, + "epoch": 2.8475573866980577, + "grad_norm": 0.0576171875, + "learning_rate": 1.6294753388163886e-07, + "loss": 0.4841885566711426, + "mean_token_accuracy": 0.8477579735219478, + "num_tokens": 93383250.0, + "step": 1210 + }, + { + "entropy": 0.4832633834332228, + "epoch": 2.8711006474396705, + "grad_norm": 0.05859375, + "learning_rate": 1.1739980212245983e-07, + "loss": 0.49979395866394044, + "mean_token_accuracy": 0.849021689593792, + "num_tokens": 94238149.0, + "step": 1220 + }, + { + "entropy": 0.5064708769321442, + "epoch": 2.8946439081812834, + "grad_norm": 0.05517578125, + "learning_rate": 7.926546270915603e-08, + "loss": 0.523139476776123, + "mean_token_accuracy": 0.8406466241925955, + "num_tokens": 95043273.0, + "step": 1230 + }, + { + "entropy": 0.47456965055316686, + "epoch": 2.9181871689228958, + "grad_norm": 0.0478515625, + "learning_rate": 4.8573121961347226e-08, + "loss": 0.4729292392730713, + "mean_token_accuracy": 0.8510482702404261, + "num_tokens": 95859784.0, + "step": 1240 + }, + { + "entropy": 0.4945228073745966, + "epoch": 2.9417304296645086, + "grad_norm": 0.0576171875, + "learning_rate": 2.5345803613617826e-08, + "loss": 0.49330801963806153, + "mean_token_accuracy": 0.8454886879771948, + "num_tokens": 96662576.0, + "step": 1250 + }, + { + "entropy": 0.48427404602989554, + "epoch": 2.9652736904061214, + "grad_norm": 0.051513671875, + "learning_rate": 9.600931544362369e-09, + "loss": 0.4869691848754883, + "mean_token_accuracy": 0.8485142230987549, + "num_tokens": 97491833.0, + "step": 1260 + }, + { + "entropy": 0.48224083045497534, + "epoch": 2.988816951147734, + "grad_norm": 0.052734375, + "learning_rate": 1.3503167053285916e-09, + "loss": 0.4842352390289307, + "mean_token_accuracy": 0.8483625873923302, + "num_tokens": 98341011.0, + "step": 1270 + }, + { + "epoch": 3.0, + "eval_entropy": 0.49332542625885434, + "eval_loss": 0.4793912470340729, + "eval_mean_token_accuracy": 0.8440024263365975, + "eval_num_tokens": 98771934.0, + "eval_runtime": 125.3024, + "eval_samples_per_second": 2.857, + "eval_steps_per_second": 2.857, + "step": 1275 + } + ], + "logging_steps": 10, + "max_steps": 1275, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 8.35260937439957e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/last-checkpoint/training_args.bin b/last-checkpoint/training_args.bin new file mode 100644 index 0000000..ae7081c --- /dev/null +++ b/last-checkpoint/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7449cf0b96242b2258ef9698e09bf5e0288b6bb534b12d5f70eaefbd44a471d +size 5304 diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..5b07aba --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9822507c09f03b5474cec78cc118ee618cd3069c353f5144d8d0083adfb886af +size 3441185608 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..9fd0fb4 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..ae7081c --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7449cf0b96242b2258ef9698e09bf5e0288b6bb534b12d5f70eaefbd44a471d +size 5304