commit 584122fc9894710ba56782857ff62f337a8d211e Author: ModelHub XC Date: Mon Jul 13 04:40:09 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: akhauriyash/DeepSeek-R1-Distill-Qwen-1.5B-SpeculativeReasoner Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..137ceb8 --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +--- +base_model: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B +datasets: akhauriyash/OpenR1_Math_SpeculativeReasoning +library_name: transformers +model_name: DeepSeek-R1-Distill-Qwen-1.5B-SpeculativeReasoner +tags: +- generated_from_trainer +- open-r1 +- trl +- sft +licence: license +--- + +# Model Card for DeepSeek-R1-Distill-Qwen-1.5B-SpeculativeReasoner + +This model is a fine-tuned version of [deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B) on the [akhauriyash/OpenR1_Math_SpeculativeReasoning](https://huggingface.co/datasets/akhauriyash/OpenR1_Math_SpeculativeReasoning) dataset. +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="akhauriyash/DeepSeek-R1-Distill-Qwen-1.5B-SpeculativeReasoner", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/akhauriyash/SpeculativeReasoning/runs/f3qpjel6) + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.16.1 +- Transformers: 4.50.0 +- Pytorch: 2.5.1 +- Datasets: 3.5.0 +- Tokenizers: 0.21.1 + +## Citations + + + +Cite TRL as: + +```bibtex +@misc{vonwerra2022trl, + title = {{TRL: Transformer Reinforcement Learning}}, + author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec}, + year = 2020, + journal = {GitHub repository}, + publisher = {GitHub}, + howpublished = {\url{https://github.com/huggingface/trl}} +} +``` \ No newline at end of file diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..9f9c85a --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "total_flos": 287156144701440.0, + "train_loss": 0.4225423020029825, + "train_runtime": 9655.7571, + "train_samples": 18521, + "train_samples_per_second": 2.082, + "train_steps_per_second": 0.033 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..bc19948 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "max_position_embeddings": 131072, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 10000, + "sliding_window": 4096, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.50.0", + "use_cache": true, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 151665 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..67bc30b --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 151646, + "do_sample": true, + "eos_token_id": 151643, + "temperature": 0.6, + "top_p": 0.95, + "transformers_version": "4.50.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..3532a91 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1583476e6f048db6022428146b9b8a2326371fccd0ea27791ab62c92305af444 +size 3552549728 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..d53f816 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,17 @@ +{ + "bos_token": { + "content": "<｜begin▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<｜end▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<｜end▁of▁sentence｜>" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1a2db24 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e20ddafc659ba90242154b55275402edeca0715e5dbb30f56815a4ce081f4893 +size 11422778 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..46f1cc8 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,195 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": null, + "added_tokens_decoder": { + "151643": { + "content": "<｜end▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<｜User｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151645": { + "content": "<｜Assistant｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151646": { + "content": "<｜begin▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|EOT|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151648": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151649": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "<｜begin▁of▁sentence｜>", + "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<｜end▁of▁sentence｜>", + "extra_special_tokens": {}, + "legacy": true, + "model_max_length": 16384, + "pad_token": "<｜end▁of▁sentence｜>", + "sp_model_kwargs": {}, + "tokenizer_class": "LlamaTokenizerFast", + "unk_token": null, + "use_default_system_prompt": false +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..9f9c85a --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "total_flos": 287156144701440.0, + "train_loss": 0.4225423020029825, + "train_runtime": 9655.7571, + "train_samples": 18521, + "train_samples_per_second": 2.082, + "train_steps_per_second": 0.033 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..d80fa5f --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,2563 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 315, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.009523809523809525, + "grad_norm": 1.049260690190334, + "learning_rate": 3.125e-06, + "loss": 0.5385, + "num_tokens": 1048576.0, + "step": 1 + }, + { + "epoch": 0.01904761904761905, + "grad_norm": 1.0789843180440266, + "learning_rate": 6.25e-06, + "loss": 0.5658, + "num_tokens": 2097152.0, + "step": 2 + }, + { + "epoch": 0.02857142857142857, + "grad_norm": 4.94991346565488, + "learning_rate": 9.375000000000001e-06, + "loss": 0.7228, + "num_tokens": 3141437.0, + "step": 3 + }, + { + "epoch": 0.0380952380952381, + "grad_norm": 1.0754586349371367, + "learning_rate": 1.25e-05, + "loss": 0.5616, + "num_tokens": 4185063.0, + "step": 4 + }, + { + "epoch": 0.047619047619047616, + "grad_norm": 0.8560328603848789, + "learning_rate": 1.5625e-05, + "loss": 0.5392, + "num_tokens": 5233639.0, + "step": 5 + }, + { + "epoch": 0.05714285714285714, + "grad_norm": 0.7730008783640578, + "learning_rate": 1.8750000000000002e-05, + "loss": 0.531, + "num_tokens": 6269804.0, + "step": 6 + }, + { + "epoch": 0.06666666666666667, + "grad_norm": 0.45738907826104275, + "learning_rate": 2.1875e-05, + "loss": 0.5089, + "num_tokens": 7313218.0, + "step": 7 + }, + { + "epoch": 0.0761904761904762, + "grad_norm": 3.284809119752342, + "learning_rate": 2.5e-05, + "loss": 0.6136, + "num_tokens": 8361794.0, + "step": 8 + }, + { + "epoch": 0.08571428571428572, + "grad_norm": 1.0574311652743773, + "learning_rate": 2.8125000000000003e-05, + "loss": 0.5128, + "num_tokens": 9409278.0, + "step": 9 + }, + { + "epoch": 0.09523809523809523, + "grad_norm": 1.0261676590479574, + "learning_rate": 3.125e-05, + "loss": 0.5168, + "num_tokens": 10451099.0, + "step": 10 + }, + { + "epoch": 0.10476190476190476, + "grad_norm": 0.8383592498975208, + "learning_rate": 3.4375e-05, + "loss": 0.498, + "num_tokens": 11499675.0, + "step": 11 + }, + { + "epoch": 0.11428571428571428, + "grad_norm": 2.823502000164976, + "learning_rate": 3.7500000000000003e-05, + "loss": 0.4985, + "num_tokens": 12538677.0, + "step": 12 + }, + { + "epoch": 0.12380952380952381, + "grad_norm": 0.5053425812423326, + "learning_rate": 4.0625000000000005e-05, + "loss": 0.5002, + "num_tokens": 13587253.0, + "step": 13 + }, + { + "epoch": 0.13333333333333333, + "grad_norm": 0.45098538286493733, + "learning_rate": 4.375e-05, + "loss": 0.4957, + "num_tokens": 14625318.0, + "step": 14 + }, + { + "epoch": 0.14285714285714285, + "grad_norm": 0.4109221085606176, + "learning_rate": 4.6875e-05, + "loss": 0.4815, + "num_tokens": 15673894.0, + "step": 15 + }, + { + "epoch": 0.1523809523809524, + "grad_norm": 0.4072569185255141, + "learning_rate": 5e-05, + "loss": 0.4847, + "num_tokens": 16707791.0, + "step": 16 + }, + { + "epoch": 0.1619047619047619, + "grad_norm": 0.37860012278830546, + "learning_rate": 4.999875804489839e-05, + "loss": 0.4842, + "num_tokens": 17756367.0, + "step": 17 + }, + { + "epoch": 0.17142857142857143, + "grad_norm": 0.3484656472813555, + "learning_rate": 4.999503231670043e-05, + "loss": 0.4697, + "num_tokens": 18804943.0, + "step": 18 + }, + { + "epoch": 0.18095238095238095, + "grad_norm": 0.27860562608724904, + "learning_rate": 4.998882322671165e-05, + "loss": 0.4596, + "num_tokens": 19852798.0, + "step": 19 + }, + { + "epoch": 0.19047619047619047, + "grad_norm": 0.27352927871047156, + "learning_rate": 4.9980131460390804e-05, + "loss": 0.4816, + "num_tokens": 20886414.0, + "step": 20 + }, + { + "epoch": 0.2, + "grad_norm": 0.2745545679833705, + "learning_rate": 4.996895797727421e-05, + "loss": 0.4791, + "num_tokens": 21934990.0, + "step": 21 + }, + { + "epoch": 0.20952380952380953, + "grad_norm": 0.4930503806235754, + "learning_rate": 4.995530401086981e-05, + "loss": 0.4593, + "num_tokens": 22979721.0, + "step": 22 + }, + { + "epoch": 0.21904761904761905, + "grad_norm": 0.22214918712494278, + "learning_rate": 4.993917106852099e-05, + "loss": 0.4651, + "num_tokens": 24028297.0, + "step": 23 + }, + { + "epoch": 0.22857142857142856, + "grad_norm": 0.22655600309552515, + "learning_rate": 4.9920560931240214e-05, + "loss": 0.4723, + "num_tokens": 25065891.0, + "step": 24 + }, + { + "epoch": 0.23809523809523808, + "grad_norm": 0.2385439925375394, + "learning_rate": 4.9899475653512364e-05, + "loss": 0.4876, + "num_tokens": 26111357.0, + "step": 25 + }, + { + "epoch": 0.24761904761904763, + "grad_norm": 0.17815843515733643, + "learning_rate": 4.987591756306794e-05, + "loss": 0.4493, + "num_tokens": 27148962.0, + "step": 26 + }, + { + "epoch": 0.2571428571428571, + "grad_norm": 0.18802692981067173, + "learning_rate": 4.984988926062612e-05, + "loss": 0.4773, + "num_tokens": 28178810.0, + "step": 27 + }, + { + "epoch": 0.26666666666666666, + "grad_norm": 0.1900671022720311, + "learning_rate": 4.982139361960761e-05, + "loss": 0.4409, + "num_tokens": 29227386.0, + "step": 28 + }, + { + "epoch": 0.2761904761904762, + "grad_norm": 0.1945623932976909, + "learning_rate": 4.979043378581744e-05, + "loss": 0.4668, + "num_tokens": 30265535.0, + "step": 29 + }, + { + "epoch": 0.2857142857142857, + "grad_norm": 0.1816340082978734, + "learning_rate": 4.975701317709773e-05, + "loss": 0.4573, + "num_tokens": 31314111.0, + "step": 30 + }, + { + "epoch": 0.29523809523809524, + "grad_norm": 0.14504815987687575, + "learning_rate": 4.972113548295028e-05, + "loss": 0.4513, + "num_tokens": 32341497.0, + "step": 31 + }, + { + "epoch": 0.3047619047619048, + "grad_norm": 0.16268051991890234, + "learning_rate": 4.968280466412934e-05, + "loss": 0.4709, + "num_tokens": 33390073.0, + "step": 32 + }, + { + "epoch": 0.3142857142857143, + "grad_norm": 0.16881317376789234, + "learning_rate": 4.9642024952204335e-05, + "loss": 0.4782, + "num_tokens": 34425866.0, + "step": 33 + }, + { + "epoch": 0.3238095238095238, + "grad_norm": 0.15534352950358957, + "learning_rate": 4.959880084909271e-05, + "loss": 0.4753, + "num_tokens": 35474442.0, + "step": 34 + }, + { + "epoch": 0.3333333333333333, + "grad_norm": 0.15632760882189625, + "learning_rate": 4.9553137126562944e-05, + "loss": 0.4551, + "num_tokens": 36518938.0, + "step": 35 + }, + { + "epoch": 0.34285714285714286, + "grad_norm": 0.1435786925585567, + "learning_rate": 4.950503882570775e-05, + "loss": 0.4445, + "num_tokens": 37555262.0, + "step": 36 + }, + { + "epoch": 0.3523809523809524, + "grad_norm": 0.13204667286756613, + "learning_rate": 4.9454511256387606e-05, + "loss": 0.4495, + "num_tokens": 38597659.0, + "step": 37 + }, + { + "epoch": 0.3619047619047619, + "grad_norm": 0.3392838888429332, + "learning_rate": 4.9401559996644505e-05, + "loss": 0.4868, + "num_tokens": 39646235.0, + "step": 38 + }, + { + "epoch": 0.37142857142857144, + "grad_norm": 0.13927699543829483, + "learning_rate": 4.9346190892086174e-05, + "loss": 0.4568, + "num_tokens": 40694811.0, + "step": 39 + }, + { + "epoch": 0.38095238095238093, + "grad_norm": 0.12649376683576488, + "learning_rate": 4.92884100552408e-05, + "loss": 0.4509, + "num_tokens": 41743387.0, + "step": 40 + }, + { + "epoch": 0.3904761904761905, + "grad_norm": 0.12653279776873122, + "learning_rate": 4.922822386488217e-05, + "loss": 0.4603, + "num_tokens": 42780722.0, + "step": 41 + }, + { + "epoch": 0.4, + "grad_norm": 0.1332474434519712, + "learning_rate": 4.916563896532549e-05, + "loss": 0.4535, + "num_tokens": 43829298.0, + "step": 42 + }, + { + "epoch": 0.4095238095238095, + "grad_norm": 0.13272848286582178, + "learning_rate": 4.91006622656939e-05, + "loss": 0.4624, + "num_tokens": 44877874.0, + "step": 43 + }, + { + "epoch": 0.41904761904761906, + "grad_norm": 0.13426533914501868, + "learning_rate": 4.903330093915572e-05, + "loss": 0.4686, + "num_tokens": 45926450.0, + "step": 44 + }, + { + "epoch": 0.42857142857142855, + "grad_norm": 0.13743304920185773, + "learning_rate": 4.896356242213256e-05, + "loss": 0.4561, + "num_tokens": 46975026.0, + "step": 45 + }, + { + "epoch": 0.4380952380952381, + "grad_norm": 0.1568729235862451, + "learning_rate": 4.889145441347838e-05, + "loss": 0.4459, + "num_tokens": 48018851.0, + "step": 46 + }, + { + "epoch": 0.44761904761904764, + "grad_norm": 0.14434809292552514, + "learning_rate": 4.881698487362955e-05, + "loss": 0.458, + "num_tokens": 49067427.0, + "step": 47 + }, + { + "epoch": 0.45714285714285713, + "grad_norm": 0.11513468093094269, + "learning_rate": 4.8740162023726064e-05, + "loss": 0.45, + "num_tokens": 50116003.0, + "step": 48 + }, + { + "epoch": 0.4666666666666667, + "grad_norm": 0.110896239508101, + "learning_rate": 4.8660994344703965e-05, + "loss": 0.4497, + "num_tokens": 51164579.0, + "step": 49 + }, + { + "epoch": 0.47619047619047616, + "grad_norm": 0.11374390510880547, + "learning_rate": 4.8579490576359046e-05, + "loss": 0.4507, + "num_tokens": 52197176.0, + "step": 50 + }, + { + "epoch": 0.4857142857142857, + "grad_norm": 0.12338805010954729, + "learning_rate": 4.849565971638206e-05, + "loss": 0.4544, + "num_tokens": 53228031.0, + "step": 51 + }, + { + "epoch": 0.49523809523809526, + "grad_norm": 0.11362343896287345, + "learning_rate": 4.8409511019365384e-05, + "loss": 0.4549, + "num_tokens": 54276607.0, + "step": 52 + }, + { + "epoch": 0.5047619047619047, + "grad_norm": 0.10400687358440638, + "learning_rate": 4.832105399578135e-05, + "loss": 0.4437, + "num_tokens": 55317298.0, + "step": 53 + }, + { + "epoch": 0.5142857142857142, + "grad_norm": 0.11312688395203489, + "learning_rate": 4.8230298410932335e-05, + "loss": 0.4668, + "num_tokens": 56365874.0, + "step": 54 + }, + { + "epoch": 0.5238095238095238, + "grad_norm": 0.1248036429793104, + "learning_rate": 4.8137254283872696e-05, + "loss": 0.4506, + "num_tokens": 57403614.0, + "step": 55 + }, + { + "epoch": 0.5333333333333333, + "grad_norm": 0.11021878529793232, + "learning_rate": 4.804193188630273e-05, + "loss": 0.4204, + "num_tokens": 58452190.0, + "step": 56 + }, + { + "epoch": 0.5428571428571428, + "grad_norm": 0.10499980797426618, + "learning_rate": 4.79443417414347e-05, + "loss": 0.4356, + "num_tokens": 59500766.0, + "step": 57 + }, + { + "epoch": 0.5523809523809524, + "grad_norm": 0.13718876090636073, + "learning_rate": 4.784449462283113e-05, + "loss": 0.4469, + "num_tokens": 60549342.0, + "step": 58 + }, + { + "epoch": 0.5619047619047619, + "grad_norm": 0.10435055505331534, + "learning_rate": 4.774240155321541e-05, + "loss": 0.4496, + "num_tokens": 61597918.0, + "step": 59 + }, + { + "epoch": 0.5714285714285714, + "grad_norm": 0.1054851919046421, + "learning_rate": 4.763807380325497e-05, + "loss": 0.4661, + "num_tokens": 62646494.0, + "step": 60 + }, + { + "epoch": 0.580952380952381, + "grad_norm": 0.11024753578382669, + "learning_rate": 4.7531522890317074e-05, + "loss": 0.4567, + "num_tokens": 63690949.0, + "step": 61 + }, + { + "epoch": 0.5904761904761905, + "grad_norm": 0.09861659314751674, + "learning_rate": 4.7422760577197226e-05, + "loss": 0.4308, + "num_tokens": 64739525.0, + "step": 62 + }, + { + "epoch": 0.6, + "grad_norm": 0.10971910152145464, + "learning_rate": 4.731179887082075e-05, + "loss": 0.4465, + "num_tokens": 65788101.0, + "step": 63 + }, + { + "epoch": 0.6095238095238096, + "grad_norm": 0.10097521708936648, + "learning_rate": 4.719865002091718e-05, + "loss": 0.4429, + "num_tokens": 66821096.0, + "step": 64 + }, + { + "epoch": 0.6190476190476191, + "grad_norm": 0.10125890328894402, + "learning_rate": 4.708332651866798e-05, + "loss": 0.4307, + "num_tokens": 67856130.0, + "step": 65 + }, + { + "epoch": 0.6285714285714286, + "grad_norm": 0.11031399303974143, + "learning_rate": 4.6965841095327536e-05, + "loss": 0.4481, + "num_tokens": 68904706.0, + "step": 66 + }, + { + "epoch": 0.638095238095238, + "grad_norm": 0.10797444267781332, + "learning_rate": 4.6846206720817715e-05, + "loss": 0.4599, + "num_tokens": 69948115.0, + "step": 67 + }, + { + "epoch": 0.6476190476190476, + "grad_norm": 0.09759890274461364, + "learning_rate": 4.6724436602296e-05, + "loss": 0.4492, + "num_tokens": 70996691.0, + "step": 68 + }, + { + "epoch": 0.6571428571428571, + "grad_norm": 0.11528192521546547, + "learning_rate": 4.660054418269749e-05, + "loss": 0.4607, + "num_tokens": 72045267.0, + "step": 69 + }, + { + "epoch": 0.6666666666666666, + "grad_norm": 0.1056857247749398, + "learning_rate": 4.6474543139250856e-05, + "loss": 0.4415, + "num_tokens": 73093843.0, + "step": 70 + }, + { + "epoch": 0.6761904761904762, + "grad_norm": 0.10662186932792149, + "learning_rate": 4.634644738196844e-05, + "loss": 0.4624, + "num_tokens": 74142419.0, + "step": 71 + }, + { + "epoch": 0.6857142857142857, + "grad_norm": 0.11451841992401207, + "learning_rate": 4.621627105211061e-05, + "loss": 0.4372, + "num_tokens": 75175097.0, + "step": 72 + }, + { + "epoch": 0.6952380952380952, + "grad_norm": 0.10422230279153209, + "learning_rate": 4.608402852062466e-05, + "loss": 0.4559, + "num_tokens": 76214886.0, + "step": 73 + }, + { + "epoch": 0.7047619047619048, + "grad_norm": 0.09714732723739297, + "learning_rate": 4.594973438655828e-05, + "loss": 0.4388, + "num_tokens": 77263462.0, + "step": 74 + }, + { + "epoch": 0.7142857142857143, + "grad_norm": 0.10418511260106932, + "learning_rate": 4.581340347544793e-05, + "loss": 0.4404, + "num_tokens": 78312038.0, + "step": 75 + }, + { + "epoch": 0.7238095238095238, + "grad_norm": 0.09969465580989353, + "learning_rate": 4.567505083768208e-05, + "loss": 0.4312, + "num_tokens": 79346692.0, + "step": 76 + }, + { + "epoch": 0.7333333333333333, + "grad_norm": 0.09908060276284385, + "learning_rate": 4.5534691746839803e-05, + "loss": 0.4289, + "num_tokens": 80395268.0, + "step": 77 + }, + { + "epoch": 0.7428571428571429, + "grad_norm": 0.10418410447672026, + "learning_rate": 4.539234169800453e-05, + "loss": 0.4493, + "num_tokens": 81431684.0, + "step": 78 + }, + { + "epoch": 0.7523809523809524, + "grad_norm": 0.10571425638849696, + "learning_rate": 4.524801640605355e-05, + "loss": 0.4406, + "num_tokens": 82480260.0, + "step": 79 + }, + { + "epoch": 0.7619047619047619, + "grad_norm": 0.10846401802951529, + "learning_rate": 4.510173180392311e-05, + "loss": 0.4535, + "num_tokens": 83528836.0, + "step": 80 + }, + { + "epoch": 0.7714285714285715, + "grad_norm": 0.10000084455418001, + "learning_rate": 4.4953504040849445e-05, + "loss": 0.4329, + "num_tokens": 84561735.0, + "step": 81 + }, + { + "epoch": 0.780952380952381, + "grad_norm": 0.10643581912047337, + "learning_rate": 4.480334948058605e-05, + "loss": 0.4331, + "num_tokens": 85610311.0, + "step": 82 + }, + { + "epoch": 0.7904761904761904, + "grad_norm": 0.10819428529227776, + "learning_rate": 4.46512846995971e-05, + "loss": 0.4505, + "num_tokens": 86658887.0, + "step": 83 + }, + { + "epoch": 0.8, + "grad_norm": 0.09951795759379795, + "learning_rate": 4.4497326485227536e-05, + "loss": 0.4343, + "num_tokens": 87691345.0, + "step": 84 + }, + { + "epoch": 0.8095238095238095, + "grad_norm": 0.11244217242240032, + "learning_rate": 4.434149183384977e-05, + "loss": 0.4606, + "num_tokens": 88739921.0, + "step": 85 + }, + { + "epoch": 0.819047619047619, + "grad_norm": 0.11943999094411742, + "learning_rate": 4.418379794898741e-05, + "loss": 0.4556, + "num_tokens": 89788497.0, + "step": 86 + }, + { + "epoch": 0.8285714285714286, + "grad_norm": 0.10264074720662945, + "learning_rate": 4.402426223941598e-05, + "loss": 0.4252, + "num_tokens": 90837073.0, + "step": 87 + }, + { + "epoch": 0.8380952380952381, + "grad_norm": 0.11513419088503307, + "learning_rate": 4.38629023172411e-05, + "loss": 0.4393, + "num_tokens": 91885649.0, + "step": 88 + }, + { + "epoch": 0.8476190476190476, + "grad_norm": 0.11016157943031923, + "learning_rate": 4.369973599595421e-05, + "loss": 0.4454, + "num_tokens": 92934225.0, + "step": 89 + }, + { + "epoch": 0.8571428571428571, + "grad_norm": 0.1104360428324509, + "learning_rate": 4.3534781288466e-05, + "loss": 0.451, + "num_tokens": 93982801.0, + "step": 90 + }, + { + "epoch": 0.8666666666666667, + "grad_norm": 0.11707707016773751, + "learning_rate": 4.3368056405117816e-05, + "loss": 0.4714, + "num_tokens": 95029538.0, + "step": 91 + }, + { + "epoch": 0.8761904761904762, + "grad_norm": 0.10842792025010918, + "learning_rate": 4.319957975167142e-05, + "loss": 0.4513, + "num_tokens": 96078114.0, + "step": 92 + }, + { + "epoch": 0.8857142857142857, + "grad_norm": 0.10665928396337833, + "learning_rate": 4.3029369927276946e-05, + "loss": 0.4254, + "num_tokens": 97126690.0, + "step": 93 + }, + { + "epoch": 0.8952380952380953, + "grad_norm": 0.13131513526253621, + "learning_rate": 4.285744572241972e-05, + "loss": 0.4493, + "num_tokens": 98175266.0, + "step": 94 + }, + { + "epoch": 0.9047619047619048, + "grad_norm": 0.10144858729101446, + "learning_rate": 4.26838261168458e-05, + "loss": 0.4212, + "num_tokens": 99223842.0, + "step": 95 + }, + { + "epoch": 0.9142857142857143, + "grad_norm": 0.11935464726225116, + "learning_rate": 4.250853027746674e-05, + "loss": 0.4309, + "num_tokens": 100272418.0, + "step": 96 + }, + { + "epoch": 0.9238095238095239, + "grad_norm": 0.10500080087601192, + "learning_rate": 4.23315775562436e-05, + "loss": 0.4467, + "num_tokens": 101304883.0, + "step": 97 + }, + { + "epoch": 0.9333333333333333, + "grad_norm": 0.0965280902363528, + "learning_rate": 4.21529874880506e-05, + "loss": 0.4356, + "num_tokens": 102353459.0, + "step": 98 + }, + { + "epoch": 0.9428571428571428, + "grad_norm": 0.12064905063303911, + "learning_rate": 4.197277978851852e-05, + "loss": 0.4589, + "num_tokens": 103402035.0, + "step": 99 + }, + { + "epoch": 0.9523809523809523, + "grad_norm": 0.11144417369816893, + "learning_rate": 4.179097435185818e-05, + "loss": 0.448, + "num_tokens": 104450611.0, + "step": 100 + }, + { + "epoch": 0.9619047619047619, + "grad_norm": 0.10016814625639724, + "learning_rate": 4.16075912486642e-05, + "loss": 0.4468, + "num_tokens": 105491327.0, + "step": 101 + }, + { + "epoch": 0.9714285714285714, + "grad_norm": 0.10806917956160045, + "learning_rate": 4.1422650723699315e-05, + "loss": 0.4568, + "num_tokens": 106539903.0, + "step": 102 + }, + { + "epoch": 0.9809523809523809, + "grad_norm": 0.10453711479237124, + "learning_rate": 4.123617319365938e-05, + "loss": 0.4484, + "num_tokens": 107588479.0, + "step": 103 + }, + { + "epoch": 0.9904761904761905, + "grad_norm": 0.10178829326929145, + "learning_rate": 4.104817924491949e-05, + "loss": 0.4342, + "num_tokens": 108637055.0, + "step": 104 + }, + { + "epoch": 1.0, + "grad_norm": 0.10406109988368172, + "learning_rate": 4.08586896312613e-05, + "loss": 0.4282, + "num_tokens": 109685631.0, + "step": 105 + }, + { + "epoch": 1.0095238095238095, + "grad_norm": 0.11975705548408926, + "learning_rate": 4.0667725271581916e-05, + "loss": 0.4263, + "num_tokens": 110734207.0, + "step": 106 + }, + { + "epoch": 1.019047619047619, + "grad_norm": 0.10553236872926029, + "learning_rate": 4.047530724758451e-05, + "loss": 0.4112, + "num_tokens": 111778780.0, + "step": 107 + }, + { + "epoch": 1.0285714285714285, + "grad_norm": 0.10607551203352013, + "learning_rate": 4.0281456801451004e-05, + "loss": 0.404, + "num_tokens": 112827356.0, + "step": 108 + }, + { + "epoch": 1.0380952380952382, + "grad_norm": 0.10938793692440127, + "learning_rate": 4.0086195333497e-05, + "loss": 0.4078, + "num_tokens": 113875932.0, + "step": 109 + }, + { + "epoch": 1.0476190476190477, + "grad_norm": 0.1071686071116638, + "learning_rate": 3.988954439980927e-05, + "loss": 0.4167, + "num_tokens": 114924508.0, + "step": 110 + }, + { + "epoch": 1.0571428571428572, + "grad_norm": 0.1025408914908266, + "learning_rate": 3.969152570986608e-05, + "loss": 0.4085, + "num_tokens": 115959581.0, + "step": 111 + }, + { + "epoch": 1.0666666666666667, + "grad_norm": 0.10636800138293842, + "learning_rate": 3.94921611241405e-05, + "loss": 0.4054, + "num_tokens": 117008157.0, + "step": 112 + }, + { + "epoch": 1.0761904761904761, + "grad_norm": 0.1081881615102381, + "learning_rate": 3.929147265168714e-05, + "loss": 0.4305, + "num_tokens": 118053623.0, + "step": 113 + }, + { + "epoch": 1.0857142857142856, + "grad_norm": 0.10001208487517647, + "learning_rate": 3.9089482447712413e-05, + "loss": 0.3973, + "num_tokens": 119102199.0, + "step": 114 + }, + { + "epoch": 1.0952380952380953, + "grad_norm": 0.11763206163786546, + "learning_rate": 3.888621281112871e-05, + "loss": 0.4296, + "num_tokens": 120136503.0, + "step": 115 + }, + { + "epoch": 1.1047619047619048, + "grad_norm": 0.11405915337336904, + "learning_rate": 3.8681686182092664e-05, + "loss": 0.4202, + "num_tokens": 121185079.0, + "step": 116 + }, + { + "epoch": 1.1142857142857143, + "grad_norm": 0.10423458175966549, + "learning_rate": 3.847592513952787e-05, + "loss": 0.4154, + "num_tokens": 122233655.0, + "step": 117 + }, + { + "epoch": 1.1238095238095238, + "grad_norm": 0.10313766919722424, + "learning_rate": 3.826895239863222e-05, + "loss": 0.4158, + "num_tokens": 123282231.0, + "step": 118 + }, + { + "epoch": 1.1333333333333333, + "grad_norm": 0.10463792473707209, + "learning_rate": 3.806079080837031e-05, + "loss": 0.4126, + "num_tokens": 124326727.0, + "step": 119 + }, + { + "epoch": 1.1428571428571428, + "grad_norm": 0.10784633449277728, + "learning_rate": 3.785146334895093e-05, + "loss": 0.411, + "num_tokens": 125375303.0, + "step": 120 + }, + { + "epoch": 1.1523809523809523, + "grad_norm": 0.10515646509202412, + "learning_rate": 3.764099312929017e-05, + "loss": 0.4209, + "num_tokens": 126423879.0, + "step": 121 + }, + { + "epoch": 1.161904761904762, + "grad_norm": 0.10142496470617653, + "learning_rate": 3.742940338446031e-05, + "loss": 0.4235, + "num_tokens": 127472455.0, + "step": 122 + }, + { + "epoch": 1.1714285714285715, + "grad_norm": 0.09862215529598983, + "learning_rate": 3.721671747312475e-05, + "loss": 0.4191, + "num_tokens": 128521031.0, + "step": 123 + }, + { + "epoch": 1.180952380952381, + "grad_norm": 0.10402585697860553, + "learning_rate": 3.700295887495925e-05, + "loss": 0.4248, + "num_tokens": 129569607.0, + "step": 124 + }, + { + "epoch": 1.1904761904761905, + "grad_norm": 0.0954150411815143, + "learning_rate": 3.678815118805996e-05, + "loss": 0.4034, + "num_tokens": 130618183.0, + "step": 125 + }, + { + "epoch": 1.2, + "grad_norm": 0.09649322653250475, + "learning_rate": 3.657231812633821e-05, + "loss": 0.4243, + "num_tokens": 131666759.0, + "step": 126 + }, + { + "epoch": 1.2095238095238094, + "grad_norm": 0.10705598622094731, + "learning_rate": 3.6355483516902643e-05, + "loss": 0.4348, + "num_tokens": 132715335.0, + "step": 127 + }, + { + "epoch": 1.2190476190476192, + "grad_norm": 0.09629014773562172, + "learning_rate": 3.6137671297428754e-05, + "loss": 0.4173, + "num_tokens": 133763911.0, + "step": 128 + }, + { + "epoch": 1.2285714285714286, + "grad_norm": 0.10320312984761114, + "learning_rate": 3.59189055135163e-05, + "loss": 0.4255, + "num_tokens": 134806308.0, + "step": 129 + }, + { + "epoch": 1.2380952380952381, + "grad_norm": 0.09571478844489108, + "learning_rate": 3.5699210316034734e-05, + "loss": 0.4043, + "num_tokens": 135854884.0, + "step": 130 + }, + { + "epoch": 1.2476190476190476, + "grad_norm": 0.10771116268191924, + "learning_rate": 3.5478609958457056e-05, + "loss": 0.4238, + "num_tokens": 136903416.0, + "step": 131 + }, + { + "epoch": 1.2571428571428571, + "grad_norm": 0.10175401590730496, + "learning_rate": 3.525712879418233e-05, + "loss": 0.4287, + "num_tokens": 137951992.0, + "step": 132 + }, + { + "epoch": 1.2666666666666666, + "grad_norm": 0.0967070629353188, + "learning_rate": 3.503479127384719e-05, + "loss": 0.4152, + "num_tokens": 139000568.0, + "step": 133 + }, + { + "epoch": 1.276190476190476, + "grad_norm": 0.10103244767291159, + "learning_rate": 3.481162194262652e-05, + "loss": 0.4213, + "num_tokens": 140049144.0, + "step": 134 + }, + { + "epoch": 1.2857142857142856, + "grad_norm": 0.09890133185256893, + "learning_rate": 3.458764543752383e-05, + "loss": 0.4068, + "num_tokens": 141097720.0, + "step": 135 + }, + { + "epoch": 1.2952380952380953, + "grad_norm": 0.09808689779435707, + "learning_rate": 3.43628864846514e-05, + "loss": 0.4225, + "num_tokens": 142132578.0, + "step": 136 + }, + { + "epoch": 1.3047619047619048, + "grad_norm": 0.10022671320761328, + "learning_rate": 3.413736989650062e-05, + "loss": 0.4222, + "num_tokens": 143181154.0, + "step": 137 + }, + { + "epoch": 1.3142857142857143, + "grad_norm": 0.10032738710682779, + "learning_rate": 3.391112056920279e-05, + "loss": 0.4017, + "num_tokens": 144225609.0, + "step": 138 + }, + { + "epoch": 1.3238095238095238, + "grad_norm": 0.1057498033519758, + "learning_rate": 3.36841634797807e-05, + "loss": 0.428, + "num_tokens": 145270190.0, + "step": 139 + }, + { + "epoch": 1.3333333333333333, + "grad_norm": 0.11817126707644059, + "learning_rate": 3.345652368339125e-05, + "loss": 0.4118, + "num_tokens": 146318766.0, + "step": 140 + }, + { + "epoch": 1.342857142857143, + "grad_norm": 0.10355330863449409, + "learning_rate": 3.3228226310559425e-05, + "loss": 0.4107, + "num_tokens": 147346303.0, + "step": 141 + }, + { + "epoch": 1.3523809523809525, + "grad_norm": 0.09657302231772706, + "learning_rate": 3.299929656440409e-05, + "loss": 0.4168, + "num_tokens": 148394879.0, + "step": 142 + }, + { + "epoch": 1.361904761904762, + "grad_norm": 0.10870395595274093, + "learning_rate": 3.276975971785555e-05, + "loss": 0.4236, + "num_tokens": 149443455.0, + "step": 143 + }, + { + "epoch": 1.3714285714285714, + "grad_norm": 0.10346256352691864, + "learning_rate": 3.253964111086561e-05, + "loss": 0.4153, + "num_tokens": 150492031.0, + "step": 144 + }, + { + "epoch": 1.380952380952381, + "grad_norm": 0.1025323821545458, + "learning_rate": 3.230896614761007e-05, + "loss": 0.413, + "num_tokens": 151540607.0, + "step": 145 + }, + { + "epoch": 1.3904761904761904, + "grad_norm": 0.09627870411628041, + "learning_rate": 3.207776029368427e-05, + "loss": 0.3927, + "num_tokens": 152589183.0, + "step": 146 + }, + { + "epoch": 1.4, + "grad_norm": 0.10079625074324744, + "learning_rate": 3.184604907329173e-05, + "loss": 0.4027, + "num_tokens": 153637038.0, + "step": 147 + }, + { + "epoch": 1.4095238095238094, + "grad_norm": 0.10269446109684671, + "learning_rate": 3.1613858066426435e-05, + "loss": 0.4255, + "num_tokens": 154672072.0, + "step": 148 + }, + { + "epoch": 1.4190476190476191, + "grad_norm": 0.10532955467415533, + "learning_rate": 3.138121290604889e-05, + "loss": 0.4004, + "num_tokens": 155720648.0, + "step": 149 + }, + { + "epoch": 1.4285714285714286, + "grad_norm": 0.10017071431590631, + "learning_rate": 3.1148139275256306e-05, + "loss": 0.4025, + "num_tokens": 156769224.0, + "step": 150 + }, + { + "epoch": 1.438095238095238, + "grad_norm": 0.09355669085444661, + "learning_rate": 3.091466290444735e-05, + "loss": 0.3896, + "num_tokens": 157817800.0, + "step": 151 + }, + { + "epoch": 1.4476190476190476, + "grad_norm": 0.10964785569155697, + "learning_rate": 3.0680809568481545e-05, + "loss": 0.4334, + "num_tokens": 158855540.0, + "step": 152 + }, + { + "epoch": 1.457142857142857, + "grad_norm": 0.10654711217833003, + "learning_rate": 3.0446605083833884e-05, + "loss": 0.4145, + "num_tokens": 159904116.0, + "step": 153 + }, + { + "epoch": 1.4666666666666668, + "grad_norm": 0.1014670088648007, + "learning_rate": 3.021207530574477e-05, + "loss": 0.4152, + "num_tokens": 160952692.0, + "step": 154 + }, + { + "epoch": 1.4761904761904763, + "grad_norm": 0.09783720552006167, + "learning_rate": 2.997724612536571e-05, + "loss": 0.4143, + "num_tokens": 161986535.0, + "step": 155 + }, + { + "epoch": 1.4857142857142858, + "grad_norm": 0.1028820155147363, + "learning_rate": 2.9742143466901022e-05, + "loss": 0.4094, + "num_tokens": 163010012.0, + "step": 156 + }, + { + "epoch": 1.4952380952380953, + "grad_norm": 0.10375046173873649, + "learning_rate": 2.9506793284745908e-05, + "loss": 0.419, + "num_tokens": 164058588.0, + "step": 157 + }, + { + "epoch": 1.5047619047619047, + "grad_norm": 0.11747619502496238, + "learning_rate": 2.927122156062121e-05, + "loss": 0.4063, + "num_tokens": 165094029.0, + "step": 158 + }, + { + "epoch": 1.5142857142857142, + "grad_norm": 0.10380663778019883, + "learning_rate": 2.90354543007051e-05, + "loss": 0.4147, + "num_tokens": 166142605.0, + "step": 159 + }, + { + "epoch": 1.5238095238095237, + "grad_norm": 0.10173307643165348, + "learning_rate": 2.8799517532762142e-05, + "loss": 0.4151, + "num_tokens": 167191181.0, + "step": 160 + }, + { + "epoch": 1.5333333333333332, + "grad_norm": 0.10475651029374987, + "learning_rate": 2.8563437303269897e-05, + "loss": 0.4077, + "num_tokens": 168224124.0, + "step": 161 + }, + { + "epoch": 1.5428571428571427, + "grad_norm": 0.11141713760781811, + "learning_rate": 2.8327239674543523e-05, + "loss": 0.412, + "num_tokens": 169272700.0, + "step": 162 + }, + { + "epoch": 1.5523809523809524, + "grad_norm": 0.09803894305939315, + "learning_rate": 2.8090950721858566e-05, + "loss": 0.423, + "num_tokens": 170321276.0, + "step": 163 + }, + { + "epoch": 1.561904761904762, + "grad_norm": 0.10068633066029532, + "learning_rate": 2.7854596530572373e-05, + "loss": 0.4235, + "num_tokens": 171369852.0, + "step": 164 + }, + { + "epoch": 1.5714285714285714, + "grad_norm": 0.10964315592208115, + "learning_rate": 2.761820319324439e-05, + "loss": 0.4213, + "num_tokens": 172416928.0, + "step": 165 + }, + { + "epoch": 1.580952380952381, + "grad_norm": 0.10100497021797217, + "learning_rate": 2.7381796806755618e-05, + "loss": 0.4091, + "num_tokens": 173465504.0, + "step": 166 + }, + { + "epoch": 1.5904761904761906, + "grad_norm": 0.10257468860099382, + "learning_rate": 2.7145403469427632e-05, + "loss": 0.4277, + "num_tokens": 174514080.0, + "step": 167 + }, + { + "epoch": 1.6, + "grad_norm": 0.10256412501218375, + "learning_rate": 2.6909049278141446e-05, + "loss": 0.4133, + "num_tokens": 175562656.0, + "step": 168 + }, + { + "epoch": 1.6095238095238096, + "grad_norm": 0.09972039128770226, + "learning_rate": 2.6672760325456486e-05, + "loss": 0.4002, + "num_tokens": 176603347.0, + "step": 169 + }, + { + "epoch": 1.619047619047619, + "grad_norm": 0.10426974990040318, + "learning_rate": 2.6436562696730105e-05, + "loss": 0.4066, + "num_tokens": 177651923.0, + "step": 170 + }, + { + "epoch": 1.6285714285714286, + "grad_norm": 0.09764099355358319, + "learning_rate": 2.6200482467237863e-05, + "loss": 0.3986, + "num_tokens": 178700499.0, + "step": 171 + }, + { + "epoch": 1.638095238095238, + "grad_norm": 0.09783749201272511, + "learning_rate": 2.5964545699294906e-05, + "loss": 0.4098, + "num_tokens": 179739501.0, + "step": 172 + }, + { + "epoch": 1.6476190476190475, + "grad_norm": 0.10016886833829632, + "learning_rate": 2.57287784393788e-05, + "loss": 0.4039, + "num_tokens": 180775283.0, + "step": 173 + }, + { + "epoch": 1.657142857142857, + "grad_norm": 0.09275306717857602, + "learning_rate": 2.549320671525409e-05, + "loss": 0.4157, + "num_tokens": 181823859.0, + "step": 174 + }, + { + "epoch": 1.6666666666666665, + "grad_norm": 0.1077673633105214, + "learning_rate": 2.5257856533098983e-05, + "loss": 0.3923, + "num_tokens": 182872435.0, + "step": 175 + }, + { + "epoch": 1.6761904761904762, + "grad_norm": 0.09977513899163716, + "learning_rate": 2.5022753874634297e-05, + "loss": 0.4139, + "num_tokens": 183921011.0, + "step": 176 + }, + { + "epoch": 1.6857142857142857, + "grad_norm": 0.10500114960196057, + "learning_rate": 2.4787924694255237e-05, + "loss": 0.4298, + "num_tokens": 184966404.0, + "step": 177 + }, + { + "epoch": 1.6952380952380952, + "grad_norm": 0.10267740809228589, + "learning_rate": 2.4553394916166128e-05, + "loss": 0.4043, + "num_tokens": 185988428.0, + "step": 178 + }, + { + "epoch": 1.704761904761905, + "grad_norm": 0.09782384087772643, + "learning_rate": 2.4319190431518467e-05, + "loss": 0.4128, + "num_tokens": 187026022.0, + "step": 179 + }, + { + "epoch": 1.7142857142857144, + "grad_norm": 0.09935145643883452, + "learning_rate": 2.408533709555266e-05, + "loss": 0.4073, + "num_tokens": 188074598.0, + "step": 180 + }, + { + "epoch": 1.723809523809524, + "grad_norm": 0.09823975456156628, + "learning_rate": 2.3851860724743702e-05, + "loss": 0.4239, + "num_tokens": 189123174.0, + "step": 181 + }, + { + "epoch": 1.7333333333333334, + "grad_norm": 0.0935867612514239, + "learning_rate": 2.3618787093951122e-05, + "loss": 0.3959, + "num_tokens": 190161239.0, + "step": 182 + }, + { + "epoch": 1.7428571428571429, + "grad_norm": 0.0952586220940576, + "learning_rate": 2.3386141933573573e-05, + "loss": 0.4161, + "num_tokens": 191204653.0, + "step": 183 + }, + { + "epoch": 1.7523809523809524, + "grad_norm": 0.0972735469715154, + "learning_rate": 2.3153950926708284e-05, + "loss": 0.4254, + "num_tokens": 192248187.0, + "step": 184 + }, + { + "epoch": 1.7619047619047619, + "grad_norm": 0.09765335070026246, + "learning_rate": 2.2922239706315745e-05, + "loss": 0.4097, + "num_tokens": 193296763.0, + "step": 185 + }, + { + "epoch": 1.7714285714285714, + "grad_norm": 0.09395375586553344, + "learning_rate": 2.2691033852389936e-05, + "loss": 0.4158, + "num_tokens": 194332556.0, + "step": 186 + }, + { + "epoch": 1.7809523809523808, + "grad_norm": 0.0979061169174931, + "learning_rate": 2.2460358889134403e-05, + "loss": 0.4097, + "num_tokens": 195374377.0, + "step": 187 + }, + { + "epoch": 1.7904761904761903, + "grad_norm": 0.09628597338678281, + "learning_rate": 2.2230240282144454e-05, + "loss": 0.4135, + "num_tokens": 196408274.0, + "step": 188 + }, + { + "epoch": 1.8, + "grad_norm": 0.09346569128967716, + "learning_rate": 2.2000703435595925e-05, + "loss": 0.4186, + "num_tokens": 197456850.0, + "step": 189 + }, + { + "epoch": 1.8095238095238095, + "grad_norm": 0.09147276543919937, + "learning_rate": 2.1771773689440584e-05, + "loss": 0.3996, + "num_tokens": 198505426.0, + "step": 190 + }, + { + "epoch": 1.819047619047619, + "grad_norm": 0.09750710784011565, + "learning_rate": 2.154347631660876e-05, + "loss": 0.4043, + "num_tokens": 199554002.0, + "step": 191 + }, + { + "epoch": 1.8285714285714287, + "grad_norm": 0.0913058115268578, + "learning_rate": 2.13158365202193e-05, + "loss": 0.3962, + "num_tokens": 200597628.0, + "step": 192 + }, + { + "epoch": 1.8380952380952382, + "grad_norm": 0.10042865647687828, + "learning_rate": 2.108887943079721e-05, + "loss": 0.4138, + "num_tokens": 201646204.0, + "step": 193 + }, + { + "epoch": 1.8476190476190477, + "grad_norm": 0.10154366590416665, + "learning_rate": 2.086263010349938e-05, + "loss": 0.3903, + "num_tokens": 202673495.0, + "step": 194 + }, + { + "epoch": 1.8571428571428572, + "grad_norm": 0.09926045077906644, + "learning_rate": 2.0637113515348606e-05, + "loss": 0.4116, + "num_tokens": 203722071.0, + "step": 195 + }, + { + "epoch": 1.8666666666666667, + "grad_norm": 0.0996267272870035, + "learning_rate": 2.041235456247617e-05, + "loss": 0.4186, + "num_tokens": 204770647.0, + "step": 196 + }, + { + "epoch": 1.8761904761904762, + "grad_norm": 0.09030040292832316, + "learning_rate": 2.0188378057373487e-05, + "loss": 0.3981, + "num_tokens": 205819223.0, + "step": 197 + }, + { + "epoch": 1.8857142857142857, + "grad_norm": 0.09361807015787005, + "learning_rate": 1.9965208726152813e-05, + "loss": 0.418, + "num_tokens": 206867799.0, + "step": 198 + }, + { + "epoch": 1.8952380952380952, + "grad_norm": 0.0952778464179367, + "learning_rate": 1.9742871205817664e-05, + "loss": 0.4067, + "num_tokens": 207916375.0, + "step": 199 + }, + { + "epoch": 1.9047619047619047, + "grad_norm": 0.0958508990924158, + "learning_rate": 1.9521390041542946e-05, + "loss": 0.4079, + "num_tokens": 208964951.0, + "step": 200 + }, + { + "epoch": 1.9142857142857141, + "grad_norm": 0.09426032213012041, + "learning_rate": 1.930078968396527e-05, + "loss": 0.4149, + "num_tokens": 210013527.0, + "step": 201 + }, + { + "epoch": 1.9238095238095239, + "grad_norm": 0.09422254645281578, + "learning_rate": 1.9081094486483703e-05, + "loss": 0.416, + "num_tokens": 211062103.0, + "step": 202 + }, + { + "epoch": 1.9333333333333333, + "grad_norm": 0.0967208052867282, + "learning_rate": 1.8862328702571248e-05, + "loss": 0.4232, + "num_tokens": 212110679.0, + "step": 203 + }, + { + "epoch": 1.9428571428571428, + "grad_norm": 0.09663057153504559, + "learning_rate": 1.8644516483097362e-05, + "loss": 0.4232, + "num_tokens": 213148014.0, + "step": 204 + }, + { + "epoch": 1.9523809523809523, + "grad_norm": 0.09589026135548133, + "learning_rate": 1.8427681873661794e-05, + "loss": 0.4273, + "num_tokens": 214165732.0, + "step": 205 + }, + { + "epoch": 1.961904761904762, + "grad_norm": 0.09610415355364234, + "learning_rate": 1.8211848811940046e-05, + "loss": 0.4095, + "num_tokens": 215198160.0, + "step": 206 + }, + { + "epoch": 1.9714285714285715, + "grad_norm": 0.09111769904499907, + "learning_rate": 1.7997041125040758e-05, + "loss": 0.3925, + "num_tokens": 216246736.0, + "step": 207 + }, + { + "epoch": 1.980952380952381, + "grad_norm": 0.09264992556629768, + "learning_rate": 1.7783282526875256e-05, + "loss": 0.4025, + "num_tokens": 217274110.0, + "step": 208 + }, + { + "epoch": 1.9904761904761905, + "grad_norm": 0.09704193547718563, + "learning_rate": 1.7570596615539686e-05, + "loss": 0.4108, + "num_tokens": 218322686.0, + "step": 209 + }, + { + "epoch": 2.0, + "grad_norm": 0.09377388066183875, + "learning_rate": 1.7359006870709833e-05, + "loss": 0.4089, + "num_tokens": 219371262.0, + "step": 210 + }, + { + "epoch": 2.0095238095238095, + "grad_norm": 0.09938123388787974, + "learning_rate": 1.7148536651049078e-05, + "loss": 0.3893, + "num_tokens": 220419838.0, + "step": 211 + }, + { + "epoch": 2.019047619047619, + "grad_norm": 0.09776803753932861, + "learning_rate": 1.6939209191629695e-05, + "loss": 0.3867, + "num_tokens": 221466914.0, + "step": 212 + }, + { + "epoch": 2.0285714285714285, + "grad_norm": 0.0945336878870005, + "learning_rate": 1.6731047601367783e-05, + "loss": 0.3727, + "num_tokens": 222500757.0, + "step": 213 + }, + { + "epoch": 2.038095238095238, + "grad_norm": 0.10080591312139771, + "learning_rate": 1.652407486047214e-05, + "loss": 0.4044, + "num_tokens": 223534654.0, + "step": 214 + }, + { + "epoch": 2.0476190476190474, + "grad_norm": 0.09488441063056365, + "learning_rate": 1.6318313817907338e-05, + "loss": 0.3759, + "num_tokens": 224571070.0, + "step": 215 + }, + { + "epoch": 2.057142857142857, + "grad_norm": 0.10288256799622328, + "learning_rate": 1.611378718887129e-05, + "loss": 0.3998, + "num_tokens": 225614712.0, + "step": 216 + }, + { + "epoch": 2.066666666666667, + "grad_norm": 0.09288816059430816, + "learning_rate": 1.5910517552287592e-05, + "loss": 0.3869, + "num_tokens": 226663288.0, + "step": 217 + }, + { + "epoch": 2.0761904761904764, + "grad_norm": 0.10365592697137521, + "learning_rate": 1.5708527348312875e-05, + "loss": 0.4079, + "num_tokens": 227711864.0, + "step": 218 + }, + { + "epoch": 2.085714285714286, + "grad_norm": 0.10036362213052903, + "learning_rate": 1.5507838875859513e-05, + "loss": 0.3897, + "num_tokens": 228740568.0, + "step": 219 + }, + { + "epoch": 2.0952380952380953, + "grad_norm": 0.10069950166892538, + "learning_rate": 1.5308474290133928e-05, + "loss": 0.3869, + "num_tokens": 229773511.0, + "step": 220 + }, + { + "epoch": 2.104761904761905, + "grad_norm": 0.09383305136467432, + "learning_rate": 1.5110455600190732e-05, + "loss": 0.3712, + "num_tokens": 230799557.0, + "step": 221 + }, + { + "epoch": 2.1142857142857143, + "grad_norm": 0.09538010893865374, + "learning_rate": 1.4913804666503006e-05, + "loss": 0.392, + "num_tokens": 231842966.0, + "step": 222 + }, + { + "epoch": 2.123809523809524, + "grad_norm": 0.10396436607915151, + "learning_rate": 1.4718543198548998e-05, + "loss": 0.3954, + "num_tokens": 232891542.0, + "step": 223 + }, + { + "epoch": 2.1333333333333333, + "grad_norm": 0.10605230273952929, + "learning_rate": 1.4524692752415493e-05, + "loss": 0.4112, + "num_tokens": 233940118.0, + "step": 224 + }, + { + "epoch": 2.142857142857143, + "grad_norm": 0.09097903063824954, + "learning_rate": 1.4332274728418091e-05, + "loss": 0.3866, + "num_tokens": 234988694.0, + "step": 225 + }, + { + "epoch": 2.1523809523809523, + "grad_norm": 0.09118944468490846, + "learning_rate": 1.4141310368738706e-05, + "loss": 0.3803, + "num_tokens": 236037270.0, + "step": 226 + }, + { + "epoch": 2.1619047619047618, + "grad_norm": 0.09183411536635765, + "learning_rate": 1.395182075508052e-05, + "loss": 0.3908, + "num_tokens": 237085846.0, + "step": 227 + }, + { + "epoch": 2.1714285714285713, + "grad_norm": 0.10016025960797259, + "learning_rate": 1.3763826806340624e-05, + "loss": 0.3843, + "num_tokens": 238134422.0, + "step": 228 + }, + { + "epoch": 2.1809523809523808, + "grad_norm": 0.09593444671059793, + "learning_rate": 1.3577349276300687e-05, + "loss": 0.4176, + "num_tokens": 239166880.0, + "step": 229 + }, + { + "epoch": 2.1904761904761907, + "grad_norm": 0.09056580302118011, + "learning_rate": 1.3392408751335806e-05, + "loss": 0.386, + "num_tokens": 240215456.0, + "step": 230 + }, + { + "epoch": 2.2, + "grad_norm": 0.08672263238664585, + "learning_rate": 1.3209025648141837e-05, + "loss": 0.3804, + "num_tokens": 241264032.0, + "step": 231 + }, + { + "epoch": 2.2095238095238097, + "grad_norm": 0.09349340905956938, + "learning_rate": 1.3027220211481492e-05, + "loss": 0.4013, + "num_tokens": 242312608.0, + "step": 232 + }, + { + "epoch": 2.219047619047619, + "grad_norm": 0.09355849794730789, + "learning_rate": 1.2847012511949407e-05, + "loss": 0.3844, + "num_tokens": 243361184.0, + "step": 233 + }, + { + "epoch": 2.2285714285714286, + "grad_norm": 0.09744385272655379, + "learning_rate": 1.2668422443756406e-05, + "loss": 0.3935, + "num_tokens": 244409760.0, + "step": 234 + }, + { + "epoch": 2.238095238095238, + "grad_norm": 0.08680160631151776, + "learning_rate": 1.2491469722533267e-05, + "loss": 0.3688, + "num_tokens": 245437668.0, + "step": 235 + }, + { + "epoch": 2.2476190476190476, + "grad_norm": 0.09319340118791133, + "learning_rate": 1.2316173883154205e-05, + "loss": 0.4037, + "num_tokens": 246486244.0, + "step": 236 + }, + { + "epoch": 2.257142857142857, + "grad_norm": 0.09570593846989157, + "learning_rate": 1.2142554277580288e-05, + "loss": 0.4087, + "num_tokens": 247534820.0, + "step": 237 + }, + { + "epoch": 2.2666666666666666, + "grad_norm": 0.09273713845652151, + "learning_rate": 1.1970630072723054e-05, + "loss": 0.3872, + "num_tokens": 248578354.0, + "step": 238 + }, + { + "epoch": 2.276190476190476, + "grad_norm": 0.09056425329558235, + "learning_rate": 1.1800420248328584e-05, + "loss": 0.401, + "num_tokens": 249626930.0, + "step": 239 + }, + { + "epoch": 2.2857142857142856, + "grad_norm": 0.08971332705129444, + "learning_rate": 1.1631943594882181e-05, + "loss": 0.3843, + "num_tokens": 250664670.0, + "step": 240 + }, + { + "epoch": 2.295238095238095, + "grad_norm": 0.08743967702254121, + "learning_rate": 1.1465218711534009e-05, + "loss": 0.3888, + "num_tokens": 251713246.0, + "step": 241 + }, + { + "epoch": 2.3047619047619046, + "grad_norm": 0.09669581841740169, + "learning_rate": 1.1300264004045791e-05, + "loss": 0.3904, + "num_tokens": 252727243.0, + "step": 242 + }, + { + "epoch": 2.314285714285714, + "grad_norm": 0.08847551102862308, + "learning_rate": 1.1137097682758904e-05, + "loss": 0.3931, + "num_tokens": 253775819.0, + "step": 243 + }, + { + "epoch": 2.323809523809524, + "grad_norm": 0.0883622849523124, + "learning_rate": 1.097573776058403e-05, + "loss": 0.3924, + "num_tokens": 254824395.0, + "step": 244 + }, + { + "epoch": 2.3333333333333335, + "grad_norm": 0.08836368209792722, + "learning_rate": 1.0816202051012592e-05, + "loss": 0.4003, + "num_tokens": 255872971.0, + "step": 245 + }, + { + "epoch": 2.342857142857143, + "grad_norm": 0.09380865250937885, + "learning_rate": 1.0658508166150224e-05, + "loss": 0.3927, + "num_tokens": 256908412.0, + "step": 246 + }, + { + "epoch": 2.3523809523809525, + "grad_norm": 0.09222949934619636, + "learning_rate": 1.0502673514772467e-05, + "loss": 0.3865, + "num_tokens": 257956988.0, + "step": 247 + }, + { + "epoch": 2.361904761904762, + "grad_norm": 0.08680353575716368, + "learning_rate": 1.0348715300402904e-05, + "loss": 0.3963, + "num_tokens": 259005564.0, + "step": 248 + }, + { + "epoch": 2.3714285714285714, + "grad_norm": 0.08894767953247357, + "learning_rate": 1.0196650519413953e-05, + "loss": 0.3825, + "num_tokens": 260054140.0, + "step": 249 + }, + { + "epoch": 2.380952380952381, + "grad_norm": 0.08873629102361436, + "learning_rate": 1.0046495959150554e-05, + "loss": 0.3654, + "num_tokens": 261087339.0, + "step": 250 + }, + { + "epoch": 2.3904761904761904, + "grad_norm": 0.08839912229958018, + "learning_rate": 9.898268196076893e-06, + "loss": 0.3829, + "num_tokens": 262131920.0, + "step": 251 + }, + { + "epoch": 2.4, + "grad_norm": 0.0915382416553906, + "learning_rate": 9.751983593946454e-06, + "loss": 0.404, + "num_tokens": 263180496.0, + "step": 252 + }, + { + "epoch": 2.4095238095238094, + "grad_norm": 0.09258687286456545, + "learning_rate": 9.607658301995479e-06, + "loss": 0.3838, + "num_tokens": 264224992.0, + "step": 253 + }, + { + "epoch": 2.419047619047619, + "grad_norm": 0.09067757771386709, + "learning_rate": 9.465308253160207e-06, + "loss": 0.4032, + "num_tokens": 265261235.0, + "step": 254 + }, + { + "epoch": 2.4285714285714284, + "grad_norm": 0.08660235449561558, + "learning_rate": 9.324949162317918e-06, + "loss": 0.3933, + "num_tokens": 266309811.0, + "step": 255 + }, + { + "epoch": 2.4380952380952383, + "grad_norm": 0.08477723004636327, + "learning_rate": 9.186596524552071e-06, + "loss": 0.3849, + "num_tokens": 267358387.0, + "step": 256 + }, + { + "epoch": 2.447619047619048, + "grad_norm": 0.08832895308703803, + "learning_rate": 9.050265613441716e-06, + "loss": 0.3884, + "num_tokens": 268406963.0, + "step": 257 + }, + { + "epoch": 2.4571428571428573, + "grad_norm": 0.09028896770395589, + "learning_rate": 8.915971479375343e-06, + "loss": 0.3954, + "num_tokens": 269455539.0, + "step": 258 + }, + { + "epoch": 2.466666666666667, + "grad_norm": 0.08535507936244807, + "learning_rate": 8.783728947889394e-06, + "loss": 0.3974, + "num_tokens": 270504115.0, + "step": 259 + }, + { + "epoch": 2.4761904761904763, + "grad_norm": 0.08705908759384202, + "learning_rate": 8.653552618031566e-06, + "loss": 0.3906, + "num_tokens": 271552691.0, + "step": 260 + }, + { + "epoch": 2.4857142857142858, + "grad_norm": 0.0870873689987336, + "learning_rate": 8.525456860749148e-06, + "loss": 0.3906, + "num_tokens": 272601267.0, + "step": 261 + }, + { + "epoch": 2.4952380952380953, + "grad_norm": 0.09165748014494636, + "learning_rate": 8.399455817302518e-06, + "loss": 0.3715, + "num_tokens": 273649843.0, + "step": 262 + }, + { + "epoch": 2.5047619047619047, + "grad_norm": 0.09350940712177117, + "learning_rate": 8.275563397704008e-06, + "loss": 0.3915, + "num_tokens": 274698419.0, + "step": 263 + }, + { + "epoch": 2.5142857142857142, + "grad_norm": 0.08755269735890109, + "learning_rate": 8.15379327918229e-06, + "loss": 0.385, + "num_tokens": 275746995.0, + "step": 264 + }, + { + "epoch": 2.5238095238095237, + "grad_norm": 0.08946794484656653, + "learning_rate": 8.034158904672464e-06, + "loss": 0.3998, + "num_tokens": 276795571.0, + "step": 265 + }, + { + "epoch": 2.533333333333333, + "grad_norm": 0.08832293863323495, + "learning_rate": 7.916673481332023e-06, + "loss": 0.3931, + "num_tokens": 277839396.0, + "step": 266 + }, + { + "epoch": 2.5428571428571427, + "grad_norm": 0.08623467394541917, + "learning_rate": 7.80134997908282e-06, + "loss": 0.3789, + "num_tokens": 278880112.0, + "step": 267 + }, + { + "epoch": 2.552380952380952, + "grad_norm": 0.08906185793908447, + "learning_rate": 7.688201129179253e-06, + "loss": 0.3804, + "num_tokens": 279901273.0, + "step": 268 + }, + { + "epoch": 2.5619047619047617, + "grad_norm": 0.08704875351515759, + "learning_rate": 7.57723942280278e-06, + "loss": 0.3936, + "num_tokens": 280949849.0, + "step": 269 + }, + { + "epoch": 2.571428571428571, + "grad_norm": 0.0908241593075488, + "learning_rate": 7.468477109682939e-06, + "loss": 0.3956, + "num_tokens": 281982527.0, + "step": 270 + }, + { + "epoch": 2.580952380952381, + "grad_norm": 0.08451740456409636, + "learning_rate": 7.361926196745029e-06, + "loss": 0.3804, + "num_tokens": 283031103.0, + "step": 271 + }, + { + "epoch": 2.5904761904761906, + "grad_norm": 0.17055052389187877, + "learning_rate": 7.257598446784601e-06, + "loss": 0.4125, + "num_tokens": 284065587.0, + "step": 272 + }, + { + "epoch": 2.6, + "grad_norm": 0.08617217503344544, + "learning_rate": 7.155505377168878e-06, + "loss": 0.3876, + "num_tokens": 285110160.0, + "step": 273 + }, + { + "epoch": 2.6095238095238096, + "grad_norm": 0.08922032833324223, + "learning_rate": 7.0556582585653005e-06, + "loss": 0.4008, + "num_tokens": 286158736.0, + "step": 274 + }, + { + "epoch": 2.619047619047619, + "grad_norm": 0.08553753681831719, + "learning_rate": 6.958068113697271e-06, + "loss": 0.3823, + "num_tokens": 287207312.0, + "step": 275 + }, + { + "epoch": 2.6285714285714286, + "grad_norm": 0.08225391128213397, + "learning_rate": 6.862745716127312e-06, + "loss": 0.3801, + "num_tokens": 288255167.0, + "step": 276 + }, + { + "epoch": 2.638095238095238, + "grad_norm": 0.08662018226588346, + "learning_rate": 6.769701589067673e-06, + "loss": 0.3953, + "num_tokens": 289303743.0, + "step": 277 + }, + { + "epoch": 2.6476190476190475, + "grad_norm": 0.08526982029844157, + "learning_rate": 6.678946004218656e-06, + "loss": 0.3745, + "num_tokens": 290352319.0, + "step": 278 + }, + { + "epoch": 2.657142857142857, + "grad_norm": 0.0892892277251501, + "learning_rate": 6.590488980634621e-06, + "loss": 0.3886, + "num_tokens": 291389924.0, + "step": 279 + }, + { + "epoch": 2.6666666666666665, + "grad_norm": 0.08740330934668722, + "learning_rate": 6.504340283617945e-06, + "loss": 0.3812, + "num_tokens": 292438500.0, + "step": 280 + }, + { + "epoch": 2.6761904761904765, + "grad_norm": 0.086559933823555, + "learning_rate": 6.42050942364096e-06, + "loss": 0.3982, + "num_tokens": 293487076.0, + "step": 281 + }, + { + "epoch": 2.685714285714286, + "grad_norm": 0.08894610557736665, + "learning_rate": 6.339005655296042e-06, + "loss": 0.4123, + "num_tokens": 294535652.0, + "step": 282 + }, + { + "epoch": 2.6952380952380954, + "grad_norm": 0.08964722806361651, + "learning_rate": 6.259837976273936e-06, + "loss": 0.3948, + "num_tokens": 295584228.0, + "step": 283 + }, + { + "epoch": 2.704761904761905, + "grad_norm": 0.08974916941590187, + "learning_rate": 6.183015126370456e-06, + "loss": 0.3908, + "num_tokens": 296632804.0, + "step": 284 + }, + { + "epoch": 2.7142857142857144, + "grad_norm": 0.08621800035719321, + "learning_rate": 6.108545586521626e-06, + "loss": 0.3907, + "num_tokens": 297670869.0, + "step": 285 + }, + { + "epoch": 2.723809523809524, + "grad_norm": 0.08832749227824972, + "learning_rate": 6.036437577867446e-06, + "loss": 0.3835, + "num_tokens": 298719445.0, + "step": 286 + }, + { + "epoch": 2.7333333333333334, + "grad_norm": 0.08602423863031433, + "learning_rate": 5.966699060844288e-06, + "loss": 0.395, + "num_tokens": 299768021.0, + "step": 287 + }, + { + "epoch": 2.742857142857143, + "grad_norm": 0.08498192695435394, + "learning_rate": 5.899337734306109e-06, + "loss": 0.379, + "num_tokens": 300816597.0, + "step": 288 + }, + { + "epoch": 2.7523809523809524, + "grad_norm": 0.08307980900807126, + "learning_rate": 5.834361034674521e-06, + "loss": 0.3699, + "num_tokens": 301861052.0, + "step": 289 + }, + { + "epoch": 2.761904761904762, + "grad_norm": 0.08566641060598595, + "learning_rate": 5.771776135117838e-06, + "loss": 0.383, + "num_tokens": 302891955.0, + "step": 290 + }, + { + "epoch": 2.7714285714285714, + "grad_norm": 0.0836883508324955, + "learning_rate": 5.711589944759201e-06, + "loss": 0.3848, + "num_tokens": 303940531.0, + "step": 291 + }, + { + "epoch": 2.780952380952381, + "grad_norm": 0.08305006837890677, + "learning_rate": 5.65380910791383e-06, + "loss": 0.378, + "num_tokens": 304983945.0, + "step": 292 + }, + { + "epoch": 2.7904761904761903, + "grad_norm": 0.0860162465052085, + "learning_rate": 5.598440003355505e-06, + "loss": 0.398, + "num_tokens": 306032521.0, + "step": 293 + }, + { + "epoch": 2.8, + "grad_norm": 0.08720885221348973, + "learning_rate": 5.545488743612393e-06, + "loss": 0.387, + "num_tokens": 307081097.0, + "step": 294 + }, + { + "epoch": 2.8095238095238093, + "grad_norm": 0.08553460337578449, + "learning_rate": 5.494961174292244e-06, + "loss": 0.4015, + "num_tokens": 308115955.0, + "step": 295 + }, + { + "epoch": 2.819047619047619, + "grad_norm": 0.08315689006826711, + "learning_rate": 5.4468628734370595e-06, + "loss": 0.3691, + "num_tokens": 309148420.0, + "step": 296 + }, + { + "epoch": 2.8285714285714287, + "grad_norm": 0.08660069976632101, + "learning_rate": 5.401199150907295e-06, + "loss": 0.3982, + "num_tokens": 310196996.0, + "step": 297 + }, + { + "epoch": 2.8380952380952382, + "grad_norm": 0.09173080598609032, + "learning_rate": 5.35797504779567e-06, + "loss": 0.3992, + "num_tokens": 311245572.0, + "step": 298 + }, + { + "epoch": 2.8476190476190477, + "grad_norm": 0.08376794111360732, + "learning_rate": 5.317195335870665e-06, + "loss": 0.3741, + "num_tokens": 312294148.0, + "step": 299 + }, + { + "epoch": 2.857142857142857, + "grad_norm": 0.08670819869393387, + "learning_rate": 5.278864517049726e-06, + "loss": 0.3784, + "num_tokens": 313340885.0, + "step": 300 + }, + { + "epoch": 2.8666666666666667, + "grad_norm": 0.08459915010710338, + "learning_rate": 5.242986822902276e-06, + "loss": 0.3882, + "num_tokens": 314389461.0, + "step": 301 + }, + { + "epoch": 2.876190476190476, + "grad_norm": 0.08507043143310072, + "learning_rate": 5.209566214182558e-06, + "loss": 0.3814, + "num_tokens": 315438037.0, + "step": 302 + }, + { + "epoch": 2.8857142857142857, + "grad_norm": 0.08610471258721382, + "learning_rate": 5.178606380392399e-06, + "loss": 0.3966, + "num_tokens": 316486613.0, + "step": 303 + }, + { + "epoch": 2.895238095238095, + "grad_norm": 0.08731429972743228, + "learning_rate": 5.150110739373885e-06, + "loss": 0.3989, + "num_tokens": 317526402.0, + "step": 304 + }, + { + "epoch": 2.9047619047619047, + "grad_norm": 0.08777662497316624, + "learning_rate": 5.1240824369320615e-06, + "loss": 0.3883, + "num_tokens": 318574978.0, + "step": 305 + }, + { + "epoch": 2.914285714285714, + "grad_norm": 0.0887627325620308, + "learning_rate": 5.10052434648764e-06, + "loss": 0.3939, + "num_tokens": 319623554.0, + "step": 306 + }, + { + "epoch": 2.923809523809524, + "grad_norm": 0.09904295109514709, + "learning_rate": 5.079439068759786e-06, + "loss": 0.4064, + "num_tokens": 320672130.0, + "step": 307 + }, + { + "epoch": 2.9333333333333336, + "grad_norm": 0.08491496803282857, + "learning_rate": 5.0608289314790114e-06, + "loss": 0.3698, + "num_tokens": 321720706.0, + "step": 308 + }, + { + "epoch": 2.942857142857143, + "grad_norm": 0.0852440176407223, + "learning_rate": 5.044695989130197e-06, + "loss": 0.3887, + "num_tokens": 322769282.0, + "step": 309 + }, + { + "epoch": 2.9523809523809526, + "grad_norm": 0.08773303183288964, + "learning_rate": 5.031042022725793e-06, + "loss": 0.4018, + "num_tokens": 323817858.0, + "step": 310 + }, + { + "epoch": 2.961904761904762, + "grad_norm": 0.08198705761675795, + "learning_rate": 5.019868539609196e-06, + "loss": 0.3733, + "num_tokens": 324866434.0, + "step": 311 + }, + { + "epoch": 2.9714285714285715, + "grad_norm": 0.08878406705500722, + "learning_rate": 5.011176773288352e-06, + "loss": 0.3978, + "num_tokens": 325915010.0, + "step": 312 + }, + { + "epoch": 2.980952380952381, + "grad_norm": 0.09230604959152837, + "learning_rate": 5.004967683299575e-06, + "loss": 0.4028, + "num_tokens": 326963586.0, + "step": 313 + }, + { + "epoch": 2.9904761904761905, + "grad_norm": 0.08995130727798627, + "learning_rate": 5.001241955101616e-06, + "loss": 0.4194, + "num_tokens": 328008317.0, + "step": 314 + }, + { + "epoch": 3.0, + "grad_norm": 0.07991456010161288, + "learning_rate": 5e-06, + "loss": 0.3719, + "num_tokens": 329056893.0, + "step": 315 + }, + { + "epoch": 3.0, + "step": 315, + "total_flos": 287156144701440.0, + "train_loss": 0.4225423020029825, + "train_runtime": 9655.7571, + "train_samples_per_second": 2.082, + "train_steps_per_second": 0.033 + } + ], + "logging_steps": 1, + "max_steps": 315, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 287156144701440.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..dc1334b --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54bb79b729a51c5079ba67bf899f87d7cff7e804813384e3fa7ac18af23f2b74 +size 9912