commit ea98d0a1867083c050cb0e9403acf711fa74eed9 Author: ModelHub XC Date: Sun Aug 23 06:58:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: gshasiri/SmolLM3-SFT-Second-Round Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..517e8f5 --- /dev/null +++ b/README.md @@ -0,0 +1,59 @@ +--- +base_model: gshasiri/SmolLM3-Mid-Second-Round +library_name: transformers +model_name: SmolLM3-SFT-Second-Round +tags: +- generated_from_trainer +- sft +- trl +- alignment-handbook +licence: license +--- + +# Model Card for SmolLM3-SFT-Second-Round + +This model is a fine-tuned version of [gshasiri/SmolLM3-Mid-Second-Round](https://huggingface.co/gshasiri/SmolLM3-Mid-Second-Round). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="gshasiri/SmolLM3-SFT-Second-Round", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/shamanework-pl/huggingface/runs/npyo7nk5) + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.25.1 +- Transformers: 4.57.1 +- Pytorch: 2.6.0+cu126 +- Datasets: 4.4.1 +- Tokenizers: 0.22.1 + +## Citations + + + +Cite TRL as: + +```bibtex +@misc{vonwerra2022trl, + title = {{TRL: Transformer Reinforcement Learning}}, + author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec}, + year = 2020, + journal = {GitHub repository}, + publisher = {GitHub}, + howpublished = {\url{https://github.com/huggingface/trl}} +} +``` \ No newline at end of file diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..dbc96b3 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 5.0, + "total_flos": 3832823860199424.0, + "train_loss": 0.9150249538750485, + "train_runtime": 89344.5669, + "train_samples": 1444084, + "train_samples_per_second": 1.655, + "train_steps_per_second": 0.013 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..b481759 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,96 @@ +{# ───── defaults ───── #} +{%- if enable_thinking is not defined -%} +{%- set enable_thinking = true -%} +{%- endif -%} + +{# ───── reasoning mode ───── #} +{%- if enable_thinking -%} + {%- set reasoning_mode = "/think" -%} +{%- else -%} + {%- set reasoning_mode = "/no_think" -%} +{%- endif -%} + +{# ───── header (system message) ───── #} +{{- "<|im_start|>system\n" -}} + +{%- if messages[0].role == "system" -%} + {%- set system_message = messages[0].content -%} + {%- if "/no_think" in system_message -%} + {%- set reasoning_mode = "/no_think" -%} + {%- elif "/think" in system_message -%} + {%- set reasoning_mode = "/think" -%} + {%- endif -%} + {%- set custom_instructions = system_message.replace("/no_think", "").replace("/think", "").rstrip() -%} +{%- endif -%} + +{%- if "/system_override" in system_message -%} + {{- custom_instructions.replace("/system_override", "").rstrip() -}} + {{- "<|im_end|>\n" -}} +{%- else -%} + {{- "## Metadata\n\n" -}} + {{- "Knowledge Cutoff Date: June 2025\n" -}} + {%- set today = strftime_now("%d %B %Y") -%} + {{- "Today Date: " ~ today ~ "\n" -}} + {{- "Reasoning Mode: " + reasoning_mode + "\n\n" -}} + + {{- "## Custom Instructions\n\n" -}} + {%- if custom_instructions -%} + {{- custom_instructions + "\n\n" -}} + {%- elif reasoning_mode == "/think" -%} + {{- "You are a helpful AI assistant named SmolLM, trained by Hugging Face. Your role as an assistant involves thoroughly exploring questions through a systematic thinking process before providing the final precise and accurate solutions. This requires engaging in a comprehensive cycle of analysis, summarizing, exploration, reassessment, reflection, backtracking, and iteration to develop well-considered thinking process. Please structure your response into two main sections: Thought and Solution using the specified format: Thought section Solution section. In the Thought section, detail your reasoning process in steps. Each step should include detailed considerations such as analysing questions, summarizing relevant findings, brainstorming new ideas, verifying the accuracy of the current steps, refining any errors, and revisiting previous steps. In the Solution section, based on various attempts, explorations, and reflections from the Thought section, systematically present the final solution that you deem correct. The Solution section should be logical, accurate, and concise and detail necessary steps needed to reach the conclusion.\n\n" -}} + {%- else -%} + {{- "You are a helpful AI assistant named SmolLM, trained by Hugging Face.\n\n" -}} + {%- endif -%} + + {{- "## Tools\n\n" -}} + {{- "### XML Tools\n\n" -}} + {%- if tools -%} + {%- set ns = namespace(xml_tool_string="You may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n\n") -%} + {%- for tool in tools -%} + {%- set ns.xml_tool_string = ns.xml_tool_string ~ (tool | tojson) ~ "\n" -%} + {%- endfor -%} + {%- set xml_tools = ns.xml_tool_string + "\n\nFor each function call, return a json object with function name and arguments within XML tags." -%} + {%- endif -%} + {%- if xml_tools -%} + {{- xml_tools -}} + {%- else -%} + {{- "None" -}} + {%- endif -%} + {{- "\n\n" -}} + {{- "### Python Tools\n\n" -}} + {%- if python_tools -%} + {{- python_tools -}} + {%- else -%} + {{- "None" -}} + {%- endif -%} + {{- "\n\n" -}} + {{- "<|im_end|>\n" -}} +{%- endif -%} + +{# ───── main loop ───── #} +{%- for message in messages -%} + {%- set content = message.content if message.content is string else "" -%} + {%- if message.role == "user" -%} + {{ "<|im_start|>" + message.role + "\n" + content + "<|im_end|>\n" }} + {%- elif message.role == "assistant" -%} + {% generation %} + {%- if reasoning_mode == "/think" -%} + {{ "<|im_start|>assistant\n" + content.lstrip("\n") + "<|im_end|>\n" }} + {%- else -%} + {{ "<|im_start|>assistant\n" + "\n\n\n" + content.lstrip("\n") + "<|im_end|>\n" }} + {%- endif -%} + {% endgeneration %} + + {%- elif message.role == "tool" -%} + {{ "<|im_start|>" + "user\n" + content + "<|im_end|>\n" }} + {%- endif -%} +{%- endfor -%} + +{# ───── generation prompt ───── #} +{%- if add_generation_prompt -%} + {%- if reasoning_mode == "/think" -%} + {{ "<|im_start|>assistant\n" }} + {%- else -%} + {{ "<|im_start|>assistant\n" + "\n\n\n" }} + {%- endif -%} +{%- endif -%} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..45c637a --- /dev/null +++ b/config.json @@ -0,0 +1,35 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128012, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..e8c9e21 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128012, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..185e4a3 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acf8caf58d44c3a9dddf50b4384b228f7464fd7402d77284c8f4f1a8dd1d45c0 +size 2471645608 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..0656edf --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,10 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": "<|im_end|>" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..f342589 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e7c979daf2c715603b21e094ce7e032280b007311a070cdf98ed708c492d614 +size 17209792 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..7aa903e --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2062 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128014": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128015": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128016": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128017": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128018": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|im_end|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..dbc96b3 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 5.0, + "total_flos": 3832823860199424.0, + "train_loss": 0.9150249538750485, + "train_runtime": 89344.5669, + "train_samples": 1444084, + "train_samples_per_second": 1.655, + "train_steps_per_second": 0.013 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..adc3833 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,9323 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 1160, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004326663061114116, + "grad_norm": 9.830283616452624, + "learning_rate": 0.0, + "loss": 1.3476, + "num_tokens": 8052727.0, + "step": 1 + }, + { + "epoch": 0.008653326122228232, + "grad_norm": 9.519867321484186, + "learning_rate": 5.714285714285715e-07, + "loss": 1.3642, + "num_tokens": 16316249.0, + "step": 2 + }, + { + "epoch": 0.012979989183342347, + "grad_norm": 9.442336551776892, + "learning_rate": 1.142857142857143e-06, + "loss": 1.367, + "num_tokens": 24414956.0, + "step": 3 + }, + { + "epoch": 0.017306652244456464, + "grad_norm": 9.471009290013246, + "learning_rate": 1.7142857142857145e-06, + "loss": 1.3358, + "num_tokens": 32432061.0, + "step": 4 + }, + { + "epoch": 0.02163331530557058, + "grad_norm": 9.115653849817324, + "learning_rate": 2.285714285714286e-06, + "loss": 1.3613, + "num_tokens": 40560264.0, + "step": 5 + }, + { + "epoch": 0.025959978366684695, + "grad_norm": 6.542355004343838, + "learning_rate": 2.8571428571428573e-06, + "loss": 1.3277, + "num_tokens": 48746761.0, + "step": 6 + }, + { + "epoch": 0.03028664142779881, + "grad_norm": 3.9396532324883506, + "learning_rate": 3.428571428571429e-06, + "loss": 1.2881, + "num_tokens": 56923208.0, + "step": 7 + }, + { + "epoch": 0.03461330448891293, + "grad_norm": 2.877023388671469, + "learning_rate": 4.000000000000001e-06, + "loss": 1.2755, + "num_tokens": 65037147.0, + "step": 8 + }, + { + "epoch": 0.038939967550027044, + "grad_norm": 2.636083130941496, + "learning_rate": 4.571428571428572e-06, + "loss": 1.2177, + "num_tokens": 73157157.0, + "step": 9 + }, + { + "epoch": 0.04326663061114116, + "grad_norm": 2.8265457962852625, + "learning_rate": 5.142857142857142e-06, + "loss": 1.2418, + "num_tokens": 81330425.0, + "step": 10 + }, + { + "epoch": 0.047593293672255274, + "grad_norm": 2.4691679478623354, + "learning_rate": 5.7142857142857145e-06, + "loss": 1.2406, + "num_tokens": 89515900.0, + "step": 11 + }, + { + "epoch": 0.05191995673336939, + "grad_norm": 1.3604966781466097, + "learning_rate": 6.285714285714286e-06, + "loss": 1.2218, + "num_tokens": 97822905.0, + "step": 12 + }, + { + "epoch": 0.056246619794483504, + "grad_norm": 1.4377360792946812, + "learning_rate": 6.857142857142858e-06, + "loss": 1.2148, + "num_tokens": 105891211.0, + "step": 13 + }, + { + "epoch": 0.06057328285559762, + "grad_norm": 1.1167760820906318, + "learning_rate": 7.428571428571429e-06, + "loss": 1.1911, + "num_tokens": 114159655.0, + "step": 14 + }, + { + "epoch": 0.06489994591671173, + "grad_norm": 0.8336030501572852, + "learning_rate": 8.000000000000001e-06, + "loss": 1.1865, + "num_tokens": 122302268.0, + "step": 15 + }, + { + "epoch": 0.06922660897782586, + "grad_norm": 0.8902289923264906, + "learning_rate": 8.571428571428571e-06, + "loss": 1.1802, + "num_tokens": 130363765.0, + "step": 16 + }, + { + "epoch": 0.07355327203893997, + "grad_norm": 0.7890837270197739, + "learning_rate": 9.142857142857144e-06, + "loss": 1.1546, + "num_tokens": 138559833.0, + "step": 17 + }, + { + "epoch": 0.07787993510005409, + "grad_norm": 0.6619514307664508, + "learning_rate": 9.714285714285715e-06, + "loss": 1.1679, + "num_tokens": 146796904.0, + "step": 18 + }, + { + "epoch": 0.0822065981611682, + "grad_norm": 0.7437862013859642, + "learning_rate": 1.0285714285714285e-05, + "loss": 1.1696, + "num_tokens": 155019734.0, + "step": 19 + }, + { + "epoch": 0.08653326122228232, + "grad_norm": 0.5923572964078235, + "learning_rate": 1.0857142857142858e-05, + "loss": 1.1397, + "num_tokens": 163012614.0, + "step": 20 + }, + { + "epoch": 0.09085992428339643, + "grad_norm": 0.614737751373971, + "learning_rate": 1.1428571428571429e-05, + "loss": 1.1332, + "num_tokens": 171242469.0, + "step": 21 + }, + { + "epoch": 0.09518658734451055, + "grad_norm": 0.5921550598589083, + "learning_rate": 1.2e-05, + "loss": 1.1474, + "num_tokens": 179418820.0, + "step": 22 + }, + { + "epoch": 0.09951325040562466, + "grad_norm": 0.5628528294416506, + "learning_rate": 1.2571428571428572e-05, + "loss": 1.1199, + "num_tokens": 187617438.0, + "step": 23 + }, + { + "epoch": 0.10383991346673878, + "grad_norm": 0.4637445426933222, + "learning_rate": 1.3142857142857145e-05, + "loss": 1.1421, + "num_tokens": 195684109.0, + "step": 24 + }, + { + "epoch": 0.10816657652785289, + "grad_norm": 0.5454906393782035, + "learning_rate": 1.3714285714285716e-05, + "loss": 1.1434, + "num_tokens": 203827960.0, + "step": 25 + }, + { + "epoch": 0.11249323958896701, + "grad_norm": 0.464326953548454, + "learning_rate": 1.4285714285714287e-05, + "loss": 1.0968, + "num_tokens": 211915123.0, + "step": 26 + }, + { + "epoch": 0.11681990265008113, + "grad_norm": 0.5241833371081568, + "learning_rate": 1.4857142857142858e-05, + "loss": 1.1213, + "num_tokens": 220176060.0, + "step": 27 + }, + { + "epoch": 0.12114656571119524, + "grad_norm": 0.4222502236528627, + "learning_rate": 1.542857142857143e-05, + "loss": 1.119, + "num_tokens": 228562361.0, + "step": 28 + }, + { + "epoch": 0.12547322877230935, + "grad_norm": 0.40197376998712087, + "learning_rate": 1.6000000000000003e-05, + "loss": 1.1004, + "num_tokens": 236738124.0, + "step": 29 + }, + { + "epoch": 0.12979989183342347, + "grad_norm": 0.667263167983757, + "learning_rate": 1.6571428571428574e-05, + "loss": 1.1035, + "num_tokens": 244925231.0, + "step": 30 + }, + { + "epoch": 0.1341265548945376, + "grad_norm": 0.5276754605829086, + "learning_rate": 1.7142857142857142e-05, + "loss": 1.0955, + "num_tokens": 253126364.0, + "step": 31 + }, + { + "epoch": 0.13845321795565171, + "grad_norm": 0.508680706529472, + "learning_rate": 1.7714285714285717e-05, + "loss": 1.0788, + "num_tokens": 261432036.0, + "step": 32 + }, + { + "epoch": 0.1427798810167658, + "grad_norm": 0.4736854224697208, + "learning_rate": 1.8285714285714288e-05, + "loss": 1.0754, + "num_tokens": 269412990.0, + "step": 33 + }, + { + "epoch": 0.14710654407787993, + "grad_norm": 0.5644038888965046, + "learning_rate": 1.885714285714286e-05, + "loss": 1.0812, + "num_tokens": 277507256.0, + "step": 34 + }, + { + "epoch": 0.15143320713899405, + "grad_norm": 0.517430144442927, + "learning_rate": 1.942857142857143e-05, + "loss": 1.0518, + "num_tokens": 285825070.0, + "step": 35 + }, + { + "epoch": 0.15575987020010817, + "grad_norm": 0.4767328121960403, + "learning_rate": 2e-05, + "loss": 1.0746, + "num_tokens": 293998939.0, + "step": 36 + }, + { + "epoch": 0.16008653326122227, + "grad_norm": 0.6265822085481365, + "learning_rate": 1.9999964908096047e-05, + "loss": 1.0779, + "num_tokens": 302026611.0, + "step": 37 + }, + { + "epoch": 0.1644131963223364, + "grad_norm": 0.5421794033981302, + "learning_rate": 1.9999859632657835e-05, + "loss": 1.0567, + "num_tokens": 310311612.0, + "step": 38 + }, + { + "epoch": 0.1687398593834505, + "grad_norm": 0.5630990560279432, + "learning_rate": 1.9999684174506328e-05, + "loss": 1.079, + "num_tokens": 318541443.0, + "step": 39 + }, + { + "epoch": 0.17306652244456464, + "grad_norm": 0.48788951615244724, + "learning_rate": 1.999943853500978e-05, + "loss": 1.0547, + "num_tokens": 326739815.0, + "step": 40 + }, + { + "epoch": 0.17739318550567876, + "grad_norm": 0.5340588580376897, + "learning_rate": 1.9999122716083737e-05, + "loss": 1.0934, + "num_tokens": 334937951.0, + "step": 41 + }, + { + "epoch": 0.18171984856679285, + "grad_norm": 0.64525962255221, + "learning_rate": 1.9998736720191024e-05, + "loss": 1.0549, + "num_tokens": 342906928.0, + "step": 42 + }, + { + "epoch": 0.18604651162790697, + "grad_norm": 0.6322467483553018, + "learning_rate": 1.999828055034171e-05, + "loss": 1.0708, + "num_tokens": 351105939.0, + "step": 43 + }, + { + "epoch": 0.1903731746890211, + "grad_norm": 0.5286887426663199, + "learning_rate": 1.99977542100931e-05, + "loss": 1.0398, + "num_tokens": 359264834.0, + "step": 44 + }, + { + "epoch": 0.19469983775013522, + "grad_norm": 0.5550761990980296, + "learning_rate": 1.99971577035497e-05, + "loss": 1.0603, + "num_tokens": 367366715.0, + "step": 45 + }, + { + "epoch": 0.1990265008112493, + "grad_norm": 0.6881955496478498, + "learning_rate": 1.999649103536319e-05, + "loss": 1.05, + "num_tokens": 375621618.0, + "step": 46 + }, + { + "epoch": 0.20335316387236343, + "grad_norm": 0.5029731593462167, + "learning_rate": 1.9995754210732382e-05, + "loss": 1.0313, + "num_tokens": 383988397.0, + "step": 47 + }, + { + "epoch": 0.20767982693347756, + "grad_norm": 0.5868421371084077, + "learning_rate": 1.999494723540318e-05, + "loss": 1.0394, + "num_tokens": 392268195.0, + "step": 48 + }, + { + "epoch": 0.21200648999459168, + "grad_norm": 0.4194973171959154, + "learning_rate": 1.9994070115668543e-05, + "loss": 1.0573, + "num_tokens": 400505785.0, + "step": 49 + }, + { + "epoch": 0.21633315305570577, + "grad_norm": 0.6636892110494055, + "learning_rate": 1.9993122858368424e-05, + "loss": 1.0393, + "num_tokens": 408726600.0, + "step": 50 + }, + { + "epoch": 0.2206598161168199, + "grad_norm": 0.49726970914407065, + "learning_rate": 1.9992105470889727e-05, + "loss": 1.0386, + "num_tokens": 416846182.0, + "step": 51 + }, + { + "epoch": 0.22498647917793402, + "grad_norm": 0.46057751369498073, + "learning_rate": 1.9991017961166245e-05, + "loss": 1.0313, + "num_tokens": 425045657.0, + "step": 52 + }, + { + "epoch": 0.22931314223904814, + "grad_norm": 0.5270662555668391, + "learning_rate": 1.9989860337678596e-05, + "loss": 1.0201, + "num_tokens": 433167472.0, + "step": 53 + }, + { + "epoch": 0.23363980530016226, + "grad_norm": 0.39550370941134155, + "learning_rate": 1.998863260945416e-05, + "loss": 1.0342, + "num_tokens": 441300779.0, + "step": 54 + }, + { + "epoch": 0.23796646836127636, + "grad_norm": 0.4888851069343672, + "learning_rate": 1.998733478606701e-05, + "loss": 1.0691, + "num_tokens": 449442687.0, + "step": 55 + }, + { + "epoch": 0.24229313142239048, + "grad_norm": 0.39030633583317975, + "learning_rate": 1.998596687763783e-05, + "loss": 1.0456, + "num_tokens": 457634060.0, + "step": 56 + }, + { + "epoch": 0.2466197944835046, + "grad_norm": 0.6037448745855647, + "learning_rate": 1.998452889483385e-05, + "loss": 1.0366, + "num_tokens": 465796468.0, + "step": 57 + }, + { + "epoch": 0.2509464575446187, + "grad_norm": 0.6407808858535493, + "learning_rate": 1.9983020848868745e-05, + "loss": 1.0206, + "num_tokens": 473933547.0, + "step": 58 + }, + { + "epoch": 0.2552731206057328, + "grad_norm": 0.46606104818607985, + "learning_rate": 1.9981442751502562e-05, + "loss": 1.0496, + "num_tokens": 482199473.0, + "step": 59 + }, + { + "epoch": 0.25959978366684694, + "grad_norm": 0.6021787006302769, + "learning_rate": 1.9979794615041623e-05, + "loss": 1.0166, + "num_tokens": 490414620.0, + "step": 60 + }, + { + "epoch": 0.26392644672796106, + "grad_norm": 0.5266763718081924, + "learning_rate": 1.997807645233842e-05, + "loss": 1.0254, + "num_tokens": 498570024.0, + "step": 61 + }, + { + "epoch": 0.2682531097890752, + "grad_norm": 0.4983776079049174, + "learning_rate": 1.9976288276791537e-05, + "loss": 1.0173, + "num_tokens": 506624102.0, + "step": 62 + }, + { + "epoch": 0.2725797728501893, + "grad_norm": 0.5496598622125858, + "learning_rate": 1.9974430102345526e-05, + "loss": 1.0362, + "num_tokens": 514728880.0, + "step": 63 + }, + { + "epoch": 0.27690643591130343, + "grad_norm": 0.589170138343133, + "learning_rate": 1.9972501943490805e-05, + "loss": 1.0625, + "num_tokens": 522847133.0, + "step": 64 + }, + { + "epoch": 0.28123309897241755, + "grad_norm": 0.4407621194070745, + "learning_rate": 1.9970503815263543e-05, + "loss": 1.0225, + "num_tokens": 531177868.0, + "step": 65 + }, + { + "epoch": 0.2855597620335316, + "grad_norm": 0.6501450077847399, + "learning_rate": 1.9968435733245542e-05, + "loss": 1.0239, + "num_tokens": 539425133.0, + "step": 66 + }, + { + "epoch": 0.28988642509464574, + "grad_norm": 0.43216047155533227, + "learning_rate": 1.9966297713564123e-05, + "loss": 1.0144, + "num_tokens": 547581282.0, + "step": 67 + }, + { + "epoch": 0.29421308815575986, + "grad_norm": 0.5556684552496532, + "learning_rate": 1.9964089772891998e-05, + "loss": 1.0173, + "num_tokens": 555792752.0, + "step": 68 + }, + { + "epoch": 0.298539751216874, + "grad_norm": 0.7190736915743237, + "learning_rate": 1.9961811928447124e-05, + "loss": 1.0127, + "num_tokens": 564088940.0, + "step": 69 + }, + { + "epoch": 0.3028664142779881, + "grad_norm": 0.41044522554276425, + "learning_rate": 1.9959464197992592e-05, + "loss": 1.035, + "num_tokens": 572385295.0, + "step": 70 + }, + { + "epoch": 0.3071930773391022, + "grad_norm": 0.6675286198708803, + "learning_rate": 1.995704659983648e-05, + "loss": 1.0446, + "num_tokens": 580633107.0, + "step": 71 + }, + { + "epoch": 0.31151974040021635, + "grad_norm": 0.4328501896592201, + "learning_rate": 1.9954559152831705e-05, + "loss": 1.0149, + "num_tokens": 588798366.0, + "step": 72 + }, + { + "epoch": 0.31584640346133047, + "grad_norm": 0.5482919084513325, + "learning_rate": 1.995200187637587e-05, + "loss": 0.9969, + "num_tokens": 596892276.0, + "step": 73 + }, + { + "epoch": 0.32017306652244454, + "grad_norm": 0.507456750208431, + "learning_rate": 1.9949374790411134e-05, + "loss": 0.991, + "num_tokens": 605149242.0, + "step": 74 + }, + { + "epoch": 0.32449972958355866, + "grad_norm": 0.49632476049381336, + "learning_rate": 1.9946677915424045e-05, + "loss": 1.0204, + "num_tokens": 613362319.0, + "step": 75 + }, + { + "epoch": 0.3288263926446728, + "grad_norm": 0.4301450511003034, + "learning_rate": 1.994391127244537e-05, + "loss": 1.0056, + "num_tokens": 621521100.0, + "step": 76 + }, + { + "epoch": 0.3331530557057869, + "grad_norm": 0.48013026268841275, + "learning_rate": 1.994107488304995e-05, + "loss": 0.9868, + "num_tokens": 629745992.0, + "step": 77 + }, + { + "epoch": 0.337479718766901, + "grad_norm": 0.5983787594346031, + "learning_rate": 1.993816876935652e-05, + "loss": 1.0229, + "num_tokens": 637832836.0, + "step": 78 + }, + { + "epoch": 0.34180638182801515, + "grad_norm": 26.575512954760693, + "learning_rate": 1.9935192954027537e-05, + "loss": 1.0564, + "num_tokens": 645907409.0, + "step": 79 + }, + { + "epoch": 0.34613304488912927, + "grad_norm": 1.4541880315573934, + "learning_rate": 1.9932147460269007e-05, + "loss": 1.0222, + "num_tokens": 654167462.0, + "step": 80 + }, + { + "epoch": 0.3504597079502434, + "grad_norm": 0.5841765044403455, + "learning_rate": 1.9929032311830303e-05, + "loss": 1.0087, + "num_tokens": 662204064.0, + "step": 81 + }, + { + "epoch": 0.3547863710113575, + "grad_norm": 1.500712678274268, + "learning_rate": 1.9925847533003976e-05, + "loss": 1.0113, + "num_tokens": 670433365.0, + "step": 82 + }, + { + "epoch": 0.3591130340724716, + "grad_norm": 1.325926976275402, + "learning_rate": 1.9922593148625573e-05, + "loss": 1.0311, + "num_tokens": 678720994.0, + "step": 83 + }, + { + "epoch": 0.3634396971335857, + "grad_norm": 1.0543056547697374, + "learning_rate": 1.9919269184073435e-05, + "loss": 0.999, + "num_tokens": 686714379.0, + "step": 84 + }, + { + "epoch": 0.3677663601946998, + "grad_norm": 1.0529439053182181, + "learning_rate": 1.9915875665268508e-05, + "loss": 1.0291, + "num_tokens": 694695261.0, + "step": 85 + }, + { + "epoch": 0.37209302325581395, + "grad_norm": 0.9231954262268605, + "learning_rate": 1.9912412618674134e-05, + "loss": 1.0029, + "num_tokens": 702823623.0, + "step": 86 + }, + { + "epoch": 0.37641968631692807, + "grad_norm": 0.7975118177707352, + "learning_rate": 1.9908880071295844e-05, + "loss": 1.0073, + "num_tokens": 711011110.0, + "step": 87 + }, + { + "epoch": 0.3807463493780422, + "grad_norm": 0.9230504017826145, + "learning_rate": 1.990527805068115e-05, + "loss": 1.008, + "num_tokens": 719208127.0, + "step": 88 + }, + { + "epoch": 0.3850730124391563, + "grad_norm": 0.660664210099823, + "learning_rate": 1.9901606584919336e-05, + "loss": 1.0172, + "num_tokens": 727411464.0, + "step": 89 + }, + { + "epoch": 0.38939967550027044, + "grad_norm": 1.0875954707517952, + "learning_rate": 1.989786570264123e-05, + "loss": 1.0147, + "num_tokens": 735493608.0, + "step": 90 + }, + { + "epoch": 0.39372633856138456, + "grad_norm": 0.7907821011014159, + "learning_rate": 1.9894055433018977e-05, + "loss": 0.994, + "num_tokens": 743732759.0, + "step": 91 + }, + { + "epoch": 0.3980530016224986, + "grad_norm": 0.8910322978516826, + "learning_rate": 1.9890175805765834e-05, + "loss": 1.0202, + "num_tokens": 751865544.0, + "step": 92 + }, + { + "epoch": 0.40237966468361275, + "grad_norm": 0.7695500261748419, + "learning_rate": 1.9886226851135904e-05, + "loss": 1.0141, + "num_tokens": 760139894.0, + "step": 93 + }, + { + "epoch": 0.40670632774472687, + "grad_norm": 0.8994347554488774, + "learning_rate": 1.988220859992394e-05, + "loss": 1.0248, + "num_tokens": 768419002.0, + "step": 94 + }, + { + "epoch": 0.411032990805841, + "grad_norm": 0.6906293884515684, + "learning_rate": 1.987812108346506e-05, + "loss": 1.0187, + "num_tokens": 776591346.0, + "step": 95 + }, + { + "epoch": 0.4153596538669551, + "grad_norm": 0.7446677976909126, + "learning_rate": 1.9873964333634546e-05, + "loss": 0.9974, + "num_tokens": 784634577.0, + "step": 96 + }, + { + "epoch": 0.41968631692806924, + "grad_norm": 0.5955551932390857, + "learning_rate": 1.9869738382847567e-05, + "loss": 0.9849, + "num_tokens": 792742873.0, + "step": 97 + }, + { + "epoch": 0.42401297998918336, + "grad_norm": 0.6859328236547411, + "learning_rate": 1.9865443264058936e-05, + "loss": 0.9806, + "num_tokens": 801014620.0, + "step": 98 + }, + { + "epoch": 0.4283396430502975, + "grad_norm": 0.5248164568656007, + "learning_rate": 1.9861079010762852e-05, + "loss": 1.0073, + "num_tokens": 809152151.0, + "step": 99 + }, + { + "epoch": 0.43266630611141155, + "grad_norm": 0.7243377980961461, + "learning_rate": 1.9856645656992637e-05, + "loss": 0.9871, + "num_tokens": 817301143.0, + "step": 100 + }, + { + "epoch": 0.43699296917252567, + "grad_norm": 0.5059074807233143, + "learning_rate": 1.9852143237320475e-05, + "loss": 1.0167, + "num_tokens": 825647595.0, + "step": 101 + }, + { + "epoch": 0.4413196322336398, + "grad_norm": 0.7665037416386092, + "learning_rate": 1.9847571786857142e-05, + "loss": 0.9919, + "num_tokens": 833758794.0, + "step": 102 + }, + { + "epoch": 0.4456462952947539, + "grad_norm": 0.5081776963476131, + "learning_rate": 1.984293134125172e-05, + "loss": 1.0054, + "num_tokens": 842044960.0, + "step": 103 + }, + { + "epoch": 0.44997295835586804, + "grad_norm": 0.7155152862197922, + "learning_rate": 1.9838221936691347e-05, + "loss": 1.0144, + "num_tokens": 850345972.0, + "step": 104 + }, + { + "epoch": 0.45429962141698216, + "grad_norm": 0.5308617288908505, + "learning_rate": 1.9833443609900896e-05, + "loss": 0.9641, + "num_tokens": 858665135.0, + "step": 105 + }, + { + "epoch": 0.4586262844780963, + "grad_norm": 0.676182296039779, + "learning_rate": 1.9828596398142725e-05, + "loss": 0.9764, + "num_tokens": 866900205.0, + "step": 106 + }, + { + "epoch": 0.4629529475392104, + "grad_norm": 0.6719380880080895, + "learning_rate": 1.9823680339216363e-05, + "loss": 1.0044, + "num_tokens": 875168554.0, + "step": 107 + }, + { + "epoch": 0.4672796106003245, + "grad_norm": 0.6650874505944395, + "learning_rate": 1.9818695471458224e-05, + "loss": 0.9973, + "num_tokens": 883496687.0, + "step": 108 + }, + { + "epoch": 0.4716062736614386, + "grad_norm": 0.6952076536924146, + "learning_rate": 1.9813641833741308e-05, + "loss": 1.0065, + "num_tokens": 891733362.0, + "step": 109 + }, + { + "epoch": 0.4759329367225527, + "grad_norm": 0.5271501068526363, + "learning_rate": 1.9808519465474898e-05, + "loss": 1.0226, + "num_tokens": 899945294.0, + "step": 110 + }, + { + "epoch": 0.48025959978366684, + "grad_norm": 0.5668392230235835, + "learning_rate": 1.9803328406604252e-05, + "loss": 1.0086, + "num_tokens": 908023952.0, + "step": 111 + }, + { + "epoch": 0.48458626284478096, + "grad_norm": 0.5811252624940818, + "learning_rate": 1.979806869761029e-05, + "loss": 1.032, + "num_tokens": 916262845.0, + "step": 112 + }, + { + "epoch": 0.4889129259058951, + "grad_norm": 0.5310269926089376, + "learning_rate": 1.9792740379509274e-05, + "loss": 0.9863, + "num_tokens": 924557446.0, + "step": 113 + }, + { + "epoch": 0.4932395889670092, + "grad_norm": 0.5329155373278363, + "learning_rate": 1.9787343493852508e-05, + "loss": 0.9889, + "num_tokens": 932686279.0, + "step": 114 + }, + { + "epoch": 0.4975662520281233, + "grad_norm": 0.5014405141680862, + "learning_rate": 1.9781878082725982e-05, + "loss": 1.001, + "num_tokens": 940962503.0, + "step": 115 + }, + { + "epoch": 0.5018929150892374, + "grad_norm": 0.5242960997988234, + "learning_rate": 1.977634418875007e-05, + "loss": 0.9781, + "num_tokens": 949185131.0, + "step": 116 + }, + { + "epoch": 0.5062195781503516, + "grad_norm": 0.514927476240931, + "learning_rate": 1.9770741855079197e-05, + "loss": 0.992, + "num_tokens": 957358766.0, + "step": 117 + }, + { + "epoch": 0.5105462412114656, + "grad_norm": 0.5961640262436182, + "learning_rate": 1.976507112540148e-05, + "loss": 0.9657, + "num_tokens": 965609614.0, + "step": 118 + }, + { + "epoch": 0.5148729042725798, + "grad_norm": 0.37671367462811034, + "learning_rate": 1.9759332043938408e-05, + "loss": 0.9909, + "num_tokens": 973732746.0, + "step": 119 + }, + { + "epoch": 0.5191995673336939, + "grad_norm": 0.5709908896565365, + "learning_rate": 1.9753524655444495e-05, + "loss": 0.9943, + "num_tokens": 981895080.0, + "step": 120 + }, + { + "epoch": 0.5235262303948081, + "grad_norm": 0.5393322671236704, + "learning_rate": 1.974764900520692e-05, + "loss": 0.9807, + "num_tokens": 990134999.0, + "step": 121 + }, + { + "epoch": 0.5278528934559221, + "grad_norm": 0.4782617423698582, + "learning_rate": 1.9741705139045183e-05, + "loss": 0.9694, + "num_tokens": 998352958.0, + "step": 122 + }, + { + "epoch": 0.5321795565170362, + "grad_norm": 0.45441671351177826, + "learning_rate": 1.9735693103310747e-05, + "loss": 0.9886, + "num_tokens": 1006510181.0, + "step": 123 + }, + { + "epoch": 0.5365062195781504, + "grad_norm": 0.5465803757865236, + "learning_rate": 1.9729612944886677e-05, + "loss": 0.9805, + "num_tokens": 1014696620.0, + "step": 124 + }, + { + "epoch": 0.5408328826392644, + "grad_norm": 0.5776084742066251, + "learning_rate": 1.9723464711187267e-05, + "loss": 0.9961, + "num_tokens": 1022837328.0, + "step": 125 + }, + { + "epoch": 0.5451595457003786, + "grad_norm": 0.43888568948045675, + "learning_rate": 1.9717248450157682e-05, + "loss": 1.0001, + "num_tokens": 1031063641.0, + "step": 126 + }, + { + "epoch": 0.5494862087614927, + "grad_norm": 0.6288688904629286, + "learning_rate": 1.9710964210273567e-05, + "loss": 0.9876, + "num_tokens": 1039147546.0, + "step": 127 + }, + { + "epoch": 0.5538128718226069, + "grad_norm": 0.5148111749819088, + "learning_rate": 1.9704612040540698e-05, + "loss": 0.9863, + "num_tokens": 1047288118.0, + "step": 128 + }, + { + "epoch": 0.5581395348837209, + "grad_norm": 0.5079798494743828, + "learning_rate": 1.969819199049456e-05, + "loss": 0.9965, + "num_tokens": 1055150222.0, + "step": 129 + }, + { + "epoch": 0.5624661979448351, + "grad_norm": 0.6459772466570576, + "learning_rate": 1.9691704110199997e-05, + "loss": 1.0051, + "num_tokens": 1063527758.0, + "step": 130 + }, + { + "epoch": 0.5667928610059492, + "grad_norm": 0.3616574518249611, + "learning_rate": 1.9685148450250802e-05, + "loss": 0.994, + "num_tokens": 1071865630.0, + "step": 131 + }, + { + "epoch": 0.5711195240670632, + "grad_norm": 0.516177320289666, + "learning_rate": 1.9678525061769332e-05, + "loss": 0.995, + "num_tokens": 1080093780.0, + "step": 132 + }, + { + "epoch": 0.5754461871281774, + "grad_norm": 0.5155532270149339, + "learning_rate": 1.96718339964061e-05, + "loss": 0.9524, + "num_tokens": 1088310915.0, + "step": 133 + }, + { + "epoch": 0.5797728501892915, + "grad_norm": 0.6016781959009333, + "learning_rate": 1.9665075306339373e-05, + "loss": 1.0022, + "num_tokens": 1096512949.0, + "step": 134 + }, + { + "epoch": 0.5840995132504057, + "grad_norm": 0.3830738893089929, + "learning_rate": 1.9658249044274773e-05, + "loss": 0.9928, + "num_tokens": 1104653577.0, + "step": 135 + }, + { + "epoch": 0.5884261763115197, + "grad_norm": 5.211589683449916, + "learning_rate": 1.965135526344487e-05, + "loss": 0.9846, + "num_tokens": 1112704677.0, + "step": 136 + }, + { + "epoch": 0.5927528393726339, + "grad_norm": 1.173818152854464, + "learning_rate": 1.964439401760875e-05, + "loss": 0.9863, + "num_tokens": 1120871489.0, + "step": 137 + }, + { + "epoch": 0.597079502433748, + "grad_norm": 0.5590208361578028, + "learning_rate": 1.9637365361051602e-05, + "loss": 0.9908, + "num_tokens": 1129071346.0, + "step": 138 + }, + { + "epoch": 0.601406165494862, + "grad_norm": 0.8868179793868765, + "learning_rate": 1.9630269348584303e-05, + "loss": 1.0092, + "num_tokens": 1137112125.0, + "step": 139 + }, + { + "epoch": 0.6057328285559762, + "grad_norm": 0.7902134012530393, + "learning_rate": 1.9623106035542988e-05, + "loss": 1.0034, + "num_tokens": 1145154797.0, + "step": 140 + }, + { + "epoch": 0.6100594916170903, + "grad_norm": 0.6298334469626393, + "learning_rate": 1.9615875477788607e-05, + "loss": 0.9912, + "num_tokens": 1153287437.0, + "step": 141 + }, + { + "epoch": 0.6143861546782045, + "grad_norm": 0.6474737648673636, + "learning_rate": 1.9608577731706502e-05, + "loss": 0.9627, + "num_tokens": 1161393271.0, + "step": 142 + }, + { + "epoch": 0.6187128177393185, + "grad_norm": 0.6224814543201583, + "learning_rate": 1.9601212854205965e-05, + "loss": 0.9823, + "num_tokens": 1169669678.0, + "step": 143 + }, + { + "epoch": 0.6230394808004327, + "grad_norm": 0.565179608540916, + "learning_rate": 1.959378090271979e-05, + "loss": 0.9714, + "num_tokens": 1177849833.0, + "step": 144 + }, + { + "epoch": 0.6273661438615468, + "grad_norm": 0.7629800032025918, + "learning_rate": 1.9586281935203823e-05, + "loss": 0.99, + "num_tokens": 1186030845.0, + "step": 145 + }, + { + "epoch": 0.6316928069226609, + "grad_norm": 0.6016888565376319, + "learning_rate": 1.9578716010136524e-05, + "loss": 0.9875, + "num_tokens": 1194180535.0, + "step": 146 + }, + { + "epoch": 0.636019469983775, + "grad_norm": 0.5855131134683734, + "learning_rate": 1.9571083186518495e-05, + "loss": 0.9969, + "num_tokens": 1202349525.0, + "step": 147 + }, + { + "epoch": 0.6403461330448891, + "grad_norm": 0.6110006455597233, + "learning_rate": 1.956338352387203e-05, + "loss": 0.9934, + "num_tokens": 1210452873.0, + "step": 148 + }, + { + "epoch": 0.6446727961060033, + "grad_norm": 0.4623062335027479, + "learning_rate": 1.955561708224064e-05, + "loss": 0.9696, + "num_tokens": 1218704252.0, + "step": 149 + }, + { + "epoch": 0.6489994591671173, + "grad_norm": 0.5626496069977094, + "learning_rate": 1.9547783922188605e-05, + "loss": 0.9779, + "num_tokens": 1226893769.0, + "step": 150 + }, + { + "epoch": 0.6533261222282315, + "grad_norm": 0.5345465422920194, + "learning_rate": 1.953988410480047e-05, + "loss": 0.9828, + "num_tokens": 1235209341.0, + "step": 151 + }, + { + "epoch": 0.6576527852893456, + "grad_norm": 8.764088138276044, + "learning_rate": 1.9531917691680605e-05, + "loss": 0.9842, + "num_tokens": 1243563275.0, + "step": 152 + }, + { + "epoch": 0.6619794483504597, + "grad_norm": 0.8366248889512555, + "learning_rate": 1.95238847449527e-05, + "loss": 0.9959, + "num_tokens": 1251780215.0, + "step": 153 + }, + { + "epoch": 0.6663061114115738, + "grad_norm": 0.65022275007566, + "learning_rate": 1.9515785327259283e-05, + "loss": 0.9742, + "num_tokens": 1259817249.0, + "step": 154 + }, + { + "epoch": 0.670632774472688, + "grad_norm": 0.5856608330177381, + "learning_rate": 1.950761950176125e-05, + "loss": 0.968, + "num_tokens": 1268123631.0, + "step": 155 + }, + { + "epoch": 0.674959437533802, + "grad_norm": 0.5815004857711259, + "learning_rate": 1.949938733213733e-05, + "loss": 0.9863, + "num_tokens": 1276298604.0, + "step": 156 + }, + { + "epoch": 0.6792861005949161, + "grad_norm": 0.5173351166894457, + "learning_rate": 1.9491088882583653e-05, + "loss": 0.9771, + "num_tokens": 1284421448.0, + "step": 157 + }, + { + "epoch": 0.6836127636560303, + "grad_norm": 0.6198867246439916, + "learning_rate": 1.948272421781319e-05, + "loss": 0.9676, + "num_tokens": 1292627003.0, + "step": 158 + }, + { + "epoch": 0.6879394267171444, + "grad_norm": 0.5000354785807577, + "learning_rate": 1.9474293403055273e-05, + "loss": 0.99, + "num_tokens": 1300823442.0, + "step": 159 + }, + { + "epoch": 0.6922660897782585, + "grad_norm": 0.5767004692781212, + "learning_rate": 1.9465796504055095e-05, + "loss": 0.9866, + "num_tokens": 1308893801.0, + "step": 160 + }, + { + "epoch": 0.6965927528393726, + "grad_norm": 0.48248590328029517, + "learning_rate": 1.9457233587073177e-05, + "loss": 0.9675, + "num_tokens": 1317094649.0, + "step": 161 + }, + { + "epoch": 0.7009194159004868, + "grad_norm": 0.508005752307088, + "learning_rate": 1.9448604718884868e-05, + "loss": 0.9717, + "num_tokens": 1325279137.0, + "step": 162 + }, + { + "epoch": 0.7052460789616009, + "grad_norm": 0.4648308346506714, + "learning_rate": 1.9439909966779816e-05, + "loss": 0.9708, + "num_tokens": 1333616465.0, + "step": 163 + }, + { + "epoch": 0.709572742022715, + "grad_norm": 0.8715498782209189, + "learning_rate": 1.943114939856144e-05, + "loss": 0.9465, + "num_tokens": 1341875716.0, + "step": 164 + }, + { + "epoch": 0.7138994050838291, + "grad_norm": 0.5235352618915897, + "learning_rate": 1.942232308254642e-05, + "loss": 0.982, + "num_tokens": 1350163497.0, + "step": 165 + }, + { + "epoch": 0.7182260681449432, + "grad_norm": 0.5736863060766292, + "learning_rate": 1.941343108756413e-05, + "loss": 0.9668, + "num_tokens": 1358471450.0, + "step": 166 + }, + { + "epoch": 0.7225527312060573, + "grad_norm": 0.5807069471784715, + "learning_rate": 1.9404473482956143e-05, + "loss": 0.9774, + "num_tokens": 1366767300.0, + "step": 167 + }, + { + "epoch": 0.7268793942671714, + "grad_norm": 0.5350146728521313, + "learning_rate": 1.9395450338575655e-05, + "loss": 0.957, + "num_tokens": 1374860079.0, + "step": 168 + }, + { + "epoch": 0.7312060573282856, + "grad_norm": 0.6298172838153421, + "learning_rate": 1.938636172478695e-05, + "loss": 0.9552, + "num_tokens": 1383032035.0, + "step": 169 + }, + { + "epoch": 0.7355327203893997, + "grad_norm": 0.4724906362910472, + "learning_rate": 1.937720771246488e-05, + "loss": 0.9661, + "num_tokens": 1391382107.0, + "step": 170 + }, + { + "epoch": 0.7398593834505138, + "grad_norm": 0.40655819017325623, + "learning_rate": 1.9367988372994264e-05, + "loss": 0.9888, + "num_tokens": 1399520998.0, + "step": 171 + }, + { + "epoch": 0.7441860465116279, + "grad_norm": 0.5296123044360186, + "learning_rate": 1.9358703778269362e-05, + "loss": 0.958, + "num_tokens": 1407729420.0, + "step": 172 + }, + { + "epoch": 0.7485127095727421, + "grad_norm": 0.5881566511232578, + "learning_rate": 1.934935400069331e-05, + "loss": 0.9844, + "num_tokens": 1415936565.0, + "step": 173 + }, + { + "epoch": 0.7528393726338561, + "grad_norm": 0.3423356730054581, + "learning_rate": 1.933993911317755e-05, + "loss": 0.9531, + "num_tokens": 1424051233.0, + "step": 174 + }, + { + "epoch": 0.7571660356949702, + "grad_norm": 0.6217907701320499, + "learning_rate": 1.933045918914127e-05, + "loss": 0.9774, + "num_tokens": 1432218028.0, + "step": 175 + }, + { + "epoch": 0.7614926987560844, + "grad_norm": 0.439174507769625, + "learning_rate": 1.932091430251082e-05, + "loss": 0.9951, + "num_tokens": 1440529142.0, + "step": 176 + }, + { + "epoch": 0.7658193618171985, + "grad_norm": 5.018307710610472, + "learning_rate": 1.931130452771914e-05, + "loss": 0.9831, + "num_tokens": 1448651650.0, + "step": 177 + }, + { + "epoch": 0.7701460248783126, + "grad_norm": 1.036148648364116, + "learning_rate": 1.930162993970519e-05, + "loss": 0.9605, + "num_tokens": 1456909194.0, + "step": 178 + }, + { + "epoch": 0.7744726879394267, + "grad_norm": 0.5935362801766706, + "learning_rate": 1.9291890613913353e-05, + "loss": 0.9707, + "num_tokens": 1465040408.0, + "step": 179 + }, + { + "epoch": 0.7787993510005409, + "grad_norm": 0.7778583519012636, + "learning_rate": 1.9282086626292835e-05, + "loss": 0.9568, + "num_tokens": 1473270226.0, + "step": 180 + }, + { + "epoch": 0.7831260140616549, + "grad_norm": 0.801578696787655, + "learning_rate": 1.9272218053297113e-05, + "loss": 0.9821, + "num_tokens": 1481459987.0, + "step": 181 + }, + { + "epoch": 0.7874526771227691, + "grad_norm": 0.6757381881745876, + "learning_rate": 1.9262284971883293e-05, + "loss": 0.9626, + "num_tokens": 1489496080.0, + "step": 182 + }, + { + "epoch": 0.7917793401838832, + "grad_norm": 0.7517854861798496, + "learning_rate": 1.925228745951155e-05, + "loss": 0.9919, + "num_tokens": 1497776422.0, + "step": 183 + }, + { + "epoch": 0.7961060032449973, + "grad_norm": 0.6808277302065322, + "learning_rate": 1.9242225594144487e-05, + "loss": 0.9775, + "num_tokens": 1505883048.0, + "step": 184 + }, + { + "epoch": 0.8004326663061114, + "grad_norm": 0.6350339248320397, + "learning_rate": 1.9232099454246547e-05, + "loss": 0.9581, + "num_tokens": 1514015227.0, + "step": 185 + }, + { + "epoch": 0.8047593293672255, + "grad_norm": 0.5934537904581213, + "learning_rate": 1.922190911878341e-05, + "loss": 0.9652, + "num_tokens": 1521975633.0, + "step": 186 + }, + { + "epoch": 0.8090859924283397, + "grad_norm": 0.5157410704143146, + "learning_rate": 1.9211654667221356e-05, + "loss": 0.9695, + "num_tokens": 1530141608.0, + "step": 187 + }, + { + "epoch": 0.8134126554894537, + "grad_norm": 0.6963524421881355, + "learning_rate": 1.9201336179526662e-05, + "loss": 0.971, + "num_tokens": 1538284060.0, + "step": 188 + }, + { + "epoch": 0.8177393185505679, + "grad_norm": 0.48945767329559603, + "learning_rate": 1.9190953736164962e-05, + "loss": 0.9854, + "num_tokens": 1546570178.0, + "step": 189 + }, + { + "epoch": 0.822065981611682, + "grad_norm": 0.6298060617221776, + "learning_rate": 1.918050741810064e-05, + "loss": 0.9595, + "num_tokens": 1554819493.0, + "step": 190 + }, + { + "epoch": 0.826392644672796, + "grad_norm": 0.5923902354962175, + "learning_rate": 1.916999730679618e-05, + "loss": 0.9436, + "num_tokens": 1562945757.0, + "step": 191 + }, + { + "epoch": 0.8307193077339102, + "grad_norm": 0.40079556271463335, + "learning_rate": 1.9159423484211542e-05, + "loss": 0.9744, + "num_tokens": 1571028714.0, + "step": 192 + }, + { + "epoch": 0.8350459707950243, + "grad_norm": 0.7154075197188315, + "learning_rate": 1.9148786032803516e-05, + "loss": 0.9773, + "num_tokens": 1578905113.0, + "step": 193 + }, + { + "epoch": 0.8393726338561385, + "grad_norm": 0.4902553566691951, + "learning_rate": 1.9138085035525088e-05, + "loss": 0.9661, + "num_tokens": 1587105154.0, + "step": 194 + }, + { + "epoch": 0.8436992969172525, + "grad_norm": 0.7781965230914352, + "learning_rate": 1.912732057582479e-05, + "loss": 0.9501, + "num_tokens": 1595124375.0, + "step": 195 + }, + { + "epoch": 0.8480259599783667, + "grad_norm": 0.5813304419716897, + "learning_rate": 1.9116492737646025e-05, + "loss": 0.9595, + "num_tokens": 1603125013.0, + "step": 196 + }, + { + "epoch": 0.8523526230394808, + "grad_norm": 0.6753700971945712, + "learning_rate": 1.9105601605426464e-05, + "loss": 0.9436, + "num_tokens": 1611231596.0, + "step": 197 + }, + { + "epoch": 0.856679286100595, + "grad_norm": 0.6245159188540251, + "learning_rate": 1.909464726409734e-05, + "loss": 0.9534, + "num_tokens": 1619468595.0, + "step": 198 + }, + { + "epoch": 0.861005949161709, + "grad_norm": 3.968292033372073, + "learning_rate": 1.9083629799082806e-05, + "loss": 1.0366, + "num_tokens": 1627687009.0, + "step": 199 + }, + { + "epoch": 0.8653326122228231, + "grad_norm": 1.0824031283568236, + "learning_rate": 1.9072549296299272e-05, + "loss": 0.9639, + "num_tokens": 1635741823.0, + "step": 200 + }, + { + "epoch": 0.8696592752839373, + "grad_norm": 0.5941762165219101, + "learning_rate": 1.9061405842154716e-05, + "loss": 0.9921, + "num_tokens": 1643822955.0, + "step": 201 + }, + { + "epoch": 0.8739859383450513, + "grad_norm": 0.9442326347332348, + "learning_rate": 1.9050199523548042e-05, + "loss": 0.9544, + "num_tokens": 1652169611.0, + "step": 202 + }, + { + "epoch": 0.8783126014061655, + "grad_norm": 0.9101744792913391, + "learning_rate": 1.9038930427868367e-05, + "loss": 0.9984, + "num_tokens": 1660273693.0, + "step": 203 + }, + { + "epoch": 0.8826392644672796, + "grad_norm": 0.6722336561034881, + "learning_rate": 1.9027598642994357e-05, + "loss": 0.9795, + "num_tokens": 1668328193.0, + "step": 204 + }, + { + "epoch": 0.8869659275283938, + "grad_norm": 0.6998798459455463, + "learning_rate": 1.901620425729356e-05, + "loss": 0.9613, + "num_tokens": 1676550574.0, + "step": 205 + }, + { + "epoch": 0.8912925905895078, + "grad_norm": 0.7234571791964981, + "learning_rate": 1.900474735962168e-05, + "loss": 0.9685, + "num_tokens": 1684819316.0, + "step": 206 + }, + { + "epoch": 0.895619253650622, + "grad_norm": 0.6119990230186566, + "learning_rate": 1.89932280393219e-05, + "loss": 0.9748, + "num_tokens": 1693142528.0, + "step": 207 + }, + { + "epoch": 0.8999459167117361, + "grad_norm": 0.623729491234639, + "learning_rate": 1.8981646386224205e-05, + "loss": 0.9532, + "num_tokens": 1701382334.0, + "step": 208 + }, + { + "epoch": 0.9042725797728501, + "grad_norm": 0.6283265364320373, + "learning_rate": 1.8970002490644643e-05, + "loss": 0.9744, + "num_tokens": 1709452029.0, + "step": 209 + }, + { + "epoch": 0.9085992428339643, + "grad_norm": 0.5784972319269428, + "learning_rate": 1.8958296443384655e-05, + "loss": 0.9714, + "num_tokens": 1717633966.0, + "step": 210 + }, + { + "epoch": 0.9129259058950784, + "grad_norm": 0.5762173383368797, + "learning_rate": 1.8946528335730344e-05, + "loss": 0.9558, + "num_tokens": 1725809798.0, + "step": 211 + }, + { + "epoch": 0.9172525689561926, + "grad_norm": 0.5188011012079045, + "learning_rate": 1.8934698259451784e-05, + "loss": 0.932, + "num_tokens": 1733760440.0, + "step": 212 + }, + { + "epoch": 0.9215792320173066, + "grad_norm": 0.616836368277132, + "learning_rate": 1.8922806306802283e-05, + "loss": 0.9589, + "num_tokens": 1741769950.0, + "step": 213 + }, + { + "epoch": 0.9259058950784208, + "grad_norm": 0.45595945221870204, + "learning_rate": 1.891085257051768e-05, + "loss": 0.9575, + "num_tokens": 1749934288.0, + "step": 214 + }, + { + "epoch": 0.9302325581395349, + "grad_norm": 0.5570921341366776, + "learning_rate": 1.8898837143815604e-05, + "loss": 0.9895, + "num_tokens": 1758259441.0, + "step": 215 + }, + { + "epoch": 0.934559221200649, + "grad_norm": 0.4417768988783611, + "learning_rate": 1.8886760120394774e-05, + "loss": 0.9461, + "num_tokens": 1766468586.0, + "step": 216 + }, + { + "epoch": 0.9388858842617631, + "grad_norm": 0.5440442593054226, + "learning_rate": 1.8874621594434242e-05, + "loss": 0.9711, + "num_tokens": 1774646798.0, + "step": 217 + }, + { + "epoch": 0.9432125473228772, + "grad_norm": 0.4305725930525812, + "learning_rate": 1.8862421660592673e-05, + "loss": 0.9606, + "num_tokens": 1782926524.0, + "step": 218 + }, + { + "epoch": 0.9475392103839914, + "grad_norm": 0.5070054432980237, + "learning_rate": 1.8850160414007595e-05, + "loss": 0.9857, + "num_tokens": 1791004823.0, + "step": 219 + }, + { + "epoch": 0.9518658734451054, + "grad_norm": 0.49783160996596737, + "learning_rate": 1.883783795029468e-05, + "loss": 0.9705, + "num_tokens": 1799153931.0, + "step": 220 + }, + { + "epoch": 0.9561925365062196, + "grad_norm": 0.4178709401695967, + "learning_rate": 1.8825454365546974e-05, + "loss": 0.9493, + "num_tokens": 1807314631.0, + "step": 221 + }, + { + "epoch": 0.9605191995673337, + "grad_norm": 0.54579102780289, + "learning_rate": 1.8813009756334156e-05, + "loss": 0.9732, + "num_tokens": 1815508610.0, + "step": 222 + }, + { + "epoch": 0.9648458626284478, + "grad_norm": 0.5531609776420421, + "learning_rate": 1.8800504219701788e-05, + "loss": 0.9591, + "num_tokens": 1823847575.0, + "step": 223 + }, + { + "epoch": 0.9691725256895619, + "grad_norm": 0.4474460460545339, + "learning_rate": 1.8787937853170563e-05, + "loss": 0.9552, + "num_tokens": 1832073535.0, + "step": 224 + }, + { + "epoch": 0.9734991887506761, + "grad_norm": 0.5158352231097937, + "learning_rate": 1.8775310754735518e-05, + "loss": 0.9614, + "num_tokens": 1840367653.0, + "step": 225 + }, + { + "epoch": 0.9778258518117902, + "grad_norm": 0.45041507287886795, + "learning_rate": 1.8762623022865317e-05, + "loss": 0.9584, + "num_tokens": 1848435257.0, + "step": 226 + }, + { + "epoch": 0.9821525148729042, + "grad_norm": 0.46329345840375313, + "learning_rate": 1.874987475650144e-05, + "loss": 0.9434, + "num_tokens": 1856515858.0, + "step": 227 + }, + { + "epoch": 0.9864791779340184, + "grad_norm": 0.4705252177689208, + "learning_rate": 1.873706605505742e-05, + "loss": 0.9558, + "num_tokens": 1864752312.0, + "step": 228 + }, + { + "epoch": 0.9908058409951325, + "grad_norm": 0.4601714494448067, + "learning_rate": 1.8724197018418092e-05, + "loss": 0.9692, + "num_tokens": 1873006067.0, + "step": 229 + }, + { + "epoch": 0.9951325040562466, + "grad_norm": 0.6185847242049543, + "learning_rate": 1.8711267746938787e-05, + "loss": 0.9695, + "num_tokens": 1881215073.0, + "step": 230 + }, + { + "epoch": 0.9994591671173607, + "grad_norm": 0.4101842265307164, + "learning_rate": 1.869827834144456e-05, + "loss": 0.9636, + "num_tokens": 1889561273.0, + "step": 231 + }, + { + "epoch": 1.0, + "grad_norm": 0.7040526526338052, + "learning_rate": 1.8685228903229408e-05, + "loss": 0.9194, + "num_tokens": 1890568792.0, + "step": 232 + }, + { + "epoch": 1.004326663061114, + "grad_norm": 0.6626410793255663, + "learning_rate": 1.8672119534055465e-05, + "loss": 0.948, + "num_tokens": 1898861182.0, + "step": 233 + }, + { + "epoch": 1.0086533261222281, + "grad_norm": 0.5985974181574858, + "learning_rate": 1.8658950336152227e-05, + "loss": 0.9591, + "num_tokens": 1907114575.0, + "step": 234 + }, + { + "epoch": 1.0129799891833424, + "grad_norm": 0.5608529698001458, + "learning_rate": 1.864572141221575e-05, + "loss": 0.949, + "num_tokens": 1915123299.0, + "step": 235 + }, + { + "epoch": 1.0173066522444565, + "grad_norm": 0.5896508033160583, + "learning_rate": 1.8632432865407835e-05, + "loss": 0.9361, + "num_tokens": 1923293463.0, + "step": 236 + }, + { + "epoch": 1.0216333153055706, + "grad_norm": 0.6247724053913459, + "learning_rate": 1.861908479935524e-05, + "loss": 0.9466, + "num_tokens": 1931345202.0, + "step": 237 + }, + { + "epoch": 1.0259599783666846, + "grad_norm": 0.4242474671709629, + "learning_rate": 1.8605677318148872e-05, + "loss": 0.9425, + "num_tokens": 1939503734.0, + "step": 238 + }, + { + "epoch": 1.030286641427799, + "grad_norm": 0.6481719380451512, + "learning_rate": 1.859221052634295e-05, + "loss": 0.9595, + "num_tokens": 1947562008.0, + "step": 239 + }, + { + "epoch": 1.034613304488913, + "grad_norm": 0.5785038674585137, + "learning_rate": 1.8578684528954232e-05, + "loss": 0.9469, + "num_tokens": 1955830446.0, + "step": 240 + }, + { + "epoch": 1.038939967550027, + "grad_norm": 0.5182893960147562, + "learning_rate": 1.8565099431461158e-05, + "loss": 0.9289, + "num_tokens": 1964022793.0, + "step": 241 + }, + { + "epoch": 1.043266630611141, + "grad_norm": 0.4663310434959851, + "learning_rate": 1.8551455339803053e-05, + "loss": 0.9625, + "num_tokens": 1972176095.0, + "step": 242 + }, + { + "epoch": 1.0475932936722552, + "grad_norm": 2.4019432307703217, + "learning_rate": 1.8537752360379277e-05, + "loss": 0.9486, + "num_tokens": 1980479570.0, + "step": 243 + }, + { + "epoch": 1.0519199567333695, + "grad_norm": 0.8544175463447066, + "learning_rate": 1.852399060004842e-05, + "loss": 0.9399, + "num_tokens": 1988590168.0, + "step": 244 + }, + { + "epoch": 1.0562466197944835, + "grad_norm": 0.5179045655090389, + "learning_rate": 1.8510170166127453e-05, + "loss": 0.9397, + "num_tokens": 1996900398.0, + "step": 245 + }, + { + "epoch": 1.0605732828555976, + "grad_norm": 0.6622106221311604, + "learning_rate": 1.8496291166390898e-05, + "loss": 0.9556, + "num_tokens": 2005170686.0, + "step": 246 + }, + { + "epoch": 1.0648999459167117, + "grad_norm": 0.6757903920362329, + "learning_rate": 1.848235370906998e-05, + "loss": 0.922, + "num_tokens": 2013474129.0, + "step": 247 + }, + { + "epoch": 1.069226608977826, + "grad_norm": 0.5216596091763227, + "learning_rate": 1.8468357902851788e-05, + "loss": 0.9457, + "num_tokens": 2021541879.0, + "step": 248 + }, + { + "epoch": 1.07355327203894, + "grad_norm": 0.6223658854192845, + "learning_rate": 1.845430385687844e-05, + "loss": 0.9272, + "num_tokens": 2029719336.0, + "step": 249 + }, + { + "epoch": 1.077879935100054, + "grad_norm": 0.4535927621134612, + "learning_rate": 1.84401916807462e-05, + "loss": 0.9424, + "num_tokens": 2037903029.0, + "step": 250 + }, + { + "epoch": 1.0822065981611682, + "grad_norm": 0.7997470320978654, + "learning_rate": 1.8426021484504655e-05, + "loss": 0.9424, + "num_tokens": 2046073146.0, + "step": 251 + }, + { + "epoch": 1.0865332612222822, + "grad_norm": 0.5126221066639043, + "learning_rate": 1.8411793378655847e-05, + "loss": 0.942, + "num_tokens": 2054348442.0, + "step": 252 + }, + { + "epoch": 1.0908599242833965, + "grad_norm": 0.7938516302653984, + "learning_rate": 1.83975074741534e-05, + "loss": 0.9399, + "num_tokens": 2062371725.0, + "step": 253 + }, + { + "epoch": 1.0951865873445106, + "grad_norm": 0.5313132219013962, + "learning_rate": 1.8383163882401664e-05, + "loss": 0.9407, + "num_tokens": 2070611501.0, + "step": 254 + }, + { + "epoch": 1.0995132504056246, + "grad_norm": 0.6728801915369794, + "learning_rate": 1.836876271525485e-05, + "loss": 0.9465, + "num_tokens": 2078684967.0, + "step": 255 + }, + { + "epoch": 1.1038399134667387, + "grad_norm": 0.49575776162838636, + "learning_rate": 1.8354304085016157e-05, + "loss": 0.9262, + "num_tokens": 2086960480.0, + "step": 256 + }, + { + "epoch": 1.1081665765278528, + "grad_norm": 0.6120552108025855, + "learning_rate": 1.8339788104436886e-05, + "loss": 0.9443, + "num_tokens": 2095042239.0, + "step": 257 + }, + { + "epoch": 1.112493239588967, + "grad_norm": 0.5014185175162884, + "learning_rate": 1.8325214886715567e-05, + "loss": 0.9375, + "num_tokens": 2103025535.0, + "step": 258 + }, + { + "epoch": 1.1168199026500811, + "grad_norm": 0.47896338127837235, + "learning_rate": 1.8310584545497075e-05, + "loss": 0.9457, + "num_tokens": 2111120140.0, + "step": 259 + }, + { + "epoch": 1.1211465657111952, + "grad_norm": 0.4297415409601369, + "learning_rate": 1.829589719487176e-05, + "loss": 0.9214, + "num_tokens": 2119414579.0, + "step": 260 + }, + { + "epoch": 1.1254732287723093, + "grad_norm": 0.48279932645064866, + "learning_rate": 1.8281152949374527e-05, + "loss": 0.9113, + "num_tokens": 2127479341.0, + "step": 261 + }, + { + "epoch": 1.1297998918334236, + "grad_norm": 0.384654569118448, + "learning_rate": 1.8266351923983967e-05, + "loss": 0.9356, + "num_tokens": 2135457476.0, + "step": 262 + }, + { + "epoch": 1.1341265548945376, + "grad_norm": 0.5350794948575581, + "learning_rate": 1.8251494234121445e-05, + "loss": 0.9249, + "num_tokens": 2143716350.0, + "step": 263 + }, + { + "epoch": 1.1384532179556517, + "grad_norm": 0.6083072007426356, + "learning_rate": 1.823657999565021e-05, + "loss": 0.9537, + "num_tokens": 2152027890.0, + "step": 264 + }, + { + "epoch": 1.1427798810167658, + "grad_norm": 0.40855364181266673, + "learning_rate": 1.8221609324874503e-05, + "loss": 0.9149, + "num_tokens": 2160186039.0, + "step": 265 + }, + { + "epoch": 1.14710654407788, + "grad_norm": 0.4365335649133271, + "learning_rate": 1.82065823385386e-05, + "loss": 0.9691, + "num_tokens": 2168446273.0, + "step": 266 + }, + { + "epoch": 1.151433207138994, + "grad_norm": 0.416231817579171, + "learning_rate": 1.819149915382598e-05, + "loss": 0.9624, + "num_tokens": 2176451082.0, + "step": 267 + }, + { + "epoch": 1.1557598702001082, + "grad_norm": 0.5827166915685855, + "learning_rate": 1.8176359888358332e-05, + "loss": 0.9449, + "num_tokens": 2184668485.0, + "step": 268 + }, + { + "epoch": 1.1600865332612222, + "grad_norm": 0.5389094717897734, + "learning_rate": 1.8161164660194697e-05, + "loss": 0.9442, + "num_tokens": 2192927865.0, + "step": 269 + }, + { + "epoch": 1.1644131963223363, + "grad_norm": 0.41906435961564176, + "learning_rate": 1.814591358783052e-05, + "loss": 0.9254, + "num_tokens": 2200965797.0, + "step": 270 + }, + { + "epoch": 1.1687398593834506, + "grad_norm": 0.4698013821548138, + "learning_rate": 1.813060679019672e-05, + "loss": 0.9346, + "num_tokens": 2209198752.0, + "step": 271 + }, + { + "epoch": 1.1730665224445647, + "grad_norm": 0.4183058129628398, + "learning_rate": 1.811524438665878e-05, + "loss": 0.9169, + "num_tokens": 2217390535.0, + "step": 272 + }, + { + "epoch": 1.1773931855056787, + "grad_norm": 0.5682861942844173, + "learning_rate": 1.809982649701581e-05, + "loss": 0.9391, + "num_tokens": 2225526040.0, + "step": 273 + }, + { + "epoch": 1.1817198485667928, + "grad_norm": 0.5074575323794233, + "learning_rate": 1.808435324149961e-05, + "loss": 0.9547, + "num_tokens": 2233735491.0, + "step": 274 + }, + { + "epoch": 1.1860465116279069, + "grad_norm": 0.42430824346970375, + "learning_rate": 1.806882474077374e-05, + "loss": 0.9267, + "num_tokens": 2241878787.0, + "step": 275 + }, + { + "epoch": 1.1903731746890212, + "grad_norm": 0.48127693874438393, + "learning_rate": 1.805324111593256e-05, + "loss": 0.9211, + "num_tokens": 2250094355.0, + "step": 276 + }, + { + "epoch": 1.1946998377501352, + "grad_norm": 0.4353231937290414, + "learning_rate": 1.8037602488500313e-05, + "loss": 0.9653, + "num_tokens": 2258299091.0, + "step": 277 + }, + { + "epoch": 1.1990265008112493, + "grad_norm": 0.5910526694031867, + "learning_rate": 1.8021908980430153e-05, + "loss": 0.9292, + "num_tokens": 2266449279.0, + "step": 278 + }, + { + "epoch": 1.2033531638723634, + "grad_norm": 3.763333072447578, + "learning_rate": 1.8006160714103213e-05, + "loss": 0.9548, + "num_tokens": 2274482818.0, + "step": 279 + }, + { + "epoch": 1.2076798269334776, + "grad_norm": 0.6292428620950098, + "learning_rate": 1.7990357812327634e-05, + "loss": 0.9704, + "num_tokens": 2282694695.0, + "step": 280 + }, + { + "epoch": 1.2120064899945917, + "grad_norm": 0.41843225864270045, + "learning_rate": 1.797450039833762e-05, + "loss": 0.933, + "num_tokens": 2290919952.0, + "step": 281 + }, + { + "epoch": 1.2163331530557058, + "grad_norm": 0.44745463158115345, + "learning_rate": 1.7958588595792467e-05, + "loss": 0.9504, + "num_tokens": 2299070716.0, + "step": 282 + }, + { + "epoch": 1.2206598161168198, + "grad_norm": 0.5962193932975942, + "learning_rate": 1.794262252877561e-05, + "loss": 0.9302, + "num_tokens": 2307171538.0, + "step": 283 + }, + { + "epoch": 1.2249864791779341, + "grad_norm": 0.40599074608865326, + "learning_rate": 1.7926602321793652e-05, + "loss": 0.9367, + "num_tokens": 2315286698.0, + "step": 284 + }, + { + "epoch": 1.2293131422390482, + "grad_norm": 0.5178527789244168, + "learning_rate": 1.791052809977538e-05, + "loss": 0.931, + "num_tokens": 2323354561.0, + "step": 285 + }, + { + "epoch": 1.2336398053001623, + "grad_norm": 0.6231083117832663, + "learning_rate": 1.7894399988070804e-05, + "loss": 0.9483, + "num_tokens": 2331568026.0, + "step": 286 + }, + { + "epoch": 1.2379664683612763, + "grad_norm": 0.4601882476297517, + "learning_rate": 1.787821811245018e-05, + "loss": 0.9357, + "num_tokens": 2339732034.0, + "step": 287 + }, + { + "epoch": 1.2422931314223904, + "grad_norm": 0.4203019097604034, + "learning_rate": 1.7861982599103033e-05, + "loss": 0.9388, + "num_tokens": 2348068160.0, + "step": 288 + }, + { + "epoch": 1.2466197944835047, + "grad_norm": 0.4986325902222409, + "learning_rate": 1.7845693574637145e-05, + "loss": 0.9385, + "num_tokens": 2356383379.0, + "step": 289 + }, + { + "epoch": 1.2509464575446188, + "grad_norm": 0.4801075949400898, + "learning_rate": 1.7829351166077613e-05, + "loss": 0.9489, + "num_tokens": 2364586281.0, + "step": 290 + }, + { + "epoch": 1.2552731206057328, + "grad_norm": 0.4029618499271581, + "learning_rate": 1.781295550086581e-05, + "loss": 0.9419, + "num_tokens": 2372923490.0, + "step": 291 + }, + { + "epoch": 1.2595997836668469, + "grad_norm": 2.7260117173100893, + "learning_rate": 1.779650670685843e-05, + "loss": 0.941, + "num_tokens": 2381125415.0, + "step": 292 + }, + { + "epoch": 1.263926446727961, + "grad_norm": 0.9250548440509712, + "learning_rate": 1.7780004912326482e-05, + "loss": 0.9205, + "num_tokens": 2389257237.0, + "step": 293 + }, + { + "epoch": 1.2682531097890752, + "grad_norm": 0.42527760432678413, + "learning_rate": 1.7763450245954265e-05, + "loss": 0.9085, + "num_tokens": 2397461313.0, + "step": 294 + }, + { + "epoch": 1.2725797728501893, + "grad_norm": 0.6538369899455708, + "learning_rate": 1.7746842836838397e-05, + "loss": 0.937, + "num_tokens": 2405703036.0, + "step": 295 + }, + { + "epoch": 1.2769064359113034, + "grad_norm": 0.515682709615586, + "learning_rate": 1.773018281448679e-05, + "loss": 0.9264, + "num_tokens": 2413784609.0, + "step": 296 + }, + { + "epoch": 1.2812330989724177, + "grad_norm": 0.5575905269955272, + "learning_rate": 1.7713470308817642e-05, + "loss": 0.9406, + "num_tokens": 2422039119.0, + "step": 297 + }, + { + "epoch": 1.2855597620335315, + "grad_norm": 0.5066298136768553, + "learning_rate": 1.769670545015843e-05, + "loss": 0.9437, + "num_tokens": 2430342231.0, + "step": 298 + }, + { + "epoch": 1.2898864250946458, + "grad_norm": 0.5292123277180285, + "learning_rate": 1.7679888369244895e-05, + "loss": 0.936, + "num_tokens": 2438659746.0, + "step": 299 + }, + { + "epoch": 1.2942130881557599, + "grad_norm": 0.47408732214016464, + "learning_rate": 1.7663019197220003e-05, + "loss": 0.9229, + "num_tokens": 2446791972.0, + "step": 300 + }, + { + "epoch": 1.298539751216874, + "grad_norm": 0.42325831697178434, + "learning_rate": 1.7646098065632956e-05, + "loss": 0.9172, + "num_tokens": 2454970534.0, + "step": 301 + }, + { + "epoch": 1.3028664142779882, + "grad_norm": 0.5226063865140929, + "learning_rate": 1.7629125106438132e-05, + "loss": 0.9416, + "num_tokens": 2463285369.0, + "step": 302 + }, + { + "epoch": 1.3071930773391023, + "grad_norm": 0.4178871768930848, + "learning_rate": 1.7612100451994077e-05, + "loss": 0.9331, + "num_tokens": 2471588270.0, + "step": 303 + }, + { + "epoch": 1.3115197404002163, + "grad_norm": 0.45742867604044, + "learning_rate": 1.759502423506246e-05, + "loss": 0.9177, + "num_tokens": 2479928296.0, + "step": 304 + }, + { + "epoch": 1.3158464034613304, + "grad_norm": 0.4329041029339811, + "learning_rate": 1.7577896588807065e-05, + "loss": 0.9498, + "num_tokens": 2488083060.0, + "step": 305 + }, + { + "epoch": 1.3201730665224445, + "grad_norm": 0.549195113071723, + "learning_rate": 1.7560717646792704e-05, + "loss": 0.9414, + "num_tokens": 2496207113.0, + "step": 306 + }, + { + "epoch": 1.3244997295835588, + "grad_norm": 0.5007519079251725, + "learning_rate": 1.7543487542984227e-05, + "loss": 0.9283, + "num_tokens": 2504363006.0, + "step": 307 + }, + { + "epoch": 1.3288263926446728, + "grad_norm": 0.4302664656481686, + "learning_rate": 1.752620641174544e-05, + "loss": 0.9505, + "num_tokens": 2512601158.0, + "step": 308 + }, + { + "epoch": 1.333153055705787, + "grad_norm": 0.40762150238903927, + "learning_rate": 1.750887438783808e-05, + "loss": 0.9503, + "num_tokens": 2520870232.0, + "step": 309 + }, + { + "epoch": 1.337479718766901, + "grad_norm": 0.4292085185421004, + "learning_rate": 1.749149160642075e-05, + "loss": 0.9108, + "num_tokens": 2529131046.0, + "step": 310 + }, + { + "epoch": 1.341806381828015, + "grad_norm": 0.4228339871228847, + "learning_rate": 1.7474058203047863e-05, + "loss": 0.9462, + "num_tokens": 2537471218.0, + "step": 311 + }, + { + "epoch": 1.3461330448891293, + "grad_norm": 1.5874863004755355, + "learning_rate": 1.745657431366861e-05, + "loss": 0.9609, + "num_tokens": 2545506472.0, + "step": 312 + }, + { + "epoch": 1.3504597079502434, + "grad_norm": 0.7384571833500205, + "learning_rate": 1.743904007462587e-05, + "loss": 0.934, + "num_tokens": 2553731166.0, + "step": 313 + }, + { + "epoch": 1.3547863710113575, + "grad_norm": 0.610328026086027, + "learning_rate": 1.742145562265516e-05, + "loss": 0.9265, + "num_tokens": 2561896975.0, + "step": 314 + }, + { + "epoch": 1.3591130340724715, + "grad_norm": 0.4307803879828114, + "learning_rate": 1.7403821094883572e-05, + "loss": 0.9516, + "num_tokens": 2570093665.0, + "step": 315 + }, + { + "epoch": 1.3634396971335856, + "grad_norm": 0.6954869446118173, + "learning_rate": 1.738613662882869e-05, + "loss": 0.9159, + "num_tokens": 2578141084.0, + "step": 316 + }, + { + "epoch": 1.3677663601946999, + "grad_norm": 0.4045801076886558, + "learning_rate": 1.7368402362397537e-05, + "loss": 0.9318, + "num_tokens": 2586214821.0, + "step": 317 + }, + { + "epoch": 1.372093023255814, + "grad_norm": 0.579678200044439, + "learning_rate": 1.7350618433885487e-05, + "loss": 0.9378, + "num_tokens": 2594387028.0, + "step": 318 + }, + { + "epoch": 1.376419686316928, + "grad_norm": 0.3822113687393793, + "learning_rate": 1.7332784981975183e-05, + "loss": 0.9157, + "num_tokens": 2602470905.0, + "step": 319 + }, + { + "epoch": 1.3807463493780423, + "grad_norm": 0.6217370944940714, + "learning_rate": 1.731490214573547e-05, + "loss": 0.9193, + "num_tokens": 2610641482.0, + "step": 320 + }, + { + "epoch": 1.3850730124391564, + "grad_norm": 0.6439737236380032, + "learning_rate": 1.729697006462029e-05, + "loss": 0.907, + "num_tokens": 2618922646.0, + "step": 321 + }, + { + "epoch": 1.3893996755002704, + "grad_norm": 0.5322347455129888, + "learning_rate": 1.7278988878467616e-05, + "loss": 0.936, + "num_tokens": 2626926397.0, + "step": 322 + }, + { + "epoch": 1.3937263385613845, + "grad_norm": 0.4240695105863202, + "learning_rate": 1.7260958727498358e-05, + "loss": 0.9247, + "num_tokens": 2635094154.0, + "step": 323 + }, + { + "epoch": 1.3980530016224986, + "grad_norm": 0.5561764174540474, + "learning_rate": 1.7242879752315246e-05, + "loss": 0.9698, + "num_tokens": 2643445876.0, + "step": 324 + }, + { + "epoch": 1.4023796646836129, + "grad_norm": 0.40691918158283824, + "learning_rate": 1.722475209390176e-05, + "loss": 0.9106, + "num_tokens": 2651591795.0, + "step": 325 + }, + { + "epoch": 1.406706327744727, + "grad_norm": 0.45792723832547827, + "learning_rate": 1.720657589362103e-05, + "loss": 0.9346, + "num_tokens": 2659861323.0, + "step": 326 + }, + { + "epoch": 1.411032990805841, + "grad_norm": 0.4488207500812395, + "learning_rate": 1.7188351293214707e-05, + "loss": 0.9451, + "num_tokens": 2668107560.0, + "step": 327 + }, + { + "epoch": 1.415359653866955, + "grad_norm": 0.47382786788182957, + "learning_rate": 1.7170078434801893e-05, + "loss": 0.9328, + "num_tokens": 2676232248.0, + "step": 328 + }, + { + "epoch": 1.4196863169280691, + "grad_norm": 0.43399640416231045, + "learning_rate": 1.7151757460878006e-05, + "loss": 0.9275, + "num_tokens": 2684384984.0, + "step": 329 + }, + { + "epoch": 1.4240129799891834, + "grad_norm": 0.4321757247463678, + "learning_rate": 1.713338851431368e-05, + "loss": 0.9209, + "num_tokens": 2692518029.0, + "step": 330 + }, + { + "epoch": 1.4283396430502975, + "grad_norm": 0.48743738131257114, + "learning_rate": 1.7114971738353652e-05, + "loss": 0.9326, + "num_tokens": 2700641781.0, + "step": 331 + }, + { + "epoch": 1.4326663061114115, + "grad_norm": 0.4913348565936382, + "learning_rate": 1.7096507276615638e-05, + "loss": 0.932, + "num_tokens": 2708896364.0, + "step": 332 + }, + { + "epoch": 1.4369929691725256, + "grad_norm": 0.3982647519978175, + "learning_rate": 1.707799527308922e-05, + "loss": 0.9316, + "num_tokens": 2717049617.0, + "step": 333 + }, + { + "epoch": 1.4413196322336397, + "grad_norm": 0.5331893643746559, + "learning_rate": 1.7059435872134725e-05, + "loss": 0.9116, + "num_tokens": 2725284170.0, + "step": 334 + }, + { + "epoch": 1.445646295294754, + "grad_norm": 0.4118566837407902, + "learning_rate": 1.7040829218482083e-05, + "loss": 0.9387, + "num_tokens": 2733249697.0, + "step": 335 + }, + { + "epoch": 1.449972958355868, + "grad_norm": 0.46144127335069157, + "learning_rate": 1.7022175457229726e-05, + "loss": 0.9325, + "num_tokens": 2741353739.0, + "step": 336 + }, + { + "epoch": 1.454299621416982, + "grad_norm": 0.41858735688690185, + "learning_rate": 1.7003474733843423e-05, + "loss": 0.9385, + "num_tokens": 2749511134.0, + "step": 337 + }, + { + "epoch": 1.4586262844780964, + "grad_norm": 0.4326003002340522, + "learning_rate": 1.6984727194155172e-05, + "loss": 0.9275, + "num_tokens": 2757719879.0, + "step": 338 + }, + { + "epoch": 1.4629529475392105, + "grad_norm": 0.39724771390633457, + "learning_rate": 1.696593298436206e-05, + "loss": 0.9401, + "num_tokens": 2765893352.0, + "step": 339 + }, + { + "epoch": 1.4672796106003245, + "grad_norm": 0.5464279340276693, + "learning_rate": 1.6947092251025103e-05, + "loss": 0.9356, + "num_tokens": 2773975845.0, + "step": 340 + }, + { + "epoch": 1.4716062736614386, + "grad_norm": 0.4144238615293452, + "learning_rate": 1.6928205141068125e-05, + "loss": 0.9431, + "num_tokens": 2782109223.0, + "step": 341 + }, + { + "epoch": 1.4759329367225527, + "grad_norm": 0.4740865678072649, + "learning_rate": 1.690927180177661e-05, + "loss": 0.9295, + "num_tokens": 2790405211.0, + "step": 342 + }, + { + "epoch": 1.480259599783667, + "grad_norm": 0.41564607569834605, + "learning_rate": 1.6890292380796534e-05, + "loss": 0.9343, + "num_tokens": 2798580056.0, + "step": 343 + }, + { + "epoch": 1.484586262844781, + "grad_norm": 5.166935270490044, + "learning_rate": 1.687126702613324e-05, + "loss": 0.9755, + "num_tokens": 2806820921.0, + "step": 344 + }, + { + "epoch": 1.488912925905895, + "grad_norm": 0.9110427426431987, + "learning_rate": 1.685219588615026e-05, + "loss": 0.9445, + "num_tokens": 2814825722.0, + "step": 345 + }, + { + "epoch": 1.4932395889670091, + "grad_norm": 0.45812237180426035, + "learning_rate": 1.683307910956818e-05, + "loss": 0.9205, + "num_tokens": 2822967388.0, + "step": 346 + }, + { + "epoch": 1.4975662520281232, + "grad_norm": 0.7212083852760688, + "learning_rate": 1.6813916845463462e-05, + "loss": 0.9364, + "num_tokens": 2831049494.0, + "step": 347 + }, + { + "epoch": 1.5018929150892375, + "grad_norm": 0.6925188881067171, + "learning_rate": 1.6794709243267288e-05, + "loss": 0.9172, + "num_tokens": 2839301325.0, + "step": 348 + }, + { + "epoch": 1.5062195781503516, + "grad_norm": 0.49848671288645746, + "learning_rate": 1.6775456452764398e-05, + "loss": 0.9519, + "num_tokens": 2847487520.0, + "step": 349 + }, + { + "epoch": 1.5105462412114656, + "grad_norm": 0.6085115780592638, + "learning_rate": 1.6756158624091923e-05, + "loss": 0.9348, + "num_tokens": 2855822484.0, + "step": 350 + }, + { + "epoch": 1.51487290427258, + "grad_norm": 0.49102437864607534, + "learning_rate": 1.673681590773821e-05, + "loss": 0.9415, + "num_tokens": 2864139814.0, + "step": 351 + }, + { + "epoch": 1.5191995673336938, + "grad_norm": 0.5263169888215856, + "learning_rate": 1.671742845454164e-05, + "loss": 0.9231, + "num_tokens": 2872296506.0, + "step": 352 + }, + { + "epoch": 1.523526230394808, + "grad_norm": 0.5177341024812663, + "learning_rate": 1.6697996415689473e-05, + "loss": 0.9354, + "num_tokens": 2880434577.0, + "step": 353 + }, + { + "epoch": 1.5278528934559221, + "grad_norm": 0.4612875515949862, + "learning_rate": 1.667851994271665e-05, + "loss": 0.9059, + "num_tokens": 2888531402.0, + "step": 354 + }, + { + "epoch": 1.5321795565170362, + "grad_norm": 0.5727137014112862, + "learning_rate": 1.6658999187504615e-05, + "loss": 0.9167, + "num_tokens": 2896572454.0, + "step": 355 + }, + { + "epoch": 1.5365062195781505, + "grad_norm": 0.41823093858734256, + "learning_rate": 1.6639434302280145e-05, + "loss": 0.917, + "num_tokens": 2904737717.0, + "step": 356 + }, + { + "epoch": 1.5408328826392643, + "grad_norm": 0.6777094486165214, + "learning_rate": 1.6619825439614143e-05, + "loss": 0.933, + "num_tokens": 2912854159.0, + "step": 357 + }, + { + "epoch": 1.5451595457003786, + "grad_norm": 0.5160985581118991, + "learning_rate": 1.660017275242046e-05, + "loss": 0.9266, + "num_tokens": 2920913619.0, + "step": 358 + }, + { + "epoch": 1.5494862087614927, + "grad_norm": 0.6729337006148577, + "learning_rate": 1.6580476393954694e-05, + "loss": 0.9045, + "num_tokens": 2929112623.0, + "step": 359 + }, + { + "epoch": 1.5538128718226067, + "grad_norm": 0.5608135343619238, + "learning_rate": 1.6560736517813013e-05, + "loss": 0.9244, + "num_tokens": 2937273800.0, + "step": 360 + }, + { + "epoch": 1.558139534883721, + "grad_norm": 0.6852364347957023, + "learning_rate": 1.6540953277930925e-05, + "loss": 0.9344, + "num_tokens": 2945612356.0, + "step": 361 + }, + { + "epoch": 1.562466197944835, + "grad_norm": 0.465666173687953, + "learning_rate": 1.6521126828582118e-05, + "loss": 0.9221, + "num_tokens": 2953805654.0, + "step": 362 + }, + { + "epoch": 1.5667928610059492, + "grad_norm": 0.7171922804612683, + "learning_rate": 1.6501257324377227e-05, + "loss": 0.9417, + "num_tokens": 2962056242.0, + "step": 363 + }, + { + "epoch": 1.5711195240670632, + "grad_norm": 0.5433057833659239, + "learning_rate": 1.648134492026263e-05, + "loss": 0.9447, + "num_tokens": 2970248144.0, + "step": 364 + }, + { + "epoch": 1.5754461871281773, + "grad_norm": 0.5703270113708258, + "learning_rate": 1.6461389771519263e-05, + "loss": 0.9084, + "num_tokens": 2978407405.0, + "step": 365 + }, + { + "epoch": 1.5797728501892916, + "grad_norm": 0.5377284539952227, + "learning_rate": 1.6441392033761378e-05, + "loss": 0.9277, + "num_tokens": 2986581353.0, + "step": 366 + }, + { + "epoch": 1.5840995132504057, + "grad_norm": 0.5543175733013811, + "learning_rate": 1.6421351862935348e-05, + "loss": 0.9375, + "num_tokens": 2994845303.0, + "step": 367 + }, + { + "epoch": 1.5884261763115197, + "grad_norm": 0.49435998462840264, + "learning_rate": 1.6401269415318462e-05, + "loss": 0.9116, + "num_tokens": 3002783957.0, + "step": 368 + }, + { + "epoch": 1.592752839372634, + "grad_norm": 0.5415581262563944, + "learning_rate": 1.6381144847517672e-05, + "loss": 0.9156, + "num_tokens": 3011071399.0, + "step": 369 + }, + { + "epoch": 1.5970795024337479, + "grad_norm": 0.44702591691128174, + "learning_rate": 1.6360978316468404e-05, + "loss": 0.9167, + "num_tokens": 3019260204.0, + "step": 370 + }, + { + "epoch": 1.6014061654948621, + "grad_norm": 0.5369169002409531, + "learning_rate": 1.6340769979433314e-05, + "loss": 0.9249, + "num_tokens": 3027381946.0, + "step": 371 + }, + { + "epoch": 1.6057328285559762, + "grad_norm": 0.4036413923248667, + "learning_rate": 1.632051999400108e-05, + "loss": 0.9232, + "num_tokens": 3035634956.0, + "step": 372 + }, + { + "epoch": 1.6100594916170903, + "grad_norm": 0.5909816450871015, + "learning_rate": 1.6300228518085148e-05, + "loss": 0.9287, + "num_tokens": 3043836858.0, + "step": 373 + }, + { + "epoch": 1.6143861546782046, + "grad_norm": 0.4024810283961205, + "learning_rate": 1.6279895709922534e-05, + "loss": 0.9313, + "num_tokens": 3051951149.0, + "step": 374 + }, + { + "epoch": 1.6187128177393184, + "grad_norm": 0.5651764879246354, + "learning_rate": 1.625952172807255e-05, + "loss": 0.9082, + "num_tokens": 3060171329.0, + "step": 375 + }, + { + "epoch": 1.6230394808004327, + "grad_norm": 0.7076832324695906, + "learning_rate": 1.6239106731415604e-05, + "loss": 0.911, + "num_tokens": 3068406742.0, + "step": 376 + }, + { + "epoch": 1.6273661438615468, + "grad_norm": 0.5624449796241844, + "learning_rate": 1.6218650879151946e-05, + "loss": 0.9268, + "num_tokens": 3076648922.0, + "step": 377 + }, + { + "epoch": 1.6316928069226608, + "grad_norm": 0.4542645735176373, + "learning_rate": 1.6198154330800408e-05, + "loss": 0.9329, + "num_tokens": 3084760568.0, + "step": 378 + }, + { + "epoch": 1.6360194699837751, + "grad_norm": 0.5048456648778138, + "learning_rate": 1.6177617246197206e-05, + "loss": 0.9338, + "num_tokens": 3092895455.0, + "step": 379 + }, + { + "epoch": 1.640346133044889, + "grad_norm": 0.500093898184155, + "learning_rate": 1.615703978549464e-05, + "loss": 0.9067, + "num_tokens": 3100964651.0, + "step": 380 + }, + { + "epoch": 1.6446727961060033, + "grad_norm": 0.47333990305328766, + "learning_rate": 1.6136422109159887e-05, + "loss": 0.9197, + "num_tokens": 3109142378.0, + "step": 381 + }, + { + "epoch": 1.6489994591671173, + "grad_norm": 0.4562283281745153, + "learning_rate": 1.611576437797373e-05, + "loss": 0.9246, + "num_tokens": 3117299434.0, + "step": 382 + }, + { + "epoch": 1.6533261222282314, + "grad_norm": 0.46912085795087555, + "learning_rate": 1.60950667530293e-05, + "loss": 0.9277, + "num_tokens": 3125571167.0, + "step": 383 + }, + { + "epoch": 1.6576527852893457, + "grad_norm": 0.5201385710628031, + "learning_rate": 1.607432939573084e-05, + "loss": 0.9148, + "num_tokens": 3133697525.0, + "step": 384 + }, + { + "epoch": 1.6619794483504597, + "grad_norm": 0.37660757557206814, + "learning_rate": 1.605355246779243e-05, + "loss": 0.9508, + "num_tokens": 3141822978.0, + "step": 385 + }, + { + "epoch": 1.6663061114115738, + "grad_norm": 0.551184808866067, + "learning_rate": 1.6032736131236722e-05, + "loss": 0.902, + "num_tokens": 3150015018.0, + "step": 386 + }, + { + "epoch": 1.670632774472688, + "grad_norm": 0.3901710925044271, + "learning_rate": 1.6011880548393694e-05, + "loss": 0.9034, + "num_tokens": 3158336582.0, + "step": 387 + }, + { + "epoch": 1.674959437533802, + "grad_norm": 0.520033700897056, + "learning_rate": 1.5990985881899367e-05, + "loss": 0.9272, + "num_tokens": 3166540130.0, + "step": 388 + }, + { + "epoch": 1.6792861005949162, + "grad_norm": 0.5353166178330362, + "learning_rate": 1.597005229469455e-05, + "loss": 0.9145, + "num_tokens": 3174680579.0, + "step": 389 + }, + { + "epoch": 1.6836127636560303, + "grad_norm": 0.3976708445593737, + "learning_rate": 1.594907995002356e-05, + "loss": 0.9204, + "num_tokens": 3182889631.0, + "step": 390 + }, + { + "epoch": 1.6879394267171444, + "grad_norm": 0.4784070195017845, + "learning_rate": 1.5928069011432955e-05, + "loss": 0.9088, + "num_tokens": 3190935444.0, + "step": 391 + }, + { + "epoch": 1.6922660897782587, + "grad_norm": 0.47369072605947965, + "learning_rate": 1.590701964277025e-05, + "loss": 0.9246, + "num_tokens": 3199139933.0, + "step": 392 + }, + { + "epoch": 1.6965927528393725, + "grad_norm": 0.508975977127289, + "learning_rate": 1.588593200818266e-05, + "loss": 0.9221, + "num_tokens": 3207493693.0, + "step": 393 + }, + { + "epoch": 1.7009194159004868, + "grad_norm": 0.4338301735630921, + "learning_rate": 1.5864806272115786e-05, + "loss": 0.9119, + "num_tokens": 3215722302.0, + "step": 394 + }, + { + "epoch": 1.7052460789616009, + "grad_norm": 0.45012723691222956, + "learning_rate": 1.5843642599312365e-05, + "loss": 0.9131, + "num_tokens": 3223931960.0, + "step": 395 + }, + { + "epoch": 1.709572742022715, + "grad_norm": 0.5743856871396514, + "learning_rate": 1.582244115481097e-05, + "loss": 0.9182, + "num_tokens": 3232077781.0, + "step": 396 + }, + { + "epoch": 1.7138994050838292, + "grad_norm": 0.34475583620132555, + "learning_rate": 1.5801202103944725e-05, + "loss": 0.9101, + "num_tokens": 3240178362.0, + "step": 397 + }, + { + "epoch": 1.718226068144943, + "grad_norm": 0.4699145821831058, + "learning_rate": 1.577992561234001e-05, + "loss": 0.9324, + "num_tokens": 3248415998.0, + "step": 398 + }, + { + "epoch": 1.7225527312060573, + "grad_norm": 0.4918593807700145, + "learning_rate": 1.5758611845915188e-05, + "loss": 0.9352, + "num_tokens": 3256514429.0, + "step": 399 + }, + { + "epoch": 1.7268793942671714, + "grad_norm": 0.46058113786272287, + "learning_rate": 1.573726097087928e-05, + "loss": 0.9295, + "num_tokens": 3264657809.0, + "step": 400 + }, + { + "epoch": 1.7312060573282855, + "grad_norm": 0.3939951301191064, + "learning_rate": 1.5715873153730713e-05, + "loss": 0.9092, + "num_tokens": 3272908368.0, + "step": 401 + }, + { + "epoch": 1.7355327203893998, + "grad_norm": 0.41937619102996104, + "learning_rate": 1.5694448561255972e-05, + "loss": 0.9045, + "num_tokens": 3281278346.0, + "step": 402 + }, + { + "epoch": 1.7398593834505138, + "grad_norm": 0.5603562215899213, + "learning_rate": 1.5672987360528334e-05, + "loss": 0.9026, + "num_tokens": 3289563469.0, + "step": 403 + }, + { + "epoch": 1.744186046511628, + "grad_norm": 0.36939883821258745, + "learning_rate": 1.5651489718906553e-05, + "loss": 0.9298, + "num_tokens": 3297848756.0, + "step": 404 + }, + { + "epoch": 1.7485127095727422, + "grad_norm": 0.5522494345369162, + "learning_rate": 1.5629955804033558e-05, + "loss": 0.9232, + "num_tokens": 3305947270.0, + "step": 405 + }, + { + "epoch": 1.752839372633856, + "grad_norm": 0.5087487713374635, + "learning_rate": 1.5608385783835145e-05, + "loss": 0.9005, + "num_tokens": 3314154548.0, + "step": 406 + }, + { + "epoch": 1.7571660356949703, + "grad_norm": 0.35613076878477273, + "learning_rate": 1.558677982651866e-05, + "loss": 0.9153, + "num_tokens": 3322401023.0, + "step": 407 + }, + { + "epoch": 1.7614926987560844, + "grad_norm": 3.004718095464454, + "learning_rate": 1.5565138100571703e-05, + "loss": 1.0005, + "num_tokens": 3330591089.0, + "step": 408 + }, + { + "epoch": 1.7658193618171985, + "grad_norm": 0.8945369496941992, + "learning_rate": 1.5543460774760798e-05, + "loss": 0.9205, + "num_tokens": 3338808458.0, + "step": 409 + }, + { + "epoch": 1.7701460248783127, + "grad_norm": 0.4352376852362294, + "learning_rate": 1.5521748018130082e-05, + "loss": 0.9584, + "num_tokens": 3347174654.0, + "step": 410 + }, + { + "epoch": 1.7744726879394266, + "grad_norm": 0.609757165223338, + "learning_rate": 1.55e-05, + "loss": 0.9516, + "num_tokens": 3355320718.0, + "step": 411 + }, + { + "epoch": 1.7787993510005409, + "grad_norm": 0.5520473688674168, + "learning_rate": 1.5478216889965965e-05, + "loss": 0.9333, + "num_tokens": 3363459971.0, + "step": 412 + }, + { + "epoch": 1.783126014061655, + "grad_norm": 0.49281661126977533, + "learning_rate": 1.545639885789704e-05, + "loss": 0.923, + "num_tokens": 3371739156.0, + "step": 413 + }, + { + "epoch": 1.787452677122769, + "grad_norm": 0.5560643264576769, + "learning_rate": 1.5434546073934625e-05, + "loss": 0.9361, + "num_tokens": 3379788636.0, + "step": 414 + }, + { + "epoch": 1.7917793401838833, + "grad_norm": 0.44209595144570524, + "learning_rate": 1.541265870849112e-05, + "loss": 0.9262, + "num_tokens": 3387924962.0, + "step": 415 + }, + { + "epoch": 1.7961060032449971, + "grad_norm": 0.6070128313460539, + "learning_rate": 1.5390736932248595e-05, + "loss": 0.8898, + "num_tokens": 3395936261.0, + "step": 416 + }, + { + "epoch": 1.8004326663061114, + "grad_norm": 0.47206598080267503, + "learning_rate": 1.5368780916157466e-05, + "loss": 0.9333, + "num_tokens": 3403999071.0, + "step": 417 + }, + { + "epoch": 1.8047593293672255, + "grad_norm": 0.5474905062663444, + "learning_rate": 1.5346790831435157e-05, + "loss": 0.938, + "num_tokens": 3412038810.0, + "step": 418 + }, + { + "epoch": 1.8090859924283396, + "grad_norm": 0.4304182016730308, + "learning_rate": 1.5324766849564766e-05, + "loss": 0.9107, + "num_tokens": 3420225284.0, + "step": 419 + }, + { + "epoch": 1.8134126554894539, + "grad_norm": 0.4888120564153342, + "learning_rate": 1.5302709142293732e-05, + "loss": 0.941, + "num_tokens": 3428374233.0, + "step": 420 + }, + { + "epoch": 1.817739318550568, + "grad_norm": 0.47697295556152397, + "learning_rate": 1.528061788163248e-05, + "loss": 0.888, + "num_tokens": 3436609204.0, + "step": 421 + }, + { + "epoch": 1.822065981611682, + "grad_norm": 0.3999854836952636, + "learning_rate": 1.52584932398531e-05, + "loss": 0.9302, + "num_tokens": 3444701961.0, + "step": 422 + }, + { + "epoch": 1.826392644672796, + "grad_norm": 0.4885292531539521, + "learning_rate": 1.5236335389487997e-05, + "loss": 0.8906, + "num_tokens": 3452748752.0, + "step": 423 + }, + { + "epoch": 1.8307193077339101, + "grad_norm": 0.4368358294354215, + "learning_rate": 1.5214144503328532e-05, + "loss": 0.8998, + "num_tokens": 3460973876.0, + "step": 424 + }, + { + "epoch": 1.8350459707950244, + "grad_norm": 0.37936931936739093, + "learning_rate": 1.5191920754423698e-05, + "loss": 0.9142, + "num_tokens": 3469299324.0, + "step": 425 + }, + { + "epoch": 1.8393726338561385, + "grad_norm": 0.43837542594614076, + "learning_rate": 1.5169664316078758e-05, + "loss": 0.9067, + "num_tokens": 3477387515.0, + "step": 426 + }, + { + "epoch": 1.8436992969172525, + "grad_norm": 0.4744991932675852, + "learning_rate": 1.514737536185388e-05, + "loss": 0.9419, + "num_tokens": 3485617970.0, + "step": 427 + }, + { + "epoch": 1.8480259599783668, + "grad_norm": 0.35152350730870363, + "learning_rate": 1.512505406556281e-05, + "loss": 0.9073, + "num_tokens": 3493787044.0, + "step": 428 + }, + { + "epoch": 1.8523526230394807, + "grad_norm": 0.44138744966131516, + "learning_rate": 1.5102700601271503e-05, + "loss": 0.9106, + "num_tokens": 3501956330.0, + "step": 429 + }, + { + "epoch": 1.856679286100595, + "grad_norm": 0.38057698169738785, + "learning_rate": 1.5080315143296758e-05, + "loss": 0.9233, + "num_tokens": 3510115204.0, + "step": 430 + }, + { + "epoch": 1.861005949161709, + "grad_norm": 0.38874077070561314, + "learning_rate": 1.5057897866204878e-05, + "loss": 0.9051, + "num_tokens": 3518347535.0, + "step": 431 + }, + { + "epoch": 1.865332612222823, + "grad_norm": 0.39275793677415505, + "learning_rate": 1.5035448944810293e-05, + "loss": 0.9257, + "num_tokens": 3526549333.0, + "step": 432 + }, + { + "epoch": 1.8696592752839374, + "grad_norm": 0.40697953022097266, + "learning_rate": 1.5012968554174198e-05, + "loss": 0.9154, + "num_tokens": 3534791430.0, + "step": 433 + }, + { + "epoch": 1.8739859383450512, + "grad_norm": 0.37570193407129493, + "learning_rate": 1.4990456869603193e-05, + "loss": 0.9184, + "num_tokens": 3542974024.0, + "step": 434 + }, + { + "epoch": 1.8783126014061655, + "grad_norm": 0.362561840859063, + "learning_rate": 1.4967914066647928e-05, + "loss": 0.9146, + "num_tokens": 3551132133.0, + "step": 435 + }, + { + "epoch": 1.8826392644672796, + "grad_norm": 0.3965765830883574, + "learning_rate": 1.4945340321101698e-05, + "loss": 0.9364, + "num_tokens": 3559359244.0, + "step": 436 + }, + { + "epoch": 1.8869659275283936, + "grad_norm": 0.5150730160741546, + "learning_rate": 1.4922735808999107e-05, + "loss": 0.9164, + "num_tokens": 3567410487.0, + "step": 437 + }, + { + "epoch": 1.891292590589508, + "grad_norm": 0.33322705254699836, + "learning_rate": 1.4900100706614686e-05, + "loss": 0.9251, + "num_tokens": 3575554406.0, + "step": 438 + }, + { + "epoch": 1.895619253650622, + "grad_norm": 0.4908146102288755, + "learning_rate": 1.4877435190461506e-05, + "loss": 0.92, + "num_tokens": 3583884834.0, + "step": 439 + }, + { + "epoch": 1.899945916711736, + "grad_norm": 0.38292660340308166, + "learning_rate": 1.4854739437289814e-05, + "loss": 0.9227, + "num_tokens": 3591993768.0, + "step": 440 + }, + { + "epoch": 1.9042725797728501, + "grad_norm": 0.36720114565545237, + "learning_rate": 1.4832013624085654e-05, + "loss": 0.9271, + "num_tokens": 3600194803.0, + "step": 441 + }, + { + "epoch": 1.9085992428339642, + "grad_norm": 0.4250143841777569, + "learning_rate": 1.4809257928069487e-05, + "loss": 0.9459, + "num_tokens": 3608329500.0, + "step": 442 + }, + { + "epoch": 1.9129259058950785, + "grad_norm": 0.4735816487326457, + "learning_rate": 1.4786472526694795e-05, + "loss": 0.9132, + "num_tokens": 3616669854.0, + "step": 443 + }, + { + "epoch": 1.9172525689561926, + "grad_norm": 0.380098999872213, + "learning_rate": 1.4763657597646713e-05, + "loss": 0.9376, + "num_tokens": 3624932650.0, + "step": 444 + }, + { + "epoch": 1.9215792320173066, + "grad_norm": 0.4158549718051883, + "learning_rate": 1.4740813318840652e-05, + "loss": 0.9211, + "num_tokens": 3633032770.0, + "step": 445 + }, + { + "epoch": 1.925905895078421, + "grad_norm": 0.31575001074360326, + "learning_rate": 1.4717939868420878e-05, + "loss": 0.9035, + "num_tokens": 3641129330.0, + "step": 446 + }, + { + "epoch": 1.9302325581395348, + "grad_norm": 0.4034639529737966, + "learning_rate": 1.4695037424759153e-05, + "loss": 0.9202, + "num_tokens": 3649430974.0, + "step": 447 + }, + { + "epoch": 1.934559221200649, + "grad_norm": 0.3506560677262555, + "learning_rate": 1.4672106166453337e-05, + "loss": 0.9221, + "num_tokens": 3657593705.0, + "step": 448 + }, + { + "epoch": 1.9388858842617631, + "grad_norm": 0.3963998799799475, + "learning_rate": 1.4649146272325984e-05, + "loss": 0.8893, + "num_tokens": 3665724037.0, + "step": 449 + }, + { + "epoch": 1.9432125473228772, + "grad_norm": 0.4294346243936522, + "learning_rate": 1.4626157921422965e-05, + "loss": 0.9101, + "num_tokens": 3673874726.0, + "step": 450 + }, + { + "epoch": 1.9475392103839915, + "grad_norm": 0.42780411562160314, + "learning_rate": 1.4603141293012057e-05, + "loss": 0.9249, + "num_tokens": 3682087260.0, + "step": 451 + }, + { + "epoch": 1.9518658734451053, + "grad_norm": 0.40847910505075763, + "learning_rate": 1.458009656658155e-05, + "loss": 0.9191, + "num_tokens": 3690401312.0, + "step": 452 + }, + { + "epoch": 1.9561925365062196, + "grad_norm": 0.3334618328736044, + "learning_rate": 1.4557023921838851e-05, + "loss": 0.9035, + "num_tokens": 3698507592.0, + "step": 453 + }, + { + "epoch": 1.9605191995673337, + "grad_norm": 0.3402318115896455, + "learning_rate": 1.4533923538709076e-05, + "loss": 0.9286, + "num_tokens": 3706566790.0, + "step": 454 + }, + { + "epoch": 1.9648458626284477, + "grad_norm": 0.43132218994493254, + "learning_rate": 1.4510795597333658e-05, + "loss": 0.9313, + "num_tokens": 3714713753.0, + "step": 455 + }, + { + "epoch": 1.969172525689562, + "grad_norm": 0.29891560690244473, + "learning_rate": 1.4487640278068929e-05, + "loss": 0.9034, + "num_tokens": 3722946902.0, + "step": 456 + }, + { + "epoch": 1.973499188750676, + "grad_norm": 0.4604931133480665, + "learning_rate": 1.4464457761484716e-05, + "loss": 0.9005, + "num_tokens": 3731204684.0, + "step": 457 + }, + { + "epoch": 1.9778258518117902, + "grad_norm": 0.47252276433364016, + "learning_rate": 1.4441248228362943e-05, + "loss": 0.8941, + "num_tokens": 3739251077.0, + "step": 458 + }, + { + "epoch": 1.9821525148729042, + "grad_norm": 0.3487883107809127, + "learning_rate": 1.4418011859696213e-05, + "loss": 0.9397, + "num_tokens": 3747482869.0, + "step": 459 + }, + { + "epoch": 1.9864791779340183, + "grad_norm": 0.3755971588438606, + "learning_rate": 1.4394748836686392e-05, + "loss": 0.9228, + "num_tokens": 3755645582.0, + "step": 460 + }, + { + "epoch": 1.9908058409951326, + "grad_norm": 0.38594868284375883, + "learning_rate": 1.437145934074321e-05, + "loss": 0.9076, + "num_tokens": 3763943960.0, + "step": 461 + }, + { + "epoch": 1.9951325040562466, + "grad_norm": 0.441193518196438, + "learning_rate": 1.4348143553482834e-05, + "loss": 0.896, + "num_tokens": 3771940937.0, + "step": 462 + }, + { + "epoch": 1.9994591671173607, + "grad_norm": 0.40886370735522437, + "learning_rate": 1.4324801656726457e-05, + "loss": 0.9404, + "num_tokens": 3780130044.0, + "step": 463 + }, + { + "epoch": 2.0, + "grad_norm": 0.5241648188516919, + "learning_rate": 1.4301433832498879e-05, + "loss": 0.9439, + "num_tokens": 3781178378.0, + "step": 464 + }, + { + "epoch": 2.0043266630611143, + "grad_norm": 0.5422765165308258, + "learning_rate": 1.4278040263027087e-05, + "loss": 0.9069, + "num_tokens": 3789243561.0, + "step": 465 + }, + { + "epoch": 2.008653326122228, + "grad_norm": 0.5153690241659417, + "learning_rate": 1.425462113073883e-05, + "loss": 0.8842, + "num_tokens": 3797347144.0, + "step": 466 + }, + { + "epoch": 2.0129799891833424, + "grad_norm": 0.3861162280683008, + "learning_rate": 1.4231176618261218e-05, + "loss": 0.8991, + "num_tokens": 3805478951.0, + "step": 467 + }, + { + "epoch": 2.0173066522444563, + "grad_norm": 0.6093514964728659, + "learning_rate": 1.4207706908419257e-05, + "loss": 0.8803, + "num_tokens": 3813714819.0, + "step": 468 + }, + { + "epoch": 2.0216333153055706, + "grad_norm": 0.4184709872797328, + "learning_rate": 1.4184212184234465e-05, + "loss": 0.8942, + "num_tokens": 3821851596.0, + "step": 469 + }, + { + "epoch": 2.025959978366685, + "grad_norm": 0.6526954073871744, + "learning_rate": 1.416069262892342e-05, + "loss": 0.9123, + "num_tokens": 3829825816.0, + "step": 470 + }, + { + "epoch": 2.0302866414277987, + "grad_norm": 0.3979475276126333, + "learning_rate": 1.4137148425896338e-05, + "loss": 0.922, + "num_tokens": 3838031840.0, + "step": 471 + }, + { + "epoch": 2.034613304488913, + "grad_norm": 0.69870506602204, + "learning_rate": 1.4113579758755645e-05, + "loss": 0.8937, + "num_tokens": 3846220365.0, + "step": 472 + }, + { + "epoch": 2.038939967550027, + "grad_norm": 0.5030836339371385, + "learning_rate": 1.4089986811294537e-05, + "loss": 0.9052, + "num_tokens": 3854344876.0, + "step": 473 + }, + { + "epoch": 2.043266630611141, + "grad_norm": 0.6731424291091427, + "learning_rate": 1.4066369767495567e-05, + "loss": 0.9031, + "num_tokens": 3862414371.0, + "step": 474 + }, + { + "epoch": 2.0475932936722554, + "grad_norm": 0.5661826912895142, + "learning_rate": 1.4042728811529175e-05, + "loss": 0.878, + "num_tokens": 3870558161.0, + "step": 475 + }, + { + "epoch": 2.0519199567333692, + "grad_norm": 0.5450287285946428, + "learning_rate": 1.4019064127752298e-05, + "loss": 0.8989, + "num_tokens": 3878550807.0, + "step": 476 + }, + { + "epoch": 2.0562466197944835, + "grad_norm": 0.501789175723529, + "learning_rate": 1.399537590070688e-05, + "loss": 0.8864, + "num_tokens": 3886777664.0, + "step": 477 + }, + { + "epoch": 2.060573282855598, + "grad_norm": 0.4408542670305424, + "learning_rate": 1.3971664315118483e-05, + "loss": 0.8653, + "num_tokens": 3894968237.0, + "step": 478 + }, + { + "epoch": 2.0648999459167117, + "grad_norm": 0.5202978543579215, + "learning_rate": 1.3947929555894813e-05, + "loss": 0.9173, + "num_tokens": 3903256977.0, + "step": 479 + }, + { + "epoch": 2.069226608977826, + "grad_norm": 0.4098601771985662, + "learning_rate": 1.392417180812429e-05, + "loss": 0.8718, + "num_tokens": 3911521747.0, + "step": 480 + }, + { + "epoch": 2.07355327203894, + "grad_norm": 5.22436981787343, + "learning_rate": 1.3900391257074601e-05, + "loss": 0.8893, + "num_tokens": 3919640464.0, + "step": 481 + }, + { + "epoch": 2.077879935100054, + "grad_norm": 1.043535388054233, + "learning_rate": 1.3876588088191264e-05, + "loss": 0.9115, + "num_tokens": 3927853518.0, + "step": 482 + }, + { + "epoch": 2.0822065981611684, + "grad_norm": 0.6560959194948016, + "learning_rate": 1.3852762487096168e-05, + "loss": 0.9033, + "num_tokens": 3935981726.0, + "step": 483 + }, + { + "epoch": 2.086533261222282, + "grad_norm": 0.687254059937301, + "learning_rate": 1.3828914639586138e-05, + "loss": 0.9148, + "num_tokens": 3944268930.0, + "step": 484 + }, + { + "epoch": 2.0908599242833965, + "grad_norm": 0.8815742044415361, + "learning_rate": 1.3805044731631475e-05, + "loss": 0.9062, + "num_tokens": 3952495730.0, + "step": 485 + }, + { + "epoch": 2.0951865873445104, + "grad_norm": 0.5215145270109305, + "learning_rate": 1.3781152949374527e-05, + "loss": 0.9165, + "num_tokens": 3960536763.0, + "step": 486 + }, + { + "epoch": 2.0995132504056246, + "grad_norm": 0.6314748042637296, + "learning_rate": 1.3757239479128204e-05, + "loss": 0.8679, + "num_tokens": 3968809657.0, + "step": 487 + }, + { + "epoch": 2.103839913466739, + "grad_norm": 0.6542582395643257, + "learning_rate": 1.373330450737455e-05, + "loss": 0.8879, + "num_tokens": 3976993049.0, + "step": 488 + }, + { + "epoch": 2.1081665765278528, + "grad_norm": 0.5017338218634015, + "learning_rate": 1.3709348220763287e-05, + "loss": 0.8987, + "num_tokens": 3985177412.0, + "step": 489 + }, + { + "epoch": 2.112493239588967, + "grad_norm": 0.5990144255847849, + "learning_rate": 1.3685370806110343e-05, + "loss": 0.8933, + "num_tokens": 3993425392.0, + "step": 490 + }, + { + "epoch": 2.1168199026500814, + "grad_norm": 0.552156147958245, + "learning_rate": 1.3661372450396422e-05, + "loss": 0.9139, + "num_tokens": 4001507279.0, + "step": 491 + }, + { + "epoch": 2.121146565711195, + "grad_norm": 0.45794058264855625, + "learning_rate": 1.3637353340765518e-05, + "loss": 0.9018, + "num_tokens": 4009520377.0, + "step": 492 + }, + { + "epoch": 2.1254732287723095, + "grad_norm": 0.5149005425946213, + "learning_rate": 1.3613313664523476e-05, + "loss": 0.9161, + "num_tokens": 4017809831.0, + "step": 493 + }, + { + "epoch": 2.1297998918334233, + "grad_norm": 0.4481923256993226, + "learning_rate": 1.3589253609136517e-05, + "loss": 0.8994, + "num_tokens": 4026017732.0, + "step": 494 + }, + { + "epoch": 2.1341265548945376, + "grad_norm": 0.47721449742552385, + "learning_rate": 1.3565173362229787e-05, + "loss": 0.9217, + "num_tokens": 4034155377.0, + "step": 495 + }, + { + "epoch": 2.138453217955652, + "grad_norm": 0.3903427614906859, + "learning_rate": 1.354107311158589e-05, + "loss": 0.9082, + "num_tokens": 4042436597.0, + "step": 496 + }, + { + "epoch": 2.1427798810167658, + "grad_norm": 0.4913115498977301, + "learning_rate": 1.3516953045143421e-05, + "loss": 0.933, + "num_tokens": 4050699644.0, + "step": 497 + }, + { + "epoch": 2.14710654407788, + "grad_norm": 0.3849035129446924, + "learning_rate": 1.3492813350995501e-05, + "loss": 0.8915, + "num_tokens": 4059016980.0, + "step": 498 + }, + { + "epoch": 2.151433207138994, + "grad_norm": 0.47042453957254404, + "learning_rate": 1.3468654217388322e-05, + "loss": 0.8871, + "num_tokens": 4067101869.0, + "step": 499 + }, + { + "epoch": 2.155759870200108, + "grad_norm": 0.490969270645771, + "learning_rate": 1.344447583271965e-05, + "loss": 0.8766, + "num_tokens": 4075178331.0, + "step": 500 + }, + { + "epoch": 2.1600865332612225, + "grad_norm": 0.33223719945376884, + "learning_rate": 1.342027838553739e-05, + "loss": 0.9186, + "num_tokens": 4083256684.0, + "step": 501 + }, + { + "epoch": 2.1644131963223363, + "grad_norm": 0.5387819016196778, + "learning_rate": 1.3396062064538103e-05, + "loss": 0.8906, + "num_tokens": 4091559760.0, + "step": 502 + }, + { + "epoch": 2.1687398593834506, + "grad_norm": 0.3859472115796784, + "learning_rate": 1.3371827058565517e-05, + "loss": 0.8789, + "num_tokens": 4099632591.0, + "step": 503 + }, + { + "epoch": 2.1730665224445644, + "grad_norm": 0.5086761457506247, + "learning_rate": 1.3347573556609075e-05, + "loss": 0.8797, + "num_tokens": 4107816226.0, + "step": 504 + }, + { + "epoch": 2.1773931855056787, + "grad_norm": 0.39661156622035815, + "learning_rate": 1.332330174780246e-05, + "loss": 0.8875, + "num_tokens": 4115954439.0, + "step": 505 + }, + { + "epoch": 2.181719848566793, + "grad_norm": 0.3764861176145268, + "learning_rate": 1.3299011821422116e-05, + "loss": 0.8803, + "num_tokens": 4124198489.0, + "step": 506 + }, + { + "epoch": 2.186046511627907, + "grad_norm": 0.41751737408571854, + "learning_rate": 1.3274703966885765e-05, + "loss": 0.8993, + "num_tokens": 4132294680.0, + "step": 507 + }, + { + "epoch": 2.190373174689021, + "grad_norm": 0.3057459758427046, + "learning_rate": 1.3250378373750941e-05, + "loss": 0.8976, + "num_tokens": 4140483674.0, + "step": 508 + }, + { + "epoch": 2.194699837750135, + "grad_norm": 0.42643227352485064, + "learning_rate": 1.3226035231713504e-05, + "loss": 0.9049, + "num_tokens": 4148649672.0, + "step": 509 + }, + { + "epoch": 2.1990265008112493, + "grad_norm": 0.46618047581555655, + "learning_rate": 1.3201674730606166e-05, + "loss": 0.8956, + "num_tokens": 4156639423.0, + "step": 510 + }, + { + "epoch": 2.2033531638723636, + "grad_norm": 0.4982014829182684, + "learning_rate": 1.317729706039701e-05, + "loss": 0.8991, + "num_tokens": 4164691703.0, + "step": 511 + }, + { + "epoch": 2.2076798269334774, + "grad_norm": 0.41635929468225186, + "learning_rate": 1.3152902411188007e-05, + "loss": 0.8884, + "num_tokens": 4172989319.0, + "step": 512 + }, + { + "epoch": 2.2120064899945917, + "grad_norm": 0.49596696921284295, + "learning_rate": 1.3128490973213523e-05, + "loss": 0.9025, + "num_tokens": 4181159956.0, + "step": 513 + }, + { + "epoch": 2.2163331530557056, + "grad_norm": 0.4761010656135519, + "learning_rate": 1.3104062936838863e-05, + "loss": 0.9058, + "num_tokens": 4189205775.0, + "step": 514 + }, + { + "epoch": 2.22065981611682, + "grad_norm": 0.35219697520585835, + "learning_rate": 1.3079618492558763e-05, + "loss": 0.888, + "num_tokens": 4197565164.0, + "step": 515 + }, + { + "epoch": 2.224986479177934, + "grad_norm": 0.6452550918075549, + "learning_rate": 1.3055157830995904e-05, + "loss": 0.8747, + "num_tokens": 4205782442.0, + "step": 516 + }, + { + "epoch": 2.229313142239048, + "grad_norm": 0.42573649584919887, + "learning_rate": 1.3030681142899437e-05, + "loss": 0.8741, + "num_tokens": 4214093886.0, + "step": 517 + }, + { + "epoch": 2.2336398053001623, + "grad_norm": 0.5898852809349857, + "learning_rate": 1.3006188619143505e-05, + "loss": 0.9112, + "num_tokens": 4222331718.0, + "step": 518 + }, + { + "epoch": 2.2379664683612766, + "grad_norm": 0.4541426408026875, + "learning_rate": 1.2981680450725715e-05, + "loss": 0.9025, + "num_tokens": 4230459145.0, + "step": 519 + }, + { + "epoch": 2.2422931314223904, + "grad_norm": 0.5766512243496518, + "learning_rate": 1.2957156828765694e-05, + "loss": 0.868, + "num_tokens": 4238746589.0, + "step": 520 + }, + { + "epoch": 2.2466197944835047, + "grad_norm": 0.448686413568374, + "learning_rate": 1.2932617944503572e-05, + "loss": 0.8906, + "num_tokens": 4246956154.0, + "step": 521 + }, + { + "epoch": 2.2509464575446185, + "grad_norm": 0.43921170748327987, + "learning_rate": 1.2908063989298493e-05, + "loss": 0.8801, + "num_tokens": 4255214700.0, + "step": 522 + }, + { + "epoch": 2.255273120605733, + "grad_norm": 0.5005618099822559, + "learning_rate": 1.2883495154627138e-05, + "loss": 0.8904, + "num_tokens": 4263358624.0, + "step": 523 + }, + { + "epoch": 2.259599783666847, + "grad_norm": 0.41197867680738215, + "learning_rate": 1.2858911632082211e-05, + "loss": 0.9135, + "num_tokens": 4271534636.0, + "step": 524 + }, + { + "epoch": 2.263926446727961, + "grad_norm": 0.5392686281513562, + "learning_rate": 1.2834313613370966e-05, + "loss": 0.8877, + "num_tokens": 4279665673.0, + "step": 525 + }, + { + "epoch": 2.2682531097890752, + "grad_norm": 0.4429863130819827, + "learning_rate": 1.2809701290313683e-05, + "loss": 0.8956, + "num_tokens": 4287803928.0, + "step": 526 + }, + { + "epoch": 2.2725797728501895, + "grad_norm": 0.5441386068010123, + "learning_rate": 1.278507485484221e-05, + "loss": 0.8859, + "num_tokens": 4296030380.0, + "step": 527 + }, + { + "epoch": 2.2769064359113034, + "grad_norm": 0.4417689718800534, + "learning_rate": 1.2760434498998434e-05, + "loss": 0.887, + "num_tokens": 4304345627.0, + "step": 528 + }, + { + "epoch": 2.2812330989724177, + "grad_norm": 0.4605808816666947, + "learning_rate": 1.27357804149328e-05, + "loss": 0.8954, + "num_tokens": 4312432489.0, + "step": 529 + }, + { + "epoch": 2.2855597620335315, + "grad_norm": 0.40523421317548425, + "learning_rate": 1.2711112794902813e-05, + "loss": 0.8833, + "num_tokens": 4320683171.0, + "step": 530 + }, + { + "epoch": 2.289886425094646, + "grad_norm": 0.3837996969644585, + "learning_rate": 1.2686431831271523e-05, + "loss": 0.9086, + "num_tokens": 4328870977.0, + "step": 531 + }, + { + "epoch": 2.29421308815576, + "grad_norm": 0.4458009886663266, + "learning_rate": 1.2661737716506043e-05, + "loss": 0.9164, + "num_tokens": 4337133930.0, + "step": 532 + }, + { + "epoch": 2.298539751216874, + "grad_norm": 0.30634058243987183, + "learning_rate": 1.2637030643176042e-05, + "loss": 0.8976, + "num_tokens": 4345336541.0, + "step": 533 + }, + { + "epoch": 2.302866414277988, + "grad_norm": 0.47613446343318006, + "learning_rate": 1.2612310803952244e-05, + "loss": 0.8888, + "num_tokens": 4353492438.0, + "step": 534 + }, + { + "epoch": 2.307193077339102, + "grad_norm": 0.3796992331114352, + "learning_rate": 1.2587578391604913e-05, + "loss": 0.8884, + "num_tokens": 4361493269.0, + "step": 535 + }, + { + "epoch": 2.3115197404002163, + "grad_norm": 0.39010998592625656, + "learning_rate": 1.2562833599002376e-05, + "loss": 0.9032, + "num_tokens": 4369684563.0, + "step": 536 + }, + { + "epoch": 2.3158464034613306, + "grad_norm": 0.45201164324424087, + "learning_rate": 1.2538076619109492e-05, + "loss": 0.8911, + "num_tokens": 4377762651.0, + "step": 537 + }, + { + "epoch": 2.3201730665224445, + "grad_norm": 0.35743872217910533, + "learning_rate": 1.2513307644986165e-05, + "loss": 0.8742, + "num_tokens": 4385903905.0, + "step": 538 + }, + { + "epoch": 2.3244997295835588, + "grad_norm": 0.4210282530591209, + "learning_rate": 1.2488526869785831e-05, + "loss": 0.905, + "num_tokens": 4394126116.0, + "step": 539 + }, + { + "epoch": 2.3288263926446726, + "grad_norm": 0.3007621884507567, + "learning_rate": 1.2463734486753953e-05, + "loss": 0.9164, + "num_tokens": 4402377429.0, + "step": 540 + }, + { + "epoch": 2.333153055705787, + "grad_norm": 0.378423074269168, + "learning_rate": 1.2438930689226516e-05, + "loss": 0.9041, + "num_tokens": 4410674177.0, + "step": 541 + }, + { + "epoch": 2.337479718766901, + "grad_norm": 0.33382032370453096, + "learning_rate": 1.241411567062851e-05, + "loss": 0.9135, + "num_tokens": 4418722831.0, + "step": 542 + }, + { + "epoch": 2.341806381828015, + "grad_norm": 0.3536677689410112, + "learning_rate": 1.2389289624472443e-05, + "loss": 0.8922, + "num_tokens": 4426969902.0, + "step": 543 + }, + { + "epoch": 2.3461330448891293, + "grad_norm": 0.37160832203179445, + "learning_rate": 1.2364452744356803e-05, + "loss": 0.922, + "num_tokens": 4435284177.0, + "step": 544 + }, + { + "epoch": 2.350459707950243, + "grad_norm": 0.4268232421691109, + "learning_rate": 1.2339605223964571e-05, + "loss": 0.892, + "num_tokens": 4443545910.0, + "step": 545 + }, + { + "epoch": 2.3547863710113575, + "grad_norm": 0.3503498221352052, + "learning_rate": 1.2314747257061705e-05, + "loss": 0.9006, + "num_tokens": 4451722923.0, + "step": 546 + }, + { + "epoch": 2.3591130340724717, + "grad_norm": 0.37290534806009157, + "learning_rate": 1.2289879037495626e-05, + "loss": 0.9024, + "num_tokens": 4460046124.0, + "step": 547 + }, + { + "epoch": 2.3634396971335856, + "grad_norm": 0.4316845985445569, + "learning_rate": 1.22650007591937e-05, + "loss": 0.9144, + "num_tokens": 4468236197.0, + "step": 548 + }, + { + "epoch": 2.3677663601947, + "grad_norm": 0.317584498993522, + "learning_rate": 1.2240112616161743e-05, + "loss": 0.8767, + "num_tokens": 4476467540.0, + "step": 549 + }, + { + "epoch": 2.3720930232558137, + "grad_norm": 0.4637631181993098, + "learning_rate": 1.2215214802482493e-05, + "loss": 0.8802, + "num_tokens": 4484673065.0, + "step": 550 + }, + { + "epoch": 2.376419686316928, + "grad_norm": 0.33738170986416594, + "learning_rate": 1.2190307512314104e-05, + "loss": 0.8917, + "num_tokens": 4492871729.0, + "step": 551 + }, + { + "epoch": 2.3807463493780423, + "grad_norm": 0.4270069173682444, + "learning_rate": 1.2165390939888622e-05, + "loss": 0.9035, + "num_tokens": 4501066221.0, + "step": 552 + }, + { + "epoch": 2.385073012439156, + "grad_norm": 0.34789430123336185, + "learning_rate": 1.2140465279510494e-05, + "loss": 0.9326, + "num_tokens": 4509160581.0, + "step": 553 + }, + { + "epoch": 2.3893996755002704, + "grad_norm": 0.32081847830938576, + "learning_rate": 1.2115530725555016e-05, + "loss": 0.9163, + "num_tokens": 4517344308.0, + "step": 554 + }, + { + "epoch": 2.3937263385613847, + "grad_norm": 0.33444118047928384, + "learning_rate": 1.2090587472466857e-05, + "loss": 0.8794, + "num_tokens": 4525618071.0, + "step": 555 + }, + { + "epoch": 2.3980530016224986, + "grad_norm": 0.2992085906551059, + "learning_rate": 1.2065635714758513e-05, + "loss": 0.8956, + "num_tokens": 4533941114.0, + "step": 556 + }, + { + "epoch": 2.402379664683613, + "grad_norm": 0.3203712666549659, + "learning_rate": 1.2040675647008796e-05, + "loss": 0.8887, + "num_tokens": 4541971697.0, + "step": 557 + }, + { + "epoch": 2.4067063277447267, + "grad_norm": 0.3451809680005225, + "learning_rate": 1.2015707463861334e-05, + "loss": 0.9004, + "num_tokens": 4549997841.0, + "step": 558 + }, + { + "epoch": 2.411032990805841, + "grad_norm": 0.31267383852413533, + "learning_rate": 1.199073136002304e-05, + "loss": 0.9222, + "num_tokens": 4558137326.0, + "step": 559 + }, + { + "epoch": 2.4153596538669553, + "grad_norm": 0.29805430871778626, + "learning_rate": 1.1965747530262581e-05, + "loss": 0.9212, + "num_tokens": 4566374347.0, + "step": 560 + }, + { + "epoch": 2.419686316928069, + "grad_norm": 0.31898258772400107, + "learning_rate": 1.1940756169408882e-05, + "loss": 0.9119, + "num_tokens": 4574648491.0, + "step": 561 + }, + { + "epoch": 2.4240129799891834, + "grad_norm": 0.3627168888485216, + "learning_rate": 1.1915757472349598e-05, + "loss": 0.9009, + "num_tokens": 4582658063.0, + "step": 562 + }, + { + "epoch": 2.4283396430502977, + "grad_norm": 0.3553066552157547, + "learning_rate": 1.1890751634029586e-05, + "loss": 0.9042, + "num_tokens": 4590774858.0, + "step": 563 + }, + { + "epoch": 2.4326663061114115, + "grad_norm": 0.35422932275188285, + "learning_rate": 1.18657388494494e-05, + "loss": 0.8772, + "num_tokens": 4599017274.0, + "step": 564 + }, + { + "epoch": 2.436992969172526, + "grad_norm": 0.35203129069473865, + "learning_rate": 1.1840719313663758e-05, + "loss": 0.9115, + "num_tokens": 4607135318.0, + "step": 565 + }, + { + "epoch": 2.4413196322336397, + "grad_norm": 0.31112640843269307, + "learning_rate": 1.1815693221780024e-05, + "loss": 0.9098, + "num_tokens": 4615474613.0, + "step": 566 + }, + { + "epoch": 2.445646295294754, + "grad_norm": 0.39983072727217445, + "learning_rate": 1.1790660768956692e-05, + "loss": 0.9021, + "num_tokens": 4623551232.0, + "step": 567 + }, + { + "epoch": 2.4499729583558683, + "grad_norm": 0.3359276704304084, + "learning_rate": 1.1765622150401855e-05, + "loss": 0.9131, + "num_tokens": 4631791745.0, + "step": 568 + }, + { + "epoch": 2.454299621416982, + "grad_norm": 0.30658787934757037, + "learning_rate": 1.1740577561371692e-05, + "loss": 0.8809, + "num_tokens": 4639925251.0, + "step": 569 + }, + { + "epoch": 2.4586262844780964, + "grad_norm": 0.33029895890907185, + "learning_rate": 1.1715527197168938e-05, + "loss": 0.8888, + "num_tokens": 4648195859.0, + "step": 570 + }, + { + "epoch": 2.4629529475392102, + "grad_norm": 0.3404940914638666, + "learning_rate": 1.1690471253141368e-05, + "loss": 0.8929, + "num_tokens": 4656405792.0, + "step": 571 + }, + { + "epoch": 2.4672796106003245, + "grad_norm": 0.32079939149081554, + "learning_rate": 1.1665409924680266e-05, + "loss": 0.9207, + "num_tokens": 4664557465.0, + "step": 572 + }, + { + "epoch": 2.471606273661439, + "grad_norm": 0.3425408462447035, + "learning_rate": 1.1640343407218904e-05, + "loss": 0.8946, + "num_tokens": 4672603475.0, + "step": 573 + }, + { + "epoch": 2.4759329367225527, + "grad_norm": 0.32001961470934287, + "learning_rate": 1.1615271896231019e-05, + "loss": 0.8988, + "num_tokens": 4680703421.0, + "step": 574 + }, + { + "epoch": 2.480259599783667, + "grad_norm": 0.3202839765016672, + "learning_rate": 1.1590195587229297e-05, + "loss": 0.9185, + "num_tokens": 4689056017.0, + "step": 575 + }, + { + "epoch": 2.484586262844781, + "grad_norm": 0.29722760097055645, + "learning_rate": 1.1565114675763823e-05, + "loss": 0.8975, + "num_tokens": 4697063429.0, + "step": 576 + }, + { + "epoch": 2.488912925905895, + "grad_norm": 0.30856318584000425, + "learning_rate": 1.1540029357420588e-05, + "loss": 0.8808, + "num_tokens": 4705239026.0, + "step": 577 + }, + { + "epoch": 2.4932395889670094, + "grad_norm": 0.38882569269149825, + "learning_rate": 1.1514939827819945e-05, + "loss": 0.9082, + "num_tokens": 4713384759.0, + "step": 578 + }, + { + "epoch": 2.497566252028123, + "grad_norm": 0.40139771927488926, + "learning_rate": 1.1489846282615083e-05, + "loss": 0.8829, + "num_tokens": 4721626785.0, + "step": 579 + }, + { + "epoch": 2.5018929150892375, + "grad_norm": 0.34335350306399, + "learning_rate": 1.1464748917490512e-05, + "loss": 0.914, + "num_tokens": 4729834275.0, + "step": 580 + }, + { + "epoch": 2.5062195781503513, + "grad_norm": 0.4753715822388907, + "learning_rate": 1.1439647928160523e-05, + "loss": 0.899, + "num_tokens": 4738027261.0, + "step": 581 + }, + { + "epoch": 2.5105462412114656, + "grad_norm": 0.3442168740085722, + "learning_rate": 1.1414543510367673e-05, + "loss": 0.895, + "num_tokens": 4746277623.0, + "step": 582 + }, + { + "epoch": 2.51487290427258, + "grad_norm": 0.4069832614615236, + "learning_rate": 1.1389435859881255e-05, + "loss": 0.8906, + "num_tokens": 4754421101.0, + "step": 583 + }, + { + "epoch": 2.5191995673336938, + "grad_norm": 0.35977176719889054, + "learning_rate": 1.1364325172495773e-05, + "loss": 0.8888, + "num_tokens": 4762547901.0, + "step": 584 + }, + { + "epoch": 2.523526230394808, + "grad_norm": 0.4252958801538233, + "learning_rate": 1.1339211644029412e-05, + "loss": 0.8722, + "num_tokens": 4770798207.0, + "step": 585 + }, + { + "epoch": 2.527852893455922, + "grad_norm": 0.30316577547639456, + "learning_rate": 1.1314095470322512e-05, + "loss": 0.8595, + "num_tokens": 4778977049.0, + "step": 586 + }, + { + "epoch": 2.532179556517036, + "grad_norm": 0.4455349067338303, + "learning_rate": 1.1288976847236034e-05, + "loss": 0.8894, + "num_tokens": 4787227025.0, + "step": 587 + }, + { + "epoch": 2.5365062195781505, + "grad_norm": 0.29602450144689696, + "learning_rate": 1.1263855970650058e-05, + "loss": 0.9132, + "num_tokens": 4795283248.0, + "step": 588 + }, + { + "epoch": 2.5408328826392643, + "grad_norm": 0.34383847177388316, + "learning_rate": 1.1238733036462215e-05, + "loss": 0.9156, + "num_tokens": 4803501308.0, + "step": 589 + }, + { + "epoch": 2.5451595457003786, + "grad_norm": 0.3723835981546577, + "learning_rate": 1.1213608240586201e-05, + "loss": 0.8958, + "num_tokens": 4811710416.0, + "step": 590 + }, + { + "epoch": 2.5494862087614925, + "grad_norm": 0.3367284113081445, + "learning_rate": 1.1188481778950214e-05, + "loss": 0.8901, + "num_tokens": 4819731488.0, + "step": 591 + }, + { + "epoch": 2.5538128718226067, + "grad_norm": 0.31794500304475976, + "learning_rate": 1.1163353847495455e-05, + "loss": 0.8693, + "num_tokens": 4827981453.0, + "step": 592 + }, + { + "epoch": 2.558139534883721, + "grad_norm": 0.3425376938786226, + "learning_rate": 1.1138224642174578e-05, + "loss": 0.8858, + "num_tokens": 4836186521.0, + "step": 593 + }, + { + "epoch": 2.5624661979448353, + "grad_norm": 0.28500836440746563, + "learning_rate": 1.1113094358950177e-05, + "loss": 0.9027, + "num_tokens": 4844286515.0, + "step": 594 + }, + { + "epoch": 2.566792861005949, + "grad_norm": 0.30502564032613017, + "learning_rate": 1.1087963193793242e-05, + "loss": 0.885, + "num_tokens": 4852585782.0, + "step": 595 + }, + { + "epoch": 2.571119524067063, + "grad_norm": 0.3211366240751949, + "learning_rate": 1.1062831342681655e-05, + "loss": 0.8729, + "num_tokens": 4860742609.0, + "step": 596 + }, + { + "epoch": 2.5754461871281773, + "grad_norm": 0.29166956888421985, + "learning_rate": 1.1037699001598636e-05, + "loss": 0.8786, + "num_tokens": 4868966110.0, + "step": 597 + }, + { + "epoch": 2.5797728501892916, + "grad_norm": 0.3173866441895712, + "learning_rate": 1.1012566366531232e-05, + "loss": 0.9068, + "num_tokens": 4877218398.0, + "step": 598 + }, + { + "epoch": 2.584099513250406, + "grad_norm": 0.3807799752277812, + "learning_rate": 1.0987433633468771e-05, + "loss": 0.9093, + "num_tokens": 4885335516.0, + "step": 599 + }, + { + "epoch": 2.5884261763115197, + "grad_norm": 0.3164738700395768, + "learning_rate": 1.0962300998401367e-05, + "loss": 0.8549, + "num_tokens": 4893631383.0, + "step": 600 + }, + { + "epoch": 2.592752839372634, + "grad_norm": 0.3328602469053286, + "learning_rate": 1.0937168657318347e-05, + "loss": 0.9028, + "num_tokens": 4901565081.0, + "step": 601 + }, + { + "epoch": 2.597079502433748, + "grad_norm": 0.297551701728602, + "learning_rate": 1.091203680620676e-05, + "loss": 0.8965, + "num_tokens": 4909757173.0, + "step": 602 + }, + { + "epoch": 2.601406165494862, + "grad_norm": 0.3580233004919111, + "learning_rate": 1.0886905641049828e-05, + "loss": 0.8709, + "num_tokens": 4917882555.0, + "step": 603 + }, + { + "epoch": 2.6057328285559764, + "grad_norm": 0.28720000678541124, + "learning_rate": 1.0861775357825424e-05, + "loss": 0.8986, + "num_tokens": 4925934620.0, + "step": 604 + }, + { + "epoch": 2.6100594916170903, + "grad_norm": 0.3989709284138721, + "learning_rate": 1.0836646152504548e-05, + "loss": 0.9066, + "num_tokens": 4934185315.0, + "step": 605 + }, + { + "epoch": 2.6143861546782046, + "grad_norm": 0.3156022403153613, + "learning_rate": 1.0811518221049787e-05, + "loss": 0.8708, + "num_tokens": 4942214670.0, + "step": 606 + }, + { + "epoch": 2.6187128177393184, + "grad_norm": 0.3481613692304499, + "learning_rate": 1.0786391759413805e-05, + "loss": 0.8756, + "num_tokens": 4950206457.0, + "step": 607 + }, + { + "epoch": 2.6230394808004327, + "grad_norm": 0.34726504730568397, + "learning_rate": 1.0761266963537786e-05, + "loss": 0.9102, + "num_tokens": 4958463551.0, + "step": 608 + }, + { + "epoch": 2.627366143861547, + "grad_norm": 0.2910625169951151, + "learning_rate": 1.0736144029349947e-05, + "loss": 0.8818, + "num_tokens": 4966753648.0, + "step": 609 + }, + { + "epoch": 2.631692806922661, + "grad_norm": 0.3535519095817945, + "learning_rate": 1.0711023152763967e-05, + "loss": 0.8843, + "num_tokens": 4975030329.0, + "step": 610 + }, + { + "epoch": 2.636019469983775, + "grad_norm": 0.28849559876127817, + "learning_rate": 1.0685904529677496e-05, + "loss": 0.908, + "num_tokens": 4983247374.0, + "step": 611 + }, + { + "epoch": 2.640346133044889, + "grad_norm": 0.31903626709421223, + "learning_rate": 1.066078835597059e-05, + "loss": 0.884, + "num_tokens": 4991473168.0, + "step": 612 + }, + { + "epoch": 2.6446727961060033, + "grad_norm": 0.32930239979884796, + "learning_rate": 1.063567482750423e-05, + "loss": 0.9069, + "num_tokens": 4999694941.0, + "step": 613 + }, + { + "epoch": 2.6489994591671175, + "grad_norm": 0.34141820158955455, + "learning_rate": 1.061056414011875e-05, + "loss": 0.8774, + "num_tokens": 5007979085.0, + "step": 614 + }, + { + "epoch": 2.6533261222282314, + "grad_norm": 0.3750960694219387, + "learning_rate": 1.0585456489632328e-05, + "loss": 0.8861, + "num_tokens": 5016215816.0, + "step": 615 + }, + { + "epoch": 2.6576527852893457, + "grad_norm": 0.352619945957704, + "learning_rate": 1.0560352071839482e-05, + "loss": 0.8848, + "num_tokens": 5024521758.0, + "step": 616 + }, + { + "epoch": 2.6619794483504595, + "grad_norm": 0.33946373499788846, + "learning_rate": 1.0535251082509493e-05, + "loss": 0.8996, + "num_tokens": 5032634421.0, + "step": 617 + }, + { + "epoch": 2.666306111411574, + "grad_norm": 0.30467171790858133, + "learning_rate": 1.0510153717384922e-05, + "loss": 0.8685, + "num_tokens": 5040880486.0, + "step": 618 + }, + { + "epoch": 2.670632774472688, + "grad_norm": 0.3575369543056855, + "learning_rate": 1.0485060172180058e-05, + "loss": 0.9038, + "num_tokens": 5049149323.0, + "step": 619 + }, + { + "epoch": 2.674959437533802, + "grad_norm": 0.2960685119145039, + "learning_rate": 1.0459970642579419e-05, + "loss": 0.9053, + "num_tokens": 5057361701.0, + "step": 620 + }, + { + "epoch": 2.6792861005949162, + "grad_norm": 0.3472369133727984, + "learning_rate": 1.0434885324236182e-05, + "loss": 0.8785, + "num_tokens": 5065638682.0, + "step": 621 + }, + { + "epoch": 2.68361276365603, + "grad_norm": 0.2815076136802943, + "learning_rate": 1.040980441277071e-05, + "loss": 0.8904, + "num_tokens": 5073722049.0, + "step": 622 + }, + { + "epoch": 2.6879394267171444, + "grad_norm": 0.2866689838654004, + "learning_rate": 1.0384728103768984e-05, + "loss": 0.9035, + "num_tokens": 5081846217.0, + "step": 623 + }, + { + "epoch": 2.6922660897782587, + "grad_norm": 0.28272740601345764, + "learning_rate": 1.0359656592781102e-05, + "loss": 0.9138, + "num_tokens": 5089660242.0, + "step": 624 + }, + { + "epoch": 2.6965927528393725, + "grad_norm": 0.3264203702647893, + "learning_rate": 1.0334590075319737e-05, + "loss": 0.8966, + "num_tokens": 5097931621.0, + "step": 625 + }, + { + "epoch": 2.700919415900487, + "grad_norm": 0.28868592516003155, + "learning_rate": 1.0309528746858633e-05, + "loss": 0.8797, + "num_tokens": 5106097847.0, + "step": 626 + }, + { + "epoch": 2.7052460789616006, + "grad_norm": 0.30775509726486944, + "learning_rate": 1.0284472802831064e-05, + "loss": 0.9281, + "num_tokens": 5114398989.0, + "step": 627 + }, + { + "epoch": 2.709572742022715, + "grad_norm": 0.30031736597058745, + "learning_rate": 1.0259422438628311e-05, + "loss": 0.8936, + "num_tokens": 5122600868.0, + "step": 628 + }, + { + "epoch": 2.713899405083829, + "grad_norm": 0.2662062912262921, + "learning_rate": 1.023437784959815e-05, + "loss": 0.849, + "num_tokens": 5130885364.0, + "step": 629 + }, + { + "epoch": 2.718226068144943, + "grad_norm": 0.29840707582281184, + "learning_rate": 1.0209339231043314e-05, + "loss": 0.9203, + "num_tokens": 5138887813.0, + "step": 630 + }, + { + "epoch": 2.7225527312060573, + "grad_norm": 0.2800828156252148, + "learning_rate": 1.0184306778219982e-05, + "loss": 0.9051, + "num_tokens": 5147045471.0, + "step": 631 + }, + { + "epoch": 2.726879394267171, + "grad_norm": 0.2890616701290833, + "learning_rate": 1.0159280686336247e-05, + "loss": 0.8873, + "num_tokens": 5155159784.0, + "step": 632 + }, + { + "epoch": 2.7312060573282855, + "grad_norm": 0.3770969578330538, + "learning_rate": 1.0134261150550607e-05, + "loss": 0.9053, + "num_tokens": 5163179462.0, + "step": 633 + }, + { + "epoch": 2.7355327203893998, + "grad_norm": 0.30456565611589637, + "learning_rate": 1.0109248365970417e-05, + "loss": 0.8856, + "num_tokens": 5171319460.0, + "step": 634 + }, + { + "epoch": 2.739859383450514, + "grad_norm": 0.3086865854248741, + "learning_rate": 1.0084242527650405e-05, + "loss": 0.8703, + "num_tokens": 5179270550.0, + "step": 635 + }, + { + "epoch": 2.744186046511628, + "grad_norm": 0.27491855139001636, + "learning_rate": 1.005924383059112e-05, + "loss": 0.8811, + "num_tokens": 5187435125.0, + "step": 636 + }, + { + "epoch": 2.748512709572742, + "grad_norm": 0.27277429189566194, + "learning_rate": 1.003425246973742e-05, + "loss": 0.8791, + "num_tokens": 5195409994.0, + "step": 637 + }, + { + "epoch": 2.752839372633856, + "grad_norm": 0.28644006901483415, + "learning_rate": 1.0009268639976963e-05, + "loss": 0.9017, + "num_tokens": 5203712757.0, + "step": 638 + }, + { + "epoch": 2.7571660356949703, + "grad_norm": 0.28155823844689637, + "learning_rate": 9.984292536138667e-06, + "loss": 0.8769, + "num_tokens": 5211956495.0, + "step": 639 + }, + { + "epoch": 2.7614926987560846, + "grad_norm": 0.2696206937218685, + "learning_rate": 9.959324352991208e-06, + "loss": 0.8973, + "num_tokens": 5220023229.0, + "step": 640 + }, + { + "epoch": 2.7658193618171985, + "grad_norm": 0.2854344135320555, + "learning_rate": 9.934364285241492e-06, + "loss": 0.8903, + "num_tokens": 5228362461.0, + "step": 641 + }, + { + "epoch": 2.7701460248783127, + "grad_norm": 0.3000933803499447, + "learning_rate": 9.90941252753315e-06, + "loss": 0.8725, + "num_tokens": 5236623225.0, + "step": 642 + }, + { + "epoch": 2.7744726879394266, + "grad_norm": 0.3642772434319433, + "learning_rate": 9.884469274444985e-06, + "loss": 0.8863, + "num_tokens": 5244813013.0, + "step": 643 + }, + { + "epoch": 2.778799351000541, + "grad_norm": 0.2768661519280543, + "learning_rate": 9.859534720489512e-06, + "loss": 0.8711, + "num_tokens": 5252905538.0, + "step": 644 + }, + { + "epoch": 2.783126014061655, + "grad_norm": 0.31452774076143336, + "learning_rate": 9.834609060111379e-06, + "loss": 0.8743, + "num_tokens": 5261288078.0, + "step": 645 + }, + { + "epoch": 2.787452677122769, + "grad_norm": 0.4747032921203691, + "learning_rate": 9.809692487685897e-06, + "loss": 0.8933, + "num_tokens": 5269606148.0, + "step": 646 + }, + { + "epoch": 2.7917793401838833, + "grad_norm": 0.33078446487962, + "learning_rate": 9.784785197517508e-06, + "loss": 0.9042, + "num_tokens": 5277750171.0, + "step": 647 + }, + { + "epoch": 2.796106003244997, + "grad_norm": 0.36826853310476776, + "learning_rate": 9.75988738383826e-06, + "loss": 0.8989, + "num_tokens": 5285961333.0, + "step": 648 + }, + { + "epoch": 2.8004326663061114, + "grad_norm": 0.3123467381298433, + "learning_rate": 9.734999240806305e-06, + "loss": 0.8829, + "num_tokens": 5294182374.0, + "step": 649 + }, + { + "epoch": 2.8047593293672257, + "grad_norm": 0.3867041392177723, + "learning_rate": 9.710120962504377e-06, + "loss": 0.8755, + "num_tokens": 5302446174.0, + "step": 650 + }, + { + "epoch": 2.8090859924283396, + "grad_norm": 0.2776124073705401, + "learning_rate": 9.685252742938298e-06, + "loss": 0.8968, + "num_tokens": 5310551086.0, + "step": 651 + }, + { + "epoch": 2.813412655489454, + "grad_norm": 0.3845207479011459, + "learning_rate": 9.660394776035432e-06, + "loss": 0.9014, + "num_tokens": 5318840387.0, + "step": 652 + }, + { + "epoch": 2.8177393185505677, + "grad_norm": 0.30148956643727487, + "learning_rate": 9.635547255643203e-06, + "loss": 0.8982, + "num_tokens": 5327015192.0, + "step": 653 + }, + { + "epoch": 2.822065981611682, + "grad_norm": 0.3130559684163155, + "learning_rate": 9.610710375527561e-06, + "loss": 0.8797, + "num_tokens": 5335031135.0, + "step": 654 + }, + { + "epoch": 2.8263926446727963, + "grad_norm": 0.31472555122791535, + "learning_rate": 9.585884329371496e-06, + "loss": 0.9212, + "num_tokens": 5343224601.0, + "step": 655 + }, + { + "epoch": 2.83071930773391, + "grad_norm": 0.25020915401162147, + "learning_rate": 9.561069310773487e-06, + "loss": 0.895, + "num_tokens": 5351554352.0, + "step": 656 + }, + { + "epoch": 2.8350459707950244, + "grad_norm": 0.3523693517817847, + "learning_rate": 9.536265513246048e-06, + "loss": 0.871, + "num_tokens": 5359610835.0, + "step": 657 + }, + { + "epoch": 2.8393726338561383, + "grad_norm": 0.2944061388137856, + "learning_rate": 9.511473130214173e-06, + "loss": 0.9074, + "num_tokens": 5367791567.0, + "step": 658 + }, + { + "epoch": 2.8436992969172525, + "grad_norm": 0.3397949423205344, + "learning_rate": 9.486692355013838e-06, + "loss": 0.8604, + "num_tokens": 5375974593.0, + "step": 659 + }, + { + "epoch": 2.848025959978367, + "grad_norm": 0.34578326907119866, + "learning_rate": 9.461923380890513e-06, + "loss": 0.8914, + "num_tokens": 5384206152.0, + "step": 660 + }, + { + "epoch": 2.8523526230394807, + "grad_norm": 0.3099052068516999, + "learning_rate": 9.437166400997629e-06, + "loss": 0.9098, + "num_tokens": 5392340308.0, + "step": 661 + }, + { + "epoch": 2.856679286100595, + "grad_norm": 0.3330654169372942, + "learning_rate": 9.41242160839509e-06, + "loss": 0.881, + "num_tokens": 5400514161.0, + "step": 662 + }, + { + "epoch": 2.861005949161709, + "grad_norm": 0.3106850166387637, + "learning_rate": 9.387689196047761e-06, + "loss": 0.907, + "num_tokens": 5408657654.0, + "step": 663 + }, + { + "epoch": 2.865332612222823, + "grad_norm": 0.3574083181631956, + "learning_rate": 9.36296935682396e-06, + "loss": 0.8927, + "num_tokens": 5416776368.0, + "step": 664 + }, + { + "epoch": 2.8696592752839374, + "grad_norm": 0.290557352215357, + "learning_rate": 9.33826228349396e-06, + "loss": 0.8924, + "num_tokens": 5425086900.0, + "step": 665 + }, + { + "epoch": 2.8739859383450512, + "grad_norm": 0.28000180324733526, + "learning_rate": 9.313568168728478e-06, + "loss": 0.8935, + "num_tokens": 5433140424.0, + "step": 666 + }, + { + "epoch": 2.8783126014061655, + "grad_norm": 0.31769065711635947, + "learning_rate": 9.28888720509719e-06, + "loss": 0.8614, + "num_tokens": 5441442622.0, + "step": 667 + }, + { + "epoch": 2.8826392644672794, + "grad_norm": 0.27727088376360864, + "learning_rate": 9.264219585067197e-06, + "loss": 0.9023, + "num_tokens": 5449701935.0, + "step": 668 + }, + { + "epoch": 2.8869659275283936, + "grad_norm": 0.29101936651255117, + "learning_rate": 9.239565501001568e-06, + "loss": 0.8878, + "num_tokens": 5457715414.0, + "step": 669 + }, + { + "epoch": 2.891292590589508, + "grad_norm": 0.30312540853792236, + "learning_rate": 9.214925145157793e-06, + "loss": 0.8883, + "num_tokens": 5465935059.0, + "step": 670 + }, + { + "epoch": 2.8956192536506222, + "grad_norm": 0.28196012659966063, + "learning_rate": 9.190298709686321e-06, + "loss": 0.8802, + "num_tokens": 5474148386.0, + "step": 671 + }, + { + "epoch": 2.899945916711736, + "grad_norm": 0.3178842625956468, + "learning_rate": 9.165686386629039e-06, + "loss": 0.8917, + "num_tokens": 5482381222.0, + "step": 672 + }, + { + "epoch": 2.90427257977285, + "grad_norm": 0.27564326944519846, + "learning_rate": 9.141088367917792e-06, + "loss": 0.888, + "num_tokens": 5490531423.0, + "step": 673 + }, + { + "epoch": 2.908599242833964, + "grad_norm": 0.31474094535019453, + "learning_rate": 9.116504845372865e-06, + "loss": 0.8698, + "num_tokens": 5498609096.0, + "step": 674 + }, + { + "epoch": 2.9129259058950785, + "grad_norm": 0.3004041075359756, + "learning_rate": 9.091936010701513e-06, + "loss": 0.9076, + "num_tokens": 5506861596.0, + "step": 675 + }, + { + "epoch": 2.917252568956193, + "grad_norm": 0.2629707095047107, + "learning_rate": 9.067382055496431e-06, + "loss": 0.9039, + "num_tokens": 5514903668.0, + "step": 676 + }, + { + "epoch": 2.9215792320173066, + "grad_norm": 0.30005760556170413, + "learning_rate": 9.042843171234307e-06, + "loss": 0.9021, + "num_tokens": 5523195565.0, + "step": 677 + }, + { + "epoch": 2.925905895078421, + "grad_norm": 0.2914414454104958, + "learning_rate": 9.018319549274288e-06, + "loss": 0.8918, + "num_tokens": 5531475347.0, + "step": 678 + }, + { + "epoch": 2.9302325581395348, + "grad_norm": 0.28821264421620624, + "learning_rate": 8.993811380856496e-06, + "loss": 0.8777, + "num_tokens": 5539577790.0, + "step": 679 + }, + { + "epoch": 2.934559221200649, + "grad_norm": 0.30787424475971104, + "learning_rate": 8.969318857100564e-06, + "loss": 0.8854, + "num_tokens": 5547763430.0, + "step": 680 + }, + { + "epoch": 2.9388858842617633, + "grad_norm": 0.2636942402552585, + "learning_rate": 8.944842169004099e-06, + "loss": 0.8936, + "num_tokens": 5555947477.0, + "step": 681 + }, + { + "epoch": 2.943212547322877, + "grad_norm": 0.2974349461333662, + "learning_rate": 8.920381507441243e-06, + "loss": 0.8724, + "num_tokens": 5564189004.0, + "step": 682 + }, + { + "epoch": 2.9475392103839915, + "grad_norm": 0.3256931023562487, + "learning_rate": 8.89593706316114e-06, + "loss": 0.876, + "num_tokens": 5572530889.0, + "step": 683 + }, + { + "epoch": 2.9518658734451053, + "grad_norm": 0.3088224268802468, + "learning_rate": 8.87150902678648e-06, + "loss": 0.8951, + "num_tokens": 5580617357.0, + "step": 684 + }, + { + "epoch": 2.9561925365062196, + "grad_norm": 0.2989445166837416, + "learning_rate": 8.847097588811998e-06, + "loss": 0.8973, + "num_tokens": 5588871813.0, + "step": 685 + }, + { + "epoch": 2.960519199567334, + "grad_norm": 0.2692266807757265, + "learning_rate": 8.822702939602991e-06, + "loss": 0.8884, + "num_tokens": 5597032429.0, + "step": 686 + }, + { + "epoch": 2.9648458626284477, + "grad_norm": 0.3009280564750761, + "learning_rate": 8.798325269393837e-06, + "loss": 0.8791, + "num_tokens": 5605270382.0, + "step": 687 + }, + { + "epoch": 2.969172525689562, + "grad_norm": 0.28498661719018553, + "learning_rate": 8.773964768286496e-06, + "loss": 0.8928, + "num_tokens": 5613488862.0, + "step": 688 + }, + { + "epoch": 2.973499188750676, + "grad_norm": 0.2987351632275756, + "learning_rate": 8.749621626249064e-06, + "loss": 0.8831, + "num_tokens": 5621722714.0, + "step": 689 + }, + { + "epoch": 2.97782585181179, + "grad_norm": 0.343507112710914, + "learning_rate": 8.725296033114236e-06, + "loss": 0.8788, + "num_tokens": 5629918986.0, + "step": 690 + }, + { + "epoch": 2.9821525148729044, + "grad_norm": 0.26460061847429295, + "learning_rate": 8.700988178577887e-06, + "loss": 0.8665, + "num_tokens": 5638035878.0, + "step": 691 + }, + { + "epoch": 2.9864791779340183, + "grad_norm": 0.38885910183036854, + "learning_rate": 8.676698252197542e-06, + "loss": 0.9019, + "num_tokens": 5646264741.0, + "step": 692 + }, + { + "epoch": 2.9908058409951326, + "grad_norm": 0.3037808422160528, + "learning_rate": 8.652426443390931e-06, + "loss": 0.8835, + "num_tokens": 5654482295.0, + "step": 693 + }, + { + "epoch": 2.9951325040562464, + "grad_norm": 0.3341227066886302, + "learning_rate": 8.628172941434488e-06, + "loss": 0.8996, + "num_tokens": 5662561825.0, + "step": 694 + }, + { + "epoch": 2.9994591671173607, + "grad_norm": 0.3282512911305376, + "learning_rate": 8.603937935461901e-06, + "loss": 0.8691, + "num_tokens": 5670739658.0, + "step": 695 + }, + { + "epoch": 3.0, + "grad_norm": 0.5132604225693898, + "learning_rate": 8.57972161446261e-06, + "loss": 0.8374, + "num_tokens": 5671788117.0, + "step": 696 + }, + { + "epoch": 3.0043266630611143, + "grad_norm": 0.46571042314664285, + "learning_rate": 8.55552416728035e-06, + "loss": 0.8552, + "num_tokens": 5679907376.0, + "step": 697 + }, + { + "epoch": 3.008653326122228, + "grad_norm": 0.3342024251891964, + "learning_rate": 8.531345782611683e-06, + "loss": 0.8626, + "num_tokens": 5688098573.0, + "step": 698 + }, + { + "epoch": 3.0129799891833424, + "grad_norm": 0.3805010635796787, + "learning_rate": 8.5071866490045e-06, + "loss": 0.8645, + "num_tokens": 5696307538.0, + "step": 699 + }, + { + "epoch": 3.0173066522444563, + "grad_norm": 0.3102324497487666, + "learning_rate": 8.483046954856585e-06, + "loss": 0.8869, + "num_tokens": 5704458745.0, + "step": 700 + }, + { + "epoch": 3.0216333153055706, + "grad_norm": 0.32358120108707883, + "learning_rate": 8.458926888414112e-06, + "loss": 0.8888, + "num_tokens": 5712749212.0, + "step": 701 + }, + { + "epoch": 3.025959978366685, + "grad_norm": 0.29092581435493775, + "learning_rate": 8.434826637770217e-06, + "loss": 0.8614, + "num_tokens": 5720871990.0, + "step": 702 + }, + { + "epoch": 3.0302866414277987, + "grad_norm": 0.32479943430802005, + "learning_rate": 8.410746390863487e-06, + "loss": 0.8413, + "num_tokens": 5728998047.0, + "step": 703 + }, + { + "epoch": 3.034613304488913, + "grad_norm": 0.3009589967203895, + "learning_rate": 8.386686335476529e-06, + "loss": 0.8613, + "num_tokens": 5737061266.0, + "step": 704 + }, + { + "epoch": 3.038939967550027, + "grad_norm": 0.325288590783515, + "learning_rate": 8.362646659234485e-06, + "loss": 0.8535, + "num_tokens": 5745295628.0, + "step": 705 + }, + { + "epoch": 3.043266630611141, + "grad_norm": 0.320931169639338, + "learning_rate": 8.338627549603585e-06, + "loss": 0.8623, + "num_tokens": 5753603305.0, + "step": 706 + }, + { + "epoch": 3.0475932936722554, + "grad_norm": 0.32043179663147336, + "learning_rate": 8.314629193889658e-06, + "loss": 0.8675, + "num_tokens": 5761788608.0, + "step": 707 + }, + { + "epoch": 3.0519199567333692, + "grad_norm": 0.33943891029475853, + "learning_rate": 8.290651779236718e-06, + "loss": 0.8658, + "num_tokens": 5770006613.0, + "step": 708 + }, + { + "epoch": 3.0562466197944835, + "grad_norm": 0.30406229707992777, + "learning_rate": 8.266695492625454e-06, + "loss": 0.8873, + "num_tokens": 5778112084.0, + "step": 709 + }, + { + "epoch": 3.060573282855598, + "grad_norm": 0.35714472944774817, + "learning_rate": 8.242760520871797e-06, + "loss": 0.8777, + "num_tokens": 5786313328.0, + "step": 710 + }, + { + "epoch": 3.0648999459167117, + "grad_norm": 0.30168450003299485, + "learning_rate": 8.218847050625476e-06, + "loss": 0.857, + "num_tokens": 5794405561.0, + "step": 711 + }, + { + "epoch": 3.069226608977826, + "grad_norm": 0.31518748220458154, + "learning_rate": 8.194955268368526e-06, + "loss": 0.8665, + "num_tokens": 5802547640.0, + "step": 712 + }, + { + "epoch": 3.07355327203894, + "grad_norm": 0.3147924239111273, + "learning_rate": 8.171085360413867e-06, + "loss": 0.8765, + "num_tokens": 5810681438.0, + "step": 713 + }, + { + "epoch": 3.077879935100054, + "grad_norm": 0.2849806500120854, + "learning_rate": 8.147237512903834e-06, + "loss": 0.8566, + "num_tokens": 5818654130.0, + "step": 714 + }, + { + "epoch": 3.0822065981611684, + "grad_norm": 0.29042499603601274, + "learning_rate": 8.12341191180874e-06, + "loss": 0.8684, + "num_tokens": 5826922460.0, + "step": 715 + }, + { + "epoch": 3.086533261222282, + "grad_norm": 0.26696439211759415, + "learning_rate": 8.099608742925403e-06, + "loss": 0.8654, + "num_tokens": 5835151883.0, + "step": 716 + }, + { + "epoch": 3.0908599242833965, + "grad_norm": 0.28094391640838656, + "learning_rate": 8.075828191875714e-06, + "loss": 0.872, + "num_tokens": 5843263950.0, + "step": 717 + }, + { + "epoch": 3.0951865873445104, + "grad_norm": 0.2656584855668805, + "learning_rate": 8.052070444105188e-06, + "loss": 0.8536, + "num_tokens": 5851439557.0, + "step": 718 + }, + { + "epoch": 3.0995132504056246, + "grad_norm": 0.2945175948743297, + "learning_rate": 8.028335684881517e-06, + "loss": 0.8699, + "num_tokens": 5859515896.0, + "step": 719 + }, + { + "epoch": 3.103839913466739, + "grad_norm": 0.26691367581230224, + "learning_rate": 8.00462409929312e-06, + "loss": 0.8496, + "num_tokens": 5867765162.0, + "step": 720 + }, + { + "epoch": 3.1081665765278528, + "grad_norm": 0.2998684136951534, + "learning_rate": 7.980935872247706e-06, + "loss": 0.8456, + "num_tokens": 5875913424.0, + "step": 721 + }, + { + "epoch": 3.112493239588967, + "grad_norm": 0.26583765858547925, + "learning_rate": 7.957271188470828e-06, + "loss": 0.8694, + "num_tokens": 5884271974.0, + "step": 722 + }, + { + "epoch": 3.1168199026500814, + "grad_norm": 0.30237831485637084, + "learning_rate": 7.933630232504437e-06, + "loss": 0.877, + "num_tokens": 5892387066.0, + "step": 723 + }, + { + "epoch": 3.121146565711195, + "grad_norm": 0.2864329930462281, + "learning_rate": 7.910013188705464e-06, + "loss": 0.8733, + "num_tokens": 5900624981.0, + "step": 724 + }, + { + "epoch": 3.1254732287723095, + "grad_norm": 0.2715384888380705, + "learning_rate": 7.88642024124436e-06, + "loss": 0.8825, + "num_tokens": 5908904823.0, + "step": 725 + }, + { + "epoch": 3.1297998918334233, + "grad_norm": 0.311681470639809, + "learning_rate": 7.862851574103665e-06, + "loss": 0.8775, + "num_tokens": 5916758246.0, + "step": 726 + }, + { + "epoch": 3.1341265548945376, + "grad_norm": 0.2563445781326313, + "learning_rate": 7.839307371076581e-06, + "loss": 0.8714, + "num_tokens": 5924835719.0, + "step": 727 + }, + { + "epoch": 3.138453217955652, + "grad_norm": 0.2802012921576202, + "learning_rate": 7.815787815765536e-06, + "loss": 0.8928, + "num_tokens": 5933215022.0, + "step": 728 + }, + { + "epoch": 3.1427798810167658, + "grad_norm": 0.25503455234753647, + "learning_rate": 7.792293091580746e-06, + "loss": 0.8888, + "num_tokens": 5941428338.0, + "step": 729 + }, + { + "epoch": 3.14710654407788, + "grad_norm": 0.32595121514073616, + "learning_rate": 7.768823381738786e-06, + "loss": 0.8589, + "num_tokens": 5949505495.0, + "step": 730 + }, + { + "epoch": 3.151433207138994, + "grad_norm": 0.27274196555510377, + "learning_rate": 7.74537886926117e-06, + "loss": 0.8642, + "num_tokens": 5957735147.0, + "step": 731 + }, + { + "epoch": 3.155759870200108, + "grad_norm": 0.2789944149889373, + "learning_rate": 7.721959736972918e-06, + "loss": 0.8903, + "num_tokens": 5965961603.0, + "step": 732 + }, + { + "epoch": 3.1600865332612225, + "grad_norm": 0.26146167942472176, + "learning_rate": 7.698566167501124e-06, + "loss": 0.8551, + "num_tokens": 5974180680.0, + "step": 733 + }, + { + "epoch": 3.1644131963223363, + "grad_norm": 0.25603147298921575, + "learning_rate": 7.675198343273546e-06, + "loss": 0.8552, + "num_tokens": 5982369604.0, + "step": 734 + }, + { + "epoch": 3.1687398593834506, + "grad_norm": 0.2855161266232447, + "learning_rate": 7.651856446517172e-06, + "loss": 0.8645, + "num_tokens": 5990516155.0, + "step": 735 + }, + { + "epoch": 3.1730665224445644, + "grad_norm": 0.3038997304425538, + "learning_rate": 7.628540659256791e-06, + "loss": 0.8826, + "num_tokens": 5998781123.0, + "step": 736 + }, + { + "epoch": 3.1773931855056787, + "grad_norm": 0.26482858588663494, + "learning_rate": 7.605251163313614e-06, + "loss": 0.874, + "num_tokens": 6006962197.0, + "step": 737 + }, + { + "epoch": 3.181719848566793, + "grad_norm": 0.2718010445462845, + "learning_rate": 7.581988140303791e-06, + "loss": 0.8665, + "num_tokens": 6015012764.0, + "step": 738 + }, + { + "epoch": 3.186046511627907, + "grad_norm": 0.27324822810863625, + "learning_rate": 7.558751771637059e-06, + "loss": 0.862, + "num_tokens": 6023064873.0, + "step": 739 + }, + { + "epoch": 3.190373174689021, + "grad_norm": 3.237581947119273, + "learning_rate": 7.535542238515285e-06, + "loss": 0.8589, + "num_tokens": 6031035576.0, + "step": 740 + }, + { + "epoch": 3.194699837750135, + "grad_norm": 0.34930824914750813, + "learning_rate": 7.512359721931075e-06, + "loss": 0.896, + "num_tokens": 6038996187.0, + "step": 741 + }, + { + "epoch": 3.1990265008112493, + "grad_norm": 0.26052001901962446, + "learning_rate": 7.489204402666344e-06, + "loss": 0.8848, + "num_tokens": 6047077307.0, + "step": 742 + }, + { + "epoch": 3.2033531638723636, + "grad_norm": 0.29239010446917674, + "learning_rate": 7.466076461290925e-06, + "loss": 0.8818, + "num_tokens": 6055424890.0, + "step": 743 + }, + { + "epoch": 3.2076798269334774, + "grad_norm": 0.2761724819978436, + "learning_rate": 7.442976078161155e-06, + "loss": 0.8706, + "num_tokens": 6063730975.0, + "step": 744 + }, + { + "epoch": 3.2120064899945917, + "grad_norm": 0.2804796057596528, + "learning_rate": 7.419903433418454e-06, + "loss": 0.8961, + "num_tokens": 6071978906.0, + "step": 745 + }, + { + "epoch": 3.2163331530557056, + "grad_norm": 0.27865836011530204, + "learning_rate": 7.396858706987948e-06, + "loss": 0.8768, + "num_tokens": 6080158959.0, + "step": 746 + }, + { + "epoch": 3.22065981611682, + "grad_norm": 0.25469964443618087, + "learning_rate": 7.373842078577038e-06, + "loss": 0.8721, + "num_tokens": 6088351610.0, + "step": 747 + }, + { + "epoch": 3.224986479177934, + "grad_norm": 0.27048905589356725, + "learning_rate": 7.350853727674019e-06, + "loss": 0.8565, + "num_tokens": 6096565394.0, + "step": 748 + }, + { + "epoch": 3.229313142239048, + "grad_norm": 0.25324063527866647, + "learning_rate": 7.327893833546666e-06, + "loss": 0.8913, + "num_tokens": 6104716930.0, + "step": 749 + }, + { + "epoch": 3.2336398053001623, + "grad_norm": 0.2803811077859664, + "learning_rate": 7.3049625752408485e-06, + "loss": 0.8717, + "num_tokens": 6112890728.0, + "step": 750 + }, + { + "epoch": 3.2379664683612766, + "grad_norm": 0.2559650504558686, + "learning_rate": 7.282060131579125e-06, + "loss": 0.8476, + "num_tokens": 6121176505.0, + "step": 751 + }, + { + "epoch": 3.2422931314223904, + "grad_norm": 0.30073692133249874, + "learning_rate": 7.259186681159349e-06, + "loss": 0.8543, + "num_tokens": 6129439002.0, + "step": 752 + }, + { + "epoch": 3.2466197944835047, + "grad_norm": 0.27823059250238474, + "learning_rate": 7.236342402353289e-06, + "loss": 0.8712, + "num_tokens": 6137620370.0, + "step": 753 + }, + { + "epoch": 3.2509464575446185, + "grad_norm": 0.29665899851410604, + "learning_rate": 7.213527473305211e-06, + "loss": 0.8739, + "num_tokens": 6145856796.0, + "step": 754 + }, + { + "epoch": 3.255273120605733, + "grad_norm": 0.30015185552293183, + "learning_rate": 7.1907420719305185e-06, + "loss": 0.8568, + "num_tokens": 6154173109.0, + "step": 755 + }, + { + "epoch": 3.259599783666847, + "grad_norm": 0.25583534620528137, + "learning_rate": 7.167986375914347e-06, + "loss": 0.8769, + "num_tokens": 6162507632.0, + "step": 756 + }, + { + "epoch": 3.263926446727961, + "grad_norm": 0.28002638199246893, + "learning_rate": 7.145260562710188e-06, + "loss": 0.8926, + "num_tokens": 6170703161.0, + "step": 757 + }, + { + "epoch": 3.2682531097890752, + "grad_norm": 0.24659664099544912, + "learning_rate": 7.1225648095384994e-06, + "loss": 0.8779, + "num_tokens": 6178954520.0, + "step": 758 + }, + { + "epoch": 3.2725797728501895, + "grad_norm": 0.2710399557896545, + "learning_rate": 7.099899293385317e-06, + "loss": 0.8812, + "num_tokens": 6187067364.0, + "step": 759 + }, + { + "epoch": 3.2769064359113034, + "grad_norm": 0.258142205593885, + "learning_rate": 7.077264191000895e-06, + "loss": 0.8765, + "num_tokens": 6195345851.0, + "step": 760 + }, + { + "epoch": 3.2812330989724177, + "grad_norm": 0.2554335029899288, + "learning_rate": 7.054659678898304e-06, + "loss": 0.8715, + "num_tokens": 6203640158.0, + "step": 761 + }, + { + "epoch": 3.2855597620335315, + "grad_norm": 0.30356204954405863, + "learning_rate": 7.032085933352075e-06, + "loss": 0.8983, + "num_tokens": 6211763964.0, + "step": 762 + }, + { + "epoch": 3.289886425094646, + "grad_norm": 0.2487852274847125, + "learning_rate": 7.0095431303968035e-06, + "loss": 0.8609, + "num_tokens": 6220014772.0, + "step": 763 + }, + { + "epoch": 3.29421308815576, + "grad_norm": 0.29502519688481116, + "learning_rate": 6.987031445825805e-06, + "loss": 0.8738, + "num_tokens": 6228248225.0, + "step": 764 + }, + { + "epoch": 3.298539751216874, + "grad_norm": 0.2594345910033793, + "learning_rate": 6.964551055189712e-06, + "loss": 0.8499, + "num_tokens": 6236342489.0, + "step": 765 + }, + { + "epoch": 3.302866414277988, + "grad_norm": 0.2915471104348337, + "learning_rate": 6.942102133795126e-06, + "loss": 0.8717, + "num_tokens": 6244415800.0, + "step": 766 + }, + { + "epoch": 3.307193077339102, + "grad_norm": 0.2771556147262842, + "learning_rate": 6.919684856703244e-06, + "loss": 0.8691, + "num_tokens": 6252502921.0, + "step": 767 + }, + { + "epoch": 3.3115197404002163, + "grad_norm": 0.2514371148450061, + "learning_rate": 6.897299398728503e-06, + "loss": 0.8824, + "num_tokens": 6260666865.0, + "step": 768 + }, + { + "epoch": 3.3158464034613306, + "grad_norm": 0.27879871569793124, + "learning_rate": 6.874945934437192e-06, + "loss": 0.8508, + "num_tokens": 6268955901.0, + "step": 769 + }, + { + "epoch": 3.3201730665224445, + "grad_norm": 0.27528454639183825, + "learning_rate": 6.852624638146123e-06, + "loss": 0.8687, + "num_tokens": 6276986279.0, + "step": 770 + }, + { + "epoch": 3.3244997295835588, + "grad_norm": 0.25582524411716656, + "learning_rate": 6.830335683921248e-06, + "loss": 0.8793, + "num_tokens": 6285269008.0, + "step": 771 + }, + { + "epoch": 3.3288263926446726, + "grad_norm": 0.25012268065695037, + "learning_rate": 6.808079245576303e-06, + "loss": 0.8859, + "num_tokens": 6293301665.0, + "step": 772 + }, + { + "epoch": 3.333153055705787, + "grad_norm": 0.2668023276937087, + "learning_rate": 6.785855496671471e-06, + "loss": 0.8855, + "num_tokens": 6301573370.0, + "step": 773 + }, + { + "epoch": 3.337479718766901, + "grad_norm": 0.28226938600147894, + "learning_rate": 6.763664610512007e-06, + "loss": 0.8457, + "num_tokens": 6309720037.0, + "step": 774 + }, + { + "epoch": 3.341806381828015, + "grad_norm": 0.27071889817078265, + "learning_rate": 6.741506760146903e-06, + "loss": 0.8689, + "num_tokens": 6317882247.0, + "step": 775 + }, + { + "epoch": 3.3461330448891293, + "grad_norm": 0.25696343033765107, + "learning_rate": 6.719382118367523e-06, + "loss": 0.8872, + "num_tokens": 6326180976.0, + "step": 776 + }, + { + "epoch": 3.350459707950243, + "grad_norm": 0.24129317914946216, + "learning_rate": 6.697290857706271e-06, + "loss": 0.8804, + "num_tokens": 6334413371.0, + "step": 777 + }, + { + "epoch": 3.3547863710113575, + "grad_norm": 0.2638457532033502, + "learning_rate": 6.675233150435234e-06, + "loss": 0.8569, + "num_tokens": 6342736129.0, + "step": 778 + }, + { + "epoch": 3.3591130340724717, + "grad_norm": 0.2717639004632756, + "learning_rate": 6.653209168564847e-06, + "loss": 0.8772, + "num_tokens": 6350884456.0, + "step": 779 + }, + { + "epoch": 3.3634396971335856, + "grad_norm": 0.24993590323974443, + "learning_rate": 6.631219083842535e-06, + "loss": 0.867, + "num_tokens": 6358963541.0, + "step": 780 + }, + { + "epoch": 3.3677663601947, + "grad_norm": 0.26173447676070083, + "learning_rate": 6.609263067751406e-06, + "loss": 0.8852, + "num_tokens": 6367137156.0, + "step": 781 + }, + { + "epoch": 3.3720930232558137, + "grad_norm": 0.2333411693047713, + "learning_rate": 6.587341291508882e-06, + "loss": 0.8594, + "num_tokens": 6375245387.0, + "step": 782 + }, + { + "epoch": 3.376419686316928, + "grad_norm": 0.24068791860551728, + "learning_rate": 6.565453926065377e-06, + "loss": 0.8569, + "num_tokens": 6383366695.0, + "step": 783 + }, + { + "epoch": 3.3807463493780423, + "grad_norm": 0.22652753425293787, + "learning_rate": 6.543601142102964e-06, + "loss": 0.8566, + "num_tokens": 6391535970.0, + "step": 784 + }, + { + "epoch": 3.385073012439156, + "grad_norm": 0.26036759949354427, + "learning_rate": 6.521783110034038e-06, + "loss": 0.8635, + "num_tokens": 6399663481.0, + "step": 785 + }, + { + "epoch": 3.3893996755002704, + "grad_norm": 0.261127078943389, + "learning_rate": 6.500000000000003e-06, + "loss": 0.8447, + "num_tokens": 6407885851.0, + "step": 786 + }, + { + "epoch": 3.3937263385613847, + "grad_norm": 0.2415423153134818, + "learning_rate": 6.478251981869919e-06, + "loss": 0.8678, + "num_tokens": 6415976374.0, + "step": 787 + }, + { + "epoch": 3.3980530016224986, + "grad_norm": 0.2452370962103664, + "learning_rate": 6.4565392252392066e-06, + "loss": 0.8604, + "num_tokens": 6424180948.0, + "step": 788 + }, + { + "epoch": 3.402379664683613, + "grad_norm": 0.2839442645594292, + "learning_rate": 6.434861899428299e-06, + "loss": 0.8678, + "num_tokens": 6432412961.0, + "step": 789 + }, + { + "epoch": 3.4067063277447267, + "grad_norm": 0.2537667574029015, + "learning_rate": 6.41322017348134e-06, + "loss": 0.884, + "num_tokens": 6440384132.0, + "step": 790 + }, + { + "epoch": 3.411032990805841, + "grad_norm": 0.31301518238606135, + "learning_rate": 6.391614216164859e-06, + "loss": 0.8953, + "num_tokens": 6448589837.0, + "step": 791 + }, + { + "epoch": 3.4153596538669553, + "grad_norm": 0.23017617381504482, + "learning_rate": 6.370044195966443e-06, + "loss": 0.8721, + "num_tokens": 6456680358.0, + "step": 792 + }, + { + "epoch": 3.419686316928069, + "grad_norm": 0.2545081225482966, + "learning_rate": 6.3485102810934495e-06, + "loss": 0.8898, + "num_tokens": 6464933820.0, + "step": 793 + }, + { + "epoch": 3.4240129799891834, + "grad_norm": 0.25607065470566615, + "learning_rate": 6.327012639471668e-06, + "loss": 0.8759, + "num_tokens": 6473109749.0, + "step": 794 + }, + { + "epoch": 3.4283396430502977, + "grad_norm": 0.2435900638297003, + "learning_rate": 6.305551438744031e-06, + "loss": 0.8761, + "num_tokens": 6481346949.0, + "step": 795 + }, + { + "epoch": 3.4326663061114115, + "grad_norm": 0.27156499733279055, + "learning_rate": 6.2841268462692894e-06, + "loss": 0.8815, + "num_tokens": 6489527978.0, + "step": 796 + }, + { + "epoch": 3.436992969172526, + "grad_norm": 0.24974081466991993, + "learning_rate": 6.262739029120721e-06, + "loss": 0.879, + "num_tokens": 6497765536.0, + "step": 797 + }, + { + "epoch": 3.4413196322336397, + "grad_norm": 0.28512251325377547, + "learning_rate": 6.241388154084815e-06, + "loss": 0.8355, + "num_tokens": 6506034667.0, + "step": 798 + }, + { + "epoch": 3.445646295294754, + "grad_norm": 0.2577085947742597, + "learning_rate": 6.220074387659993e-06, + "loss": 0.8729, + "num_tokens": 6514211607.0, + "step": 799 + }, + { + "epoch": 3.4499729583558683, + "grad_norm": 0.23948299441168738, + "learning_rate": 6.198797896055277e-06, + "loss": 0.8505, + "num_tokens": 6522418657.0, + "step": 800 + }, + { + "epoch": 3.454299621416982, + "grad_norm": 0.2632314377682141, + "learning_rate": 6.177558845189029e-06, + "loss": 0.8812, + "num_tokens": 6530597396.0, + "step": 801 + }, + { + "epoch": 3.4586262844780964, + "grad_norm": 0.23940563288517072, + "learning_rate": 6.156357400687636e-06, + "loss": 0.8754, + "num_tokens": 6538658527.0, + "step": 802 + }, + { + "epoch": 3.4629529475392102, + "grad_norm": 0.25524547959853006, + "learning_rate": 6.135193727884217e-06, + "loss": 0.8564, + "num_tokens": 6546799151.0, + "step": 803 + }, + { + "epoch": 3.4672796106003245, + "grad_norm": 0.822853012219854, + "learning_rate": 6.114067991817345e-06, + "loss": 0.871, + "num_tokens": 6554902212.0, + "step": 804 + }, + { + "epoch": 3.471606273661439, + "grad_norm": 0.37646816694051527, + "learning_rate": 6.09298035722975e-06, + "loss": 0.8872, + "num_tokens": 6562951258.0, + "step": 805 + }, + { + "epoch": 3.4759329367225527, + "grad_norm": 0.27969225784194046, + "learning_rate": 6.07193098856705e-06, + "loss": 0.8697, + "num_tokens": 6571067947.0, + "step": 806 + }, + { + "epoch": 3.480259599783667, + "grad_norm": 0.299270732602528, + "learning_rate": 6.050920049976443e-06, + "loss": 0.8734, + "num_tokens": 6579204839.0, + "step": 807 + }, + { + "epoch": 3.484586262844781, + "grad_norm": 0.2817869382303203, + "learning_rate": 6.029947705305453e-06, + "loss": 0.8552, + "num_tokens": 6587394333.0, + "step": 808 + }, + { + "epoch": 3.488912925905895, + "grad_norm": 0.25533645633798413, + "learning_rate": 6.009014118100638e-06, + "loss": 0.8619, + "num_tokens": 6595531380.0, + "step": 809 + }, + { + "epoch": 3.4932395889670094, + "grad_norm": 0.2757106986988144, + "learning_rate": 5.988119451606312e-06, + "loss": 0.8736, + "num_tokens": 6603668023.0, + "step": 810 + }, + { + "epoch": 3.497566252028123, + "grad_norm": 0.23819533166161425, + "learning_rate": 5.96726386876328e-06, + "loss": 0.8474, + "num_tokens": 6611873512.0, + "step": 811 + }, + { + "epoch": 3.5018929150892375, + "grad_norm": 0.2718684086684025, + "learning_rate": 5.946447532207571e-06, + "loss": 0.8471, + "num_tokens": 6620173912.0, + "step": 812 + }, + { + "epoch": 3.5062195781503513, + "grad_norm": 0.2492179940880177, + "learning_rate": 5.92567060426916e-06, + "loss": 0.8839, + "num_tokens": 6628431770.0, + "step": 813 + }, + { + "epoch": 3.5105462412114656, + "grad_norm": 0.2589122041948524, + "learning_rate": 5.904933246970699e-06, + "loss": 0.8584, + "num_tokens": 6636570118.0, + "step": 814 + }, + { + "epoch": 3.51487290427258, + "grad_norm": 0.24561063755619578, + "learning_rate": 5.884235622026278e-06, + "loss": 0.8694, + "num_tokens": 6644641468.0, + "step": 815 + }, + { + "epoch": 3.5191995673336938, + "grad_norm": 0.2422170344395523, + "learning_rate": 5.863577890840116e-06, + "loss": 0.8922, + "num_tokens": 6652786136.0, + "step": 816 + }, + { + "epoch": 3.523526230394808, + "grad_norm": 0.25633241978816124, + "learning_rate": 5.842960214505366e-06, + "loss": 0.8792, + "num_tokens": 6661043504.0, + "step": 817 + }, + { + "epoch": 3.527852893455922, + "grad_norm": 0.23458045701247796, + "learning_rate": 5.8223827538027974e-06, + "loss": 0.892, + "num_tokens": 6669246154.0, + "step": 818 + }, + { + "epoch": 3.532179556517036, + "grad_norm": 0.24478746089429998, + "learning_rate": 5.801845669199594e-06, + "loss": 0.8779, + "num_tokens": 6677425374.0, + "step": 819 + }, + { + "epoch": 3.5365062195781505, + "grad_norm": 0.23356051530524108, + "learning_rate": 5.781349120848057e-06, + "loss": 0.8684, + "num_tokens": 6685537605.0, + "step": 820 + }, + { + "epoch": 3.5408328826392643, + "grad_norm": 0.25578166763060434, + "learning_rate": 5.760893268584398e-06, + "loss": 0.8768, + "num_tokens": 6693722191.0, + "step": 821 + }, + { + "epoch": 3.5451595457003786, + "grad_norm": 0.24297738556688706, + "learning_rate": 5.740478271927452e-06, + "loss": 0.891, + "num_tokens": 6701918956.0, + "step": 822 + }, + { + "epoch": 3.5494862087614925, + "grad_norm": 0.23564171248748056, + "learning_rate": 5.720104290077469e-06, + "loss": 0.8617, + "num_tokens": 6710029466.0, + "step": 823 + }, + { + "epoch": 3.5538128718226067, + "grad_norm": 0.2633781290994519, + "learning_rate": 5.6997714819148534e-06, + "loss": 0.8654, + "num_tokens": 6718173687.0, + "step": 824 + }, + { + "epoch": 3.558139534883721, + "grad_norm": 0.23081286282684868, + "learning_rate": 5.679480005998923e-06, + "loss": 0.8618, + "num_tokens": 6726435102.0, + "step": 825 + }, + { + "epoch": 3.5624661979448353, + "grad_norm": 0.24010000335558374, + "learning_rate": 5.659230020566689e-06, + "loss": 0.889, + "num_tokens": 6734692268.0, + "step": 826 + }, + { + "epoch": 3.566792861005949, + "grad_norm": 0.24660166305164508, + "learning_rate": 5.639021683531598e-06, + "loss": 0.8681, + "num_tokens": 6742951944.0, + "step": 827 + }, + { + "epoch": 3.571119524067063, + "grad_norm": 0.23651779717321664, + "learning_rate": 5.618855152482334e-06, + "loss": 0.8674, + "num_tokens": 6751195202.0, + "step": 828 + }, + { + "epoch": 3.5754461871281773, + "grad_norm": 0.2648880121835487, + "learning_rate": 5.5987305846815425e-06, + "loss": 0.8818, + "num_tokens": 6759329049.0, + "step": 829 + }, + { + "epoch": 3.5797728501892916, + "grad_norm": 0.23397061173999484, + "learning_rate": 5.578648137064655e-06, + "loss": 0.8781, + "num_tokens": 6767359316.0, + "step": 830 + }, + { + "epoch": 3.584099513250406, + "grad_norm": 0.26782518944680506, + "learning_rate": 5.558607966238627e-06, + "loss": 0.8747, + "num_tokens": 6775484941.0, + "step": 831 + }, + { + "epoch": 3.5884261763115197, + "grad_norm": 0.25297771415198006, + "learning_rate": 5.5386102284807395e-06, + "loss": 0.8715, + "num_tokens": 6783629197.0, + "step": 832 + }, + { + "epoch": 3.592752839372634, + "grad_norm": 0.24344142207353725, + "learning_rate": 5.518655079737371e-06, + "loss": 0.8702, + "num_tokens": 6791822522.0, + "step": 833 + }, + { + "epoch": 3.597079502433748, + "grad_norm": 0.24866449099851073, + "learning_rate": 5.498742675622777e-06, + "loss": 0.9028, + "num_tokens": 6799891082.0, + "step": 834 + }, + { + "epoch": 3.601406165494862, + "grad_norm": 0.2309579377927138, + "learning_rate": 5.478873171417884e-06, + "loss": 0.8622, + "num_tokens": 6807929736.0, + "step": 835 + }, + { + "epoch": 3.6057328285559764, + "grad_norm": 0.25768153330484456, + "learning_rate": 5.459046722069077e-06, + "loss": 0.8808, + "num_tokens": 6816078199.0, + "step": 836 + }, + { + "epoch": 3.6100594916170903, + "grad_norm": 0.22946890852236013, + "learning_rate": 5.439263482186993e-06, + "loss": 0.8875, + "num_tokens": 6824351479.0, + "step": 837 + }, + { + "epoch": 3.6143861546782046, + "grad_norm": 0.284350627497672, + "learning_rate": 5.419523606045307e-06, + "loss": 0.8634, + "num_tokens": 6832648960.0, + "step": 838 + }, + { + "epoch": 3.6187128177393184, + "grad_norm": 0.2461254509930547, + "learning_rate": 5.399827247579543e-06, + "loss": 0.8788, + "num_tokens": 6840799187.0, + "step": 839 + }, + { + "epoch": 3.6230394808004327, + "grad_norm": 0.25376519838917316, + "learning_rate": 5.3801745603858606e-06, + "loss": 0.8698, + "num_tokens": 6849039361.0, + "step": 840 + }, + { + "epoch": 3.627366143861547, + "grad_norm": 0.264870737974905, + "learning_rate": 5.36056569771986e-06, + "loss": 0.8628, + "num_tokens": 6857202285.0, + "step": 841 + }, + { + "epoch": 3.631692806922661, + "grad_norm": 0.25854125632719377, + "learning_rate": 5.341000812495387e-06, + "loss": 0.8455, + "num_tokens": 6865442272.0, + "step": 842 + }, + { + "epoch": 3.636019469983775, + "grad_norm": 0.2714656211256253, + "learning_rate": 5.3214800572833535e-06, + "loss": 0.8883, + "num_tokens": 6873686134.0, + "step": 843 + }, + { + "epoch": 3.640346133044889, + "grad_norm": 0.24643298718947976, + "learning_rate": 5.302003584310531e-06, + "loss": 0.8671, + "num_tokens": 6881934773.0, + "step": 844 + }, + { + "epoch": 3.6446727961060033, + "grad_norm": 0.24828687409016623, + "learning_rate": 5.282571545458361e-06, + "loss": 0.8924, + "num_tokens": 6890167439.0, + "step": 845 + }, + { + "epoch": 3.6489994591671175, + "grad_norm": 0.23017624085228425, + "learning_rate": 5.263184092261793e-06, + "loss": 0.8655, + "num_tokens": 6898084286.0, + "step": 846 + }, + { + "epoch": 3.6533261222282314, + "grad_norm": 0.25068184011797584, + "learning_rate": 5.243841375908079e-06, + "loss": 0.8952, + "num_tokens": 6906136822.0, + "step": 847 + }, + { + "epoch": 3.6576527852893457, + "grad_norm": 0.22590219136010012, + "learning_rate": 5.2245435472356075e-06, + "loss": 0.8596, + "num_tokens": 6914342775.0, + "step": 848 + }, + { + "epoch": 3.6619794483504595, + "grad_norm": 0.28142059088247345, + "learning_rate": 5.205290756732717e-06, + "loss": 0.8725, + "num_tokens": 6922575948.0, + "step": 849 + }, + { + "epoch": 3.666306111411574, + "grad_norm": 0.24061764424713128, + "learning_rate": 5.186083154536545e-06, + "loss": 0.8792, + "num_tokens": 6930711473.0, + "step": 850 + }, + { + "epoch": 3.670632774472688, + "grad_norm": 0.24535210553929054, + "learning_rate": 5.166920890431822e-06, + "loss": 0.8642, + "num_tokens": 6939089933.0, + "step": 851 + }, + { + "epoch": 3.674959437533802, + "grad_norm": 0.27331874962756486, + "learning_rate": 5.147804113849739e-06, + "loss": 0.8681, + "num_tokens": 6947404283.0, + "step": 852 + }, + { + "epoch": 3.6792861005949162, + "grad_norm": 0.24193385525958766, + "learning_rate": 5.128732973866764e-06, + "loss": 0.8794, + "num_tokens": 6955647078.0, + "step": 853 + }, + { + "epoch": 3.68361276365603, + "grad_norm": 0.26035041478942234, + "learning_rate": 5.109707619203468e-06, + "loss": 0.8667, + "num_tokens": 6963899007.0, + "step": 854 + }, + { + "epoch": 3.6879394267171444, + "grad_norm": 0.24464880558869317, + "learning_rate": 5.090728198223393e-06, + "loss": 0.8627, + "num_tokens": 6972095624.0, + "step": 855 + }, + { + "epoch": 3.6922660897782587, + "grad_norm": 0.2679596293250479, + "learning_rate": 5.071794858931875e-06, + "loss": 0.8879, + "num_tokens": 6980432335.0, + "step": 856 + }, + { + "epoch": 3.6965927528393725, + "grad_norm": 0.2301512690565832, + "learning_rate": 5.052907748974902e-06, + "loss": 0.8435, + "num_tokens": 6988564397.0, + "step": 857 + }, + { + "epoch": 3.700919415900487, + "grad_norm": 0.271844027728891, + "learning_rate": 5.034067015637945e-06, + "loss": 0.8856, + "num_tokens": 6996765969.0, + "step": 858 + }, + { + "epoch": 3.7052460789616006, + "grad_norm": 0.21679771415042162, + "learning_rate": 5.015272805844829e-06, + "loss": 0.8938, + "num_tokens": 7005080433.0, + "step": 859 + }, + { + "epoch": 3.709572742022715, + "grad_norm": 0.25873549893156994, + "learning_rate": 4.996525266156582e-06, + "loss": 0.8628, + "num_tokens": 7013240989.0, + "step": 860 + }, + { + "epoch": 3.713899405083829, + "grad_norm": 0.2384884080212786, + "learning_rate": 4.977824542770279e-06, + "loss": 0.8396, + "num_tokens": 7021510989.0, + "step": 861 + }, + { + "epoch": 3.718226068144943, + "grad_norm": 0.261797066838615, + "learning_rate": 4.959170781517917e-06, + "loss": 0.8531, + "num_tokens": 7029805069.0, + "step": 862 + }, + { + "epoch": 3.7225527312060573, + "grad_norm": 0.24558072291623043, + "learning_rate": 4.940564127865276e-06, + "loss": 0.8582, + "num_tokens": 7038026239.0, + "step": 863 + }, + { + "epoch": 3.726879394267171, + "grad_norm": 0.24713485535544746, + "learning_rate": 4.92200472691078e-06, + "loss": 0.8827, + "num_tokens": 7046185960.0, + "step": 864 + }, + { + "epoch": 3.7312060573282855, + "grad_norm": 0.23845612907877414, + "learning_rate": 4.903492723384366e-06, + "loss": 0.8785, + "num_tokens": 7054363219.0, + "step": 865 + }, + { + "epoch": 3.7355327203893998, + "grad_norm": 0.22606839767231876, + "learning_rate": 4.885028261646354e-06, + "loss": 0.867, + "num_tokens": 7062407105.0, + "step": 866 + }, + { + "epoch": 3.739859383450514, + "grad_norm": 0.24424189437762311, + "learning_rate": 4.866611485686323e-06, + "loss": 0.8972, + "num_tokens": 7070565428.0, + "step": 867 + }, + { + "epoch": 3.744186046511628, + "grad_norm": 0.23995639109377395, + "learning_rate": 4.848242539121998e-06, + "loss": 0.8641, + "num_tokens": 7078632819.0, + "step": 868 + }, + { + "epoch": 3.748512709572742, + "grad_norm": 0.23424864631147535, + "learning_rate": 4.8299215651981095e-06, + "loss": 0.8847, + "num_tokens": 7086774735.0, + "step": 869 + }, + { + "epoch": 3.752839372633856, + "grad_norm": 0.24669935609823532, + "learning_rate": 4.8116487067852945e-06, + "loss": 0.8559, + "num_tokens": 7094926736.0, + "step": 870 + }, + { + "epoch": 3.7571660356949703, + "grad_norm": 0.22046021680839034, + "learning_rate": 4.793424106378972e-06, + "loss": 0.8618, + "num_tokens": 7103147358.0, + "step": 871 + }, + { + "epoch": 3.7614926987560846, + "grad_norm": 0.2395354316786138, + "learning_rate": 4.77524790609824e-06, + "loss": 0.8883, + "num_tokens": 7111357604.0, + "step": 872 + }, + { + "epoch": 3.7658193618171985, + "grad_norm": 0.2365151865427725, + "learning_rate": 4.7571202476847575e-06, + "loss": 0.8487, + "num_tokens": 7119470672.0, + "step": 873 + }, + { + "epoch": 3.7701460248783127, + "grad_norm": 0.25387276765750244, + "learning_rate": 4.739041272501643e-06, + "loss": 0.8495, + "num_tokens": 7127678886.0, + "step": 874 + }, + { + "epoch": 3.7744726879394266, + "grad_norm": 0.243430748812654, + "learning_rate": 4.721011121532384e-06, + "loss": 0.8786, + "num_tokens": 7135834358.0, + "step": 875 + }, + { + "epoch": 3.778799351000541, + "grad_norm": 0.2253824574379863, + "learning_rate": 4.703029935379711e-06, + "loss": 0.8885, + "num_tokens": 7144080341.0, + "step": 876 + }, + { + "epoch": 3.783126014061655, + "grad_norm": 0.25595198805615155, + "learning_rate": 4.685097854264535e-06, + "loss": 0.8787, + "num_tokens": 7152048641.0, + "step": 877 + }, + { + "epoch": 3.787452677122769, + "grad_norm": 0.2151870609226196, + "learning_rate": 4.66721501802482e-06, + "loss": 0.8502, + "num_tokens": 7160159340.0, + "step": 878 + }, + { + "epoch": 3.7917793401838833, + "grad_norm": 0.23498172585247432, + "learning_rate": 4.649381566114517e-06, + "loss": 0.8562, + "num_tokens": 7168386764.0, + "step": 879 + }, + { + "epoch": 3.796106003244997, + "grad_norm": 0.2202317264271157, + "learning_rate": 4.631597637602465e-06, + "loss": 0.8339, + "num_tokens": 7176392255.0, + "step": 880 + }, + { + "epoch": 3.8004326663061114, + "grad_norm": 0.22348936099151778, + "learning_rate": 4.613863371171314e-06, + "loss": 0.8828, + "num_tokens": 7184640858.0, + "step": 881 + }, + { + "epoch": 3.8047593293672257, + "grad_norm": 0.2320891008196228, + "learning_rate": 4.5961789051164325e-06, + "loss": 0.873, + "num_tokens": 7192875182.0, + "step": 882 + }, + { + "epoch": 3.8090859924283396, + "grad_norm": 0.21780432331671123, + "learning_rate": 4.578544377344841e-06, + "loss": 0.8538, + "num_tokens": 7201076591.0, + "step": 883 + }, + { + "epoch": 3.813412655489454, + "grad_norm": 0.2229548120930206, + "learning_rate": 4.560959925374133e-06, + "loss": 0.8658, + "num_tokens": 7209160410.0, + "step": 884 + }, + { + "epoch": 3.8177393185505677, + "grad_norm": 0.2204686154860945, + "learning_rate": 4.543425686331394e-06, + "loss": 0.8558, + "num_tokens": 7217251112.0, + "step": 885 + }, + { + "epoch": 3.822065981611682, + "grad_norm": 0.21889738307844428, + "learning_rate": 4.525941796952142e-06, + "loss": 0.9103, + "num_tokens": 7225589569.0, + "step": 886 + }, + { + "epoch": 3.8263926446727963, + "grad_norm": 0.22195229341976094, + "learning_rate": 4.5085083935792566e-06, + "loss": 0.8746, + "num_tokens": 7233839255.0, + "step": 887 + }, + { + "epoch": 3.83071930773391, + "grad_norm": 0.22203328389903487, + "learning_rate": 4.491125612161924e-06, + "loss": 0.8675, + "num_tokens": 7242039779.0, + "step": 888 + }, + { + "epoch": 3.8350459707950244, + "grad_norm": 0.23433836266559419, + "learning_rate": 4.47379358825456e-06, + "loss": 0.8829, + "num_tokens": 7250299144.0, + "step": 889 + }, + { + "epoch": 3.8393726338561383, + "grad_norm": 0.22003609539379673, + "learning_rate": 4.456512457015775e-06, + "loss": 0.8776, + "num_tokens": 7258605561.0, + "step": 890 + }, + { + "epoch": 3.8436992969172525, + "grad_norm": 0.23521493912813957, + "learning_rate": 4.4392823532072984e-06, + "loss": 0.9054, + "num_tokens": 7266742349.0, + "step": 891 + }, + { + "epoch": 3.848025959978367, + "grad_norm": 0.22815542527706334, + "learning_rate": 4.422103411192941e-06, + "loss": 0.8749, + "num_tokens": 7274939437.0, + "step": 892 + }, + { + "epoch": 3.8523526230394807, + "grad_norm": 0.22310220098274597, + "learning_rate": 4.404975764937541e-06, + "loss": 0.8784, + "num_tokens": 7283217746.0, + "step": 893 + }, + { + "epoch": 3.856679286100595, + "grad_norm": 0.23665733625687027, + "learning_rate": 4.387899548005927e-06, + "loss": 0.8827, + "num_tokens": 7291342620.0, + "step": 894 + }, + { + "epoch": 3.861005949161709, + "grad_norm": 0.2357274515626233, + "learning_rate": 4.370874893561872e-06, + "loss": 0.8923, + "num_tokens": 7299513806.0, + "step": 895 + }, + { + "epoch": 3.865332612222823, + "grad_norm": 0.24194775225375628, + "learning_rate": 4.353901934367045e-06, + "loss": 0.8684, + "num_tokens": 7307509871.0, + "step": 896 + }, + { + "epoch": 3.8696592752839374, + "grad_norm": 0.21625785056749258, + "learning_rate": 4.336980802779998e-06, + "loss": 0.8736, + "num_tokens": 7315632369.0, + "step": 897 + }, + { + "epoch": 3.8739859383450512, + "grad_norm": 0.2376011450614481, + "learning_rate": 4.320111630755109e-06, + "loss": 0.8825, + "num_tokens": 7323748995.0, + "step": 898 + }, + { + "epoch": 3.8783126014061655, + "grad_norm": 0.22914479254242603, + "learning_rate": 4.303294549841573e-06, + "loss": 0.8979, + "num_tokens": 7331960561.0, + "step": 899 + }, + { + "epoch": 3.8826392644672794, + "grad_norm": 0.24273658078009602, + "learning_rate": 4.286529691182362e-06, + "loss": 0.9077, + "num_tokens": 7340214994.0, + "step": 900 + }, + { + "epoch": 3.8869659275283936, + "grad_norm": 0.22765819958364839, + "learning_rate": 4.269817185513215e-06, + "loss": 0.8751, + "num_tokens": 7348363099.0, + "step": 901 + }, + { + "epoch": 3.891292590589508, + "grad_norm": 0.2412190604210739, + "learning_rate": 4.253157163161605e-06, + "loss": 0.8409, + "num_tokens": 7356665002.0, + "step": 902 + }, + { + "epoch": 3.8956192536506222, + "grad_norm": 0.23267678662848995, + "learning_rate": 4.236549754045737e-06, + "loss": 0.8513, + "num_tokens": 7364879198.0, + "step": 903 + }, + { + "epoch": 3.899945916711736, + "grad_norm": 0.24617821294263423, + "learning_rate": 4.2199950876735215e-06, + "loss": 0.8773, + "num_tokens": 7373070906.0, + "step": 904 + }, + { + "epoch": 3.90427257977285, + "grad_norm": 0.25099801574181024, + "learning_rate": 4.203493293141569e-06, + "loss": 0.8891, + "num_tokens": 7381382956.0, + "step": 905 + }, + { + "epoch": 3.908599242833964, + "grad_norm": 0.24621890662823137, + "learning_rate": 4.187044499134194e-06, + "loss": 0.87, + "num_tokens": 7389566035.0, + "step": 906 + }, + { + "epoch": 3.9129259058950785, + "grad_norm": 0.2450439123209906, + "learning_rate": 4.170648833922391e-06, + "loss": 0.8671, + "num_tokens": 7397828026.0, + "step": 907 + }, + { + "epoch": 3.917252568956193, + "grad_norm": 0.24148342943530646, + "learning_rate": 4.154306425362856e-06, + "loss": 0.8529, + "num_tokens": 7406033799.0, + "step": 908 + }, + { + "epoch": 3.9215792320173066, + "grad_norm": 0.25508465052024193, + "learning_rate": 4.1380174008969685e-06, + "loss": 0.8635, + "num_tokens": 7414155897.0, + "step": 909 + }, + { + "epoch": 3.925905895078421, + "grad_norm": 0.22308785083920818, + "learning_rate": 4.121781887549819e-06, + "loss": 0.8965, + "num_tokens": 7422421950.0, + "step": 910 + }, + { + "epoch": 3.9302325581395348, + "grad_norm": 0.21770015216002292, + "learning_rate": 4.1056000119292e-06, + "loss": 0.8514, + "num_tokens": 7430613952.0, + "step": 911 + }, + { + "epoch": 3.934559221200649, + "grad_norm": 0.2266454296767021, + "learning_rate": 4.089471900224625e-06, + "loss": 0.8966, + "num_tokens": 7438787578.0, + "step": 912 + }, + { + "epoch": 3.9388858842617633, + "grad_norm": 0.22380927100300407, + "learning_rate": 4.07339767820635e-06, + "loss": 0.8926, + "num_tokens": 7447030738.0, + "step": 913 + }, + { + "epoch": 3.943212547322877, + "grad_norm": 0.2424857416532719, + "learning_rate": 4.057377471224389e-06, + "loss": 0.8675, + "num_tokens": 7455115841.0, + "step": 914 + }, + { + "epoch": 3.9475392103839915, + "grad_norm": 0.4561860168642841, + "learning_rate": 4.041411404207534e-06, + "loss": 0.8495, + "num_tokens": 7463324922.0, + "step": 915 + }, + { + "epoch": 3.9518658734451053, + "grad_norm": 0.2447660248000196, + "learning_rate": 4.025499601662385e-06, + "loss": 0.8627, + "num_tokens": 7471433641.0, + "step": 916 + }, + { + "epoch": 3.9561925365062196, + "grad_norm": 0.22717301609727106, + "learning_rate": 4.009642187672371e-06, + "loss": 0.8625, + "num_tokens": 7479647760.0, + "step": 917 + }, + { + "epoch": 3.960519199567334, + "grad_norm": 0.23146279567894976, + "learning_rate": 3.99383928589679e-06, + "loss": 0.851, + "num_tokens": 7487931251.0, + "step": 918 + }, + { + "epoch": 3.9648458626284477, + "grad_norm": 0.2227424680999003, + "learning_rate": 3.9780910195698505e-06, + "loss": 0.8407, + "num_tokens": 7496101225.0, + "step": 919 + }, + { + "epoch": 3.969172525689562, + "grad_norm": 0.22760470701948257, + "learning_rate": 3.9623975114996905e-06, + "loss": 0.8811, + "num_tokens": 7504063654.0, + "step": 920 + }, + { + "epoch": 3.973499188750676, + "grad_norm": 0.21794459346072656, + "learning_rate": 3.946758884067443e-06, + "loss": 0.8838, + "num_tokens": 7512165076.0, + "step": 921 + }, + { + "epoch": 3.97782585181179, + "grad_norm": 0.21058550976215654, + "learning_rate": 3.9311752592262636e-06, + "loss": 0.8794, + "num_tokens": 7520168097.0, + "step": 922 + }, + { + "epoch": 3.9821525148729044, + "grad_norm": 0.2210143730720937, + "learning_rate": 3.915646758500391e-06, + "loss": 0.8799, + "num_tokens": 7528384996.0, + "step": 923 + }, + { + "epoch": 3.9864791779340183, + "grad_norm": 0.21168643720099292, + "learning_rate": 3.900173502984195e-06, + "loss": 0.8644, + "num_tokens": 7536667574.0, + "step": 924 + }, + { + "epoch": 3.9908058409951326, + "grad_norm": 0.2257362665475205, + "learning_rate": 3.884755613341223e-06, + "loss": 0.8691, + "num_tokens": 7544881945.0, + "step": 925 + }, + { + "epoch": 3.9951325040562464, + "grad_norm": 0.21540201264645625, + "learning_rate": 3.8693932098032845e-06, + "loss": 0.8738, + "num_tokens": 7553115253.0, + "step": 926 + }, + { + "epoch": 3.9994591671173607, + "grad_norm": 0.2220835897447074, + "learning_rate": 3.854086412169482e-06, + "loss": 0.8607, + "num_tokens": 7561403316.0, + "step": 927 + }, + { + "epoch": 4.0, + "grad_norm": 0.5180877481536112, + "learning_rate": 3.838835339805301e-06, + "loss": 0.8743, + "num_tokens": 7562397899.0, + "step": 928 + }, + { + "epoch": 4.004326663061114, + "grad_norm": 0.25591003328303946, + "learning_rate": 3.8236401116416686e-06, + "loss": 0.8683, + "num_tokens": 7570694341.0, + "step": 929 + }, + { + "epoch": 4.008653326122229, + "grad_norm": 0.23130299920493375, + "learning_rate": 3.8085008461740245e-06, + "loss": 0.8518, + "num_tokens": 7578913010.0, + "step": 930 + }, + { + "epoch": 4.012979989183342, + "grad_norm": 0.21942409398345594, + "learning_rate": 3.7934176614614004e-06, + "loss": 0.8574, + "num_tokens": 7587176043.0, + "step": 931 + }, + { + "epoch": 4.017306652244456, + "grad_norm": 0.23602893651738854, + "learning_rate": 3.778390675125503e-06, + "loss": 0.88, + "num_tokens": 7595361150.0, + "step": 932 + }, + { + "epoch": 4.0216333153055706, + "grad_norm": 0.22963261312250266, + "learning_rate": 3.7634200043497886e-06, + "loss": 0.8696, + "num_tokens": 7603603319.0, + "step": 933 + }, + { + "epoch": 4.025959978366685, + "grad_norm": 0.23821751613972522, + "learning_rate": 3.7485057658785564e-06, + "loss": 0.8583, + "num_tokens": 7611629746.0, + "step": 934 + }, + { + "epoch": 4.030286641427799, + "grad_norm": 0.24202628415040636, + "learning_rate": 3.733648076016035e-06, + "loss": 0.8529, + "num_tokens": 7619480000.0, + "step": 935 + }, + { + "epoch": 4.0346133044889125, + "grad_norm": 0.2412207126177947, + "learning_rate": 3.7188470506254746e-06, + "loss": 0.8413, + "num_tokens": 7627668546.0, + "step": 936 + }, + { + "epoch": 4.038939967550027, + "grad_norm": 0.25144531477738197, + "learning_rate": 3.704102805128242e-06, + "loss": 0.8611, + "num_tokens": 7636016252.0, + "step": 937 + }, + { + "epoch": 4.043266630611141, + "grad_norm": 0.2098155309817567, + "learning_rate": 3.6894154545029255e-06, + "loss": 0.8781, + "num_tokens": 7644242332.0, + "step": 938 + }, + { + "epoch": 4.047593293672255, + "grad_norm": 0.2577295507503897, + "learning_rate": 3.6747851132844392e-06, + "loss": 0.8537, + "num_tokens": 7652459876.0, + "step": 939 + }, + { + "epoch": 4.05191995673337, + "grad_norm": 0.23110292079530256, + "learning_rate": 3.660211895563117e-06, + "loss": 0.855, + "num_tokens": 7660516751.0, + "step": 940 + }, + { + "epoch": 4.056246619794483, + "grad_norm": 0.23131121218263148, + "learning_rate": 3.6456959149838443e-06, + "loss": 0.8323, + "num_tokens": 7668597513.0, + "step": 941 + }, + { + "epoch": 4.060573282855597, + "grad_norm": 0.250846106465643, + "learning_rate": 3.6312372847451503e-06, + "loss": 0.8607, + "num_tokens": 7676742840.0, + "step": 942 + }, + { + "epoch": 4.064899945916712, + "grad_norm": 0.22886392433701483, + "learning_rate": 3.61683611759834e-06, + "loss": 0.8397, + "num_tokens": 7685065882.0, + "step": 943 + }, + { + "epoch": 4.069226608977826, + "grad_norm": 0.2690960212842193, + "learning_rate": 3.6024925258466028e-06, + "loss": 0.8472, + "num_tokens": 7693158562.0, + "step": 944 + }, + { + "epoch": 4.07355327203894, + "grad_norm": 0.23376964425733063, + "learning_rate": 3.5882066213441537e-06, + "loss": 0.8657, + "num_tokens": 7701397857.0, + "step": 945 + }, + { + "epoch": 4.077879935100054, + "grad_norm": 0.22588301718988346, + "learning_rate": 3.573978515495345e-06, + "loss": 0.8842, + "num_tokens": 7709555471.0, + "step": 946 + }, + { + "epoch": 4.082206598161168, + "grad_norm": 0.2174811506315356, + "learning_rate": 3.559808319253801e-06, + "loss": 0.8534, + "num_tokens": 7717790959.0, + "step": 947 + }, + { + "epoch": 4.086533261222282, + "grad_norm": 0.2253456579939054, + "learning_rate": 3.545696143121563e-06, + "loss": 0.8767, + "num_tokens": 7726157391.0, + "step": 948 + }, + { + "epoch": 4.0908599242833965, + "grad_norm": 0.2346147116122069, + "learning_rate": 3.5316420971482133e-06, + "loss": 0.8565, + "num_tokens": 7734232042.0, + "step": 949 + }, + { + "epoch": 4.095186587344511, + "grad_norm": 0.2216997510075399, + "learning_rate": 3.5176462909300257e-06, + "loss": 0.8531, + "num_tokens": 7742544120.0, + "step": 950 + }, + { + "epoch": 4.099513250405625, + "grad_norm": 0.2374983543316505, + "learning_rate": 3.5037088336091054e-06, + "loss": 0.8709, + "num_tokens": 7750722436.0, + "step": 951 + }, + { + "epoch": 4.1038399134667385, + "grad_norm": 0.22376234315222962, + "learning_rate": 3.48982983387255e-06, + "loss": 0.8743, + "num_tokens": 7758886506.0, + "step": 952 + }, + { + "epoch": 4.108166576527853, + "grad_norm": 0.2281977610944927, + "learning_rate": 3.476009399951582e-06, + "loss": 0.8546, + "num_tokens": 7767004240.0, + "step": 953 + }, + { + "epoch": 4.112493239588967, + "grad_norm": 0.2099695039921276, + "learning_rate": 3.4622476396207254e-06, + "loss": 0.8724, + "num_tokens": 7775163716.0, + "step": 954 + }, + { + "epoch": 4.116819902650081, + "grad_norm": 0.22705602607925848, + "learning_rate": 3.4485446601969507e-06, + "loss": 0.8477, + "num_tokens": 7783152253.0, + "step": 955 + }, + { + "epoch": 4.121146565711196, + "grad_norm": 0.21306056074521926, + "learning_rate": 3.4349005685388424e-06, + "loss": 0.8389, + "num_tokens": 7791264021.0, + "step": 956 + }, + { + "epoch": 4.125473228772309, + "grad_norm": 0.20090147827660462, + "learning_rate": 3.4213154710457708e-06, + "loss": 0.8796, + "num_tokens": 7799608435.0, + "step": 957 + }, + { + "epoch": 4.129799891833423, + "grad_norm": 0.21493122053824765, + "learning_rate": 3.4077894736570515e-06, + "loss": 0.8314, + "num_tokens": 7807667064.0, + "step": 958 + }, + { + "epoch": 4.134126554894538, + "grad_norm": 0.2059753610753284, + "learning_rate": 3.3943226818511333e-06, + "loss": 0.833, + "num_tokens": 7815990109.0, + "step": 959 + }, + { + "epoch": 4.138453217955652, + "grad_norm": 0.21198245077209585, + "learning_rate": 3.3809152006447587e-06, + "loss": 0.853, + "num_tokens": 7824114384.0, + "step": 960 + }, + { + "epoch": 4.142779881016766, + "grad_norm": 0.20381984846414497, + "learning_rate": 3.367567134592167e-06, + "loss": 0.8393, + "num_tokens": 7832371624.0, + "step": 961 + }, + { + "epoch": 4.14710654407788, + "grad_norm": 0.21493506431737103, + "learning_rate": 3.354278587784253e-06, + "loss": 0.8504, + "num_tokens": 7840569518.0, + "step": 962 + }, + { + "epoch": 4.151433207138994, + "grad_norm": 0.2048642755786179, + "learning_rate": 3.341049663847775e-06, + "loss": 0.8751, + "num_tokens": 7848910772.0, + "step": 963 + }, + { + "epoch": 4.155759870200108, + "grad_norm": 0.21067916197180275, + "learning_rate": 3.3278804659445384e-06, + "loss": 0.8443, + "num_tokens": 7857199697.0, + "step": 964 + }, + { + "epoch": 4.1600865332612225, + "grad_norm": 0.22092411444057108, + "learning_rate": 3.3147710967705948e-06, + "loss": 0.8425, + "num_tokens": 7865241297.0, + "step": 965 + }, + { + "epoch": 4.164413196322337, + "grad_norm": 0.22847833135320852, + "learning_rate": 3.301721658555441e-06, + "loss": 0.8321, + "num_tokens": 7873515182.0, + "step": 966 + }, + { + "epoch": 4.16873985938345, + "grad_norm": 0.21533532210759604, + "learning_rate": 3.288732253061214e-06, + "loss": 0.8679, + "num_tokens": 7881690202.0, + "step": 967 + }, + { + "epoch": 4.173066522444564, + "grad_norm": 0.22082466357661767, + "learning_rate": 3.2758029815819105e-06, + "loss": 0.8384, + "num_tokens": 7889898062.0, + "step": 968 + }, + { + "epoch": 4.177393185505679, + "grad_norm": 0.21187994596968757, + "learning_rate": 3.2629339449425813e-06, + "loss": 0.8656, + "num_tokens": 7898068459.0, + "step": 969 + }, + { + "epoch": 4.181719848566793, + "grad_norm": 0.21264696690044585, + "learning_rate": 3.2501252434985642e-06, + "loss": 0.8653, + "num_tokens": 7906290749.0, + "step": 970 + }, + { + "epoch": 4.186046511627907, + "grad_norm": 0.22388568106518758, + "learning_rate": 3.237376977134683e-06, + "loss": 0.8723, + "num_tokens": 7914461990.0, + "step": 971 + }, + { + "epoch": 4.190373174689021, + "grad_norm": 0.2185567653978264, + "learning_rate": 3.2246892452644827e-06, + "loss": 0.8615, + "num_tokens": 7922673754.0, + "step": 972 + }, + { + "epoch": 4.194699837750135, + "grad_norm": 0.20821639294582098, + "learning_rate": 3.212062146829442e-06, + "loss": 0.8473, + "num_tokens": 7930839650.0, + "step": 973 + }, + { + "epoch": 4.199026500811249, + "grad_norm": 0.2169071579237775, + "learning_rate": 3.1994957802982153e-06, + "loss": 0.8791, + "num_tokens": 7939062759.0, + "step": 974 + }, + { + "epoch": 4.203353163872364, + "grad_norm": 0.22586825315004452, + "learning_rate": 3.1869902436658484e-06, + "loss": 0.8487, + "num_tokens": 7947181658.0, + "step": 975 + }, + { + "epoch": 4.207679826933478, + "grad_norm": 0.20465574736788683, + "learning_rate": 3.1745456344530296e-06, + "loss": 0.8365, + "num_tokens": 7955475853.0, + "step": 976 + }, + { + "epoch": 4.212006489994591, + "grad_norm": 0.23451583071171617, + "learning_rate": 3.1621620497053225e-06, + "loss": 0.8583, + "num_tokens": 7963605535.0, + "step": 977 + }, + { + "epoch": 4.2163331530557056, + "grad_norm": 0.21021820011152162, + "learning_rate": 3.149839585992407e-06, + "loss": 0.8442, + "num_tokens": 7971771095.0, + "step": 978 + }, + { + "epoch": 4.22065981611682, + "grad_norm": 0.2219459473177528, + "learning_rate": 3.1375783394073323e-06, + "loss": 0.8578, + "num_tokens": 7979836050.0, + "step": 979 + }, + { + "epoch": 4.224986479177934, + "grad_norm": 0.216570841875195, + "learning_rate": 3.125378405565762e-06, + "loss": 0.8678, + "num_tokens": 7988104269.0, + "step": 980 + }, + { + "epoch": 4.229313142239048, + "grad_norm": 0.21189127738762853, + "learning_rate": 3.11323987960523e-06, + "loss": 0.8447, + "num_tokens": 7996301506.0, + "step": 981 + }, + { + "epoch": 4.233639805300163, + "grad_norm": 0.20397361398391256, + "learning_rate": 3.1011628561843982e-06, + "loss": 0.8448, + "num_tokens": 8004443351.0, + "step": 982 + }, + { + "epoch": 4.237966468361276, + "grad_norm": 0.21711331197170478, + "learning_rate": 3.0891474294823253e-06, + "loss": 0.8456, + "num_tokens": 8012595110.0, + "step": 983 + }, + { + "epoch": 4.24229313142239, + "grad_norm": 0.20594669470073923, + "learning_rate": 3.0771936931977185e-06, + "loss": 0.8364, + "num_tokens": 8020696638.0, + "step": 984 + }, + { + "epoch": 4.246619794483505, + "grad_norm": 0.20660621838635748, + "learning_rate": 3.065301740548219e-06, + "loss": 0.8591, + "num_tokens": 8028997120.0, + "step": 985 + }, + { + "epoch": 4.250946457544619, + "grad_norm": 0.21334801460062347, + "learning_rate": 3.053471664269658e-06, + "loss": 0.8447, + "num_tokens": 8037246568.0, + "step": 986 + }, + { + "epoch": 4.255273120605732, + "grad_norm": 0.1997395867867868, + "learning_rate": 3.0417035566153498e-06, + "loss": 0.8503, + "num_tokens": 8045382073.0, + "step": 987 + }, + { + "epoch": 4.259599783666847, + "grad_norm": 0.20649808810674367, + "learning_rate": 3.029997509355361e-06, + "loss": 0.8514, + "num_tokens": 8053500671.0, + "step": 988 + }, + { + "epoch": 4.263926446727961, + "grad_norm": 0.20101444191026274, + "learning_rate": 3.018353613775798e-06, + "loss": 0.8655, + "num_tokens": 8061856544.0, + "step": 989 + }, + { + "epoch": 4.268253109789075, + "grad_norm": 0.20019259823425603, + "learning_rate": 3.0067719606781e-06, + "loss": 0.8391, + "num_tokens": 8069951026.0, + "step": 990 + }, + { + "epoch": 4.2725797728501895, + "grad_norm": 0.21186236516022863, + "learning_rate": 2.9952526403783227e-06, + "loss": 0.8726, + "num_tokens": 8078195640.0, + "step": 991 + }, + { + "epoch": 4.276906435911304, + "grad_norm": 0.21045324934209927, + "learning_rate": 2.9837957427064413e-06, + "loss": 0.8542, + "num_tokens": 8086372358.0, + "step": 992 + }, + { + "epoch": 4.281233098972417, + "grad_norm": 0.2127565330851415, + "learning_rate": 2.9724013570056436e-06, + "loss": 0.873, + "num_tokens": 8094412706.0, + "step": 993 + }, + { + "epoch": 4.2855597620335315, + "grad_norm": 0.21127003833828553, + "learning_rate": 2.9610695721316396e-06, + "loss": 0.8478, + "num_tokens": 8102660141.0, + "step": 994 + }, + { + "epoch": 4.289886425094646, + "grad_norm": 0.20968739559742613, + "learning_rate": 2.949800476451963e-06, + "loss": 0.8482, + "num_tokens": 8110936041.0, + "step": 995 + }, + { + "epoch": 4.29421308815576, + "grad_norm": 0.21115683131715796, + "learning_rate": 2.9385941578452842e-06, + "loss": 0.8667, + "num_tokens": 8119007237.0, + "step": 996 + }, + { + "epoch": 4.298539751216874, + "grad_norm": 0.23437004354364685, + "learning_rate": 2.9274507037007312e-06, + "loss": 0.8832, + "num_tokens": 8127105294.0, + "step": 997 + }, + { + "epoch": 4.302866414277988, + "grad_norm": 0.21295730128285184, + "learning_rate": 2.9163702009171936e-06, + "loss": 0.8521, + "num_tokens": 8135216486.0, + "step": 998 + }, + { + "epoch": 4.307193077339102, + "grad_norm": 0.2105296464815066, + "learning_rate": 2.9053527359026605e-06, + "loss": 0.8373, + "num_tokens": 8143450807.0, + "step": 999 + }, + { + "epoch": 4.311519740400216, + "grad_norm": 0.2030718327262675, + "learning_rate": 2.8943983945735375e-06, + "loss": 0.8715, + "num_tokens": 8151571833.0, + "step": 1000 + }, + { + "epoch": 4.315846403461331, + "grad_norm": 0.22271248778522185, + "learning_rate": 2.883507262353977e-06, + "loss": 0.8615, + "num_tokens": 8159666786.0, + "step": 1001 + }, + { + "epoch": 4.320173066522445, + "grad_norm": 0.19950024521569348, + "learning_rate": 2.8726794241752163e-06, + "loss": 0.8464, + "num_tokens": 8167894590.0, + "step": 1002 + }, + { + "epoch": 4.324499729583558, + "grad_norm": 0.20153685175387673, + "learning_rate": 2.861914964474913e-06, + "loss": 0.8782, + "num_tokens": 8176082220.0, + "step": 1003 + }, + { + "epoch": 4.328826392644673, + "grad_norm": 0.22279504804051867, + "learning_rate": 2.8512139671964844e-06, + "loss": 0.8626, + "num_tokens": 8184300094.0, + "step": 1004 + }, + { + "epoch": 4.333153055705787, + "grad_norm": 0.2038091869757905, + "learning_rate": 2.8405765157884615e-06, + "loss": 0.8599, + "num_tokens": 8192455955.0, + "step": 1005 + }, + { + "epoch": 4.337479718766901, + "grad_norm": 0.22490828762253212, + "learning_rate": 2.830002693203823e-06, + "loss": 0.8526, + "num_tokens": 8200561775.0, + "step": 1006 + }, + { + "epoch": 4.3418063818280155, + "grad_norm": 0.2071771838853218, + "learning_rate": 2.8194925818993617e-06, + "loss": 0.8316, + "num_tokens": 8208489256.0, + "step": 1007 + }, + { + "epoch": 4.346133044889129, + "grad_norm": 0.20946397412793513, + "learning_rate": 2.8090462638350397e-06, + "loss": 0.8852, + "num_tokens": 8216532606.0, + "step": 1008 + }, + { + "epoch": 4.350459707950243, + "grad_norm": 0.2083799669198408, + "learning_rate": 2.7986638204733407e-06, + "loss": 0.852, + "num_tokens": 8224748749.0, + "step": 1009 + }, + { + "epoch": 4.3547863710113575, + "grad_norm": 0.20181338473914553, + "learning_rate": 2.788345332778646e-06, + "loss": 0.8519, + "num_tokens": 8232966676.0, + "step": 1010 + }, + { + "epoch": 4.359113034072472, + "grad_norm": 0.21484350895517648, + "learning_rate": 2.778090881216592e-06, + "loss": 0.8625, + "num_tokens": 8241115667.0, + "step": 1011 + }, + { + "epoch": 4.363439697133586, + "grad_norm": 0.20599622655226485, + "learning_rate": 2.7679005457534557e-06, + "loss": 0.8622, + "num_tokens": 8249319753.0, + "step": 1012 + }, + { + "epoch": 4.367766360194699, + "grad_norm": 0.20423535038888477, + "learning_rate": 2.757774405855519e-06, + "loss": 0.8342, + "num_tokens": 8257538643.0, + "step": 1013 + }, + { + "epoch": 4.372093023255814, + "grad_norm": 0.20136810455892376, + "learning_rate": 2.747712540488454e-06, + "loss": 0.8445, + "num_tokens": 8265880026.0, + "step": 1014 + }, + { + "epoch": 4.376419686316928, + "grad_norm": 0.20790306861235458, + "learning_rate": 2.737715028116707e-06, + "loss": 0.8676, + "num_tokens": 8274049024.0, + "step": 1015 + }, + { + "epoch": 4.380746349378042, + "grad_norm": 0.23342970120509718, + "learning_rate": 2.727781946702891e-06, + "loss": 0.8544, + "num_tokens": 8281959691.0, + "step": 1016 + }, + { + "epoch": 4.385073012439157, + "grad_norm": 0.20738049716539703, + "learning_rate": 2.717913373707167e-06, + "loss": 0.8514, + "num_tokens": 8289922698.0, + "step": 1017 + }, + { + "epoch": 4.38939967550027, + "grad_norm": 0.20760307455167779, + "learning_rate": 2.708109386086653e-06, + "loss": 0.8349, + "num_tokens": 8298030837.0, + "step": 1018 + }, + { + "epoch": 4.393726338561384, + "grad_norm": 0.19864761050881158, + "learning_rate": 2.6983700602948116e-06, + "loss": 0.8694, + "num_tokens": 8306262362.0, + "step": 1019 + }, + { + "epoch": 4.398053001622499, + "grad_norm": 0.21699012361528822, + "learning_rate": 2.68869547228086e-06, + "loss": 0.8457, + "num_tokens": 8314188668.0, + "step": 1020 + }, + { + "epoch": 4.402379664683613, + "grad_norm": 0.19662960094490703, + "learning_rate": 2.679085697489183e-06, + "loss": 0.8383, + "num_tokens": 8322428029.0, + "step": 1021 + }, + { + "epoch": 4.406706327744727, + "grad_norm": 0.19203720012864794, + "learning_rate": 2.6695408108587314e-06, + "loss": 0.8584, + "num_tokens": 8330710298.0, + "step": 1022 + }, + { + "epoch": 4.411032990805841, + "grad_norm": 0.20180059430853428, + "learning_rate": 2.6600608868224516e-06, + "loss": 0.8576, + "num_tokens": 8338859871.0, + "step": 1023 + }, + { + "epoch": 4.415359653866955, + "grad_norm": 0.20509198340134852, + "learning_rate": 2.6506459993066918e-06, + "loss": 0.891, + "num_tokens": 8347072869.0, + "step": 1024 + }, + { + "epoch": 4.419686316928069, + "grad_norm": 0.19374201929028256, + "learning_rate": 2.6412962217306415e-06, + "loss": 0.8887, + "num_tokens": 8355257664.0, + "step": 1025 + }, + { + "epoch": 4.424012979989183, + "grad_norm": 0.2021307218042221, + "learning_rate": 2.632011627005738e-06, + "loss": 0.8406, + "num_tokens": 8363403289.0, + "step": 1026 + }, + { + "epoch": 4.428339643050298, + "grad_norm": 0.1993100650934687, + "learning_rate": 2.6227922875351196e-06, + "loss": 0.8399, + "num_tokens": 8371515317.0, + "step": 1027 + }, + { + "epoch": 4.432666306111411, + "grad_norm": 0.21451807058523048, + "learning_rate": 2.6136382752130486e-06, + "loss": 0.8738, + "num_tokens": 8379708656.0, + "step": 1028 + }, + { + "epoch": 4.436992969172525, + "grad_norm": 0.19724417743227204, + "learning_rate": 2.6045496614243485e-06, + "loss": 0.8434, + "num_tokens": 8387884404.0, + "step": 1029 + }, + { + "epoch": 4.44131963223364, + "grad_norm": 0.21148662130322912, + "learning_rate": 2.5955265170438593e-06, + "loss": 0.8398, + "num_tokens": 8396000688.0, + "step": 1030 + }, + { + "epoch": 4.445646295294754, + "grad_norm": 0.21249678382475493, + "learning_rate": 2.5865689124358707e-06, + "loss": 0.8797, + "num_tokens": 8404242280.0, + "step": 1031 + }, + { + "epoch": 4.449972958355868, + "grad_norm": 0.2122850708785224, + "learning_rate": 2.5776769174535835e-06, + "loss": 0.8503, + "num_tokens": 8412353855.0, + "step": 1032 + }, + { + "epoch": 4.4542996214169825, + "grad_norm": 0.19496518393450699, + "learning_rate": 2.56885060143856e-06, + "loss": 0.8549, + "num_tokens": 8420679140.0, + "step": 1033 + }, + { + "epoch": 4.458626284478096, + "grad_norm": 0.21546001780818982, + "learning_rate": 2.560090033220187e-06, + "loss": 0.8421, + "num_tokens": 8428798464.0, + "step": 1034 + }, + { + "epoch": 4.46295294753921, + "grad_norm": 0.2058699626060394, + "learning_rate": 2.5513952811151338e-06, + "loss": 0.858, + "num_tokens": 8437125225.0, + "step": 1035 + }, + { + "epoch": 4.4672796106003245, + "grad_norm": 0.20739910298684777, + "learning_rate": 2.5427664129268253e-06, + "loss": 0.853, + "num_tokens": 8445225870.0, + "step": 1036 + }, + { + "epoch": 4.471606273661439, + "grad_norm": 0.20012787441123944, + "learning_rate": 2.5342034959449075e-06, + "loss": 0.8841, + "num_tokens": 8453398833.0, + "step": 1037 + }, + { + "epoch": 4.475932936722553, + "grad_norm": 0.20785445530514798, + "learning_rate": 2.5257065969447297e-06, + "loss": 0.8746, + "num_tokens": 8461501083.0, + "step": 1038 + }, + { + "epoch": 4.4802595997836665, + "grad_norm": 0.20147268740752025, + "learning_rate": 2.5172757821868144e-06, + "loss": 0.8773, + "num_tokens": 8469658382.0, + "step": 1039 + }, + { + "epoch": 4.484586262844781, + "grad_norm": 0.20579225843061455, + "learning_rate": 2.5089111174163483e-06, + "loss": 0.8686, + "num_tokens": 8477816663.0, + "step": 1040 + }, + { + "epoch": 4.488912925905895, + "grad_norm": 0.209417939508779, + "learning_rate": 2.5006126678626714e-06, + "loss": 0.8455, + "num_tokens": 8485918441.0, + "step": 1041 + }, + { + "epoch": 4.493239588967009, + "grad_norm": 0.2020873397348721, + "learning_rate": 2.492380498238756e-06, + "loss": 0.8655, + "num_tokens": 8494098053.0, + "step": 1042 + }, + { + "epoch": 4.497566252028124, + "grad_norm": 0.20422973644356235, + "learning_rate": 2.4842146727407173e-06, + "loss": 0.8733, + "num_tokens": 8502267994.0, + "step": 1043 + }, + { + "epoch": 4.501892915089237, + "grad_norm": 0.1986755135857136, + "learning_rate": 2.4761152550473024e-06, + "loss": 0.864, + "num_tokens": 8510479741.0, + "step": 1044 + }, + { + "epoch": 4.506219578150351, + "grad_norm": 0.20328455515310134, + "learning_rate": 2.468082308319397e-06, + "loss": 0.8471, + "num_tokens": 8518702560.0, + "step": 1045 + }, + { + "epoch": 4.510546241211466, + "grad_norm": 0.20584656306811688, + "learning_rate": 2.4601158951995314e-06, + "loss": 0.854, + "num_tokens": 8526638511.0, + "step": 1046 + }, + { + "epoch": 4.51487290427258, + "grad_norm": 0.1979600701432307, + "learning_rate": 2.4522160778113993e-06, + "loss": 0.8407, + "num_tokens": 8534913477.0, + "step": 1047 + }, + { + "epoch": 4.519199567333694, + "grad_norm": 0.2006337799928146, + "learning_rate": 2.44438291775936e-06, + "loss": 0.8655, + "num_tokens": 8543149752.0, + "step": 1048 + }, + { + "epoch": 4.523526230394808, + "grad_norm": 0.20019310238339189, + "learning_rate": 2.4366164761279707e-06, + "loss": 0.8589, + "num_tokens": 8551310173.0, + "step": 1049 + }, + { + "epoch": 4.527852893455922, + "grad_norm": 0.1998137123379213, + "learning_rate": 2.4289168134815065e-06, + "loss": 0.8823, + "num_tokens": 8559541643.0, + "step": 1050 + }, + { + "epoch": 4.532179556517036, + "grad_norm": 0.1956711324757758, + "learning_rate": 2.421283989863478e-06, + "loss": 0.8324, + "num_tokens": 8567789572.0, + "step": 1051 + }, + { + "epoch": 4.5365062195781505, + "grad_norm": 0.20015618587739664, + "learning_rate": 2.41371806479618e-06, + "loss": 0.8795, + "num_tokens": 8576052312.0, + "step": 1052 + }, + { + "epoch": 4.540832882639265, + "grad_norm": 0.21117882552665548, + "learning_rate": 2.406219097280214e-06, + "loss": 0.8842, + "num_tokens": 8584205318.0, + "step": 1053 + }, + { + "epoch": 4.545159545700379, + "grad_norm": 0.1996970868528108, + "learning_rate": 2.398787145794037e-06, + "loss": 0.8793, + "num_tokens": 8592311560.0, + "step": 1054 + }, + { + "epoch": 4.5494862087614925, + "grad_norm": 0.20097451063537922, + "learning_rate": 2.3914222682934986e-06, + "loss": 0.8549, + "num_tokens": 8600521759.0, + "step": 1055 + }, + { + "epoch": 4.553812871822607, + "grad_norm": 0.20371311117093258, + "learning_rate": 2.3841245222113953e-06, + "loss": 0.8666, + "num_tokens": 8608496460.0, + "step": 1056 + }, + { + "epoch": 4.558139534883721, + "grad_norm": 0.20934454792497098, + "learning_rate": 2.3768939644570143e-06, + "loss": 0.8341, + "num_tokens": 8616656011.0, + "step": 1057 + }, + { + "epoch": 4.562466197944835, + "grad_norm": 0.20323979716662224, + "learning_rate": 2.3697306514156978e-06, + "loss": 0.8663, + "num_tokens": 8624909497.0, + "step": 1058 + }, + { + "epoch": 4.566792861005949, + "grad_norm": 0.1964754956482057, + "learning_rate": 2.3626346389484005e-06, + "loss": 0.851, + "num_tokens": 8633066281.0, + "step": 1059 + }, + { + "epoch": 4.571119524067063, + "grad_norm": 0.2005006441414875, + "learning_rate": 2.3556059823912524e-06, + "loss": 0.8321, + "num_tokens": 8641075767.0, + "step": 1060 + }, + { + "epoch": 4.575446187128177, + "grad_norm": 0.19309520329441, + "learning_rate": 2.34864473655513e-06, + "loss": 0.8712, + "num_tokens": 8649204746.0, + "step": 1061 + }, + { + "epoch": 4.579772850189292, + "grad_norm": 0.20389140138429787, + "learning_rate": 2.341750955725227e-06, + "loss": 0.876, + "num_tokens": 8657339951.0, + "step": 1062 + }, + { + "epoch": 4.584099513250406, + "grad_norm": 0.20670382800347903, + "learning_rate": 2.334924693660631e-06, + "loss": 0.8541, + "num_tokens": 8665580801.0, + "step": 1063 + }, + { + "epoch": 4.58842617631152, + "grad_norm": 0.21934359163148062, + "learning_rate": 2.328166003593904e-06, + "loss": 0.8522, + "num_tokens": 8673721932.0, + "step": 1064 + }, + { + "epoch": 4.592752839372634, + "grad_norm": 0.21400905424612607, + "learning_rate": 2.3214749382306696e-06, + "loss": 0.8573, + "num_tokens": 8681948289.0, + "step": 1065 + }, + { + "epoch": 4.597079502433748, + "grad_norm": 0.2030055681254076, + "learning_rate": 2.3148515497491976e-06, + "loss": 0.8616, + "num_tokens": 8689866977.0, + "step": 1066 + }, + { + "epoch": 4.601406165494862, + "grad_norm": 0.2240242181087473, + "learning_rate": 2.308295889800004e-06, + "loss": 0.8628, + "num_tokens": 8698129144.0, + "step": 1067 + }, + { + "epoch": 4.605732828555976, + "grad_norm": 0.2108010422812071, + "learning_rate": 2.3018080095054418e-06, + "loss": 0.8562, + "num_tokens": 8706408710.0, + "step": 1068 + }, + { + "epoch": 4.61005949161709, + "grad_norm": 0.20356640308770632, + "learning_rate": 2.2953879594593046e-06, + "loss": 0.8868, + "num_tokens": 8714492500.0, + "step": 1069 + }, + { + "epoch": 4.614386154678204, + "grad_norm": 0.21152231608157082, + "learning_rate": 2.2890357897264325e-06, + "loss": 0.8626, + "num_tokens": 8722598010.0, + "step": 1070 + }, + { + "epoch": 4.618712817739318, + "grad_norm": 0.1989867053059923, + "learning_rate": 2.2827515498423204e-06, + "loss": 0.8909, + "num_tokens": 8730667889.0, + "step": 1071 + }, + { + "epoch": 4.623039480800433, + "grad_norm": 0.27277061856130735, + "learning_rate": 2.276535288812734e-06, + "loss": 0.8693, + "num_tokens": 8738937181.0, + "step": 1072 + }, + { + "epoch": 4.627366143861547, + "grad_norm": 0.21108720998300526, + "learning_rate": 2.2703870551133246e-06, + "loss": 0.8611, + "num_tokens": 8747212287.0, + "step": 1073 + }, + { + "epoch": 4.631692806922661, + "grad_norm": 0.19739661137620035, + "learning_rate": 2.264306896689255e-06, + "loss": 0.8576, + "num_tokens": 8755501228.0, + "step": 1074 + }, + { + "epoch": 4.636019469983775, + "grad_norm": 0.22385027404243815, + "learning_rate": 2.2582948609548205e-06, + "loss": 0.8702, + "num_tokens": 8763516888.0, + "step": 1075 + }, + { + "epoch": 4.640346133044889, + "grad_norm": 0.2272057959644768, + "learning_rate": 2.2523509947930847e-06, + "loss": 0.8523, + "num_tokens": 8771723101.0, + "step": 1076 + }, + { + "epoch": 4.644672796106003, + "grad_norm": 0.21466493038592643, + "learning_rate": 2.2464753445555083e-06, + "loss": 0.8571, + "num_tokens": 8779915380.0, + "step": 1077 + }, + { + "epoch": 4.6489994591671175, + "grad_norm": 0.24086017128692577, + "learning_rate": 2.2406679560615948e-06, + "loss": 0.8605, + "num_tokens": 8788127167.0, + "step": 1078 + }, + { + "epoch": 4.653326122228232, + "grad_norm": 0.2210737869072607, + "learning_rate": 2.2349288745985235e-06, + "loss": 0.8464, + "num_tokens": 8796385368.0, + "step": 1079 + }, + { + "epoch": 4.657652785289345, + "grad_norm": 0.22713293616921146, + "learning_rate": 2.229258144920805e-06, + "loss": 0.8461, + "num_tokens": 8804694802.0, + "step": 1080 + }, + { + "epoch": 4.6619794483504595, + "grad_norm": 0.21838849606060426, + "learning_rate": 2.223655811249931e-06, + "loss": 0.8831, + "num_tokens": 8812788017.0, + "step": 1081 + }, + { + "epoch": 4.666306111411574, + "grad_norm": 0.2143229091691115, + "learning_rate": 2.218121917274023e-06, + "loss": 0.8606, + "num_tokens": 8820992434.0, + "step": 1082 + }, + { + "epoch": 4.670632774472688, + "grad_norm": 0.21205279048684345, + "learning_rate": 2.2126565061474972e-06, + "loss": 0.875, + "num_tokens": 8829297906.0, + "step": 1083 + }, + { + "epoch": 4.674959437533802, + "grad_norm": 0.19606526188494686, + "learning_rate": 2.207259620490727e-06, + "loss": 0.8389, + "num_tokens": 8837598626.0, + "step": 1084 + }, + { + "epoch": 4.679286100594916, + "grad_norm": 0.21699881561935982, + "learning_rate": 2.2019313023897142e-06, + "loss": 0.8298, + "num_tokens": 8845644022.0, + "step": 1085 + }, + { + "epoch": 4.68361276365603, + "grad_norm": 0.22543962157078043, + "learning_rate": 2.1966715933957493e-06, + "loss": 0.8827, + "num_tokens": 8853593606.0, + "step": 1086 + }, + { + "epoch": 4.687939426717144, + "grad_norm": 0.19780128568604136, + "learning_rate": 2.191480534525103e-06, + "loss": 0.8411, + "num_tokens": 8861737787.0, + "step": 1087 + }, + { + "epoch": 4.692266089778259, + "grad_norm": 0.21919735942158233, + "learning_rate": 2.1863581662586945e-06, + "loss": 0.8554, + "num_tokens": 8869895071.0, + "step": 1088 + }, + { + "epoch": 4.696592752839373, + "grad_norm": 0.20996646061510832, + "learning_rate": 2.1813045285417785e-06, + "loss": 0.8601, + "num_tokens": 8878094051.0, + "step": 1089 + }, + { + "epoch": 4.700919415900486, + "grad_norm": 0.1987955644777749, + "learning_rate": 2.1763196607836393e-06, + "loss": 0.8535, + "num_tokens": 8886195133.0, + "step": 1090 + }, + { + "epoch": 4.705246078961601, + "grad_norm": 0.22311738864556185, + "learning_rate": 2.1714036018572764e-06, + "loss": 0.8711, + "num_tokens": 8894243498.0, + "step": 1091 + }, + { + "epoch": 4.709572742022715, + "grad_norm": 0.1992678620430286, + "learning_rate": 2.1665563900991043e-06, + "loss": 0.8549, + "num_tokens": 8902270183.0, + "step": 1092 + }, + { + "epoch": 4.713899405083829, + "grad_norm": 0.21002943196809834, + "learning_rate": 2.1617780633086545e-06, + "loss": 0.8723, + "num_tokens": 8910350327.0, + "step": 1093 + }, + { + "epoch": 4.7182260681449435, + "grad_norm": 0.20368217921766293, + "learning_rate": 2.1570686587482796e-06, + "loss": 0.8611, + "num_tokens": 8918480147.0, + "step": 1094 + }, + { + "epoch": 4.722552731206058, + "grad_norm": 0.20924443503979412, + "learning_rate": 2.1524282131428615e-06, + "loss": 0.878, + "num_tokens": 8926628507.0, + "step": 1095 + }, + { + "epoch": 4.726879394267171, + "grad_norm": 0.1929404196134999, + "learning_rate": 2.147856762679528e-06, + "loss": 0.8759, + "num_tokens": 8934963683.0, + "step": 1096 + }, + { + "epoch": 4.7312060573282855, + "grad_norm": 0.21398049353891876, + "learning_rate": 2.143354343007367e-06, + "loss": 0.855, + "num_tokens": 8943013513.0, + "step": 1097 + }, + { + "epoch": 4.7355327203894, + "grad_norm": 0.21200009065223316, + "learning_rate": 2.1389209892371525e-06, + "loss": 0.8668, + "num_tokens": 8951290822.0, + "step": 1098 + }, + { + "epoch": 4.739859383450514, + "grad_norm": 0.19031198479728464, + "learning_rate": 2.1345567359410665e-06, + "loss": 0.8234, + "num_tokens": 8959545093.0, + "step": 1099 + }, + { + "epoch": 4.7441860465116275, + "grad_norm": 0.21809881855681928, + "learning_rate": 2.1302616171524356e-06, + "loss": 0.8577, + "num_tokens": 8967855311.0, + "step": 1100 + }, + { + "epoch": 4.748512709572742, + "grad_norm": 0.20797275404291496, + "learning_rate": 2.1260356663654562e-06, + "loss": 0.848, + "num_tokens": 8976044488.0, + "step": 1101 + }, + { + "epoch": 4.752839372633856, + "grad_norm": 0.19440266018134847, + "learning_rate": 2.1218789165349425e-06, + "loss": 0.8627, + "num_tokens": 8984357314.0, + "step": 1102 + }, + { + "epoch": 4.75716603569497, + "grad_norm": 0.20997610073576503, + "learning_rate": 2.117791400076065e-06, + "loss": 0.8721, + "num_tokens": 8992576761.0, + "step": 1103 + }, + { + "epoch": 4.761492698756085, + "grad_norm": 0.20853472034710416, + "learning_rate": 2.113773148864097e-06, + "loss": 0.8568, + "num_tokens": 9000730014.0, + "step": 1104 + }, + { + "epoch": 4.765819361817199, + "grad_norm": 0.1950896985708432, + "learning_rate": 2.1098241942341703e-06, + "loss": 0.8583, + "num_tokens": 9008984932.0, + "step": 1105 + }, + { + "epoch": 4.770146024878312, + "grad_norm": 0.21236078108705764, + "learning_rate": 2.105944566981025e-06, + "loss": 0.8659, + "num_tokens": 9017153205.0, + "step": 1106 + }, + { + "epoch": 4.774472687939427, + "grad_norm": 0.20387892509759897, + "learning_rate": 2.1021342973587747e-06, + "loss": 0.8428, + "num_tokens": 9025258528.0, + "step": 1107 + }, + { + "epoch": 4.778799351000541, + "grad_norm": 0.19382799420820787, + "learning_rate": 2.098393415080667e-06, + "loss": 0.8832, + "num_tokens": 9033594046.0, + "step": 1108 + }, + { + "epoch": 4.783126014061655, + "grad_norm": 2.7908103634883124, + "learning_rate": 2.0947219493188515e-06, + "loss": 0.8403, + "num_tokens": 9041606475.0, + "step": 1109 + }, + { + "epoch": 4.7874526771227695, + "grad_norm": 0.23045958796275307, + "learning_rate": 2.0911199287041585e-06, + "loss": 0.8587, + "num_tokens": 9049879423.0, + "step": 1110 + }, + { + "epoch": 4.791779340183883, + "grad_norm": 0.20865917780722662, + "learning_rate": 2.087587381325867e-06, + "loss": 0.872, + "num_tokens": 9057921465.0, + "step": 1111 + }, + { + "epoch": 4.796106003244997, + "grad_norm": 0.20650218846876126, + "learning_rate": 2.0841243347314926e-06, + "loss": 0.8632, + "num_tokens": 9066182183.0, + "step": 1112 + }, + { + "epoch": 4.800432666306111, + "grad_norm": 0.2066697931474049, + "learning_rate": 2.080730815926566e-06, + "loss": 0.8534, + "num_tokens": 9074410085.0, + "step": 1113 + }, + { + "epoch": 4.804759329367226, + "grad_norm": 0.20393990938229173, + "learning_rate": 2.0774068513744294e-06, + "loss": 0.8577, + "num_tokens": 9082547243.0, + "step": 1114 + }, + { + "epoch": 4.80908599242834, + "grad_norm": 0.21959324220210089, + "learning_rate": 2.0741524669960258e-06, + "loss": 0.8509, + "num_tokens": 9090617728.0, + "step": 1115 + }, + { + "epoch": 4.813412655489453, + "grad_norm": 0.2051147526854198, + "learning_rate": 2.0709676881697004e-06, + "loss": 0.828, + "num_tokens": 9098948978.0, + "step": 1116 + }, + { + "epoch": 4.817739318550568, + "grad_norm": 0.21326189461300965, + "learning_rate": 2.0678525397309945e-06, + "loss": 0.867, + "num_tokens": 9107105697.0, + "step": 1117 + }, + { + "epoch": 4.822065981611682, + "grad_norm": 0.2085453061644322, + "learning_rate": 2.0648070459724656e-06, + "loss": 0.8631, + "num_tokens": 9115275228.0, + "step": 1118 + }, + { + "epoch": 4.826392644672796, + "grad_norm": 0.20940377079066355, + "learning_rate": 2.061831230643482e-06, + "loss": 0.8534, + "num_tokens": 9123634716.0, + "step": 1119 + }, + { + "epoch": 4.830719307733911, + "grad_norm": 0.21123146856924704, + "learning_rate": 2.0589251169500524e-06, + "loss": 0.8603, + "num_tokens": 9131839110.0, + "step": 1120 + }, + { + "epoch": 4.835045970795024, + "grad_norm": 0.2149869350167206, + "learning_rate": 2.056088727554633e-06, + "loss": 0.847, + "num_tokens": 9140126543.0, + "step": 1121 + }, + { + "epoch": 4.839372633856138, + "grad_norm": 0.19518065998387582, + "learning_rate": 2.0533220845759586e-06, + "loss": 0.8375, + "num_tokens": 9148451220.0, + "step": 1122 + }, + { + "epoch": 4.8436992969172525, + "grad_norm": 0.19363161868705317, + "learning_rate": 2.0506252095888685e-06, + "loss": 0.8113, + "num_tokens": 9156748029.0, + "step": 1123 + }, + { + "epoch": 4.848025959978367, + "grad_norm": 0.19750138349419968, + "learning_rate": 2.0479981236241335e-06, + "loss": 0.8614, + "num_tokens": 9164970638.0, + "step": 1124 + }, + { + "epoch": 4.852352623039481, + "grad_norm": 0.22333789400351034, + "learning_rate": 2.0454408471682986e-06, + "loss": 0.8586, + "num_tokens": 9173209500.0, + "step": 1125 + }, + { + "epoch": 4.856679286100595, + "grad_norm": 0.20742015733872982, + "learning_rate": 2.0429534001635194e-06, + "loss": 0.85, + "num_tokens": 9181435799.0, + "step": 1126 + }, + { + "epoch": 4.861005949161709, + "grad_norm": 0.19540059382211195, + "learning_rate": 2.0405358020074076e-06, + "loss": 0.8494, + "num_tokens": 9189661099.0, + "step": 1127 + }, + { + "epoch": 4.865332612222823, + "grad_norm": 0.21671512555598077, + "learning_rate": 2.0381880715528786e-06, + "loss": 0.8382, + "num_tokens": 9197821370.0, + "step": 1128 + }, + { + "epoch": 4.869659275283937, + "grad_norm": 0.21664128348754388, + "learning_rate": 2.0359102271080062e-06, + "loss": 0.8844, + "num_tokens": 9206073397.0, + "step": 1129 + }, + { + "epoch": 4.873985938345052, + "grad_norm": 0.19799962673850283, + "learning_rate": 2.0337022864358786e-06, + "loss": 0.844, + "num_tokens": 9214342872.0, + "step": 1130 + }, + { + "epoch": 4.878312601406165, + "grad_norm": 0.2133979471385915, + "learning_rate": 2.031564266754461e-06, + "loss": 0.8602, + "num_tokens": 9222383875.0, + "step": 1131 + }, + { + "epoch": 4.882639264467279, + "grad_norm": 0.1899247421311208, + "learning_rate": 2.0294961847364616e-06, + "loss": 0.8612, + "num_tokens": 9230602185.0, + "step": 1132 + }, + { + "epoch": 4.886965927528394, + "grad_norm": 0.19083365902010194, + "learning_rate": 2.0274980565091975e-06, + "loss": 0.8369, + "num_tokens": 9238800108.0, + "step": 1133 + }, + { + "epoch": 4.891292590589508, + "grad_norm": 0.20606804271210444, + "learning_rate": 2.025569897654475e-06, + "loss": 0.8689, + "num_tokens": 9246887907.0, + "step": 1134 + }, + { + "epoch": 4.895619253650622, + "grad_norm": 0.1919997676540761, + "learning_rate": 2.0237117232084633e-06, + "loss": 0.8728, + "num_tokens": 9255010084.0, + "step": 1135 + }, + { + "epoch": 4.8999459167117365, + "grad_norm": 0.19628375998232303, + "learning_rate": 2.0219235476615828e-06, + "loss": 0.8624, + "num_tokens": 9263224558.0, + "step": 1136 + }, + { + "epoch": 4.90427257977285, + "grad_norm": 0.1904333095794769, + "learning_rate": 2.0202053849583807e-06, + "loss": 0.8513, + "num_tokens": 9271422323.0, + "step": 1137 + }, + { + "epoch": 4.908599242833964, + "grad_norm": 0.1963855052762015, + "learning_rate": 2.0185572484974404e-06, + "loss": 0.8529, + "num_tokens": 9279531030.0, + "step": 1138 + }, + { + "epoch": 4.9129259058950785, + "grad_norm": 0.19660001032098826, + "learning_rate": 2.0169791511312564e-06, + "loss": 0.8535, + "num_tokens": 9287666160.0, + "step": 1139 + }, + { + "epoch": 4.917252568956193, + "grad_norm": 0.19391627559032526, + "learning_rate": 2.0154711051661524e-06, + "loss": 0.8669, + "num_tokens": 9295927271.0, + "step": 1140 + }, + { + "epoch": 4.921579232017306, + "grad_norm": 0.19438624850004038, + "learning_rate": 2.014033122362171e-06, + "loss": 0.8439, + "num_tokens": 9304234235.0, + "step": 1141 + }, + { + "epoch": 4.9259058950784205, + "grad_norm": 0.18807083464628718, + "learning_rate": 2.0126652139329934e-06, + "loss": 0.8712, + "num_tokens": 9312414685.0, + "step": 1142 + }, + { + "epoch": 4.930232558139535, + "grad_norm": 0.18695265450163262, + "learning_rate": 2.0113673905458433e-06, + "loss": 0.8638, + "num_tokens": 9320798188.0, + "step": 1143 + }, + { + "epoch": 4.934559221200649, + "grad_norm": 0.2055374573159018, + "learning_rate": 2.0101396623214068e-06, + "loss": 0.8718, + "num_tokens": 9328950678.0, + "step": 1144 + }, + { + "epoch": 4.938885884261763, + "grad_norm": 0.19193609134086684, + "learning_rate": 2.008982038833758e-06, + "loss": 0.8646, + "num_tokens": 9337182102.0, + "step": 1145 + }, + { + "epoch": 4.943212547322878, + "grad_norm": 0.19450720264110566, + "learning_rate": 2.0078945291102746e-06, + "loss": 0.8407, + "num_tokens": 9345324719.0, + "step": 1146 + }, + { + "epoch": 4.947539210383991, + "grad_norm": 0.1994417218682754, + "learning_rate": 2.0068771416315785e-06, + "loss": 0.852, + "num_tokens": 9353540951.0, + "step": 1147 + }, + { + "epoch": 4.951865873445105, + "grad_norm": 0.18791718843786184, + "learning_rate": 2.0059298843314594e-06, + "loss": 0.8571, + "num_tokens": 9361885796.0, + "step": 1148 + }, + { + "epoch": 4.95619253650622, + "grad_norm": 0.19351012793173114, + "learning_rate": 2.005052764596822e-06, + "loss": 0.8782, + "num_tokens": 9370144897.0, + "step": 1149 + }, + { + "epoch": 4.960519199567334, + "grad_norm": 0.19586829147183368, + "learning_rate": 2.0042457892676203e-06, + "loss": 0.8458, + "num_tokens": 9378399195.0, + "step": 1150 + }, + { + "epoch": 4.964845862628448, + "grad_norm": 0.19870574246867262, + "learning_rate": 2.003508964636811e-06, + "loss": 0.8629, + "num_tokens": 9386612011.0, + "step": 1151 + }, + { + "epoch": 4.969172525689562, + "grad_norm": 0.19700347694935025, + "learning_rate": 2.0028422964503007e-06, + "loss": 0.8646, + "num_tokens": 9394937098.0, + "step": 1152 + }, + { + "epoch": 4.973499188750676, + "grad_norm": 0.1961337078006437, + "learning_rate": 2.002245789906901e-06, + "loss": 0.8565, + "num_tokens": 9403311814.0, + "step": 1153 + }, + { + "epoch": 4.97782585181179, + "grad_norm": 0.20077657955499584, + "learning_rate": 2.0017194496582903e-06, + "loss": 0.8541, + "num_tokens": 9411436654.0, + "step": 1154 + }, + { + "epoch": 4.9821525148729044, + "grad_norm": 0.20610885410416396, + "learning_rate": 2.001263279808977e-06, + "loss": 0.8706, + "num_tokens": 9419510594.0, + "step": 1155 + }, + { + "epoch": 4.986479177934019, + "grad_norm": 0.20443771770097768, + "learning_rate": 2.0008772839162623e-06, + "loss": 0.86, + "num_tokens": 9427814762.0, + "step": 1156 + }, + { + "epoch": 4.990805840995132, + "grad_norm": 0.19815150840072435, + "learning_rate": 2.000561464990222e-06, + "loss": 0.8592, + "num_tokens": 9435805130.0, + "step": 1157 + }, + { + "epoch": 4.995132504056246, + "grad_norm": 0.2064122207419476, + "learning_rate": 2.0003158254936748e-06, + "loss": 0.8346, + "num_tokens": 9443971606.0, + "step": 1158 + }, + { + "epoch": 4.999459167117361, + "grad_norm": 0.2047105562353002, + "learning_rate": 2.000140367342166e-06, + "loss": 0.8615, + "num_tokens": 9452005208.0, + "step": 1159 + }, + { + "epoch": 5.0, + "grad_norm": 0.5085605102663149, + "learning_rate": 2.000035091903955e-06, + "loss": 0.8453, + "num_tokens": 9453007658.0, + "step": 1160 + }, + { + "epoch": 5.0, + "step": 1160, + "total_flos": 3832823860199424.0, + "train_loss": 0.9150249538750485, + "train_runtime": 89344.5669, + "train_samples_per_second": 1.655, + "train_steps_per_second": 0.013 + } + ], + "logging_steps": 1, + "max_steps": 1160, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 3832823860199424.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..ffe646e --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fb9aa8cfb60f1b136b324382439d80e6a25d7f8864d262af3f04c7f104032a2 +size 12024