commit b0e2f62ef25ba85e3d4b238f12a649e4ce70ab34 Author: ModelHub XC Date: Wed Sep 2 07:14:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Neelectric/Llama-3.2-1B-Instruct_SFT_sciencev00.01 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..665ffff --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +--- +base_model: meta-llama/Llama-3.2-1B-Instruct +datasets: Neelectric/MoT_science_Llama3_4096toks +library_name: transformers +model_name: Llama-3.2-1B-Instruct_SFT_sciencev00.01 +tags: +- generated_from_trainer +- sft +- trl +- open-r1 +licence: license +--- + +# Model Card for Llama-3.2-1B-Instruct_SFT_sciencev00.01 + +This model is a fine-tuned version of [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) on the [Neelectric/MoT_science_Llama3_4096toks](https://huggingface.co/datasets/Neelectric/MoT_science_Llama3_4096toks) dataset. +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="Neelectric/Llama-3.2-1B-Instruct_SFT_sciencev00.01", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/neelectric/open-r1_science/runs/r3s0tzuv) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 1.0.0.dev0 +- Transformers: 4.57.6 +- Pytorch: 2.9.0 +- Datasets: 4.8.3 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..4935120 --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "total_flos": 1.377941890090926e+18, + "train_loss": 1.2427511761063024, + "train_runtime": 1307.8696, + "train_samples": 145693, + "train_samples_per_second": 111.397, + "train_steps_per_second": 0.436 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..0ab931a --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,121 @@ +{{- bos_token }} +{%- if custom_tools is defined %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if not tools_in_user_message is defined %} + {%- set tools_in_user_message = true %} +{%- endif %} +{%- if not date_string is defined %} + {%- set date_string = "26 Jul 2024" %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- set system_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} +{%- else %} + {%- set system_message = "You are a helpful AI Assistant that provides well-reasoned and detailed responses. You first think about the reasoning process as an internal monologue and then provide the user with the answer. Respond in the following format: +... + + +... +" %} +{%- endif %} + +{#- System message + builtin tools #} +{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} +{%- if builtin_tools is defined or tools is not none %} + {{- "Environment: ipython\n" }} +{%- endif %} +{%- if builtin_tools is defined %} + {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}} +{%- endif %} +{{- "Cutting Knowledge Date: December 2023\n" }} +{{- "Today Date: " + date_string + "\n\n" }} +{%- if tools is not none and not tools_in_user_message %} + {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} +{%- endif %} +{{- system_message }} +{{- "<|eot_id|>" }} + +{#- Custom tools are passed in a user message with some extra guidance #} +{%- if tools_in_user_message and not tools is none %} + {#- Extract the first user message so we can plug it in here #} + {%- if messages | length != 0 %} + {%- set first_user_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} + {%- else %} + {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} + {%- endif %} + {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} + {{- "Given the following functions, please respond with a JSON for a function call " }} + {{- "with its proper arguments that best answers the given prompt.\n\n" }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} + {{- first_user_message + "<|eot_id|>"}} +{%- endif %} + +{%- for message in messages %} + {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} + {%- if message['role'] == 'assistant' %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} + {% generation %} + {{- message['content'] | trim + '<|eot_id|>' }} + {% endgeneration %} + {%- else %} + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} + {%- endif %} + {%- elif 'tool_calls' in message %} + {%- if not message.tool_calls|length == 1 %} + {{- raise_exception("This model only supports single tool-calls at once!") }} + {%- endif %} + {%- set tool_call = message.tool_calls[0].function %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} + {% generation %} + {%- if builtin_tools is defined and tool_call.name in builtin_tools %} + {{- "<|python_tag|>" + tool_call.name + ".call(" }} + {%- for arg_name, arg_val in tool_call.arguments | items %} + {{- arg_name + '="' + arg_val + '"' }} + {%- if not loop.last %} + {{- ", " }} + {%- endif %} + {%- endfor %} + {{- ")" }} + {%- else %} + {{- '{"name": "' + tool_call.name + '", ' }} + {{- '"parameters": ' }} + {{- tool_call.arguments | tojson }} + {{- "}" }} + {%- endif %} + {%- if builtin_tools is defined %} + {{- "<|eom_id|>" }} + {%- else %} + {{- "<|eot_id|>" }} + {%- endif %} + {% endgeneration %} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} + {%- if message.content is mapping or message.content is iterable %} + {{- message.content | tojson }} + {%- else %} + {{- message.content }} + {%- endif %} + {{- "<|eot_id|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..9912fbf --- /dev/null +++ b/config.json @@ -0,0 +1,35 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128009, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..1996dc1 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,8 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128009, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.6" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..f888943 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dbd5523170c4a227933d60069b2cc5ac92af5e94f881ef61c10563114c17ff4 +size 2996982344 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e8f05fa --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,10 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": "<|eot_id|>" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8b0c7c1 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2062 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|eot_id|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..4935120 --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "total_flos": 1.377941890090926e+18, + "train_loss": 1.2427511761063024, + "train_runtime": 1307.8696, + "train_samples": 145693, + "train_samples_per_second": 111.397, + "train_steps_per_second": 0.436 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..f27e11d --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,5173 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 570, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0017543859649122807, + "grad_norm": 6.017632007598877, + "learning_rate": 0.0, + "loss": 1.789, + "mean_token_accuracy": 0.5678549408912659, + "num_tokens": 429478.0, + "step": 1 + }, + { + "epoch": 0.0035087719298245615, + "grad_norm": 6.3739118576049805, + "learning_rate": 8.771929824561404e-07, + "loss": 1.7812, + "mean_token_accuracy": 0.5701305270195007, + "num_tokens": 824562.0, + "step": 2 + }, + { + "epoch": 0.005263157894736842, + "grad_norm": 6.420792579650879, + "learning_rate": 1.7543859649122807e-06, + "loss": 1.7894, + "mean_token_accuracy": 0.5683269500732422, + "num_tokens": 1228422.0, + "step": 3 + }, + { + "epoch": 0.007017543859649123, + "grad_norm": 5.904900074005127, + "learning_rate": 2.631578947368421e-06, + "loss": 1.7839, + "mean_token_accuracy": 0.5682697296142578, + "num_tokens": 1633223.0, + "step": 4 + }, + { + "epoch": 0.008771929824561403, + "grad_norm": 4.648497581481934, + "learning_rate": 3.5087719298245615e-06, + "loss": 1.7545, + "mean_token_accuracy": 0.5710222721099854, + "num_tokens": 2060143.0, + "step": 5 + }, + { + "epoch": 0.010526315789473684, + "grad_norm": 3.5128204822540283, + "learning_rate": 4.3859649122807014e-06, + "loss": 1.7077, + "mean_token_accuracy": 0.5757781267166138, + "num_tokens": 2503690.0, + "step": 6 + }, + { + "epoch": 0.012280701754385965, + "grad_norm": 3.261033296585083, + "learning_rate": 5.263157894736842e-06, + "loss": 1.6709, + "mean_token_accuracy": 0.5826667547225952, + "num_tokens": 2904426.0, + "step": 7 + }, + { + "epoch": 0.014035087719298246, + "grad_norm": 2.875699520111084, + "learning_rate": 6.140350877192982e-06, + "loss": 1.6231, + "mean_token_accuracy": 0.5871899724006653, + "num_tokens": 3299360.0, + "step": 8 + }, + { + "epoch": 0.015789473684210527, + "grad_norm": 2.4652302265167236, + "learning_rate": 7.017543859649123e-06, + "loss": 1.5943, + "mean_token_accuracy": 0.594679594039917, + "num_tokens": 3709604.0, + "step": 9 + }, + { + "epoch": 0.017543859649122806, + "grad_norm": 1.9093081951141357, + "learning_rate": 7.894736842105263e-06, + "loss": 1.5604, + "mean_token_accuracy": 0.5997968912124634, + "num_tokens": 4122427.0, + "step": 10 + }, + { + "epoch": 0.01929824561403509, + "grad_norm": 2.614924669265747, + "learning_rate": 8.771929824561403e-06, + "loss": 1.5783, + "mean_token_accuracy": 0.593652606010437, + "num_tokens": 4549800.0, + "step": 11 + }, + { + "epoch": 0.021052631578947368, + "grad_norm": 2.1088147163391113, + "learning_rate": 9.649122807017545e-06, + "loss": 1.5559, + "mean_token_accuracy": 0.5986191630363464, + "num_tokens": 4972367.0, + "step": 12 + }, + { + "epoch": 0.02280701754385965, + "grad_norm": 1.6221849918365479, + "learning_rate": 1.0526315789473684e-05, + "loss": 1.5255, + "mean_token_accuracy": 0.6020771265029907, + "num_tokens": 5381784.0, + "step": 13 + }, + { + "epoch": 0.02456140350877193, + "grad_norm": 1.4450629949569702, + "learning_rate": 1.1403508771929824e-05, + "loss": 1.4783, + "mean_token_accuracy": 0.6113154292106628, + "num_tokens": 5783941.0, + "step": 14 + }, + { + "epoch": 0.02631578947368421, + "grad_norm": 1.5197447538375854, + "learning_rate": 1.2280701754385964e-05, + "loss": 1.4628, + "mean_token_accuracy": 0.6142332553863525, + "num_tokens": 6173946.0, + "step": 15 + }, + { + "epoch": 0.028070175438596492, + "grad_norm": 1.2897140979766846, + "learning_rate": 1.3157894736842106e-05, + "loss": 1.4814, + "mean_token_accuracy": 0.6120102405548096, + "num_tokens": 6588292.0, + "step": 16 + }, + { + "epoch": 0.02982456140350877, + "grad_norm": 1.1544889211654663, + "learning_rate": 1.4035087719298246e-05, + "loss": 1.4375, + "mean_token_accuracy": 0.6202995181083679, + "num_tokens": 7005165.0, + "step": 17 + }, + { + "epoch": 0.031578947368421054, + "grad_norm": 1.0963561534881592, + "learning_rate": 1.4912280701754386e-05, + "loss": 1.4532, + "mean_token_accuracy": 0.6155336499214172, + "num_tokens": 7422596.0, + "step": 18 + }, + { + "epoch": 0.03333333333333333, + "grad_norm": 1.1144042015075684, + "learning_rate": 1.5789473684210526e-05, + "loss": 1.4038, + "mean_token_accuracy": 0.6256119608879089, + "num_tokens": 7812857.0, + "step": 19 + }, + { + "epoch": 0.03508771929824561, + "grad_norm": 1.0167980194091797, + "learning_rate": 1.6666666666666667e-05, + "loss": 1.4478, + "mean_token_accuracy": 0.6166914105415344, + "num_tokens": 8244992.0, + "step": 20 + }, + { + "epoch": 0.03684210526315789, + "grad_norm": 0.9302054047584534, + "learning_rate": 1.7543859649122806e-05, + "loss": 1.4172, + "mean_token_accuracy": 0.623745858669281, + "num_tokens": 8664134.0, + "step": 21 + }, + { + "epoch": 0.03859649122807018, + "grad_norm": 0.9185428023338318, + "learning_rate": 1.8421052631578947e-05, + "loss": 1.4214, + "mean_token_accuracy": 0.6193904280662537, + "num_tokens": 9082923.0, + "step": 22 + }, + { + "epoch": 0.04035087719298246, + "grad_norm": 0.897540807723999, + "learning_rate": 1.929824561403509e-05, + "loss": 1.4179, + "mean_token_accuracy": 0.6220552325248718, + "num_tokens": 9497934.0, + "step": 23 + }, + { + "epoch": 0.042105263157894736, + "grad_norm": 0.9716966152191162, + "learning_rate": 2.0175438596491227e-05, + "loss": 1.4451, + "mean_token_accuracy": 0.6146813035011292, + "num_tokens": 9923716.0, + "step": 24 + }, + { + "epoch": 0.043859649122807015, + "grad_norm": 0.8881113529205322, + "learning_rate": 2.105263157894737e-05, + "loss": 1.389, + "mean_token_accuracy": 0.6272114515304565, + "num_tokens": 10353257.0, + "step": 25 + }, + { + "epoch": 0.0456140350877193, + "grad_norm": 0.8260245323181152, + "learning_rate": 2.1929824561403507e-05, + "loss": 1.3968, + "mean_token_accuracy": 0.6252262592315674, + "num_tokens": 10767965.0, + "step": 26 + }, + { + "epoch": 0.04736842105263158, + "grad_norm": 0.9216469526290894, + "learning_rate": 2.280701754385965e-05, + "loss": 1.3966, + "mean_token_accuracy": 0.6255799531936646, + "num_tokens": 11188411.0, + "step": 27 + }, + { + "epoch": 0.04912280701754386, + "grad_norm": 0.8580957651138306, + "learning_rate": 2.368421052631579e-05, + "loss": 1.3669, + "mean_token_accuracy": 0.6314458250999451, + "num_tokens": 11597031.0, + "step": 28 + }, + { + "epoch": 0.05087719298245614, + "grad_norm": 0.8926440477371216, + "learning_rate": 2.456140350877193e-05, + "loss": 1.396, + "mean_token_accuracy": 0.6241698861122131, + "num_tokens": 11996650.0, + "step": 29 + }, + { + "epoch": 0.05263157894736842, + "grad_norm": 0.9144163131713867, + "learning_rate": 2.5438596491228074e-05, + "loss": 1.379, + "mean_token_accuracy": 0.6287108659744263, + "num_tokens": 12414351.0, + "step": 30 + }, + { + "epoch": 0.054385964912280704, + "grad_norm": 0.8055355548858643, + "learning_rate": 2.6315789473684212e-05, + "loss": 1.3788, + "mean_token_accuracy": 0.6278951168060303, + "num_tokens": 12853897.0, + "step": 31 + }, + { + "epoch": 0.056140350877192984, + "grad_norm": 0.8602283000946045, + "learning_rate": 2.7192982456140354e-05, + "loss": 1.351, + "mean_token_accuracy": 0.6331835389137268, + "num_tokens": 13246819.0, + "step": 32 + }, + { + "epoch": 0.05789473684210526, + "grad_norm": 0.8178858757019043, + "learning_rate": 2.8070175438596492e-05, + "loss": 1.3842, + "mean_token_accuracy": 0.6260945200920105, + "num_tokens": 13668134.0, + "step": 33 + }, + { + "epoch": 0.05964912280701754, + "grad_norm": 0.9001460671424866, + "learning_rate": 2.8947368421052634e-05, + "loss": 1.3501, + "mean_token_accuracy": 0.634337842464447, + "num_tokens": 14077677.0, + "step": 34 + }, + { + "epoch": 0.06140350877192982, + "grad_norm": 0.8717350959777832, + "learning_rate": 2.9824561403508772e-05, + "loss": 1.3674, + "mean_token_accuracy": 0.6300797462463379, + "num_tokens": 14502863.0, + "step": 35 + }, + { + "epoch": 0.06315789473684211, + "grad_norm": 0.8879193663597107, + "learning_rate": 3.0701754385964913e-05, + "loss": 1.3596, + "mean_token_accuracy": 0.6309319138526917, + "num_tokens": 14940331.0, + "step": 36 + }, + { + "epoch": 0.06491228070175438, + "grad_norm": 0.992550790309906, + "learning_rate": 3.157894736842105e-05, + "loss": 1.3454, + "mean_token_accuracy": 0.6333013772964478, + "num_tokens": 15343369.0, + "step": 37 + }, + { + "epoch": 0.06666666666666667, + "grad_norm": 0.9964364171028137, + "learning_rate": 3.24561403508772e-05, + "loss": 1.3564, + "mean_token_accuracy": 0.6310689449310303, + "num_tokens": 15752951.0, + "step": 38 + }, + { + "epoch": 0.06842105263157895, + "grad_norm": 0.932989239692688, + "learning_rate": 3.3333333333333335e-05, + "loss": 1.3199, + "mean_token_accuracy": 0.6407071352005005, + "num_tokens": 16147599.0, + "step": 39 + }, + { + "epoch": 0.07017543859649122, + "grad_norm": 0.8105142712593079, + "learning_rate": 3.421052631578947e-05, + "loss": 1.3589, + "mean_token_accuracy": 0.6311861872673035, + "num_tokens": 16557268.0, + "step": 40 + }, + { + "epoch": 0.07192982456140351, + "grad_norm": 0.7962929010391235, + "learning_rate": 3.508771929824561e-05, + "loss": 1.3239, + "mean_token_accuracy": 0.6389889121055603, + "num_tokens": 16950170.0, + "step": 41 + }, + { + "epoch": 0.07368421052631578, + "grad_norm": 1.0149195194244385, + "learning_rate": 3.5964912280701756e-05, + "loss": 1.3312, + "mean_token_accuracy": 0.6372909545898438, + "num_tokens": 17369691.0, + "step": 42 + }, + { + "epoch": 0.07543859649122807, + "grad_norm": 0.9699402451515198, + "learning_rate": 3.6842105263157895e-05, + "loss": 1.3893, + "mean_token_accuracy": 0.6237154006958008, + "num_tokens": 17777844.0, + "step": 43 + }, + { + "epoch": 0.07719298245614035, + "grad_norm": 1.0976829528808594, + "learning_rate": 3.771929824561404e-05, + "loss": 1.3361, + "mean_token_accuracy": 0.6373094320297241, + "num_tokens": 18191802.0, + "step": 44 + }, + { + "epoch": 0.07894736842105263, + "grad_norm": 1.0797998905181885, + "learning_rate": 3.859649122807018e-05, + "loss": 1.318, + "mean_token_accuracy": 0.6395655870437622, + "num_tokens": 18608551.0, + "step": 45 + }, + { + "epoch": 0.08070175438596491, + "grad_norm": 0.9435285329818726, + "learning_rate": 3.9473684210526316e-05, + "loss": 1.3797, + "mean_token_accuracy": 0.6251543760299683, + "num_tokens": 19020275.0, + "step": 46 + }, + { + "epoch": 0.0824561403508772, + "grad_norm": 0.8945981860160828, + "learning_rate": 4.0350877192982455e-05, + "loss": 1.3035, + "mean_token_accuracy": 0.6435585021972656, + "num_tokens": 19416792.0, + "step": 47 + }, + { + "epoch": 0.08421052631578947, + "grad_norm": 1.0841565132141113, + "learning_rate": 4.12280701754386e-05, + "loss": 1.332, + "mean_token_accuracy": 0.6361314058303833, + "num_tokens": 19842380.0, + "step": 48 + }, + { + "epoch": 0.08596491228070176, + "grad_norm": 0.9735652804374695, + "learning_rate": 4.210526315789474e-05, + "loss": 1.3417, + "mean_token_accuracy": 0.634251058101654, + "num_tokens": 20257340.0, + "step": 49 + }, + { + "epoch": 0.08771929824561403, + "grad_norm": 1.2134379148483276, + "learning_rate": 4.298245614035088e-05, + "loss": 1.331, + "mean_token_accuracy": 0.6350133419036865, + "num_tokens": 20693853.0, + "step": 50 + }, + { + "epoch": 0.08947368421052632, + "grad_norm": 0.9178745150566101, + "learning_rate": 4.3859649122807014e-05, + "loss": 1.3148, + "mean_token_accuracy": 0.6392862200737, + "num_tokens": 21145069.0, + "step": 51 + }, + { + "epoch": 0.0912280701754386, + "grad_norm": 1.3387223482131958, + "learning_rate": 4.473684210526316e-05, + "loss": 1.3342, + "mean_token_accuracy": 0.6354435682296753, + "num_tokens": 21548907.0, + "step": 52 + }, + { + "epoch": 0.09298245614035087, + "grad_norm": 1.0784825086593628, + "learning_rate": 4.56140350877193e-05, + "loss": 1.3265, + "mean_token_accuracy": 0.6369085907936096, + "num_tokens": 21947011.0, + "step": 53 + }, + { + "epoch": 0.09473684210526316, + "grad_norm": 1.2568986415863037, + "learning_rate": 4.649122807017544e-05, + "loss": 1.3571, + "mean_token_accuracy": 0.6294127106666565, + "num_tokens": 22384995.0, + "step": 54 + }, + { + "epoch": 0.09649122807017543, + "grad_norm": 1.0275850296020508, + "learning_rate": 4.736842105263158e-05, + "loss": 1.2884, + "mean_token_accuracy": 0.6456564664840698, + "num_tokens": 22774976.0, + "step": 55 + }, + { + "epoch": 0.09824561403508772, + "grad_norm": 1.0982813835144043, + "learning_rate": 4.824561403508772e-05, + "loss": 1.348, + "mean_token_accuracy": 0.631426990032196, + "num_tokens": 23192796.0, + "step": 56 + }, + { + "epoch": 0.1, + "grad_norm": 1.0445140600204468, + "learning_rate": 4.912280701754386e-05, + "loss": 1.3079, + "mean_token_accuracy": 0.6404247283935547, + "num_tokens": 23610316.0, + "step": 57 + }, + { + "epoch": 0.10175438596491228, + "grad_norm": 1.2474071979522705, + "learning_rate": 5e-05, + "loss": 1.3636, + "mean_token_accuracy": 0.6277468204498291, + "num_tokens": 24043550.0, + "step": 58 + }, + { + "epoch": 0.10350877192982456, + "grad_norm": 1.0543211698532104, + "learning_rate": 5e-05, + "loss": 1.3177, + "mean_token_accuracy": 0.6384046077728271, + "num_tokens": 24482051.0, + "step": 59 + }, + { + "epoch": 0.10526315789473684, + "grad_norm": 1.116968035697937, + "learning_rate": 5e-05, + "loss": 1.3167, + "mean_token_accuracy": 0.6382825374603271, + "num_tokens": 24889567.0, + "step": 60 + }, + { + "epoch": 0.10701754385964912, + "grad_norm": 1.0612415075302124, + "learning_rate": 5e-05, + "loss": 1.3182, + "mean_token_accuracy": 0.638080358505249, + "num_tokens": 25296636.0, + "step": 61 + }, + { + "epoch": 0.10877192982456141, + "grad_norm": 1.183564305305481, + "learning_rate": 5e-05, + "loss": 1.2879, + "mean_token_accuracy": 0.6456817388534546, + "num_tokens": 25694614.0, + "step": 62 + }, + { + "epoch": 0.11052631578947368, + "grad_norm": 1.0575547218322754, + "learning_rate": 5e-05, + "loss": 1.2985, + "mean_token_accuracy": 0.6416627168655396, + "num_tokens": 26107689.0, + "step": 63 + }, + { + "epoch": 0.11228070175438597, + "grad_norm": 0.8870218396186829, + "learning_rate": 5e-05, + "loss": 1.2959, + "mean_token_accuracy": 0.6414470672607422, + "num_tokens": 26522427.0, + "step": 64 + }, + { + "epoch": 0.11403508771929824, + "grad_norm": 0.9101042747497559, + "learning_rate": 5e-05, + "loss": 1.3092, + "mean_token_accuracy": 0.6399649381637573, + "num_tokens": 26934123.0, + "step": 65 + }, + { + "epoch": 0.11578947368421053, + "grad_norm": 1.1583350896835327, + "learning_rate": 5e-05, + "loss": 1.3095, + "mean_token_accuracy": 0.6401656866073608, + "num_tokens": 27358915.0, + "step": 66 + }, + { + "epoch": 0.11754385964912281, + "grad_norm": 0.9792513251304626, + "learning_rate": 5e-05, + "loss": 1.3003, + "mean_token_accuracy": 0.643873929977417, + "num_tokens": 27775556.0, + "step": 67 + }, + { + "epoch": 0.11929824561403508, + "grad_norm": 0.9242398738861084, + "learning_rate": 5e-05, + "loss": 1.3114, + "mean_token_accuracy": 0.6396855711936951, + "num_tokens": 28174439.0, + "step": 68 + }, + { + "epoch": 0.12105263157894737, + "grad_norm": 0.9884592294692993, + "learning_rate": 5e-05, + "loss": 1.2798, + "mean_token_accuracy": 0.6470383405685425, + "num_tokens": 28591718.0, + "step": 69 + }, + { + "epoch": 0.12280701754385964, + "grad_norm": 1.0803687572479248, + "learning_rate": 5e-05, + "loss": 1.2928, + "mean_token_accuracy": 0.6437596082687378, + "num_tokens": 29010174.0, + "step": 70 + }, + { + "epoch": 0.12456140350877193, + "grad_norm": 0.8759295344352722, + "learning_rate": 5e-05, + "loss": 1.2995, + "mean_token_accuracy": 0.6421941518783569, + "num_tokens": 29430243.0, + "step": 71 + }, + { + "epoch": 0.12631578947368421, + "grad_norm": 0.8425408005714417, + "learning_rate": 5e-05, + "loss": 1.3015, + "mean_token_accuracy": 0.6406317949295044, + "num_tokens": 29852311.0, + "step": 72 + }, + { + "epoch": 0.1280701754385965, + "grad_norm": 1.0176916122436523, + "learning_rate": 5e-05, + "loss": 1.2829, + "mean_token_accuracy": 0.6447360515594482, + "num_tokens": 30267287.0, + "step": 73 + }, + { + "epoch": 0.12982456140350876, + "grad_norm": 1.0264191627502441, + "learning_rate": 5e-05, + "loss": 1.3046, + "mean_token_accuracy": 0.6393584609031677, + "num_tokens": 30715583.0, + "step": 74 + }, + { + "epoch": 0.13157894736842105, + "grad_norm": 1.084442377090454, + "learning_rate": 5e-05, + "loss": 1.3144, + "mean_token_accuracy": 0.6398887038230896, + "num_tokens": 31143599.0, + "step": 75 + }, + { + "epoch": 0.13333333333333333, + "grad_norm": 0.9729044437408447, + "learning_rate": 5e-05, + "loss": 1.3006, + "mean_token_accuracy": 0.6409599781036377, + "num_tokens": 31573952.0, + "step": 76 + }, + { + "epoch": 0.13508771929824562, + "grad_norm": 0.9918591976165771, + "learning_rate": 5e-05, + "loss": 1.3204, + "mean_token_accuracy": 0.6364854574203491, + "num_tokens": 31985309.0, + "step": 77 + }, + { + "epoch": 0.1368421052631579, + "grad_norm": 0.9986845850944519, + "learning_rate": 5e-05, + "loss": 1.3223, + "mean_token_accuracy": 0.6345254778862, + "num_tokens": 32422743.0, + "step": 78 + }, + { + "epoch": 0.13859649122807016, + "grad_norm": 1.152156114578247, + "learning_rate": 5e-05, + "loss": 1.3091, + "mean_token_accuracy": 0.6381345391273499, + "num_tokens": 32830001.0, + "step": 79 + }, + { + "epoch": 0.14035087719298245, + "grad_norm": 0.8428153395652771, + "learning_rate": 5e-05, + "loss": 1.2726, + "mean_token_accuracy": 0.6470234990119934, + "num_tokens": 33242795.0, + "step": 80 + }, + { + "epoch": 0.14210526315789473, + "grad_norm": 0.8668903708457947, + "learning_rate": 5e-05, + "loss": 1.2785, + "mean_token_accuracy": 0.6454793214797974, + "num_tokens": 33631163.0, + "step": 81 + }, + { + "epoch": 0.14385964912280702, + "grad_norm": 1.1364459991455078, + "learning_rate": 5e-05, + "loss": 1.3175, + "mean_token_accuracy": 0.6366526484489441, + "num_tokens": 34030802.0, + "step": 82 + }, + { + "epoch": 0.1456140350877193, + "grad_norm": 1.1409999132156372, + "learning_rate": 5e-05, + "loss": 1.3003, + "mean_token_accuracy": 0.6391071081161499, + "num_tokens": 34443782.0, + "step": 83 + }, + { + "epoch": 0.14736842105263157, + "grad_norm": 0.9906778335571289, + "learning_rate": 5e-05, + "loss": 1.3122, + "mean_token_accuracy": 0.6378949880599976, + "num_tokens": 34854553.0, + "step": 84 + }, + { + "epoch": 0.14912280701754385, + "grad_norm": 1.0259820222854614, + "learning_rate": 5e-05, + "loss": 1.2919, + "mean_token_accuracy": 0.6433262825012207, + "num_tokens": 35265577.0, + "step": 85 + }, + { + "epoch": 0.15087719298245614, + "grad_norm": 0.9070889949798584, + "learning_rate": 5e-05, + "loss": 1.2858, + "mean_token_accuracy": 0.6451277732849121, + "num_tokens": 35705670.0, + "step": 86 + }, + { + "epoch": 0.15263157894736842, + "grad_norm": 1.1579855680465698, + "learning_rate": 5e-05, + "loss": 1.2963, + "mean_token_accuracy": 0.6409980058670044, + "num_tokens": 36117458.0, + "step": 87 + }, + { + "epoch": 0.1543859649122807, + "grad_norm": 1.029868245124817, + "learning_rate": 5e-05, + "loss": 1.2861, + "mean_token_accuracy": 0.6440685987472534, + "num_tokens": 36528322.0, + "step": 88 + }, + { + "epoch": 0.156140350877193, + "grad_norm": 0.9426443576812744, + "learning_rate": 5e-05, + "loss": 1.2878, + "mean_token_accuracy": 0.6442275047302246, + "num_tokens": 36937089.0, + "step": 89 + }, + { + "epoch": 0.15789473684210525, + "grad_norm": 1.075119137763977, + "learning_rate": 5e-05, + "loss": 1.2699, + "mean_token_accuracy": 0.6478577852249146, + "num_tokens": 37347449.0, + "step": 90 + }, + { + "epoch": 0.15964912280701754, + "grad_norm": 0.8840885758399963, + "learning_rate": 5e-05, + "loss": 1.2885, + "mean_token_accuracy": 0.6444318294525146, + "num_tokens": 37768132.0, + "step": 91 + }, + { + "epoch": 0.16140350877192983, + "grad_norm": 1.062712550163269, + "learning_rate": 5e-05, + "loss": 1.2781, + "mean_token_accuracy": 0.6460012197494507, + "num_tokens": 38173420.0, + "step": 92 + }, + { + "epoch": 0.1631578947368421, + "grad_norm": 1.09293794631958, + "learning_rate": 5e-05, + "loss": 1.2708, + "mean_token_accuracy": 0.6461946964263916, + "num_tokens": 38564492.0, + "step": 93 + }, + { + "epoch": 0.1649122807017544, + "grad_norm": 0.9275119304656982, + "learning_rate": 5e-05, + "loss": 1.2896, + "mean_token_accuracy": 0.6428093910217285, + "num_tokens": 38973529.0, + "step": 94 + }, + { + "epoch": 0.16666666666666666, + "grad_norm": 0.9005506038665771, + "learning_rate": 5e-05, + "loss": 1.3194, + "mean_token_accuracy": 0.6357496976852417, + "num_tokens": 39394470.0, + "step": 95 + }, + { + "epoch": 0.16842105263157894, + "grad_norm": 1.001502275466919, + "learning_rate": 5e-05, + "loss": 1.3001, + "mean_token_accuracy": 0.6399731040000916, + "num_tokens": 39817247.0, + "step": 96 + }, + { + "epoch": 0.17017543859649123, + "grad_norm": 1.0487033128738403, + "learning_rate": 5e-05, + "loss": 1.3036, + "mean_token_accuracy": 0.6400885581970215, + "num_tokens": 40219578.0, + "step": 97 + }, + { + "epoch": 0.17192982456140352, + "grad_norm": 0.9323050379753113, + "learning_rate": 5e-05, + "loss": 1.2965, + "mean_token_accuracy": 0.6414350271224976, + "num_tokens": 40631135.0, + "step": 98 + }, + { + "epoch": 0.1736842105263158, + "grad_norm": 1.3655850887298584, + "learning_rate": 5e-05, + "loss": 1.2779, + "mean_token_accuracy": 0.6456343531608582, + "num_tokens": 41060591.0, + "step": 99 + }, + { + "epoch": 0.17543859649122806, + "grad_norm": 0.8818299770355225, + "learning_rate": 5e-05, + "loss": 1.3043, + "mean_token_accuracy": 0.6394057869911194, + "num_tokens": 41491310.0, + "step": 100 + }, + { + "epoch": 0.17719298245614035, + "grad_norm": 1.2532066106796265, + "learning_rate": 5e-05, + "loss": 1.2849, + "mean_token_accuracy": 0.6430491805076599, + "num_tokens": 41928251.0, + "step": 101 + }, + { + "epoch": 0.17894736842105263, + "grad_norm": 1.1177780628204346, + "learning_rate": 5e-05, + "loss": 1.2725, + "mean_token_accuracy": 0.6462083458900452, + "num_tokens": 42328746.0, + "step": 102 + }, + { + "epoch": 0.18070175438596492, + "grad_norm": 1.0585683584213257, + "learning_rate": 5e-05, + "loss": 1.3166, + "mean_token_accuracy": 0.638184666633606, + "num_tokens": 42770651.0, + "step": 103 + }, + { + "epoch": 0.1824561403508772, + "grad_norm": 1.074300765991211, + "learning_rate": 5e-05, + "loss": 1.2882, + "mean_token_accuracy": 0.6429480314254761, + "num_tokens": 43173759.0, + "step": 104 + }, + { + "epoch": 0.18421052631578946, + "grad_norm": 0.975164532661438, + "learning_rate": 5e-05, + "loss": 1.2736, + "mean_token_accuracy": 0.6454344987869263, + "num_tokens": 43604295.0, + "step": 105 + }, + { + "epoch": 0.18596491228070175, + "grad_norm": 0.8710235357284546, + "learning_rate": 5e-05, + "loss": 1.2766, + "mean_token_accuracy": 0.6456890106201172, + "num_tokens": 44008348.0, + "step": 106 + }, + { + "epoch": 0.18771929824561404, + "grad_norm": 0.9731278419494629, + "learning_rate": 5e-05, + "loss": 1.2937, + "mean_token_accuracy": 0.6415528059005737, + "num_tokens": 44444613.0, + "step": 107 + }, + { + "epoch": 0.18947368421052632, + "grad_norm": 0.9545327425003052, + "learning_rate": 5e-05, + "loss": 1.2686, + "mean_token_accuracy": 0.6495999097824097, + "num_tokens": 44875312.0, + "step": 108 + }, + { + "epoch": 0.1912280701754386, + "grad_norm": 1.001120686531067, + "learning_rate": 5e-05, + "loss": 1.2816, + "mean_token_accuracy": 0.6427603363990784, + "num_tokens": 45273670.0, + "step": 109 + }, + { + "epoch": 0.19298245614035087, + "grad_norm": 0.8974591493606567, + "learning_rate": 5e-05, + "loss": 1.257, + "mean_token_accuracy": 0.649543285369873, + "num_tokens": 45696861.0, + "step": 110 + }, + { + "epoch": 0.19473684210526315, + "grad_norm": 0.9619162082672119, + "learning_rate": 5e-05, + "loss": 1.284, + "mean_token_accuracy": 0.6437926888465881, + "num_tokens": 46115387.0, + "step": 111 + }, + { + "epoch": 0.19649122807017544, + "grad_norm": 0.9367555379867554, + "learning_rate": 5e-05, + "loss": 1.2696, + "mean_token_accuracy": 0.6463337540626526, + "num_tokens": 46521933.0, + "step": 112 + }, + { + "epoch": 0.19824561403508772, + "grad_norm": 1.0860817432403564, + "learning_rate": 5e-05, + "loss": 1.3046, + "mean_token_accuracy": 0.6390475034713745, + "num_tokens": 46941357.0, + "step": 113 + }, + { + "epoch": 0.2, + "grad_norm": 0.9335904121398926, + "learning_rate": 5e-05, + "loss": 1.2592, + "mean_token_accuracy": 0.6510353088378906, + "num_tokens": 47343683.0, + "step": 114 + }, + { + "epoch": 0.20175438596491227, + "grad_norm": 0.8602105379104614, + "learning_rate": 5e-05, + "loss": 1.2734, + "mean_token_accuracy": 0.646478533744812, + "num_tokens": 47787665.0, + "step": 115 + }, + { + "epoch": 0.20350877192982456, + "grad_norm": 0.9017706513404846, + "learning_rate": 5e-05, + "loss": 1.2658, + "mean_token_accuracy": 0.6482557058334351, + "num_tokens": 48190313.0, + "step": 116 + }, + { + "epoch": 0.20526315789473684, + "grad_norm": 0.9685434699058533, + "learning_rate": 5e-05, + "loss": 1.2713, + "mean_token_accuracy": 0.6463505625724792, + "num_tokens": 48630247.0, + "step": 117 + }, + { + "epoch": 0.20701754385964913, + "grad_norm": 1.1000430583953857, + "learning_rate": 5e-05, + "loss": 1.277, + "mean_token_accuracy": 0.6445934176445007, + "num_tokens": 49009657.0, + "step": 118 + }, + { + "epoch": 0.20877192982456141, + "grad_norm": 1.1366989612579346, + "learning_rate": 5e-05, + "loss": 1.2512, + "mean_token_accuracy": 0.6498565673828125, + "num_tokens": 49398873.0, + "step": 119 + }, + { + "epoch": 0.21052631578947367, + "grad_norm": 0.7835362553596497, + "learning_rate": 5e-05, + "loss": 1.2567, + "mean_token_accuracy": 0.6495886445045471, + "num_tokens": 49804381.0, + "step": 120 + }, + { + "epoch": 0.21228070175438596, + "grad_norm": 1.0764087438583374, + "learning_rate": 5e-05, + "loss": 1.2544, + "mean_token_accuracy": 0.6497900485992432, + "num_tokens": 50213766.0, + "step": 121 + }, + { + "epoch": 0.21403508771929824, + "grad_norm": 0.9324831962585449, + "learning_rate": 5e-05, + "loss": 1.2663, + "mean_token_accuracy": 0.6485564708709717, + "num_tokens": 50625155.0, + "step": 122 + }, + { + "epoch": 0.21578947368421053, + "grad_norm": 0.8722991943359375, + "learning_rate": 5e-05, + "loss": 1.2719, + "mean_token_accuracy": 0.6460986137390137, + "num_tokens": 51028769.0, + "step": 123 + }, + { + "epoch": 0.21754385964912282, + "grad_norm": 0.9531785249710083, + "learning_rate": 5e-05, + "loss": 1.2602, + "mean_token_accuracy": 0.6484909057617188, + "num_tokens": 51442144.0, + "step": 124 + }, + { + "epoch": 0.21929824561403508, + "grad_norm": 0.8308611512184143, + "learning_rate": 5e-05, + "loss": 1.2511, + "mean_token_accuracy": 0.6517787575721741, + "num_tokens": 51847840.0, + "step": 125 + }, + { + "epoch": 0.22105263157894736, + "grad_norm": 1.1263482570648193, + "learning_rate": 5e-05, + "loss": 1.2585, + "mean_token_accuracy": 0.6480927467346191, + "num_tokens": 52284344.0, + "step": 126 + }, + { + "epoch": 0.22280701754385965, + "grad_norm": 0.8966045379638672, + "learning_rate": 5e-05, + "loss": 1.3014, + "mean_token_accuracy": 0.6384085416793823, + "num_tokens": 52685505.0, + "step": 127 + }, + { + "epoch": 0.22456140350877193, + "grad_norm": 1.0184638500213623, + "learning_rate": 5e-05, + "loss": 1.2747, + "mean_token_accuracy": 0.6446154117584229, + "num_tokens": 53110059.0, + "step": 128 + }, + { + "epoch": 0.22631578947368422, + "grad_norm": 1.0314264297485352, + "learning_rate": 5e-05, + "loss": 1.2499, + "mean_token_accuracy": 0.6499871015548706, + "num_tokens": 53502719.0, + "step": 129 + }, + { + "epoch": 0.22807017543859648, + "grad_norm": 0.88434898853302, + "learning_rate": 5e-05, + "loss": 1.2475, + "mean_token_accuracy": 0.6521512866020203, + "num_tokens": 53917317.0, + "step": 130 + }, + { + "epoch": 0.22982456140350876, + "grad_norm": 0.9266948699951172, + "learning_rate": 5e-05, + "loss": 1.2803, + "mean_token_accuracy": 0.6440666913986206, + "num_tokens": 54338305.0, + "step": 131 + }, + { + "epoch": 0.23157894736842105, + "grad_norm": 0.864328145980835, + "learning_rate": 5e-05, + "loss": 1.2592, + "mean_token_accuracy": 0.6478369235992432, + "num_tokens": 54754376.0, + "step": 132 + }, + { + "epoch": 0.23333333333333334, + "grad_norm": 1.0081030130386353, + "learning_rate": 5e-05, + "loss": 1.3031, + "mean_token_accuracy": 0.6385481357574463, + "num_tokens": 55177668.0, + "step": 133 + }, + { + "epoch": 0.23508771929824562, + "grad_norm": 0.8136110305786133, + "learning_rate": 5e-05, + "loss": 1.2658, + "mean_token_accuracy": 0.6473321318626404, + "num_tokens": 55588413.0, + "step": 134 + }, + { + "epoch": 0.23684210526315788, + "grad_norm": 0.8085804581642151, + "learning_rate": 5e-05, + "loss": 1.2482, + "mean_token_accuracy": 0.6528279781341553, + "num_tokens": 56002661.0, + "step": 135 + }, + { + "epoch": 0.23859649122807017, + "grad_norm": 1.0122929811477661, + "learning_rate": 5e-05, + "loss": 1.2482, + "mean_token_accuracy": 0.6517050266265869, + "num_tokens": 56399978.0, + "step": 136 + }, + { + "epoch": 0.24035087719298245, + "grad_norm": 0.8344688415527344, + "learning_rate": 5e-05, + "loss": 1.2549, + "mean_token_accuracy": 0.6500000953674316, + "num_tokens": 56805057.0, + "step": 137 + }, + { + "epoch": 0.24210526315789474, + "grad_norm": 0.8131964802742004, + "learning_rate": 5e-05, + "loss": 1.2722, + "mean_token_accuracy": 0.6463589072227478, + "num_tokens": 57221548.0, + "step": 138 + }, + { + "epoch": 0.24385964912280703, + "grad_norm": 1.1147828102111816, + "learning_rate": 5e-05, + "loss": 1.2709, + "mean_token_accuracy": 0.6457227468490601, + "num_tokens": 57619205.0, + "step": 139 + }, + { + "epoch": 0.24561403508771928, + "grad_norm": 1.0612682104110718, + "learning_rate": 5e-05, + "loss": 1.2503, + "mean_token_accuracy": 0.6517500281333923, + "num_tokens": 58029104.0, + "step": 140 + }, + { + "epoch": 0.24736842105263157, + "grad_norm": 0.832512378692627, + "learning_rate": 5e-05, + "loss": 1.2874, + "mean_token_accuracy": 0.6432777643203735, + "num_tokens": 58444372.0, + "step": 141 + }, + { + "epoch": 0.24912280701754386, + "grad_norm": 1.3216387033462524, + "learning_rate": 5e-05, + "loss": 1.2851, + "mean_token_accuracy": 0.6411096453666687, + "num_tokens": 58859539.0, + "step": 142 + }, + { + "epoch": 0.25087719298245614, + "grad_norm": 0.8598228693008423, + "learning_rate": 5e-05, + "loss": 1.2934, + "mean_token_accuracy": 0.6409814357757568, + "num_tokens": 59268101.0, + "step": 143 + }, + { + "epoch": 0.25263157894736843, + "grad_norm": 1.094240427017212, + "learning_rate": 5e-05, + "loss": 1.2475, + "mean_token_accuracy": 0.6517050862312317, + "num_tokens": 59708311.0, + "step": 144 + }, + { + "epoch": 0.2543859649122807, + "grad_norm": 0.881372332572937, + "learning_rate": 5e-05, + "loss": 1.2454, + "mean_token_accuracy": 0.6524955034255981, + "num_tokens": 60135806.0, + "step": 145 + }, + { + "epoch": 0.256140350877193, + "grad_norm": 0.8415367007255554, + "learning_rate": 5e-05, + "loss": 1.2919, + "mean_token_accuracy": 0.6415248513221741, + "num_tokens": 60546162.0, + "step": 146 + }, + { + "epoch": 0.2578947368421053, + "grad_norm": 1.0249412059783936, + "learning_rate": 5e-05, + "loss": 1.2453, + "mean_token_accuracy": 0.6507151126861572, + "num_tokens": 60977521.0, + "step": 147 + }, + { + "epoch": 0.2596491228070175, + "grad_norm": 0.8630224466323853, + "learning_rate": 5e-05, + "loss": 1.2531, + "mean_token_accuracy": 0.6503190994262695, + "num_tokens": 61372998.0, + "step": 148 + }, + { + "epoch": 0.2614035087719298, + "grad_norm": 0.9597001671791077, + "learning_rate": 5e-05, + "loss": 1.2774, + "mean_token_accuracy": 0.6439586877822876, + "num_tokens": 61781320.0, + "step": 149 + }, + { + "epoch": 0.2631578947368421, + "grad_norm": 0.8958455920219421, + "learning_rate": 5e-05, + "loss": 1.2582, + "mean_token_accuracy": 0.6471173763275146, + "num_tokens": 62199535.0, + "step": 150 + }, + { + "epoch": 0.2649122807017544, + "grad_norm": 0.8937548398971558, + "learning_rate": 5e-05, + "loss": 1.2566, + "mean_token_accuracy": 0.6492927670478821, + "num_tokens": 62632976.0, + "step": 151 + }, + { + "epoch": 0.26666666666666666, + "grad_norm": 0.9271047115325928, + "learning_rate": 5e-05, + "loss": 1.2362, + "mean_token_accuracy": 0.6543413400650024, + "num_tokens": 63043041.0, + "step": 152 + }, + { + "epoch": 0.26842105263157895, + "grad_norm": 0.951982855796814, + "learning_rate": 5e-05, + "loss": 1.247, + "mean_token_accuracy": 0.6496573686599731, + "num_tokens": 63453330.0, + "step": 153 + }, + { + "epoch": 0.27017543859649124, + "grad_norm": 0.7908352017402649, + "learning_rate": 5e-05, + "loss": 1.2781, + "mean_token_accuracy": 0.643843412399292, + "num_tokens": 63864814.0, + "step": 154 + }, + { + "epoch": 0.2719298245614035, + "grad_norm": 1.1237787008285522, + "learning_rate": 5e-05, + "loss": 1.2393, + "mean_token_accuracy": 0.6528310179710388, + "num_tokens": 64280965.0, + "step": 155 + }, + { + "epoch": 0.2736842105263158, + "grad_norm": 0.8140657544136047, + "learning_rate": 5e-05, + "loss": 1.2507, + "mean_token_accuracy": 0.6514922380447388, + "num_tokens": 64672526.0, + "step": 156 + }, + { + "epoch": 0.2754385964912281, + "grad_norm": 1.0491621494293213, + "learning_rate": 5e-05, + "loss": 1.2305, + "mean_token_accuracy": 0.6557638645172119, + "num_tokens": 65077313.0, + "step": 157 + }, + { + "epoch": 0.2771929824561403, + "grad_norm": 0.8243358135223389, + "learning_rate": 5e-05, + "loss": 1.2465, + "mean_token_accuracy": 0.6493934392929077, + "num_tokens": 65487692.0, + "step": 158 + }, + { + "epoch": 0.2789473684210526, + "grad_norm": 0.8356885313987732, + "learning_rate": 5e-05, + "loss": 1.2401, + "mean_token_accuracy": 0.6533178091049194, + "num_tokens": 65905410.0, + "step": 159 + }, + { + "epoch": 0.2807017543859649, + "grad_norm": 0.8304692506790161, + "learning_rate": 5e-05, + "loss": 1.2757, + "mean_token_accuracy": 0.6427402496337891, + "num_tokens": 66310475.0, + "step": 160 + }, + { + "epoch": 0.2824561403508772, + "grad_norm": 0.8690595030784607, + "learning_rate": 5e-05, + "loss": 1.2644, + "mean_token_accuracy": 0.6469794511795044, + "num_tokens": 66727367.0, + "step": 161 + }, + { + "epoch": 0.28421052631578947, + "grad_norm": 0.8000058531761169, + "learning_rate": 5e-05, + "loss": 1.2816, + "mean_token_accuracy": 0.6434546709060669, + "num_tokens": 67154750.0, + "step": 162 + }, + { + "epoch": 0.28596491228070176, + "grad_norm": 0.9528214335441589, + "learning_rate": 5e-05, + "loss": 1.245, + "mean_token_accuracy": 0.6497441530227661, + "num_tokens": 67571403.0, + "step": 163 + }, + { + "epoch": 0.28771929824561404, + "grad_norm": 0.8060006499290466, + "learning_rate": 5e-05, + "loss": 1.2289, + "mean_token_accuracy": 0.6528938412666321, + "num_tokens": 67947644.0, + "step": 164 + }, + { + "epoch": 0.2894736842105263, + "grad_norm": 0.9064860343933105, + "learning_rate": 5e-05, + "loss": 1.2529, + "mean_token_accuracy": 0.6477084755897522, + "num_tokens": 68362596.0, + "step": 165 + }, + { + "epoch": 0.2912280701754386, + "grad_norm": 0.7802819013595581, + "learning_rate": 5e-05, + "loss": 1.2352, + "mean_token_accuracy": 0.6546307802200317, + "num_tokens": 68768455.0, + "step": 166 + }, + { + "epoch": 0.2929824561403509, + "grad_norm": 0.8065477609634399, + "learning_rate": 5e-05, + "loss": 1.2641, + "mean_token_accuracy": 0.6465767025947571, + "num_tokens": 69182971.0, + "step": 167 + }, + { + "epoch": 0.29473684210526313, + "grad_norm": 0.9150968194007874, + "learning_rate": 5e-05, + "loss": 1.25, + "mean_token_accuracy": 0.6500002145767212, + "num_tokens": 69633484.0, + "step": 168 + }, + { + "epoch": 0.2964912280701754, + "grad_norm": 0.760566234588623, + "learning_rate": 5e-05, + "loss": 1.2728, + "mean_token_accuracy": 0.643431544303894, + "num_tokens": 70069352.0, + "step": 169 + }, + { + "epoch": 0.2982456140350877, + "grad_norm": 0.8697807192802429, + "learning_rate": 5e-05, + "loss": 1.2482, + "mean_token_accuracy": 0.6508249640464783, + "num_tokens": 70478616.0, + "step": 170 + }, + { + "epoch": 0.3, + "grad_norm": 0.7884939312934875, + "learning_rate": 5e-05, + "loss": 1.2725, + "mean_token_accuracy": 0.6448128819465637, + "num_tokens": 70905952.0, + "step": 171 + }, + { + "epoch": 0.3017543859649123, + "grad_norm": 0.8294349312782288, + "learning_rate": 5e-05, + "loss": 1.2557, + "mean_token_accuracy": 0.6493710279464722, + "num_tokens": 71317263.0, + "step": 172 + }, + { + "epoch": 0.30350877192982456, + "grad_norm": 0.7906805276870728, + "learning_rate": 5e-05, + "loss": 1.2651, + "mean_token_accuracy": 0.6471047401428223, + "num_tokens": 71764261.0, + "step": 173 + }, + { + "epoch": 0.30526315789473685, + "grad_norm": 0.894378125667572, + "learning_rate": 5e-05, + "loss": 1.2575, + "mean_token_accuracy": 0.648348867893219, + "num_tokens": 72179518.0, + "step": 174 + }, + { + "epoch": 0.30701754385964913, + "grad_norm": 0.7740491628646851, + "learning_rate": 5e-05, + "loss": 1.2452, + "mean_token_accuracy": 0.6501320600509644, + "num_tokens": 72594932.0, + "step": 175 + }, + { + "epoch": 0.3087719298245614, + "grad_norm": 0.7505267858505249, + "learning_rate": 5e-05, + "loss": 1.2502, + "mean_token_accuracy": 0.6511836647987366, + "num_tokens": 73005473.0, + "step": 176 + }, + { + "epoch": 0.3105263157894737, + "grad_norm": 0.7503657937049866, + "learning_rate": 5e-05, + "loss": 1.2458, + "mean_token_accuracy": 0.6500927209854126, + "num_tokens": 73397415.0, + "step": 177 + }, + { + "epoch": 0.312280701754386, + "grad_norm": 0.8414142727851868, + "learning_rate": 5e-05, + "loss": 1.2534, + "mean_token_accuracy": 0.6492934226989746, + "num_tokens": 73793964.0, + "step": 178 + }, + { + "epoch": 0.3140350877192982, + "grad_norm": 0.7966176867485046, + "learning_rate": 5e-05, + "loss": 1.2281, + "mean_token_accuracy": 0.6537145972251892, + "num_tokens": 74195270.0, + "step": 179 + }, + { + "epoch": 0.3157894736842105, + "grad_norm": 0.7663098573684692, + "learning_rate": 5e-05, + "loss": 1.2444, + "mean_token_accuracy": 0.6517476439476013, + "num_tokens": 74603802.0, + "step": 180 + }, + { + "epoch": 0.3175438596491228, + "grad_norm": 0.8110319375991821, + "learning_rate": 5e-05, + "loss": 1.2485, + "mean_token_accuracy": 0.6502121090888977, + "num_tokens": 75026083.0, + "step": 181 + }, + { + "epoch": 0.3192982456140351, + "grad_norm": 0.8121880292892456, + "learning_rate": 5e-05, + "loss": 1.2486, + "mean_token_accuracy": 0.6483826637268066, + "num_tokens": 75426052.0, + "step": 182 + }, + { + "epoch": 0.32105263157894737, + "grad_norm": 0.8323014378547668, + "learning_rate": 5e-05, + "loss": 1.2511, + "mean_token_accuracy": 0.6503880023956299, + "num_tokens": 75830225.0, + "step": 183 + }, + { + "epoch": 0.32280701754385965, + "grad_norm": 0.8791164755821228, + "learning_rate": 5e-05, + "loss": 1.2509, + "mean_token_accuracy": 0.6500139236450195, + "num_tokens": 76241940.0, + "step": 184 + }, + { + "epoch": 0.32456140350877194, + "grad_norm": 0.7512854337692261, + "learning_rate": 5e-05, + "loss": 1.2344, + "mean_token_accuracy": 0.6520208120346069, + "num_tokens": 76663142.0, + "step": 185 + }, + { + "epoch": 0.3263157894736842, + "grad_norm": 0.8191966414451599, + "learning_rate": 5e-05, + "loss": 1.2301, + "mean_token_accuracy": 0.6554604768753052, + "num_tokens": 77102129.0, + "step": 186 + }, + { + "epoch": 0.3280701754385965, + "grad_norm": 0.8626450896263123, + "learning_rate": 5e-05, + "loss": 1.255, + "mean_token_accuracy": 0.6475114822387695, + "num_tokens": 77508487.0, + "step": 187 + }, + { + "epoch": 0.3298245614035088, + "grad_norm": 0.7672626972198486, + "learning_rate": 5e-05, + "loss": 1.2236, + "mean_token_accuracy": 0.6557425856590271, + "num_tokens": 77908351.0, + "step": 188 + }, + { + "epoch": 0.33157894736842103, + "grad_norm": 0.6808663606643677, + "learning_rate": 5e-05, + "loss": 1.2259, + "mean_token_accuracy": 0.6549999713897705, + "num_tokens": 78320845.0, + "step": 189 + }, + { + "epoch": 0.3333333333333333, + "grad_norm": 0.8139033913612366, + "learning_rate": 5e-05, + "loss": 1.234, + "mean_token_accuracy": 0.6540912389755249, + "num_tokens": 78732383.0, + "step": 190 + }, + { + "epoch": 0.3350877192982456, + "grad_norm": 0.8501596450805664, + "learning_rate": 5e-05, + "loss": 1.235, + "mean_token_accuracy": 0.6527209281921387, + "num_tokens": 79123256.0, + "step": 191 + }, + { + "epoch": 0.3368421052631579, + "grad_norm": 0.8173626661300659, + "learning_rate": 5e-05, + "loss": 1.2333, + "mean_token_accuracy": 0.6546882390975952, + "num_tokens": 79525484.0, + "step": 192 + }, + { + "epoch": 0.3385964912280702, + "grad_norm": 0.6363052725791931, + "learning_rate": 5e-05, + "loss": 1.2397, + "mean_token_accuracy": 0.652570366859436, + "num_tokens": 79955140.0, + "step": 193 + }, + { + "epoch": 0.34035087719298246, + "grad_norm": 0.8814049959182739, + "learning_rate": 5e-05, + "loss": 1.2543, + "mean_token_accuracy": 0.6487300992012024, + "num_tokens": 80387845.0, + "step": 194 + }, + { + "epoch": 0.34210526315789475, + "grad_norm": 0.7822277545928955, + "learning_rate": 5e-05, + "loss": 1.249, + "mean_token_accuracy": 0.6507469415664673, + "num_tokens": 80810849.0, + "step": 195 + }, + { + "epoch": 0.34385964912280703, + "grad_norm": 0.907059907913208, + "learning_rate": 5e-05, + "loss": 1.2532, + "mean_token_accuracy": 0.6494332551956177, + "num_tokens": 81226626.0, + "step": 196 + }, + { + "epoch": 0.3456140350877193, + "grad_norm": 0.782198965549469, + "learning_rate": 5e-05, + "loss": 1.2717, + "mean_token_accuracy": 0.6448840498924255, + "num_tokens": 81648045.0, + "step": 197 + }, + { + "epoch": 0.3473684210526316, + "grad_norm": 0.8873505592346191, + "learning_rate": 5e-05, + "loss": 1.2373, + "mean_token_accuracy": 0.6524488925933838, + "num_tokens": 82055621.0, + "step": 198 + }, + { + "epoch": 0.34912280701754383, + "grad_norm": 0.782770037651062, + "learning_rate": 5e-05, + "loss": 1.2589, + "mean_token_accuracy": 0.6471732258796692, + "num_tokens": 82454820.0, + "step": 199 + }, + { + "epoch": 0.3508771929824561, + "grad_norm": 0.9466656446456909, + "learning_rate": 5e-05, + "loss": 1.2538, + "mean_token_accuracy": 0.6485123634338379, + "num_tokens": 82881912.0, + "step": 200 + }, + { + "epoch": 0.3526315789473684, + "grad_norm": 0.748227596282959, + "learning_rate": 5e-05, + "loss": 1.2543, + "mean_token_accuracy": 0.6485024094581604, + "num_tokens": 83294708.0, + "step": 201 + }, + { + "epoch": 0.3543859649122807, + "grad_norm": 0.8454561233520508, + "learning_rate": 5e-05, + "loss": 1.2473, + "mean_token_accuracy": 0.6512226462364197, + "num_tokens": 83692153.0, + "step": 202 + }, + { + "epoch": 0.356140350877193, + "grad_norm": 0.7926602959632874, + "learning_rate": 5e-05, + "loss": 1.2345, + "mean_token_accuracy": 0.6536177396774292, + "num_tokens": 84106215.0, + "step": 203 + }, + { + "epoch": 0.35789473684210527, + "grad_norm": 0.8172314167022705, + "learning_rate": 5e-05, + "loss": 1.2567, + "mean_token_accuracy": 0.6489985585212708, + "num_tokens": 84533326.0, + "step": 204 + }, + { + "epoch": 0.35964912280701755, + "grad_norm": 0.7534179091453552, + "learning_rate": 5e-05, + "loss": 1.2456, + "mean_token_accuracy": 0.6519287824630737, + "num_tokens": 84958198.0, + "step": 205 + }, + { + "epoch": 0.36140350877192984, + "grad_norm": 0.8493759632110596, + "learning_rate": 5e-05, + "loss": 1.2516, + "mean_token_accuracy": 0.6489816904067993, + "num_tokens": 85375870.0, + "step": 206 + }, + { + "epoch": 0.3631578947368421, + "grad_norm": 0.7335429787635803, + "learning_rate": 5e-05, + "loss": 1.2535, + "mean_token_accuracy": 0.649667501449585, + "num_tokens": 85800348.0, + "step": 207 + }, + { + "epoch": 0.3649122807017544, + "grad_norm": 0.8852811455726624, + "learning_rate": 5e-05, + "loss": 1.2247, + "mean_token_accuracy": 0.65558260679245, + "num_tokens": 86195245.0, + "step": 208 + }, + { + "epoch": 0.36666666666666664, + "grad_norm": 0.8402389883995056, + "learning_rate": 5e-05, + "loss": 1.2482, + "mean_token_accuracy": 0.6527358293533325, + "num_tokens": 86599515.0, + "step": 209 + }, + { + "epoch": 0.3684210526315789, + "grad_norm": 0.7256657481193542, + "learning_rate": 5e-05, + "loss": 1.2789, + "mean_token_accuracy": 0.6426239013671875, + "num_tokens": 87039537.0, + "step": 210 + }, + { + "epoch": 0.3701754385964912, + "grad_norm": 0.882832944393158, + "learning_rate": 5e-05, + "loss": 1.2278, + "mean_token_accuracy": 0.6572034358978271, + "num_tokens": 87426029.0, + "step": 211 + }, + { + "epoch": 0.3719298245614035, + "grad_norm": 0.8152451515197754, + "learning_rate": 5e-05, + "loss": 1.2418, + "mean_token_accuracy": 0.6507000923156738, + "num_tokens": 87817987.0, + "step": 212 + }, + { + "epoch": 0.3736842105263158, + "grad_norm": 0.7267266511917114, + "learning_rate": 5e-05, + "loss": 1.2136, + "mean_token_accuracy": 0.6583740711212158, + "num_tokens": 88218779.0, + "step": 213 + }, + { + "epoch": 0.37543859649122807, + "grad_norm": 0.7972964644432068, + "learning_rate": 5e-05, + "loss": 1.2228, + "mean_token_accuracy": 0.6542617082595825, + "num_tokens": 88610670.0, + "step": 214 + }, + { + "epoch": 0.37719298245614036, + "grad_norm": 0.9238396286964417, + "learning_rate": 5e-05, + "loss": 1.2587, + "mean_token_accuracy": 0.648353099822998, + "num_tokens": 89031094.0, + "step": 215 + }, + { + "epoch": 0.37894736842105264, + "grad_norm": 0.8439246416091919, + "learning_rate": 5e-05, + "loss": 1.2613, + "mean_token_accuracy": 0.6463220715522766, + "num_tokens": 89465235.0, + "step": 216 + }, + { + "epoch": 0.38070175438596493, + "grad_norm": 0.852899968624115, + "learning_rate": 5e-05, + "loss": 1.2292, + "mean_token_accuracy": 0.6539123058319092, + "num_tokens": 89867145.0, + "step": 217 + }, + { + "epoch": 0.3824561403508772, + "grad_norm": 0.8784600496292114, + "learning_rate": 5e-05, + "loss": 1.2273, + "mean_token_accuracy": 0.6555376648902893, + "num_tokens": 90262494.0, + "step": 218 + }, + { + "epoch": 0.38421052631578945, + "grad_norm": 0.9005212187767029, + "learning_rate": 5e-05, + "loss": 1.2513, + "mean_token_accuracy": 0.649271547794342, + "num_tokens": 90671131.0, + "step": 219 + }, + { + "epoch": 0.38596491228070173, + "grad_norm": 0.7955536246299744, + "learning_rate": 5e-05, + "loss": 1.2409, + "mean_token_accuracy": 0.6508429050445557, + "num_tokens": 91092593.0, + "step": 220 + }, + { + "epoch": 0.387719298245614, + "grad_norm": 1.0016906261444092, + "learning_rate": 5e-05, + "loss": 1.2586, + "mean_token_accuracy": 0.6467268466949463, + "num_tokens": 91501248.0, + "step": 221 + }, + { + "epoch": 0.3894736842105263, + "grad_norm": 0.844469428062439, + "learning_rate": 5e-05, + "loss": 1.2239, + "mean_token_accuracy": 0.6546355485916138, + "num_tokens": 91913007.0, + "step": 222 + }, + { + "epoch": 0.3912280701754386, + "grad_norm": 0.729566752910614, + "learning_rate": 5e-05, + "loss": 1.2197, + "mean_token_accuracy": 0.6564855575561523, + "num_tokens": 92321188.0, + "step": 223 + }, + { + "epoch": 0.3929824561403509, + "grad_norm": 1.079853892326355, + "learning_rate": 5e-05, + "loss": 1.2475, + "mean_token_accuracy": 0.6499646902084351, + "num_tokens": 92726934.0, + "step": 224 + }, + { + "epoch": 0.39473684210526316, + "grad_norm": 0.7690060138702393, + "learning_rate": 5e-05, + "loss": 1.222, + "mean_token_accuracy": 0.6556497812271118, + "num_tokens": 93128966.0, + "step": 225 + }, + { + "epoch": 0.39649122807017545, + "grad_norm": 0.7643059492111206, + "learning_rate": 5e-05, + "loss": 1.2578, + "mean_token_accuracy": 0.6472760438919067, + "num_tokens": 93567917.0, + "step": 226 + }, + { + "epoch": 0.39824561403508774, + "grad_norm": 0.9275525808334351, + "learning_rate": 5e-05, + "loss": 1.2377, + "mean_token_accuracy": 0.651978075504303, + "num_tokens": 93979268.0, + "step": 227 + }, + { + "epoch": 0.4, + "grad_norm": 0.7879064679145813, + "learning_rate": 5e-05, + "loss": 1.2062, + "mean_token_accuracy": 0.660293459892273, + "num_tokens": 94367195.0, + "step": 228 + }, + { + "epoch": 0.4017543859649123, + "grad_norm": 0.775917649269104, + "learning_rate": 5e-05, + "loss": 1.2253, + "mean_token_accuracy": 0.654182493686676, + "num_tokens": 94759639.0, + "step": 229 + }, + { + "epoch": 0.40350877192982454, + "grad_norm": 0.8188601136207581, + "learning_rate": 5e-05, + "loss": 1.2545, + "mean_token_accuracy": 0.6471498012542725, + "num_tokens": 95191668.0, + "step": 230 + }, + { + "epoch": 0.4052631578947368, + "grad_norm": 0.7463585138320923, + "learning_rate": 5e-05, + "loss": 1.2284, + "mean_token_accuracy": 0.6549192667007446, + "num_tokens": 95614874.0, + "step": 231 + }, + { + "epoch": 0.4070175438596491, + "grad_norm": 0.798812747001648, + "learning_rate": 5e-05, + "loss": 1.1995, + "mean_token_accuracy": 0.660561203956604, + "num_tokens": 95990695.0, + "step": 232 + }, + { + "epoch": 0.4087719298245614, + "grad_norm": 0.7233158349990845, + "learning_rate": 5e-05, + "loss": 1.2207, + "mean_token_accuracy": 0.6556422710418701, + "num_tokens": 96377947.0, + "step": 233 + }, + { + "epoch": 0.4105263157894737, + "grad_norm": 0.7123532295227051, + "learning_rate": 5e-05, + "loss": 1.2362, + "mean_token_accuracy": 0.6533983945846558, + "num_tokens": 96787894.0, + "step": 234 + }, + { + "epoch": 0.41228070175438597, + "grad_norm": 0.7054610252380371, + "learning_rate": 5e-05, + "loss": 1.2357, + "mean_token_accuracy": 0.6522467136383057, + "num_tokens": 97203762.0, + "step": 235 + }, + { + "epoch": 0.41403508771929826, + "grad_norm": 0.8345780372619629, + "learning_rate": 5e-05, + "loss": 1.2407, + "mean_token_accuracy": 0.6512540578842163, + "num_tokens": 97598494.0, + "step": 236 + }, + { + "epoch": 0.41578947368421054, + "grad_norm": 0.7388120293617249, + "learning_rate": 5e-05, + "loss": 1.2544, + "mean_token_accuracy": 0.6488337516784668, + "num_tokens": 98011465.0, + "step": 237 + }, + { + "epoch": 0.41754385964912283, + "grad_norm": 0.764832079410553, + "learning_rate": 5e-05, + "loss": 1.2583, + "mean_token_accuracy": 0.647980809211731, + "num_tokens": 98447067.0, + "step": 238 + }, + { + "epoch": 0.4192982456140351, + "grad_norm": 0.7196218967437744, + "learning_rate": 5e-05, + "loss": 1.2238, + "mean_token_accuracy": 0.6545816659927368, + "num_tokens": 98854782.0, + "step": 239 + }, + { + "epoch": 0.42105263157894735, + "grad_norm": 0.8150844573974609, + "learning_rate": 5e-05, + "loss": 1.2303, + "mean_token_accuracy": 0.6536305546760559, + "num_tokens": 99276423.0, + "step": 240 + }, + { + "epoch": 0.42280701754385963, + "grad_norm": 0.8302487730979919, + "learning_rate": 5e-05, + "loss": 1.2336, + "mean_token_accuracy": 0.6537948846817017, + "num_tokens": 99705527.0, + "step": 241 + }, + { + "epoch": 0.4245614035087719, + "grad_norm": 0.8534227609634399, + "learning_rate": 5e-05, + "loss": 1.2542, + "mean_token_accuracy": 0.6488999724388123, + "num_tokens": 100144851.0, + "step": 242 + }, + { + "epoch": 0.4263157894736842, + "grad_norm": 0.7815830111503601, + "learning_rate": 5e-05, + "loss": 1.1953, + "mean_token_accuracy": 0.6612969636917114, + "num_tokens": 100546756.0, + "step": 243 + }, + { + "epoch": 0.4280701754385965, + "grad_norm": 0.8883622288703918, + "learning_rate": 5e-05, + "loss": 1.2206, + "mean_token_accuracy": 0.6570008993148804, + "num_tokens": 100961732.0, + "step": 244 + }, + { + "epoch": 0.4298245614035088, + "grad_norm": 0.8061820864677429, + "learning_rate": 5e-05, + "loss": 1.2462, + "mean_token_accuracy": 0.6509603261947632, + "num_tokens": 101396866.0, + "step": 245 + }, + { + "epoch": 0.43157894736842106, + "grad_norm": 0.9414984583854675, + "learning_rate": 5e-05, + "loss": 1.237, + "mean_token_accuracy": 0.652214527130127, + "num_tokens": 101823395.0, + "step": 246 + }, + { + "epoch": 0.43333333333333335, + "grad_norm": 0.7871649265289307, + "learning_rate": 5e-05, + "loss": 1.2196, + "mean_token_accuracy": 0.6553509831428528, + "num_tokens": 102218896.0, + "step": 247 + }, + { + "epoch": 0.43508771929824563, + "grad_norm": 0.7410905957221985, + "learning_rate": 5e-05, + "loss": 1.1993, + "mean_token_accuracy": 0.6595052480697632, + "num_tokens": 102635943.0, + "step": 248 + }, + { + "epoch": 0.4368421052631579, + "grad_norm": 0.7503636479377747, + "learning_rate": 5e-05, + "loss": 1.2169, + "mean_token_accuracy": 0.6566457748413086, + "num_tokens": 103060426.0, + "step": 249 + }, + { + "epoch": 0.43859649122807015, + "grad_norm": 0.7568004131317139, + "learning_rate": 5e-05, + "loss": 1.2137, + "mean_token_accuracy": 0.6576935052871704, + "num_tokens": 103480891.0, + "step": 250 + }, + { + "epoch": 0.44035087719298244, + "grad_norm": 0.7525817155838013, + "learning_rate": 5e-05, + "loss": 1.2096, + "mean_token_accuracy": 0.658031702041626, + "num_tokens": 103896368.0, + "step": 251 + }, + { + "epoch": 0.4421052631578947, + "grad_norm": 0.8335546255111694, + "learning_rate": 5e-05, + "loss": 1.2239, + "mean_token_accuracy": 0.6559743881225586, + "num_tokens": 104318424.0, + "step": 252 + }, + { + "epoch": 0.443859649122807, + "grad_norm": 0.8194493651390076, + "learning_rate": 5e-05, + "loss": 1.2217, + "mean_token_accuracy": 0.6560364365577698, + "num_tokens": 104743917.0, + "step": 253 + }, + { + "epoch": 0.4456140350877193, + "grad_norm": 0.7434910535812378, + "learning_rate": 5e-05, + "loss": 1.2101, + "mean_token_accuracy": 0.6583675146102905, + "num_tokens": 105146836.0, + "step": 254 + }, + { + "epoch": 0.4473684210526316, + "grad_norm": 0.7820823788642883, + "learning_rate": 5e-05, + "loss": 1.2374, + "mean_token_accuracy": 0.651545524597168, + "num_tokens": 105552302.0, + "step": 255 + }, + { + "epoch": 0.44912280701754387, + "grad_norm": 0.764985978603363, + "learning_rate": 5e-05, + "loss": 1.2684, + "mean_token_accuracy": 0.6438525319099426, + "num_tokens": 105963414.0, + "step": 256 + }, + { + "epoch": 0.45087719298245615, + "grad_norm": 0.795109748840332, + "learning_rate": 5e-05, + "loss": 1.2376, + "mean_token_accuracy": 0.6508457660675049, + "num_tokens": 106368736.0, + "step": 257 + }, + { + "epoch": 0.45263157894736844, + "grad_norm": 0.79286789894104, + "learning_rate": 5e-05, + "loss": 1.2048, + "mean_token_accuracy": 0.6594696044921875, + "num_tokens": 106769771.0, + "step": 258 + }, + { + "epoch": 0.4543859649122807, + "grad_norm": 0.809396505355835, + "learning_rate": 5e-05, + "loss": 1.2238, + "mean_token_accuracy": 0.6554226279258728, + "num_tokens": 107170029.0, + "step": 259 + }, + { + "epoch": 0.45614035087719296, + "grad_norm": 0.7759262323379517, + "learning_rate": 5e-05, + "loss": 1.2106, + "mean_token_accuracy": 0.6567337512969971, + "num_tokens": 107576140.0, + "step": 260 + }, + { + "epoch": 0.45789473684210524, + "grad_norm": 0.9430484771728516, + "learning_rate": 5e-05, + "loss": 1.2252, + "mean_token_accuracy": 0.6534851789474487, + "num_tokens": 108009067.0, + "step": 261 + }, + { + "epoch": 0.45964912280701753, + "grad_norm": 0.7474607825279236, + "learning_rate": 5e-05, + "loss": 1.2153, + "mean_token_accuracy": 0.6566612124443054, + "num_tokens": 108434633.0, + "step": 262 + }, + { + "epoch": 0.4614035087719298, + "grad_norm": 0.8174511194229126, + "learning_rate": 5e-05, + "loss": 1.2326, + "mean_token_accuracy": 0.6518517732620239, + "num_tokens": 108878862.0, + "step": 263 + }, + { + "epoch": 0.4631578947368421, + "grad_norm": 0.9160625338554382, + "learning_rate": 5e-05, + "loss": 1.212, + "mean_token_accuracy": 0.6583468914031982, + "num_tokens": 109287222.0, + "step": 264 + }, + { + "epoch": 0.4649122807017544, + "grad_norm": 0.765175998210907, + "learning_rate": 5e-05, + "loss": 1.2139, + "mean_token_accuracy": 0.6560186743736267, + "num_tokens": 109693157.0, + "step": 265 + }, + { + "epoch": 0.4666666666666667, + "grad_norm": 1.0045809745788574, + "learning_rate": 5e-05, + "loss": 1.2054, + "mean_token_accuracy": 0.6584377288818359, + "num_tokens": 110099662.0, + "step": 266 + }, + { + "epoch": 0.46842105263157896, + "grad_norm": 0.8009559512138367, + "learning_rate": 5e-05, + "loss": 1.2303, + "mean_token_accuracy": 0.6532865762710571, + "num_tokens": 110511759.0, + "step": 267 + }, + { + "epoch": 0.47017543859649125, + "grad_norm": 0.8285786509513855, + "learning_rate": 5e-05, + "loss": 1.2128, + "mean_token_accuracy": 0.6573644876480103, + "num_tokens": 110927626.0, + "step": 268 + }, + { + "epoch": 0.47192982456140353, + "grad_norm": 0.8812404870986938, + "learning_rate": 5e-05, + "loss": 1.2263, + "mean_token_accuracy": 0.6543660163879395, + "num_tokens": 111371879.0, + "step": 269 + }, + { + "epoch": 0.47368421052631576, + "grad_norm": 0.742110013961792, + "learning_rate": 5e-05, + "loss": 1.2318, + "mean_token_accuracy": 0.652706503868103, + "num_tokens": 111784069.0, + "step": 270 + }, + { + "epoch": 0.47543859649122805, + "grad_norm": 0.891385018825531, + "learning_rate": 5e-05, + "loss": 1.2048, + "mean_token_accuracy": 0.658243715763092, + "num_tokens": 112186870.0, + "step": 271 + }, + { + "epoch": 0.47719298245614034, + "grad_norm": 0.8773000240325928, + "learning_rate": 5e-05, + "loss": 1.213, + "mean_token_accuracy": 0.6567518711090088, + "num_tokens": 112616585.0, + "step": 272 + }, + { + "epoch": 0.4789473684210526, + "grad_norm": 0.7605869770050049, + "learning_rate": 5e-05, + "loss": 1.2077, + "mean_token_accuracy": 0.6579781770706177, + "num_tokens": 113009798.0, + "step": 273 + }, + { + "epoch": 0.4807017543859649, + "grad_norm": 0.8880680799484253, + "learning_rate": 5e-05, + "loss": 1.2017, + "mean_token_accuracy": 0.6584115624427795, + "num_tokens": 113410244.0, + "step": 274 + }, + { + "epoch": 0.4824561403508772, + "grad_norm": 0.7250128388404846, + "learning_rate": 5e-05, + "loss": 1.2179, + "mean_token_accuracy": 0.6570006608963013, + "num_tokens": 113805215.0, + "step": 275 + }, + { + "epoch": 0.4842105263157895, + "grad_norm": 0.7507626414299011, + "learning_rate": 5e-05, + "loss": 1.2544, + "mean_token_accuracy": 0.6478323340415955, + "num_tokens": 114224914.0, + "step": 276 + }, + { + "epoch": 0.48596491228070177, + "grad_norm": 0.8163794875144958, + "learning_rate": 5e-05, + "loss": 1.1876, + "mean_token_accuracy": 0.6628009080886841, + "num_tokens": 114606381.0, + "step": 277 + }, + { + "epoch": 0.48771929824561405, + "grad_norm": 0.696807324886322, + "learning_rate": 5e-05, + "loss": 1.2254, + "mean_token_accuracy": 0.6540670990943909, + "num_tokens": 115021603.0, + "step": 278 + }, + { + "epoch": 0.48947368421052634, + "grad_norm": 0.9132812023162842, + "learning_rate": 5e-05, + "loss": 1.2234, + "mean_token_accuracy": 0.654281735420227, + "num_tokens": 115427417.0, + "step": 279 + }, + { + "epoch": 0.49122807017543857, + "grad_norm": 0.6871014833450317, + "learning_rate": 5e-05, + "loss": 1.2358, + "mean_token_accuracy": 0.6517217755317688, + "num_tokens": 115868301.0, + "step": 280 + }, + { + "epoch": 0.49298245614035086, + "grad_norm": 0.852879524230957, + "learning_rate": 5e-05, + "loss": 1.1941, + "mean_token_accuracy": 0.660997748374939, + "num_tokens": 116278418.0, + "step": 281 + }, + { + "epoch": 0.49473684210526314, + "grad_norm": 0.7810425162315369, + "learning_rate": 5e-05, + "loss": 1.2197, + "mean_token_accuracy": 0.6550916433334351, + "num_tokens": 116681770.0, + "step": 282 + }, + { + "epoch": 0.4964912280701754, + "grad_norm": 0.8541464805603027, + "learning_rate": 5e-05, + "loss": 1.2146, + "mean_token_accuracy": 0.6579118967056274, + "num_tokens": 117054940.0, + "step": 283 + }, + { + "epoch": 0.4982456140350877, + "grad_norm": 0.6937265396118164, + "learning_rate": 5e-05, + "loss": 1.2207, + "mean_token_accuracy": 0.6556277275085449, + "num_tokens": 117477299.0, + "step": 284 + }, + { + "epoch": 0.5, + "grad_norm": 0.8826410174369812, + "learning_rate": 5e-05, + "loss": 1.2271, + "mean_token_accuracy": 0.6536678671836853, + "num_tokens": 117885331.0, + "step": 285 + }, + { + "epoch": 0.5017543859649123, + "grad_norm": 0.9776446223258972, + "learning_rate": 5e-05, + "loss": 1.2378, + "mean_token_accuracy": 0.6510674953460693, + "num_tokens": 118307149.0, + "step": 286 + }, + { + "epoch": 0.5035087719298246, + "grad_norm": 0.8107390999794006, + "learning_rate": 5e-05, + "loss": 1.1968, + "mean_token_accuracy": 0.6598985195159912, + "num_tokens": 118697262.0, + "step": 287 + }, + { + "epoch": 0.5052631578947369, + "grad_norm": 0.8725271821022034, + "learning_rate": 5e-05, + "loss": 1.2292, + "mean_token_accuracy": 0.6528037786483765, + "num_tokens": 119124053.0, + "step": 288 + }, + { + "epoch": 0.5070175438596491, + "grad_norm": 0.854211688041687, + "learning_rate": 5e-05, + "loss": 1.2345, + "mean_token_accuracy": 0.6516417860984802, + "num_tokens": 119569185.0, + "step": 289 + }, + { + "epoch": 0.5087719298245614, + "grad_norm": 0.7967270612716675, + "learning_rate": 5e-05, + "loss": 1.2136, + "mean_token_accuracy": 0.6577362418174744, + "num_tokens": 119987270.0, + "step": 290 + }, + { + "epoch": 0.5105263157894737, + "grad_norm": 0.8669372200965881, + "learning_rate": 5e-05, + "loss": 1.2129, + "mean_token_accuracy": 0.6562959551811218, + "num_tokens": 120377177.0, + "step": 291 + }, + { + "epoch": 0.512280701754386, + "grad_norm": 0.7417237162590027, + "learning_rate": 5e-05, + "loss": 1.204, + "mean_token_accuracy": 0.6589335799217224, + "num_tokens": 120794750.0, + "step": 292 + }, + { + "epoch": 0.5140350877192983, + "grad_norm": 0.7018514275550842, + "learning_rate": 5e-05, + "loss": 1.2239, + "mean_token_accuracy": 0.6558222770690918, + "num_tokens": 121223883.0, + "step": 293 + }, + { + "epoch": 0.5157894736842106, + "grad_norm": 0.807005763053894, + "learning_rate": 5e-05, + "loss": 1.21, + "mean_token_accuracy": 0.6572495102882385, + "num_tokens": 121644108.0, + "step": 294 + }, + { + "epoch": 0.5175438596491229, + "grad_norm": 0.7636781930923462, + "learning_rate": 5e-05, + "loss": 1.2161, + "mean_token_accuracy": 0.6554030776023865, + "num_tokens": 122051612.0, + "step": 295 + }, + { + "epoch": 0.519298245614035, + "grad_norm": 0.7048824429512024, + "learning_rate": 5e-05, + "loss": 1.2041, + "mean_token_accuracy": 0.6592545509338379, + "num_tokens": 122430847.0, + "step": 296 + }, + { + "epoch": 0.5210526315789473, + "grad_norm": 0.8004878163337708, + "learning_rate": 5e-05, + "loss": 1.2076, + "mean_token_accuracy": 0.6585075259208679, + "num_tokens": 122849234.0, + "step": 297 + }, + { + "epoch": 0.5228070175438596, + "grad_norm": 0.7337815165519714, + "learning_rate": 5e-05, + "loss": 1.2265, + "mean_token_accuracy": 0.6551846861839294, + "num_tokens": 123258290.0, + "step": 298 + }, + { + "epoch": 0.5245614035087719, + "grad_norm": 0.6424617767333984, + "learning_rate": 5e-05, + "loss": 1.2207, + "mean_token_accuracy": 0.6552677154541016, + "num_tokens": 123663735.0, + "step": 299 + }, + { + "epoch": 0.5263157894736842, + "grad_norm": 0.7560967206954956, + "learning_rate": 5e-05, + "loss": 1.2234, + "mean_token_accuracy": 0.6535024642944336, + "num_tokens": 124058727.0, + "step": 300 + }, + { + "epoch": 0.5280701754385965, + "grad_norm": 0.744430661201477, + "learning_rate": 5e-05, + "loss": 1.2261, + "mean_token_accuracy": 0.6548186540603638, + "num_tokens": 124484021.0, + "step": 301 + }, + { + "epoch": 0.5298245614035088, + "grad_norm": 0.6977707743644714, + "learning_rate": 5e-05, + "loss": 1.1961, + "mean_token_accuracy": 0.6600746512413025, + "num_tokens": 124902607.0, + "step": 302 + }, + { + "epoch": 0.531578947368421, + "grad_norm": 0.699184238910675, + "learning_rate": 5e-05, + "loss": 1.2214, + "mean_token_accuracy": 0.6543452143669128, + "num_tokens": 125338175.0, + "step": 303 + }, + { + "epoch": 0.5333333333333333, + "grad_norm": 0.7235009670257568, + "learning_rate": 5e-05, + "loss": 1.2135, + "mean_token_accuracy": 0.6568456292152405, + "num_tokens": 125757431.0, + "step": 304 + }, + { + "epoch": 0.5350877192982456, + "grad_norm": 0.7637624740600586, + "learning_rate": 5e-05, + "loss": 1.2027, + "mean_token_accuracy": 0.659853994846344, + "num_tokens": 126155529.0, + "step": 305 + }, + { + "epoch": 0.5368421052631579, + "grad_norm": 0.6867282390594482, + "learning_rate": 5e-05, + "loss": 1.204, + "mean_token_accuracy": 0.6584248542785645, + "num_tokens": 126572012.0, + "step": 306 + }, + { + "epoch": 0.5385964912280702, + "grad_norm": 0.7795471549034119, + "learning_rate": 5e-05, + "loss": 1.2209, + "mean_token_accuracy": 0.6548125743865967, + "num_tokens": 126990573.0, + "step": 307 + }, + { + "epoch": 0.5403508771929825, + "grad_norm": 0.7895427346229553, + "learning_rate": 5e-05, + "loss": 1.1999, + "mean_token_accuracy": 0.6595349907875061, + "num_tokens": 127418976.0, + "step": 308 + }, + { + "epoch": 0.5421052631578948, + "grad_norm": 0.6661955118179321, + "learning_rate": 5e-05, + "loss": 1.1751, + "mean_token_accuracy": 0.6655969023704529, + "num_tokens": 127827614.0, + "step": 309 + }, + { + "epoch": 0.543859649122807, + "grad_norm": 0.8003215193748474, + "learning_rate": 5e-05, + "loss": 1.2098, + "mean_token_accuracy": 0.6574652194976807, + "num_tokens": 128258313.0, + "step": 310 + }, + { + "epoch": 0.5456140350877193, + "grad_norm": 0.7345336675643921, + "learning_rate": 5e-05, + "loss": 1.231, + "mean_token_accuracy": 0.6522745490074158, + "num_tokens": 128702682.0, + "step": 311 + }, + { + "epoch": 0.5473684210526316, + "grad_norm": 0.7758187055587769, + "learning_rate": 5e-05, + "loss": 1.2035, + "mean_token_accuracy": 0.659572422504425, + "num_tokens": 129094449.0, + "step": 312 + }, + { + "epoch": 0.5491228070175439, + "grad_norm": 0.8047874569892883, + "learning_rate": 5e-05, + "loss": 1.2059, + "mean_token_accuracy": 0.6584779620170593, + "num_tokens": 129492795.0, + "step": 313 + }, + { + "epoch": 0.5508771929824562, + "grad_norm": 0.7046459913253784, + "learning_rate": 5e-05, + "loss": 1.2087, + "mean_token_accuracy": 0.6579834818840027, + "num_tokens": 129923670.0, + "step": 314 + }, + { + "epoch": 0.5526315789473685, + "grad_norm": 0.9518388509750366, + "learning_rate": 5e-05, + "loss": 1.1942, + "mean_token_accuracy": 0.6631639003753662, + "num_tokens": 130333012.0, + "step": 315 + }, + { + "epoch": 0.5543859649122806, + "grad_norm": 0.7315335869789124, + "learning_rate": 5e-05, + "loss": 1.1916, + "mean_token_accuracy": 0.6623898148536682, + "num_tokens": 130749305.0, + "step": 316 + }, + { + "epoch": 0.5561403508771929, + "grad_norm": 0.7444988489151001, + "learning_rate": 5e-05, + "loss": 1.2364, + "mean_token_accuracy": 0.6515935659408569, + "num_tokens": 131169421.0, + "step": 317 + }, + { + "epoch": 0.5578947368421052, + "grad_norm": 0.9178642630577087, + "learning_rate": 5e-05, + "loss": 1.1918, + "mean_token_accuracy": 0.6635771989822388, + "num_tokens": 131574788.0, + "step": 318 + }, + { + "epoch": 0.5596491228070175, + "grad_norm": 0.6425471901893616, + "learning_rate": 5e-05, + "loss": 1.2085, + "mean_token_accuracy": 0.6575384140014648, + "num_tokens": 131984459.0, + "step": 319 + }, + { + "epoch": 0.5614035087719298, + "grad_norm": 1.0934267044067383, + "learning_rate": 5e-05, + "loss": 1.2447, + "mean_token_accuracy": 0.6507349014282227, + "num_tokens": 132415165.0, + "step": 320 + }, + { + "epoch": 0.5631578947368421, + "grad_norm": 0.7396553754806519, + "learning_rate": 5e-05, + "loss": 1.1873, + "mean_token_accuracy": 0.6627764701843262, + "num_tokens": 132838824.0, + "step": 321 + }, + { + "epoch": 0.5649122807017544, + "grad_norm": 1.1760458946228027, + "learning_rate": 5e-05, + "loss": 1.1948, + "mean_token_accuracy": 0.6606582999229431, + "num_tokens": 133247120.0, + "step": 322 + }, + { + "epoch": 0.5666666666666667, + "grad_norm": 0.9091032147407532, + "learning_rate": 5e-05, + "loss": 1.2072, + "mean_token_accuracy": 0.658053457736969, + "num_tokens": 133672706.0, + "step": 323 + }, + { + "epoch": 0.5684210526315789, + "grad_norm": 1.0296709537506104, + "learning_rate": 5e-05, + "loss": 1.2211, + "mean_token_accuracy": 0.657611608505249, + "num_tokens": 134079103.0, + "step": 324 + }, + { + "epoch": 0.5701754385964912, + "grad_norm": 0.9069951772689819, + "learning_rate": 5e-05, + "loss": 1.2107, + "mean_token_accuracy": 0.6570094227790833, + "num_tokens": 134494826.0, + "step": 325 + }, + { + "epoch": 0.5719298245614035, + "grad_norm": 0.9754378199577332, + "learning_rate": 5e-05, + "loss": 1.2207, + "mean_token_accuracy": 0.6552340388298035, + "num_tokens": 134919504.0, + "step": 326 + }, + { + "epoch": 0.5736842105263158, + "grad_norm": 0.8345258235931396, + "learning_rate": 5e-05, + "loss": 1.2144, + "mean_token_accuracy": 0.6559135317802429, + "num_tokens": 135331962.0, + "step": 327 + }, + { + "epoch": 0.5754385964912281, + "grad_norm": 0.9506780505180359, + "learning_rate": 5e-05, + "loss": 1.2099, + "mean_token_accuracy": 0.6563049554824829, + "num_tokens": 135745710.0, + "step": 328 + }, + { + "epoch": 0.5771929824561404, + "grad_norm": 0.7616828083992004, + "learning_rate": 5e-05, + "loss": 1.216, + "mean_token_accuracy": 0.6563494205474854, + "num_tokens": 136174418.0, + "step": 329 + }, + { + "epoch": 0.5789473684210527, + "grad_norm": 0.9776095747947693, + "learning_rate": 5e-05, + "loss": 1.2181, + "mean_token_accuracy": 0.656294584274292, + "num_tokens": 136597949.0, + "step": 330 + }, + { + "epoch": 0.5807017543859649, + "grad_norm": 0.7999969720840454, + "learning_rate": 5e-05, + "loss": 1.2319, + "mean_token_accuracy": 0.6528905034065247, + "num_tokens": 137047696.0, + "step": 331 + }, + { + "epoch": 0.5824561403508772, + "grad_norm": 0.9681965708732605, + "learning_rate": 5e-05, + "loss": 1.194, + "mean_token_accuracy": 0.6618871688842773, + "num_tokens": 137428701.0, + "step": 332 + }, + { + "epoch": 0.5842105263157895, + "grad_norm": 0.853967010974884, + "learning_rate": 5e-05, + "loss": 1.1991, + "mean_token_accuracy": 0.6608738899230957, + "num_tokens": 137823083.0, + "step": 333 + }, + { + "epoch": 0.5859649122807018, + "grad_norm": 0.8977996110916138, + "learning_rate": 5e-05, + "loss": 1.2526, + "mean_token_accuracy": 0.6461295485496521, + "num_tokens": 138250591.0, + "step": 334 + }, + { + "epoch": 0.5877192982456141, + "grad_norm": 0.8242368698120117, + "learning_rate": 5e-05, + "loss": 1.2061, + "mean_token_accuracy": 0.6575807332992554, + "num_tokens": 138660562.0, + "step": 335 + }, + { + "epoch": 0.5894736842105263, + "grad_norm": 0.855155348777771, + "learning_rate": 5e-05, + "loss": 1.1957, + "mean_token_accuracy": 0.6602746248245239, + "num_tokens": 139077597.0, + "step": 336 + }, + { + "epoch": 0.5912280701754385, + "grad_norm": 0.7887840270996094, + "learning_rate": 5e-05, + "loss": 1.2152, + "mean_token_accuracy": 0.6552134156227112, + "num_tokens": 139482698.0, + "step": 337 + }, + { + "epoch": 0.5929824561403508, + "grad_norm": 0.8065133690834045, + "learning_rate": 5e-05, + "loss": 1.2215, + "mean_token_accuracy": 0.6540716290473938, + "num_tokens": 139906177.0, + "step": 338 + }, + { + "epoch": 0.5947368421052631, + "grad_norm": 0.7055850625038147, + "learning_rate": 5e-05, + "loss": 1.2109, + "mean_token_accuracy": 0.6572525501251221, + "num_tokens": 140319741.0, + "step": 339 + }, + { + "epoch": 0.5964912280701754, + "grad_norm": 0.9528288841247559, + "learning_rate": 5e-05, + "loss": 1.2057, + "mean_token_accuracy": 0.657723605632782, + "num_tokens": 140731809.0, + "step": 340 + }, + { + "epoch": 0.5982456140350877, + "grad_norm": 0.7323077917098999, + "learning_rate": 5e-05, + "loss": 1.1864, + "mean_token_accuracy": 0.6634937524795532, + "num_tokens": 141126659.0, + "step": 341 + }, + { + "epoch": 0.6, + "grad_norm": 0.9974225163459778, + "learning_rate": 5e-05, + "loss": 1.2166, + "mean_token_accuracy": 0.6553642749786377, + "num_tokens": 141551619.0, + "step": 342 + }, + { + "epoch": 0.6017543859649123, + "grad_norm": 0.8490902781486511, + "learning_rate": 5e-05, + "loss": 1.2049, + "mean_token_accuracy": 0.6593654155731201, + "num_tokens": 141970744.0, + "step": 343 + }, + { + "epoch": 0.6035087719298246, + "grad_norm": 0.7998098134994507, + "learning_rate": 5e-05, + "loss": 1.198, + "mean_token_accuracy": 0.6596667766571045, + "num_tokens": 142395401.0, + "step": 344 + }, + { + "epoch": 0.6052631578947368, + "grad_norm": 0.9548800587654114, + "learning_rate": 5e-05, + "loss": 1.2089, + "mean_token_accuracy": 0.6570626497268677, + "num_tokens": 142827910.0, + "step": 345 + }, + { + "epoch": 0.6070175438596491, + "grad_norm": 0.7498379349708557, + "learning_rate": 5e-05, + "loss": 1.2128, + "mean_token_accuracy": 0.6553232669830322, + "num_tokens": 143256693.0, + "step": 346 + }, + { + "epoch": 0.6087719298245614, + "grad_norm": 0.78290194272995, + "learning_rate": 5e-05, + "loss": 1.1947, + "mean_token_accuracy": 0.6607247591018677, + "num_tokens": 143670113.0, + "step": 347 + }, + { + "epoch": 0.6105263157894737, + "grad_norm": 0.7073200345039368, + "learning_rate": 5e-05, + "loss": 1.2138, + "mean_token_accuracy": 0.6558977365493774, + "num_tokens": 144085012.0, + "step": 348 + }, + { + "epoch": 0.612280701754386, + "grad_norm": 0.6999359130859375, + "learning_rate": 5e-05, + "loss": 1.2178, + "mean_token_accuracy": 0.6541372537612915, + "num_tokens": 144511304.0, + "step": 349 + }, + { + "epoch": 0.6140350877192983, + "grad_norm": 0.7888347506523132, + "learning_rate": 5e-05, + "loss": 1.1763, + "mean_token_accuracy": 0.6656613349914551, + "num_tokens": 144903389.0, + "step": 350 + }, + { + "epoch": 0.6157894736842106, + "grad_norm": 0.6909470558166504, + "learning_rate": 5e-05, + "loss": 1.2047, + "mean_token_accuracy": 0.6575745344161987, + "num_tokens": 145317796.0, + "step": 351 + }, + { + "epoch": 0.6175438596491228, + "grad_norm": 0.7789744734764099, + "learning_rate": 5e-05, + "loss": 1.2127, + "mean_token_accuracy": 0.6574368476867676, + "num_tokens": 145728692.0, + "step": 352 + }, + { + "epoch": 0.6192982456140351, + "grad_norm": 0.8671786785125732, + "learning_rate": 5e-05, + "loss": 1.2057, + "mean_token_accuracy": 0.6586547493934631, + "num_tokens": 146149428.0, + "step": 353 + }, + { + "epoch": 0.6210526315789474, + "grad_norm": 0.7211107611656189, + "learning_rate": 5e-05, + "loss": 1.2187, + "mean_token_accuracy": 0.6552145481109619, + "num_tokens": 146567803.0, + "step": 354 + }, + { + "epoch": 0.6228070175438597, + "grad_norm": 0.825245201587677, + "learning_rate": 5e-05, + "loss": 1.2368, + "mean_token_accuracy": 0.6506693363189697, + "num_tokens": 146996287.0, + "step": 355 + }, + { + "epoch": 0.624561403508772, + "grad_norm": 0.6972028613090515, + "learning_rate": 5e-05, + "loss": 1.1866, + "mean_token_accuracy": 0.6610680222511292, + "num_tokens": 147406032.0, + "step": 356 + }, + { + "epoch": 0.6263157894736842, + "grad_norm": 0.9675116539001465, + "learning_rate": 5e-05, + "loss": 1.2344, + "mean_token_accuracy": 0.6520998477935791, + "num_tokens": 147818711.0, + "step": 357 + }, + { + "epoch": 0.6280701754385964, + "grad_norm": 0.6987552642822266, + "learning_rate": 5e-05, + "loss": 1.2036, + "mean_token_accuracy": 0.6586868762969971, + "num_tokens": 148224782.0, + "step": 358 + }, + { + "epoch": 0.6298245614035087, + "grad_norm": 0.7363338470458984, + "learning_rate": 5e-05, + "loss": 1.1921, + "mean_token_accuracy": 0.6602431535720825, + "num_tokens": 148638041.0, + "step": 359 + }, + { + "epoch": 0.631578947368421, + "grad_norm": 0.7094611525535583, + "learning_rate": 5e-05, + "loss": 1.1785, + "mean_token_accuracy": 0.6655131578445435, + "num_tokens": 149034209.0, + "step": 360 + }, + { + "epoch": 0.6333333333333333, + "grad_norm": 0.6311611533164978, + "learning_rate": 5e-05, + "loss": 1.1991, + "mean_token_accuracy": 0.6584141850471497, + "num_tokens": 149462197.0, + "step": 361 + }, + { + "epoch": 0.6350877192982456, + "grad_norm": 0.746914267539978, + "learning_rate": 5e-05, + "loss": 1.2058, + "mean_token_accuracy": 0.6585882902145386, + "num_tokens": 149874309.0, + "step": 362 + }, + { + "epoch": 0.6368421052631579, + "grad_norm": 0.700130820274353, + "learning_rate": 5e-05, + "loss": 1.1954, + "mean_token_accuracy": 0.6615002751350403, + "num_tokens": 150278354.0, + "step": 363 + }, + { + "epoch": 0.6385964912280702, + "grad_norm": 0.7632087469100952, + "learning_rate": 5e-05, + "loss": 1.2099, + "mean_token_accuracy": 0.6557667255401611, + "num_tokens": 150696393.0, + "step": 364 + }, + { + "epoch": 0.6403508771929824, + "grad_norm": 0.6703248023986816, + "learning_rate": 5e-05, + "loss": 1.1982, + "mean_token_accuracy": 0.6600207090377808, + "num_tokens": 151097966.0, + "step": 365 + }, + { + "epoch": 0.6421052631578947, + "grad_norm": 0.7099396586418152, + "learning_rate": 5e-05, + "loss": 1.1887, + "mean_token_accuracy": 0.6626266241073608, + "num_tokens": 151487261.0, + "step": 366 + }, + { + "epoch": 0.643859649122807, + "grad_norm": 0.6625206470489502, + "learning_rate": 5e-05, + "loss": 1.2288, + "mean_token_accuracy": 0.6534708738327026, + "num_tokens": 151915716.0, + "step": 367 + }, + { + "epoch": 0.6456140350877193, + "grad_norm": 0.726699709892273, + "learning_rate": 5e-05, + "loss": 1.1861, + "mean_token_accuracy": 0.6615396738052368, + "num_tokens": 152333716.0, + "step": 368 + }, + { + "epoch": 0.6473684210526316, + "grad_norm": 0.6541877388954163, + "learning_rate": 5e-05, + "loss": 1.2095, + "mean_token_accuracy": 0.6560462713241577, + "num_tokens": 152758004.0, + "step": 369 + }, + { + "epoch": 0.6491228070175439, + "grad_norm": 0.7980314493179321, + "learning_rate": 5e-05, + "loss": 1.2129, + "mean_token_accuracy": 0.6552513241767883, + "num_tokens": 153175726.0, + "step": 370 + }, + { + "epoch": 0.6508771929824562, + "grad_norm": 0.6738728880882263, + "learning_rate": 5e-05, + "loss": 1.1736, + "mean_token_accuracy": 0.6657727956771851, + "num_tokens": 153573526.0, + "step": 371 + }, + { + "epoch": 0.6526315789473685, + "grad_norm": 0.7453187108039856, + "learning_rate": 5e-05, + "loss": 1.2045, + "mean_token_accuracy": 0.6587833166122437, + "num_tokens": 154001303.0, + "step": 372 + }, + { + "epoch": 0.6543859649122807, + "grad_norm": 0.6349968314170837, + "learning_rate": 5e-05, + "loss": 1.1711, + "mean_token_accuracy": 0.6651227474212646, + "num_tokens": 154409874.0, + "step": 373 + }, + { + "epoch": 0.656140350877193, + "grad_norm": 0.7447376251220703, + "learning_rate": 5e-05, + "loss": 1.2015, + "mean_token_accuracy": 0.6600010395050049, + "num_tokens": 154826554.0, + "step": 374 + }, + { + "epoch": 0.6578947368421053, + "grad_norm": 0.7133269906044006, + "learning_rate": 5e-05, + "loss": 1.1959, + "mean_token_accuracy": 0.659218430519104, + "num_tokens": 155230585.0, + "step": 375 + }, + { + "epoch": 0.6596491228070176, + "grad_norm": 0.7269437909126282, + "learning_rate": 5e-05, + "loss": 1.1868, + "mean_token_accuracy": 0.6625350713729858, + "num_tokens": 155636974.0, + "step": 376 + }, + { + "epoch": 0.6614035087719298, + "grad_norm": 0.7790443301200867, + "learning_rate": 5e-05, + "loss": 1.2028, + "mean_token_accuracy": 0.6589221954345703, + "num_tokens": 156063068.0, + "step": 377 + }, + { + "epoch": 0.6631578947368421, + "grad_norm": 0.7665508985519409, + "learning_rate": 5e-05, + "loss": 1.2252, + "mean_token_accuracy": 0.6543481349945068, + "num_tokens": 156510643.0, + "step": 378 + }, + { + "epoch": 0.6649122807017543, + "grad_norm": 0.73358553647995, + "learning_rate": 5e-05, + "loss": 1.208, + "mean_token_accuracy": 0.6577810049057007, + "num_tokens": 156947423.0, + "step": 379 + }, + { + "epoch": 0.6666666666666666, + "grad_norm": 0.7150813341140747, + "learning_rate": 5e-05, + "loss": 1.1982, + "mean_token_accuracy": 0.6588643789291382, + "num_tokens": 157370426.0, + "step": 380 + }, + { + "epoch": 0.6684210526315789, + "grad_norm": 0.7256612777709961, + "learning_rate": 5e-05, + "loss": 1.1948, + "mean_token_accuracy": 0.6601337790489197, + "num_tokens": 157782963.0, + "step": 381 + }, + { + "epoch": 0.6701754385964912, + "grad_norm": 0.6820621490478516, + "learning_rate": 5e-05, + "loss": 1.2126, + "mean_token_accuracy": 0.656718373298645, + "num_tokens": 158180387.0, + "step": 382 + }, + { + "epoch": 0.6719298245614035, + "grad_norm": 0.6703466176986694, + "learning_rate": 5e-05, + "loss": 1.2153, + "mean_token_accuracy": 0.654987096786499, + "num_tokens": 158582401.0, + "step": 383 + }, + { + "epoch": 0.6736842105263158, + "grad_norm": 0.7091477513313293, + "learning_rate": 5e-05, + "loss": 1.1954, + "mean_token_accuracy": 0.66131591796875, + "num_tokens": 158991919.0, + "step": 384 + }, + { + "epoch": 0.6754385964912281, + "grad_norm": 0.7758155465126038, + "learning_rate": 5e-05, + "loss": 1.1926, + "mean_token_accuracy": 0.6596799492835999, + "num_tokens": 159393180.0, + "step": 385 + }, + { + "epoch": 0.6771929824561403, + "grad_norm": 0.7306463718414307, + "learning_rate": 5e-05, + "loss": 1.1978, + "mean_token_accuracy": 0.6593434810638428, + "num_tokens": 159800687.0, + "step": 386 + }, + { + "epoch": 0.6789473684210526, + "grad_norm": 0.7373037338256836, + "learning_rate": 5e-05, + "loss": 1.174, + "mean_token_accuracy": 0.6654571890830994, + "num_tokens": 160206677.0, + "step": 387 + }, + { + "epoch": 0.6807017543859649, + "grad_norm": 0.7122206091880798, + "learning_rate": 5e-05, + "loss": 1.2132, + "mean_token_accuracy": 0.6562681198120117, + "num_tokens": 160623973.0, + "step": 388 + }, + { + "epoch": 0.6824561403508772, + "grad_norm": 0.8452529907226562, + "learning_rate": 5e-05, + "loss": 1.2136, + "mean_token_accuracy": 0.6565843820571899, + "num_tokens": 161033575.0, + "step": 389 + }, + { + "epoch": 0.6842105263157895, + "grad_norm": 0.7118136286735535, + "learning_rate": 5e-05, + "loss": 1.1694, + "mean_token_accuracy": 0.6655601859092712, + "num_tokens": 161420334.0, + "step": 390 + }, + { + "epoch": 0.6859649122807018, + "grad_norm": 0.7205653190612793, + "learning_rate": 5e-05, + "loss": 1.1928, + "mean_token_accuracy": 0.6603007912635803, + "num_tokens": 161823214.0, + "step": 391 + }, + { + "epoch": 0.6877192982456141, + "grad_norm": 0.6563616991043091, + "learning_rate": 5e-05, + "loss": 1.1981, + "mean_token_accuracy": 0.6605692505836487, + "num_tokens": 162253222.0, + "step": 392 + }, + { + "epoch": 0.6894736842105263, + "grad_norm": 0.6877943277359009, + "learning_rate": 5e-05, + "loss": 1.2002, + "mean_token_accuracy": 0.6587787866592407, + "num_tokens": 162673396.0, + "step": 393 + }, + { + "epoch": 0.6912280701754386, + "grad_norm": 0.6227843761444092, + "learning_rate": 5e-05, + "loss": 1.1749, + "mean_token_accuracy": 0.664422869682312, + "num_tokens": 163095701.0, + "step": 394 + }, + { + "epoch": 0.6929824561403509, + "grad_norm": 0.6978362798690796, + "learning_rate": 5e-05, + "loss": 1.2018, + "mean_token_accuracy": 0.6577540636062622, + "num_tokens": 163503487.0, + "step": 395 + }, + { + "epoch": 0.6947368421052632, + "grad_norm": 0.6628575921058655, + "learning_rate": 5e-05, + "loss": 1.1815, + "mean_token_accuracy": 0.6655293107032776, + "num_tokens": 163908638.0, + "step": 396 + }, + { + "epoch": 0.6964912280701754, + "grad_norm": 0.6469285488128662, + "learning_rate": 5e-05, + "loss": 1.1925, + "mean_token_accuracy": 0.6593499779701233, + "num_tokens": 164321664.0, + "step": 397 + }, + { + "epoch": 0.6982456140350877, + "grad_norm": 0.6442559361457825, + "learning_rate": 5e-05, + "loss": 1.1913, + "mean_token_accuracy": 0.6608812212944031, + "num_tokens": 164739198.0, + "step": 398 + }, + { + "epoch": 0.7, + "grad_norm": 0.6495915055274963, + "learning_rate": 5e-05, + "loss": 1.1757, + "mean_token_accuracy": 0.6637240648269653, + "num_tokens": 165130707.0, + "step": 399 + }, + { + "epoch": 0.7017543859649122, + "grad_norm": 0.6610386371612549, + "learning_rate": 5e-05, + "loss": 1.1897, + "mean_token_accuracy": 0.6602373123168945, + "num_tokens": 165544679.0, + "step": 400 + }, + { + "epoch": 0.7035087719298245, + "grad_norm": 0.6443278789520264, + "learning_rate": 5e-05, + "loss": 1.1743, + "mean_token_accuracy": 0.6640031337738037, + "num_tokens": 165963969.0, + "step": 401 + }, + { + "epoch": 0.7052631578947368, + "grad_norm": 0.7723492980003357, + "learning_rate": 5e-05, + "loss": 1.1852, + "mean_token_accuracy": 0.6626303791999817, + "num_tokens": 166342479.0, + "step": 402 + }, + { + "epoch": 0.7070175438596491, + "grad_norm": 0.6442627310752869, + "learning_rate": 5e-05, + "loss": 1.2004, + "mean_token_accuracy": 0.6594836711883545, + "num_tokens": 166746524.0, + "step": 403 + }, + { + "epoch": 0.7087719298245614, + "grad_norm": 0.7476648688316345, + "learning_rate": 5e-05, + "loss": 1.2077, + "mean_token_accuracy": 0.6561458110809326, + "num_tokens": 167166144.0, + "step": 404 + }, + { + "epoch": 0.7105263157894737, + "grad_norm": 0.722258985042572, + "learning_rate": 5e-05, + "loss": 1.196, + "mean_token_accuracy": 0.6601729393005371, + "num_tokens": 167576053.0, + "step": 405 + }, + { + "epoch": 0.712280701754386, + "grad_norm": 0.6938480138778687, + "learning_rate": 5e-05, + "loss": 1.1929, + "mean_token_accuracy": 0.6615767478942871, + "num_tokens": 168005716.0, + "step": 406 + }, + { + "epoch": 0.7140350877192982, + "grad_norm": 0.7194964289665222, + "learning_rate": 5e-05, + "loss": 1.17, + "mean_token_accuracy": 0.6662665009498596, + "num_tokens": 168425061.0, + "step": 407 + }, + { + "epoch": 0.7157894736842105, + "grad_norm": 0.6797705888748169, + "learning_rate": 5e-05, + "loss": 1.197, + "mean_token_accuracy": 0.6588425636291504, + "num_tokens": 168832540.0, + "step": 408 + }, + { + "epoch": 0.7175438596491228, + "grad_norm": 0.6420835256576538, + "learning_rate": 5e-05, + "loss": 1.18, + "mean_token_accuracy": 0.663701057434082, + "num_tokens": 169243355.0, + "step": 409 + }, + { + "epoch": 0.7192982456140351, + "grad_norm": 0.7660720348358154, + "learning_rate": 5e-05, + "loss": 1.2121, + "mean_token_accuracy": 0.6573351621627808, + "num_tokens": 169678758.0, + "step": 410 + }, + { + "epoch": 0.7210526315789474, + "grad_norm": 0.6421164274215698, + "learning_rate": 5e-05, + "loss": 1.2035, + "mean_token_accuracy": 0.6576613187789917, + "num_tokens": 170107843.0, + "step": 411 + }, + { + "epoch": 0.7228070175438597, + "grad_norm": 0.6630116701126099, + "learning_rate": 5e-05, + "loss": 1.1959, + "mean_token_accuracy": 0.660021960735321, + "num_tokens": 170512414.0, + "step": 412 + }, + { + "epoch": 0.724561403508772, + "grad_norm": 0.6951940059661865, + "learning_rate": 5e-05, + "loss": 1.2118, + "mean_token_accuracy": 0.6564462184906006, + "num_tokens": 170927891.0, + "step": 413 + }, + { + "epoch": 0.7263157894736842, + "grad_norm": 0.7424482703208923, + "learning_rate": 5e-05, + "loss": 1.2296, + "mean_token_accuracy": 0.65247642993927, + "num_tokens": 171347602.0, + "step": 414 + }, + { + "epoch": 0.7280701754385965, + "grad_norm": 0.6758566498756409, + "learning_rate": 5e-05, + "loss": 1.1711, + "mean_token_accuracy": 0.6659772992134094, + "num_tokens": 171764325.0, + "step": 415 + }, + { + "epoch": 0.7298245614035088, + "grad_norm": 0.738978922367096, + "learning_rate": 5e-05, + "loss": 1.2116, + "mean_token_accuracy": 0.6572073101997375, + "num_tokens": 172161316.0, + "step": 416 + }, + { + "epoch": 0.7315789473684211, + "grad_norm": 0.7489046454429626, + "learning_rate": 5e-05, + "loss": 1.1733, + "mean_token_accuracy": 0.6655478477478027, + "num_tokens": 172557979.0, + "step": 417 + }, + { + "epoch": 0.7333333333333333, + "grad_norm": 0.6416832804679871, + "learning_rate": 5e-05, + "loss": 1.1938, + "mean_token_accuracy": 0.66144198179245, + "num_tokens": 172977293.0, + "step": 418 + }, + { + "epoch": 0.7350877192982456, + "grad_norm": 0.7506982684135437, + "learning_rate": 5e-05, + "loss": 1.1919, + "mean_token_accuracy": 0.6598606705665588, + "num_tokens": 173386905.0, + "step": 419 + }, + { + "epoch": 0.7368421052631579, + "grad_norm": 0.6610329151153564, + "learning_rate": 5e-05, + "loss": 1.1715, + "mean_token_accuracy": 0.6652780175209045, + "num_tokens": 173790039.0, + "step": 420 + }, + { + "epoch": 0.7385964912280701, + "grad_norm": 0.8175221681594849, + "learning_rate": 5e-05, + "loss": 1.1689, + "mean_token_accuracy": 0.6665405035018921, + "num_tokens": 174200214.0, + "step": 421 + }, + { + "epoch": 0.7403508771929824, + "grad_norm": 0.7499369978904724, + "learning_rate": 5e-05, + "loss": 1.1812, + "mean_token_accuracy": 0.662172794342041, + "num_tokens": 174594155.0, + "step": 422 + }, + { + "epoch": 0.7421052631578947, + "grad_norm": 0.7348382472991943, + "learning_rate": 5e-05, + "loss": 1.1758, + "mean_token_accuracy": 0.6644183993339539, + "num_tokens": 175025672.0, + "step": 423 + }, + { + "epoch": 0.743859649122807, + "grad_norm": 0.6634716391563416, + "learning_rate": 5e-05, + "loss": 1.2127, + "mean_token_accuracy": 0.6543850898742676, + "num_tokens": 175452636.0, + "step": 424 + }, + { + "epoch": 0.7456140350877193, + "grad_norm": 0.7941571474075317, + "learning_rate": 5e-05, + "loss": 1.2025, + "mean_token_accuracy": 0.6584181785583496, + "num_tokens": 175879906.0, + "step": 425 + }, + { + "epoch": 0.7473684210526316, + "grad_norm": 0.6681484580039978, + "learning_rate": 5e-05, + "loss": 1.1921, + "mean_token_accuracy": 0.6602107882499695, + "num_tokens": 176292162.0, + "step": 426 + }, + { + "epoch": 0.7491228070175439, + "grad_norm": 0.6817023158073425, + "learning_rate": 5e-05, + "loss": 1.1837, + "mean_token_accuracy": 0.6628162860870361, + "num_tokens": 176720725.0, + "step": 427 + }, + { + "epoch": 0.7508771929824561, + "grad_norm": 0.7154136300086975, + "learning_rate": 5e-05, + "loss": 1.2116, + "mean_token_accuracy": 0.6570006608963013, + "num_tokens": 177161243.0, + "step": 428 + }, + { + "epoch": 0.7526315789473684, + "grad_norm": 0.6473210453987122, + "learning_rate": 5e-05, + "loss": 1.1741, + "mean_token_accuracy": 0.6632194519042969, + "num_tokens": 177562007.0, + "step": 429 + }, + { + "epoch": 0.7543859649122807, + "grad_norm": 0.6447071433067322, + "learning_rate": 5e-05, + "loss": 1.2026, + "mean_token_accuracy": 0.658052921295166, + "num_tokens": 177983323.0, + "step": 430 + }, + { + "epoch": 0.756140350877193, + "grad_norm": 0.6229775547981262, + "learning_rate": 5e-05, + "loss": 1.1981, + "mean_token_accuracy": 0.6606621742248535, + "num_tokens": 178409419.0, + "step": 431 + }, + { + "epoch": 0.7578947368421053, + "grad_norm": 0.6685755252838135, + "learning_rate": 5e-05, + "loss": 1.1944, + "mean_token_accuracy": 0.6610383987426758, + "num_tokens": 178832025.0, + "step": 432 + }, + { + "epoch": 0.7596491228070176, + "grad_norm": 0.6298593878746033, + "learning_rate": 5e-05, + "loss": 1.2002, + "mean_token_accuracy": 0.6584385633468628, + "num_tokens": 179246477.0, + "step": 433 + }, + { + "epoch": 0.7614035087719299, + "grad_norm": 0.7148140668869019, + "learning_rate": 5e-05, + "loss": 1.2058, + "mean_token_accuracy": 0.658037006855011, + "num_tokens": 179666041.0, + "step": 434 + }, + { + "epoch": 0.7631578947368421, + "grad_norm": 0.6938548684120178, + "learning_rate": 5e-05, + "loss": 1.1955, + "mean_token_accuracy": 0.6596259474754333, + "num_tokens": 180092225.0, + "step": 435 + }, + { + "epoch": 0.7649122807017544, + "grad_norm": 0.7944298386573792, + "learning_rate": 5e-05, + "loss": 1.18, + "mean_token_accuracy": 0.6638443470001221, + "num_tokens": 180498123.0, + "step": 436 + }, + { + "epoch": 0.7666666666666667, + "grad_norm": 0.6547574400901794, + "learning_rate": 5e-05, + "loss": 1.1813, + "mean_token_accuracy": 0.6638117432594299, + "num_tokens": 180904735.0, + "step": 437 + }, + { + "epoch": 0.7684210526315789, + "grad_norm": 0.677518904209137, + "learning_rate": 5e-05, + "loss": 1.183, + "mean_token_accuracy": 0.6625351905822754, + "num_tokens": 181297827.0, + "step": 438 + }, + { + "epoch": 0.7701754385964912, + "grad_norm": 0.6108941435813904, + "learning_rate": 5e-05, + "loss": 1.1937, + "mean_token_accuracy": 0.6619063019752502, + "num_tokens": 181722096.0, + "step": 439 + }, + { + "epoch": 0.7719298245614035, + "grad_norm": 0.7047777771949768, + "learning_rate": 5e-05, + "loss": 1.1727, + "mean_token_accuracy": 0.6657708883285522, + "num_tokens": 182124515.0, + "step": 440 + }, + { + "epoch": 0.7736842105263158, + "grad_norm": 0.7206352353096008, + "learning_rate": 5e-05, + "loss": 1.2011, + "mean_token_accuracy": 0.657702624797821, + "num_tokens": 182542265.0, + "step": 441 + }, + { + "epoch": 0.775438596491228, + "grad_norm": 0.6818886995315552, + "learning_rate": 5e-05, + "loss": 1.1942, + "mean_token_accuracy": 0.6611602902412415, + "num_tokens": 182942368.0, + "step": 442 + }, + { + "epoch": 0.7771929824561403, + "grad_norm": 0.7941774725914001, + "learning_rate": 5e-05, + "loss": 1.1874, + "mean_token_accuracy": 0.6601078510284424, + "num_tokens": 183350652.0, + "step": 443 + }, + { + "epoch": 0.7789473684210526, + "grad_norm": 0.6606309413909912, + "learning_rate": 5e-05, + "loss": 1.195, + "mean_token_accuracy": 0.6604108810424805, + "num_tokens": 183786290.0, + "step": 444 + }, + { + "epoch": 0.7807017543859649, + "grad_norm": 0.6471251249313354, + "learning_rate": 5e-05, + "loss": 1.1875, + "mean_token_accuracy": 0.6615259647369385, + "num_tokens": 184191985.0, + "step": 445 + }, + { + "epoch": 0.7824561403508772, + "grad_norm": 0.6947017312049866, + "learning_rate": 5e-05, + "loss": 1.209, + "mean_token_accuracy": 0.655540406703949, + "num_tokens": 184617087.0, + "step": 446 + }, + { + "epoch": 0.7842105263157895, + "grad_norm": 0.6960648894309998, + "learning_rate": 5e-05, + "loss": 1.1874, + "mean_token_accuracy": 0.6617487072944641, + "num_tokens": 185046113.0, + "step": 447 + }, + { + "epoch": 0.7859649122807018, + "grad_norm": 0.6484550833702087, + "learning_rate": 5e-05, + "loss": 1.1588, + "mean_token_accuracy": 0.667752742767334, + "num_tokens": 185440210.0, + "step": 448 + }, + { + "epoch": 0.787719298245614, + "grad_norm": 0.6320661902427673, + "learning_rate": 5e-05, + "loss": 1.1997, + "mean_token_accuracy": 0.6575033664703369, + "num_tokens": 185866240.0, + "step": 449 + }, + { + "epoch": 0.7894736842105263, + "grad_norm": 0.6956246495246887, + "learning_rate": 5e-05, + "loss": 1.165, + "mean_token_accuracy": 0.6674104928970337, + "num_tokens": 186262559.0, + "step": 450 + }, + { + "epoch": 0.7912280701754386, + "grad_norm": 0.7143962383270264, + "learning_rate": 5e-05, + "loss": 1.2132, + "mean_token_accuracy": 0.6562595367431641, + "num_tokens": 186677017.0, + "step": 451 + }, + { + "epoch": 0.7929824561403509, + "grad_norm": 0.6602803468704224, + "learning_rate": 5e-05, + "loss": 1.207, + "mean_token_accuracy": 0.6583276987075806, + "num_tokens": 187108135.0, + "step": 452 + }, + { + "epoch": 0.7947368421052632, + "grad_norm": 0.685832142829895, + "learning_rate": 5e-05, + "loss": 1.1764, + "mean_token_accuracy": 0.6635936498641968, + "num_tokens": 187533913.0, + "step": 453 + }, + { + "epoch": 0.7964912280701755, + "grad_norm": 0.6198664903640747, + "learning_rate": 5e-05, + "loss": 1.1988, + "mean_token_accuracy": 0.6600010395050049, + "num_tokens": 187943777.0, + "step": 454 + }, + { + "epoch": 0.7982456140350878, + "grad_norm": 0.6220059990882874, + "learning_rate": 5e-05, + "loss": 1.1812, + "mean_token_accuracy": 0.6627987027168274, + "num_tokens": 188365013.0, + "step": 455 + }, + { + "epoch": 0.8, + "grad_norm": 0.6069747805595398, + "learning_rate": 5e-05, + "loss": 1.1831, + "mean_token_accuracy": 0.6625472903251648, + "num_tokens": 188805546.0, + "step": 456 + }, + { + "epoch": 0.8017543859649123, + "grad_norm": 0.6519631743431091, + "learning_rate": 5e-05, + "loss": 1.1652, + "mean_token_accuracy": 0.6660283803939819, + "num_tokens": 189204261.0, + "step": 457 + }, + { + "epoch": 0.8035087719298246, + "grad_norm": 0.7023555040359497, + "learning_rate": 5e-05, + "loss": 1.1899, + "mean_token_accuracy": 0.6618468165397644, + "num_tokens": 189609457.0, + "step": 458 + }, + { + "epoch": 0.8052631578947368, + "grad_norm": 0.7214339971542358, + "learning_rate": 5e-05, + "loss": 1.1939, + "mean_token_accuracy": 0.659986138343811, + "num_tokens": 190024904.0, + "step": 459 + }, + { + "epoch": 0.8070175438596491, + "grad_norm": 0.6715176105499268, + "learning_rate": 5e-05, + "loss": 1.1759, + "mean_token_accuracy": 0.6641864776611328, + "num_tokens": 190435843.0, + "step": 460 + }, + { + "epoch": 0.8087719298245614, + "grad_norm": 0.7698047161102295, + "learning_rate": 5e-05, + "loss": 1.195, + "mean_token_accuracy": 0.6599795818328857, + "num_tokens": 190844879.0, + "step": 461 + }, + { + "epoch": 0.8105263157894737, + "grad_norm": 0.734397292137146, + "learning_rate": 5e-05, + "loss": 1.1796, + "mean_token_accuracy": 0.6632078289985657, + "num_tokens": 191240668.0, + "step": 462 + }, + { + "epoch": 0.8122807017543859, + "grad_norm": 0.7458048462867737, + "learning_rate": 5e-05, + "loss": 1.1925, + "mean_token_accuracy": 0.6606088876724243, + "num_tokens": 191649912.0, + "step": 463 + }, + { + "epoch": 0.8140350877192982, + "grad_norm": 0.8290356993675232, + "learning_rate": 5e-05, + "loss": 1.1792, + "mean_token_accuracy": 0.6643441915512085, + "num_tokens": 192065093.0, + "step": 464 + }, + { + "epoch": 0.8157894736842105, + "grad_norm": 0.6535662412643433, + "learning_rate": 5e-05, + "loss": 1.1985, + "mean_token_accuracy": 0.6606610417366028, + "num_tokens": 192458897.0, + "step": 465 + }, + { + "epoch": 0.8175438596491228, + "grad_norm": 0.7194675803184509, + "learning_rate": 5e-05, + "loss": 1.1938, + "mean_token_accuracy": 0.6604172587394714, + "num_tokens": 192871837.0, + "step": 466 + }, + { + "epoch": 0.8192982456140351, + "grad_norm": 0.686488687992096, + "learning_rate": 5e-05, + "loss": 1.1742, + "mean_token_accuracy": 0.6656160950660706, + "num_tokens": 193297231.0, + "step": 467 + }, + { + "epoch": 0.8210526315789474, + "grad_norm": 0.7032576203346252, + "learning_rate": 5e-05, + "loss": 1.1476, + "mean_token_accuracy": 0.6692982316017151, + "num_tokens": 193684403.0, + "step": 468 + }, + { + "epoch": 0.8228070175438597, + "grad_norm": 0.7109776735305786, + "learning_rate": 5e-05, + "loss": 1.1981, + "mean_token_accuracy": 0.6577274799346924, + "num_tokens": 194113357.0, + "step": 469 + }, + { + "epoch": 0.8245614035087719, + "grad_norm": 0.7627134919166565, + "learning_rate": 5e-05, + "loss": 1.1821, + "mean_token_accuracy": 0.6613542437553406, + "num_tokens": 194526469.0, + "step": 470 + }, + { + "epoch": 0.8263157894736842, + "grad_norm": 0.6512476801872253, + "learning_rate": 5e-05, + "loss": 1.2159, + "mean_token_accuracy": 0.6537120342254639, + "num_tokens": 194953909.0, + "step": 471 + }, + { + "epoch": 0.8280701754385965, + "grad_norm": 0.6464257836341858, + "learning_rate": 5e-05, + "loss": 1.2098, + "mean_token_accuracy": 0.655582070350647, + "num_tokens": 195388769.0, + "step": 472 + }, + { + "epoch": 0.8298245614035088, + "grad_norm": 0.7239501476287842, + "learning_rate": 5e-05, + "loss": 1.2033, + "mean_token_accuracy": 0.6582602858543396, + "num_tokens": 195818916.0, + "step": 473 + }, + { + "epoch": 0.8315789473684211, + "grad_norm": 0.6906052231788635, + "learning_rate": 5e-05, + "loss": 1.1812, + "mean_token_accuracy": 0.6626557111740112, + "num_tokens": 196233034.0, + "step": 474 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 0.6602339148521423, + "learning_rate": 5e-05, + "loss": 1.2233, + "mean_token_accuracy": 0.6518707871437073, + "num_tokens": 196660034.0, + "step": 475 + }, + { + "epoch": 0.8350877192982457, + "grad_norm": 0.6464066505432129, + "learning_rate": 5e-05, + "loss": 1.2022, + "mean_token_accuracy": 0.6586562395095825, + "num_tokens": 197080673.0, + "step": 476 + }, + { + "epoch": 0.8368421052631579, + "grad_norm": 0.6947892904281616, + "learning_rate": 5e-05, + "loss": 1.1799, + "mean_token_accuracy": 0.66390061378479, + "num_tokens": 197493003.0, + "step": 477 + }, + { + "epoch": 0.8385964912280702, + "grad_norm": 0.7277309894561768, + "learning_rate": 5e-05, + "loss": 1.1946, + "mean_token_accuracy": 0.6587433815002441, + "num_tokens": 197923635.0, + "step": 478 + }, + { + "epoch": 0.8403508771929824, + "grad_norm": 0.6581082940101624, + "learning_rate": 5e-05, + "loss": 1.1864, + "mean_token_accuracy": 0.6605808734893799, + "num_tokens": 198339854.0, + "step": 479 + }, + { + "epoch": 0.8421052631578947, + "grad_norm": 0.775787353515625, + "learning_rate": 5e-05, + "loss": 1.2254, + "mean_token_accuracy": 0.6519793272018433, + "num_tokens": 198768483.0, + "step": 480 + }, + { + "epoch": 0.843859649122807, + "grad_norm": 0.7034079432487488, + "learning_rate": 5e-05, + "loss": 1.2004, + "mean_token_accuracy": 0.6575764417648315, + "num_tokens": 199190842.0, + "step": 481 + }, + { + "epoch": 0.8456140350877193, + "grad_norm": 0.7499285340309143, + "learning_rate": 5e-05, + "loss": 1.2057, + "mean_token_accuracy": 0.657794713973999, + "num_tokens": 199620649.0, + "step": 482 + }, + { + "epoch": 0.8473684210526315, + "grad_norm": 0.6442464590072632, + "learning_rate": 5e-05, + "loss": 1.1845, + "mean_token_accuracy": 0.6626601219177246, + "num_tokens": 200023406.0, + "step": 483 + }, + { + "epoch": 0.8491228070175438, + "grad_norm": 0.6592628359794617, + "learning_rate": 5e-05, + "loss": 1.162, + "mean_token_accuracy": 0.667891263961792, + "num_tokens": 200430239.0, + "step": 484 + }, + { + "epoch": 0.8508771929824561, + "grad_norm": 0.6535340547561646, + "learning_rate": 5e-05, + "loss": 1.1766, + "mean_token_accuracy": 0.6650654673576355, + "num_tokens": 200833864.0, + "step": 485 + }, + { + "epoch": 0.8526315789473684, + "grad_norm": 0.6712135076522827, + "learning_rate": 5e-05, + "loss": 1.1788, + "mean_token_accuracy": 0.6628377437591553, + "num_tokens": 201241869.0, + "step": 486 + }, + { + "epoch": 0.8543859649122807, + "grad_norm": 0.7537463903427124, + "learning_rate": 5e-05, + "loss": 1.173, + "mean_token_accuracy": 0.66404128074646, + "num_tokens": 201658653.0, + "step": 487 + }, + { + "epoch": 0.856140350877193, + "grad_norm": 0.6496484875679016, + "learning_rate": 5e-05, + "loss": 1.1947, + "mean_token_accuracy": 0.659920871257782, + "num_tokens": 202086503.0, + "step": 488 + }, + { + "epoch": 0.8578947368421053, + "grad_norm": 0.7035285830497742, + "learning_rate": 5e-05, + "loss": 1.1848, + "mean_token_accuracy": 0.6625712513923645, + "num_tokens": 202509306.0, + "step": 489 + }, + { + "epoch": 0.8596491228070176, + "grad_norm": 0.6473917365074158, + "learning_rate": 5e-05, + "loss": 1.1621, + "mean_token_accuracy": 0.6654759645462036, + "num_tokens": 202909087.0, + "step": 490 + }, + { + "epoch": 0.8614035087719298, + "grad_norm": 0.7222784757614136, + "learning_rate": 5e-05, + "loss": 1.1878, + "mean_token_accuracy": 0.6602382659912109, + "num_tokens": 203337615.0, + "step": 491 + }, + { + "epoch": 0.8631578947368421, + "grad_norm": 0.6569610834121704, + "learning_rate": 5e-05, + "loss": 1.1861, + "mean_token_accuracy": 0.6620035767555237, + "num_tokens": 203754527.0, + "step": 492 + }, + { + "epoch": 0.8649122807017544, + "grad_norm": 0.6757194399833679, + "learning_rate": 5e-05, + "loss": 1.1596, + "mean_token_accuracy": 0.6687979698181152, + "num_tokens": 204166768.0, + "step": 493 + }, + { + "epoch": 0.8666666666666667, + "grad_norm": 0.6936085820198059, + "learning_rate": 5e-05, + "loss": 1.1778, + "mean_token_accuracy": 0.6632059812545776, + "num_tokens": 204554076.0, + "step": 494 + }, + { + "epoch": 0.868421052631579, + "grad_norm": 0.6749398112297058, + "learning_rate": 5e-05, + "loss": 1.2125, + "mean_token_accuracy": 0.6559756994247437, + "num_tokens": 204962578.0, + "step": 495 + }, + { + "epoch": 0.8701754385964913, + "grad_norm": 0.6298931837081909, + "learning_rate": 5e-05, + "loss": 1.1391, + "mean_token_accuracy": 0.6719960570335388, + "num_tokens": 205358271.0, + "step": 496 + }, + { + "epoch": 0.8719298245614036, + "grad_norm": 0.6448364853858948, + "learning_rate": 5e-05, + "loss": 1.195, + "mean_token_accuracy": 0.6596945524215698, + "num_tokens": 205786890.0, + "step": 497 + }, + { + "epoch": 0.8736842105263158, + "grad_norm": 0.8100766539573669, + "learning_rate": 5e-05, + "loss": 1.1876, + "mean_token_accuracy": 0.6618038415908813, + "num_tokens": 206193572.0, + "step": 498 + }, + { + "epoch": 0.875438596491228, + "grad_norm": 0.6642587780952454, + "learning_rate": 5e-05, + "loss": 1.1601, + "mean_token_accuracy": 0.6680558323860168, + "num_tokens": 206607450.0, + "step": 499 + }, + { + "epoch": 0.8771929824561403, + "grad_norm": 0.8082707524299622, + "learning_rate": 5e-05, + "loss": 1.1663, + "mean_token_accuracy": 0.665715754032135, + "num_tokens": 207005126.0, + "step": 500 + }, + { + "epoch": 0.8789473684210526, + "grad_norm": 0.7876964211463928, + "learning_rate": 5e-05, + "loss": 1.1682, + "mean_token_accuracy": 0.6669149994850159, + "num_tokens": 207394794.0, + "step": 501 + }, + { + "epoch": 0.8807017543859649, + "grad_norm": 0.6647291779518127, + "learning_rate": 5e-05, + "loss": 1.1872, + "mean_token_accuracy": 0.6603274345397949, + "num_tokens": 207783539.0, + "step": 502 + }, + { + "epoch": 0.8824561403508772, + "grad_norm": 0.6698031425476074, + "learning_rate": 5e-05, + "loss": 1.1901, + "mean_token_accuracy": 0.6615040302276611, + "num_tokens": 208196198.0, + "step": 503 + }, + { + "epoch": 0.8842105263157894, + "grad_norm": 0.7431910634040833, + "learning_rate": 5e-05, + "loss": 1.1847, + "mean_token_accuracy": 0.6631199717521667, + "num_tokens": 208602598.0, + "step": 504 + }, + { + "epoch": 0.8859649122807017, + "grad_norm": 0.5865485072135925, + "learning_rate": 5e-05, + "loss": 1.1768, + "mean_token_accuracy": 0.6640779972076416, + "num_tokens": 209039139.0, + "step": 505 + }, + { + "epoch": 0.887719298245614, + "grad_norm": 0.7942367792129517, + "learning_rate": 5e-05, + "loss": 1.1846, + "mean_token_accuracy": 0.6616294384002686, + "num_tokens": 209437950.0, + "step": 506 + }, + { + "epoch": 0.8894736842105263, + "grad_norm": 0.7096176147460938, + "learning_rate": 5e-05, + "loss": 1.191, + "mean_token_accuracy": 0.6605682969093323, + "num_tokens": 209862559.0, + "step": 507 + }, + { + "epoch": 0.8912280701754386, + "grad_norm": 0.5890538096427917, + "learning_rate": 5e-05, + "loss": 1.2079, + "mean_token_accuracy": 0.656398355960846, + "num_tokens": 210284769.0, + "step": 508 + }, + { + "epoch": 0.8929824561403509, + "grad_norm": 0.6946651935577393, + "learning_rate": 5e-05, + "loss": 1.1907, + "mean_token_accuracy": 0.6619917154312134, + "num_tokens": 210708859.0, + "step": 509 + }, + { + "epoch": 0.8947368421052632, + "grad_norm": 0.7260644435882568, + "learning_rate": 5e-05, + "loss": 1.1865, + "mean_token_accuracy": 0.6614673137664795, + "num_tokens": 211130168.0, + "step": 510 + }, + { + "epoch": 0.8964912280701754, + "grad_norm": 0.654033899307251, + "learning_rate": 5e-05, + "loss": 1.1897, + "mean_token_accuracy": 0.6604927778244019, + "num_tokens": 211529753.0, + "step": 511 + }, + { + "epoch": 0.8982456140350877, + "grad_norm": 0.7885578274726868, + "learning_rate": 5e-05, + "loss": 1.2024, + "mean_token_accuracy": 0.656429648399353, + "num_tokens": 211963430.0, + "step": 512 + }, + { + "epoch": 0.9, + "grad_norm": 0.655292809009552, + "learning_rate": 5e-05, + "loss": 1.1997, + "mean_token_accuracy": 0.6585859656333923, + "num_tokens": 212387404.0, + "step": 513 + }, + { + "epoch": 0.9017543859649123, + "grad_norm": 0.7295964360237122, + "learning_rate": 5e-05, + "loss": 1.1586, + "mean_token_accuracy": 0.6679401397705078, + "num_tokens": 212802763.0, + "step": 514 + }, + { + "epoch": 0.9035087719298246, + "grad_norm": 0.746534526348114, + "learning_rate": 5e-05, + "loss": 1.157, + "mean_token_accuracy": 0.6686700582504272, + "num_tokens": 213219406.0, + "step": 515 + }, + { + "epoch": 0.9052631578947369, + "grad_norm": 0.762389600276947, + "learning_rate": 5e-05, + "loss": 1.1812, + "mean_token_accuracy": 0.662742555141449, + "num_tokens": 213623710.0, + "step": 516 + }, + { + "epoch": 0.9070175438596492, + "grad_norm": 0.7516655921936035, + "learning_rate": 5e-05, + "loss": 1.1579, + "mean_token_accuracy": 0.6695092916488647, + "num_tokens": 214053806.0, + "step": 517 + }, + { + "epoch": 0.9087719298245615, + "grad_norm": 0.7952564358711243, + "learning_rate": 5e-05, + "loss": 1.1877, + "mean_token_accuracy": 0.6593484878540039, + "num_tokens": 214457394.0, + "step": 518 + }, + { + "epoch": 0.9105263157894737, + "grad_norm": 0.7105541825294495, + "learning_rate": 5e-05, + "loss": 1.1545, + "mean_token_accuracy": 0.6694096326828003, + "num_tokens": 214841472.0, + "step": 519 + }, + { + "epoch": 0.9122807017543859, + "grad_norm": 0.716000497341156, + "learning_rate": 5e-05, + "loss": 1.1931, + "mean_token_accuracy": 0.6585883498191833, + "num_tokens": 215261046.0, + "step": 520 + }, + { + "epoch": 0.9140350877192982, + "grad_norm": 0.7107662558555603, + "learning_rate": 5e-05, + "loss": 1.1587, + "mean_token_accuracy": 0.6692352294921875, + "num_tokens": 215664423.0, + "step": 521 + }, + { + "epoch": 0.9157894736842105, + "grad_norm": 0.7039586901664734, + "learning_rate": 5e-05, + "loss": 1.1683, + "mean_token_accuracy": 0.6676132678985596, + "num_tokens": 216072733.0, + "step": 522 + }, + { + "epoch": 0.9175438596491228, + "grad_norm": 0.791039764881134, + "learning_rate": 5e-05, + "loss": 1.1815, + "mean_token_accuracy": 0.6626390814781189, + "num_tokens": 216490157.0, + "step": 523 + }, + { + "epoch": 0.9192982456140351, + "grad_norm": 0.6723469495773315, + "learning_rate": 5e-05, + "loss": 1.1989, + "mean_token_accuracy": 0.6579208970069885, + "num_tokens": 216933714.0, + "step": 524 + }, + { + "epoch": 0.9210526315789473, + "grad_norm": 0.827643871307373, + "learning_rate": 5e-05, + "loss": 1.1855, + "mean_token_accuracy": 0.6619547605514526, + "num_tokens": 217360826.0, + "step": 525 + }, + { + "epoch": 0.9228070175438596, + "grad_norm": 0.6785314083099365, + "learning_rate": 5e-05, + "loss": 1.1696, + "mean_token_accuracy": 0.665156364440918, + "num_tokens": 217767964.0, + "step": 526 + }, + { + "epoch": 0.9245614035087719, + "grad_norm": 0.9181134104728699, + "learning_rate": 5e-05, + "loss": 1.1656, + "mean_token_accuracy": 0.6664823293685913, + "num_tokens": 218182502.0, + "step": 527 + }, + { + "epoch": 0.9263157894736842, + "grad_norm": 0.6506194472312927, + "learning_rate": 5e-05, + "loss": 1.1716, + "mean_token_accuracy": 0.6655004024505615, + "num_tokens": 218599164.0, + "step": 528 + }, + { + "epoch": 0.9280701754385965, + "grad_norm": 0.8218353390693665, + "learning_rate": 5e-05, + "loss": 1.1578, + "mean_token_accuracy": 0.6688790321350098, + "num_tokens": 219038853.0, + "step": 529 + }, + { + "epoch": 0.9298245614035088, + "grad_norm": 0.7387883067131042, + "learning_rate": 5e-05, + "loss": 1.1776, + "mean_token_accuracy": 0.663921594619751, + "num_tokens": 219455763.0, + "step": 530 + }, + { + "epoch": 0.9315789473684211, + "grad_norm": 0.7437043786048889, + "learning_rate": 5e-05, + "loss": 1.1873, + "mean_token_accuracy": 0.6606233716011047, + "num_tokens": 219857096.0, + "step": 531 + }, + { + "epoch": 0.9333333333333333, + "grad_norm": 0.7363501191139221, + "learning_rate": 5e-05, + "loss": 1.1604, + "mean_token_accuracy": 0.6665854454040527, + "num_tokens": 220269043.0, + "step": 532 + }, + { + "epoch": 0.9350877192982456, + "grad_norm": 0.6709656715393066, + "learning_rate": 5e-05, + "loss": 1.1904, + "mean_token_accuracy": 0.6600399017333984, + "num_tokens": 220650778.0, + "step": 533 + }, + { + "epoch": 0.9368421052631579, + "grad_norm": 0.6969729065895081, + "learning_rate": 5e-05, + "loss": 1.1875, + "mean_token_accuracy": 0.6613034009933472, + "num_tokens": 221055825.0, + "step": 534 + }, + { + "epoch": 0.9385964912280702, + "grad_norm": 0.6233665943145752, + "learning_rate": 5e-05, + "loss": 1.1959, + "mean_token_accuracy": 0.657875657081604, + "num_tokens": 221459868.0, + "step": 535 + }, + { + "epoch": 0.9403508771929825, + "grad_norm": 0.6261913776397705, + "learning_rate": 5e-05, + "loss": 1.1543, + "mean_token_accuracy": 0.667622447013855, + "num_tokens": 221860324.0, + "step": 536 + }, + { + "epoch": 0.9421052631578948, + "grad_norm": 0.6652575135231018, + "learning_rate": 5e-05, + "loss": 1.1624, + "mean_token_accuracy": 0.6667903065681458, + "num_tokens": 222291251.0, + "step": 537 + }, + { + "epoch": 0.9438596491228071, + "grad_norm": 0.5863179564476013, + "learning_rate": 5e-05, + "loss": 1.1642, + "mean_token_accuracy": 0.6657068729400635, + "num_tokens": 222699559.0, + "step": 538 + }, + { + "epoch": 0.9456140350877194, + "grad_norm": 0.6168889999389648, + "learning_rate": 5e-05, + "loss": 1.1762, + "mean_token_accuracy": 0.664853572845459, + "num_tokens": 223156357.0, + "step": 539 + }, + { + "epoch": 0.9473684210526315, + "grad_norm": 0.678865373134613, + "learning_rate": 5e-05, + "loss": 1.1677, + "mean_token_accuracy": 0.6656055450439453, + "num_tokens": 223562411.0, + "step": 540 + }, + { + "epoch": 0.9491228070175438, + "grad_norm": 0.6533005237579346, + "learning_rate": 5e-05, + "loss": 1.1897, + "mean_token_accuracy": 0.6595481038093567, + "num_tokens": 223988252.0, + "step": 541 + }, + { + "epoch": 0.9508771929824561, + "grad_norm": 0.6714079976081848, + "learning_rate": 5e-05, + "loss": 1.1757, + "mean_token_accuracy": 0.6642969846725464, + "num_tokens": 224401217.0, + "step": 542 + }, + { + "epoch": 0.9526315789473684, + "grad_norm": 0.6550496816635132, + "learning_rate": 5e-05, + "loss": 1.1632, + "mean_token_accuracy": 0.6655498743057251, + "num_tokens": 224807482.0, + "step": 543 + }, + { + "epoch": 0.9543859649122807, + "grad_norm": 0.7725386619567871, + "learning_rate": 5e-05, + "loss": 1.1771, + "mean_token_accuracy": 0.6629381775856018, + "num_tokens": 225220680.0, + "step": 544 + }, + { + "epoch": 0.956140350877193, + "grad_norm": 0.6665419340133667, + "learning_rate": 5e-05, + "loss": 1.2241, + "mean_token_accuracy": 0.6517584919929504, + "num_tokens": 225640539.0, + "step": 545 + }, + { + "epoch": 0.9578947368421052, + "grad_norm": 0.8708847165107727, + "learning_rate": 5e-05, + "loss": 1.1678, + "mean_token_accuracy": 0.6656101942062378, + "num_tokens": 226035924.0, + "step": 546 + }, + { + "epoch": 0.9596491228070175, + "grad_norm": 0.7241430878639221, + "learning_rate": 5e-05, + "loss": 1.1502, + "mean_token_accuracy": 0.6692663431167603, + "num_tokens": 226425232.0, + "step": 547 + }, + { + "epoch": 0.9614035087719298, + "grad_norm": 0.8228368759155273, + "learning_rate": 5e-05, + "loss": 1.1575, + "mean_token_accuracy": 0.6680091619491577, + "num_tokens": 226802383.0, + "step": 548 + }, + { + "epoch": 0.9631578947368421, + "grad_norm": 0.7209904789924622, + "learning_rate": 5e-05, + "loss": 1.1825, + "mean_token_accuracy": 0.6617472171783447, + "num_tokens": 227227610.0, + "step": 549 + }, + { + "epoch": 0.9649122807017544, + "grad_norm": 0.7651249766349792, + "learning_rate": 5e-05, + "loss": 1.1878, + "mean_token_accuracy": 0.6614329814910889, + "num_tokens": 227657882.0, + "step": 550 + }, + { + "epoch": 0.9666666666666667, + "grad_norm": 0.7364425659179688, + "learning_rate": 5e-05, + "loss": 1.1605, + "mean_token_accuracy": 0.6667755842208862, + "num_tokens": 228081460.0, + "step": 551 + }, + { + "epoch": 0.968421052631579, + "grad_norm": 0.7921107411384583, + "learning_rate": 5e-05, + "loss": 1.2029, + "mean_token_accuracy": 0.6565576195716858, + "num_tokens": 228507388.0, + "step": 552 + }, + { + "epoch": 0.9701754385964912, + "grad_norm": 0.726375162601471, + "learning_rate": 5e-05, + "loss": 1.1982, + "mean_token_accuracy": 0.6580156683921814, + "num_tokens": 228935033.0, + "step": 553 + }, + { + "epoch": 0.9719298245614035, + "grad_norm": 0.7788160443305969, + "learning_rate": 5e-05, + "loss": 1.1381, + "mean_token_accuracy": 0.6721576452255249, + "num_tokens": 229325690.0, + "step": 554 + }, + { + "epoch": 0.9736842105263158, + "grad_norm": 0.7351102232933044, + "learning_rate": 5e-05, + "loss": 1.1455, + "mean_token_accuracy": 0.6698772311210632, + "num_tokens": 229726549.0, + "step": 555 + }, + { + "epoch": 0.9754385964912281, + "grad_norm": 0.7573174834251404, + "learning_rate": 5e-05, + "loss": 1.1726, + "mean_token_accuracy": 0.663779616355896, + "num_tokens": 230122274.0, + "step": 556 + }, + { + "epoch": 0.9771929824561404, + "grad_norm": 0.7851194143295288, + "learning_rate": 5e-05, + "loss": 1.1838, + "mean_token_accuracy": 0.6609175205230713, + "num_tokens": 230526104.0, + "step": 557 + }, + { + "epoch": 0.9789473684210527, + "grad_norm": 0.7095761299133301, + "learning_rate": 5e-05, + "loss": 1.1489, + "mean_token_accuracy": 0.6705464124679565, + "num_tokens": 230941199.0, + "step": 558 + }, + { + "epoch": 0.980701754385965, + "grad_norm": 0.723162829875946, + "learning_rate": 5e-05, + "loss": 1.1612, + "mean_token_accuracy": 0.667212724685669, + "num_tokens": 231367872.0, + "step": 559 + }, + { + "epoch": 0.9824561403508771, + "grad_norm": 0.6912490725517273, + "learning_rate": 5e-05, + "loss": 1.1907, + "mean_token_accuracy": 0.6610735058784485, + "num_tokens": 231804511.0, + "step": 560 + }, + { + "epoch": 0.9842105263157894, + "grad_norm": 0.7252631187438965, + "learning_rate": 5e-05, + "loss": 1.1532, + "mean_token_accuracy": 0.6693242192268372, + "num_tokens": 232201554.0, + "step": 561 + }, + { + "epoch": 0.9859649122807017, + "grad_norm": 0.6097338795661926, + "learning_rate": 5e-05, + "loss": 1.1685, + "mean_token_accuracy": 0.6653523445129395, + "num_tokens": 232632247.0, + "step": 562 + }, + { + "epoch": 0.987719298245614, + "grad_norm": 0.6657564043998718, + "learning_rate": 5e-05, + "loss": 1.1781, + "mean_token_accuracy": 0.661851167678833, + "num_tokens": 233040299.0, + "step": 563 + }, + { + "epoch": 0.9894736842105263, + "grad_norm": 0.7004382610321045, + "learning_rate": 5e-05, + "loss": 1.1837, + "mean_token_accuracy": 0.6624906659126282, + "num_tokens": 233476898.0, + "step": 564 + }, + { + "epoch": 0.9912280701754386, + "grad_norm": 0.6217873096466064, + "learning_rate": 5e-05, + "loss": 1.154, + "mean_token_accuracy": 0.668339729309082, + "num_tokens": 233898203.0, + "step": 565 + }, + { + "epoch": 0.9929824561403509, + "grad_norm": 0.6709463596343994, + "learning_rate": 5e-05, + "loss": 1.2068, + "mean_token_accuracy": 0.656959056854248, + "num_tokens": 234316491.0, + "step": 566 + }, + { + "epoch": 0.9947368421052631, + "grad_norm": 0.6852229237556458, + "learning_rate": 5e-05, + "loss": 1.1701, + "mean_token_accuracy": 0.6647340059280396, + "num_tokens": 234725455.0, + "step": 567 + }, + { + "epoch": 0.9964912280701754, + "grad_norm": 0.6170585751533508, + "learning_rate": 5e-05, + "loss": 1.1583, + "mean_token_accuracy": 0.6682082414627075, + "num_tokens": 235152094.0, + "step": 568 + }, + { + "epoch": 0.9982456140350877, + "grad_norm": 0.6941448450088501, + "learning_rate": 5e-05, + "loss": 1.1723, + "mean_token_accuracy": 0.6631691455841064, + "num_tokens": 235558911.0, + "step": 569 + }, + { + "epoch": 1.0, + "grad_norm": 0.5916408896446228, + "learning_rate": 5e-05, + "loss": 1.1826, + "mean_token_accuracy": 0.6617175340652466, + "num_tokens": 235994347.0, + "step": 570 + }, + { + "epoch": 1.0, + "step": 570, + "total_flos": 1.377941890090926e+18, + "train_loss": 1.2427511761063024, + "train_runtime": 1307.8696, + "train_samples_per_second": 111.397, + "train_steps_per_second": 0.436 + } + ], + "logging_steps": 1, + "max_steps": 570, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 285, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.377941890090926e+18, + "train_batch_size": 64, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..61e7913 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb879d0c49ee5762ad527eb5643313b8dd9ebcde6ea3c5dd264e56dff06ce2a2 +size 13329