From ef6501d01cbe222404dfda56c7af1c3403069ca6 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 31 Aug 2026 10:14:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: Neelectric/Llama-3.2-1B-Instruct_SFT_sciencev00.03 Source: Original Platform --- .gitattributes | 36 + README.md | 60 + all_results.json | 8 + chat_template.jinja | 121 + config.json | 35 + generation_config.json | 8 + model.safetensors | 3 + special_tokens_map.json | 10 + tokenizer.json | 3 + tokenizer_config.json | 2062 ++++++++++++++ train_results.json | 8 + trainer_state.json | 5713 +++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 13 files changed, 8070 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..5269155 --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +--- +base_model: meta-llama/Llama-3.2-1B-Instruct +datasets: Neelectric/MoT_science_Llama3_2048toks +library_name: transformers +model_name: Llama-3.2-1B-Instruct_SFT_sciencev00.03 +tags: +- generated_from_trainer +- trl +- open-r1 +- sft +licence: license +--- + +# Model Card for Llama-3.2-1B-Instruct_SFT_sciencev00.03 + +This model is a fine-tuned version of [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) on the [Neelectric/MoT_science_Llama3_2048toks](https://huggingface.co/datasets/Neelectric/MoT_science_Llama3_2048toks) dataset. +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="Neelectric/Llama-3.2-1B-Instruct_SFT_sciencev00.03", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/neelectric/open-r1_science/runs/70zwq4xi) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 1.0.0.dev0 +- Transformers: 4.57.6 +- Pytorch: 2.9.0 +- Datasets: 4.8.3 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..b3a31d3 --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "total_flos": 2.1853937174671524e+18, + "train_loss": 1.0672233128358448, + "train_runtime": 1837.0629, + "train_samples": 107517, + "train_samples_per_second": 175.58, + "train_steps_per_second": 0.343 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..0ab931a --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,121 @@ +{{- bos_token }} +{%- if custom_tools is defined %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if not tools_in_user_message is defined %} + {%- set tools_in_user_message = true %} +{%- endif %} +{%- if not date_string is defined %} + {%- set date_string = "26 Jul 2024" %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- set system_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} +{%- else %} + {%- set system_message = "You are a helpful AI Assistant that provides well-reasoned and detailed responses. You first think about the reasoning process as an internal monologue and then provide the user with the answer. Respond in the following format: +... + + +... +" %} +{%- endif %} + +{#- System message + builtin tools #} +{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} +{%- if builtin_tools is defined or tools is not none %} + {{- "Environment: ipython\n" }} +{%- endif %} +{%- if builtin_tools is defined %} + {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}} +{%- endif %} +{{- "Cutting Knowledge Date: December 2023\n" }} +{{- "Today Date: " + date_string + "\n\n" }} +{%- if tools is not none and not tools_in_user_message %} + {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} +{%- endif %} +{{- system_message }} +{{- "<|eot_id|>" }} + +{#- Custom tools are passed in a user message with some extra guidance #} +{%- if tools_in_user_message and not tools is none %} + {#- Extract the first user message so we can plug it in here #} + {%- if messages | length != 0 %} + {%- set first_user_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} + {%- else %} + {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} + {%- endif %} + {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} + {{- "Given the following functions, please respond with a JSON for a function call " }} + {{- "with its proper arguments that best answers the given prompt.\n\n" }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} + {{- first_user_message + "<|eot_id|>"}} +{%- endif %} + +{%- for message in messages %} + {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} + {%- if message['role'] == 'assistant' %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} + {% generation %} + {{- message['content'] | trim + '<|eot_id|>' }} + {% endgeneration %} + {%- else %} + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} + {%- endif %} + {%- elif 'tool_calls' in message %} + {%- if not message.tool_calls|length == 1 %} + {{- raise_exception("This model only supports single tool-calls at once!") }} + {%- endif %} + {%- set tool_call = message.tool_calls[0].function %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} + {% generation %} + {%- if builtin_tools is defined and tool_call.name in builtin_tools %} + {{- "<|python_tag|>" + tool_call.name + ".call(" }} + {%- for arg_name, arg_val in tool_call.arguments | items %} + {{- arg_name + '="' + arg_val + '"' }} + {%- if not loop.last %} + {{- ", " }} + {%- endif %} + {%- endfor %} + {{- ")" }} + {%- else %} + {{- '{"name": "' + tool_call.name + '", ' }} + {{- '"parameters": ' }} + {{- tool_call.arguments | tojson }} + {{- "}" }} + {%- endif %} + {%- if builtin_tools is defined %} + {{- "<|eom_id|>" }} + {%- else %} + {{- "<|eot_id|>" }} + {%- endif %} + {% endgeneration %} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} + {%- if message.content is mapping or message.content is iterable %} + {{- message.content | tojson }} + {%- else %} + {{- message.content }} + {%- endif %} + {{- "<|eot_id|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..9912fbf --- /dev/null +++ b/config.json @@ -0,0 +1,35 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128009, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..1996dc1 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,8 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128009, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.6" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..d40e9d1 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cdf7633be940fe3840da14e651da08f0defea480d24783e26e8a5ab9ae04d03 +size 2996982344 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e8f05fa --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,10 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": "<|eot_id|>" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8b0c7c1 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2062 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|eot_id|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..b3a31d3 --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "total_flos": 2.1853937174671524e+18, + "train_loss": 1.0672233128358448, + "train_runtime": 1837.0629, + "train_samples": 107517, + "train_samples_per_second": 175.58, + "train_steps_per_second": 0.343 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..ccaa03e --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,5713 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 630, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004761904761904762, + "grad_norm": 8.287551879882812, + "learning_rate": 0.0, + "loss": 1.7656, + "mean_token_accuracy": 0.5768666863441467, + "num_tokens": 582781.0, + "step": 1 + }, + { + "epoch": 0.009523809523809525, + "grad_norm": 8.260395050048828, + "learning_rate": 7.936507936507937e-07, + "loss": 1.7728, + "mean_token_accuracy": 0.5752322673797607, + "num_tokens": 1163696.0, + "step": 2 + }, + { + "epoch": 0.014285714285714285, + "grad_norm": 8.0443696975708, + "learning_rate": 1.5873015873015873e-06, + "loss": 1.7745, + "mean_token_accuracy": 0.5747243762016296, + "num_tokens": 1762000.0, + "step": 3 + }, + { + "epoch": 0.01904761904761905, + "grad_norm": 7.40345573425293, + "learning_rate": 2.3809523809523808e-06, + "loss": 1.7639, + "mean_token_accuracy": 0.5749228000640869, + "num_tokens": 2363228.0, + "step": 4 + }, + { + "epoch": 0.023809523809523808, + "grad_norm": 5.93461799621582, + "learning_rate": 3.1746031746031746e-06, + "loss": 1.7486, + "mean_token_accuracy": 0.5752354264259338, + "num_tokens": 2968748.0, + "step": 5 + }, + { + "epoch": 0.02857142857142857, + "grad_norm": 4.171263694763184, + "learning_rate": 3.968253968253968e-06, + "loss": 1.6533, + "mean_token_accuracy": 0.5862137079238892, + "num_tokens": 3564062.0, + "step": 6 + }, + { + "epoch": 0.03333333333333333, + "grad_norm": 4.0062642097473145, + "learning_rate": 4.7619047619047615e-06, + "loss": 1.6529, + "mean_token_accuracy": 0.5845167636871338, + "num_tokens": 4140352.0, + "step": 7 + }, + { + "epoch": 0.0380952380952381, + "grad_norm": 2.9634792804718018, + "learning_rate": 5.555555555555556e-06, + "loss": 1.5874, + "mean_token_accuracy": 0.5953940749168396, + "num_tokens": 4748067.0, + "step": 8 + }, + { + "epoch": 0.04285714285714286, + "grad_norm": 2.9049317836761475, + "learning_rate": 6.349206349206349e-06, + "loss": 1.5549, + "mean_token_accuracy": 0.60173499584198, + "num_tokens": 5333791.0, + "step": 9 + }, + { + "epoch": 0.047619047619047616, + "grad_norm": 2.269242763519287, + "learning_rate": 7.142857142857143e-06, + "loss": 1.5464, + "mean_token_accuracy": 0.6025688648223877, + "num_tokens": 5923564.0, + "step": 10 + }, + { + "epoch": 0.05238095238095238, + "grad_norm": 3.1737020015716553, + "learning_rate": 7.936507936507936e-06, + "loss": 1.4959, + "mean_token_accuracy": 0.6135456562042236, + "num_tokens": 6528559.0, + "step": 11 + }, + { + "epoch": 0.05714285714285714, + "grad_norm": 2.8201091289520264, + "learning_rate": 8.73015873015873e-06, + "loss": 1.4915, + "mean_token_accuracy": 0.6120684146881104, + "num_tokens": 7118765.0, + "step": 12 + }, + { + "epoch": 0.06190476190476191, + "grad_norm": 2.0840981006622314, + "learning_rate": 9.523809523809523e-06, + "loss": 1.472, + "mean_token_accuracy": 0.6149868369102478, + "num_tokens": 7709226.0, + "step": 13 + }, + { + "epoch": 0.06666666666666667, + "grad_norm": 1.5017718076705933, + "learning_rate": 1.0317460317460318e-05, + "loss": 1.4304, + "mean_token_accuracy": 0.6240715384483337, + "num_tokens": 8298984.0, + "step": 14 + }, + { + "epoch": 0.07142857142857142, + "grad_norm": 1.772209882736206, + "learning_rate": 1.1111111111111112e-05, + "loss": 1.4155, + "mean_token_accuracy": 0.6238371729850769, + "num_tokens": 8875624.0, + "step": 15 + }, + { + "epoch": 0.0761904761904762, + "grad_norm": 1.635779857635498, + "learning_rate": 1.1904761904761905e-05, + "loss": 1.413, + "mean_token_accuracy": 0.6242497563362122, + "num_tokens": 9448671.0, + "step": 16 + }, + { + "epoch": 0.08095238095238096, + "grad_norm": 1.3846430778503418, + "learning_rate": 1.2698412698412699e-05, + "loss": 1.3944, + "mean_token_accuracy": 0.629358172416687, + "num_tokens": 10049546.0, + "step": 17 + }, + { + "epoch": 0.08571428571428572, + "grad_norm": 1.1820706129074097, + "learning_rate": 1.3492063492063492e-05, + "loss": 1.3912, + "mean_token_accuracy": 0.6291320323944092, + "num_tokens": 10644905.0, + "step": 18 + }, + { + "epoch": 0.09047619047619047, + "grad_norm": 1.140526294708252, + "learning_rate": 1.4285714285714285e-05, + "loss": 1.3864, + "mean_token_accuracy": 0.6298290491104126, + "num_tokens": 11239583.0, + "step": 19 + }, + { + "epoch": 0.09523809523809523, + "grad_norm": 1.0532516241073608, + "learning_rate": 1.5079365079365079e-05, + "loss": 1.3639, + "mean_token_accuracy": 0.6349307894706726, + "num_tokens": 11827320.0, + "step": 20 + }, + { + "epoch": 0.1, + "grad_norm": 0.9888433218002319, + "learning_rate": 1.5873015873015872e-05, + "loss": 1.3397, + "mean_token_accuracy": 0.6381456851959229, + "num_tokens": 12425511.0, + "step": 21 + }, + { + "epoch": 0.10476190476190476, + "grad_norm": 0.8685981035232544, + "learning_rate": 1.6666666666666667e-05, + "loss": 1.3508, + "mean_token_accuracy": 0.6360146999359131, + "num_tokens": 13015708.0, + "step": 22 + }, + { + "epoch": 0.10952380952380952, + "grad_norm": 0.9491143822669983, + "learning_rate": 1.746031746031746e-05, + "loss": 1.3372, + "mean_token_accuracy": 0.6386818289756775, + "num_tokens": 13608875.0, + "step": 23 + }, + { + "epoch": 0.11428571428571428, + "grad_norm": 0.8696007132530212, + "learning_rate": 1.8253968253968254e-05, + "loss": 1.3447, + "mean_token_accuracy": 0.6366680860519409, + "num_tokens": 14204297.0, + "step": 24 + }, + { + "epoch": 0.11904761904761904, + "grad_norm": 0.8588811159133911, + "learning_rate": 1.9047619047619046e-05, + "loss": 1.3363, + "mean_token_accuracy": 0.6377676725387573, + "num_tokens": 14782206.0, + "step": 25 + }, + { + "epoch": 0.12380952380952381, + "grad_norm": 0.9697495698928833, + "learning_rate": 1.984126984126984e-05, + "loss": 1.3192, + "mean_token_accuracy": 0.6418357491493225, + "num_tokens": 15377829.0, + "step": 26 + }, + { + "epoch": 0.12857142857142856, + "grad_norm": 0.9554264545440674, + "learning_rate": 2.0634920634920636e-05, + "loss": 1.3176, + "mean_token_accuracy": 0.64220130443573, + "num_tokens": 15975819.0, + "step": 27 + }, + { + "epoch": 0.13333333333333333, + "grad_norm": 0.8665664792060852, + "learning_rate": 2.1428571428571428e-05, + "loss": 1.3081, + "mean_token_accuracy": 0.6432693004608154, + "num_tokens": 16577839.0, + "step": 28 + }, + { + "epoch": 0.1380952380952381, + "grad_norm": 0.8949389457702637, + "learning_rate": 2.2222222222222223e-05, + "loss": 1.3049, + "mean_token_accuracy": 0.6440079808235168, + "num_tokens": 17164831.0, + "step": 29 + }, + { + "epoch": 0.14285714285714285, + "grad_norm": 0.8900285959243774, + "learning_rate": 2.3015873015873015e-05, + "loss": 1.2963, + "mean_token_accuracy": 0.6464663743972778, + "num_tokens": 17770476.0, + "step": 30 + }, + { + "epoch": 0.14761904761904762, + "grad_norm": 0.8141711950302124, + "learning_rate": 2.380952380952381e-05, + "loss": 1.2874, + "mean_token_accuracy": 0.6475774049758911, + "num_tokens": 18360862.0, + "step": 31 + }, + { + "epoch": 0.1523809523809524, + "grad_norm": 0.7370226979255676, + "learning_rate": 2.4603174603174602e-05, + "loss": 1.3027, + "mean_token_accuracy": 0.6437569856643677, + "num_tokens": 18944338.0, + "step": 32 + }, + { + "epoch": 0.15714285714285714, + "grad_norm": 1.0480390787124634, + "learning_rate": 2.5396825396825397e-05, + "loss": 1.2813, + "mean_token_accuracy": 0.6486168503761292, + "num_tokens": 19540189.0, + "step": 33 + }, + { + "epoch": 0.1619047619047619, + "grad_norm": 0.7855182886123657, + "learning_rate": 2.6190476190476192e-05, + "loss": 1.2813, + "mean_token_accuracy": 0.6489517688751221, + "num_tokens": 20138131.0, + "step": 34 + }, + { + "epoch": 0.16666666666666666, + "grad_norm": 0.8817404508590698, + "learning_rate": 2.6984126984126984e-05, + "loss": 1.2992, + "mean_token_accuracy": 0.644393801689148, + "num_tokens": 20735187.0, + "step": 35 + }, + { + "epoch": 0.17142857142857143, + "grad_norm": 1.100029706954956, + "learning_rate": 2.777777777777778e-05, + "loss": 1.2781, + "mean_token_accuracy": 0.6481138467788696, + "num_tokens": 21316383.0, + "step": 36 + }, + { + "epoch": 0.1761904761904762, + "grad_norm": 0.9269927740097046, + "learning_rate": 2.857142857142857e-05, + "loss": 1.2596, + "mean_token_accuracy": 0.6538517475128174, + "num_tokens": 21910626.0, + "step": 37 + }, + { + "epoch": 0.18095238095238095, + "grad_norm": 0.7763503193855286, + "learning_rate": 2.9365079365079366e-05, + "loss": 1.283, + "mean_token_accuracy": 0.6467125415802002, + "num_tokens": 22503955.0, + "step": 38 + }, + { + "epoch": 0.18571428571428572, + "grad_norm": 0.9469724297523499, + "learning_rate": 3.0158730158730158e-05, + "loss": 1.2789, + "mean_token_accuracy": 0.6480865478515625, + "num_tokens": 23093310.0, + "step": 39 + }, + { + "epoch": 0.19047619047619047, + "grad_norm": 0.9910866618156433, + "learning_rate": 3.095238095238095e-05, + "loss": 1.2535, + "mean_token_accuracy": 0.6537003517150879, + "num_tokens": 23681028.0, + "step": 40 + }, + { + "epoch": 0.19523809523809524, + "grad_norm": 0.8376869559288025, + "learning_rate": 3.1746031746031745e-05, + "loss": 1.2551, + "mean_token_accuracy": 0.6538482904434204, + "num_tokens": 24275532.0, + "step": 41 + }, + { + "epoch": 0.2, + "grad_norm": 0.8315210938453674, + "learning_rate": 3.253968253968254e-05, + "loss": 1.2594, + "mean_token_accuracy": 0.6529629230499268, + "num_tokens": 24868054.0, + "step": 42 + }, + { + "epoch": 0.20476190476190476, + "grad_norm": 0.9830642342567444, + "learning_rate": 3.3333333333333335e-05, + "loss": 1.2651, + "mean_token_accuracy": 0.6512579917907715, + "num_tokens": 25459842.0, + "step": 43 + }, + { + "epoch": 0.20952380952380953, + "grad_norm": 1.1936790943145752, + "learning_rate": 3.412698412698413e-05, + "loss": 1.275, + "mean_token_accuracy": 0.6482353210449219, + "num_tokens": 26051340.0, + "step": 44 + }, + { + "epoch": 0.21428571428571427, + "grad_norm": 0.7959531545639038, + "learning_rate": 3.492063492063492e-05, + "loss": 1.2535, + "mean_token_accuracy": 0.6540487408638, + "num_tokens": 26635240.0, + "step": 45 + }, + { + "epoch": 0.21904761904761905, + "grad_norm": 0.8500040769577026, + "learning_rate": 3.571428571428572e-05, + "loss": 1.2552, + "mean_token_accuracy": 0.6532554626464844, + "num_tokens": 27228570.0, + "step": 46 + }, + { + "epoch": 0.22380952380952382, + "grad_norm": 1.065184473991394, + "learning_rate": 3.650793650793651e-05, + "loss": 1.2508, + "mean_token_accuracy": 0.65426105260849, + "num_tokens": 27825958.0, + "step": 47 + }, + { + "epoch": 0.22857142857142856, + "grad_norm": 0.93488609790802, + "learning_rate": 3.730158730158731e-05, + "loss": 1.2471, + "mean_token_accuracy": 0.653727650642395, + "num_tokens": 28418470.0, + "step": 48 + }, + { + "epoch": 0.23333333333333334, + "grad_norm": 0.8569839596748352, + "learning_rate": 3.809523809523809e-05, + "loss": 1.2435, + "mean_token_accuracy": 0.6544537544250488, + "num_tokens": 29013060.0, + "step": 49 + }, + { + "epoch": 0.23809523809523808, + "grad_norm": 1.1500039100646973, + "learning_rate": 3.888888888888889e-05, + "loss": 1.2556, + "mean_token_accuracy": 0.6510920524597168, + "num_tokens": 29624709.0, + "step": 50 + }, + { + "epoch": 0.24285714285714285, + "grad_norm": 1.1380728483200073, + "learning_rate": 3.968253968253968e-05, + "loss": 1.2487, + "mean_token_accuracy": 0.6535077095031738, + "num_tokens": 30227928.0, + "step": 51 + }, + { + "epoch": 0.24761904761904763, + "grad_norm": 0.8196055889129639, + "learning_rate": 4.047619047619048e-05, + "loss": 1.2509, + "mean_token_accuracy": 0.6520772576332092, + "num_tokens": 30823383.0, + "step": 52 + }, + { + "epoch": 0.2523809523809524, + "grad_norm": 0.9668157696723938, + "learning_rate": 4.126984126984127e-05, + "loss": 1.2332, + "mean_token_accuracy": 0.6577878594398499, + "num_tokens": 31418593.0, + "step": 53 + }, + { + "epoch": 0.2571428571428571, + "grad_norm": 1.0810840129852295, + "learning_rate": 4.2063492063492065e-05, + "loss": 1.2263, + "mean_token_accuracy": 0.6584261655807495, + "num_tokens": 32008377.0, + "step": 54 + }, + { + "epoch": 0.2619047619047619, + "grad_norm": 0.7663713693618774, + "learning_rate": 4.2857142857142856e-05, + "loss": 1.2342, + "mean_token_accuracy": 0.655688464641571, + "num_tokens": 32600302.0, + "step": 55 + }, + { + "epoch": 0.26666666666666666, + "grad_norm": 1.0739625692367554, + "learning_rate": 4.3650793650793655e-05, + "loss": 1.2368, + "mean_token_accuracy": 0.6552919149398804, + "num_tokens": 33201147.0, + "step": 56 + }, + { + "epoch": 0.2714285714285714, + "grad_norm": 1.0154988765716553, + "learning_rate": 4.4444444444444447e-05, + "loss": 1.2191, + "mean_token_accuracy": 0.6595079898834229, + "num_tokens": 33790565.0, + "step": 57 + }, + { + "epoch": 0.2761904761904762, + "grad_norm": 1.041022777557373, + "learning_rate": 4.523809523809524e-05, + "loss": 1.2242, + "mean_token_accuracy": 0.6588901281356812, + "num_tokens": 34387187.0, + "step": 58 + }, + { + "epoch": 0.28095238095238095, + "grad_norm": 1.3004519939422607, + "learning_rate": 4.603174603174603e-05, + "loss": 1.2347, + "mean_token_accuracy": 0.656902551651001, + "num_tokens": 35000480.0, + "step": 59 + }, + { + "epoch": 0.2857142857142857, + "grad_norm": 1.0212417840957642, + "learning_rate": 4.682539682539683e-05, + "loss": 1.2166, + "mean_token_accuracy": 0.6611014604568481, + "num_tokens": 35588577.0, + "step": 60 + }, + { + "epoch": 0.2904761904761905, + "grad_norm": 1.020236611366272, + "learning_rate": 4.761904761904762e-05, + "loss": 1.2215, + "mean_token_accuracy": 0.659095287322998, + "num_tokens": 36179186.0, + "step": 61 + }, + { + "epoch": 0.29523809523809524, + "grad_norm": 1.1656397581100464, + "learning_rate": 4.841269841269841e-05, + "loss": 1.2306, + "mean_token_accuracy": 0.6576155424118042, + "num_tokens": 36787338.0, + "step": 62 + }, + { + "epoch": 0.3, + "grad_norm": 0.9852789044380188, + "learning_rate": 4.9206349206349204e-05, + "loss": 1.2194, + "mean_token_accuracy": 0.6592267155647278, + "num_tokens": 37376232.0, + "step": 63 + }, + { + "epoch": 0.3047619047619048, + "grad_norm": 1.0741709470748901, + "learning_rate": 5e-05, + "loss": 1.2121, + "mean_token_accuracy": 0.6610227227210999, + "num_tokens": 37965066.0, + "step": 64 + }, + { + "epoch": 0.30952380952380953, + "grad_norm": 1.2849828004837036, + "learning_rate": 5e-05, + "loss": 1.2258, + "mean_token_accuracy": 0.6583669185638428, + "num_tokens": 38556474.0, + "step": 65 + }, + { + "epoch": 0.3142857142857143, + "grad_norm": 1.2494887113571167, + "learning_rate": 5e-05, + "loss": 1.2377, + "mean_token_accuracy": 0.6551421284675598, + "num_tokens": 39153957.0, + "step": 66 + }, + { + "epoch": 0.319047619047619, + "grad_norm": 1.1150826215744019, + "learning_rate": 5e-05, + "loss": 1.2252, + "mean_token_accuracy": 0.6574854850769043, + "num_tokens": 39743079.0, + "step": 67 + }, + { + "epoch": 0.3238095238095238, + "grad_norm": 0.867920458316803, + "learning_rate": 5e-05, + "loss": 1.2076, + "mean_token_accuracy": 0.6632025837898254, + "num_tokens": 40341422.0, + "step": 68 + }, + { + "epoch": 0.32857142857142857, + "grad_norm": 1.3973991870880127, + "learning_rate": 5e-05, + "loss": 1.2113, + "mean_token_accuracy": 0.6611201763153076, + "num_tokens": 40930579.0, + "step": 69 + }, + { + "epoch": 0.3333333333333333, + "grad_norm": 1.280426263809204, + "learning_rate": 5e-05, + "loss": 1.2061, + "mean_token_accuracy": 0.6619083881378174, + "num_tokens": 41521392.0, + "step": 70 + }, + { + "epoch": 0.3380952380952381, + "grad_norm": 1.076837182044983, + "learning_rate": 5e-05, + "loss": 1.2226, + "mean_token_accuracy": 0.658558189868927, + "num_tokens": 42126117.0, + "step": 71 + }, + { + "epoch": 0.34285714285714286, + "grad_norm": 0.8278794288635254, + "learning_rate": 5e-05, + "loss": 1.2, + "mean_token_accuracy": 0.6640195846557617, + "num_tokens": 42717085.0, + "step": 72 + }, + { + "epoch": 0.3476190476190476, + "grad_norm": 1.4381461143493652, + "learning_rate": 5e-05, + "loss": 1.2075, + "mean_token_accuracy": 0.6619212031364441, + "num_tokens": 43300932.0, + "step": 73 + }, + { + "epoch": 0.3523809523809524, + "grad_norm": 1.0904641151428223, + "learning_rate": 5e-05, + "loss": 1.2197, + "mean_token_accuracy": 0.6577476263046265, + "num_tokens": 43885512.0, + "step": 74 + }, + { + "epoch": 0.35714285714285715, + "grad_norm": 1.2840248346328735, + "learning_rate": 5e-05, + "loss": 1.2073, + "mean_token_accuracy": 0.6614718437194824, + "num_tokens": 44482626.0, + "step": 75 + }, + { + "epoch": 0.3619047619047619, + "grad_norm": 1.0928597450256348, + "learning_rate": 5e-05, + "loss": 1.2138, + "mean_token_accuracy": 0.6594449281692505, + "num_tokens": 45073331.0, + "step": 76 + }, + { + "epoch": 0.36666666666666664, + "grad_norm": 1.3326466083526611, + "learning_rate": 5e-05, + "loss": 1.2136, + "mean_token_accuracy": 0.6602523326873779, + "num_tokens": 45683001.0, + "step": 77 + }, + { + "epoch": 0.37142857142857144, + "grad_norm": 0.9598885178565979, + "learning_rate": 5e-05, + "loss": 1.2174, + "mean_token_accuracy": 0.6601754426956177, + "num_tokens": 46280871.0, + "step": 78 + }, + { + "epoch": 0.3761904761904762, + "grad_norm": 0.9827783703804016, + "learning_rate": 5e-05, + "loss": 1.1955, + "mean_token_accuracy": 0.6635642647743225, + "num_tokens": 46860927.0, + "step": 79 + }, + { + "epoch": 0.38095238095238093, + "grad_norm": 1.2690178155899048, + "learning_rate": 5e-05, + "loss": 1.2121, + "mean_token_accuracy": 0.6599856019020081, + "num_tokens": 47450747.0, + "step": 80 + }, + { + "epoch": 0.38571428571428573, + "grad_norm": 0.9417280554771423, + "learning_rate": 5e-05, + "loss": 1.2092, + "mean_token_accuracy": 0.6623343825340271, + "num_tokens": 48053355.0, + "step": 81 + }, + { + "epoch": 0.3904761904761905, + "grad_norm": 0.8554616570472717, + "learning_rate": 5e-05, + "loss": 1.1784, + "mean_token_accuracy": 0.6671629548072815, + "num_tokens": 48654406.0, + "step": 82 + }, + { + "epoch": 0.3952380952380952, + "grad_norm": 1.1767923831939697, + "learning_rate": 5e-05, + "loss": 1.1889, + "mean_token_accuracy": 0.6649227142333984, + "num_tokens": 49253902.0, + "step": 83 + }, + { + "epoch": 0.4, + "grad_norm": 0.8467581868171692, + "learning_rate": 5e-05, + "loss": 1.2042, + "mean_token_accuracy": 0.6616181135177612, + "num_tokens": 49851728.0, + "step": 84 + }, + { + "epoch": 0.40476190476190477, + "grad_norm": 1.162235975265503, + "learning_rate": 5e-05, + "loss": 1.2124, + "mean_token_accuracy": 0.6599748730659485, + "num_tokens": 50456288.0, + "step": 85 + }, + { + "epoch": 0.4095238095238095, + "grad_norm": 0.7732124924659729, + "learning_rate": 5e-05, + "loss": 1.2147, + "mean_token_accuracy": 0.6592139005661011, + "num_tokens": 51058176.0, + "step": 86 + }, + { + "epoch": 0.4142857142857143, + "grad_norm": 1.1737257242202759, + "learning_rate": 5e-05, + "loss": 1.202, + "mean_token_accuracy": 0.6626513004302979, + "num_tokens": 51665178.0, + "step": 87 + }, + { + "epoch": 0.41904761904761906, + "grad_norm": 1.1886131763458252, + "learning_rate": 5e-05, + "loss": 1.1871, + "mean_token_accuracy": 0.6647380590438843, + "num_tokens": 52237427.0, + "step": 88 + }, + { + "epoch": 0.4238095238095238, + "grad_norm": 0.9629137516021729, + "learning_rate": 5e-05, + "loss": 1.2096, + "mean_token_accuracy": 0.6598652005195618, + "num_tokens": 52850605.0, + "step": 89 + }, + { + "epoch": 0.42857142857142855, + "grad_norm": 1.1625380516052246, + "learning_rate": 5e-05, + "loss": 1.1726, + "mean_token_accuracy": 0.6677452325820923, + "num_tokens": 53435907.0, + "step": 90 + }, + { + "epoch": 0.43333333333333335, + "grad_norm": 1.0230753421783447, + "learning_rate": 5e-05, + "loss": 1.208, + "mean_token_accuracy": 0.6604536771774292, + "num_tokens": 54032690.0, + "step": 91 + }, + { + "epoch": 0.4380952380952381, + "grad_norm": 1.1533459424972534, + "learning_rate": 5e-05, + "loss": 1.1946, + "mean_token_accuracy": 0.664383053779602, + "num_tokens": 54631908.0, + "step": 92 + }, + { + "epoch": 0.44285714285714284, + "grad_norm": 1.1243093013763428, + "learning_rate": 5e-05, + "loss": 1.189, + "mean_token_accuracy": 0.66476970911026, + "num_tokens": 55218598.0, + "step": 93 + }, + { + "epoch": 0.44761904761904764, + "grad_norm": 0.8233395218849182, + "learning_rate": 5e-05, + "loss": 1.2095, + "mean_token_accuracy": 0.659702479839325, + "num_tokens": 55814642.0, + "step": 94 + }, + { + "epoch": 0.4523809523809524, + "grad_norm": 1.1013133525848389, + "learning_rate": 5e-05, + "loss": 1.1839, + "mean_token_accuracy": 0.6664130091667175, + "num_tokens": 56410000.0, + "step": 95 + }, + { + "epoch": 0.45714285714285713, + "grad_norm": 1.107353925704956, + "learning_rate": 5e-05, + "loss": 1.1929, + "mean_token_accuracy": 0.6642213463783264, + "num_tokens": 57001902.0, + "step": 96 + }, + { + "epoch": 0.46190476190476193, + "grad_norm": 1.1815845966339111, + "learning_rate": 5e-05, + "loss": 1.2038, + "mean_token_accuracy": 0.6620410680770874, + "num_tokens": 57612227.0, + "step": 97 + }, + { + "epoch": 0.4666666666666667, + "grad_norm": 0.8402481079101562, + "learning_rate": 5e-05, + "loss": 1.1875, + "mean_token_accuracy": 0.6639487743377686, + "num_tokens": 58198983.0, + "step": 98 + }, + { + "epoch": 0.4714285714285714, + "grad_norm": 1.0578429698944092, + "learning_rate": 5e-05, + "loss": 1.2076, + "mean_token_accuracy": 0.660643458366394, + "num_tokens": 58805739.0, + "step": 99 + }, + { + "epoch": 0.47619047619047616, + "grad_norm": 1.1170893907546997, + "learning_rate": 5e-05, + "loss": 1.1766, + "mean_token_accuracy": 0.6673372983932495, + "num_tokens": 59386596.0, + "step": 100 + }, + { + "epoch": 0.48095238095238096, + "grad_norm": 0.829818606376648, + "learning_rate": 5e-05, + "loss": 1.1836, + "mean_token_accuracy": 0.6650121212005615, + "num_tokens": 59976677.0, + "step": 101 + }, + { + "epoch": 0.4857142857142857, + "grad_norm": 1.1198710203170776, + "learning_rate": 5e-05, + "loss": 1.1965, + "mean_token_accuracy": 0.6630198955535889, + "num_tokens": 60581023.0, + "step": 102 + }, + { + "epoch": 0.49047619047619045, + "grad_norm": 1.0050970315933228, + "learning_rate": 5e-05, + "loss": 1.1868, + "mean_token_accuracy": 0.6658117771148682, + "num_tokens": 61177587.0, + "step": 103 + }, + { + "epoch": 0.49523809523809526, + "grad_norm": 0.9390196800231934, + "learning_rate": 5e-05, + "loss": 1.1815, + "mean_token_accuracy": 0.6655706167221069, + "num_tokens": 61759739.0, + "step": 104 + }, + { + "epoch": 0.5, + "grad_norm": 0.8263099789619446, + "learning_rate": 5e-05, + "loss": 1.1889, + "mean_token_accuracy": 0.6644082069396973, + "num_tokens": 62343623.0, + "step": 105 + }, + { + "epoch": 0.5047619047619047, + "grad_norm": 1.0954078435897827, + "learning_rate": 5e-05, + "loss": 1.1819, + "mean_token_accuracy": 0.6660667061805725, + "num_tokens": 62936609.0, + "step": 106 + }, + { + "epoch": 0.5095238095238095, + "grad_norm": 1.0054658651351929, + "learning_rate": 5e-05, + "loss": 1.1798, + "mean_token_accuracy": 0.6665365695953369, + "num_tokens": 63540035.0, + "step": 107 + }, + { + "epoch": 0.5142857142857142, + "grad_norm": 0.7802014946937561, + "learning_rate": 5e-05, + "loss": 1.1849, + "mean_token_accuracy": 0.6653165817260742, + "num_tokens": 64137406.0, + "step": 108 + }, + { + "epoch": 0.5190476190476191, + "grad_norm": 0.917020857334137, + "learning_rate": 5e-05, + "loss": 1.1689, + "mean_token_accuracy": 0.6689997911453247, + "num_tokens": 64747343.0, + "step": 109 + }, + { + "epoch": 0.5238095238095238, + "grad_norm": 0.8191772103309631, + "learning_rate": 5e-05, + "loss": 1.1684, + "mean_token_accuracy": 0.6685223579406738, + "num_tokens": 65340433.0, + "step": 110 + }, + { + "epoch": 0.5285714285714286, + "grad_norm": 0.9644578695297241, + "learning_rate": 5e-05, + "loss": 1.1649, + "mean_token_accuracy": 0.6692923307418823, + "num_tokens": 65928375.0, + "step": 111 + }, + { + "epoch": 0.5333333333333333, + "grad_norm": 0.8696696758270264, + "learning_rate": 5e-05, + "loss": 1.1894, + "mean_token_accuracy": 0.6640723943710327, + "num_tokens": 66527455.0, + "step": 112 + }, + { + "epoch": 0.5380952380952381, + "grad_norm": 0.7868416905403137, + "learning_rate": 5e-05, + "loss": 1.1593, + "mean_token_accuracy": 0.6710422039031982, + "num_tokens": 67120147.0, + "step": 113 + }, + { + "epoch": 0.5428571428571428, + "grad_norm": 1.0771127939224243, + "learning_rate": 5e-05, + "loss": 1.1796, + "mean_token_accuracy": 0.6659225225448608, + "num_tokens": 67726850.0, + "step": 114 + }, + { + "epoch": 0.5476190476190477, + "grad_norm": 1.0433073043823242, + "learning_rate": 5e-05, + "loss": 1.1744, + "mean_token_accuracy": 0.6663403511047363, + "num_tokens": 68316713.0, + "step": 115 + }, + { + "epoch": 0.5523809523809524, + "grad_norm": 0.919708788394928, + "learning_rate": 5e-05, + "loss": 1.1772, + "mean_token_accuracy": 0.6663476228713989, + "num_tokens": 68892365.0, + "step": 116 + }, + { + "epoch": 0.5571428571428572, + "grad_norm": 1.0221799612045288, + "learning_rate": 5e-05, + "loss": 1.1841, + "mean_token_accuracy": 0.665060818195343, + "num_tokens": 69505524.0, + "step": 117 + }, + { + "epoch": 0.5619047619047619, + "grad_norm": 0.8631717562675476, + "learning_rate": 5e-05, + "loss": 1.1636, + "mean_token_accuracy": 0.6692487597465515, + "num_tokens": 70095302.0, + "step": 118 + }, + { + "epoch": 0.5666666666666667, + "grad_norm": 0.9916826486587524, + "learning_rate": 5e-05, + "loss": 1.1782, + "mean_token_accuracy": 0.6660374402999878, + "num_tokens": 70694971.0, + "step": 119 + }, + { + "epoch": 0.5714285714285714, + "grad_norm": 1.0251036882400513, + "learning_rate": 5e-05, + "loss": 1.1767, + "mean_token_accuracy": 0.6667168736457825, + "num_tokens": 71279415.0, + "step": 120 + }, + { + "epoch": 0.5761904761904761, + "grad_norm": 0.8991736173629761, + "learning_rate": 5e-05, + "loss": 1.1678, + "mean_token_accuracy": 0.668420672416687, + "num_tokens": 71869014.0, + "step": 121 + }, + { + "epoch": 0.580952380952381, + "grad_norm": 1.2064132690429688, + "learning_rate": 5e-05, + "loss": 1.1799, + "mean_token_accuracy": 0.6652439832687378, + "num_tokens": 72472715.0, + "step": 122 + }, + { + "epoch": 0.5857142857142857, + "grad_norm": 0.8267526030540466, + "learning_rate": 5e-05, + "loss": 1.1696, + "mean_token_accuracy": 0.6684820055961609, + "num_tokens": 73055740.0, + "step": 123 + }, + { + "epoch": 0.5904761904761905, + "grad_norm": 0.9388744235038757, + "learning_rate": 5e-05, + "loss": 1.152, + "mean_token_accuracy": 0.6729195713996887, + "num_tokens": 73639151.0, + "step": 124 + }, + { + "epoch": 0.5952380952380952, + "grad_norm": 1.1199097633361816, + "learning_rate": 5e-05, + "loss": 1.1721, + "mean_token_accuracy": 0.6682595610618591, + "num_tokens": 74216756.0, + "step": 125 + }, + { + "epoch": 0.6, + "grad_norm": 0.8756281733512878, + "learning_rate": 5e-05, + "loss": 1.1674, + "mean_token_accuracy": 0.6686472296714783, + "num_tokens": 74813953.0, + "step": 126 + }, + { + "epoch": 0.6047619047619047, + "grad_norm": 1.0236656665802002, + "learning_rate": 5e-05, + "loss": 1.1671, + "mean_token_accuracy": 0.6689480543136597, + "num_tokens": 75410691.0, + "step": 127 + }, + { + "epoch": 0.6095238095238096, + "grad_norm": 1.0756475925445557, + "learning_rate": 5e-05, + "loss": 1.1842, + "mean_token_accuracy": 0.6640157699584961, + "num_tokens": 75996496.0, + "step": 128 + }, + { + "epoch": 0.6142857142857143, + "grad_norm": 0.8371219038963318, + "learning_rate": 5e-05, + "loss": 1.159, + "mean_token_accuracy": 0.671413779258728, + "num_tokens": 76585198.0, + "step": 129 + }, + { + "epoch": 0.6190476190476191, + "grad_norm": 0.986903190612793, + "learning_rate": 5e-05, + "loss": 1.147, + "mean_token_accuracy": 0.6733829975128174, + "num_tokens": 77191596.0, + "step": 130 + }, + { + "epoch": 0.6238095238095238, + "grad_norm": 0.8165880441665649, + "learning_rate": 5e-05, + "loss": 1.1643, + "mean_token_accuracy": 0.6688679456710815, + "num_tokens": 77796998.0, + "step": 131 + }, + { + "epoch": 0.6285714285714286, + "grad_norm": 1.2362457513809204, + "learning_rate": 5e-05, + "loss": 1.1784, + "mean_token_accuracy": 0.6656336188316345, + "num_tokens": 78395104.0, + "step": 132 + }, + { + "epoch": 0.6333333333333333, + "grad_norm": 0.8373320698738098, + "learning_rate": 5e-05, + "loss": 1.1633, + "mean_token_accuracy": 0.6698626279830933, + "num_tokens": 78988563.0, + "step": 133 + }, + { + "epoch": 0.638095238095238, + "grad_norm": 1.0211904048919678, + "learning_rate": 5e-05, + "loss": 1.1855, + "mean_token_accuracy": 0.6629217863082886, + "num_tokens": 79599633.0, + "step": 134 + }, + { + "epoch": 0.6428571428571429, + "grad_norm": 0.8456815481185913, + "learning_rate": 5e-05, + "loss": 1.1729, + "mean_token_accuracy": 0.6674565672874451, + "num_tokens": 80196954.0, + "step": 135 + }, + { + "epoch": 0.6476190476190476, + "grad_norm": 0.970680832862854, + "learning_rate": 5e-05, + "loss": 1.1728, + "mean_token_accuracy": 0.6667064428329468, + "num_tokens": 80790959.0, + "step": 136 + }, + { + "epoch": 0.6523809523809524, + "grad_norm": 0.9108589887619019, + "learning_rate": 5e-05, + "loss": 1.1604, + "mean_token_accuracy": 0.6702724695205688, + "num_tokens": 81363350.0, + "step": 137 + }, + { + "epoch": 0.6571428571428571, + "grad_norm": 1.0496387481689453, + "learning_rate": 5e-05, + "loss": 1.1615, + "mean_token_accuracy": 0.6696370840072632, + "num_tokens": 81960356.0, + "step": 138 + }, + { + "epoch": 0.6619047619047619, + "grad_norm": 0.74150151014328, + "learning_rate": 5e-05, + "loss": 1.1687, + "mean_token_accuracy": 0.668015718460083, + "num_tokens": 82573967.0, + "step": 139 + }, + { + "epoch": 0.6666666666666666, + "grad_norm": 0.8225556015968323, + "learning_rate": 5e-05, + "loss": 1.1567, + "mean_token_accuracy": 0.6704581379890442, + "num_tokens": 83170751.0, + "step": 140 + }, + { + "epoch": 0.6714285714285714, + "grad_norm": 1.114274501800537, + "learning_rate": 5e-05, + "loss": 1.1403, + "mean_token_accuracy": 0.6749263405799866, + "num_tokens": 83744022.0, + "step": 141 + }, + { + "epoch": 0.6761904761904762, + "grad_norm": 1.064664363861084, + "learning_rate": 5e-05, + "loss": 1.1776, + "mean_token_accuracy": 0.6658495664596558, + "num_tokens": 84336667.0, + "step": 142 + }, + { + "epoch": 0.680952380952381, + "grad_norm": 0.9398872256278992, + "learning_rate": 5e-05, + "loss": 1.1573, + "mean_token_accuracy": 0.6711140871047974, + "num_tokens": 84927599.0, + "step": 143 + }, + { + "epoch": 0.6857142857142857, + "grad_norm": 0.9965890645980835, + "learning_rate": 5e-05, + "loss": 1.1573, + "mean_token_accuracy": 0.671012818813324, + "num_tokens": 85516121.0, + "step": 144 + }, + { + "epoch": 0.6904761904761905, + "grad_norm": 0.9002604484558105, + "learning_rate": 5e-05, + "loss": 1.1523, + "mean_token_accuracy": 0.671434760093689, + "num_tokens": 86106161.0, + "step": 145 + }, + { + "epoch": 0.6952380952380952, + "grad_norm": 0.9985966086387634, + "learning_rate": 5e-05, + "loss": 1.1696, + "mean_token_accuracy": 0.6681336164474487, + "num_tokens": 86723880.0, + "step": 146 + }, + { + "epoch": 0.7, + "grad_norm": 0.9383719563484192, + "learning_rate": 5e-05, + "loss": 1.1614, + "mean_token_accuracy": 0.6697560548782349, + "num_tokens": 87327542.0, + "step": 147 + }, + { + "epoch": 0.7047619047619048, + "grad_norm": 0.8549755811691284, + "learning_rate": 5e-05, + "loss": 1.1566, + "mean_token_accuracy": 0.6707610487937927, + "num_tokens": 87912557.0, + "step": 148 + }, + { + "epoch": 0.7095238095238096, + "grad_norm": 1.0576001405715942, + "learning_rate": 5e-05, + "loss": 1.1676, + "mean_token_accuracy": 0.6687466502189636, + "num_tokens": 88514499.0, + "step": 149 + }, + { + "epoch": 0.7142857142857143, + "grad_norm": 1.052337646484375, + "learning_rate": 5e-05, + "loss": 1.1486, + "mean_token_accuracy": 0.6725994944572449, + "num_tokens": 89090412.0, + "step": 150 + }, + { + "epoch": 0.719047619047619, + "grad_norm": 0.8293784856796265, + "learning_rate": 5e-05, + "loss": 1.1561, + "mean_token_accuracy": 0.6712163686752319, + "num_tokens": 89689944.0, + "step": 151 + }, + { + "epoch": 0.7238095238095238, + "grad_norm": 0.7837069034576416, + "learning_rate": 5e-05, + "loss": 1.1668, + "mean_token_accuracy": 0.6684024333953857, + "num_tokens": 90281605.0, + "step": 152 + }, + { + "epoch": 0.7285714285714285, + "grad_norm": 0.7598390579223633, + "learning_rate": 5e-05, + "loss": 1.1663, + "mean_token_accuracy": 0.6688531637191772, + "num_tokens": 90877169.0, + "step": 153 + }, + { + "epoch": 0.7333333333333333, + "grad_norm": 1.0549771785736084, + "learning_rate": 5e-05, + "loss": 1.1728, + "mean_token_accuracy": 0.6663796901702881, + "num_tokens": 91473587.0, + "step": 154 + }, + { + "epoch": 0.7380952380952381, + "grad_norm": 0.8871486186981201, + "learning_rate": 5e-05, + "loss": 1.1642, + "mean_token_accuracy": 0.6688350439071655, + "num_tokens": 92066142.0, + "step": 155 + }, + { + "epoch": 0.7428571428571429, + "grad_norm": 0.8467488288879395, + "learning_rate": 5e-05, + "loss": 1.1461, + "mean_token_accuracy": 0.6732866764068604, + "num_tokens": 92671294.0, + "step": 156 + }, + { + "epoch": 0.7476190476190476, + "grad_norm": 0.8179646134376526, + "learning_rate": 5e-05, + "loss": 1.1497, + "mean_token_accuracy": 0.6721580624580383, + "num_tokens": 93267004.0, + "step": 157 + }, + { + "epoch": 0.7523809523809524, + "grad_norm": 0.9022694826126099, + "learning_rate": 5e-05, + "loss": 1.1683, + "mean_token_accuracy": 0.6682420969009399, + "num_tokens": 93865099.0, + "step": 158 + }, + { + "epoch": 0.7571428571428571, + "grad_norm": 0.9815990924835205, + "learning_rate": 5e-05, + "loss": 1.1412, + "mean_token_accuracy": 0.6744946241378784, + "num_tokens": 94449283.0, + "step": 159 + }, + { + "epoch": 0.7619047619047619, + "grad_norm": 0.9697504043579102, + "learning_rate": 5e-05, + "loss": 1.1514, + "mean_token_accuracy": 0.6713624596595764, + "num_tokens": 95037160.0, + "step": 160 + }, + { + "epoch": 0.7666666666666667, + "grad_norm": 0.7498188018798828, + "learning_rate": 5e-05, + "loss": 1.132, + "mean_token_accuracy": 0.6764633655548096, + "num_tokens": 95620329.0, + "step": 161 + }, + { + "epoch": 0.7714285714285715, + "grad_norm": 0.8020328879356384, + "learning_rate": 5e-05, + "loss": 1.1541, + "mean_token_accuracy": 0.6700690984725952, + "num_tokens": 96202979.0, + "step": 162 + }, + { + "epoch": 0.7761904761904762, + "grad_norm": 0.8966916799545288, + "learning_rate": 5e-05, + "loss": 1.1487, + "mean_token_accuracy": 0.6717889904975891, + "num_tokens": 96794135.0, + "step": 163 + }, + { + "epoch": 0.780952380952381, + "grad_norm": 0.8855036497116089, + "learning_rate": 5e-05, + "loss": 1.1398, + "mean_token_accuracy": 0.6740757822990417, + "num_tokens": 97380180.0, + "step": 164 + }, + { + "epoch": 0.7857142857142857, + "grad_norm": 0.7727010250091553, + "learning_rate": 5e-05, + "loss": 1.1561, + "mean_token_accuracy": 0.6711185574531555, + "num_tokens": 97979583.0, + "step": 165 + }, + { + "epoch": 0.7904761904761904, + "grad_norm": 0.9088108539581299, + "learning_rate": 5e-05, + "loss": 1.1384, + "mean_token_accuracy": 0.6758732199668884, + "num_tokens": 98573453.0, + "step": 166 + }, + { + "epoch": 0.7952380952380952, + "grad_norm": 0.8021786212921143, + "learning_rate": 5e-05, + "loss": 1.1445, + "mean_token_accuracy": 0.6735219955444336, + "num_tokens": 99162518.0, + "step": 167 + }, + { + "epoch": 0.8, + "grad_norm": 0.884023129940033, + "learning_rate": 5e-05, + "loss": 1.1449, + "mean_token_accuracy": 0.6727030277252197, + "num_tokens": 99755688.0, + "step": 168 + }, + { + "epoch": 0.8047619047619048, + "grad_norm": 0.7199719548225403, + "learning_rate": 5e-05, + "loss": 1.1502, + "mean_token_accuracy": 0.6714985966682434, + "num_tokens": 100367122.0, + "step": 169 + }, + { + "epoch": 0.8095238095238095, + "grad_norm": 1.0445648431777954, + "learning_rate": 5e-05, + "loss": 1.1463, + "mean_token_accuracy": 0.671629786491394, + "num_tokens": 100966663.0, + "step": 170 + }, + { + "epoch": 0.8142857142857143, + "grad_norm": 0.9512839913368225, + "learning_rate": 5e-05, + "loss": 1.1568, + "mean_token_accuracy": 0.6707793474197388, + "num_tokens": 101561561.0, + "step": 171 + }, + { + "epoch": 0.819047619047619, + "grad_norm": 0.8089645504951477, + "learning_rate": 5e-05, + "loss": 1.1626, + "mean_token_accuracy": 0.670020341873169, + "num_tokens": 102150367.0, + "step": 172 + }, + { + "epoch": 0.8238095238095238, + "grad_norm": 0.8684815168380737, + "learning_rate": 5e-05, + "loss": 1.1527, + "mean_token_accuracy": 0.671258270740509, + "num_tokens": 102744438.0, + "step": 173 + }, + { + "epoch": 0.8285714285714286, + "grad_norm": 0.8510096669197083, + "learning_rate": 5e-05, + "loss": 1.1518, + "mean_token_accuracy": 0.6712289452552795, + "num_tokens": 103336159.0, + "step": 174 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 0.9447618722915649, + "learning_rate": 5e-05, + "loss": 1.1572, + "mean_token_accuracy": 0.6702134013175964, + "num_tokens": 103933457.0, + "step": 175 + }, + { + "epoch": 0.8380952380952381, + "grad_norm": 1.024540662765503, + "learning_rate": 5e-05, + "loss": 1.1423, + "mean_token_accuracy": 0.6720887422561646, + "num_tokens": 104528020.0, + "step": 176 + }, + { + "epoch": 0.8428571428571429, + "grad_norm": 0.6513694524765015, + "learning_rate": 5e-05, + "loss": 1.1702, + "mean_token_accuracy": 0.6670519709587097, + "num_tokens": 105126562.0, + "step": 177 + }, + { + "epoch": 0.8476190476190476, + "grad_norm": 0.8769256472587585, + "learning_rate": 5e-05, + "loss": 1.1369, + "mean_token_accuracy": 0.6742633581161499, + "num_tokens": 105730229.0, + "step": 178 + }, + { + "epoch": 0.8523809523809524, + "grad_norm": 0.7902522087097168, + "learning_rate": 5e-05, + "loss": 1.1581, + "mean_token_accuracy": 0.6698312759399414, + "num_tokens": 106338239.0, + "step": 179 + }, + { + "epoch": 0.8571428571428571, + "grad_norm": 1.0291701555252075, + "learning_rate": 5e-05, + "loss": 1.1558, + "mean_token_accuracy": 0.670113205909729, + "num_tokens": 106929782.0, + "step": 180 + }, + { + "epoch": 0.861904761904762, + "grad_norm": 0.9144110679626465, + "learning_rate": 5e-05, + "loss": 1.1493, + "mean_token_accuracy": 0.6723814010620117, + "num_tokens": 107516950.0, + "step": 181 + }, + { + "epoch": 0.8666666666666667, + "grad_norm": 0.7676694989204407, + "learning_rate": 5e-05, + "loss": 1.1386, + "mean_token_accuracy": 0.6733461022377014, + "num_tokens": 108104046.0, + "step": 182 + }, + { + "epoch": 0.8714285714285714, + "grad_norm": 0.766424834728241, + "learning_rate": 5e-05, + "loss": 1.1571, + "mean_token_accuracy": 0.6696920394897461, + "num_tokens": 108711068.0, + "step": 183 + }, + { + "epoch": 0.8761904761904762, + "grad_norm": 0.7795400619506836, + "learning_rate": 5e-05, + "loss": 1.1241, + "mean_token_accuracy": 0.6772262454032898, + "num_tokens": 109294847.0, + "step": 184 + }, + { + "epoch": 0.8809523809523809, + "grad_norm": 0.8943405747413635, + "learning_rate": 5e-05, + "loss": 1.1458, + "mean_token_accuracy": 0.6723207831382751, + "num_tokens": 109903424.0, + "step": 185 + }, + { + "epoch": 0.8857142857142857, + "grad_norm": 0.9186291098594666, + "learning_rate": 5e-05, + "loss": 1.148, + "mean_token_accuracy": 0.6714228391647339, + "num_tokens": 110515082.0, + "step": 186 + }, + { + "epoch": 0.8904761904761904, + "grad_norm": 0.8866440653800964, + "learning_rate": 5e-05, + "loss": 1.1451, + "mean_token_accuracy": 0.6727667450904846, + "num_tokens": 111105456.0, + "step": 187 + }, + { + "epoch": 0.8952380952380953, + "grad_norm": 0.8748181462287903, + "learning_rate": 5e-05, + "loss": 1.1462, + "mean_token_accuracy": 0.6725558042526245, + "num_tokens": 111699029.0, + "step": 188 + }, + { + "epoch": 0.9, + "grad_norm": 0.921633243560791, + "learning_rate": 5e-05, + "loss": 1.1231, + "mean_token_accuracy": 0.6771888732910156, + "num_tokens": 112280321.0, + "step": 189 + }, + { + "epoch": 0.9047619047619048, + "grad_norm": 0.6554481983184814, + "learning_rate": 5e-05, + "loss": 1.1236, + "mean_token_accuracy": 0.6772241592407227, + "num_tokens": 112860009.0, + "step": 190 + }, + { + "epoch": 0.9095238095238095, + "grad_norm": 0.8720636367797852, + "learning_rate": 5e-05, + "loss": 1.1444, + "mean_token_accuracy": 0.6725858449935913, + "num_tokens": 113457303.0, + "step": 191 + }, + { + "epoch": 0.9142857142857143, + "grad_norm": 0.8109149932861328, + "learning_rate": 5e-05, + "loss": 1.1382, + "mean_token_accuracy": 0.6740887761116028, + "num_tokens": 114054977.0, + "step": 192 + }, + { + "epoch": 0.919047619047619, + "grad_norm": 1.1724909543991089, + "learning_rate": 5e-05, + "loss": 1.1347, + "mean_token_accuracy": 0.6748798489570618, + "num_tokens": 114641555.0, + "step": 193 + }, + { + "epoch": 0.9238095238095239, + "grad_norm": 0.7042703628540039, + "learning_rate": 5e-05, + "loss": 1.1505, + "mean_token_accuracy": 0.6722049713134766, + "num_tokens": 115241437.0, + "step": 194 + }, + { + "epoch": 0.9285714285714286, + "grad_norm": 1.1027475595474243, + "learning_rate": 5e-05, + "loss": 1.13, + "mean_token_accuracy": 0.6760388612747192, + "num_tokens": 115845775.0, + "step": 195 + }, + { + "epoch": 0.9333333333333333, + "grad_norm": 0.766270101070404, + "learning_rate": 5e-05, + "loss": 1.136, + "mean_token_accuracy": 0.674863338470459, + "num_tokens": 116452623.0, + "step": 196 + }, + { + "epoch": 0.9380952380952381, + "grad_norm": 0.9983667135238647, + "learning_rate": 5e-05, + "loss": 1.1443, + "mean_token_accuracy": 0.6721261739730835, + "num_tokens": 117045570.0, + "step": 197 + }, + { + "epoch": 0.9428571428571428, + "grad_norm": 0.8452311158180237, + "learning_rate": 5e-05, + "loss": 1.1214, + "mean_token_accuracy": 0.6784695386886597, + "num_tokens": 117654535.0, + "step": 198 + }, + { + "epoch": 0.9476190476190476, + "grad_norm": 0.9161198735237122, + "learning_rate": 5e-05, + "loss": 1.1345, + "mean_token_accuracy": 0.675512433052063, + "num_tokens": 118247244.0, + "step": 199 + }, + { + "epoch": 0.9523809523809523, + "grad_norm": 0.8395583033561707, + "learning_rate": 5e-05, + "loss": 1.1299, + "mean_token_accuracy": 0.6765437126159668, + "num_tokens": 118843074.0, + "step": 200 + }, + { + "epoch": 0.9571428571428572, + "grad_norm": 0.7813617587089539, + "learning_rate": 5e-05, + "loss": 1.1276, + "mean_token_accuracy": 0.6768419742584229, + "num_tokens": 119445023.0, + "step": 201 + }, + { + "epoch": 0.9619047619047619, + "grad_norm": 0.8200250267982483, + "learning_rate": 5e-05, + "loss": 1.1421, + "mean_token_accuracy": 0.6734207272529602, + "num_tokens": 120045748.0, + "step": 202 + }, + { + "epoch": 0.9666666666666667, + "grad_norm": 0.7514604330062866, + "learning_rate": 5e-05, + "loss": 1.1348, + "mean_token_accuracy": 0.6745180487632751, + "num_tokens": 120635079.0, + "step": 203 + }, + { + "epoch": 0.9714285714285714, + "grad_norm": 0.774848222732544, + "learning_rate": 5e-05, + "loss": 1.1328, + "mean_token_accuracy": 0.6744321584701538, + "num_tokens": 121220486.0, + "step": 204 + }, + { + "epoch": 0.9761904761904762, + "grad_norm": 0.8506629467010498, + "learning_rate": 5e-05, + "loss": 1.1355, + "mean_token_accuracy": 0.6743907928466797, + "num_tokens": 121800676.0, + "step": 205 + }, + { + "epoch": 0.9809523809523809, + "grad_norm": 0.7962760329246521, + "learning_rate": 5e-05, + "loss": 1.139, + "mean_token_accuracy": 0.6734001040458679, + "num_tokens": 122409399.0, + "step": 206 + }, + { + "epoch": 0.9857142857142858, + "grad_norm": 0.7492257952690125, + "learning_rate": 5e-05, + "loss": 1.1191, + "mean_token_accuracy": 0.6777032017707825, + "num_tokens": 123003502.0, + "step": 207 + }, + { + "epoch": 0.9904761904761905, + "grad_norm": 0.8552586436271667, + "learning_rate": 5e-05, + "loss": 1.1371, + "mean_token_accuracy": 0.6750062108039856, + "num_tokens": 123595838.0, + "step": 208 + }, + { + "epoch": 0.9952380952380953, + "grad_norm": 0.8296043872833252, + "learning_rate": 5e-05, + "loss": 1.1328, + "mean_token_accuracy": 0.6748907566070557, + "num_tokens": 124166476.0, + "step": 209 + }, + { + "epoch": 1.0, + "grad_norm": 0.7930900454521179, + "learning_rate": 5e-05, + "loss": 1.1227, + "mean_token_accuracy": 0.6773048639297485, + "num_tokens": 124761423.0, + "step": 210 + }, + { + "epoch": 1.0047619047619047, + "grad_norm": 1.0623637437820435, + "learning_rate": 5e-05, + "loss": 1.0805, + "mean_token_accuracy": 0.6853920221328735, + "num_tokens": 125364371.0, + "step": 211 + }, + { + "epoch": 1.0095238095238095, + "grad_norm": 0.7640378475189209, + "learning_rate": 5e-05, + "loss": 1.04, + "mean_token_accuracy": 0.6945140361785889, + "num_tokens": 125954118.0, + "step": 212 + }, + { + "epoch": 1.0142857142857142, + "grad_norm": 0.8286304473876953, + "learning_rate": 5e-05, + "loss": 1.0453, + "mean_token_accuracy": 0.6930573582649231, + "num_tokens": 126544991.0, + "step": 213 + }, + { + "epoch": 1.019047619047619, + "grad_norm": 0.9058213829994202, + "learning_rate": 5e-05, + "loss": 1.0487, + "mean_token_accuracy": 0.6932561993598938, + "num_tokens": 127151772.0, + "step": 214 + }, + { + "epoch": 1.0238095238095237, + "grad_norm": 0.8991729021072388, + "learning_rate": 5e-05, + "loss": 1.0506, + "mean_token_accuracy": 0.6918134689331055, + "num_tokens": 127762517.0, + "step": 215 + }, + { + "epoch": 1.0285714285714285, + "grad_norm": 0.8837477564811707, + "learning_rate": 5e-05, + "loss": 1.0538, + "mean_token_accuracy": 0.6913543939590454, + "num_tokens": 128358892.0, + "step": 216 + }, + { + "epoch": 1.0333333333333334, + "grad_norm": 0.9099165201187134, + "learning_rate": 5e-05, + "loss": 1.0503, + "mean_token_accuracy": 0.6921998262405396, + "num_tokens": 128930309.0, + "step": 217 + }, + { + "epoch": 1.0380952380952382, + "grad_norm": 1.0228596925735474, + "learning_rate": 5e-05, + "loss": 1.0369, + "mean_token_accuracy": 0.6959341168403625, + "num_tokens": 129537678.0, + "step": 218 + }, + { + "epoch": 1.042857142857143, + "grad_norm": 0.9690142273902893, + "learning_rate": 5e-05, + "loss": 1.0509, + "mean_token_accuracy": 0.6922160983085632, + "num_tokens": 130113717.0, + "step": 219 + }, + { + "epoch": 1.0476190476190477, + "grad_norm": 0.8784680962562561, + "learning_rate": 5e-05, + "loss": 1.0594, + "mean_token_accuracy": 0.6902390718460083, + "num_tokens": 130724521.0, + "step": 220 + }, + { + "epoch": 1.0523809523809524, + "grad_norm": 0.8741403818130493, + "learning_rate": 5e-05, + "loss": 1.0437, + "mean_token_accuracy": 0.6937582492828369, + "num_tokens": 131298037.0, + "step": 221 + }, + { + "epoch": 1.0571428571428572, + "grad_norm": 0.7399629950523376, + "learning_rate": 5e-05, + "loss": 1.0497, + "mean_token_accuracy": 0.6925863027572632, + "num_tokens": 131909779.0, + "step": 222 + }, + { + "epoch": 1.061904761904762, + "grad_norm": 0.9455582499504089, + "learning_rate": 5e-05, + "loss": 1.0495, + "mean_token_accuracy": 0.6921373009681702, + "num_tokens": 132506621.0, + "step": 223 + }, + { + "epoch": 1.0666666666666667, + "grad_norm": 0.8614041805267334, + "learning_rate": 5e-05, + "loss": 1.0562, + "mean_token_accuracy": 0.6901673674583435, + "num_tokens": 133124443.0, + "step": 224 + }, + { + "epoch": 1.0714285714285714, + "grad_norm": 0.9255203604698181, + "learning_rate": 5e-05, + "loss": 1.0609, + "mean_token_accuracy": 0.6890321969985962, + "num_tokens": 133719672.0, + "step": 225 + }, + { + "epoch": 1.0761904761904761, + "grad_norm": 0.9666732549667358, + "learning_rate": 5e-05, + "loss": 1.0392, + "mean_token_accuracy": 0.693585991859436, + "num_tokens": 134309852.0, + "step": 226 + }, + { + "epoch": 1.0809523809523809, + "grad_norm": 0.8443697690963745, + "learning_rate": 5e-05, + "loss": 1.0508, + "mean_token_accuracy": 0.691076934337616, + "num_tokens": 134890952.0, + "step": 227 + }, + { + "epoch": 1.0857142857142856, + "grad_norm": 0.9407112002372742, + "learning_rate": 5e-05, + "loss": 1.0162, + "mean_token_accuracy": 0.7004079818725586, + "num_tokens": 135479588.0, + "step": 228 + }, + { + "epoch": 1.0904761904761904, + "grad_norm": 0.8152827620506287, + "learning_rate": 5e-05, + "loss": 1.0405, + "mean_token_accuracy": 0.694449782371521, + "num_tokens": 136065836.0, + "step": 229 + }, + { + "epoch": 1.0952380952380953, + "grad_norm": 0.7795569896697998, + "learning_rate": 5e-05, + "loss": 1.0399, + "mean_token_accuracy": 0.6950474977493286, + "num_tokens": 136668062.0, + "step": 230 + }, + { + "epoch": 1.1, + "grad_norm": 0.8481141924858093, + "learning_rate": 5e-05, + "loss": 1.0434, + "mean_token_accuracy": 0.6934369802474976, + "num_tokens": 137256533.0, + "step": 231 + }, + { + "epoch": 1.1047619047619048, + "grad_norm": 0.8060566186904907, + "learning_rate": 5e-05, + "loss": 1.0476, + "mean_token_accuracy": 0.6922004222869873, + "num_tokens": 137848246.0, + "step": 232 + }, + { + "epoch": 1.1095238095238096, + "grad_norm": 0.9540084600448608, + "learning_rate": 5e-05, + "loss": 1.0536, + "mean_token_accuracy": 0.6909586191177368, + "num_tokens": 138450870.0, + "step": 233 + }, + { + "epoch": 1.1142857142857143, + "grad_norm": 0.8122584819793701, + "learning_rate": 5e-05, + "loss": 1.0522, + "mean_token_accuracy": 0.691053569316864, + "num_tokens": 139048178.0, + "step": 234 + }, + { + "epoch": 1.119047619047619, + "grad_norm": 0.7982403039932251, + "learning_rate": 5e-05, + "loss": 1.0418, + "mean_token_accuracy": 0.6941355466842651, + "num_tokens": 139647536.0, + "step": 235 + }, + { + "epoch": 1.1238095238095238, + "grad_norm": 0.935192883014679, + "learning_rate": 5e-05, + "loss": 1.0458, + "mean_token_accuracy": 0.6933596134185791, + "num_tokens": 140237569.0, + "step": 236 + }, + { + "epoch": 1.1285714285714286, + "grad_norm": 0.859958291053772, + "learning_rate": 5e-05, + "loss": 1.0499, + "mean_token_accuracy": 0.6922887563705444, + "num_tokens": 140808948.0, + "step": 237 + }, + { + "epoch": 1.1333333333333333, + "grad_norm": 0.7203453183174133, + "learning_rate": 5e-05, + "loss": 1.0457, + "mean_token_accuracy": 0.6926532983779907, + "num_tokens": 141387906.0, + "step": 238 + }, + { + "epoch": 1.138095238095238, + "grad_norm": 0.9747745990753174, + "learning_rate": 5e-05, + "loss": 1.0477, + "mean_token_accuracy": 0.6923413872718811, + "num_tokens": 141991024.0, + "step": 239 + }, + { + "epoch": 1.1428571428571428, + "grad_norm": 0.7779251933097839, + "learning_rate": 5e-05, + "loss": 1.0275, + "mean_token_accuracy": 0.6975968480110168, + "num_tokens": 142585170.0, + "step": 240 + }, + { + "epoch": 1.1476190476190475, + "grad_norm": 0.792132556438446, + "learning_rate": 5e-05, + "loss": 1.0589, + "mean_token_accuracy": 0.689858078956604, + "num_tokens": 143178473.0, + "step": 241 + }, + { + "epoch": 1.1523809523809523, + "grad_norm": 0.9979913830757141, + "learning_rate": 5e-05, + "loss": 1.0538, + "mean_token_accuracy": 0.6898748874664307, + "num_tokens": 143782184.0, + "step": 242 + }, + { + "epoch": 1.157142857142857, + "grad_norm": 0.9950650334358215, + "learning_rate": 5e-05, + "loss": 1.0391, + "mean_token_accuracy": 0.6937430500984192, + "num_tokens": 144383051.0, + "step": 243 + }, + { + "epoch": 1.161904761904762, + "grad_norm": 0.709945559501648, + "learning_rate": 5e-05, + "loss": 1.0498, + "mean_token_accuracy": 0.6920516490936279, + "num_tokens": 144977872.0, + "step": 244 + }, + { + "epoch": 1.1666666666666667, + "grad_norm": 0.9771570563316345, + "learning_rate": 5e-05, + "loss": 1.05, + "mean_token_accuracy": 0.691670835018158, + "num_tokens": 145573077.0, + "step": 245 + }, + { + "epoch": 1.1714285714285715, + "grad_norm": 1.1321951150894165, + "learning_rate": 5e-05, + "loss": 1.0546, + "mean_token_accuracy": 0.6916296482086182, + "num_tokens": 146165107.0, + "step": 246 + }, + { + "epoch": 1.1761904761904762, + "grad_norm": 0.7472027540206909, + "learning_rate": 5e-05, + "loss": 1.0475, + "mean_token_accuracy": 0.6918451189994812, + "num_tokens": 146763356.0, + "step": 247 + }, + { + "epoch": 1.180952380952381, + "grad_norm": 1.0085467100143433, + "learning_rate": 5e-05, + "loss": 1.0438, + "mean_token_accuracy": 0.6936038136482239, + "num_tokens": 147358035.0, + "step": 248 + }, + { + "epoch": 1.1857142857142857, + "grad_norm": 0.9739279747009277, + "learning_rate": 5e-05, + "loss": 1.0623, + "mean_token_accuracy": 0.6881403923034668, + "num_tokens": 147955530.0, + "step": 249 + }, + { + "epoch": 1.1904761904761905, + "grad_norm": 0.8775011897087097, + "learning_rate": 5e-05, + "loss": 1.057, + "mean_token_accuracy": 0.6903431415557861, + "num_tokens": 148547950.0, + "step": 250 + }, + { + "epoch": 1.1952380952380952, + "grad_norm": 0.829648494720459, + "learning_rate": 5e-05, + "loss": 1.0388, + "mean_token_accuracy": 0.6948254108428955, + "num_tokens": 149127280.0, + "step": 251 + }, + { + "epoch": 1.2, + "grad_norm": 1.0081520080566406, + "learning_rate": 5e-05, + "loss": 1.0499, + "mean_token_accuracy": 0.6920087933540344, + "num_tokens": 149735096.0, + "step": 252 + }, + { + "epoch": 1.2047619047619047, + "grad_norm": 0.7878535389900208, + "learning_rate": 5e-05, + "loss": 1.0517, + "mean_token_accuracy": 0.6921966075897217, + "num_tokens": 150338864.0, + "step": 253 + }, + { + "epoch": 1.2095238095238094, + "grad_norm": 0.8952375054359436, + "learning_rate": 5e-05, + "loss": 1.0364, + "mean_token_accuracy": 0.6957828402519226, + "num_tokens": 150940365.0, + "step": 254 + }, + { + "epoch": 1.2142857142857142, + "grad_norm": 0.8670030236244202, + "learning_rate": 5e-05, + "loss": 1.0529, + "mean_token_accuracy": 0.6915614604949951, + "num_tokens": 151517902.0, + "step": 255 + }, + { + "epoch": 1.2190476190476192, + "grad_norm": 0.7276114225387573, + "learning_rate": 5e-05, + "loss": 1.0303, + "mean_token_accuracy": 0.6969864368438721, + "num_tokens": 152093932.0, + "step": 256 + }, + { + "epoch": 1.223809523809524, + "grad_norm": 0.8988511562347412, + "learning_rate": 5e-05, + "loss": 1.0357, + "mean_token_accuracy": 0.6958300471305847, + "num_tokens": 152690003.0, + "step": 257 + }, + { + "epoch": 1.2285714285714286, + "grad_norm": 0.8661609888076782, + "learning_rate": 5e-05, + "loss": 1.0401, + "mean_token_accuracy": 0.6942745447158813, + "num_tokens": 153278014.0, + "step": 258 + }, + { + "epoch": 1.2333333333333334, + "grad_norm": 0.754280686378479, + "learning_rate": 5e-05, + "loss": 1.0446, + "mean_token_accuracy": 0.6938430666923523, + "num_tokens": 153871781.0, + "step": 259 + }, + { + "epoch": 1.2380952380952381, + "grad_norm": 0.9164913892745972, + "learning_rate": 5e-05, + "loss": 1.0479, + "mean_token_accuracy": 0.6920982599258423, + "num_tokens": 154466124.0, + "step": 260 + }, + { + "epoch": 1.2428571428571429, + "grad_norm": 0.7773877382278442, + "learning_rate": 5e-05, + "loss": 1.0406, + "mean_token_accuracy": 0.6933274269104004, + "num_tokens": 155073053.0, + "step": 261 + }, + { + "epoch": 1.2476190476190476, + "grad_norm": 1.0469205379486084, + "learning_rate": 5e-05, + "loss": 1.0722, + "mean_token_accuracy": 0.6876776814460754, + "num_tokens": 155680694.0, + "step": 262 + }, + { + "epoch": 1.2523809523809524, + "grad_norm": 0.6867191791534424, + "learning_rate": 5e-05, + "loss": 1.0576, + "mean_token_accuracy": 0.6899943351745605, + "num_tokens": 156279612.0, + "step": 263 + }, + { + "epoch": 1.2571428571428571, + "grad_norm": 0.8443048596382141, + "learning_rate": 5e-05, + "loss": 1.0495, + "mean_token_accuracy": 0.6925054788589478, + "num_tokens": 156898086.0, + "step": 264 + }, + { + "epoch": 1.2619047619047619, + "grad_norm": 0.8098260164260864, + "learning_rate": 5e-05, + "loss": 1.0613, + "mean_token_accuracy": 0.6893327832221985, + "num_tokens": 157502996.0, + "step": 265 + }, + { + "epoch": 1.2666666666666666, + "grad_norm": 0.8153842091560364, + "learning_rate": 5e-05, + "loss": 1.0615, + "mean_token_accuracy": 0.6891594529151917, + "num_tokens": 158107778.0, + "step": 266 + }, + { + "epoch": 1.2714285714285714, + "grad_norm": 0.8298802375793457, + "learning_rate": 5e-05, + "loss": 1.0299, + "mean_token_accuracy": 0.6969125270843506, + "num_tokens": 158713147.0, + "step": 267 + }, + { + "epoch": 1.276190476190476, + "grad_norm": 0.8962613940238953, + "learning_rate": 5e-05, + "loss": 1.0362, + "mean_token_accuracy": 0.694932758808136, + "num_tokens": 159292006.0, + "step": 268 + }, + { + "epoch": 1.2809523809523808, + "grad_norm": 0.805374026298523, + "learning_rate": 5e-05, + "loss": 1.0333, + "mean_token_accuracy": 0.6958897113800049, + "num_tokens": 159889197.0, + "step": 269 + }, + { + "epoch": 1.2857142857142856, + "grad_norm": 0.8051942586898804, + "learning_rate": 5e-05, + "loss": 1.0473, + "mean_token_accuracy": 0.6925612092018127, + "num_tokens": 160485304.0, + "step": 270 + }, + { + "epoch": 1.2904761904761906, + "grad_norm": 0.8105297088623047, + "learning_rate": 5e-05, + "loss": 1.0507, + "mean_token_accuracy": 0.6918688416481018, + "num_tokens": 161092433.0, + "step": 271 + }, + { + "epoch": 1.2952380952380953, + "grad_norm": 0.9065902233123779, + "learning_rate": 5e-05, + "loss": 1.0469, + "mean_token_accuracy": 0.6926026344299316, + "num_tokens": 161683555.0, + "step": 272 + }, + { + "epoch": 1.3, + "grad_norm": 0.8195280432701111, + "learning_rate": 5e-05, + "loss": 1.0425, + "mean_token_accuracy": 0.6943745613098145, + "num_tokens": 162278973.0, + "step": 273 + }, + { + "epoch": 1.3047619047619048, + "grad_norm": 0.692061722278595, + "learning_rate": 5e-05, + "loss": 1.0463, + "mean_token_accuracy": 0.6933311223983765, + "num_tokens": 162885270.0, + "step": 274 + }, + { + "epoch": 1.3095238095238095, + "grad_norm": 0.8352709412574768, + "learning_rate": 5e-05, + "loss": 1.0425, + "mean_token_accuracy": 0.6936053037643433, + "num_tokens": 163476311.0, + "step": 275 + }, + { + "epoch": 1.3142857142857143, + "grad_norm": 0.823569655418396, + "learning_rate": 5e-05, + "loss": 1.0532, + "mean_token_accuracy": 0.6909274458885193, + "num_tokens": 164071314.0, + "step": 276 + }, + { + "epoch": 1.319047619047619, + "grad_norm": 0.8307578563690186, + "learning_rate": 5e-05, + "loss": 1.0427, + "mean_token_accuracy": 0.6931385397911072, + "num_tokens": 164663415.0, + "step": 277 + }, + { + "epoch": 1.3238095238095238, + "grad_norm": 0.8843416571617126, + "learning_rate": 5e-05, + "loss": 1.0369, + "mean_token_accuracy": 0.695106029510498, + "num_tokens": 165256997.0, + "step": 278 + }, + { + "epoch": 1.3285714285714285, + "grad_norm": 0.8213310837745667, + "learning_rate": 5e-05, + "loss": 1.043, + "mean_token_accuracy": 0.6938116550445557, + "num_tokens": 165847922.0, + "step": 279 + }, + { + "epoch": 1.3333333333333333, + "grad_norm": 0.8646283149719238, + "learning_rate": 5e-05, + "loss": 1.0463, + "mean_token_accuracy": 0.6932564973831177, + "num_tokens": 166444541.0, + "step": 280 + }, + { + "epoch": 1.3380952380952382, + "grad_norm": 0.8472998738288879, + "learning_rate": 5e-05, + "loss": 1.035, + "mean_token_accuracy": 0.6957570314407349, + "num_tokens": 167028591.0, + "step": 281 + }, + { + "epoch": 1.342857142857143, + "grad_norm": 0.9474790692329407, + "learning_rate": 5e-05, + "loss": 1.0609, + "mean_token_accuracy": 0.6893367767333984, + "num_tokens": 167627132.0, + "step": 282 + }, + { + "epoch": 1.3476190476190477, + "grad_norm": 0.8395231366157532, + "learning_rate": 5e-05, + "loss": 1.0341, + "mean_token_accuracy": 0.6959323883056641, + "num_tokens": 168226854.0, + "step": 283 + }, + { + "epoch": 1.3523809523809525, + "grad_norm": 0.8269662261009216, + "learning_rate": 5e-05, + "loss": 1.0625, + "mean_token_accuracy": 0.6890583038330078, + "num_tokens": 168833732.0, + "step": 284 + }, + { + "epoch": 1.3571428571428572, + "grad_norm": 0.7933824062347412, + "learning_rate": 5e-05, + "loss": 1.035, + "mean_token_accuracy": 0.6961338520050049, + "num_tokens": 169426970.0, + "step": 285 + }, + { + "epoch": 1.361904761904762, + "grad_norm": 0.7769210338592529, + "learning_rate": 5e-05, + "loss": 1.0608, + "mean_token_accuracy": 0.6899520754814148, + "num_tokens": 170025787.0, + "step": 286 + }, + { + "epoch": 1.3666666666666667, + "grad_norm": 1.0355615615844727, + "learning_rate": 5e-05, + "loss": 1.0593, + "mean_token_accuracy": 0.6887654066085815, + "num_tokens": 170621857.0, + "step": 287 + }, + { + "epoch": 1.3714285714285714, + "grad_norm": 0.8072195649147034, + "learning_rate": 5e-05, + "loss": 1.0551, + "mean_token_accuracy": 0.6910427808761597, + "num_tokens": 171220708.0, + "step": 288 + }, + { + "epoch": 1.3761904761904762, + "grad_norm": 0.9695478081703186, + "learning_rate": 5e-05, + "loss": 1.0649, + "mean_token_accuracy": 0.6871645450592041, + "num_tokens": 171812508.0, + "step": 289 + }, + { + "epoch": 1.380952380952381, + "grad_norm": 0.833074152469635, + "learning_rate": 5e-05, + "loss": 1.0374, + "mean_token_accuracy": 0.6945768594741821, + "num_tokens": 172401090.0, + "step": 290 + }, + { + "epoch": 1.3857142857142857, + "grad_norm": 0.939854085445404, + "learning_rate": 5e-05, + "loss": 1.0459, + "mean_token_accuracy": 0.6932987570762634, + "num_tokens": 172989201.0, + "step": 291 + }, + { + "epoch": 1.3904761904761904, + "grad_norm": 0.8128898739814758, + "learning_rate": 5e-05, + "loss": 1.0443, + "mean_token_accuracy": 0.6922272443771362, + "num_tokens": 173563807.0, + "step": 292 + }, + { + "epoch": 1.3952380952380952, + "grad_norm": 0.8483256697654724, + "learning_rate": 5e-05, + "loss": 1.0412, + "mean_token_accuracy": 0.6934754848480225, + "num_tokens": 174159574.0, + "step": 293 + }, + { + "epoch": 1.4, + "grad_norm": 0.7055822014808655, + "learning_rate": 5e-05, + "loss": 1.0596, + "mean_token_accuracy": 0.6894583106040955, + "num_tokens": 174744244.0, + "step": 294 + }, + { + "epoch": 1.4047619047619047, + "grad_norm": 1.0544077157974243, + "learning_rate": 5e-05, + "loss": 1.0451, + "mean_token_accuracy": 0.6923484802246094, + "num_tokens": 175341946.0, + "step": 295 + }, + { + "epoch": 1.4095238095238094, + "grad_norm": 0.7343186736106873, + "learning_rate": 5e-05, + "loss": 1.0178, + "mean_token_accuracy": 0.6995499730110168, + "num_tokens": 175904117.0, + "step": 296 + }, + { + "epoch": 1.4142857142857144, + "grad_norm": 1.010467767715454, + "learning_rate": 5e-05, + "loss": 1.0478, + "mean_token_accuracy": 0.6921795606613159, + "num_tokens": 176493621.0, + "step": 297 + }, + { + "epoch": 1.4190476190476191, + "grad_norm": 1.0163770914077759, + "learning_rate": 5e-05, + "loss": 1.0457, + "mean_token_accuracy": 0.6921326518058777, + "num_tokens": 177082157.0, + "step": 298 + }, + { + "epoch": 1.4238095238095239, + "grad_norm": 0.7557767033576965, + "learning_rate": 5e-05, + "loss": 1.0482, + "mean_token_accuracy": 0.6914042234420776, + "num_tokens": 177668681.0, + "step": 299 + }, + { + "epoch": 1.4285714285714286, + "grad_norm": 1.0022515058517456, + "learning_rate": 5e-05, + "loss": 1.0515, + "mean_token_accuracy": 0.6908462047576904, + "num_tokens": 178256283.0, + "step": 300 + }, + { + "epoch": 1.4333333333333333, + "grad_norm": 0.8407636284828186, + "learning_rate": 5e-05, + "loss": 1.0616, + "mean_token_accuracy": 0.6882213354110718, + "num_tokens": 178850673.0, + "step": 301 + }, + { + "epoch": 1.438095238095238, + "grad_norm": 0.8547885417938232, + "learning_rate": 5e-05, + "loss": 1.0398, + "mean_token_accuracy": 0.6946403384208679, + "num_tokens": 179457871.0, + "step": 302 + }, + { + "epoch": 1.4428571428571428, + "grad_norm": 0.839057207107544, + "learning_rate": 5e-05, + "loss": 1.0695, + "mean_token_accuracy": 0.6869980096817017, + "num_tokens": 180064071.0, + "step": 303 + }, + { + "epoch": 1.4476190476190476, + "grad_norm": 0.792688250541687, + "learning_rate": 5e-05, + "loss": 1.0262, + "mean_token_accuracy": 0.6966561675071716, + "num_tokens": 180650796.0, + "step": 304 + }, + { + "epoch": 1.4523809523809523, + "grad_norm": 0.959341287612915, + "learning_rate": 5e-05, + "loss": 1.055, + "mean_token_accuracy": 0.6915225386619568, + "num_tokens": 181246825.0, + "step": 305 + }, + { + "epoch": 1.457142857142857, + "grad_norm": 0.7509632706642151, + "learning_rate": 5e-05, + "loss": 1.0588, + "mean_token_accuracy": 0.689898669719696, + "num_tokens": 181855567.0, + "step": 306 + }, + { + "epoch": 1.461904761904762, + "grad_norm": 0.7527841925621033, + "learning_rate": 5e-05, + "loss": 1.0209, + "mean_token_accuracy": 0.6979542970657349, + "num_tokens": 182433911.0, + "step": 307 + }, + { + "epoch": 1.4666666666666668, + "grad_norm": 0.7188791632652283, + "learning_rate": 5e-05, + "loss": 1.0431, + "mean_token_accuracy": 0.6931103467941284, + "num_tokens": 183023144.0, + "step": 308 + }, + { + "epoch": 1.4714285714285715, + "grad_norm": 0.8372648358345032, + "learning_rate": 5e-05, + "loss": 1.0364, + "mean_token_accuracy": 0.6948618292808533, + "num_tokens": 183626514.0, + "step": 309 + }, + { + "epoch": 1.4761904761904763, + "grad_norm": 0.780892014503479, + "learning_rate": 5e-05, + "loss": 1.0443, + "mean_token_accuracy": 0.6924835443496704, + "num_tokens": 184221439.0, + "step": 310 + }, + { + "epoch": 1.480952380952381, + "grad_norm": 0.6577752828598022, + "learning_rate": 5e-05, + "loss": 1.0453, + "mean_token_accuracy": 0.693198561668396, + "num_tokens": 184819506.0, + "step": 311 + }, + { + "epoch": 1.4857142857142858, + "grad_norm": 0.8050316572189331, + "learning_rate": 5e-05, + "loss": 1.0488, + "mean_token_accuracy": 0.6916123628616333, + "num_tokens": 185419019.0, + "step": 312 + }, + { + "epoch": 1.4904761904761905, + "grad_norm": 0.8795943856239319, + "learning_rate": 5e-05, + "loss": 1.0377, + "mean_token_accuracy": 0.6958647966384888, + "num_tokens": 186010772.0, + "step": 313 + }, + { + "epoch": 1.4952380952380953, + "grad_norm": 0.8077270984649658, + "learning_rate": 5e-05, + "loss": 1.0347, + "mean_token_accuracy": 0.6950284242630005, + "num_tokens": 186589243.0, + "step": 314 + }, + { + "epoch": 1.5, + "grad_norm": 0.8426974415779114, + "learning_rate": 5e-05, + "loss": 1.0323, + "mean_token_accuracy": 0.6960166096687317, + "num_tokens": 187182368.0, + "step": 315 + }, + { + "epoch": 1.5047619047619047, + "grad_norm": 0.7842941284179688, + "learning_rate": 5e-05, + "loss": 1.0336, + "mean_token_accuracy": 0.696148157119751, + "num_tokens": 187763428.0, + "step": 316 + }, + { + "epoch": 1.5095238095238095, + "grad_norm": 0.835817813873291, + "learning_rate": 5e-05, + "loss": 1.029, + "mean_token_accuracy": 0.6970314979553223, + "num_tokens": 188359395.0, + "step": 317 + }, + { + "epoch": 1.5142857142857142, + "grad_norm": 0.7936996221542358, + "learning_rate": 5e-05, + "loss": 1.052, + "mean_token_accuracy": 0.6905555129051208, + "num_tokens": 188952450.0, + "step": 318 + }, + { + "epoch": 1.519047619047619, + "grad_norm": 0.8359509110450745, + "learning_rate": 5e-05, + "loss": 1.0446, + "mean_token_accuracy": 0.6925878524780273, + "num_tokens": 189535853.0, + "step": 319 + }, + { + "epoch": 1.5238095238095237, + "grad_norm": 0.741528332233429, + "learning_rate": 5e-05, + "loss": 1.0264, + "mean_token_accuracy": 0.6976981163024902, + "num_tokens": 190127386.0, + "step": 320 + }, + { + "epoch": 1.5285714285714285, + "grad_norm": 0.9778928160667419, + "learning_rate": 5e-05, + "loss": 1.0468, + "mean_token_accuracy": 0.6918396949768066, + "num_tokens": 190718877.0, + "step": 321 + }, + { + "epoch": 1.5333333333333332, + "grad_norm": 0.7786639332771301, + "learning_rate": 5e-05, + "loss": 1.0653, + "mean_token_accuracy": 0.6880618929862976, + "num_tokens": 191320553.0, + "step": 322 + }, + { + "epoch": 1.538095238095238, + "grad_norm": 0.7448099255561829, + "learning_rate": 5e-05, + "loss": 1.0367, + "mean_token_accuracy": 0.6952208280563354, + "num_tokens": 191911786.0, + "step": 323 + }, + { + "epoch": 1.5428571428571427, + "grad_norm": 0.7474610209465027, + "learning_rate": 5e-05, + "loss": 1.065, + "mean_token_accuracy": 0.6890267133712769, + "num_tokens": 192517254.0, + "step": 324 + }, + { + "epoch": 1.5476190476190477, + "grad_norm": 0.9233639240264893, + "learning_rate": 5e-05, + "loss": 1.0611, + "mean_token_accuracy": 0.6891995668411255, + "num_tokens": 193113713.0, + "step": 325 + }, + { + "epoch": 1.5523809523809524, + "grad_norm": 0.725395143032074, + "learning_rate": 5e-05, + "loss": 1.0402, + "mean_token_accuracy": 0.6938502788543701, + "num_tokens": 193718335.0, + "step": 326 + }, + { + "epoch": 1.5571428571428572, + "grad_norm": 0.7485827803611755, + "learning_rate": 5e-05, + "loss": 1.0408, + "mean_token_accuracy": 0.6929433941841125, + "num_tokens": 194303328.0, + "step": 327 + }, + { + "epoch": 1.561904761904762, + "grad_norm": 0.8035783767700195, + "learning_rate": 5e-05, + "loss": 1.0359, + "mean_token_accuracy": 0.6952745914459229, + "num_tokens": 194886509.0, + "step": 328 + }, + { + "epoch": 1.5666666666666667, + "grad_norm": 0.8925185799598694, + "learning_rate": 5e-05, + "loss": 1.0423, + "mean_token_accuracy": 0.694246232509613, + "num_tokens": 195456896.0, + "step": 329 + }, + { + "epoch": 1.5714285714285714, + "grad_norm": 0.7085704803466797, + "learning_rate": 5e-05, + "loss": 1.041, + "mean_token_accuracy": 0.6928901672363281, + "num_tokens": 196053871.0, + "step": 330 + }, + { + "epoch": 1.5761904761904761, + "grad_norm": 0.7180474400520325, + "learning_rate": 5e-05, + "loss": 1.0435, + "mean_token_accuracy": 0.6929495334625244, + "num_tokens": 196654732.0, + "step": 331 + }, + { + "epoch": 1.580952380952381, + "grad_norm": 0.7589107155799866, + "learning_rate": 5e-05, + "loss": 1.0281, + "mean_token_accuracy": 0.697223961353302, + "num_tokens": 197242864.0, + "step": 332 + }, + { + "epoch": 1.5857142857142859, + "grad_norm": 0.8662079572677612, + "learning_rate": 5e-05, + "loss": 1.0334, + "mean_token_accuracy": 0.6955124735832214, + "num_tokens": 197840214.0, + "step": 333 + }, + { + "epoch": 1.5904761904761906, + "grad_norm": 0.7313894629478455, + "learning_rate": 5e-05, + "loss": 1.0349, + "mean_token_accuracy": 0.6949942708015442, + "num_tokens": 198440572.0, + "step": 334 + }, + { + "epoch": 1.5952380952380953, + "grad_norm": 0.7622560858726501, + "learning_rate": 5e-05, + "loss": 1.0324, + "mean_token_accuracy": 0.6960766315460205, + "num_tokens": 199039184.0, + "step": 335 + }, + { + "epoch": 1.6, + "grad_norm": 0.95415860414505, + "learning_rate": 5e-05, + "loss": 1.0325, + "mean_token_accuracy": 0.6953790783882141, + "num_tokens": 199626548.0, + "step": 336 + }, + { + "epoch": 1.6047619047619048, + "grad_norm": 0.8212993741035461, + "learning_rate": 5e-05, + "loss": 1.0481, + "mean_token_accuracy": 0.692508339881897, + "num_tokens": 200222258.0, + "step": 337 + }, + { + "epoch": 1.6095238095238096, + "grad_norm": 0.7992497086524963, + "learning_rate": 5e-05, + "loss": 1.0387, + "mean_token_accuracy": 0.6952558159828186, + "num_tokens": 200819172.0, + "step": 338 + }, + { + "epoch": 1.6142857142857143, + "grad_norm": 0.8674090504646301, + "learning_rate": 5e-05, + "loss": 1.0493, + "mean_token_accuracy": 0.6913407444953918, + "num_tokens": 201413549.0, + "step": 339 + }, + { + "epoch": 1.619047619047619, + "grad_norm": 0.6464642286300659, + "learning_rate": 5e-05, + "loss": 1.0329, + "mean_token_accuracy": 0.6952366828918457, + "num_tokens": 202014069.0, + "step": 340 + }, + { + "epoch": 1.6238095238095238, + "grad_norm": 0.7744977474212646, + "learning_rate": 5e-05, + "loss": 1.0621, + "mean_token_accuracy": 0.688499927520752, + "num_tokens": 202600379.0, + "step": 341 + }, + { + "epoch": 1.6285714285714286, + "grad_norm": 0.8241460919380188, + "learning_rate": 5e-05, + "loss": 1.0412, + "mean_token_accuracy": 0.6941289305686951, + "num_tokens": 203203233.0, + "step": 342 + }, + { + "epoch": 1.6333333333333333, + "grad_norm": 0.7725429534912109, + "learning_rate": 5e-05, + "loss": 1.0364, + "mean_token_accuracy": 0.6944689154624939, + "num_tokens": 203802554.0, + "step": 343 + }, + { + "epoch": 1.638095238095238, + "grad_norm": 0.7584908604621887, + "learning_rate": 5e-05, + "loss": 1.0458, + "mean_token_accuracy": 0.6935819387435913, + "num_tokens": 204395560.0, + "step": 344 + }, + { + "epoch": 1.6428571428571428, + "grad_norm": 0.8219484686851501, + "learning_rate": 5e-05, + "loss": 1.0343, + "mean_token_accuracy": 0.6956254243850708, + "num_tokens": 205001404.0, + "step": 345 + }, + { + "epoch": 1.6476190476190475, + "grad_norm": 0.8540579080581665, + "learning_rate": 5e-05, + "loss": 1.0265, + "mean_token_accuracy": 0.6976621747016907, + "num_tokens": 205599855.0, + "step": 346 + }, + { + "epoch": 1.6523809523809523, + "grad_norm": 0.7347867488861084, + "learning_rate": 5e-05, + "loss": 1.0312, + "mean_token_accuracy": 0.6957042217254639, + "num_tokens": 206163338.0, + "step": 347 + }, + { + "epoch": 1.657142857142857, + "grad_norm": 0.8335996866226196, + "learning_rate": 5e-05, + "loss": 1.0524, + "mean_token_accuracy": 0.6909912824630737, + "num_tokens": 206741397.0, + "step": 348 + }, + { + "epoch": 1.6619047619047618, + "grad_norm": 0.7557653188705444, + "learning_rate": 5e-05, + "loss": 1.0479, + "mean_token_accuracy": 0.6910521388053894, + "num_tokens": 207323297.0, + "step": 349 + }, + { + "epoch": 1.6666666666666665, + "grad_norm": 0.751379132270813, + "learning_rate": 5e-05, + "loss": 1.0485, + "mean_token_accuracy": 0.6922094821929932, + "num_tokens": 207908589.0, + "step": 350 + }, + { + "epoch": 1.6714285714285713, + "grad_norm": 0.8020631074905396, + "learning_rate": 5e-05, + "loss": 1.058, + "mean_token_accuracy": 0.6893354058265686, + "num_tokens": 208500281.0, + "step": 351 + }, + { + "epoch": 1.6761904761904762, + "grad_norm": 0.7568333745002747, + "learning_rate": 5e-05, + "loss": 1.0402, + "mean_token_accuracy": 0.6942318081855774, + "num_tokens": 209097427.0, + "step": 352 + }, + { + "epoch": 1.680952380952381, + "grad_norm": 0.698124349117279, + "learning_rate": 5e-05, + "loss": 1.0414, + "mean_token_accuracy": 0.6933043003082275, + "num_tokens": 209704225.0, + "step": 353 + }, + { + "epoch": 1.6857142857142857, + "grad_norm": 0.7386274337768555, + "learning_rate": 5e-05, + "loss": 1.0356, + "mean_token_accuracy": 0.6949059963226318, + "num_tokens": 210313267.0, + "step": 354 + }, + { + "epoch": 1.6904761904761905, + "grad_norm": 0.8014583587646484, + "learning_rate": 5e-05, + "loss": 1.0477, + "mean_token_accuracy": 0.692074179649353, + "num_tokens": 210918588.0, + "step": 355 + }, + { + "epoch": 1.6952380952380952, + "grad_norm": 0.78595370054245, + "learning_rate": 5e-05, + "loss": 1.0443, + "mean_token_accuracy": 0.6920830011367798, + "num_tokens": 211508221.0, + "step": 356 + }, + { + "epoch": 1.7, + "grad_norm": 0.7576943635940552, + "learning_rate": 5e-05, + "loss": 1.0543, + "mean_token_accuracy": 0.6903828382492065, + "num_tokens": 212108728.0, + "step": 357 + }, + { + "epoch": 1.704761904761905, + "grad_norm": 0.8005661368370056, + "learning_rate": 5e-05, + "loss": 1.0432, + "mean_token_accuracy": 0.6928914785385132, + "num_tokens": 212705437.0, + "step": 358 + }, + { + "epoch": 1.7095238095238097, + "grad_norm": 0.6753656268119812, + "learning_rate": 5e-05, + "loss": 1.0519, + "mean_token_accuracy": 0.6903222799301147, + "num_tokens": 213300176.0, + "step": 359 + }, + { + "epoch": 1.7142857142857144, + "grad_norm": 0.7472155690193176, + "learning_rate": 5e-05, + "loss": 1.0518, + "mean_token_accuracy": 0.6901761293411255, + "num_tokens": 213890731.0, + "step": 360 + }, + { + "epoch": 1.7190476190476192, + "grad_norm": 0.7447811961174011, + "learning_rate": 5e-05, + "loss": 1.0331, + "mean_token_accuracy": 0.6944743394851685, + "num_tokens": 214471137.0, + "step": 361 + }, + { + "epoch": 1.723809523809524, + "grad_norm": 0.7896323204040527, + "learning_rate": 5e-05, + "loss": 1.039, + "mean_token_accuracy": 0.6938984394073486, + "num_tokens": 215062165.0, + "step": 362 + }, + { + "epoch": 1.7285714285714286, + "grad_norm": 1.0732545852661133, + "learning_rate": 5e-05, + "loss": 1.0458, + "mean_token_accuracy": 0.6927886009216309, + "num_tokens": 215662977.0, + "step": 363 + }, + { + "epoch": 1.7333333333333334, + "grad_norm": 0.8136167526245117, + "learning_rate": 5e-05, + "loss": 1.0518, + "mean_token_accuracy": 0.6905158758163452, + "num_tokens": 216253942.0, + "step": 364 + }, + { + "epoch": 1.7380952380952381, + "grad_norm": 0.9756861329078674, + "learning_rate": 5e-05, + "loss": 1.0461, + "mean_token_accuracy": 0.6917421221733093, + "num_tokens": 216847247.0, + "step": 365 + }, + { + "epoch": 1.7428571428571429, + "grad_norm": 0.855654776096344, + "learning_rate": 5e-05, + "loss": 1.031, + "mean_token_accuracy": 0.696205735206604, + "num_tokens": 217427979.0, + "step": 366 + }, + { + "epoch": 1.7476190476190476, + "grad_norm": 0.8152772784233093, + "learning_rate": 5e-05, + "loss": 1.0367, + "mean_token_accuracy": 0.6936303377151489, + "num_tokens": 218020622.0, + "step": 367 + }, + { + "epoch": 1.7523809523809524, + "grad_norm": 0.8420906662940979, + "learning_rate": 5e-05, + "loss": 1.0422, + "mean_token_accuracy": 0.6926963329315186, + "num_tokens": 218613768.0, + "step": 368 + }, + { + "epoch": 1.7571428571428571, + "grad_norm": 0.7701714038848877, + "learning_rate": 5e-05, + "loss": 1.0561, + "mean_token_accuracy": 0.6899920105934143, + "num_tokens": 219217863.0, + "step": 369 + }, + { + "epoch": 1.7619047619047619, + "grad_norm": 0.6349092125892639, + "learning_rate": 5e-05, + "loss": 1.043, + "mean_token_accuracy": 0.692497730255127, + "num_tokens": 219826128.0, + "step": 370 + }, + { + "epoch": 1.7666666666666666, + "grad_norm": 0.7852119207382202, + "learning_rate": 5e-05, + "loss": 1.0391, + "mean_token_accuracy": 0.6945936679840088, + "num_tokens": 220424737.0, + "step": 371 + }, + { + "epoch": 1.7714285714285714, + "grad_norm": 0.7147594690322876, + "learning_rate": 5e-05, + "loss": 1.0457, + "mean_token_accuracy": 0.6921483874320984, + "num_tokens": 221010073.0, + "step": 372 + }, + { + "epoch": 1.776190476190476, + "grad_norm": 0.8322044610977173, + "learning_rate": 5e-05, + "loss": 1.0393, + "mean_token_accuracy": 0.6936159133911133, + "num_tokens": 221614799.0, + "step": 373 + }, + { + "epoch": 1.7809523809523808, + "grad_norm": 0.6658322215080261, + "learning_rate": 5e-05, + "loss": 1.0384, + "mean_token_accuracy": 0.6939022541046143, + "num_tokens": 222215943.0, + "step": 374 + }, + { + "epoch": 1.7857142857142856, + "grad_norm": 0.8558494448661804, + "learning_rate": 5e-05, + "loss": 1.0339, + "mean_token_accuracy": 0.6956003904342651, + "num_tokens": 222803822.0, + "step": 375 + }, + { + "epoch": 1.7904761904761903, + "grad_norm": 0.6905757784843445, + "learning_rate": 5e-05, + "loss": 1.0517, + "mean_token_accuracy": 0.6905571818351746, + "num_tokens": 223409541.0, + "step": 376 + }, + { + "epoch": 1.795238095238095, + "grad_norm": 0.8789547681808472, + "learning_rate": 5e-05, + "loss": 1.0386, + "mean_token_accuracy": 0.6935067176818848, + "num_tokens": 223996446.0, + "step": 377 + }, + { + "epoch": 1.8, + "grad_norm": 0.8188236951828003, + "learning_rate": 5e-05, + "loss": 1.041, + "mean_token_accuracy": 0.6929875612258911, + "num_tokens": 224599074.0, + "step": 378 + }, + { + "epoch": 1.8047619047619048, + "grad_norm": 0.7970079183578491, + "learning_rate": 5e-05, + "loss": 1.0332, + "mean_token_accuracy": 0.6959646344184875, + "num_tokens": 225184557.0, + "step": 379 + }, + { + "epoch": 1.8095238095238095, + "grad_norm": 0.755928635597229, + "learning_rate": 5e-05, + "loss": 1.0377, + "mean_token_accuracy": 0.6945538520812988, + "num_tokens": 225774197.0, + "step": 380 + }, + { + "epoch": 1.8142857142857143, + "grad_norm": 0.778325080871582, + "learning_rate": 5e-05, + "loss": 1.03, + "mean_token_accuracy": 0.6950761079788208, + "num_tokens": 226359063.0, + "step": 381 + }, + { + "epoch": 1.819047619047619, + "grad_norm": 0.7879846096038818, + "learning_rate": 5e-05, + "loss": 1.0462, + "mean_token_accuracy": 0.6923890113830566, + "num_tokens": 226962202.0, + "step": 382 + }, + { + "epoch": 1.8238095238095238, + "grad_norm": 0.8618618845939636, + "learning_rate": 5e-05, + "loss": 1.0406, + "mean_token_accuracy": 0.6938865780830383, + "num_tokens": 227541002.0, + "step": 383 + }, + { + "epoch": 1.8285714285714287, + "grad_norm": 0.8557558655738831, + "learning_rate": 5e-05, + "loss": 1.0444, + "mean_token_accuracy": 0.6926047801971436, + "num_tokens": 228134124.0, + "step": 384 + }, + { + "epoch": 1.8333333333333335, + "grad_norm": 0.7237285375595093, + "learning_rate": 5e-05, + "loss": 1.0323, + "mean_token_accuracy": 0.6954296231269836, + "num_tokens": 228729717.0, + "step": 385 + }, + { + "epoch": 1.8380952380952382, + "grad_norm": 0.7802919745445251, + "learning_rate": 5e-05, + "loss": 1.0387, + "mean_token_accuracy": 0.6931858062744141, + "num_tokens": 229318931.0, + "step": 386 + }, + { + "epoch": 1.842857142857143, + "grad_norm": 0.7305516004562378, + "learning_rate": 5e-05, + "loss": 1.0436, + "mean_token_accuracy": 0.6932748556137085, + "num_tokens": 229923675.0, + "step": 387 + }, + { + "epoch": 1.8476190476190477, + "grad_norm": 0.7265799641609192, + "learning_rate": 5e-05, + "loss": 1.0343, + "mean_token_accuracy": 0.6948027610778809, + "num_tokens": 230514254.0, + "step": 388 + }, + { + "epoch": 1.8523809523809525, + "grad_norm": 0.7820598483085632, + "learning_rate": 5e-05, + "loss": 1.0425, + "mean_token_accuracy": 0.6930860280990601, + "num_tokens": 231113801.0, + "step": 389 + }, + { + "epoch": 1.8571428571428572, + "grad_norm": 0.8653613924980164, + "learning_rate": 5e-05, + "loss": 1.0315, + "mean_token_accuracy": 0.6949290037155151, + "num_tokens": 231709098.0, + "step": 390 + }, + { + "epoch": 1.861904761904762, + "grad_norm": 0.7150859832763672, + "learning_rate": 5e-05, + "loss": 1.0443, + "mean_token_accuracy": 0.6924254894256592, + "num_tokens": 232311276.0, + "step": 391 + }, + { + "epoch": 1.8666666666666667, + "grad_norm": 0.9967947006225586, + "learning_rate": 5e-05, + "loss": 1.0341, + "mean_token_accuracy": 0.6945633888244629, + "num_tokens": 232904607.0, + "step": 392 + }, + { + "epoch": 1.8714285714285714, + "grad_norm": 0.7220901846885681, + "learning_rate": 5e-05, + "loss": 1.0264, + "mean_token_accuracy": 0.697299599647522, + "num_tokens": 233493254.0, + "step": 393 + }, + { + "epoch": 1.8761904761904762, + "grad_norm": 0.8333368897438049, + "learning_rate": 5e-05, + "loss": 1.0373, + "mean_token_accuracy": 0.6947458982467651, + "num_tokens": 234085431.0, + "step": 394 + }, + { + "epoch": 1.880952380952381, + "grad_norm": 0.8318260312080383, + "learning_rate": 5e-05, + "loss": 1.0451, + "mean_token_accuracy": 0.6922373175621033, + "num_tokens": 234685396.0, + "step": 395 + }, + { + "epoch": 1.8857142857142857, + "grad_norm": 0.8139061331748962, + "learning_rate": 5e-05, + "loss": 1.0438, + "mean_token_accuracy": 0.6927710175514221, + "num_tokens": 235280049.0, + "step": 396 + }, + { + "epoch": 1.8904761904761904, + "grad_norm": 0.8324995636940002, + "learning_rate": 5e-05, + "loss": 1.0458, + "mean_token_accuracy": 0.6930422782897949, + "num_tokens": 235867276.0, + "step": 397 + }, + { + "epoch": 1.8952380952380952, + "grad_norm": 0.7465384006500244, + "learning_rate": 5e-05, + "loss": 1.0306, + "mean_token_accuracy": 0.6955040693283081, + "num_tokens": 236448647.0, + "step": 398 + }, + { + "epoch": 1.9, + "grad_norm": 0.8792766332626343, + "learning_rate": 5e-05, + "loss": 1.0478, + "mean_token_accuracy": 0.6906482577323914, + "num_tokens": 237054180.0, + "step": 399 + }, + { + "epoch": 1.9047619047619047, + "grad_norm": 0.770839512348175, + "learning_rate": 5e-05, + "loss": 1.0597, + "mean_token_accuracy": 0.6887972354888916, + "num_tokens": 237658218.0, + "step": 400 + }, + { + "epoch": 1.9095238095238094, + "grad_norm": 0.8590947985649109, + "learning_rate": 5e-05, + "loss": 1.0428, + "mean_token_accuracy": 0.6934290528297424, + "num_tokens": 238248393.0, + "step": 401 + }, + { + "epoch": 1.9142857142857141, + "grad_norm": 0.718254804611206, + "learning_rate": 5e-05, + "loss": 1.0294, + "mean_token_accuracy": 0.6960861682891846, + "num_tokens": 238838548.0, + "step": 402 + }, + { + "epoch": 1.919047619047619, + "grad_norm": 0.7036271691322327, + "learning_rate": 5e-05, + "loss": 1.0359, + "mean_token_accuracy": 0.6949036717414856, + "num_tokens": 239442502.0, + "step": 403 + }, + { + "epoch": 1.9238095238095239, + "grad_norm": 0.7857179045677185, + "learning_rate": 5e-05, + "loss": 1.0459, + "mean_token_accuracy": 0.6916754245758057, + "num_tokens": 240029019.0, + "step": 404 + }, + { + "epoch": 1.9285714285714286, + "grad_norm": 0.742304801940918, + "learning_rate": 5e-05, + "loss": 1.0335, + "mean_token_accuracy": 0.6950439214706421, + "num_tokens": 240623504.0, + "step": 405 + }, + { + "epoch": 1.9333333333333333, + "grad_norm": 0.7249507308006287, + "learning_rate": 5e-05, + "loss": 1.0236, + "mean_token_accuracy": 0.6975294351577759, + "num_tokens": 241217102.0, + "step": 406 + }, + { + "epoch": 1.938095238095238, + "grad_norm": 0.7120564579963684, + "learning_rate": 5e-05, + "loss": 1.0335, + "mean_token_accuracy": 0.695677638053894, + "num_tokens": 241812222.0, + "step": 407 + }, + { + "epoch": 1.9428571428571428, + "grad_norm": 0.640818178653717, + "learning_rate": 5e-05, + "loss": 1.0259, + "mean_token_accuracy": 0.6973315477371216, + "num_tokens": 242388669.0, + "step": 408 + }, + { + "epoch": 1.9476190476190476, + "grad_norm": 0.7417107224464417, + "learning_rate": 5e-05, + "loss": 1.0298, + "mean_token_accuracy": 0.6961357593536377, + "num_tokens": 242965686.0, + "step": 409 + }, + { + "epoch": 1.9523809523809523, + "grad_norm": 0.7921696305274963, + "learning_rate": 5e-05, + "loss": 1.0367, + "mean_token_accuracy": 0.6949156522750854, + "num_tokens": 243553853.0, + "step": 410 + }, + { + "epoch": 1.9571428571428573, + "grad_norm": 0.6705108880996704, + "learning_rate": 5e-05, + "loss": 1.0361, + "mean_token_accuracy": 0.6939857602119446, + "num_tokens": 244139754.0, + "step": 411 + }, + { + "epoch": 1.961904761904762, + "grad_norm": 0.7707934975624084, + "learning_rate": 5e-05, + "loss": 1.0374, + "mean_token_accuracy": 0.6933623552322388, + "num_tokens": 244736423.0, + "step": 412 + }, + { + "epoch": 1.9666666666666668, + "grad_norm": 0.6765254735946655, + "learning_rate": 5e-05, + "loss": 1.0379, + "mean_token_accuracy": 0.6947987079620361, + "num_tokens": 245337682.0, + "step": 413 + }, + { + "epoch": 1.9714285714285715, + "grad_norm": 0.8522664904594421, + "learning_rate": 5e-05, + "loss": 1.0388, + "mean_token_accuracy": 0.6939291954040527, + "num_tokens": 245952105.0, + "step": 414 + }, + { + "epoch": 1.9761904761904763, + "grad_norm": 0.7539438009262085, + "learning_rate": 5e-05, + "loss": 1.0242, + "mean_token_accuracy": 0.6971290111541748, + "num_tokens": 246540999.0, + "step": 415 + }, + { + "epoch": 1.980952380952381, + "grad_norm": 0.7162102460861206, + "learning_rate": 5e-05, + "loss": 1.0326, + "mean_token_accuracy": 0.6960386633872986, + "num_tokens": 247142303.0, + "step": 416 + }, + { + "epoch": 1.9857142857142858, + "grad_norm": 0.7146256566047668, + "learning_rate": 5e-05, + "loss": 1.0311, + "mean_token_accuracy": 0.6960699558258057, + "num_tokens": 247732988.0, + "step": 417 + }, + { + "epoch": 1.9904761904761905, + "grad_norm": 0.6776054501533508, + "learning_rate": 5e-05, + "loss": 1.0246, + "mean_token_accuracy": 0.6974151134490967, + "num_tokens": 248334744.0, + "step": 418 + }, + { + "epoch": 1.9952380952380953, + "grad_norm": 0.7252874970436096, + "learning_rate": 5e-05, + "loss": 1.0363, + "mean_token_accuracy": 0.694664716720581, + "num_tokens": 248930347.0, + "step": 419 + }, + { + "epoch": 2.0, + "grad_norm": 0.6627562046051025, + "learning_rate": 5e-05, + "loss": 1.0493, + "mean_token_accuracy": 0.6915898323059082, + "num_tokens": 249522093.0, + "step": 420 + }, + { + "epoch": 2.0047619047619047, + "grad_norm": 1.2676869630813599, + "learning_rate": 5e-05, + "loss": 0.9249, + "mean_token_accuracy": 0.7201952338218689, + "num_tokens": 250112163.0, + "step": 421 + }, + { + "epoch": 2.0095238095238095, + "grad_norm": 0.868251621723175, + "learning_rate": 5e-05, + "loss": 0.9142, + "mean_token_accuracy": 0.7226945161819458, + "num_tokens": 250690466.0, + "step": 422 + }, + { + "epoch": 2.0142857142857142, + "grad_norm": 1.239370584487915, + "learning_rate": 5e-05, + "loss": 0.9161, + "mean_token_accuracy": 0.7218768000602722, + "num_tokens": 251279220.0, + "step": 423 + }, + { + "epoch": 2.019047619047619, + "grad_norm": 1.177413821220398, + "learning_rate": 5e-05, + "loss": 0.9176, + "mean_token_accuracy": 0.7216894626617432, + "num_tokens": 251871717.0, + "step": 424 + }, + { + "epoch": 2.0238095238095237, + "grad_norm": 0.9993072152137756, + "learning_rate": 5e-05, + "loss": 0.9386, + "mean_token_accuracy": 0.7159090042114258, + "num_tokens": 252474129.0, + "step": 425 + }, + { + "epoch": 2.0285714285714285, + "grad_norm": 1.0251927375793457, + "learning_rate": 5e-05, + "loss": 0.901, + "mean_token_accuracy": 0.7260990738868713, + "num_tokens": 253069065.0, + "step": 426 + }, + { + "epoch": 2.033333333333333, + "grad_norm": 0.9324399828910828, + "learning_rate": 5e-05, + "loss": 0.9258, + "mean_token_accuracy": 0.7186737060546875, + "num_tokens": 253654392.0, + "step": 427 + }, + { + "epoch": 2.038095238095238, + "grad_norm": 0.7856104373931885, + "learning_rate": 5e-05, + "loss": 0.917, + "mean_token_accuracy": 0.7210221886634827, + "num_tokens": 254245168.0, + "step": 428 + }, + { + "epoch": 2.0428571428571427, + "grad_norm": 0.9187813401222229, + "learning_rate": 5e-05, + "loss": 0.9229, + "mean_token_accuracy": 0.7194787859916687, + "num_tokens": 254841132.0, + "step": 429 + }, + { + "epoch": 2.0476190476190474, + "grad_norm": 1.0661760568618774, + "learning_rate": 5e-05, + "loss": 0.9101, + "mean_token_accuracy": 0.7229340076446533, + "num_tokens": 255433793.0, + "step": 430 + }, + { + "epoch": 2.052380952380952, + "grad_norm": 0.7787120938301086, + "learning_rate": 5e-05, + "loss": 0.9099, + "mean_token_accuracy": 0.7235040068626404, + "num_tokens": 256015225.0, + "step": 431 + }, + { + "epoch": 2.057142857142857, + "grad_norm": 1.0310695171356201, + "learning_rate": 5e-05, + "loss": 0.9076, + "mean_token_accuracy": 0.7240118384361267, + "num_tokens": 256617333.0, + "step": 432 + }, + { + "epoch": 2.0619047619047617, + "grad_norm": 0.9616384506225586, + "learning_rate": 5e-05, + "loss": 0.9318, + "mean_token_accuracy": 0.7188043594360352, + "num_tokens": 257214120.0, + "step": 433 + }, + { + "epoch": 2.066666666666667, + "grad_norm": 0.9123485088348389, + "learning_rate": 5e-05, + "loss": 0.9129, + "mean_token_accuracy": 0.7229195833206177, + "num_tokens": 257807506.0, + "step": 434 + }, + { + "epoch": 2.0714285714285716, + "grad_norm": 0.8565008044242859, + "learning_rate": 5e-05, + "loss": 0.9327, + "mean_token_accuracy": 0.7169825434684753, + "num_tokens": 258414225.0, + "step": 435 + }, + { + "epoch": 2.0761904761904764, + "grad_norm": 0.9601689577102661, + "learning_rate": 5e-05, + "loss": 0.9076, + "mean_token_accuracy": 0.7225948572158813, + "num_tokens": 259009305.0, + "step": 436 + }, + { + "epoch": 2.080952380952381, + "grad_norm": 1.0294512510299683, + "learning_rate": 5e-05, + "loss": 0.9044, + "mean_token_accuracy": 0.7241244912147522, + "num_tokens": 259600283.0, + "step": 437 + }, + { + "epoch": 2.085714285714286, + "grad_norm": 0.7002186179161072, + "learning_rate": 5e-05, + "loss": 0.9291, + "mean_token_accuracy": 0.7189517021179199, + "num_tokens": 260203907.0, + "step": 438 + }, + { + "epoch": 2.0904761904761906, + "grad_norm": 0.970206081867218, + "learning_rate": 5e-05, + "loss": 0.924, + "mean_token_accuracy": 0.7191022038459778, + "num_tokens": 260795356.0, + "step": 439 + }, + { + "epoch": 2.0952380952380953, + "grad_norm": 0.7655655145645142, + "learning_rate": 5e-05, + "loss": 0.9011, + "mean_token_accuracy": 0.7260123491287231, + "num_tokens": 261376414.0, + "step": 440 + }, + { + "epoch": 2.1, + "grad_norm": 0.8377019762992859, + "learning_rate": 5e-05, + "loss": 0.9256, + "mean_token_accuracy": 0.7194352149963379, + "num_tokens": 261965206.0, + "step": 441 + }, + { + "epoch": 2.104761904761905, + "grad_norm": 0.8415313959121704, + "learning_rate": 5e-05, + "loss": 0.9195, + "mean_token_accuracy": 0.7209702730178833, + "num_tokens": 262561371.0, + "step": 442 + }, + { + "epoch": 2.1095238095238096, + "grad_norm": 0.7528442740440369, + "learning_rate": 5e-05, + "loss": 0.9051, + "mean_token_accuracy": 0.7249529361724854, + "num_tokens": 263159935.0, + "step": 443 + }, + { + "epoch": 2.1142857142857143, + "grad_norm": 1.149167776107788, + "learning_rate": 5e-05, + "loss": 0.9222, + "mean_token_accuracy": 0.7187386155128479, + "num_tokens": 263754299.0, + "step": 444 + }, + { + "epoch": 2.119047619047619, + "grad_norm": 0.7269712686538696, + "learning_rate": 5e-05, + "loss": 0.923, + "mean_token_accuracy": 0.7199124693870544, + "num_tokens": 264354630.0, + "step": 445 + }, + { + "epoch": 2.123809523809524, + "grad_norm": 0.8058280348777771, + "learning_rate": 5e-05, + "loss": 0.9394, + "mean_token_accuracy": 0.7152698040008545, + "num_tokens": 264964353.0, + "step": 446 + }, + { + "epoch": 2.1285714285714286, + "grad_norm": 0.9243003726005554, + "learning_rate": 5e-05, + "loss": 0.9235, + "mean_token_accuracy": 0.7199217081069946, + "num_tokens": 265561411.0, + "step": 447 + }, + { + "epoch": 2.1333333333333333, + "grad_norm": 0.8085525035858154, + "learning_rate": 5e-05, + "loss": 0.9178, + "mean_token_accuracy": 0.7206405401229858, + "num_tokens": 266155790.0, + "step": 448 + }, + { + "epoch": 2.138095238095238, + "grad_norm": 0.88421231508255, + "learning_rate": 5e-05, + "loss": 0.9214, + "mean_token_accuracy": 0.7204050421714783, + "num_tokens": 266757269.0, + "step": 449 + }, + { + "epoch": 2.142857142857143, + "grad_norm": 0.7341966032981873, + "learning_rate": 5e-05, + "loss": 0.9372, + "mean_token_accuracy": 0.7158248424530029, + "num_tokens": 267369143.0, + "step": 450 + }, + { + "epoch": 2.1476190476190475, + "grad_norm": 0.8347304463386536, + "learning_rate": 5e-05, + "loss": 0.9061, + "mean_token_accuracy": 0.7247138023376465, + "num_tokens": 267965437.0, + "step": 451 + }, + { + "epoch": 2.1523809523809523, + "grad_norm": 0.8849761486053467, + "learning_rate": 5e-05, + "loss": 0.9153, + "mean_token_accuracy": 0.7221096754074097, + "num_tokens": 268571463.0, + "step": 452 + }, + { + "epoch": 2.157142857142857, + "grad_norm": 0.7802500128746033, + "learning_rate": 5e-05, + "loss": 0.9128, + "mean_token_accuracy": 0.722852349281311, + "num_tokens": 269157041.0, + "step": 453 + }, + { + "epoch": 2.1619047619047618, + "grad_norm": 0.9233799576759338, + "learning_rate": 5e-05, + "loss": 0.9139, + "mean_token_accuracy": 0.7228530645370483, + "num_tokens": 269758195.0, + "step": 454 + }, + { + "epoch": 2.1666666666666665, + "grad_norm": 0.8539539575576782, + "learning_rate": 5e-05, + "loss": 0.9357, + "mean_token_accuracy": 0.7167932987213135, + "num_tokens": 270349613.0, + "step": 455 + }, + { + "epoch": 2.1714285714285713, + "grad_norm": 0.8928214907646179, + "learning_rate": 5e-05, + "loss": 0.9272, + "mean_token_accuracy": 0.7180873155593872, + "num_tokens": 270947174.0, + "step": 456 + }, + { + "epoch": 2.176190476190476, + "grad_norm": 0.7778669595718384, + "learning_rate": 5e-05, + "loss": 0.9342, + "mean_token_accuracy": 0.7173746824264526, + "num_tokens": 271549889.0, + "step": 457 + }, + { + "epoch": 2.1809523809523808, + "grad_norm": 0.8179411292076111, + "learning_rate": 5e-05, + "loss": 0.9085, + "mean_token_accuracy": 0.7242327928543091, + "num_tokens": 272139782.0, + "step": 458 + }, + { + "epoch": 2.185714285714286, + "grad_norm": 0.7449883222579956, + "learning_rate": 5e-05, + "loss": 0.9228, + "mean_token_accuracy": 0.7201469540596008, + "num_tokens": 272746279.0, + "step": 459 + }, + { + "epoch": 2.1904761904761907, + "grad_norm": 0.873669445514679, + "learning_rate": 5e-05, + "loss": 0.9116, + "mean_token_accuracy": 0.7229417562484741, + "num_tokens": 273348449.0, + "step": 460 + }, + { + "epoch": 2.1952380952380954, + "grad_norm": 0.9282508492469788, + "learning_rate": 5e-05, + "loss": 0.9295, + "mean_token_accuracy": 0.7189794778823853, + "num_tokens": 273949928.0, + "step": 461 + }, + { + "epoch": 2.2, + "grad_norm": 0.741436779499054, + "learning_rate": 5e-05, + "loss": 0.9176, + "mean_token_accuracy": 0.7210873365402222, + "num_tokens": 274548070.0, + "step": 462 + }, + { + "epoch": 2.204761904761905, + "grad_norm": 0.8721827268600464, + "learning_rate": 5e-05, + "loss": 0.9205, + "mean_token_accuracy": 0.7202156186103821, + "num_tokens": 275135656.0, + "step": 463 + }, + { + "epoch": 2.2095238095238097, + "grad_norm": 0.9933586120605469, + "learning_rate": 5e-05, + "loss": 0.9239, + "mean_token_accuracy": 0.7202807664871216, + "num_tokens": 275740663.0, + "step": 464 + }, + { + "epoch": 2.2142857142857144, + "grad_norm": 0.7048609852790833, + "learning_rate": 5e-05, + "loss": 0.9217, + "mean_token_accuracy": 0.7210299968719482, + "num_tokens": 276346207.0, + "step": 465 + }, + { + "epoch": 2.219047619047619, + "grad_norm": 0.8262238502502441, + "learning_rate": 5e-05, + "loss": 0.9215, + "mean_token_accuracy": 0.7200701236724854, + "num_tokens": 276940208.0, + "step": 466 + }, + { + "epoch": 2.223809523809524, + "grad_norm": 0.756079375743866, + "learning_rate": 5e-05, + "loss": 0.9364, + "mean_token_accuracy": 0.7165220379829407, + "num_tokens": 277539762.0, + "step": 467 + }, + { + "epoch": 2.2285714285714286, + "grad_norm": 0.9260476231575012, + "learning_rate": 5e-05, + "loss": 0.934, + "mean_token_accuracy": 0.716609537601471, + "num_tokens": 278136526.0, + "step": 468 + }, + { + "epoch": 2.2333333333333334, + "grad_norm": 0.8242080807685852, + "learning_rate": 5e-05, + "loss": 0.9275, + "mean_token_accuracy": 0.7184122800827026, + "num_tokens": 278723575.0, + "step": 469 + }, + { + "epoch": 2.238095238095238, + "grad_norm": 0.7585132718086243, + "learning_rate": 5e-05, + "loss": 0.9052, + "mean_token_accuracy": 0.7242292165756226, + "num_tokens": 279313692.0, + "step": 470 + }, + { + "epoch": 2.242857142857143, + "grad_norm": 0.9254531860351562, + "learning_rate": 5e-05, + "loss": 0.9335, + "mean_token_accuracy": 0.7172324657440186, + "num_tokens": 279911596.0, + "step": 471 + }, + { + "epoch": 2.2476190476190476, + "grad_norm": 0.7721312046051025, + "learning_rate": 5e-05, + "loss": 0.9192, + "mean_token_accuracy": 0.7213947772979736, + "num_tokens": 280516626.0, + "step": 472 + }, + { + "epoch": 2.2523809523809524, + "grad_norm": 0.933292031288147, + "learning_rate": 5e-05, + "loss": 0.9231, + "mean_token_accuracy": 0.7197685241699219, + "num_tokens": 281109826.0, + "step": 473 + }, + { + "epoch": 2.257142857142857, + "grad_norm": 0.7447443604469299, + "learning_rate": 5e-05, + "loss": 0.9293, + "mean_token_accuracy": 0.7192258834838867, + "num_tokens": 281705908.0, + "step": 474 + }, + { + "epoch": 2.261904761904762, + "grad_norm": 0.7889474630355835, + "learning_rate": 5e-05, + "loss": 0.9241, + "mean_token_accuracy": 0.7181416749954224, + "num_tokens": 282295646.0, + "step": 475 + }, + { + "epoch": 2.2666666666666666, + "grad_norm": 0.8643538951873779, + "learning_rate": 5e-05, + "loss": 0.9142, + "mean_token_accuracy": 0.7213423252105713, + "num_tokens": 282882984.0, + "step": 476 + }, + { + "epoch": 2.2714285714285714, + "grad_norm": 0.9072065353393555, + "learning_rate": 5e-05, + "loss": 0.9228, + "mean_token_accuracy": 0.7194763422012329, + "num_tokens": 283470314.0, + "step": 477 + }, + { + "epoch": 2.276190476190476, + "grad_norm": 0.7041733860969543, + "learning_rate": 5e-05, + "loss": 0.9243, + "mean_token_accuracy": 0.7197008728981018, + "num_tokens": 284064822.0, + "step": 478 + }, + { + "epoch": 2.280952380952381, + "grad_norm": 0.9274478554725647, + "learning_rate": 5e-05, + "loss": 0.918, + "mean_token_accuracy": 0.7210705280303955, + "num_tokens": 284659143.0, + "step": 479 + }, + { + "epoch": 2.2857142857142856, + "grad_norm": 0.7842460870742798, + "learning_rate": 5e-05, + "loss": 0.9252, + "mean_token_accuracy": 0.7193678617477417, + "num_tokens": 285260603.0, + "step": 480 + }, + { + "epoch": 2.2904761904761903, + "grad_norm": 0.8480785489082336, + "learning_rate": 5e-05, + "loss": 0.9213, + "mean_token_accuracy": 0.7197173237800598, + "num_tokens": 285858617.0, + "step": 481 + }, + { + "epoch": 2.295238095238095, + "grad_norm": 0.744513213634491, + "learning_rate": 5e-05, + "loss": 0.9258, + "mean_token_accuracy": 0.7199747562408447, + "num_tokens": 286462909.0, + "step": 482 + }, + { + "epoch": 2.3, + "grad_norm": 1.0083340406417847, + "learning_rate": 5e-05, + "loss": 0.9243, + "mean_token_accuracy": 0.7203906774520874, + "num_tokens": 287057508.0, + "step": 483 + }, + { + "epoch": 2.3047619047619046, + "grad_norm": 0.852171003818512, + "learning_rate": 5e-05, + "loss": 0.9335, + "mean_token_accuracy": 0.7167496681213379, + "num_tokens": 287647864.0, + "step": 484 + }, + { + "epoch": 2.3095238095238093, + "grad_norm": 0.7880772948265076, + "learning_rate": 5e-05, + "loss": 0.9311, + "mean_token_accuracy": 0.717706024646759, + "num_tokens": 288244654.0, + "step": 485 + }, + { + "epoch": 2.314285714285714, + "grad_norm": 0.9683517217636108, + "learning_rate": 5e-05, + "loss": 0.9407, + "mean_token_accuracy": 0.7157230377197266, + "num_tokens": 288833805.0, + "step": 486 + }, + { + "epoch": 2.319047619047619, + "grad_norm": 0.7185930609703064, + "learning_rate": 5e-05, + "loss": 0.9264, + "mean_token_accuracy": 0.717744767665863, + "num_tokens": 289430249.0, + "step": 487 + }, + { + "epoch": 2.323809523809524, + "grad_norm": 0.8448125123977661, + "learning_rate": 5e-05, + "loss": 0.9294, + "mean_token_accuracy": 0.7187172770500183, + "num_tokens": 290017640.0, + "step": 488 + }, + { + "epoch": 2.3285714285714287, + "grad_norm": 1.0417472124099731, + "learning_rate": 5e-05, + "loss": 0.9246, + "mean_token_accuracy": 0.718608558177948, + "num_tokens": 290598414.0, + "step": 489 + }, + { + "epoch": 2.3333333333333335, + "grad_norm": 0.7742448449134827, + "learning_rate": 5e-05, + "loss": 0.9176, + "mean_token_accuracy": 0.7205394506454468, + "num_tokens": 291181812.0, + "step": 490 + }, + { + "epoch": 2.3380952380952382, + "grad_norm": 0.7823668122291565, + "learning_rate": 5e-05, + "loss": 0.9278, + "mean_token_accuracy": 0.7190169095993042, + "num_tokens": 291764317.0, + "step": 491 + }, + { + "epoch": 2.342857142857143, + "grad_norm": 0.9660963416099548, + "learning_rate": 5e-05, + "loss": 0.9332, + "mean_token_accuracy": 0.7169288396835327, + "num_tokens": 292350504.0, + "step": 492 + }, + { + "epoch": 2.3476190476190477, + "grad_norm": 0.7401145696640015, + "learning_rate": 5e-05, + "loss": 0.9327, + "mean_token_accuracy": 0.7173548340797424, + "num_tokens": 292951776.0, + "step": 493 + }, + { + "epoch": 2.3523809523809525, + "grad_norm": 0.8637396097183228, + "learning_rate": 5e-05, + "loss": 0.925, + "mean_token_accuracy": 0.7193020582199097, + "num_tokens": 293532488.0, + "step": 494 + }, + { + "epoch": 2.357142857142857, + "grad_norm": 0.8286495804786682, + "learning_rate": 5e-05, + "loss": 0.913, + "mean_token_accuracy": 0.7225058078765869, + "num_tokens": 294121146.0, + "step": 495 + }, + { + "epoch": 2.361904761904762, + "grad_norm": 0.7148199081420898, + "learning_rate": 5e-05, + "loss": 0.9312, + "mean_token_accuracy": 0.717105507850647, + "num_tokens": 294726732.0, + "step": 496 + }, + { + "epoch": 2.3666666666666667, + "grad_norm": 0.7451285123825073, + "learning_rate": 5e-05, + "loss": 0.9245, + "mean_token_accuracy": 0.7194476127624512, + "num_tokens": 295306697.0, + "step": 497 + }, + { + "epoch": 2.3714285714285714, + "grad_norm": 0.8156387209892273, + "learning_rate": 5e-05, + "loss": 0.9445, + "mean_token_accuracy": 0.7139602899551392, + "num_tokens": 295894972.0, + "step": 498 + }, + { + "epoch": 2.376190476190476, + "grad_norm": 0.7706066966056824, + "learning_rate": 5e-05, + "loss": 0.9396, + "mean_token_accuracy": 0.7149537801742554, + "num_tokens": 296505345.0, + "step": 499 + }, + { + "epoch": 2.380952380952381, + "grad_norm": 0.8760485053062439, + "learning_rate": 5e-05, + "loss": 0.9247, + "mean_token_accuracy": 0.7189267873764038, + "num_tokens": 297100909.0, + "step": 500 + }, + { + "epoch": 2.3857142857142857, + "grad_norm": 0.8128599524497986, + "learning_rate": 5e-05, + "loss": 0.9296, + "mean_token_accuracy": 0.7185119390487671, + "num_tokens": 297690267.0, + "step": 501 + }, + { + "epoch": 2.3904761904761904, + "grad_norm": 0.7821332812309265, + "learning_rate": 5e-05, + "loss": 0.9214, + "mean_token_accuracy": 0.7194477915763855, + "num_tokens": 298283484.0, + "step": 502 + }, + { + "epoch": 2.395238095238095, + "grad_norm": 0.8020161986351013, + "learning_rate": 5e-05, + "loss": 0.938, + "mean_token_accuracy": 0.7154324054718018, + "num_tokens": 298880193.0, + "step": 503 + }, + { + "epoch": 2.4, + "grad_norm": 0.7699880003929138, + "learning_rate": 5e-05, + "loss": 0.9179, + "mean_token_accuracy": 0.7209833860397339, + "num_tokens": 299479995.0, + "step": 504 + }, + { + "epoch": 2.4047619047619047, + "grad_norm": 0.7420978546142578, + "learning_rate": 5e-05, + "loss": 0.9299, + "mean_token_accuracy": 0.7178776264190674, + "num_tokens": 300068384.0, + "step": 505 + }, + { + "epoch": 2.4095238095238094, + "grad_norm": 0.7923991084098816, + "learning_rate": 5e-05, + "loss": 0.942, + "mean_token_accuracy": 0.7148730158805847, + "num_tokens": 300687274.0, + "step": 506 + }, + { + "epoch": 2.414285714285714, + "grad_norm": 0.842926025390625, + "learning_rate": 5e-05, + "loss": 0.9345, + "mean_token_accuracy": 0.7167230844497681, + "num_tokens": 301288728.0, + "step": 507 + }, + { + "epoch": 2.419047619047619, + "grad_norm": 0.6903310418128967, + "learning_rate": 5e-05, + "loss": 0.9286, + "mean_token_accuracy": 0.717685878276825, + "num_tokens": 301868926.0, + "step": 508 + }, + { + "epoch": 2.4238095238095236, + "grad_norm": 0.8731923699378967, + "learning_rate": 5e-05, + "loss": 0.9228, + "mean_token_accuracy": 0.7197592258453369, + "num_tokens": 302467630.0, + "step": 509 + }, + { + "epoch": 2.4285714285714284, + "grad_norm": 0.7663103342056274, + "learning_rate": 5e-05, + "loss": 0.9473, + "mean_token_accuracy": 0.7138726711273193, + "num_tokens": 303058965.0, + "step": 510 + }, + { + "epoch": 2.4333333333333336, + "grad_norm": 0.8337984681129456, + "learning_rate": 5e-05, + "loss": 0.9469, + "mean_token_accuracy": 0.7131837606430054, + "num_tokens": 303639694.0, + "step": 511 + }, + { + "epoch": 2.4380952380952383, + "grad_norm": 0.7574532628059387, + "learning_rate": 5e-05, + "loss": 0.9244, + "mean_token_accuracy": 0.7201915383338928, + "num_tokens": 304234504.0, + "step": 512 + }, + { + "epoch": 2.442857142857143, + "grad_norm": 0.8913818597793579, + "learning_rate": 5e-05, + "loss": 0.9423, + "mean_token_accuracy": 0.7151418924331665, + "num_tokens": 304822484.0, + "step": 513 + }, + { + "epoch": 2.447619047619048, + "grad_norm": 0.8777763247489929, + "learning_rate": 5e-05, + "loss": 0.9295, + "mean_token_accuracy": 0.718494176864624, + "num_tokens": 305405226.0, + "step": 514 + }, + { + "epoch": 2.4523809523809526, + "grad_norm": 0.7090263962745667, + "learning_rate": 5e-05, + "loss": 0.9322, + "mean_token_accuracy": 0.7172112464904785, + "num_tokens": 306007153.0, + "step": 515 + }, + { + "epoch": 2.4571428571428573, + "grad_norm": 0.9021292924880981, + "learning_rate": 5e-05, + "loss": 0.9285, + "mean_token_accuracy": 0.7194123864173889, + "num_tokens": 306588836.0, + "step": 516 + }, + { + "epoch": 2.461904761904762, + "grad_norm": 0.8665414452552795, + "learning_rate": 5e-05, + "loss": 0.9417, + "mean_token_accuracy": 0.7150377035140991, + "num_tokens": 307200955.0, + "step": 517 + }, + { + "epoch": 2.466666666666667, + "grad_norm": 0.6820929646492004, + "learning_rate": 5e-05, + "loss": 0.9267, + "mean_token_accuracy": 0.7185186743736267, + "num_tokens": 307799028.0, + "step": 518 + }, + { + "epoch": 2.4714285714285715, + "grad_norm": 0.8302870392799377, + "learning_rate": 5e-05, + "loss": 0.9544, + "mean_token_accuracy": 0.7117083072662354, + "num_tokens": 308389748.0, + "step": 519 + }, + { + "epoch": 2.4761904761904763, + "grad_norm": 0.7668029069900513, + "learning_rate": 5e-05, + "loss": 0.9377, + "mean_token_accuracy": 0.71599942445755, + "num_tokens": 308978715.0, + "step": 520 + }, + { + "epoch": 2.480952380952381, + "grad_norm": 0.7976921200752258, + "learning_rate": 5e-05, + "loss": 0.9397, + "mean_token_accuracy": 0.7154068946838379, + "num_tokens": 309587298.0, + "step": 521 + }, + { + "epoch": 2.4857142857142858, + "grad_norm": 0.7976080179214478, + "learning_rate": 5e-05, + "loss": 0.9256, + "mean_token_accuracy": 0.7187434434890747, + "num_tokens": 310173585.0, + "step": 522 + }, + { + "epoch": 2.4904761904761905, + "grad_norm": 0.7799238562583923, + "learning_rate": 5e-05, + "loss": 0.9327, + "mean_token_accuracy": 0.7178182601928711, + "num_tokens": 310760313.0, + "step": 523 + }, + { + "epoch": 2.4952380952380953, + "grad_norm": 0.7197299599647522, + "learning_rate": 5e-05, + "loss": 0.9392, + "mean_token_accuracy": 0.7159803509712219, + "num_tokens": 311374002.0, + "step": 524 + }, + { + "epoch": 2.5, + "grad_norm": 0.8350206017494202, + "learning_rate": 5e-05, + "loss": 0.9318, + "mean_token_accuracy": 0.7174832820892334, + "num_tokens": 311952533.0, + "step": 525 + }, + { + "epoch": 2.5047619047619047, + "grad_norm": 0.7931807041168213, + "learning_rate": 5e-05, + "loss": 0.9278, + "mean_token_accuracy": 0.7176194787025452, + "num_tokens": 312542383.0, + "step": 526 + }, + { + "epoch": 2.5095238095238095, + "grad_norm": 0.8491684198379517, + "learning_rate": 5e-05, + "loss": 0.9368, + "mean_token_accuracy": 0.7157042026519775, + "num_tokens": 313136427.0, + "step": 527 + }, + { + "epoch": 2.5142857142857142, + "grad_norm": 0.7639768123626709, + "learning_rate": 5e-05, + "loss": 0.9245, + "mean_token_accuracy": 0.7202243804931641, + "num_tokens": 313731115.0, + "step": 528 + }, + { + "epoch": 2.519047619047619, + "grad_norm": 0.6792297959327698, + "learning_rate": 5e-05, + "loss": 0.9269, + "mean_token_accuracy": 0.7188189029693604, + "num_tokens": 314316253.0, + "step": 529 + }, + { + "epoch": 2.5238095238095237, + "grad_norm": 0.9707839488983154, + "learning_rate": 5e-05, + "loss": 0.9265, + "mean_token_accuracy": 0.7191447615623474, + "num_tokens": 314906539.0, + "step": 530 + }, + { + "epoch": 2.5285714285714285, + "grad_norm": 0.7617918848991394, + "learning_rate": 5e-05, + "loss": 0.9266, + "mean_token_accuracy": 0.7187902927398682, + "num_tokens": 315491005.0, + "step": 531 + }, + { + "epoch": 2.533333333333333, + "grad_norm": 0.8404021859169006, + "learning_rate": 5e-05, + "loss": 0.9394, + "mean_token_accuracy": 0.7145642042160034, + "num_tokens": 316086156.0, + "step": 532 + }, + { + "epoch": 2.538095238095238, + "grad_norm": 0.8707789182662964, + "learning_rate": 5e-05, + "loss": 0.9438, + "mean_token_accuracy": 0.7149832844734192, + "num_tokens": 316687284.0, + "step": 533 + }, + { + "epoch": 2.5428571428571427, + "grad_norm": 0.8554860353469849, + "learning_rate": 5e-05, + "loss": 0.9298, + "mean_token_accuracy": 0.7182329893112183, + "num_tokens": 317280976.0, + "step": 534 + }, + { + "epoch": 2.5476190476190474, + "grad_norm": 0.788116455078125, + "learning_rate": 5e-05, + "loss": 0.9368, + "mean_token_accuracy": 0.7163056135177612, + "num_tokens": 317869704.0, + "step": 535 + }, + { + "epoch": 2.552380952380952, + "grad_norm": 0.8229620456695557, + "learning_rate": 5e-05, + "loss": 0.9378, + "mean_token_accuracy": 0.7156033515930176, + "num_tokens": 318453830.0, + "step": 536 + }, + { + "epoch": 2.557142857142857, + "grad_norm": 0.6864573359489441, + "learning_rate": 5e-05, + "loss": 0.9407, + "mean_token_accuracy": 0.7147571444511414, + "num_tokens": 319036628.0, + "step": 537 + }, + { + "epoch": 2.5619047619047617, + "grad_norm": 0.7234057188034058, + "learning_rate": 5e-05, + "loss": 0.9349, + "mean_token_accuracy": 0.7168844938278198, + "num_tokens": 319641680.0, + "step": 538 + }, + { + "epoch": 2.5666666666666664, + "grad_norm": 0.8731569647789001, + "learning_rate": 5e-05, + "loss": 0.9379, + "mean_token_accuracy": 0.7156639099121094, + "num_tokens": 320235018.0, + "step": 539 + }, + { + "epoch": 2.571428571428571, + "grad_norm": 0.9141507148742676, + "learning_rate": 5e-05, + "loss": 0.9445, + "mean_token_accuracy": 0.7142863273620605, + "num_tokens": 320839523.0, + "step": 540 + }, + { + "epoch": 2.576190476190476, + "grad_norm": 0.7209877967834473, + "learning_rate": 5e-05, + "loss": 0.9343, + "mean_token_accuracy": 0.7165625095367432, + "num_tokens": 321445469.0, + "step": 541 + }, + { + "epoch": 2.580952380952381, + "grad_norm": 0.8998512029647827, + "learning_rate": 5e-05, + "loss": 0.9415, + "mean_token_accuracy": 0.7142558097839355, + "num_tokens": 322040382.0, + "step": 542 + }, + { + "epoch": 2.585714285714286, + "grad_norm": 0.7413470149040222, + "learning_rate": 5e-05, + "loss": 0.939, + "mean_token_accuracy": 0.7156506180763245, + "num_tokens": 322631980.0, + "step": 543 + }, + { + "epoch": 2.5904761904761906, + "grad_norm": 0.8033062219619751, + "learning_rate": 5e-05, + "loss": 0.9354, + "mean_token_accuracy": 0.7161453366279602, + "num_tokens": 323217003.0, + "step": 544 + }, + { + "epoch": 2.5952380952380953, + "grad_norm": 0.6921564936637878, + "learning_rate": 5e-05, + "loss": 0.9383, + "mean_token_accuracy": 0.7157716155052185, + "num_tokens": 323797882.0, + "step": 545 + }, + { + "epoch": 2.6, + "grad_norm": 0.70876544713974, + "learning_rate": 5e-05, + "loss": 0.9403, + "mean_token_accuracy": 0.7146531343460083, + "num_tokens": 324381508.0, + "step": 546 + }, + { + "epoch": 2.604761904761905, + "grad_norm": 0.7912217378616333, + "learning_rate": 5e-05, + "loss": 0.9564, + "mean_token_accuracy": 0.7106965780258179, + "num_tokens": 324979976.0, + "step": 547 + }, + { + "epoch": 2.6095238095238096, + "grad_norm": 0.830923318862915, + "learning_rate": 5e-05, + "loss": 0.9355, + "mean_token_accuracy": 0.7164920568466187, + "num_tokens": 325579147.0, + "step": 548 + }, + { + "epoch": 2.6142857142857143, + "grad_norm": 0.6366074681282043, + "learning_rate": 5e-05, + "loss": 0.9394, + "mean_token_accuracy": 0.7160769701004028, + "num_tokens": 326189724.0, + "step": 549 + }, + { + "epoch": 2.619047619047619, + "grad_norm": 0.9230322241783142, + "learning_rate": 5e-05, + "loss": 0.9393, + "mean_token_accuracy": 0.7153929471969604, + "num_tokens": 326781040.0, + "step": 550 + }, + { + "epoch": 2.623809523809524, + "grad_norm": 0.6958957314491272, + "learning_rate": 5e-05, + "loss": 0.9338, + "mean_token_accuracy": 0.7172648906707764, + "num_tokens": 327384993.0, + "step": 551 + }, + { + "epoch": 2.6285714285714286, + "grad_norm": 0.7691433429718018, + "learning_rate": 5e-05, + "loss": 0.9202, + "mean_token_accuracy": 0.7202857732772827, + "num_tokens": 327967527.0, + "step": 552 + }, + { + "epoch": 2.6333333333333333, + "grad_norm": 0.8308330774307251, + "learning_rate": 5e-05, + "loss": 0.929, + "mean_token_accuracy": 0.717819094657898, + "num_tokens": 328556111.0, + "step": 553 + }, + { + "epoch": 2.638095238095238, + "grad_norm": 0.7323694229125977, + "learning_rate": 5e-05, + "loss": 0.937, + "mean_token_accuracy": 0.7159186601638794, + "num_tokens": 329156419.0, + "step": 554 + }, + { + "epoch": 2.642857142857143, + "grad_norm": 0.790925145149231, + "learning_rate": 5e-05, + "loss": 0.9591, + "mean_token_accuracy": 0.7102572321891785, + "num_tokens": 329765795.0, + "step": 555 + }, + { + "epoch": 2.6476190476190475, + "grad_norm": 0.792596161365509, + "learning_rate": 5e-05, + "loss": 0.9401, + "mean_token_accuracy": 0.7162899971008301, + "num_tokens": 330366805.0, + "step": 556 + }, + { + "epoch": 2.6523809523809523, + "grad_norm": 0.6795808672904968, + "learning_rate": 5e-05, + "loss": 0.9243, + "mean_token_accuracy": 0.7189604043960571, + "num_tokens": 330960683.0, + "step": 557 + }, + { + "epoch": 2.657142857142857, + "grad_norm": 0.725006103515625, + "learning_rate": 5e-05, + "loss": 0.9458, + "mean_token_accuracy": 0.7142760753631592, + "num_tokens": 331566617.0, + "step": 558 + }, + { + "epoch": 2.6619047619047618, + "grad_norm": 0.7129206657409668, + "learning_rate": 5e-05, + "loss": 0.9281, + "mean_token_accuracy": 0.7185797691345215, + "num_tokens": 332146780.0, + "step": 559 + }, + { + "epoch": 2.6666666666666665, + "grad_norm": 0.6961240768432617, + "learning_rate": 5e-05, + "loss": 0.942, + "mean_token_accuracy": 0.7155134677886963, + "num_tokens": 332744565.0, + "step": 560 + }, + { + "epoch": 2.6714285714285713, + "grad_norm": 0.8725546598434448, + "learning_rate": 5e-05, + "loss": 0.9405, + "mean_token_accuracy": 0.7151129245758057, + "num_tokens": 333327246.0, + "step": 561 + }, + { + "epoch": 2.6761904761904765, + "grad_norm": 0.8665727972984314, + "learning_rate": 5e-05, + "loss": 0.9314, + "mean_token_accuracy": 0.7170665264129639, + "num_tokens": 333924279.0, + "step": 562 + }, + { + "epoch": 2.680952380952381, + "grad_norm": 0.8317073583602905, + "learning_rate": 5e-05, + "loss": 0.9364, + "mean_token_accuracy": 0.715925931930542, + "num_tokens": 334526451.0, + "step": 563 + }, + { + "epoch": 2.685714285714286, + "grad_norm": 0.6629255414009094, + "learning_rate": 5e-05, + "loss": 0.9412, + "mean_token_accuracy": 0.7153710126876831, + "num_tokens": 335121296.0, + "step": 564 + }, + { + "epoch": 2.6904761904761907, + "grad_norm": 1.0133806467056274, + "learning_rate": 5e-05, + "loss": 0.9396, + "mean_token_accuracy": 0.7147179245948792, + "num_tokens": 335705229.0, + "step": 565 + }, + { + "epoch": 2.6952380952380954, + "grad_norm": 0.8434333205223083, + "learning_rate": 5e-05, + "loss": 0.9449, + "mean_token_accuracy": 0.7144158482551575, + "num_tokens": 336296515.0, + "step": 566 + }, + { + "epoch": 2.7, + "grad_norm": 0.7621550559997559, + "learning_rate": 5e-05, + "loss": 0.9472, + "mean_token_accuracy": 0.7137055993080139, + "num_tokens": 336898581.0, + "step": 567 + }, + { + "epoch": 2.704761904761905, + "grad_norm": 0.7147604823112488, + "learning_rate": 5e-05, + "loss": 0.9526, + "mean_token_accuracy": 0.7130710482597351, + "num_tokens": 337492720.0, + "step": 568 + }, + { + "epoch": 2.7095238095238097, + "grad_norm": 0.9300767779350281, + "learning_rate": 5e-05, + "loss": 0.9394, + "mean_token_accuracy": 0.7157748937606812, + "num_tokens": 338088080.0, + "step": 569 + }, + { + "epoch": 2.7142857142857144, + "grad_norm": 0.7240475416183472, + "learning_rate": 5e-05, + "loss": 0.9386, + "mean_token_accuracy": 0.7157989740371704, + "num_tokens": 338682863.0, + "step": 570 + }, + { + "epoch": 2.719047619047619, + "grad_norm": 0.757023811340332, + "learning_rate": 5e-05, + "loss": 0.9445, + "mean_token_accuracy": 0.714621901512146, + "num_tokens": 339274595.0, + "step": 571 + }, + { + "epoch": 2.723809523809524, + "grad_norm": 0.6328327059745789, + "learning_rate": 5e-05, + "loss": 0.9429, + "mean_token_accuracy": 0.7150536775588989, + "num_tokens": 339857633.0, + "step": 572 + }, + { + "epoch": 2.7285714285714286, + "grad_norm": 0.7335704565048218, + "learning_rate": 5e-05, + "loss": 0.9286, + "mean_token_accuracy": 0.7187144756317139, + "num_tokens": 340451043.0, + "step": 573 + }, + { + "epoch": 2.7333333333333334, + "grad_norm": 0.672327995300293, + "learning_rate": 5e-05, + "loss": 0.9474, + "mean_token_accuracy": 0.7142379283905029, + "num_tokens": 341042238.0, + "step": 574 + }, + { + "epoch": 2.738095238095238, + "grad_norm": 0.6720926761627197, + "learning_rate": 5e-05, + "loss": 0.9484, + "mean_token_accuracy": 0.7132652997970581, + "num_tokens": 341644006.0, + "step": 575 + }, + { + "epoch": 2.742857142857143, + "grad_norm": 0.7574931979179382, + "learning_rate": 5e-05, + "loss": 0.9401, + "mean_token_accuracy": 0.7155739665031433, + "num_tokens": 342238235.0, + "step": 576 + }, + { + "epoch": 2.7476190476190476, + "grad_norm": 0.7247797250747681, + "learning_rate": 5e-05, + "loss": 0.933, + "mean_token_accuracy": 0.7169485092163086, + "num_tokens": 342831260.0, + "step": 577 + }, + { + "epoch": 2.7523809523809524, + "grad_norm": 0.7431164383888245, + "learning_rate": 5e-05, + "loss": 0.9456, + "mean_token_accuracy": 0.7141156792640686, + "num_tokens": 343427629.0, + "step": 578 + }, + { + "epoch": 2.757142857142857, + "grad_norm": 0.8442338705062866, + "learning_rate": 5e-05, + "loss": 0.9472, + "mean_token_accuracy": 0.7142153978347778, + "num_tokens": 344021401.0, + "step": 579 + }, + { + "epoch": 2.761904761904762, + "grad_norm": 0.8185229897499084, + "learning_rate": 5e-05, + "loss": 0.9344, + "mean_token_accuracy": 0.7175722122192383, + "num_tokens": 344610108.0, + "step": 580 + }, + { + "epoch": 2.7666666666666666, + "grad_norm": 0.8149795532226562, + "learning_rate": 5e-05, + "loss": 0.947, + "mean_token_accuracy": 0.7136144042015076, + "num_tokens": 345217241.0, + "step": 581 + }, + { + "epoch": 2.7714285714285714, + "grad_norm": 0.6615715622901917, + "learning_rate": 5e-05, + "loss": 0.9366, + "mean_token_accuracy": 0.7157867550849915, + "num_tokens": 345804258.0, + "step": 582 + }, + { + "epoch": 2.776190476190476, + "grad_norm": 0.7365272045135498, + "learning_rate": 5e-05, + "loss": 0.9406, + "mean_token_accuracy": 0.7151838541030884, + "num_tokens": 346399481.0, + "step": 583 + }, + { + "epoch": 2.780952380952381, + "grad_norm": 0.7271730303764343, + "learning_rate": 5e-05, + "loss": 0.9317, + "mean_token_accuracy": 0.717467188835144, + "num_tokens": 346997922.0, + "step": 584 + }, + { + "epoch": 2.7857142857142856, + "grad_norm": 0.7325976490974426, + "learning_rate": 5e-05, + "loss": 0.9423, + "mean_token_accuracy": 0.7154061198234558, + "num_tokens": 347597863.0, + "step": 585 + }, + { + "epoch": 2.7904761904761903, + "grad_norm": 0.7731750011444092, + "learning_rate": 5e-05, + "loss": 0.9413, + "mean_token_accuracy": 0.7148541212081909, + "num_tokens": 348201046.0, + "step": 586 + }, + { + "epoch": 2.795238095238095, + "grad_norm": 0.7586520314216614, + "learning_rate": 5e-05, + "loss": 0.9304, + "mean_token_accuracy": 0.7172263860702515, + "num_tokens": 348787326.0, + "step": 587 + }, + { + "epoch": 2.8, + "grad_norm": 0.9960665702819824, + "learning_rate": 5e-05, + "loss": 0.9437, + "mean_token_accuracy": 0.7142516374588013, + "num_tokens": 349380645.0, + "step": 588 + }, + { + "epoch": 2.8047619047619046, + "grad_norm": 0.7079677581787109, + "learning_rate": 5e-05, + "loss": 0.938, + "mean_token_accuracy": 0.715801477432251, + "num_tokens": 349983092.0, + "step": 589 + }, + { + "epoch": 2.8095238095238093, + "grad_norm": 0.8647035360336304, + "learning_rate": 5e-05, + "loss": 0.9368, + "mean_token_accuracy": 0.7164928913116455, + "num_tokens": 350576836.0, + "step": 590 + }, + { + "epoch": 2.814285714285714, + "grad_norm": 0.7630114555358887, + "learning_rate": 5e-05, + "loss": 0.9378, + "mean_token_accuracy": 0.716993510723114, + "num_tokens": 351172725.0, + "step": 591 + }, + { + "epoch": 2.819047619047619, + "grad_norm": 0.7441176772117615, + "learning_rate": 5e-05, + "loss": 0.9337, + "mean_token_accuracy": 0.7171525955200195, + "num_tokens": 351777142.0, + "step": 592 + }, + { + "epoch": 2.8238095238095235, + "grad_norm": 0.7635340094566345, + "learning_rate": 5e-05, + "loss": 0.9316, + "mean_token_accuracy": 0.717369019985199, + "num_tokens": 352361834.0, + "step": 593 + }, + { + "epoch": 2.8285714285714287, + "grad_norm": 0.7294594645500183, + "learning_rate": 5e-05, + "loss": 0.9407, + "mean_token_accuracy": 0.7157597541809082, + "num_tokens": 352964892.0, + "step": 594 + }, + { + "epoch": 2.8333333333333335, + "grad_norm": 0.7735794186592102, + "learning_rate": 5e-05, + "loss": 0.9404, + "mean_token_accuracy": 0.7143554091453552, + "num_tokens": 353563696.0, + "step": 595 + }, + { + "epoch": 2.8380952380952382, + "grad_norm": 0.7808977365493774, + "learning_rate": 5e-05, + "loss": 0.9428, + "mean_token_accuracy": 0.7150280475616455, + "num_tokens": 354162323.0, + "step": 596 + }, + { + "epoch": 2.842857142857143, + "grad_norm": 0.8053597211837769, + "learning_rate": 5e-05, + "loss": 0.9656, + "mean_token_accuracy": 0.7087494730949402, + "num_tokens": 354763224.0, + "step": 597 + }, + { + "epoch": 2.8476190476190477, + "grad_norm": 0.6898967027664185, + "learning_rate": 5e-05, + "loss": 0.9341, + "mean_token_accuracy": 0.7171127796173096, + "num_tokens": 355360011.0, + "step": 598 + }, + { + "epoch": 2.8523809523809525, + "grad_norm": 0.8251420855522156, + "learning_rate": 5e-05, + "loss": 0.9406, + "mean_token_accuracy": 0.7159343957901001, + "num_tokens": 355948770.0, + "step": 599 + }, + { + "epoch": 2.857142857142857, + "grad_norm": 0.8773722052574158, + "learning_rate": 5e-05, + "loss": 0.9506, + "mean_token_accuracy": 0.7127441763877869, + "num_tokens": 356555915.0, + "step": 600 + }, + { + "epoch": 2.861904761904762, + "grad_norm": 0.7758999466896057, + "learning_rate": 5e-05, + "loss": 0.9523, + "mean_token_accuracy": 0.7118942737579346, + "num_tokens": 357168089.0, + "step": 601 + }, + { + "epoch": 2.8666666666666667, + "grad_norm": 0.6969651579856873, + "learning_rate": 5e-05, + "loss": 0.9343, + "mean_token_accuracy": 0.7165572047233582, + "num_tokens": 357762209.0, + "step": 602 + }, + { + "epoch": 2.8714285714285714, + "grad_norm": 0.7684289813041687, + "learning_rate": 5e-05, + "loss": 0.9413, + "mean_token_accuracy": 0.7148808836936951, + "num_tokens": 358365632.0, + "step": 603 + }, + { + "epoch": 2.876190476190476, + "grad_norm": 0.7433676719665527, + "learning_rate": 5e-05, + "loss": 0.9534, + "mean_token_accuracy": 0.711889386177063, + "num_tokens": 358969038.0, + "step": 604 + }, + { + "epoch": 2.880952380952381, + "grad_norm": 0.7116939425468445, + "learning_rate": 5e-05, + "loss": 0.9366, + "mean_token_accuracy": 0.7166457176208496, + "num_tokens": 359560638.0, + "step": 605 + }, + { + "epoch": 2.8857142857142857, + "grad_norm": 0.824992299079895, + "learning_rate": 5e-05, + "loss": 0.9472, + "mean_token_accuracy": 0.7136282920837402, + "num_tokens": 360159047.0, + "step": 606 + }, + { + "epoch": 2.8904761904761904, + "grad_norm": 0.8088555335998535, + "learning_rate": 5e-05, + "loss": 0.9429, + "mean_token_accuracy": 0.7151459455490112, + "num_tokens": 360752550.0, + "step": 607 + }, + { + "epoch": 2.895238095238095, + "grad_norm": 0.7039157748222351, + "learning_rate": 5e-05, + "loss": 0.9436, + "mean_token_accuracy": 0.7141553163528442, + "num_tokens": 361348966.0, + "step": 608 + }, + { + "epoch": 2.9, + "grad_norm": 0.7326273918151855, + "learning_rate": 5e-05, + "loss": 0.9378, + "mean_token_accuracy": 0.7166075706481934, + "num_tokens": 361933541.0, + "step": 609 + }, + { + "epoch": 2.9047619047619047, + "grad_norm": 0.7972474098205566, + "learning_rate": 5e-05, + "loss": 0.9438, + "mean_token_accuracy": 0.7152196168899536, + "num_tokens": 362522689.0, + "step": 610 + }, + { + "epoch": 2.9095238095238094, + "grad_norm": 0.7569297552108765, + "learning_rate": 5e-05, + "loss": 0.9249, + "mean_token_accuracy": 0.7186825275421143, + "num_tokens": 363107779.0, + "step": 611 + }, + { + "epoch": 2.914285714285714, + "grad_norm": 0.8032863140106201, + "learning_rate": 5e-05, + "loss": 0.9384, + "mean_token_accuracy": 0.7161685228347778, + "num_tokens": 363690809.0, + "step": 612 + }, + { + "epoch": 2.919047619047619, + "grad_norm": 0.7047289609909058, + "learning_rate": 5e-05, + "loss": 0.938, + "mean_token_accuracy": 0.715911865234375, + "num_tokens": 364279923.0, + "step": 613 + }, + { + "epoch": 2.923809523809524, + "grad_norm": 0.7094652652740479, + "learning_rate": 5e-05, + "loss": 0.9282, + "mean_token_accuracy": 0.7177805304527283, + "num_tokens": 364846929.0, + "step": 614 + }, + { + "epoch": 2.928571428571429, + "grad_norm": 0.7806612253189087, + "learning_rate": 5e-05, + "loss": 0.9533, + "mean_token_accuracy": 0.71263587474823, + "num_tokens": 365427755.0, + "step": 615 + }, + { + "epoch": 2.9333333333333336, + "grad_norm": 0.8107709288597107, + "learning_rate": 5e-05, + "loss": 0.938, + "mean_token_accuracy": 0.7158170342445374, + "num_tokens": 366022879.0, + "step": 616 + }, + { + "epoch": 2.9380952380952383, + "grad_norm": 0.7008342146873474, + "learning_rate": 5e-05, + "loss": 0.9391, + "mean_token_accuracy": 0.7148101925849915, + "num_tokens": 366608198.0, + "step": 617 + }, + { + "epoch": 2.942857142857143, + "grad_norm": 0.8497748970985413, + "learning_rate": 5e-05, + "loss": 0.9402, + "mean_token_accuracy": 0.715301513671875, + "num_tokens": 367187170.0, + "step": 618 + }, + { + "epoch": 2.947619047619048, + "grad_norm": 0.7606064081192017, + "learning_rate": 5e-05, + "loss": 0.9399, + "mean_token_accuracy": 0.7149579524993896, + "num_tokens": 367778542.0, + "step": 619 + }, + { + "epoch": 2.9523809523809526, + "grad_norm": 0.7233796715736389, + "learning_rate": 5e-05, + "loss": 0.9453, + "mean_token_accuracy": 0.7133791446685791, + "num_tokens": 368374361.0, + "step": 620 + }, + { + "epoch": 2.9571428571428573, + "grad_norm": 0.9382302165031433, + "learning_rate": 5e-05, + "loss": 0.9441, + "mean_token_accuracy": 0.714198112487793, + "num_tokens": 368975961.0, + "step": 621 + }, + { + "epoch": 2.961904761904762, + "grad_norm": 0.6656110286712646, + "learning_rate": 5e-05, + "loss": 0.9202, + "mean_token_accuracy": 0.7207600474357605, + "num_tokens": 369565801.0, + "step": 622 + }, + { + "epoch": 2.966666666666667, + "grad_norm": 0.7818631529808044, + "learning_rate": 5e-05, + "loss": 0.9469, + "mean_token_accuracy": 0.7133038640022278, + "num_tokens": 370147963.0, + "step": 623 + }, + { + "epoch": 2.9714285714285715, + "grad_norm": 0.72702556848526, + "learning_rate": 5e-05, + "loss": 0.9421, + "mean_token_accuracy": 0.7146210670471191, + "num_tokens": 370730337.0, + "step": 624 + }, + { + "epoch": 2.9761904761904763, + "grad_norm": 0.7305231094360352, + "learning_rate": 5e-05, + "loss": 0.9358, + "mean_token_accuracy": 0.7157225012779236, + "num_tokens": 371313464.0, + "step": 625 + }, + { + "epoch": 2.980952380952381, + "grad_norm": 0.7342341542243958, + "learning_rate": 5e-05, + "loss": 0.9397, + "mean_token_accuracy": 0.7159093618392944, + "num_tokens": 371903211.0, + "step": 626 + }, + { + "epoch": 2.9857142857142858, + "grad_norm": 0.7552860379219055, + "learning_rate": 5e-05, + "loss": 0.9316, + "mean_token_accuracy": 0.7179030179977417, + "num_tokens": 372496314.0, + "step": 627 + }, + { + "epoch": 2.9904761904761905, + "grad_norm": 0.7640341520309448, + "learning_rate": 5e-05, + "loss": 0.9454, + "mean_token_accuracy": 0.7136441469192505, + "num_tokens": 373096610.0, + "step": 628 + }, + { + "epoch": 2.9952380952380953, + "grad_norm": 0.6567404866218567, + "learning_rate": 5e-05, + "loss": 0.9456, + "mean_token_accuracy": 0.7132456302642822, + "num_tokens": 373694165.0, + "step": 629 + }, + { + "epoch": 3.0, + "grad_norm": 0.8179944753646851, + "learning_rate": 5e-05, + "loss": 0.9331, + "mean_token_accuracy": 0.7179580926895142, + "num_tokens": 374283247.0, + "step": 630 + }, + { + "epoch": 3.0, + "step": 630, + "total_flos": 2.1853937174671524e+18, + "train_loss": 1.0672233128358448, + "train_runtime": 1837.0629, + "train_samples_per_second": 175.58, + "train_steps_per_second": 0.343 + } + ], + "logging_steps": 1, + "max_steps": 630, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 315, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.1853937174671524e+18, + "train_batch_size": 128, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..56bf4cc --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3175a475c4325d0718531465cb5f055a2f775fbd670cd58c4faefe3cba8af1d8 +size 13329