commit ab19dd3c4d43b3f52e842e2aaabaeaf22b92bde1 Author: ModelHub XC Date: Mon Jun 15 17:44:17 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: danielm1405/lr-1e-05-epochs-1.0-summ-c37f22a8 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..54b33f1 --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +--- +base_model: meta-llama/Llama-3.1-8B +library_name: transformers +tags: +- generated_from_trainer +- alignment-handbook +- sft +- trl +licence: license +--- + +# Model Card for None + +This model is a fine-tuned version of [meta-llama/Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/pw-cvlab-dmarczak/huggingface/runs/hq2pii1g) + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.24.0 +- Transformers: 4.57.1 +- Pytorch: 2.6.0+cu124.post3 +- Datasets: 4.3.0 +- Tokenizers: 0.22.1 + +## Citations + + + +Cite TRL as: + +```bibtex +@misc{vonwerra2022trl, + title = {{TRL: Transformer Reinforcement Learning}}, + author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec}, + year = 2020, + journal = {GitHub repository}, + publisher = {GitHub}, + howpublished = {\url{https://github.com/huggingface/trl}} +} +``` \ No newline at end of file diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..07b695f --- /dev/null +++ b/all_results.json @@ -0,0 +1,21 @@ +{ + "entropy": 1.6328125, + "epoch": 1.0, + "eval_entropy": 1.6205357142857142, + "eval_loss": 1.1272830963134766, + "eval_mean_token_accuracy": 0.7246029462133136, + "eval_num_tokens": 785524633.0, + "eval_runtime": 4.7136, + "eval_samples": 1000, + "eval_samples_per_second": 92.922, + "eval_steps_per_second": 1.485, + "mean_token_accuracy": 0.7037371794382731, + "num_tokens": 785524633.0, + "total_flos": 1254725330468864.0, + "train_loss": 1.377876113716714, + "train_runtime": 31760.1021, + "train_samples": 1871956, + "train_samples_per_second": 24.347, + "train_steps_per_second": 0.19, + "weight_norm": 259.0048262098604 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..1bad6a0 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,93 @@ +{{- bos_token }} +{%- if custom_tools is defined %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if not tools_in_user_message is defined %} + {%- set tools_in_user_message = true %} +{%- endif %} +{%- if not date_string is defined %} + {%- if strftime_now is defined %} + {%- set date_string = strftime_now("%d %b %Y") %} + {%- else %} + {%- set date_string = "26 Jul 2024" %} + {%- endif %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- set system_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} +{%- else %} + {%- set system_message = "" %} +{%- endif %} + +{#- System message #} +{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} +{%- if tools is not none %} + {{- "Environment: ipython\n" }} +{%- endif %} +{{- "Cutting Knowledge Date: December 2023\n" }} +{{- "Today Date: " + date_string + "\n\n" }} +{%- if tools is not none and not tools_in_user_message %} + {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} +{%- endif %} +{{- system_message }} +{{- "<|eot_id|>" }} + +{#- Custom tools are passed in a user message with some extra guidance #} +{%- if tools_in_user_message and not tools is none %} + {#- Extract the first user message so we can plug it in here #} + {%- if messages | length != 0 %} + {%- set first_user_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} + {%- else %} + {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} +{%- endif %} + {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} + {{- "Given the following functions, please respond with a JSON for a function call " }} + {{- "with its proper arguments that best answers the given prompt.\n\n" }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} + {{- first_user_message + "<|eot_id|>"}} +{%- endif %} + +{%- for message in messages %} + {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} + {%- elif 'tool_calls' in message %} + {%- if not message.tool_calls|length == 1 %} + {{- raise_exception("This model only supports single tool-calls at once!") }} + {%- endif %} + {%- set tool_call = message.tool_calls[0].function %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} + {{- '{"name": "' + tool_call.name + '", ' }} + {{- '"parameters": ' }} + {{- tool_call.arguments | tojson }} + {{- "}" }} + {{- "<|eot_id|>" }} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} + {%- if message.content is mapping or message.content is iterable %} + {{- message.content | tojson }} + {%- else %} + {{- message.content }} + {%- endif %} + {{- "<|eot_id|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..3a7db4d --- /dev/null +++ b/config.json @@ -0,0 +1,35 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 8.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "transformers_version": "4.57.1", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/eval_results.json b/eval_results.json new file mode 100644 index 0000000..91fe8f8 --- /dev/null +++ b/eval_results.json @@ -0,0 +1,12 @@ +{ + "epoch": 1.0, + "eval_entropy": 1.6205357142857142, + "eval_loss": 1.1272830963134766, + "eval_mean_token_accuracy": 0.7246029462133136, + "eval_num_tokens": 785524633.0, + "eval_runtime": 4.7136, + "eval_samples": 1000, + "eval_samples_per_second": 92.922, + "eval_steps_per_second": 1.485, + "weight_norm": 259.0048262098604 +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..19cebe5 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..e848a53 --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:067f796f28f67250e765f0a4fbc04a5b9edebd64f869e526c685427c78b07f90 +size 4976698672 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..7569412 --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc47a878f89f94ce80ea7d5221ec73697896793448d3d0281bba1fa69a8ff15d +size 4999802720 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..3e48897 --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:826284f0eebb61061c2a08824df4b9cf21c04edd74b2e12ced26e21aa0e7d034 +size 4915916176 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..74df49e --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f7eec543b72f8d537c39e296a292f0989d514ed07718aeace8ba4966493fe5c +size 1168138808 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..58a7ef4 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,299 @@ +{ + "metadata": { + "total_parameters": 266240, + "total_size": 16060522496 + }, + "weight_map": { + "lm_head.weight": "model-00004-of-00004.safetensors", + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..cfabacc --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,16 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..5346f10 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2062 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..cc2d558 --- /dev/null +++ b/train_results.json @@ -0,0 +1,13 @@ +{ + "entropy": 1.6328125, + "epoch": 1.0, + "mean_token_accuracy": 0.7037371794382731, + "num_tokens": 785524633.0, + "total_flos": 1254725330468864.0, + "train_loss": 1.377876113716714, + "train_runtime": 31760.1021, + "train_samples": 1871956, + "train_samples_per_second": 24.347, + "train_steps_per_second": 0.19, + "weight_norm": 259.0048262098604 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..c214d8c --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,6746 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 100, + "global_step": 6042, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.365625, + "epoch": 0.0016552180749813787, + "grad_norm": 296.5989440232189, + "learning_rate": 4.945054945054946e-07, + "loss": 2.1428, + "mean_token_accuracy": 0.5626892387866974, + "num_tokens": 1296615.0, + "step": 10 + }, + { + "entropy": 2.403125, + "epoch": 0.0033104361499627574, + "grad_norm": 38.429677962946826, + "learning_rate": 1.0439560439560442e-06, + "loss": 2.0233, + "mean_token_accuracy": 0.5697839587926865, + "num_tokens": 2596599.0, + "step": 20 + }, + { + "entropy": 2.35078125, + "epoch": 0.004965654224944136, + "grad_norm": 145.91558163703996, + "learning_rate": 1.5934065934065933e-06, + "loss": 1.8657, + "mean_token_accuracy": 0.5830829530954361, + "num_tokens": 3897613.0, + "step": 30 + }, + { + "entropy": 2.3234375, + "epoch": 0.006620872299925515, + "grad_norm": 3.5508531681248234, + "learning_rate": 2.1428571428571427e-06, + "loss": 1.8052, + "mean_token_accuracy": 0.587401494383812, + "num_tokens": 5197364.0, + "step": 40 + }, + { + "entropy": 2.32734375, + "epoch": 0.008276090374906894, + "grad_norm": 12.967114023768268, + "learning_rate": 2.6923076923076923e-06, + "loss": 1.7629, + "mean_token_accuracy": 0.5930066972970962, + "num_tokens": 6496874.0, + "step": 50 + }, + { + "entropy": 2.215625, + "epoch": 0.009931308449888272, + "grad_norm": 22.476372402807346, + "learning_rate": 3.2417582417582424e-06, + "loss": 1.705, + "mean_token_accuracy": 0.600334295630455, + "num_tokens": 7796445.0, + "step": 60 + }, + { + "entropy": 2.240625, + "epoch": 0.011586526524869651, + "grad_norm": 24.729519644852978, + "learning_rate": 3.7912087912087915e-06, + "loss": 1.6858, + "mean_token_accuracy": 0.6018239200115204, + "num_tokens": 9096380.0, + "step": 70 + }, + { + "entropy": 2.2265625, + "epoch": 0.01324174459985103, + "grad_norm": 5.072293692146677, + "learning_rate": 4.340659340659341e-06, + "loss": 1.6914, + "mean_token_accuracy": 0.6032298028469085, + "num_tokens": 10396160.0, + "step": 80 + }, + { + "entropy": 2.18359375, + "epoch": 0.014896962674832408, + "grad_norm": 5.810988857944605, + "learning_rate": 4.890109890109891e-06, + "loss": 1.6724, + "mean_token_accuracy": 0.6093996286392211, + "num_tokens": 11695591.0, + "step": 90 + }, + { + "entropy": 2.19375, + "epoch": 0.016552180749813787, + "grad_norm": 3.774497024365393, + "learning_rate": 5.43956043956044e-06, + "loss": 1.6692, + "mean_token_accuracy": 0.608141627907753, + "num_tokens": 12996612.0, + "step": 100 + }, + { + "epoch": 0.016552180749813787, + "eval_entropy": 2.1941964285714284, + "eval_loss": 1.6620970964431763, + "eval_mean_token_accuracy": 0.6078114765030997, + "eval_num_tokens": 12996612.0, + "eval_runtime": 4.658, + "eval_samples_per_second": 94.032, + "eval_steps_per_second": 1.503, + "step": 100 + }, + { + "entropy": 2.20859375, + "epoch": 0.018207398824795166, + "grad_norm": 3.1665337983103368, + "learning_rate": 5.989010989010989e-06, + "loss": 1.6475, + "mean_token_accuracy": 0.6088291198015213, + "num_tokens": 14295976.0, + "step": 110 + }, + { + "entropy": 2.19140625, + "epoch": 0.019862616899776545, + "grad_norm": 3.639847656480574, + "learning_rate": 6.538461538461539e-06, + "loss": 1.6307, + "mean_token_accuracy": 0.6096893697977066, + "num_tokens": 15595132.0, + "step": 120 + }, + { + "entropy": 2.15, + "epoch": 0.021517834974757923, + "grad_norm": 2.559487699244455, + "learning_rate": 7.087912087912089e-06, + "loss": 1.6354, + "mean_token_accuracy": 0.6084069937467576, + "num_tokens": 16896093.0, + "step": 130 + }, + { + "entropy": 2.11796875, + "epoch": 0.023173053049739302, + "grad_norm": 2.407328382517491, + "learning_rate": 7.637362637362638e-06, + "loss": 1.6367, + "mean_token_accuracy": 0.6078297168016433, + "num_tokens": 18199950.0, + "step": 140 + }, + { + "entropy": 2.11171875, + "epoch": 0.02482827112472068, + "grad_norm": 2.5551406118692523, + "learning_rate": 8.186813186813188e-06, + "loss": 1.6402, + "mean_token_accuracy": 0.6070237040519715, + "num_tokens": 19500472.0, + "step": 150 + }, + { + "entropy": 2.12890625, + "epoch": 0.02648348919970206, + "grad_norm": 2.4295368037010308, + "learning_rate": 8.736263736263737e-06, + "loss": 1.6507, + "mean_token_accuracy": 0.6055437594652175, + "num_tokens": 20803662.0, + "step": 160 + }, + { + "entropy": 2.1046875, + "epoch": 0.028138707274683438, + "grad_norm": 2.196243899431398, + "learning_rate": 9.285714285714288e-06, + "loss": 1.6288, + "mean_token_accuracy": 0.6093661665916443, + "num_tokens": 22102369.0, + "step": 170 + }, + { + "entropy": 2.0921875, + "epoch": 0.029793925349664817, + "grad_norm": 2.1284602552110545, + "learning_rate": 9.835164835164835e-06, + "loss": 1.6516, + "mean_token_accuracy": 0.6045530080795288, + "num_tokens": 23402124.0, + "step": 180 + }, + { + "entropy": 2.08671875, + "epoch": 0.0314491434246462, + "grad_norm": 2.2851970341534846, + "learning_rate": 9.988054607508533e-06, + "loss": 1.634, + "mean_token_accuracy": 0.607491621375084, + "num_tokens": 24703898.0, + "step": 190 + }, + { + "entropy": 2.07109375, + "epoch": 0.033104361499627574, + "grad_norm": 2.1329337596282523, + "learning_rate": 9.97098976109215e-06, + "loss": 1.6332, + "mean_token_accuracy": 0.6088778257369996, + "num_tokens": 26002612.0, + "step": 200 + }, + { + "epoch": 0.033104361499627574, + "eval_entropy": 2.005580357142857, + "eval_loss": 1.6407272815704346, + "eval_mean_token_accuracy": 0.6067083988870893, + "eval_num_tokens": 26002612.0, + "eval_runtime": 4.6484, + "eval_samples_per_second": 94.225, + "eval_steps_per_second": 1.506, + "step": 200 + }, + { + "entropy": 2.053515625, + "epoch": 0.034759579574608956, + "grad_norm": 2.1004592467884207, + "learning_rate": 9.953924914675769e-06, + "loss": 1.6598, + "mean_token_accuracy": 0.6031040459871292, + "num_tokens": 27300200.0, + "step": 210 + }, + { + "entropy": 2.073828125, + "epoch": 0.03641479764959033, + "grad_norm": 2.2991158244145695, + "learning_rate": 9.936860068259387e-06, + "loss": 1.6486, + "mean_token_accuracy": 0.6055460929870605, + "num_tokens": 28601892.0, + "step": 220 + }, + { + "entropy": 2.04140625, + "epoch": 0.038070015724571714, + "grad_norm": 2.1120485647004723, + "learning_rate": 9.919795221843003e-06, + "loss": 1.6131, + "mean_token_accuracy": 0.6110236883163452, + "num_tokens": 29900285.0, + "step": 230 + }, + { + "entropy": 2.044921875, + "epoch": 0.03972523379955309, + "grad_norm": 2.1624846028351428, + "learning_rate": 9.902730375426623e-06, + "loss": 1.6325, + "mean_token_accuracy": 0.6063523352146148, + "num_tokens": 31196785.0, + "step": 240 + }, + { + "entropy": 2.04375, + "epoch": 0.04138045187453447, + "grad_norm": 2.4404660402029847, + "learning_rate": 9.88566552901024e-06, + "loss": 1.6132, + "mean_token_accuracy": 0.613209393620491, + "num_tokens": 32497260.0, + "step": 250 + }, + { + "entropy": 2.045703125, + "epoch": 0.043035669949515847, + "grad_norm": 2.2288267806354543, + "learning_rate": 9.868600682593858e-06, + "loss": 1.622, + "mean_token_accuracy": 0.609785509109497, + "num_tokens": 33797365.0, + "step": 260 + }, + { + "entropy": 2.046875, + "epoch": 0.04469088802449723, + "grad_norm": 2.5255965439134593, + "learning_rate": 9.851535836177474e-06, + "loss": 1.6273, + "mean_token_accuracy": 0.6079647839069366, + "num_tokens": 35093601.0, + "step": 270 + }, + { + "entropy": 2.0390625, + "epoch": 0.046346106099478604, + "grad_norm": 2.1637531836026405, + "learning_rate": 9.834470989761094e-06, + "loss": 1.6157, + "mean_token_accuracy": 0.6107521414756775, + "num_tokens": 36393956.0, + "step": 280 + }, + { + "entropy": 2.036328125, + "epoch": 0.048001324174459986, + "grad_norm": 2.406219067553743, + "learning_rate": 9.81740614334471e-06, + "loss": 1.6106, + "mean_token_accuracy": 0.6119499117136001, + "num_tokens": 37694962.0, + "step": 290 + }, + { + "entropy": 2.04453125, + "epoch": 0.04965654224944136, + "grad_norm": 2.118600595916072, + "learning_rate": 9.800341296928329e-06, + "loss": 1.6294, + "mean_token_accuracy": 0.6089090555906296, + "num_tokens": 38994785.0, + "step": 300 + }, + { + "epoch": 0.04965654224944136, + "eval_entropy": 2.0513392857142856, + "eval_loss": 1.6214131116867065, + "eval_mean_token_accuracy": 0.6092906253678458, + "eval_num_tokens": 38994785.0, + "eval_runtime": 4.6472, + "eval_samples_per_second": 94.25, + "eval_steps_per_second": 1.506, + "step": 300 + }, + { + "entropy": 2.040234375, + "epoch": 0.051311760324422744, + "grad_norm": 4.770669000441156, + "learning_rate": 9.783276450511947e-06, + "loss": 1.6212, + "mean_token_accuracy": 0.609554973244667, + "num_tokens": 40292211.0, + "step": 310 + }, + { + "entropy": 2.02265625, + "epoch": 0.05296697839940412, + "grad_norm": 2.2503245189277368, + "learning_rate": 9.766211604095563e-06, + "loss": 1.6133, + "mean_token_accuracy": 0.6125765919685364, + "num_tokens": 41592815.0, + "step": 320 + }, + { + "entropy": 2.034765625, + "epoch": 0.0546221964743855, + "grad_norm": 2.68756155165337, + "learning_rate": 9.749146757679183e-06, + "loss": 1.6053, + "mean_token_accuracy": 0.6159348815679551, + "num_tokens": 42893339.0, + "step": 330 + }, + { + "entropy": 2.015625, + "epoch": 0.056277414549366876, + "grad_norm": 2.50384909167604, + "learning_rate": 9.7320819112628e-06, + "loss": 1.589, + "mean_token_accuracy": 0.6157853901386261, + "num_tokens": 44194088.0, + "step": 340 + }, + { + "entropy": 2.0171875, + "epoch": 0.05793263262434826, + "grad_norm": 2.247196358694374, + "learning_rate": 9.715017064846417e-06, + "loss": 1.5948, + "mean_token_accuracy": 0.6164389908313751, + "num_tokens": 45492774.0, + "step": 350 + }, + { + "entropy": 2.008203125, + "epoch": 0.059587850699329634, + "grad_norm": 2.1480008922719604, + "learning_rate": 9.697952218430034e-06, + "loss": 1.6044, + "mean_token_accuracy": 0.6138787299394608, + "num_tokens": 46792839.0, + "step": 360 + }, + { + "entropy": 2.02109375, + "epoch": 0.061243068774311016, + "grad_norm": 2.092803130947849, + "learning_rate": 9.680887372013654e-06, + "loss": 1.6384, + "mean_token_accuracy": 0.6074419260025025, + "num_tokens": 48093863.0, + "step": 370 + }, + { + "entropy": 2.012109375, + "epoch": 0.0628982868492924, + "grad_norm": 2.1658009537857197, + "learning_rate": 9.66382252559727e-06, + "loss": 1.6034, + "mean_token_accuracy": 0.6132849752902985, + "num_tokens": 49392359.0, + "step": 380 + }, + { + "entropy": 2.012890625, + "epoch": 0.06455350492427377, + "grad_norm": 2.127584885124077, + "learning_rate": 9.646757679180888e-06, + "loss": 1.6338, + "mean_token_accuracy": 0.608244127035141, + "num_tokens": 50689973.0, + "step": 390 + }, + { + "entropy": 2.005859375, + "epoch": 0.06620872299925515, + "grad_norm": 2.128996226850232, + "learning_rate": 9.629692832764506e-06, + "loss": 1.612, + "mean_token_accuracy": 0.6121783524751663, + "num_tokens": 51989997.0, + "step": 400 + }, + { + "epoch": 0.06620872299925515, + "eval_entropy": 2.060267857142857, + "eval_loss": 1.6094664335250854, + "eval_mean_token_accuracy": 0.6129385658672878, + "eval_num_tokens": 51989997.0, + "eval_runtime": 4.6206, + "eval_samples_per_second": 94.793, + "eval_steps_per_second": 1.515, + "step": 400 + }, + { + "entropy": 2.03046875, + "epoch": 0.06786394107423653, + "grad_norm": 2.0401280909532122, + "learning_rate": 9.612627986348123e-06, + "loss": 1.612, + "mean_token_accuracy": 0.6135723024606705, + "num_tokens": 53290884.0, + "step": 410 + }, + { + "entropy": 2.018359375, + "epoch": 0.06951915914921791, + "grad_norm": 2.2825002530465075, + "learning_rate": 9.595563139931742e-06, + "loss": 1.6108, + "mean_token_accuracy": 0.6132040470838547, + "num_tokens": 54593260.0, + "step": 420 + }, + { + "entropy": 2.018359375, + "epoch": 0.0711743772241993, + "grad_norm": 4.222598171500471, + "learning_rate": 9.578498293515359e-06, + "loss": 1.6084, + "mean_token_accuracy": 0.6129140526056289, + "num_tokens": 55894695.0, + "step": 430 + }, + { + "entropy": 2.02734375, + "epoch": 0.07282959529918066, + "grad_norm": 2.7537413700467632, + "learning_rate": 9.561433447098977e-06, + "loss": 1.5897, + "mean_token_accuracy": 0.6174709916114807, + "num_tokens": 57195830.0, + "step": 440 + }, + { + "entropy": 2.03203125, + "epoch": 0.07448481337416205, + "grad_norm": 3.286384678635548, + "learning_rate": 9.544368600682593e-06, + "loss": 1.6043, + "mean_token_accuracy": 0.6140577495098114, + "num_tokens": 58495768.0, + "step": 450 + }, + { + "entropy": 2.02734375, + "epoch": 0.07614003144914343, + "grad_norm": 6.122532488652377, + "learning_rate": 9.527303754266213e-06, + "loss": 1.5942, + "mean_token_accuracy": 0.6145596444606781, + "num_tokens": 59796738.0, + "step": 460 + }, + { + "entropy": 1.983203125, + "epoch": 0.07779524952412481, + "grad_norm": 3.2631344965934685, + "learning_rate": 9.51023890784983e-06, + "loss": 1.6184, + "mean_token_accuracy": 0.6117213189601898, + "num_tokens": 61096357.0, + "step": 470 + }, + { + "entropy": 2.005859375, + "epoch": 0.07945046759910618, + "grad_norm": 3.3859602387720873, + "learning_rate": 9.493174061433448e-06, + "loss": 1.5991, + "mean_token_accuracy": 0.614801648259163, + "num_tokens": 62397092.0, + "step": 480 + }, + { + "entropy": 2.011328125, + "epoch": 0.08110568567408756, + "grad_norm": 4.531403786278181, + "learning_rate": 9.476109215017066e-06, + "loss": 1.6046, + "mean_token_accuracy": 0.614807516336441, + "num_tokens": 63696766.0, + "step": 490 + }, + { + "entropy": 1.991796875, + "epoch": 0.08276090374906894, + "grad_norm": 3.1455204125515723, + "learning_rate": 9.459044368600684e-06, + "loss": 1.5864, + "mean_token_accuracy": 0.6172792553901673, + "num_tokens": 64997827.0, + "step": 500 + }, + { + "epoch": 0.08276090374906894, + "eval_entropy": 2.0066964285714284, + "eval_loss": 1.584367036819458, + "eval_mean_token_accuracy": 0.6167110630444118, + "eval_num_tokens": 64997827.0, + "eval_runtime": 4.6406, + "eval_samples_per_second": 94.383, + "eval_steps_per_second": 1.508, + "step": 500 + }, + { + "entropy": 1.977734375, + "epoch": 0.08441612182405032, + "grad_norm": 2.078094738308102, + "learning_rate": 9.441979522184302e-06, + "loss": 1.5861, + "mean_token_accuracy": 0.6169529289007187, + "num_tokens": 66299132.0, + "step": 510 + }, + { + "entropy": 2.007421875, + "epoch": 0.08607133989903169, + "grad_norm": 2.3391654857864856, + "learning_rate": 9.424914675767919e-06, + "loss": 1.5964, + "mean_token_accuracy": 0.6154071033000946, + "num_tokens": 67599646.0, + "step": 520 + }, + { + "entropy": 1.978125, + "epoch": 0.08772655797401308, + "grad_norm": 2.246405513615281, + "learning_rate": 9.407849829351537e-06, + "loss": 1.5772, + "mean_token_accuracy": 0.6183598071336747, + "num_tokens": 68898593.0, + "step": 530 + }, + { + "entropy": 1.990234375, + "epoch": 0.08938177604899446, + "grad_norm": 2.3053946540878294, + "learning_rate": 9.390784982935153e-06, + "loss": 1.578, + "mean_token_accuracy": 0.6187983095645905, + "num_tokens": 70197647.0, + "step": 540 + }, + { + "entropy": 1.993359375, + "epoch": 0.09103699412397584, + "grad_norm": 2.342491598490211, + "learning_rate": 9.373720136518773e-06, + "loss": 1.5853, + "mean_token_accuracy": 0.6173911958932876, + "num_tokens": 71498976.0, + "step": 550 + }, + { + "entropy": 2.00078125, + "epoch": 0.09269221219895721, + "grad_norm": 2.0322193427326765, + "learning_rate": 9.35665529010239e-06, + "loss": 1.5945, + "mean_token_accuracy": 0.616578283905983, + "num_tokens": 72796439.0, + "step": 560 + }, + { + "entropy": 2.0140625, + "epoch": 0.09434743027393859, + "grad_norm": 2.17266272324696, + "learning_rate": 9.339590443686007e-06, + "loss": 1.6053, + "mean_token_accuracy": 0.6140541821718216, + "num_tokens": 74095125.0, + "step": 570 + }, + { + "entropy": 2.0015625, + "epoch": 0.09600264834891997, + "grad_norm": 2.390187727034316, + "learning_rate": 9.322525597269626e-06, + "loss": 1.5759, + "mean_token_accuracy": 0.6184700220823288, + "num_tokens": 75394728.0, + "step": 580 + }, + { + "entropy": 1.998828125, + "epoch": 0.09765786642390135, + "grad_norm": 2.1153523306826094, + "learning_rate": 9.305460750853244e-06, + "loss": 1.5952, + "mean_token_accuracy": 0.6168525487184524, + "num_tokens": 76694969.0, + "step": 590 + }, + { + "entropy": 2.019140625, + "epoch": 0.09931308449888272, + "grad_norm": 2.2794751721789503, + "learning_rate": 9.288395904436862e-06, + "loss": 1.5883, + "mean_token_accuracy": 0.6159148007631302, + "num_tokens": 77996556.0, + "step": 600 + }, + { + "epoch": 0.09931308449888272, + "eval_entropy": 1.9553571428571428, + "eval_loss": 1.581153392791748, + "eval_mean_token_accuracy": 0.6182724918637957, + "eval_num_tokens": 77996556.0, + "eval_runtime": 4.6312, + "eval_samples_per_second": 94.577, + "eval_steps_per_second": 1.511, + "step": 600 + }, + { + "entropy": 1.973828125, + "epoch": 0.1009683025738641, + "grad_norm": 1.9794041945635394, + "learning_rate": 9.271331058020478e-06, + "loss": 1.5721, + "mean_token_accuracy": 0.6199139326810836, + "num_tokens": 79296913.0, + "step": 610 + }, + { + "entropy": 1.980078125, + "epoch": 0.10262352064884549, + "grad_norm": 2.211734909620868, + "learning_rate": 9.254266211604096e-06, + "loss": 1.5727, + "mean_token_accuracy": 0.6204076260328293, + "num_tokens": 80596724.0, + "step": 620 + }, + { + "entropy": 2.0015625, + "epoch": 0.10427873872382687, + "grad_norm": 2.061081714001349, + "learning_rate": 9.237201365187714e-06, + "loss": 1.5842, + "mean_token_accuracy": 0.6178865849971771, + "num_tokens": 81895704.0, + "step": 630 + }, + { + "entropy": 2.00625, + "epoch": 0.10593395679880824, + "grad_norm": 1.9421039503141664, + "learning_rate": 9.220136518771332e-06, + "loss": 1.5952, + "mean_token_accuracy": 0.615712583065033, + "num_tokens": 83195494.0, + "step": 640 + }, + { + "entropy": 1.989453125, + "epoch": 0.10758917487378962, + "grad_norm": 1.930462150502416, + "learning_rate": 9.203071672354949e-06, + "loss": 1.5951, + "mean_token_accuracy": 0.6163991808891296, + "num_tokens": 84496527.0, + "step": 650 + }, + { + "entropy": 1.97734375, + "epoch": 0.109244392948771, + "grad_norm": 2.2922583967330197, + "learning_rate": 9.186006825938567e-06, + "loss": 1.5748, + "mean_token_accuracy": 0.6203701078891755, + "num_tokens": 85798269.0, + "step": 660 + }, + { + "entropy": 1.96875, + "epoch": 0.11089961102375238, + "grad_norm": 2.160894825938974, + "learning_rate": 9.168941979522185e-06, + "loss": 1.5625, + "mean_token_accuracy": 0.6241954833269119, + "num_tokens": 87097897.0, + "step": 670 + }, + { + "entropy": 1.965234375, + "epoch": 0.11255482909873375, + "grad_norm": 2.1199065621355877, + "learning_rate": 9.151877133105803e-06, + "loss": 1.5493, + "mean_token_accuracy": 0.6226504802703857, + "num_tokens": 88398620.0, + "step": 680 + }, + { + "entropy": 1.978125, + "epoch": 0.11421004717371513, + "grad_norm": 2.1990569028946587, + "learning_rate": 9.134812286689421e-06, + "loss": 1.5634, + "mean_token_accuracy": 0.6204953461885452, + "num_tokens": 89699984.0, + "step": 690 + }, + { + "entropy": 1.98671875, + "epoch": 0.11586526524869652, + "grad_norm": 2.238829254355838, + "learning_rate": 9.117747440273038e-06, + "loss": 1.5552, + "mean_token_accuracy": 0.6211448013782501, + "num_tokens": 90998678.0, + "step": 700 + }, + { + "epoch": 0.11586526524869652, + "eval_entropy": 1.9888392857142858, + "eval_loss": 1.5724875926971436, + "eval_mean_token_accuracy": 0.6198033945901054, + "eval_num_tokens": 90998678.0, + "eval_runtime": 4.6551, + "eval_samples_per_second": 94.09, + "eval_steps_per_second": 1.504, + "step": 700 + }, + { + "entropy": 1.981640625, + "epoch": 0.1175204833236779, + "grad_norm": 2.073060235113114, + "learning_rate": 9.100682593856656e-06, + "loss": 1.5926, + "mean_token_accuracy": 0.616797536611557, + "num_tokens": 92299279.0, + "step": 710 + }, + { + "entropy": 1.98828125, + "epoch": 0.11917570139865927, + "grad_norm": 1.9098403846011123, + "learning_rate": 9.083617747440274e-06, + "loss": 1.5714, + "mean_token_accuracy": 0.6205732762813568, + "num_tokens": 93599292.0, + "step": 720 + }, + { + "entropy": 1.96640625, + "epoch": 0.12083091947364065, + "grad_norm": 2.2465228353868545, + "learning_rate": 9.066552901023892e-06, + "loss": 1.5439, + "mean_token_accuracy": 0.6258313655853271, + "num_tokens": 94901631.0, + "step": 730 + }, + { + "entropy": 1.9953125, + "epoch": 0.12248613754862203, + "grad_norm": 2.0984702023499304, + "learning_rate": 9.049488054607509e-06, + "loss": 1.5867, + "mean_token_accuracy": 0.6169804632663727, + "num_tokens": 96203478.0, + "step": 740 + }, + { + "entropy": 1.98125, + "epoch": 0.12414135562360341, + "grad_norm": 1.9423473559257842, + "learning_rate": 9.032423208191127e-06, + "loss": 1.5659, + "mean_token_accuracy": 0.6209818184375763, + "num_tokens": 97506582.0, + "step": 750 + }, + { + "entropy": 1.983984375, + "epoch": 0.1257965736985848, + "grad_norm": 2.2187749602547853, + "learning_rate": 9.015358361774745e-06, + "loss": 1.5912, + "mean_token_accuracy": 0.617563110589981, + "num_tokens": 98809299.0, + "step": 760 + }, + { + "entropy": 1.978515625, + "epoch": 0.12745179177356616, + "grad_norm": 2.0513524811301336, + "learning_rate": 8.998293515358363e-06, + "loss": 1.5736, + "mean_token_accuracy": 0.6196451514959336, + "num_tokens": 100108319.0, + "step": 770 + }, + { + "entropy": 1.9984375, + "epoch": 0.12910700984854753, + "grad_norm": 2.248716037890662, + "learning_rate": 8.981228668941981e-06, + "loss": 1.5724, + "mean_token_accuracy": 0.620639082789421, + "num_tokens": 101407429.0, + "step": 780 + }, + { + "entropy": 1.98046875, + "epoch": 0.13076222792352893, + "grad_norm": 2.398781077417778, + "learning_rate": 8.964163822525597e-06, + "loss": 1.571, + "mean_token_accuracy": 0.6187545627355575, + "num_tokens": 102706388.0, + "step": 790 + }, + { + "entropy": 1.973828125, + "epoch": 0.1324174459985103, + "grad_norm": 2.0189555520085363, + "learning_rate": 8.947098976109216e-06, + "loss": 1.572, + "mean_token_accuracy": 0.6211838543415069, + "num_tokens": 104004058.0, + "step": 800 + }, + { + "epoch": 0.1324174459985103, + "eval_entropy": 1.9832589285714286, + "eval_loss": 1.5580196380615234, + "eval_mean_token_accuracy": 0.6228588308606829, + "eval_num_tokens": 104004058.0, + "eval_runtime": 4.6425, + "eval_samples_per_second": 94.346, + "eval_steps_per_second": 1.508, + "step": 800 + }, + { + "entropy": 1.966796875, + "epoch": 0.1340726640734917, + "grad_norm": 2.0506987682860376, + "learning_rate": 8.930034129692834e-06, + "loss": 1.5527, + "mean_token_accuracy": 0.6225547701120376, + "num_tokens": 105304920.0, + "step": 810 + }, + { + "entropy": 1.996875, + "epoch": 0.13572788214847306, + "grad_norm": 2.157982625720176, + "learning_rate": 8.912969283276452e-06, + "loss": 1.5723, + "mean_token_accuracy": 0.6204317569732666, + "num_tokens": 106605907.0, + "step": 820 + }, + { + "entropy": 1.987890625, + "epoch": 0.13738310022345443, + "grad_norm": 2.0902514615176506, + "learning_rate": 8.895904436860068e-06, + "loss": 1.5703, + "mean_token_accuracy": 0.6199889063835144, + "num_tokens": 107908266.0, + "step": 830 + }, + { + "entropy": 1.974609375, + "epoch": 0.13903831829843583, + "grad_norm": 2.154961007344283, + "learning_rate": 8.878839590443686e-06, + "loss": 1.5489, + "mean_token_accuracy": 0.625212162733078, + "num_tokens": 109207551.0, + "step": 840 + }, + { + "entropy": 1.969140625, + "epoch": 0.1406935363734172, + "grad_norm": 2.2279652507000924, + "learning_rate": 8.861774744027304e-06, + "loss": 1.5506, + "mean_token_accuracy": 0.6249184757471085, + "num_tokens": 110508779.0, + "step": 850 + }, + { + "entropy": 1.98828125, + "epoch": 0.1423487544483986, + "grad_norm": 1.9060559552751348, + "learning_rate": 8.844709897610923e-06, + "loss": 1.5542, + "mean_token_accuracy": 0.6239151000976563, + "num_tokens": 111810299.0, + "step": 860 + }, + { + "entropy": 1.99453125, + "epoch": 0.14400397252337996, + "grad_norm": 1.9651344284873995, + "learning_rate": 8.82764505119454e-06, + "loss": 1.5256, + "mean_token_accuracy": 0.6291730135679245, + "num_tokens": 113110002.0, + "step": 870 + }, + { + "entropy": 1.9859375, + "epoch": 0.14565919059836133, + "grad_norm": 2.101989061975989, + "learning_rate": 8.810580204778157e-06, + "loss": 1.5694, + "mean_token_accuracy": 0.6209084391593933, + "num_tokens": 114410367.0, + "step": 880 + }, + { + "entropy": 1.990234375, + "epoch": 0.14731440867334272, + "grad_norm": 2.0100285781628635, + "learning_rate": 8.793515358361775e-06, + "loss": 1.5618, + "mean_token_accuracy": 0.6230337709188462, + "num_tokens": 115711552.0, + "step": 890 + }, + { + "entropy": 1.965234375, + "epoch": 0.1489696267483241, + "grad_norm": 2.1874593699913665, + "learning_rate": 8.776450511945393e-06, + "loss": 1.5656, + "mean_token_accuracy": 0.6241429626941681, + "num_tokens": 117011032.0, + "step": 900 + }, + { + "epoch": 0.1489696267483241, + "eval_entropy": 1.9229910714285714, + "eval_loss": 1.550237774848938, + "eval_mean_token_accuracy": 0.6252460139138358, + "eval_num_tokens": 117011032.0, + "eval_runtime": 4.6452, + "eval_samples_per_second": 94.291, + "eval_steps_per_second": 1.507, + "step": 900 + }, + { + "entropy": 1.954296875, + "epoch": 0.15062484482330546, + "grad_norm": 1.9668359268351572, + "learning_rate": 8.759385665529011e-06, + "loss": 1.5527, + "mean_token_accuracy": 0.6238682866096497, + "num_tokens": 118310390.0, + "step": 910 + }, + { + "entropy": 1.96953125, + "epoch": 0.15228006289828686, + "grad_norm": 2.04070906302647, + "learning_rate": 8.742320819112628e-06, + "loss": 1.5517, + "mean_token_accuracy": 0.6239574968814849, + "num_tokens": 119609177.0, + "step": 920 + }, + { + "entropy": 1.968359375, + "epoch": 0.15393528097326822, + "grad_norm": 2.2741502190790928, + "learning_rate": 8.725255972696246e-06, + "loss": 1.5482, + "mean_token_accuracy": 0.6262238144874572, + "num_tokens": 120906486.0, + "step": 930 + }, + { + "entropy": 1.96953125, + "epoch": 0.15559049904824962, + "grad_norm": 2.0892048996914596, + "learning_rate": 8.708191126279864e-06, + "loss": 1.5696, + "mean_token_accuracy": 0.6205377161502839, + "num_tokens": 122210464.0, + "step": 940 + }, + { + "entropy": 1.98046875, + "epoch": 0.157245717123231, + "grad_norm": 2.2787304021886934, + "learning_rate": 8.691126279863482e-06, + "loss": 1.5422, + "mean_token_accuracy": 0.6255671918392182, + "num_tokens": 123509402.0, + "step": 950 + }, + { + "entropy": 1.980859375, + "epoch": 0.15890093519821236, + "grad_norm": 2.105278371649376, + "learning_rate": 8.6740614334471e-06, + "loss": 1.5642, + "mean_token_accuracy": 0.6216717332601547, + "num_tokens": 124809924.0, + "step": 960 + }, + { + "entropy": 1.98359375, + "epoch": 0.16055615327319375, + "grad_norm": 2.0214334492181836, + "learning_rate": 8.656996587030717e-06, + "loss": 1.5538, + "mean_token_accuracy": 0.6259096443653107, + "num_tokens": 126109990.0, + "step": 970 + }, + { + "entropy": 1.981640625, + "epoch": 0.16221137134817512, + "grad_norm": 2.144795674947024, + "learning_rate": 8.639931740614336e-06, + "loss": 1.5666, + "mean_token_accuracy": 0.6229725420475006, + "num_tokens": 127411580.0, + "step": 980 + }, + { + "entropy": 1.981640625, + "epoch": 0.1638665894231565, + "grad_norm": 2.198129198315, + "learning_rate": 8.622866894197953e-06, + "loss": 1.5626, + "mean_token_accuracy": 0.6234712779521943, + "num_tokens": 128711422.0, + "step": 990 + }, + { + "entropy": 1.998828125, + "epoch": 0.16552180749813789, + "grad_norm": 2.106597428642529, + "learning_rate": 8.605802047781571e-06, + "loss": 1.5558, + "mean_token_accuracy": 0.6245997697114944, + "num_tokens": 130012411.0, + "step": 1000 + }, + { + "epoch": 0.16552180749813789, + "eval_entropy": 1.9542410714285714, + "eval_loss": 1.5389752388000488, + "eval_mean_token_accuracy": 0.6269884279796055, + "eval_num_tokens": 130012411.0, + "eval_runtime": 4.6464, + "eval_samples_per_second": 94.267, + "eval_steps_per_second": 1.507, + "step": 1000 + }, + { + "entropy": 1.974609375, + "epoch": 0.16717702557311925, + "grad_norm": 2.1099158978625954, + "learning_rate": 8.588737201365187e-06, + "loss": 1.547, + "mean_token_accuracy": 0.6271468341350556, + "num_tokens": 131312829.0, + "step": 1010 + }, + { + "entropy": 1.963671875, + "epoch": 0.16883224364810065, + "grad_norm": 2.30092418229146, + "learning_rate": 8.571672354948806e-06, + "loss": 1.5683, + "mean_token_accuracy": 0.6208470731973648, + "num_tokens": 132613447.0, + "step": 1020 + }, + { + "entropy": 1.963671875, + "epoch": 0.17048746172308202, + "grad_norm": 2.04251409688052, + "learning_rate": 8.554607508532424e-06, + "loss": 1.5408, + "mean_token_accuracy": 0.6264098972082138, + "num_tokens": 133913001.0, + "step": 1030 + }, + { + "entropy": 1.9765625, + "epoch": 0.17214267979806339, + "grad_norm": 2.550552020558526, + "learning_rate": 8.537542662116042e-06, + "loss": 1.5608, + "mean_token_accuracy": 0.6236230045557022, + "num_tokens": 135215222.0, + "step": 1040 + }, + { + "entropy": 1.94375, + "epoch": 0.17379789787304478, + "grad_norm": 2.51702412129481, + "learning_rate": 8.52047781569966e-06, + "loss": 1.5478, + "mean_token_accuracy": 0.6257014125585556, + "num_tokens": 136514719.0, + "step": 1050 + }, + { + "entropy": 1.959765625, + "epoch": 0.17545311594802615, + "grad_norm": 2.0839502062086717, + "learning_rate": 8.503412969283276e-06, + "loss": 1.5603, + "mean_token_accuracy": 0.6241546899080277, + "num_tokens": 137813581.0, + "step": 1060 + }, + { + "entropy": 1.94296875, + "epoch": 0.17710833402300752, + "grad_norm": 2.150392460649509, + "learning_rate": 8.486348122866896e-06, + "loss": 1.5314, + "mean_token_accuracy": 0.6284324765205384, + "num_tokens": 139115846.0, + "step": 1070 + }, + { + "entropy": 1.94140625, + "epoch": 0.17876355209798891, + "grad_norm": 1.9510136377867053, + "learning_rate": 8.469283276450513e-06, + "loss": 1.5447, + "mean_token_accuracy": 0.6269305616617202, + "num_tokens": 140416410.0, + "step": 1080 + }, + { + "entropy": 1.947265625, + "epoch": 0.18041877017297028, + "grad_norm": 2.1332217934369826, + "learning_rate": 8.45221843003413e-06, + "loss": 1.5186, + "mean_token_accuracy": 0.6297720462083817, + "num_tokens": 141719469.0, + "step": 1090 + }, + { + "entropy": 1.93828125, + "epoch": 0.18207398824795168, + "grad_norm": 2.4091033277253815, + "learning_rate": 8.435153583617747e-06, + "loss": 1.5422, + "mean_token_accuracy": 0.6254990786314011, + "num_tokens": 143016170.0, + "step": 1100 + }, + { + "epoch": 0.18207398824795168, + "eval_entropy": 1.9229910714285714, + "eval_loss": 1.5259357690811157, + "eval_mean_token_accuracy": 0.6295573881694249, + "eval_num_tokens": 143016170.0, + "eval_runtime": 4.6401, + "eval_samples_per_second": 94.395, + "eval_steps_per_second": 1.509, + "step": 1100 + }, + { + "entropy": 1.93203125, + "epoch": 0.18372920632293305, + "grad_norm": 2.1426871398928036, + "learning_rate": 8.418088737201365e-06, + "loss": 1.5266, + "mean_token_accuracy": 0.629123741388321, + "num_tokens": 144316069.0, + "step": 1110 + }, + { + "entropy": 1.9171875, + "epoch": 0.18538442439791442, + "grad_norm": 2.0800009895303266, + "learning_rate": 8.401023890784983e-06, + "loss": 1.5338, + "mean_token_accuracy": 0.6276921421289444, + "num_tokens": 145614875.0, + "step": 1120 + }, + { + "entropy": 1.924609375, + "epoch": 0.1870396424728958, + "grad_norm": 2.2979092404913457, + "learning_rate": 8.383959044368601e-06, + "loss": 1.5317, + "mean_token_accuracy": 0.6292991876602173, + "num_tokens": 146915541.0, + "step": 1130 + }, + { + "entropy": 1.95546875, + "epoch": 0.18869486054787718, + "grad_norm": 2.1579099446526904, + "learning_rate": 8.36689419795222e-06, + "loss": 1.5358, + "mean_token_accuracy": 0.6276393115520478, + "num_tokens": 148217235.0, + "step": 1140 + }, + { + "entropy": 1.93203125, + "epoch": 0.19035007862285855, + "grad_norm": 2.060993397703842, + "learning_rate": 8.349829351535836e-06, + "loss": 1.5244, + "mean_token_accuracy": 0.6309717804193496, + "num_tokens": 149518676.0, + "step": 1150 + }, + { + "entropy": 1.951171875, + "epoch": 0.19200529669783994, + "grad_norm": 2.1270794678279885, + "learning_rate": 8.332764505119456e-06, + "loss": 1.5205, + "mean_token_accuracy": 0.6302749544382096, + "num_tokens": 150818333.0, + "step": 1160 + }, + { + "entropy": 1.934375, + "epoch": 0.1936605147728213, + "grad_norm": 2.3549151173634004, + "learning_rate": 8.315699658703072e-06, + "loss": 1.5307, + "mean_token_accuracy": 0.6308447629213333, + "num_tokens": 152120159.0, + "step": 1170 + }, + { + "entropy": 1.94765625, + "epoch": 0.1953157328478027, + "grad_norm": 2.126532225065307, + "learning_rate": 8.29863481228669e-06, + "loss": 1.5357, + "mean_token_accuracy": 0.6267372697591782, + "num_tokens": 153419793.0, + "step": 1180 + }, + { + "entropy": 1.93359375, + "epoch": 0.19697095092278408, + "grad_norm": 2.121571455310252, + "learning_rate": 8.281569965870307e-06, + "loss": 1.5186, + "mean_token_accuracy": 0.6330710738897324, + "num_tokens": 154721446.0, + "step": 1190 + }, + { + "entropy": 1.94296875, + "epoch": 0.19862616899776545, + "grad_norm": 2.3001215006447238, + "learning_rate": 8.264505119453927e-06, + "loss": 1.5034, + "mean_token_accuracy": 0.6341148495674134, + "num_tokens": 156023811.0, + "step": 1200 + }, + { + "epoch": 0.19862616899776545, + "eval_entropy": 1.9174107142857142, + "eval_loss": 1.5122358798980713, + "eval_mean_token_accuracy": 0.6320508207593646, + "eval_num_tokens": 156023811.0, + "eval_runtime": 4.6171, + "eval_samples_per_second": 94.865, + "eval_steps_per_second": 1.516, + "step": 1200 + }, + { + "entropy": 1.934765625, + "epoch": 0.20028138707274684, + "grad_norm": 2.1317337808220587, + "learning_rate": 8.247440273037543e-06, + "loss": 1.5322, + "mean_token_accuracy": 0.6303939014673233, + "num_tokens": 157324923.0, + "step": 1210 + }, + { + "entropy": 1.946875, + "epoch": 0.2019366051477282, + "grad_norm": 2.272698793752354, + "learning_rate": 8.230375426621161e-06, + "loss": 1.5066, + "mean_token_accuracy": 0.6336111813783646, + "num_tokens": 158627193.0, + "step": 1220 + }, + { + "entropy": 1.933984375, + "epoch": 0.2035918232227096, + "grad_norm": 2.1851554901493127, + "learning_rate": 8.213310580204779e-06, + "loss": 1.5108, + "mean_token_accuracy": 0.6326002836227417, + "num_tokens": 159928810.0, + "step": 1230 + }, + { + "entropy": 1.925390625, + "epoch": 0.20524704129769097, + "grad_norm": 2.336498917204974, + "learning_rate": 8.196245733788396e-06, + "loss": 1.5015, + "mean_token_accuracy": 0.6343601822853089, + "num_tokens": 161229071.0, + "step": 1240 + }, + { + "entropy": 1.9234375, + "epoch": 0.20690225937267234, + "grad_norm": 2.1632585313787955, + "learning_rate": 8.179180887372015e-06, + "loss": 1.5321, + "mean_token_accuracy": 0.6294375538825989, + "num_tokens": 162528734.0, + "step": 1250 + }, + { + "entropy": 1.9296875, + "epoch": 0.20855747744765374, + "grad_norm": 2.3029966922288305, + "learning_rate": 8.162116040955632e-06, + "loss": 1.4796, + "mean_token_accuracy": 0.6387410670518875, + "num_tokens": 163827482.0, + "step": 1260 + }, + { + "entropy": 1.940234375, + "epoch": 0.2102126955226351, + "grad_norm": 2.072139203010286, + "learning_rate": 8.14505119453925e-06, + "loss": 1.5329, + "mean_token_accuracy": 0.6289373248815536, + "num_tokens": 165129752.0, + "step": 1270 + }, + { + "entropy": 1.9296875, + "epoch": 0.21186791359761648, + "grad_norm": 2.1467092517985438, + "learning_rate": 8.127986348122866e-06, + "loss": 1.5188, + "mean_token_accuracy": 0.6312744200229645, + "num_tokens": 166430829.0, + "step": 1280 + }, + { + "entropy": 1.91796875, + "epoch": 0.21352313167259787, + "grad_norm": 2.11090618549315, + "learning_rate": 8.110921501706486e-06, + "loss": 1.5085, + "mean_token_accuracy": 0.6340959340333938, + "num_tokens": 167734127.0, + "step": 1290 + }, + { + "entropy": 1.93125, + "epoch": 0.21517834974757924, + "grad_norm": 2.202357538905635, + "learning_rate": 8.093856655290103e-06, + "loss": 1.5265, + "mean_token_accuracy": 0.6316495805978775, + "num_tokens": 169034230.0, + "step": 1300 + }, + { + "epoch": 0.21517834974757924, + "eval_entropy": 1.9140625, + "eval_loss": 1.5022934675216675, + "eval_mean_token_accuracy": 0.6348826459475926, + "eval_num_tokens": 169034230.0, + "eval_runtime": 4.644, + "eval_samples_per_second": 94.316, + "eval_steps_per_second": 1.507, + "step": 1300 + }, + { + "entropy": 1.948046875, + "epoch": 0.21683356782256064, + "grad_norm": 2.223098363714227, + "learning_rate": 8.07679180887372e-06, + "loss": 1.5366, + "mean_token_accuracy": 0.6281047582626342, + "num_tokens": 170328412.0, + "step": 1310 + }, + { + "entropy": 1.93203125, + "epoch": 0.218488785897542, + "grad_norm": 2.406626748685825, + "learning_rate": 8.059726962457339e-06, + "loss": 1.5163, + "mean_token_accuracy": 0.6307426512241363, + "num_tokens": 171628469.0, + "step": 1320 + }, + { + "entropy": 1.937890625, + "epoch": 0.22014400397252337, + "grad_norm": 2.264447382177499, + "learning_rate": 8.042662116040957e-06, + "loss": 1.5096, + "mean_token_accuracy": 0.6353007078170776, + "num_tokens": 172923988.0, + "step": 1330 + }, + { + "entropy": 1.95234375, + "epoch": 0.22179922204750477, + "grad_norm": 2.131698726410324, + "learning_rate": 8.025597269624575e-06, + "loss": 1.5244, + "mean_token_accuracy": 0.6311636984348297, + "num_tokens": 174222835.0, + "step": 1340 + }, + { + "entropy": 1.93203125, + "epoch": 0.22345444012248614, + "grad_norm": 2.2465687406467576, + "learning_rate": 8.008532423208191e-06, + "loss": 1.4953, + "mean_token_accuracy": 0.6376566916704178, + "num_tokens": 175524279.0, + "step": 1350 + }, + { + "entropy": 1.94140625, + "epoch": 0.2251096581974675, + "grad_norm": 2.08355016893503, + "learning_rate": 7.99146757679181e-06, + "loss": 1.5199, + "mean_token_accuracy": 0.63325836956501, + "num_tokens": 176825510.0, + "step": 1360 + }, + { + "entropy": 1.934765625, + "epoch": 0.2267648762724489, + "grad_norm": 2.2172569535338575, + "learning_rate": 7.974402730375426e-06, + "loss": 1.523, + "mean_token_accuracy": 0.6316302210092545, + "num_tokens": 178125788.0, + "step": 1370 + }, + { + "entropy": 1.951171875, + "epoch": 0.22842009434743027, + "grad_norm": 1.8915268978713486, + "learning_rate": 7.957337883959046e-06, + "loss": 1.5228, + "mean_token_accuracy": 0.6299791485071182, + "num_tokens": 179423260.0, + "step": 1380 + }, + { + "entropy": 1.93984375, + "epoch": 0.23007531242241167, + "grad_norm": 2.076578629946283, + "learning_rate": 7.940273037542662e-06, + "loss": 1.511, + "mean_token_accuracy": 0.6327517807483674, + "num_tokens": 180723736.0, + "step": 1390 + }, + { + "entropy": 1.939453125, + "epoch": 0.23173053049739303, + "grad_norm": 2.156110498322818, + "learning_rate": 7.92320819112628e-06, + "loss": 1.5013, + "mean_token_accuracy": 0.6359281450510025, + "num_tokens": 182020934.0, + "step": 1400 + }, + { + "epoch": 0.23173053049739303, + "eval_entropy": 1.9497767857142858, + "eval_loss": 1.4890518188476562, + "eval_mean_token_accuracy": 0.6373054044587272, + "eval_num_tokens": 182020934.0, + "eval_runtime": 4.633, + "eval_samples_per_second": 94.539, + "eval_steps_per_second": 1.511, + "step": 1400 + }, + { + "entropy": 1.946484375, + "epoch": 0.2333857485723744, + "grad_norm": 1.991711629516408, + "learning_rate": 7.906143344709898e-06, + "loss": 1.5403, + "mean_token_accuracy": 0.6275832712650299, + "num_tokens": 183323778.0, + "step": 1410 + }, + { + "entropy": 1.948046875, + "epoch": 0.2350409666473558, + "grad_norm": 2.080565403386363, + "learning_rate": 7.889078498293517e-06, + "loss": 1.5127, + "mean_token_accuracy": 0.6323427587747574, + "num_tokens": 184624703.0, + "step": 1420 + }, + { + "entropy": 1.951171875, + "epoch": 0.23669618472233717, + "grad_norm": 1.9048275532334538, + "learning_rate": 7.872013651877135e-06, + "loss": 1.5122, + "mean_token_accuracy": 0.6334949612617493, + "num_tokens": 185923416.0, + "step": 1430 + }, + { + "entropy": 1.948046875, + "epoch": 0.23835140279731853, + "grad_norm": 2.215698358798685, + "learning_rate": 7.854948805460751e-06, + "loss": 1.5157, + "mean_token_accuracy": 0.6317879527807235, + "num_tokens": 187224302.0, + "step": 1440 + }, + { + "entropy": 1.95546875, + "epoch": 0.24000662087229993, + "grad_norm": 2.1518687376165757, + "learning_rate": 7.83788395904437e-06, + "loss": 1.5287, + "mean_token_accuracy": 0.6296833544969559, + "num_tokens": 188523983.0, + "step": 1450 + }, + { + "entropy": 1.9265625, + "epoch": 0.2416618389472813, + "grad_norm": 2.2250684820278486, + "learning_rate": 7.820819112627987e-06, + "loss": 1.4995, + "mean_token_accuracy": 0.635177955031395, + "num_tokens": 189823177.0, + "step": 1460 + }, + { + "entropy": 1.934765625, + "epoch": 0.2433170570222627, + "grad_norm": 2.0361968646565565, + "learning_rate": 7.803754266211605e-06, + "loss": 1.4966, + "mean_token_accuracy": 0.6359856784343719, + "num_tokens": 191122783.0, + "step": 1470 + }, + { + "entropy": 1.942578125, + "epoch": 0.24497227509724406, + "grad_norm": 2.0588272551937132, + "learning_rate": 7.786689419795222e-06, + "loss": 1.5185, + "mean_token_accuracy": 0.6343974649906159, + "num_tokens": 192423479.0, + "step": 1480 + }, + { + "entropy": 1.94921875, + "epoch": 0.24662749317222543, + "grad_norm": 2.2557471781456027, + "learning_rate": 7.76962457337884e-06, + "loss": 1.5109, + "mean_token_accuracy": 0.6324847370386124, + "num_tokens": 193722897.0, + "step": 1490 + }, + { + "entropy": 1.9125, + "epoch": 0.24828271124720683, + "grad_norm": 2.098594664432755, + "learning_rate": 7.752559726962458e-06, + "loss": 1.4998, + "mean_token_accuracy": 0.6359317243099213, + "num_tokens": 195022446.0, + "step": 1500 + }, + { + "epoch": 0.24828271124720683, + "eval_entropy": 1.875, + "eval_loss": 1.480890154838562, + "eval_mean_token_accuracy": 0.6385358657155719, + "eval_num_tokens": 195022446.0, + "eval_runtime": 4.6339, + "eval_samples_per_second": 94.521, + "eval_steps_per_second": 1.511, + "step": 1500 + }, + { + "entropy": 1.934375, + "epoch": 0.2499379293221882, + "grad_norm": 2.276831302907779, + "learning_rate": 7.735494880546076e-06, + "loss": 1.5067, + "mean_token_accuracy": 0.6327724367380142, + "num_tokens": 196323117.0, + "step": 1510 + }, + { + "entropy": 1.9109375, + "epoch": 0.2515931473971696, + "grad_norm": 2.0002167462003926, + "learning_rate": 7.718430034129694e-06, + "loss": 1.4879, + "mean_token_accuracy": 0.6374749451875686, + "num_tokens": 197623989.0, + "step": 1520 + }, + { + "entropy": 1.930859375, + "epoch": 0.25324836547215096, + "grad_norm": 2.2667644829850095, + "learning_rate": 7.70136518771331e-06, + "loss": 1.5359, + "mean_token_accuracy": 0.6296163409948349, + "num_tokens": 198924109.0, + "step": 1530 + }, + { + "entropy": 1.933203125, + "epoch": 0.25490358354713233, + "grad_norm": 2.1407108098465537, + "learning_rate": 7.684300341296929e-06, + "loss": 1.497, + "mean_token_accuracy": 0.6348809570074081, + "num_tokens": 200222323.0, + "step": 1540 + }, + { + "entropy": 1.9234375, + "epoch": 0.2565588016221137, + "grad_norm": 2.1285084037176474, + "learning_rate": 7.667235494880547e-06, + "loss": 1.4931, + "mean_token_accuracy": 0.6377875149250031, + "num_tokens": 201521701.0, + "step": 1550 + }, + { + "entropy": 1.918359375, + "epoch": 0.25821401969709507, + "grad_norm": 2.131821319625504, + "learning_rate": 7.650170648464165e-06, + "loss": 1.4954, + "mean_token_accuracy": 0.634219291806221, + "num_tokens": 202820630.0, + "step": 1560 + }, + { + "entropy": 1.918359375, + "epoch": 0.2598692377720765, + "grad_norm": 2.1349187242565084, + "learning_rate": 7.633105802047781e-06, + "loss": 1.5102, + "mean_token_accuracy": 0.6340892046689988, + "num_tokens": 204118301.0, + "step": 1570 + }, + { + "entropy": 1.912109375, + "epoch": 0.26152445584705786, + "grad_norm": 2.0953555265882695, + "learning_rate": 7.6160409556313996e-06, + "loss": 1.4942, + "mean_token_accuracy": 0.6379499018192292, + "num_tokens": 205417718.0, + "step": 1580 + }, + { + "entropy": 1.9046875, + "epoch": 0.2631796739220392, + "grad_norm": 2.3021858633248837, + "learning_rate": 7.598976109215017e-06, + "loss": 1.481, + "mean_token_accuracy": 0.6410841941833496, + "num_tokens": 206715970.0, + "step": 1590 + }, + { + "entropy": 1.91953125, + "epoch": 0.2648348919970206, + "grad_norm": 2.275827892190262, + "learning_rate": 7.581911262798636e-06, + "loss": 1.4965, + "mean_token_accuracy": 0.6389019787311554, + "num_tokens": 208016399.0, + "step": 1600 + }, + { + "epoch": 0.2648348919970206, + "eval_entropy": 1.875, + "eval_loss": 1.4675310850143433, + "eval_mean_token_accuracy": 0.6400606036186218, + "eval_num_tokens": 208016399.0, + "eval_runtime": 4.6376, + "eval_samples_per_second": 94.445, + "eval_steps_per_second": 1.509, + "step": 1600 + }, + { + "entropy": 1.905859375, + "epoch": 0.26649011007200196, + "grad_norm": 2.3387963777055463, + "learning_rate": 7.564846416382253e-06, + "loss": 1.4974, + "mean_token_accuracy": 0.6380386829376221, + "num_tokens": 209318545.0, + "step": 1610 + }, + { + "entropy": 1.90546875, + "epoch": 0.2681453281469834, + "grad_norm": 2.104858629082659, + "learning_rate": 7.547781569965871e-06, + "loss": 1.5005, + "mean_token_accuracy": 0.6362629383802414, + "num_tokens": 210618457.0, + "step": 1620 + }, + { + "entropy": 1.915625, + "epoch": 0.26980054622196475, + "grad_norm": 2.0353564179974204, + "learning_rate": 7.5307167235494884e-06, + "loss": 1.4704, + "mean_token_accuracy": 0.6419424384832382, + "num_tokens": 211919399.0, + "step": 1630 + }, + { + "entropy": 1.9015625, + "epoch": 0.2714557642969461, + "grad_norm": 2.454943415348521, + "learning_rate": 7.5136518771331065e-06, + "loss": 1.4508, + "mean_token_accuracy": 0.6464002221822739, + "num_tokens": 213216050.0, + "step": 1640 + }, + { + "entropy": 1.9109375, + "epoch": 0.2731109823719275, + "grad_norm": 2.0538581744748954, + "learning_rate": 7.496587030716725e-06, + "loss": 1.4915, + "mean_token_accuracy": 0.6380661070346832, + "num_tokens": 214518701.0, + "step": 1650 + }, + { + "entropy": 1.8921875, + "epoch": 0.27476620044690886, + "grad_norm": 2.148141331395261, + "learning_rate": 7.479522184300342e-06, + "loss": 1.4847, + "mean_token_accuracy": 0.6384107679128647, + "num_tokens": 215820203.0, + "step": 1660 + }, + { + "entropy": 1.9046875, + "epoch": 0.2764214185218903, + "grad_norm": 1.9823959043320287, + "learning_rate": 7.462457337883959e-06, + "loss": 1.4831, + "mean_token_accuracy": 0.6406153351068496, + "num_tokens": 217122786.0, + "step": 1670 + }, + { + "entropy": 1.919140625, + "epoch": 0.27807663659687165, + "grad_norm": 2.1217661278721263, + "learning_rate": 7.445392491467578e-06, + "loss": 1.4948, + "mean_token_accuracy": 0.6383452385663986, + "num_tokens": 218421034.0, + "step": 1680 + }, + { + "entropy": 1.913671875, + "epoch": 0.279731854671853, + "grad_norm": 2.2102486099557805, + "learning_rate": 7.428327645051195e-06, + "loss": 1.4964, + "mean_token_accuracy": 0.638947656750679, + "num_tokens": 219722145.0, + "step": 1690 + }, + { + "entropy": 1.905078125, + "epoch": 0.2813870727468344, + "grad_norm": 2.125824730715212, + "learning_rate": 7.411262798634813e-06, + "loss": 1.48, + "mean_token_accuracy": 0.6397389680147171, + "num_tokens": 221024561.0, + "step": 1700 + }, + { + "epoch": 0.2813870727468344, + "eval_entropy": 1.9006696428571428, + "eval_loss": 1.4548004865646362, + "eval_mean_token_accuracy": 0.6436779413904462, + "eval_num_tokens": 221024561.0, + "eval_runtime": 4.6257, + "eval_samples_per_second": 94.689, + "eval_steps_per_second": 1.513, + "step": 1700 + }, + { + "entropy": 1.905859375, + "epoch": 0.28304229082181576, + "grad_norm": 2.235022409845969, + "learning_rate": 7.394197952218431e-06, + "loss": 1.4872, + "mean_token_accuracy": 0.6388321906328202, + "num_tokens": 222327272.0, + "step": 1710 + }, + { + "entropy": 1.913671875, + "epoch": 0.2846975088967972, + "grad_norm": 2.1407701696574457, + "learning_rate": 7.377133105802048e-06, + "loss": 1.5, + "mean_token_accuracy": 0.637863889336586, + "num_tokens": 223629022.0, + "step": 1720 + }, + { + "entropy": 1.894921875, + "epoch": 0.28635272697177855, + "grad_norm": 2.2589719066229534, + "learning_rate": 7.360068259385666e-06, + "loss": 1.4695, + "mean_token_accuracy": 0.6447693794965744, + "num_tokens": 224929484.0, + "step": 1730 + }, + { + "entropy": 1.901171875, + "epoch": 0.2880079450467599, + "grad_norm": 2.0744835674215514, + "learning_rate": 7.343003412969284e-06, + "loss": 1.4692, + "mean_token_accuracy": 0.6418043613433838, + "num_tokens": 226228876.0, + "step": 1740 + }, + { + "entropy": 1.908984375, + "epoch": 0.2896631631217413, + "grad_norm": 2.0648780265220767, + "learning_rate": 7.3259385665529016e-06, + "loss": 1.4771, + "mean_token_accuracy": 0.6420573741197586, + "num_tokens": 227528347.0, + "step": 1750 + }, + { + "entropy": 1.89609375, + "epoch": 0.29131838119672265, + "grad_norm": 2.214842874232076, + "learning_rate": 7.308873720136519e-06, + "loss": 1.4284, + "mean_token_accuracy": 0.6503993839025497, + "num_tokens": 228828879.0, + "step": 1760 + }, + { + "entropy": 1.897265625, + "epoch": 0.292973599271704, + "grad_norm": 2.2271958645972494, + "learning_rate": 7.291808873720138e-06, + "loss": 1.4845, + "mean_token_accuracy": 0.6392062962055206, + "num_tokens": 230124374.0, + "step": 1770 + }, + { + "entropy": 1.891796875, + "epoch": 0.29462881734668545, + "grad_norm": 2.1018380414762547, + "learning_rate": 7.274744027303755e-06, + "loss": 1.4846, + "mean_token_accuracy": 0.6391264945268631, + "num_tokens": 231426414.0, + "step": 1780 + }, + { + "entropy": 1.882421875, + "epoch": 0.2962840354216668, + "grad_norm": 2.2058772817664574, + "learning_rate": 7.257679180887372e-06, + "loss": 1.4752, + "mean_token_accuracy": 0.641876009106636, + "num_tokens": 232728159.0, + "step": 1790 + }, + { + "entropy": 1.901953125, + "epoch": 0.2979392534966482, + "grad_norm": 2.140878217791849, + "learning_rate": 7.2406143344709904e-06, + "loss": 1.4824, + "mean_token_accuracy": 0.6405533373355865, + "num_tokens": 234029081.0, + "step": 1800 + }, + { + "epoch": 0.2979392534966482, + "eval_entropy": 1.890625, + "eval_loss": 1.439766526222229, + "eval_mean_token_accuracy": 0.6466241904667446, + "eval_num_tokens": 234029081.0, + "eval_runtime": 4.6427, + "eval_samples_per_second": 94.341, + "eval_steps_per_second": 1.508, + "step": 1800 + }, + { + "entropy": 1.900390625, + "epoch": 0.29959447157162955, + "grad_norm": 2.2708541065828034, + "learning_rate": 7.2235494880546085e-06, + "loss": 1.492, + "mean_token_accuracy": 0.6375244587659836, + "num_tokens": 235331821.0, + "step": 1810 + }, + { + "entropy": 1.895703125, + "epoch": 0.3012496896466109, + "grad_norm": 2.19740206337131, + "learning_rate": 7.206484641638226e-06, + "loss": 1.4817, + "mean_token_accuracy": 0.6404325038194656, + "num_tokens": 236632090.0, + "step": 1820 + }, + { + "entropy": 1.88515625, + "epoch": 0.30290490772159234, + "grad_norm": 2.258861588571928, + "learning_rate": 7.189419795221844e-06, + "loss": 1.4567, + "mean_token_accuracy": 0.6461021602153778, + "num_tokens": 237931501.0, + "step": 1830 + }, + { + "entropy": 1.892578125, + "epoch": 0.3045601257965737, + "grad_norm": 2.270597159397126, + "learning_rate": 7.172354948805461e-06, + "loss": 1.4737, + "mean_token_accuracy": 0.6417543858289718, + "num_tokens": 239232276.0, + "step": 1840 + }, + { + "entropy": 1.874609375, + "epoch": 0.3062153438715551, + "grad_norm": 2.2183409181832676, + "learning_rate": 7.1552901023890785e-06, + "loss": 1.4512, + "mean_token_accuracy": 0.6465785950422287, + "num_tokens": 240531922.0, + "step": 1850 + }, + { + "entropy": 1.896484375, + "epoch": 0.30787056194653645, + "grad_norm": 2.202504310058667, + "learning_rate": 7.138225255972697e-06, + "loss": 1.4566, + "mean_token_accuracy": 0.646238136291504, + "num_tokens": 241833915.0, + "step": 1860 + }, + { + "entropy": 1.902734375, + "epoch": 0.3095257800215178, + "grad_norm": 2.097835890842409, + "learning_rate": 7.121160409556315e-06, + "loss": 1.4622, + "mean_token_accuracy": 0.6443604111671448, + "num_tokens": 243132840.0, + "step": 1870 + }, + { + "entropy": 1.89296875, + "epoch": 0.31118099809649924, + "grad_norm": 2.3378637496678447, + "learning_rate": 7.104095563139932e-06, + "loss": 1.4862, + "mean_token_accuracy": 0.6414578646421433, + "num_tokens": 244433423.0, + "step": 1880 + }, + { + "entropy": 1.87578125, + "epoch": 0.3128362161714806, + "grad_norm": 2.2969652168765125, + "learning_rate": 7.08703071672355e-06, + "loss": 1.4508, + "mean_token_accuracy": 0.6470060855150223, + "num_tokens": 245736210.0, + "step": 1890 + }, + { + "entropy": 1.891015625, + "epoch": 0.314491434246462, + "grad_norm": 2.127378320600538, + "learning_rate": 7.069965870307168e-06, + "loss": 1.465, + "mean_token_accuracy": 0.6455838143825531, + "num_tokens": 247040196.0, + "step": 1900 + }, + { + "epoch": 0.314491434246462, + "eval_entropy": 1.8716517857142858, + "eval_loss": 1.436755895614624, + "eval_mean_token_accuracy": 0.6475338850702558, + "eval_num_tokens": 247040196.0, + "eval_runtime": 4.6391, + "eval_samples_per_second": 94.415, + "eval_steps_per_second": 1.509, + "step": 1900 + }, + { + "entropy": 1.890234375, + "epoch": 0.31614665232144334, + "grad_norm": 2.068121830654759, + "learning_rate": 7.0529010238907854e-06, + "loss": 1.4449, + "mean_token_accuracy": 0.6490348249673843, + "num_tokens": 248340540.0, + "step": 1910 + }, + { + "entropy": 1.88203125, + "epoch": 0.3178018703964247, + "grad_norm": 2.169311225136056, + "learning_rate": 7.0358361774744036e-06, + "loss": 1.4453, + "mean_token_accuracy": 0.6484747886657715, + "num_tokens": 249639011.0, + "step": 1920 + }, + { + "entropy": 1.891015625, + "epoch": 0.3194570884714061, + "grad_norm": 2.152603120914206, + "learning_rate": 7.018771331058021e-06, + "loss": 1.4581, + "mean_token_accuracy": 0.645383819937706, + "num_tokens": 250939297.0, + "step": 1930 + }, + { + "entropy": 1.89140625, + "epoch": 0.3211123065463875, + "grad_norm": 2.244866465707774, + "learning_rate": 7.001706484641638e-06, + "loss": 1.4722, + "mean_token_accuracy": 0.642143377661705, + "num_tokens": 252239558.0, + "step": 1940 + }, + { + "entropy": 1.885546875, + "epoch": 0.3227675246213689, + "grad_norm": 2.3051022135654016, + "learning_rate": 6.984641638225257e-06, + "loss": 1.4656, + "mean_token_accuracy": 0.6441871672868729, + "num_tokens": 253539412.0, + "step": 1950 + }, + { + "entropy": 1.88671875, + "epoch": 0.32442274269635024, + "grad_norm": 2.315557539795658, + "learning_rate": 6.967576791808874e-06, + "loss": 1.4348, + "mean_token_accuracy": 0.6483229964971542, + "num_tokens": 254837951.0, + "step": 1960 + }, + { + "entropy": 1.90390625, + "epoch": 0.3260779607713316, + "grad_norm": 2.278508142111934, + "learning_rate": 6.9505119453924916e-06, + "loss": 1.4808, + "mean_token_accuracy": 0.639566707611084, + "num_tokens": 256139930.0, + "step": 1970 + }, + { + "entropy": 1.888671875, + "epoch": 0.327733178846313, + "grad_norm": 2.1964883208570583, + "learning_rate": 6.93344709897611e-06, + "loss": 1.4701, + "mean_token_accuracy": 0.6428975909948349, + "num_tokens": 257440570.0, + "step": 1980 + }, + { + "entropy": 1.909375, + "epoch": 0.3293883969212944, + "grad_norm": 2.4036981596941542, + "learning_rate": 6.916382252559728e-06, + "loss": 1.4578, + "mean_token_accuracy": 0.6457471787929535, + "num_tokens": 258739759.0, + "step": 1990 + }, + { + "entropy": 1.880859375, + "epoch": 0.33104361499627577, + "grad_norm": 2.3869713509828894, + "learning_rate": 6.899317406143345e-06, + "loss": 1.4181, + "mean_token_accuracy": 0.6531212717294693, + "num_tokens": 260041853.0, + "step": 2000 + }, + { + "epoch": 0.33104361499627577, + "eval_entropy": 1.8705357142857142, + "eval_loss": 1.4256962537765503, + "eval_mean_token_accuracy": 0.6512183121272496, + "eval_num_tokens": 260041853.0, + "eval_runtime": 4.6577, + "eval_samples_per_second": 94.037, + "eval_steps_per_second": 1.503, + "step": 2000 + }, + { + "entropy": 1.8625, + "epoch": 0.33269883307125714, + "grad_norm": 2.7723264482575107, + "learning_rate": 6.882252559726963e-06, + "loss": 1.4411, + "mean_token_accuracy": 0.6497026890516281, + "num_tokens": 261339408.0, + "step": 2010 + }, + { + "entropy": 1.895703125, + "epoch": 0.3343540511462385, + "grad_norm": 2.3173472241233517, + "learning_rate": 6.8651877133105804e-06, + "loss": 1.4159, + "mean_token_accuracy": 0.6544720292091369, + "num_tokens": 262641005.0, + "step": 2020 + }, + { + "entropy": 1.88203125, + "epoch": 0.3360092692212199, + "grad_norm": 2.1128863553449575, + "learning_rate": 6.848122866894199e-06, + "loss": 1.4573, + "mean_token_accuracy": 0.6461355507373809, + "num_tokens": 263942521.0, + "step": 2030 + }, + { + "entropy": 1.86484375, + "epoch": 0.3376644872962013, + "grad_norm": 2.20072217323278, + "learning_rate": 6.831058020477817e-06, + "loss": 1.4421, + "mean_token_accuracy": 0.650199356675148, + "num_tokens": 265242352.0, + "step": 2040 + }, + { + "entropy": 1.90546875, + "epoch": 0.33931970537118267, + "grad_norm": 2.1084573287326687, + "learning_rate": 6.813993174061434e-06, + "loss": 1.4421, + "mean_token_accuracy": 0.6499897629022598, + "num_tokens": 266538520.0, + "step": 2050 + }, + { + "entropy": 1.882421875, + "epoch": 0.34097492344616404, + "grad_norm": 2.3201130683976254, + "learning_rate": 6.796928327645051e-06, + "loss": 1.4265, + "mean_token_accuracy": 0.6525385499000549, + "num_tokens": 267840356.0, + "step": 2060 + }, + { + "entropy": 1.88828125, + "epoch": 0.3426301415211454, + "grad_norm": 2.3997702452786815, + "learning_rate": 6.779863481228669e-06, + "loss": 1.4319, + "mean_token_accuracy": 0.6508309781551361, + "num_tokens": 269142518.0, + "step": 2070 + }, + { + "entropy": 1.880859375, + "epoch": 0.34428535959612677, + "grad_norm": 2.054869007594777, + "learning_rate": 6.7627986348122874e-06, + "loss": 1.4279, + "mean_token_accuracy": 0.6518710255622864, + "num_tokens": 270444946.0, + "step": 2080 + }, + { + "entropy": 1.876953125, + "epoch": 0.3459405776711082, + "grad_norm": 2.464982638214835, + "learning_rate": 6.745733788395905e-06, + "loss": 1.4219, + "mean_token_accuracy": 0.6548264652490616, + "num_tokens": 271740631.0, + "step": 2090 + }, + { + "entropy": 1.878515625, + "epoch": 0.34759579574608956, + "grad_norm": 2.36726320252569, + "learning_rate": 6.728668941979523e-06, + "loss": 1.4234, + "mean_token_accuracy": 0.6531784325838089, + "num_tokens": 273040080.0, + "step": 2100 + }, + { + "epoch": 0.34759579574608956, + "eval_entropy": 1.8705357142857142, + "eval_loss": 1.418318510055542, + "eval_mean_token_accuracy": 0.6528186542647225, + "eval_num_tokens": 273040080.0, + "eval_runtime": 4.6306, + "eval_samples_per_second": 94.588, + "eval_steps_per_second": 1.512, + "step": 2100 + }, + { + "entropy": 1.907421875, + "epoch": 0.34925101382107093, + "grad_norm": 2.080607856504551, + "learning_rate": 6.71160409556314e-06, + "loss": 1.4595, + "mean_token_accuracy": 0.6444210410118103, + "num_tokens": 274339463.0, + "step": 2110 + }, + { + "entropy": 1.87578125, + "epoch": 0.3509062318960523, + "grad_norm": 2.2732894162989083, + "learning_rate": 6.694539249146759e-06, + "loss": 1.4417, + "mean_token_accuracy": 0.6476331979036332, + "num_tokens": 275637060.0, + "step": 2120 + }, + { + "entropy": 1.8859375, + "epoch": 0.35256144997103367, + "grad_norm": 2.2109900353045906, + "learning_rate": 6.677474402730376e-06, + "loss": 1.4241, + "mean_token_accuracy": 0.6523648470640182, + "num_tokens": 276939398.0, + "step": 2130 + }, + { + "entropy": 1.885546875, + "epoch": 0.35421666804601504, + "grad_norm": 2.1737385103420372, + "learning_rate": 6.6604095563139936e-06, + "loss": 1.4451, + "mean_token_accuracy": 0.6472997695207596, + "num_tokens": 278241264.0, + "step": 2140 + }, + { + "entropy": 1.872265625, + "epoch": 0.35587188612099646, + "grad_norm": 2.432750325790286, + "learning_rate": 6.643344709897611e-06, + "loss": 1.3901, + "mean_token_accuracy": 0.6598243981599807, + "num_tokens": 279544118.0, + "step": 2150 + }, + { + "entropy": 1.883984375, + "epoch": 0.35752710419597783, + "grad_norm": 2.204172122905628, + "learning_rate": 6.62627986348123e-06, + "loss": 1.4321, + "mean_token_accuracy": 0.6505872458219528, + "num_tokens": 280845451.0, + "step": 2160 + }, + { + "entropy": 1.875, + "epoch": 0.3591823222709592, + "grad_norm": 2.336376172364065, + "learning_rate": 6.609215017064847e-06, + "loss": 1.4533, + "mean_token_accuracy": 0.6473734349012374, + "num_tokens": 282142256.0, + "step": 2170 + }, + { + "entropy": 1.87890625, + "epoch": 0.36083754034594057, + "grad_norm": 2.14990265922693, + "learning_rate": 6.592150170648464e-06, + "loss": 1.4096, + "mean_token_accuracy": 0.6571146130561829, + "num_tokens": 283444105.0, + "step": 2180 + }, + { + "entropy": 1.873046875, + "epoch": 0.36249275842092193, + "grad_norm": 2.429311765710193, + "learning_rate": 6.5750853242320824e-06, + "loss": 1.4329, + "mean_token_accuracy": 0.6502751678228378, + "num_tokens": 284744476.0, + "step": 2190 + }, + { + "entropy": 1.896875, + "epoch": 0.36414797649590336, + "grad_norm": 2.4148109850950283, + "learning_rate": 6.5580204778157e-06, + "loss": 1.4576, + "mean_token_accuracy": 0.6490479379892349, + "num_tokens": 286043749.0, + "step": 2200 + }, + { + "epoch": 0.36414797649590336, + "eval_entropy": 1.8772321428571428, + "eval_loss": 1.416571855545044, + "eval_mean_token_accuracy": 0.6539189304624285, + "eval_num_tokens": 286043749.0, + "eval_runtime": 4.6412, + "eval_samples_per_second": 94.371, + "eval_steps_per_second": 1.508, + "step": 2200 + }, + { + "entropy": 1.884375, + "epoch": 0.3658031945708847, + "grad_norm": 2.195653647079439, + "learning_rate": 6.540955631399319e-06, + "loss": 1.4444, + "mean_token_accuracy": 0.6483281642198563, + "num_tokens": 287344292.0, + "step": 2210 + }, + { + "entropy": 1.893359375, + "epoch": 0.3674584126458661, + "grad_norm": 2.2041406908065175, + "learning_rate": 6.523890784982936e-06, + "loss": 1.4159, + "mean_token_accuracy": 0.6535877704620361, + "num_tokens": 288643882.0, + "step": 2220 + }, + { + "entropy": 1.887109375, + "epoch": 0.36911363072084746, + "grad_norm": 2.3182630944578793, + "learning_rate": 6.506825938566553e-06, + "loss": 1.4291, + "mean_token_accuracy": 0.6539813995361328, + "num_tokens": 289942333.0, + "step": 2230 + }, + { + "entropy": 1.88671875, + "epoch": 0.37076884879582883, + "grad_norm": 2.5849136264069443, + "learning_rate": 6.4897610921501705e-06, + "loss": 1.4604, + "mean_token_accuracy": 0.6455789625644683, + "num_tokens": 291241408.0, + "step": 2240 + }, + { + "entropy": 1.889453125, + "epoch": 0.37242406687081026, + "grad_norm": 2.305119497532868, + "learning_rate": 6.4726962457337894e-06, + "loss": 1.4268, + "mean_token_accuracy": 0.6520987033843995, + "num_tokens": 292542158.0, + "step": 2250 + }, + { + "entropy": 1.88984375, + "epoch": 0.3740792849457916, + "grad_norm": 2.148829580476403, + "learning_rate": 6.455631399317407e-06, + "loss": 1.4279, + "mean_token_accuracy": 0.6514461934566498, + "num_tokens": 293843930.0, + "step": 2260 + }, + { + "entropy": 1.88671875, + "epoch": 0.375734503020773, + "grad_norm": 2.4759420910255825, + "learning_rate": 6.438566552901025e-06, + "loss": 1.4423, + "mean_token_accuracy": 0.6494761973619461, + "num_tokens": 295145592.0, + "step": 2270 + }, + { + "entropy": 1.88359375, + "epoch": 0.37738972109575436, + "grad_norm": 2.4464026783780124, + "learning_rate": 6.421501706484642e-06, + "loss": 1.4243, + "mean_token_accuracy": 0.6532855361700058, + "num_tokens": 296446908.0, + "step": 2280 + }, + { + "entropy": 1.8609375, + "epoch": 0.37904493917073573, + "grad_norm": 2.463897913458643, + "learning_rate": 6.404436860068259e-06, + "loss": 1.3923, + "mean_token_accuracy": 0.6611443072557449, + "num_tokens": 297748128.0, + "step": 2290 + }, + { + "entropy": 1.8765625, + "epoch": 0.3807001572457171, + "grad_norm": 2.575680623664601, + "learning_rate": 6.387372013651878e-06, + "loss": 1.3799, + "mean_token_accuracy": 0.6633019357919693, + "num_tokens": 299048937.0, + "step": 2300 + }, + { + "epoch": 0.3807001572457171, + "eval_entropy": 1.8716517857142858, + "eval_loss": 1.4085102081298828, + "eval_mean_token_accuracy": 0.6557413424764361, + "eval_num_tokens": 299048937.0, + "eval_runtime": 4.6375, + "eval_samples_per_second": 94.448, + "eval_steps_per_second": 1.509, + "step": 2300 + }, + { + "entropy": 1.878125, + "epoch": 0.3823553753206985, + "grad_norm": 2.1327085038843023, + "learning_rate": 6.3703071672354956e-06, + "loss": 1.4454, + "mean_token_accuracy": 0.6473503053188324, + "num_tokens": 300349295.0, + "step": 2310 + }, + { + "entropy": 1.8765625, + "epoch": 0.3840105933956799, + "grad_norm": 2.1132493692865064, + "learning_rate": 6.353242320819113e-06, + "loss": 1.4422, + "mean_token_accuracy": 0.651432928442955, + "num_tokens": 301647274.0, + "step": 2320 + }, + { + "entropy": 1.90546875, + "epoch": 0.38566581147066126, + "grad_norm": 2.3426472249278754, + "learning_rate": 6.33617747440273e-06, + "loss": 1.4326, + "mean_token_accuracy": 0.6515222698450088, + "num_tokens": 302949183.0, + "step": 2330 + }, + { + "entropy": 1.843359375, + "epoch": 0.3873210295456426, + "grad_norm": 2.4163903542520515, + "learning_rate": 6.319112627986349e-06, + "loss": 1.3932, + "mean_token_accuracy": 0.6594741076231003, + "num_tokens": 304246973.0, + "step": 2340 + }, + { + "entropy": 1.85703125, + "epoch": 0.388976247620624, + "grad_norm": 2.2528155912847487, + "learning_rate": 6.302047781569966e-06, + "loss": 1.3947, + "mean_token_accuracy": 0.6590730786323548, + "num_tokens": 305550454.0, + "step": 2350 + }, + { + "entropy": 1.900390625, + "epoch": 0.3906314656956054, + "grad_norm": 2.1907048388210337, + "learning_rate": 6.2849829351535844e-06, + "loss": 1.4275, + "mean_token_accuracy": 0.6556181490421296, + "num_tokens": 306849766.0, + "step": 2360 + }, + { + "entropy": 1.866015625, + "epoch": 0.3922866837705868, + "grad_norm": 2.2982449610803117, + "learning_rate": 6.267918088737202e-06, + "loss": 1.4026, + "mean_token_accuracy": 0.6576106518507003, + "num_tokens": 308148337.0, + "step": 2370 + }, + { + "entropy": 1.887890625, + "epoch": 0.39394190184556815, + "grad_norm": 2.3857127257987765, + "learning_rate": 6.25085324232082e-06, + "loss": 1.4045, + "mean_token_accuracy": 0.6589350372552871, + "num_tokens": 309450378.0, + "step": 2380 + }, + { + "entropy": 1.901953125, + "epoch": 0.3955971199205495, + "grad_norm": 1.937167359399426, + "learning_rate": 6.233788395904438e-06, + "loss": 1.4633, + "mean_token_accuracy": 0.6443170815706253, + "num_tokens": 310750566.0, + "step": 2390 + }, + { + "entropy": 1.904296875, + "epoch": 0.3972523379955309, + "grad_norm": 2.0943680508904445, + "learning_rate": 6.216723549488055e-06, + "loss": 1.4298, + "mean_token_accuracy": 0.6526560723781586, + "num_tokens": 312049296.0, + "step": 2400 + }, + { + "epoch": 0.3972523379955309, + "eval_entropy": 1.8638392857142858, + "eval_loss": 1.3830530643463135, + "eval_mean_token_accuracy": 0.6598115733691624, + "eval_num_tokens": 312049296.0, + "eval_runtime": 4.6302, + "eval_samples_per_second": 94.595, + "eval_steps_per_second": 1.512, + "step": 2400 + }, + { + "entropy": 1.891796875, + "epoch": 0.3989075560705123, + "grad_norm": 2.158207635320603, + "learning_rate": 6.1996587030716725e-06, + "loss": 1.4532, + "mean_token_accuracy": 0.6486242443323136, + "num_tokens": 313352151.0, + "step": 2410 + }, + { + "entropy": 1.846875, + "epoch": 0.4005627741454937, + "grad_norm": 2.3444280540391276, + "learning_rate": 6.18259385665529e-06, + "loss": 1.3864, + "mean_token_accuracy": 0.6619591265916824, + "num_tokens": 314651035.0, + "step": 2420 + }, + { + "entropy": 1.876953125, + "epoch": 0.40221799222047505, + "grad_norm": 2.7665384044557926, + "learning_rate": 6.165529010238909e-06, + "loss": 1.4259, + "mean_token_accuracy": 0.6533846348524094, + "num_tokens": 315951219.0, + "step": 2430 + }, + { + "entropy": 1.86484375, + "epoch": 0.4038732102954564, + "grad_norm": 2.4973845567823436, + "learning_rate": 6.148464163822526e-06, + "loss": 1.405, + "mean_token_accuracy": 0.6572460770606995, + "num_tokens": 317250541.0, + "step": 2440 + }, + { + "entropy": 1.882421875, + "epoch": 0.4055284283704378, + "grad_norm": 2.4010323854032, + "learning_rate": 6.131399317406144e-06, + "loss": 1.3896, + "mean_token_accuracy": 0.6622798711061477, + "num_tokens": 318553072.0, + "step": 2450 + }, + { + "entropy": 1.853125, + "epoch": 0.4071836464454192, + "grad_norm": 2.1342622134162124, + "learning_rate": 6.114334470989761e-06, + "loss": 1.397, + "mean_token_accuracy": 0.6604621589183808, + "num_tokens": 319853175.0, + "step": 2460 + }, + { + "entropy": 1.864453125, + "epoch": 0.4088388645204006, + "grad_norm": 2.334109320914987, + "learning_rate": 6.0972696245733794e-06, + "loss": 1.4114, + "mean_token_accuracy": 0.6570899724960327, + "num_tokens": 321151756.0, + "step": 2470 + }, + { + "entropy": 1.8828125, + "epoch": 0.41049408259538195, + "grad_norm": 2.4408453149373934, + "learning_rate": 6.0802047781569976e-06, + "loss": 1.4223, + "mean_token_accuracy": 0.6532759517431259, + "num_tokens": 322450712.0, + "step": 2480 + }, + { + "entropy": 1.876953125, + "epoch": 0.4121493006703633, + "grad_norm": 2.461061749568398, + "learning_rate": 6.063139931740615e-06, + "loss": 1.4179, + "mean_token_accuracy": 0.6554657787084579, + "num_tokens": 323750940.0, + "step": 2490 + }, + { + "entropy": 1.85546875, + "epoch": 0.4138045187453447, + "grad_norm": 2.3415548388201914, + "learning_rate": 6.046075085324232e-06, + "loss": 1.3673, + "mean_token_accuracy": 0.6667586147785187, + "num_tokens": 325053047.0, + "step": 2500 + }, + { + "epoch": 0.4138045187453447, + "eval_entropy": 1.8839285714285714, + "eval_loss": 1.3725452423095703, + "eval_mean_token_accuracy": 0.6629791940961566, + "eval_num_tokens": 325053047.0, + "eval_runtime": 4.6471, + "eval_samples_per_second": 94.253, + "eval_steps_per_second": 1.506, + "step": 2500 + }, + { + "entropy": 1.88046875, + "epoch": 0.41545973682032605, + "grad_norm": 2.474380950145421, + "learning_rate": 6.029010238907851e-06, + "loss": 1.4285, + "mean_token_accuracy": 0.653350755572319, + "num_tokens": 326356692.0, + "step": 2510 + }, + { + "entropy": 1.879296875, + "epoch": 0.4171149548953075, + "grad_norm": 2.1197057439589493, + "learning_rate": 6.011945392491468e-06, + "loss": 1.3901, + "mean_token_accuracy": 0.6600858956575394, + "num_tokens": 327658898.0, + "step": 2520 + }, + { + "entropy": 1.874609375, + "epoch": 0.41877017297028885, + "grad_norm": 2.0889439877328306, + "learning_rate": 5.994880546075086e-06, + "loss": 1.4058, + "mean_token_accuracy": 0.6582729011774063, + "num_tokens": 328961954.0, + "step": 2530 + }, + { + "entropy": 1.889453125, + "epoch": 0.4204253910452702, + "grad_norm": 2.3492289183836217, + "learning_rate": 5.977815699658704e-06, + "loss": 1.4159, + "mean_token_accuracy": 0.6550557106733322, + "num_tokens": 330260148.0, + "step": 2540 + }, + { + "entropy": 1.8828125, + "epoch": 0.4220806091202516, + "grad_norm": 2.8608517593391904, + "learning_rate": 5.960750853242321e-06, + "loss": 1.4014, + "mean_token_accuracy": 0.6602959513664246, + "num_tokens": 331559508.0, + "step": 2550 + }, + { + "entropy": 1.892578125, + "epoch": 0.42373582719523295, + "grad_norm": 2.401392627933927, + "learning_rate": 5.943686006825939e-06, + "loss": 1.3911, + "mean_token_accuracy": 0.6638623833656311, + "num_tokens": 332858506.0, + "step": 2560 + }, + { + "entropy": 1.88125, + "epoch": 0.4253910452702144, + "grad_norm": 2.275675480559417, + "learning_rate": 5.926621160409557e-06, + "loss": 1.4152, + "mean_token_accuracy": 0.6586078435182572, + "num_tokens": 334159196.0, + "step": 2570 + }, + { + "entropy": 1.87734375, + "epoch": 0.42704626334519574, + "grad_norm": 2.2090149766460834, + "learning_rate": 5.9095563139931745e-06, + "loss": 1.3988, + "mean_token_accuracy": 0.6604704469442367, + "num_tokens": 335461048.0, + "step": 2580 + }, + { + "entropy": 1.87265625, + "epoch": 0.4287014814201771, + "grad_norm": 2.467637754461087, + "learning_rate": 5.892491467576792e-06, + "loss": 1.3946, + "mean_token_accuracy": 0.6610612541437149, + "num_tokens": 336760355.0, + "step": 2590 + }, + { + "entropy": 1.860546875, + "epoch": 0.4303566994951585, + "grad_norm": 2.545665749965741, + "learning_rate": 5.875426621160411e-06, + "loss": 1.4032, + "mean_token_accuracy": 0.6592549920082093, + "num_tokens": 338062231.0, + "step": 2600 + }, + { + "epoch": 0.4303566994951585, + "eval_entropy": 1.8571428571428572, + "eval_loss": 1.3682761192321777, + "eval_mean_token_accuracy": 0.6636263472693307, + "eval_num_tokens": 338062231.0, + "eval_runtime": 4.6469, + "eval_samples_per_second": 94.256, + "eval_steps_per_second": 1.506, + "step": 2600 + }, + { + "entropy": 1.8796875, + "epoch": 0.43201191757013985, + "grad_norm": 2.350592803977708, + "learning_rate": 5.858361774744028e-06, + "loss": 1.4264, + "mean_token_accuracy": 0.6542666226625442, + "num_tokens": 339360241.0, + "step": 2610 + }, + { + "entropy": 1.871875, + "epoch": 0.43366713564512127, + "grad_norm": 2.2885308304181304, + "learning_rate": 5.841296928327645e-06, + "loss": 1.3936, + "mean_token_accuracy": 0.6600595533847808, + "num_tokens": 340655181.0, + "step": 2620 + }, + { + "entropy": 1.8796875, + "epoch": 0.43532235372010264, + "grad_norm": 2.3369525118402934, + "learning_rate": 5.824232081911263e-06, + "loss": 1.3882, + "mean_token_accuracy": 0.6615833342075348, + "num_tokens": 341955029.0, + "step": 2630 + }, + { + "entropy": 1.859765625, + "epoch": 0.436977571795084, + "grad_norm": 2.260938942837787, + "learning_rate": 5.807167235494881e-06, + "loss": 1.386, + "mean_token_accuracy": 0.6609073966741562, + "num_tokens": 343256815.0, + "step": 2640 + }, + { + "entropy": 1.878125, + "epoch": 0.4386327898700654, + "grad_norm": 2.199411958244908, + "learning_rate": 5.790102389078499e-06, + "loss": 1.4232, + "mean_token_accuracy": 0.6562960803508758, + "num_tokens": 344556457.0, + "step": 2650 + }, + { + "entropy": 1.862890625, + "epoch": 0.44028800794504674, + "grad_norm": 2.6825555541088324, + "learning_rate": 5.773037542662117e-06, + "loss": 1.3955, + "mean_token_accuracy": 0.6610384434461594, + "num_tokens": 345857991.0, + "step": 2660 + }, + { + "entropy": 1.88203125, + "epoch": 0.4419432260200281, + "grad_norm": 2.2019115075827904, + "learning_rate": 5.755972696245734e-06, + "loss": 1.4116, + "mean_token_accuracy": 0.6570022106170654, + "num_tokens": 347159633.0, + "step": 2670 + }, + { + "entropy": 1.859765625, + "epoch": 0.44359844409500954, + "grad_norm": 2.313861641137639, + "learning_rate": 5.738907849829351e-06, + "loss": 1.4065, + "mean_token_accuracy": 0.6575842171907424, + "num_tokens": 348460255.0, + "step": 2680 + }, + { + "entropy": 1.849609375, + "epoch": 0.4452536621699909, + "grad_norm": 2.369761293580437, + "learning_rate": 5.72184300341297e-06, + "loss": 1.394, + "mean_token_accuracy": 0.6596699029207229, + "num_tokens": 349759819.0, + "step": 2690 + }, + { + "entropy": 1.856640625, + "epoch": 0.4469088802449723, + "grad_norm": 2.515491177557878, + "learning_rate": 5.704778156996588e-06, + "loss": 1.3819, + "mean_token_accuracy": 0.6650141805410386, + "num_tokens": 351060953.0, + "step": 2700 + }, + { + "epoch": 0.4469088802449723, + "eval_entropy": 1.8660714285714286, + "eval_loss": 1.3611005544662476, + "eval_mean_token_accuracy": 0.6652760761124747, + "eval_num_tokens": 351060953.0, + "eval_runtime": 4.6186, + "eval_samples_per_second": 94.834, + "eval_steps_per_second": 1.516, + "step": 2700 + }, + { + "entropy": 1.883984375, + "epoch": 0.44856409831995364, + "grad_norm": 2.064177796178853, + "learning_rate": 5.687713310580205e-06, + "loss": 1.4031, + "mean_token_accuracy": 0.6593500882387161, + "num_tokens": 352363783.0, + "step": 2710 + }, + { + "entropy": 1.872265625, + "epoch": 0.450219316394935, + "grad_norm": 2.1049422553582056, + "learning_rate": 5.670648464163823e-06, + "loss": 1.3795, + "mean_token_accuracy": 0.6652626752853393, + "num_tokens": 353664551.0, + "step": 2720 + }, + { + "entropy": 1.87265625, + "epoch": 0.45187453446991643, + "grad_norm": 2.1368048829650323, + "learning_rate": 5.653583617747441e-06, + "loss": 1.376, + "mean_token_accuracy": 0.6659601867198944, + "num_tokens": 354967692.0, + "step": 2730 + }, + { + "entropy": 1.88125, + "epoch": 0.4535297525448978, + "grad_norm": 2.0234287828136823, + "learning_rate": 5.636518771331058e-06, + "loss": 1.4072, + "mean_token_accuracy": 0.6563500553369522, + "num_tokens": 356269041.0, + "step": 2740 + }, + { + "entropy": 1.85625, + "epoch": 0.45518497061987917, + "grad_norm": 2.020197610278999, + "learning_rate": 5.6194539249146765e-06, + "loss": 1.3847, + "mean_token_accuracy": 0.6622072458267212, + "num_tokens": 357569645.0, + "step": 2750 + }, + { + "entropy": 1.851953125, + "epoch": 0.45684018869486054, + "grad_norm": 2.126300477003854, + "learning_rate": 5.602389078498294e-06, + "loss": 1.3784, + "mean_token_accuracy": 0.6656763285398484, + "num_tokens": 358871131.0, + "step": 2760 + }, + { + "entropy": 1.8609375, + "epoch": 0.4584954067698419, + "grad_norm": 2.505842657999584, + "learning_rate": 5.585324232081911e-06, + "loss": 1.3843, + "mean_token_accuracy": 0.6638334959745407, + "num_tokens": 360171994.0, + "step": 2770 + }, + { + "entropy": 1.871484375, + "epoch": 0.46015062484482333, + "grad_norm": 2.4892783080141596, + "learning_rate": 5.56825938566553e-06, + "loss": 1.3825, + "mean_token_accuracy": 0.6639627814292908, + "num_tokens": 361467558.0, + "step": 2780 + }, + { + "entropy": 1.855078125, + "epoch": 0.4618058429198047, + "grad_norm": 2.42270302454476, + "learning_rate": 5.551194539249147e-06, + "loss": 1.3833, + "mean_token_accuracy": 0.664381018280983, + "num_tokens": 362768853.0, + "step": 2790 + }, + { + "entropy": 1.8640625, + "epoch": 0.46346106099478607, + "grad_norm": 2.4921260967668255, + "learning_rate": 5.5341296928327645e-06, + "loss": 1.3898, + "mean_token_accuracy": 0.6628597468137741, + "num_tokens": 364071305.0, + "step": 2800 + }, + { + "epoch": 0.46346106099478607, + "eval_entropy": 1.8191964285714286, + "eval_loss": 1.3539565801620483, + "eval_mean_token_accuracy": 0.6665515984807696, + "eval_num_tokens": 364071305.0, + "eval_runtime": 4.6482, + "eval_samples_per_second": 94.229, + "eval_steps_per_second": 1.506, + "step": 2800 + }, + { + "entropy": 1.85859375, + "epoch": 0.46511627906976744, + "grad_norm": 2.330189076649679, + "learning_rate": 5.517064846416383e-06, + "loss": 1.3862, + "mean_token_accuracy": 0.6642320215702057, + "num_tokens": 365370283.0, + "step": 2810 + }, + { + "entropy": 1.858203125, + "epoch": 0.4667714971447488, + "grad_norm": 2.147772808369429, + "learning_rate": 5.500000000000001e-06, + "loss": 1.3782, + "mean_token_accuracy": 0.663418909907341, + "num_tokens": 366670554.0, + "step": 2820 + }, + { + "entropy": 1.858984375, + "epoch": 0.46842671521973017, + "grad_norm": 2.2251778720032003, + "learning_rate": 5.482935153583618e-06, + "loss": 1.3778, + "mean_token_accuracy": 0.6643008470535279, + "num_tokens": 367967565.0, + "step": 2830 + }, + { + "entropy": 1.833984375, + "epoch": 0.4700819332947116, + "grad_norm": 2.4251244824961056, + "learning_rate": 5.465870307167236e-06, + "loss": 1.3742, + "mean_token_accuracy": 0.6641975939273834, + "num_tokens": 369268487.0, + "step": 2840 + }, + { + "entropy": 1.866015625, + "epoch": 0.47173715136969296, + "grad_norm": 2.3552430922271044, + "learning_rate": 5.448805460750853e-06, + "loss": 1.4049, + "mean_token_accuracy": 0.659347090125084, + "num_tokens": 370566033.0, + "step": 2850 + }, + { + "entropy": 1.846484375, + "epoch": 0.47339236944467433, + "grad_norm": 2.0419227866121066, + "learning_rate": 5.431740614334472e-06, + "loss": 1.3655, + "mean_token_accuracy": 0.6685224354267121, + "num_tokens": 371865920.0, + "step": 2860 + }, + { + "entropy": 1.85859375, + "epoch": 0.4750475875196557, + "grad_norm": 2.384018219502059, + "learning_rate": 5.4146757679180896e-06, + "loss": 1.3634, + "mean_token_accuracy": 0.6672281265258789, + "num_tokens": 373164238.0, + "step": 2870 + }, + { + "entropy": 1.84140625, + "epoch": 0.47670280559463707, + "grad_norm": 2.516442784576793, + "learning_rate": 5.397610921501707e-06, + "loss": 1.3658, + "mean_token_accuracy": 0.6661449372768402, + "num_tokens": 374464158.0, + "step": 2880 + }, + { + "entropy": 1.8328125, + "epoch": 0.4783580236696185, + "grad_norm": 2.3280490006366685, + "learning_rate": 5.380546075085324e-06, + "loss": 1.341, + "mean_token_accuracy": 0.672097560763359, + "num_tokens": 375764327.0, + "step": 2890 + }, + { + "entropy": 1.851953125, + "epoch": 0.48001324174459986, + "grad_norm": 2.2839661885774656, + "learning_rate": 5.363481228668942e-06, + "loss": 1.3819, + "mean_token_accuracy": 0.6647409856319427, + "num_tokens": 377064094.0, + "step": 2900 + }, + { + "epoch": 0.48001324174459986, + "eval_entropy": 1.8370535714285714, + "eval_loss": 1.3412119150161743, + "eval_mean_token_accuracy": 0.6705723660332816, + "eval_num_tokens": 377064094.0, + "eval_runtime": 4.6436, + "eval_samples_per_second": 94.323, + "eval_steps_per_second": 1.507, + "step": 2900 + }, + { + "entropy": 1.84765625, + "epoch": 0.48166845981958123, + "grad_norm": 2.2113973452849995, + "learning_rate": 5.34641638225256e-06, + "loss": 1.338, + "mean_token_accuracy": 0.6756518632173538, + "num_tokens": 378364200.0, + "step": 2910 + }, + { + "entropy": 1.849609375, + "epoch": 0.4833236778945626, + "grad_norm": 2.5927143294330026, + "learning_rate": 5.3293515358361784e-06, + "loss": 1.3607, + "mean_token_accuracy": 0.6681739538908005, + "num_tokens": 379664074.0, + "step": 2920 + }, + { + "entropy": 1.84765625, + "epoch": 0.48497889596954397, + "grad_norm": 2.366264782839681, + "learning_rate": 5.312286689419796e-06, + "loss": 1.375, + "mean_token_accuracy": 0.6652745813131332, + "num_tokens": 380964576.0, + "step": 2930 + }, + { + "entropy": 1.83828125, + "epoch": 0.4866341140445254, + "grad_norm": 2.3228535707728595, + "learning_rate": 5.295221843003413e-06, + "loss": 1.3828, + "mean_token_accuracy": 0.6642748475074768, + "num_tokens": 382263551.0, + "step": 2940 + }, + { + "entropy": 1.855078125, + "epoch": 0.48828933211950676, + "grad_norm": 2.6933663139618256, + "learning_rate": 5.278156996587032e-06, + "loss": 1.3693, + "mean_token_accuracy": 0.6665320217609405, + "num_tokens": 383565767.0, + "step": 2950 + }, + { + "entropy": 1.84140625, + "epoch": 0.4899445501944881, + "grad_norm": 2.2837486294049536, + "learning_rate": 5.261092150170649e-06, + "loss": 1.3737, + "mean_token_accuracy": 0.6653079032897949, + "num_tokens": 384864152.0, + "step": 2960 + }, + { + "entropy": 1.8546875, + "epoch": 0.4915997682694695, + "grad_norm": 2.3858560142694842, + "learning_rate": 5.2440273037542665e-06, + "loss": 1.374, + "mean_token_accuracy": 0.6661663770675659, + "num_tokens": 386164559.0, + "step": 2970 + }, + { + "entropy": 1.8375, + "epoch": 0.49325498634445086, + "grad_norm": 2.2762554386402445, + "learning_rate": 5.226962457337884e-06, + "loss": 1.3796, + "mean_token_accuracy": 0.6645379483699798, + "num_tokens": 387467105.0, + "step": 2980 + }, + { + "entropy": 1.82265625, + "epoch": 0.4949102044194323, + "grad_norm": 2.3906641280486802, + "learning_rate": 5.209897610921502e-06, + "loss": 1.34, + "mean_token_accuracy": 0.6719705700874329, + "num_tokens": 388768010.0, + "step": 2990 + }, + { + "entropy": 1.83515625, + "epoch": 0.49656542249441366, + "grad_norm": 2.495405891988979, + "learning_rate": 5.19283276450512e-06, + "loss": 1.3544, + "mean_token_accuracy": 0.6707408487796783, + "num_tokens": 390066424.0, + "step": 3000 + }, + { + "epoch": 0.49656542249441366, + "eval_entropy": 1.8258928571428572, + "eval_loss": 1.3202171325683594, + "eval_mean_token_accuracy": 0.6758410164288112, + "eval_num_tokens": 390066424.0, + "eval_runtime": 4.6327, + "eval_samples_per_second": 94.545, + "eval_steps_per_second": 1.511, + "step": 3000 + }, + { + "entropy": 1.841015625, + "epoch": 0.498220640569395, + "grad_norm": 2.2079109224615747, + "learning_rate": 5.175767918088738e-06, + "loss": 1.3743, + "mean_token_accuracy": 0.6664914667606354, + "num_tokens": 391367935.0, + "step": 3010 + }, + { + "entropy": 1.85, + "epoch": 0.4998758586443764, + "grad_norm": 2.849077097969467, + "learning_rate": 5.158703071672355e-06, + "loss": 1.4169, + "mean_token_accuracy": 0.6588717699050903, + "num_tokens": 392669110.0, + "step": 3020 + }, + { + "entropy": 1.8578125, + "epoch": 0.5015310767193578, + "grad_norm": 2.4098536210374606, + "learning_rate": 5.141638225255973e-06, + "loss": 1.4041, + "mean_token_accuracy": 0.6602443128824234, + "num_tokens": 393970739.0, + "step": 3030 + }, + { + "entropy": 1.8359375, + "epoch": 0.5031862947943392, + "grad_norm": 2.6686735509519663, + "learning_rate": 5.1245733788395916e-06, + "loss": 1.3449, + "mean_token_accuracy": 0.6725923418998718, + "num_tokens": 395268277.0, + "step": 3040 + }, + { + "entropy": 1.836328125, + "epoch": 0.5048415128693206, + "grad_norm": 2.359826306712244, + "learning_rate": 5.107508532423209e-06, + "loss": 1.3591, + "mean_token_accuracy": 0.6698974579572677, + "num_tokens": 396568293.0, + "step": 3050 + }, + { + "entropy": 1.844921875, + "epoch": 0.5064967309443019, + "grad_norm": 2.1319730264519, + "learning_rate": 5.090443686006826e-06, + "loss": 1.363, + "mean_token_accuracy": 0.6698909074068069, + "num_tokens": 397869580.0, + "step": 3060 + }, + { + "entropy": 1.8390625, + "epoch": 0.5081519490192833, + "grad_norm": 2.29474829305848, + "learning_rate": 5.073378839590443e-06, + "loss": 1.3652, + "mean_token_accuracy": 0.6668981403112412, + "num_tokens": 399168474.0, + "step": 3070 + }, + { + "entropy": 1.839453125, + "epoch": 0.5098071670942647, + "grad_norm": 2.2971589355610367, + "learning_rate": 5.056313993174062e-06, + "loss": 1.3445, + "mean_token_accuracy": 0.6741679042577744, + "num_tokens": 400468335.0, + "step": 3080 + }, + { + "entropy": 1.834765625, + "epoch": 0.511462385169246, + "grad_norm": 2.468202732313567, + "learning_rate": 5.03924914675768e-06, + "loss": 1.3685, + "mean_token_accuracy": 0.6682452589273453, + "num_tokens": 401769882.0, + "step": 3090 + }, + { + "entropy": 1.83359375, + "epoch": 0.5131176032442274, + "grad_norm": 2.3844714399986877, + "learning_rate": 5.022184300341298e-06, + "loss": 1.3681, + "mean_token_accuracy": 0.6684703230857849, + "num_tokens": 403069886.0, + "step": 3100 + }, + { + "epoch": 0.5131176032442274, + "eval_entropy": 1.7756696428571428, + "eval_loss": 1.3115850687026978, + "eval_mean_token_accuracy": 0.6780931438718524, + "eval_num_tokens": 403069886.0, + "eval_runtime": 4.6546, + "eval_samples_per_second": 94.1, + "eval_steps_per_second": 1.504, + "step": 3100 + }, + { + "entropy": 1.83515625, + "epoch": 0.5147728213192088, + "grad_norm": 2.3823841149863534, + "learning_rate": 5.005119453924915e-06, + "loss": 1.3697, + "mean_token_accuracy": 0.6664821773767471, + "num_tokens": 404371913.0, + "step": 3110 + }, + { + "entropy": 1.8, + "epoch": 0.5164280393941901, + "grad_norm": 2.198066165370959, + "learning_rate": 4.988054607508533e-06, + "loss": 1.3461, + "mean_token_accuracy": 0.6731403648853302, + "num_tokens": 405672549.0, + "step": 3120 + }, + { + "entropy": 1.837109375, + "epoch": 0.5180832574691716, + "grad_norm": 2.5833415375178324, + "learning_rate": 4.97098976109215e-06, + "loss": 1.3494, + "mean_token_accuracy": 0.6712380439043045, + "num_tokens": 406973923.0, + "step": 3130 + }, + { + "entropy": 1.827734375, + "epoch": 0.519738475544153, + "grad_norm": 2.357994041626405, + "learning_rate": 4.9539249146757685e-06, + "loss": 1.3582, + "mean_token_accuracy": 0.671397989988327, + "num_tokens": 408272972.0, + "step": 3140 + }, + { + "entropy": 1.83515625, + "epoch": 0.5213936936191343, + "grad_norm": 2.4878283619458563, + "learning_rate": 4.936860068259386e-06, + "loss": 1.3582, + "mean_token_accuracy": 0.6711713135242462, + "num_tokens": 409572605.0, + "step": 3150 + }, + { + "entropy": 1.84609375, + "epoch": 0.5230489116941157, + "grad_norm": 2.6031800283002786, + "learning_rate": 4.919795221843004e-06, + "loss": 1.359, + "mean_token_accuracy": 0.670652449131012, + "num_tokens": 410872583.0, + "step": 3160 + }, + { + "entropy": 1.838671875, + "epoch": 0.5247041297690971, + "grad_norm": 2.940282155246058, + "learning_rate": 4.902730375426621e-06, + "loss": 1.344, + "mean_token_accuracy": 0.6723409295082092, + "num_tokens": 412172584.0, + "step": 3170 + }, + { + "entropy": 1.833203125, + "epoch": 0.5263593478440785, + "grad_norm": 2.2475086086499743, + "learning_rate": 4.885665529010239e-06, + "loss": 1.3312, + "mean_token_accuracy": 0.6760436564683914, + "num_tokens": 413470828.0, + "step": 3180 + }, + { + "entropy": 1.826953125, + "epoch": 0.5280145659190598, + "grad_norm": 2.3351982773227684, + "learning_rate": 4.868600682593857e-06, + "loss": 1.3274, + "mean_token_accuracy": 0.6764830023050308, + "num_tokens": 414770752.0, + "step": 3190 + }, + { + "entropy": 1.8359375, + "epoch": 0.5296697839940412, + "grad_norm": 2.4292649367548225, + "learning_rate": 4.8515358361774755e-06, + "loss": 1.3507, + "mean_token_accuracy": 0.6708489090204239, + "num_tokens": 416073874.0, + "step": 3200 + }, + { + "epoch": 0.5296697839940412, + "eval_entropy": 1.8080357142857142, + "eval_loss": 1.2953251600265503, + "eval_mean_token_accuracy": 0.681542558329446, + "eval_num_tokens": 416073874.0, + "eval_runtime": 4.6523, + "eval_samples_per_second": 94.147, + "eval_steps_per_second": 1.505, + "step": 3200 + }, + { + "entropy": 1.845703125, + "epoch": 0.5313250020690226, + "grad_norm": 2.2257809019937045, + "learning_rate": 4.834470989761093e-06, + "loss": 1.3231, + "mean_token_accuracy": 0.6768483996391297, + "num_tokens": 417376616.0, + "step": 3210 + }, + { + "entropy": 1.850390625, + "epoch": 0.5329802201440039, + "grad_norm": 2.4938723825923854, + "learning_rate": 4.81740614334471e-06, + "loss": 1.3271, + "mean_token_accuracy": 0.6758997827768326, + "num_tokens": 418678076.0, + "step": 3220 + }, + { + "entropy": 1.842578125, + "epoch": 0.5346354382189854, + "grad_norm": 2.7524652727723624, + "learning_rate": 4.800341296928328e-06, + "loss": 1.3313, + "mean_token_accuracy": 0.6761848002672195, + "num_tokens": 419977870.0, + "step": 3230 + }, + { + "entropy": 1.83828125, + "epoch": 0.5362906562939668, + "grad_norm": 2.68595058367095, + "learning_rate": 4.783276450511945e-06, + "loss": 1.3342, + "mean_token_accuracy": 0.6764870315790177, + "num_tokens": 421279801.0, + "step": 3240 + }, + { + "entropy": 1.824609375, + "epoch": 0.5379458743689481, + "grad_norm": 2.4573223286484134, + "learning_rate": 4.7662116040955635e-06, + "loss": 1.3676, + "mean_token_accuracy": 0.6678244799375535, + "num_tokens": 422581431.0, + "step": 3250 + }, + { + "entropy": 1.840234375, + "epoch": 0.5396010924439295, + "grad_norm": 2.3668287029615844, + "learning_rate": 4.749146757679181e-06, + "loss": 1.3629, + "mean_token_accuracy": 0.6695770233869552, + "num_tokens": 423880493.0, + "step": 3260 + }, + { + "entropy": 1.825, + "epoch": 0.5412563105189109, + "grad_norm": 2.3989879772221405, + "learning_rate": 4.732081911262799e-06, + "loss": 1.3301, + "mean_token_accuracy": 0.6756509155035019, + "num_tokens": 425178114.0, + "step": 3270 + }, + { + "entropy": 1.80703125, + "epoch": 0.5429115285938922, + "grad_norm": 2.789468079108899, + "learning_rate": 4.715017064846417e-06, + "loss": 1.3077, + "mean_token_accuracy": 0.6824726164340973, + "num_tokens": 426478477.0, + "step": 3280 + }, + { + "entropy": 1.821484375, + "epoch": 0.5445667466688736, + "grad_norm": 2.4578444302457387, + "learning_rate": 4.697952218430035e-06, + "loss": 1.322, + "mean_token_accuracy": 0.676766362786293, + "num_tokens": 427780439.0, + "step": 3290 + }, + { + "entropy": 1.83828125, + "epoch": 0.546221964743855, + "grad_norm": 2.5925045183487714, + "learning_rate": 4.680887372013652e-06, + "loss": 1.3252, + "mean_token_accuracy": 0.6769931107759476, + "num_tokens": 429080245.0, + "step": 3300 + }, + { + "epoch": 0.546221964743855, + "eval_entropy": 1.8125, + "eval_loss": 1.2867203950881958, + "eval_mean_token_accuracy": 0.6842466677938189, + "eval_num_tokens": 429080245.0, + "eval_runtime": 4.6445, + "eval_samples_per_second": 94.305, + "eval_steps_per_second": 1.507, + "step": 3300 + }, + { + "entropy": 1.833203125, + "epoch": 0.5478771828188364, + "grad_norm": 2.7548954523241904, + "learning_rate": 4.6638225255972705e-06, + "loss": 1.3057, + "mean_token_accuracy": 0.6804468154907226, + "num_tokens": 430378133.0, + "step": 3310 + }, + { + "entropy": 1.830078125, + "epoch": 0.5495324008938177, + "grad_norm": 2.341118468516948, + "learning_rate": 4.646757679180888e-06, + "loss": 1.3452, + "mean_token_accuracy": 0.6735901236534119, + "num_tokens": 431679110.0, + "step": 3320 + }, + { + "entropy": 1.80859375, + "epoch": 0.5511876189687991, + "grad_norm": 2.551754049152379, + "learning_rate": 4.629692832764506e-06, + "loss": 1.3218, + "mean_token_accuracy": 0.679484286904335, + "num_tokens": 432977085.0, + "step": 3330 + }, + { + "entropy": 1.82578125, + "epoch": 0.5528428370437806, + "grad_norm": 2.366094976827691, + "learning_rate": 4.612627986348123e-06, + "loss": 1.3292, + "mean_token_accuracy": 0.6760279566049576, + "num_tokens": 434273299.0, + "step": 3340 + }, + { + "entropy": 1.82578125, + "epoch": 0.5544980551187619, + "grad_norm": 2.3722561496809984, + "learning_rate": 4.59556313993174e-06, + "loss": 1.3166, + "mean_token_accuracy": 0.6784065276384353, + "num_tokens": 435573398.0, + "step": 3350 + }, + { + "entropy": 1.81171875, + "epoch": 0.5561532731937433, + "grad_norm": 2.0355372772065494, + "learning_rate": 4.5784982935153585e-06, + "loss": 1.3007, + "mean_token_accuracy": 0.6822363317012787, + "num_tokens": 436874573.0, + "step": 3360 + }, + { + "entropy": 1.829296875, + "epoch": 0.5578084912687247, + "grad_norm": 2.4387706368066038, + "learning_rate": 4.561433447098977e-06, + "loss": 1.2961, + "mean_token_accuracy": 0.685641062259674, + "num_tokens": 438174785.0, + "step": 3370 + }, + { + "entropy": 1.840234375, + "epoch": 0.559463709343706, + "grad_norm": 2.3573429060877995, + "learning_rate": 4.544368600682595e-06, + "loss": 1.3102, + "mean_token_accuracy": 0.6811522036790848, + "num_tokens": 439474232.0, + "step": 3380 + }, + { + "entropy": 1.844140625, + "epoch": 0.5611189274186874, + "grad_norm": 2.4568367949761623, + "learning_rate": 4.527303754266212e-06, + "loss": 1.3303, + "mean_token_accuracy": 0.6769051164388656, + "num_tokens": 440773612.0, + "step": 3390 + }, + { + "entropy": 1.838671875, + "epoch": 0.5627741454936688, + "grad_norm": 2.4840814042765937, + "learning_rate": 4.51023890784983e-06, + "loss": 1.3188, + "mean_token_accuracy": 0.6786750167608261, + "num_tokens": 442073202.0, + "step": 3400 + }, + { + "epoch": 0.5627741454936688, + "eval_entropy": 1.8191964285714286, + "eval_loss": 1.2771695852279663, + "eval_mean_token_accuracy": 0.685752774987902, + "eval_num_tokens": 442073202.0, + "eval_runtime": 4.6338, + "eval_samples_per_second": 94.522, + "eval_steps_per_second": 1.511, + "step": 3400 + }, + { + "entropy": 1.8296875, + "epoch": 0.5644293635686501, + "grad_norm": 2.2719967260249407, + "learning_rate": 4.493174061433447e-06, + "loss": 1.3533, + "mean_token_accuracy": 0.6717742949724197, + "num_tokens": 443371796.0, + "step": 3410 + }, + { + "entropy": 1.83515625, + "epoch": 0.5660845816436315, + "grad_norm": 2.4565586938419544, + "learning_rate": 4.4761092150170655e-06, + "loss": 1.3096, + "mean_token_accuracy": 0.6821169286966324, + "num_tokens": 444669730.0, + "step": 3420 + }, + { + "entropy": 1.825390625, + "epoch": 0.5677397997186129, + "grad_norm": 2.3777193339338494, + "learning_rate": 4.459044368600683e-06, + "loss": 1.3244, + "mean_token_accuracy": 0.6772226631641388, + "num_tokens": 445969223.0, + "step": 3430 + }, + { + "entropy": 1.820703125, + "epoch": 0.5693950177935944, + "grad_norm": 2.503425647686177, + "learning_rate": 4.441979522184301e-06, + "loss": 1.3128, + "mean_token_accuracy": 0.6818882197141647, + "num_tokens": 447265461.0, + "step": 3440 + }, + { + "entropy": 1.810546875, + "epoch": 0.5710502358685757, + "grad_norm": 2.3221555143995323, + "learning_rate": 4.424914675767918e-06, + "loss": 1.2979, + "mean_token_accuracy": 0.6835970938205719, + "num_tokens": 448566683.0, + "step": 3450 + }, + { + "entropy": 1.81171875, + "epoch": 0.5727054539435571, + "grad_norm": 2.400790694785465, + "learning_rate": 4.407849829351536e-06, + "loss": 1.343, + "mean_token_accuracy": 0.6763038247823715, + "num_tokens": 449866891.0, + "step": 3460 + }, + { + "entropy": 1.812890625, + "epoch": 0.5743606720185385, + "grad_norm": 2.3714310350187544, + "learning_rate": 4.390784982935154e-06, + "loss": 1.3342, + "mean_token_accuracy": 0.6768683284521103, + "num_tokens": 451166791.0, + "step": 3470 + }, + { + "entropy": 1.79140625, + "epoch": 0.5760158900935198, + "grad_norm": 2.2387228733442006, + "learning_rate": 4.373720136518772e-06, + "loss": 1.2902, + "mean_token_accuracy": 0.6857896953821182, + "num_tokens": 452466659.0, + "step": 3480 + }, + { + "entropy": 1.80234375, + "epoch": 0.5776711081685012, + "grad_norm": 2.4803595444057835, + "learning_rate": 4.35665529010239e-06, + "loss": 1.3089, + "mean_token_accuracy": 0.682360765337944, + "num_tokens": 453766413.0, + "step": 3490 + }, + { + "entropy": 1.798828125, + "epoch": 0.5793263262434826, + "grad_norm": 2.5239072510497995, + "learning_rate": 4.339590443686007e-06, + "loss": 1.3106, + "mean_token_accuracy": 0.6819195687770844, + "num_tokens": 455064666.0, + "step": 3500 + }, + { + "epoch": 0.5793263262434826, + "eval_entropy": 1.8258928571428572, + "eval_loss": 1.2698050737380981, + "eval_mean_token_accuracy": 0.6885324205671038, + "eval_num_tokens": 455064666.0, + "eval_runtime": 4.6325, + "eval_samples_per_second": 94.549, + "eval_steps_per_second": 1.511, + "step": 3500 + }, + { + "entropy": 1.823828125, + "epoch": 0.5809815443184639, + "grad_norm": 2.460931233447082, + "learning_rate": 4.322525597269625e-06, + "loss": 1.317, + "mean_token_accuracy": 0.67956483066082, + "num_tokens": 456361817.0, + "step": 3510 + }, + { + "entropy": 1.828515625, + "epoch": 0.5826367623934453, + "grad_norm": 2.4176257720370655, + "learning_rate": 4.305460750853242e-06, + "loss": 1.313, + "mean_token_accuracy": 0.6804495811462402, + "num_tokens": 457660945.0, + "step": 3520 + }, + { + "entropy": 1.812109375, + "epoch": 0.5842919804684267, + "grad_norm": 2.346219719715966, + "learning_rate": 4.2883959044368605e-06, + "loss": 1.2924, + "mean_token_accuracy": 0.6861444622278213, + "num_tokens": 458960620.0, + "step": 3530 + }, + { + "entropy": 1.816015625, + "epoch": 0.585947198543408, + "grad_norm": 2.9094889805055013, + "learning_rate": 4.271331058020478e-06, + "loss": 1.2955, + "mean_token_accuracy": 0.685803547501564, + "num_tokens": 460262518.0, + "step": 3540 + }, + { + "entropy": 1.8265625, + "epoch": 0.5876024166183895, + "grad_norm": 2.5877597601989892, + "learning_rate": 4.254266211604096e-06, + "loss": 1.3261, + "mean_token_accuracy": 0.6782096236944198, + "num_tokens": 461561255.0, + "step": 3550 + }, + { + "entropy": 1.79140625, + "epoch": 0.5892576346933709, + "grad_norm": 2.356347557069381, + "learning_rate": 4.237201365187714e-06, + "loss": 1.2915, + "mean_token_accuracy": 0.6834725469350815, + "num_tokens": 462860934.0, + "step": 3560 + }, + { + "entropy": 1.80546875, + "epoch": 0.5909128527683523, + "grad_norm": 2.2366756789286217, + "learning_rate": 4.220136518771332e-06, + "loss": 1.3136, + "mean_token_accuracy": 0.6794704407453537, + "num_tokens": 464161290.0, + "step": 3570 + }, + { + "entropy": 1.794921875, + "epoch": 0.5925680708433336, + "grad_norm": 2.423150425360731, + "learning_rate": 4.203071672354949e-06, + "loss": 1.3222, + "mean_token_accuracy": 0.6798444479703903, + "num_tokens": 465464886.0, + "step": 3580 + }, + { + "entropy": 1.8296875, + "epoch": 0.594223288918315, + "grad_norm": 2.562122175931704, + "learning_rate": 4.186006825938567e-06, + "loss": 1.3325, + "mean_token_accuracy": 0.6753124177455903, + "num_tokens": 466764701.0, + "step": 3590 + }, + { + "entropy": 1.7875, + "epoch": 0.5958785069932964, + "grad_norm": 2.5676584428288995, + "learning_rate": 4.168941979522185e-06, + "loss": 1.3078, + "mean_token_accuracy": 0.6855162173509598, + "num_tokens": 468064635.0, + "step": 3600 + }, + { + "epoch": 0.5958785069932964, + "eval_entropy": 1.7979910714285714, + "eval_loss": 1.2558246850967407, + "eval_mean_token_accuracy": 0.6921522447041103, + "eval_num_tokens": 468064635.0, + "eval_runtime": 4.6297, + "eval_samples_per_second": 94.606, + "eval_steps_per_second": 1.512, + "step": 3600 + }, + { + "entropy": 1.827734375, + "epoch": 0.5975337250682777, + "grad_norm": 2.321889844072915, + "learning_rate": 4.151877133105802e-06, + "loss": 1.2924, + "mean_token_accuracy": 0.6863483697175979, + "num_tokens": 469364602.0, + "step": 3610 + }, + { + "entropy": 1.8140625, + "epoch": 0.5991889431432591, + "grad_norm": 2.18950682176646, + "learning_rate": 4.13481228668942e-06, + "loss": 1.3194, + "mean_token_accuracy": 0.6795646101236343, + "num_tokens": 470665060.0, + "step": 3620 + }, + { + "entropy": 1.823828125, + "epoch": 0.6008441612182405, + "grad_norm": 2.279362261925571, + "learning_rate": 4.117747440273037e-06, + "loss": 1.317, + "mean_token_accuracy": 0.6790448695421218, + "num_tokens": 471965837.0, + "step": 3630 + }, + { + "entropy": 1.819140625, + "epoch": 0.6024993792932218, + "grad_norm": 2.3570730066443963, + "learning_rate": 4.1006825938566555e-06, + "loss": 1.3129, + "mean_token_accuracy": 0.6793660551309586, + "num_tokens": 473264135.0, + "step": 3640 + }, + { + "entropy": 1.8109375, + "epoch": 0.6041545973682032, + "grad_norm": 2.130551720125765, + "learning_rate": 4.083617747440274e-06, + "loss": 1.3009, + "mean_token_accuracy": 0.6838442623615265, + "num_tokens": 474563771.0, + "step": 3650 + }, + { + "entropy": 1.791796875, + "epoch": 0.6058098154431847, + "grad_norm": 2.6307567059057697, + "learning_rate": 4.066552901023892e-06, + "loss": 1.299, + "mean_token_accuracy": 0.6834118366241455, + "num_tokens": 475862106.0, + "step": 3660 + }, + { + "entropy": 1.796875, + "epoch": 0.607465033518166, + "grad_norm": 2.4728304381382165, + "learning_rate": 4.049488054607509e-06, + "loss": 1.2681, + "mean_token_accuracy": 0.6902280569076538, + "num_tokens": 477162981.0, + "step": 3670 + }, + { + "entropy": 1.781640625, + "epoch": 0.6091202515931474, + "grad_norm": 2.640973230155348, + "learning_rate": 4.032423208191127e-06, + "loss": 1.2601, + "mean_token_accuracy": 0.6910527884960175, + "num_tokens": 478464377.0, + "step": 3680 + }, + { + "entropy": 1.783203125, + "epoch": 0.6107754696681288, + "grad_norm": 2.4115504400652656, + "learning_rate": 4.015358361774744e-06, + "loss": 1.2814, + "mean_token_accuracy": 0.6887993007898331, + "num_tokens": 479763882.0, + "step": 3690 + }, + { + "entropy": 1.79765625, + "epoch": 0.6124306877431102, + "grad_norm": 2.291989992285925, + "learning_rate": 3.998293515358362e-06, + "loss": 1.337, + "mean_token_accuracy": 0.6748594999313354, + "num_tokens": 481060913.0, + "step": 3700 + }, + { + "epoch": 0.6124306877431102, + "eval_entropy": 1.7745535714285714, + "eval_loss": 1.2481906414031982, + "eval_mean_token_accuracy": 0.6934785928045001, + "eval_num_tokens": 481060913.0, + "eval_runtime": 4.9475, + "eval_samples_per_second": 88.53, + "eval_steps_per_second": 1.415, + "step": 3700 + }, + { + "entropy": 1.78515625, + "epoch": 0.6140859058180915, + "grad_norm": 2.3244808172607594, + "learning_rate": 3.98122866894198e-06, + "loss": 1.2876, + "mean_token_accuracy": 0.6881023228168488, + "num_tokens": 482361128.0, + "step": 3710 + }, + { + "entropy": 1.787109375, + "epoch": 0.6157411238930729, + "grad_norm": 2.8117944153616903, + "learning_rate": 3.964163822525597e-06, + "loss": 1.2995, + "mean_token_accuracy": 0.684839951992035, + "num_tokens": 483663419.0, + "step": 3720 + }, + { + "entropy": 1.7671875, + "epoch": 0.6173963419680543, + "grad_norm": 2.817409087353589, + "learning_rate": 3.947098976109215e-06, + "loss": 1.2659, + "mean_token_accuracy": 0.691384556889534, + "num_tokens": 484962184.0, + "step": 3730 + }, + { + "entropy": 1.773046875, + "epoch": 0.6190515600430356, + "grad_norm": 2.226520647124414, + "learning_rate": 3.930034129692833e-06, + "loss": 1.2474, + "mean_token_accuracy": 0.6959296852350235, + "num_tokens": 486260811.0, + "step": 3740 + }, + { + "entropy": 1.798046875, + "epoch": 0.620706778118017, + "grad_norm": 2.5735228501076555, + "learning_rate": 3.912969283276451e-06, + "loss": 1.3236, + "mean_token_accuracy": 0.6822678565979003, + "num_tokens": 487561531.0, + "step": 3750 + }, + { + "entropy": 1.779296875, + "epoch": 0.6223619961929985, + "grad_norm": 2.447174533520884, + "learning_rate": 3.895904436860069e-06, + "loss": 1.2958, + "mean_token_accuracy": 0.6845273405313492, + "num_tokens": 488861951.0, + "step": 3760 + }, + { + "entropy": 1.78984375, + "epoch": 0.6240172142679798, + "grad_norm": 2.4781939483520214, + "learning_rate": 3.878839590443687e-06, + "loss": 1.2632, + "mean_token_accuracy": 0.6936646103858948, + "num_tokens": 490163501.0, + "step": 3770 + }, + { + "entropy": 1.7921875, + "epoch": 0.6256724323429612, + "grad_norm": 2.519562363065026, + "learning_rate": 3.861774744027304e-06, + "loss": 1.3081, + "mean_token_accuracy": 0.6813121765851975, + "num_tokens": 491460058.0, + "step": 3780 + }, + { + "entropy": 1.7671875, + "epoch": 0.6273276504179426, + "grad_norm": 2.265734211520595, + "learning_rate": 3.844709897610922e-06, + "loss": 1.2758, + "mean_token_accuracy": 0.6888975769281387, + "num_tokens": 492759704.0, + "step": 3790 + }, + { + "entropy": 1.790625, + "epoch": 0.628982868492924, + "grad_norm": 2.3801180758463207, + "learning_rate": 3.827645051194539e-06, + "loss": 1.3012, + "mean_token_accuracy": 0.683932626247406, + "num_tokens": 494058961.0, + "step": 3800 + }, + { + "epoch": 0.628982868492924, + "eval_entropy": 1.7723214285714286, + "eval_loss": 1.2402316331863403, + "eval_mean_token_accuracy": 0.6955270596912929, + "eval_num_tokens": 494058961.0, + "eval_runtime": 4.6291, + "eval_samples_per_second": 94.618, + "eval_steps_per_second": 1.512, + "step": 3800 + }, + { + "entropy": 1.800390625, + "epoch": 0.6306380865679053, + "grad_norm": 2.389274520435617, + "learning_rate": 3.810580204778157e-06, + "loss": 1.3079, + "mean_token_accuracy": 0.6813450068235397, + "num_tokens": 495358317.0, + "step": 3810 + }, + { + "entropy": 1.776171875, + "epoch": 0.6322933046428867, + "grad_norm": 2.6168551657979173, + "learning_rate": 3.793515358361775e-06, + "loss": 1.3105, + "mean_token_accuracy": 0.6829809457063675, + "num_tokens": 496659564.0, + "step": 3820 + }, + { + "entropy": 1.80078125, + "epoch": 0.6339485227178681, + "grad_norm": 2.4837099358753214, + "learning_rate": 3.7764505119453924e-06, + "loss": 1.3043, + "mean_token_accuracy": 0.6842409640550613, + "num_tokens": 497959970.0, + "step": 3830 + }, + { + "entropy": 1.7890625, + "epoch": 0.6356037407928494, + "grad_norm": 2.462238064482019, + "learning_rate": 3.7593856655290106e-06, + "loss": 1.274, + "mean_token_accuracy": 0.6902779757976532, + "num_tokens": 499260147.0, + "step": 3840 + }, + { + "entropy": 1.787109375, + "epoch": 0.6372589588678308, + "grad_norm": 2.5367590916494778, + "learning_rate": 3.7423208191126282e-06, + "loss": 1.3001, + "mean_token_accuracy": 0.685394424200058, + "num_tokens": 500561708.0, + "step": 3850 + }, + { + "entropy": 1.798828125, + "epoch": 0.6389141769428122, + "grad_norm": 2.5557035511771056, + "learning_rate": 3.7252559726962464e-06, + "loss": 1.2937, + "mean_token_accuracy": 0.6848995119333268, + "num_tokens": 501864915.0, + "step": 3860 + }, + { + "entropy": 1.76015625, + "epoch": 0.6405693950177936, + "grad_norm": 2.5823767016291046, + "learning_rate": 3.7081911262798636e-06, + "loss": 1.2834, + "mean_token_accuracy": 0.6893062323331833, + "num_tokens": 503164142.0, + "step": 3870 + }, + { + "entropy": 1.7703125, + "epoch": 0.642224613092775, + "grad_norm": 2.4148430310975817, + "learning_rate": 3.6911262798634817e-06, + "loss": 1.265, + "mean_token_accuracy": 0.6926558434963226, + "num_tokens": 504466371.0, + "step": 3880 + }, + { + "entropy": 1.763671875, + "epoch": 0.6438798311677564, + "grad_norm": 2.4225120808688536, + "learning_rate": 3.674061433447099e-06, + "loss": 1.2865, + "mean_token_accuracy": 0.6877583533525466, + "num_tokens": 505767882.0, + "step": 3890 + }, + { + "entropy": 1.764453125, + "epoch": 0.6455350492427377, + "grad_norm": 2.9462812132859324, + "learning_rate": 3.656996587030717e-06, + "loss": 1.2915, + "mean_token_accuracy": 0.6885056257247925, + "num_tokens": 507069225.0, + "step": 3900 + }, + { + "epoch": 0.6455350492427377, + "eval_entropy": 1.7645089285714286, + "eval_loss": 1.2312456369400024, + "eval_mean_token_accuracy": 0.6976498706000192, + "eval_num_tokens": 507069225.0, + "eval_runtime": 4.6273, + "eval_samples_per_second": 94.656, + "eval_steps_per_second": 1.513, + "step": 3900 + }, + { + "entropy": 1.793359375, + "epoch": 0.6471902673177191, + "grad_norm": 2.565460829563857, + "learning_rate": 3.639931740614335e-06, + "loss": 1.2724, + "mean_token_accuracy": 0.6899377435445786, + "num_tokens": 508367116.0, + "step": 3910 + }, + { + "entropy": 1.75546875, + "epoch": 0.6488454853927005, + "grad_norm": 2.4364788519616347, + "learning_rate": 3.622866894197953e-06, + "loss": 1.2546, + "mean_token_accuracy": 0.6945523023605347, + "num_tokens": 509666752.0, + "step": 3920 + }, + { + "entropy": 1.783203125, + "epoch": 0.6505007034676819, + "grad_norm": 2.682827780609857, + "learning_rate": 3.60580204778157e-06, + "loss": 1.2916, + "mean_token_accuracy": 0.6872588962316513, + "num_tokens": 510967470.0, + "step": 3930 + }, + { + "entropy": 1.768359375, + "epoch": 0.6521559215426632, + "grad_norm": 2.505180818791301, + "learning_rate": 3.588737201365188e-06, + "loss": 1.3186, + "mean_token_accuracy": 0.6803337544202804, + "num_tokens": 512267944.0, + "step": 3940 + }, + { + "entropy": 1.762890625, + "epoch": 0.6538111396176446, + "grad_norm": 2.7620551382084577, + "learning_rate": 3.571672354948806e-06, + "loss": 1.2392, + "mean_token_accuracy": 0.6974693089723587, + "num_tokens": 513565434.0, + "step": 3950 + }, + { + "entropy": 1.772265625, + "epoch": 0.655466357692626, + "grad_norm": 2.497777317257079, + "learning_rate": 3.5546075085324233e-06, + "loss": 1.2846, + "mean_token_accuracy": 0.6888549089431762, + "num_tokens": 514865208.0, + "step": 3960 + }, + { + "entropy": 1.7875, + "epoch": 0.6571215757676074, + "grad_norm": 2.5164869720311427, + "learning_rate": 3.5375426621160414e-06, + "loss": 1.2578, + "mean_token_accuracy": 0.694903028011322, + "num_tokens": 516167457.0, + "step": 3970 + }, + { + "entropy": 1.7953125, + "epoch": 0.6587767938425888, + "grad_norm": 2.5590798094366867, + "learning_rate": 3.520477815699659e-06, + "loss": 1.2693, + "mean_token_accuracy": 0.6902157008647919, + "num_tokens": 517468370.0, + "step": 3980 + }, + { + "entropy": 1.790234375, + "epoch": 0.6604320119175702, + "grad_norm": 2.5198694448149324, + "learning_rate": 3.5034129692832767e-06, + "loss": 1.2958, + "mean_token_accuracy": 0.6859830439090728, + "num_tokens": 518768058.0, + "step": 3990 + }, + { + "entropy": 1.78828125, + "epoch": 0.6620872299925515, + "grad_norm": 2.6820619603914038, + "learning_rate": 3.4863481228668944e-06, + "loss": 1.277, + "mean_token_accuracy": 0.6900451958179474, + "num_tokens": 520070630.0, + "step": 4000 + }, + { + "epoch": 0.6620872299925515, + "eval_entropy": 1.7700892857142858, + "eval_loss": 1.2253153324127197, + "eval_mean_token_accuracy": 0.7001319016729083, + "eval_num_tokens": 520070630.0, + "eval_runtime": 4.6477, + "eval_samples_per_second": 94.241, + "eval_steps_per_second": 1.506, + "step": 4000 + }, + { + "entropy": 1.78984375, + "epoch": 0.6637424480675329, + "grad_norm": 2.6810540105689946, + "learning_rate": 3.4692832764505126e-06, + "loss": 1.2987, + "mean_token_accuracy": 0.6859483391046524, + "num_tokens": 521373078.0, + "step": 4010 + }, + { + "entropy": 1.78359375, + "epoch": 0.6653976661425143, + "grad_norm": 2.2587728217077827, + "learning_rate": 3.45221843003413e-06, + "loss": 1.3004, + "mean_token_accuracy": 0.6832431077957153, + "num_tokens": 522673704.0, + "step": 4020 + }, + { + "entropy": 1.77734375, + "epoch": 0.6670528842174956, + "grad_norm": 2.593555939923691, + "learning_rate": 3.435153583617748e-06, + "loss": 1.256, + "mean_token_accuracy": 0.6960073351860047, + "num_tokens": 523972839.0, + "step": 4030 + }, + { + "entropy": 1.7734375, + "epoch": 0.668708102292477, + "grad_norm": 2.3770300202655807, + "learning_rate": 3.4180887372013656e-06, + "loss": 1.2773, + "mean_token_accuracy": 0.6914051204919816, + "num_tokens": 525274687.0, + "step": 4040 + }, + { + "entropy": 1.763671875, + "epoch": 0.6703633203674584, + "grad_norm": 2.8480260935632282, + "learning_rate": 3.401023890784983e-06, + "loss": 1.2567, + "mean_token_accuracy": 0.6958839118480682, + "num_tokens": 526574689.0, + "step": 4050 + }, + { + "entropy": 1.790234375, + "epoch": 0.6720185384424398, + "grad_norm": 2.3851622892604127, + "learning_rate": 3.383959044368601e-06, + "loss": 1.3012, + "mean_token_accuracy": 0.6852335572242737, + "num_tokens": 527876009.0, + "step": 4060 + }, + { + "entropy": 1.775390625, + "epoch": 0.6736737565174211, + "grad_norm": 2.5290431696690012, + "learning_rate": 3.3668941979522187e-06, + "loss": 1.3086, + "mean_token_accuracy": 0.6832914292812348, + "num_tokens": 529177024.0, + "step": 4070 + }, + { + "entropy": 1.780859375, + "epoch": 0.6753289745924026, + "grad_norm": 2.4279229429301132, + "learning_rate": 3.3498293515358364e-06, + "loss": 1.2572, + "mean_token_accuracy": 0.6937553316354752, + "num_tokens": 530479823.0, + "step": 4080 + }, + { + "entropy": 1.773828125, + "epoch": 0.676984192667384, + "grad_norm": 2.4527193592967453, + "learning_rate": 3.332764505119454e-06, + "loss": 1.2739, + "mean_token_accuracy": 0.6902639389038085, + "num_tokens": 531780941.0, + "step": 4090 + }, + { + "entropy": 1.769140625, + "epoch": 0.6786394107423653, + "grad_norm": 2.281918205687874, + "learning_rate": 3.315699658703072e-06, + "loss": 1.298, + "mean_token_accuracy": 0.6876232415437699, + "num_tokens": 533081218.0, + "step": 4100 + }, + { + "epoch": 0.6786394107423653, + "eval_entropy": 1.7790178571428572, + "eval_loss": 1.2168325185775757, + "eval_mean_token_accuracy": 0.7016116891588483, + "eval_num_tokens": 533081218.0, + "eval_runtime": 4.6347, + "eval_samples_per_second": 94.505, + "eval_steps_per_second": 1.51, + "step": 4100 + }, + { + "entropy": 1.78984375, + "epoch": 0.6802946288173467, + "grad_norm": 2.5227123053180898, + "learning_rate": 3.2986348122866894e-06, + "loss": 1.2856, + "mean_token_accuracy": 0.6883559882640838, + "num_tokens": 534379489.0, + "step": 4110 + }, + { + "entropy": 1.7828125, + "epoch": 0.6819498468923281, + "grad_norm": 2.630391942926362, + "learning_rate": 3.2815699658703076e-06, + "loss": 1.2984, + "mean_token_accuracy": 0.6847425669431686, + "num_tokens": 535679789.0, + "step": 4120 + }, + { + "entropy": 1.768359375, + "epoch": 0.6836050649673094, + "grad_norm": 2.4176699346985773, + "learning_rate": 3.2645051194539253e-06, + "loss": 1.2695, + "mean_token_accuracy": 0.6933052957057952, + "num_tokens": 536982731.0, + "step": 4130 + }, + { + "entropy": 1.775, + "epoch": 0.6852602830422908, + "grad_norm": 2.724888227177248, + "learning_rate": 3.2474402730375434e-06, + "loss": 1.2505, + "mean_token_accuracy": 0.6960378557443618, + "num_tokens": 538282388.0, + "step": 4140 + }, + { + "entropy": 1.75703125, + "epoch": 0.6869155011172722, + "grad_norm": 2.3271884939966414, + "learning_rate": 3.2303754266211606e-06, + "loss": 1.2636, + "mean_token_accuracy": 0.6938478142023087, + "num_tokens": 539581542.0, + "step": 4150 + }, + { + "entropy": 1.7734375, + "epoch": 0.6885707191922535, + "grad_norm": 2.3636341549823414, + "learning_rate": 3.2133105802047783e-06, + "loss": 1.2825, + "mean_token_accuracy": 0.6883620142936706, + "num_tokens": 540881133.0, + "step": 4160 + }, + { + "entropy": 1.76015625, + "epoch": 0.6902259372672349, + "grad_norm": 2.5295761625670203, + "learning_rate": 3.196245733788396e-06, + "loss": 1.2529, + "mean_token_accuracy": 0.69583360850811, + "num_tokens": 542183231.0, + "step": 4170 + }, + { + "entropy": 1.76484375, + "epoch": 0.6918811553422164, + "grad_norm": 2.7549985947980558, + "learning_rate": 3.1791808873720137e-06, + "loss": 1.293, + "mean_token_accuracy": 0.6881015658378601, + "num_tokens": 543483649.0, + "step": 4180 + }, + { + "entropy": 1.755859375, + "epoch": 0.6935363734171978, + "grad_norm": 2.096072229476976, + "learning_rate": 3.162116040955632e-06, + "loss": 1.2776, + "mean_token_accuracy": 0.6914682954549789, + "num_tokens": 544786904.0, + "step": 4190 + }, + { + "entropy": 1.762890625, + "epoch": 0.6951915914921791, + "grad_norm": 2.382657849169538, + "learning_rate": 3.145051194539249e-06, + "loss": 1.2592, + "mean_token_accuracy": 0.694176259636879, + "num_tokens": 546087106.0, + "step": 4200 + }, + { + "epoch": 0.6951915914921791, + "eval_entropy": 1.7566964285714286, + "eval_loss": 1.2030233144760132, + "eval_mean_token_accuracy": 0.7050757493291583, + "eval_num_tokens": 546087106.0, + "eval_runtime": 4.6346, + "eval_samples_per_second": 94.506, + "eval_steps_per_second": 1.51, + "step": 4200 + }, + { + "entropy": 1.770703125, + "epoch": 0.6968468095671605, + "grad_norm": 2.552516836595105, + "learning_rate": 3.127986348122867e-06, + "loss": 1.2774, + "mean_token_accuracy": 0.6900360703468322, + "num_tokens": 547387657.0, + "step": 4210 + }, + { + "entropy": 1.7796875, + "epoch": 0.6985020276421419, + "grad_norm": 2.5452673124911653, + "learning_rate": 3.110921501706485e-06, + "loss": 1.292, + "mean_token_accuracy": 0.6870338559150696, + "num_tokens": 548687865.0, + "step": 4220 + }, + { + "entropy": 1.750390625, + "epoch": 0.7001572457171232, + "grad_norm": 2.5771885640482903, + "learning_rate": 3.093856655290103e-06, + "loss": 1.1953, + "mean_token_accuracy": 0.7086612612009049, + "num_tokens": 549986829.0, + "step": 4230 + }, + { + "entropy": 1.748046875, + "epoch": 0.7018124637921046, + "grad_norm": 2.477052333275593, + "learning_rate": 3.0767918088737203e-06, + "loss": 1.254, + "mean_token_accuracy": 0.6968858271837235, + "num_tokens": 551287936.0, + "step": 4240 + }, + { + "entropy": 1.756640625, + "epoch": 0.703467681867086, + "grad_norm": 2.23510805750647, + "learning_rate": 3.0597269624573384e-06, + "loss": 1.2486, + "mean_token_accuracy": 0.6987336874008179, + "num_tokens": 552588492.0, + "step": 4250 + }, + { + "entropy": 1.749609375, + "epoch": 0.7051228999420673, + "grad_norm": 2.319349853459115, + "learning_rate": 3.0426621160409556e-06, + "loss": 1.2518, + "mean_token_accuracy": 0.6969301968812942, + "num_tokens": 553888162.0, + "step": 4260 + }, + { + "entropy": 1.750390625, + "epoch": 0.7067781180170487, + "grad_norm": 2.5540084168546158, + "learning_rate": 3.0255972696245738e-06, + "loss": 1.2238, + "mean_token_accuracy": 0.702789381146431, + "num_tokens": 555187588.0, + "step": 4270 + }, + { + "entropy": 1.780078125, + "epoch": 0.7084333360920301, + "grad_norm": 2.40690080015461, + "learning_rate": 3.0085324232081914e-06, + "loss": 1.3117, + "mean_token_accuracy": 0.6837745904922485, + "num_tokens": 556490332.0, + "step": 4280 + }, + { + "entropy": 1.756640625, + "epoch": 0.7100885541670116, + "grad_norm": 2.431898158382137, + "learning_rate": 2.9914675767918087e-06, + "loss": 1.2531, + "mean_token_accuracy": 0.6963717520236969, + "num_tokens": 557788397.0, + "step": 4290 + }, + { + "entropy": 1.777734375, + "epoch": 0.7117437722419929, + "grad_norm": 2.754425167088839, + "learning_rate": 2.974402730375427e-06, + "loss": 1.2764, + "mean_token_accuracy": 0.6893212735652924, + "num_tokens": 559088280.0, + "step": 4300 + }, + { + "epoch": 0.7117437722419929, + "eval_entropy": 1.7332589285714286, + "eval_loss": 1.1991610527038574, + "eval_mean_token_accuracy": 0.7060475519725254, + "eval_num_tokens": 559088280.0, + "eval_runtime": 4.6363, + "eval_samples_per_second": 94.472, + "eval_steps_per_second": 1.51, + "step": 4300 + }, + { + "entropy": 1.75390625, + "epoch": 0.7133989903169743, + "grad_norm": 2.7751127777787397, + "learning_rate": 2.9573378839590445e-06, + "loss": 1.2731, + "mean_token_accuracy": 0.6897863805294037, + "num_tokens": 560387442.0, + "step": 4310 + }, + { + "entropy": 1.76875, + "epoch": 0.7150542083919557, + "grad_norm": 2.5697243566958035, + "learning_rate": 2.9402730375426626e-06, + "loss": 1.2891, + "mean_token_accuracy": 0.6885288000106812, + "num_tokens": 561688041.0, + "step": 4320 + }, + { + "entropy": 1.772265625, + "epoch": 0.716709426466937, + "grad_norm": 2.7945730285726245, + "learning_rate": 2.92320819112628e-06, + "loss": 1.2619, + "mean_token_accuracy": 0.693383127450943, + "num_tokens": 562986759.0, + "step": 4330 + }, + { + "entropy": 1.7640625, + "epoch": 0.7183646445419184, + "grad_norm": 2.5141949245877138, + "learning_rate": 2.906143344709898e-06, + "loss": 1.2712, + "mean_token_accuracy": 0.6909885942935944, + "num_tokens": 564287509.0, + "step": 4340 + }, + { + "entropy": 1.74921875, + "epoch": 0.7200198626168998, + "grad_norm": 2.3903562704134855, + "learning_rate": 2.8890784982935157e-06, + "loss": 1.2523, + "mean_token_accuracy": 0.6963708102703094, + "num_tokens": 565588409.0, + "step": 4350 + }, + { + "entropy": 1.747265625, + "epoch": 0.7216750806918811, + "grad_norm": 2.6449879459569856, + "learning_rate": 2.8720136518771334e-06, + "loss": 1.2603, + "mean_token_accuracy": 0.6954372197389602, + "num_tokens": 566888759.0, + "step": 4360 + }, + { + "entropy": 1.7734375, + "epoch": 0.7233302987668625, + "grad_norm": 2.286701287064059, + "learning_rate": 2.854948805460751e-06, + "loss": 1.2319, + "mean_token_accuracy": 0.7022051870822906, + "num_tokens": 568190244.0, + "step": 4370 + }, + { + "entropy": 1.75703125, + "epoch": 0.7249855168418439, + "grad_norm": 2.5083895920809796, + "learning_rate": 2.837883959044369e-06, + "loss": 1.256, + "mean_token_accuracy": 0.6956620275974273, + "num_tokens": 569491818.0, + "step": 4380 + }, + { + "entropy": 1.766796875, + "epoch": 0.7266407349168252, + "grad_norm": 2.331149409901928, + "learning_rate": 2.8208191126279865e-06, + "loss": 1.2492, + "mean_token_accuracy": 0.6981422275304794, + "num_tokens": 570792064.0, + "step": 4390 + }, + { + "entropy": 1.75, + "epoch": 0.7282959529918067, + "grad_norm": 2.376076120887949, + "learning_rate": 2.803754266211604e-06, + "loss": 1.2292, + "mean_token_accuracy": 0.7025991410017014, + "num_tokens": 572092803.0, + "step": 4400 + }, + { + "epoch": 0.7282959529918067, + "eval_entropy": 1.7410714285714286, + "eval_loss": 1.193205714225769, + "eval_mean_token_accuracy": 0.7080071994236538, + "eval_num_tokens": 572092803.0, + "eval_runtime": 4.641, + "eval_samples_per_second": 94.376, + "eval_steps_per_second": 1.508, + "step": 4400 + }, + { + "entropy": 1.753125, + "epoch": 0.7299511710667881, + "grad_norm": 2.515631236274595, + "learning_rate": 2.7866894197952223e-06, + "loss": 1.2084, + "mean_token_accuracy": 0.7063405781984329, + "num_tokens": 573390170.0, + "step": 4410 + }, + { + "entropy": 1.7625, + "epoch": 0.7316063891417695, + "grad_norm": 2.3462644620311726, + "learning_rate": 2.7696245733788395e-06, + "loss": 1.2486, + "mean_token_accuracy": 0.6986642211675644, + "num_tokens": 574689954.0, + "step": 4420 + }, + { + "entropy": 1.744921875, + "epoch": 0.7332616072167508, + "grad_norm": 2.5581477788952793, + "learning_rate": 2.7525597269624576e-06, + "loss": 1.2311, + "mean_token_accuracy": 0.7014442980289459, + "num_tokens": 575991011.0, + "step": 4430 + }, + { + "entropy": 1.776171875, + "epoch": 0.7349168252917322, + "grad_norm": 2.4889835522309056, + "learning_rate": 2.7354948805460753e-06, + "loss": 1.2691, + "mean_token_accuracy": 0.6928168714046479, + "num_tokens": 577290521.0, + "step": 4440 + }, + { + "entropy": 1.74453125, + "epoch": 0.7365720433667136, + "grad_norm": 2.54796810477095, + "learning_rate": 2.718430034129693e-06, + "loss": 1.2679, + "mean_token_accuracy": 0.6912356495857239, + "num_tokens": 578590616.0, + "step": 4450 + }, + { + "entropy": 1.76640625, + "epoch": 0.7382272614416949, + "grad_norm": 2.5990075296474893, + "learning_rate": 2.7013651877133107e-06, + "loss": 1.2339, + "mean_token_accuracy": 0.6998564571142196, + "num_tokens": 579891138.0, + "step": 4460 + }, + { + "entropy": 1.7359375, + "epoch": 0.7398824795166763, + "grad_norm": 2.4333220958494977, + "learning_rate": 2.684300341296929e-06, + "loss": 1.2613, + "mean_token_accuracy": 0.6963325083255768, + "num_tokens": 581189690.0, + "step": 4470 + }, + { + "entropy": 1.749609375, + "epoch": 0.7415376975916577, + "grad_norm": 2.390817792453385, + "learning_rate": 2.667235494880546e-06, + "loss": 1.2006, + "mean_token_accuracy": 0.7076130002737046, + "num_tokens": 582489917.0, + "step": 4480 + }, + { + "entropy": 1.730078125, + "epoch": 0.743192915666639, + "grad_norm": 2.476190623339874, + "learning_rate": 2.650170648464164e-06, + "loss": 1.2407, + "mean_token_accuracy": 0.701440492272377, + "num_tokens": 583789982.0, + "step": 4490 + }, + { + "entropy": 1.772265625, + "epoch": 0.7448481337416205, + "grad_norm": 2.5558887717849252, + "learning_rate": 2.633105802047782e-06, + "loss": 1.2562, + "mean_token_accuracy": 0.6959700375795365, + "num_tokens": 585091467.0, + "step": 4500 + }, + { + "epoch": 0.7448481337416205, + "eval_entropy": 1.7354910714285714, + "eval_loss": 1.1860095262527466, + "eval_mean_token_accuracy": 0.708976013319833, + "eval_num_tokens": 585091467.0, + "eval_runtime": 4.6247, + "eval_samples_per_second": 94.708, + "eval_steps_per_second": 1.514, + "step": 4500 + }, + { + "entropy": 1.755859375, + "epoch": 0.7465033518166019, + "grad_norm": 2.3322020567081534, + "learning_rate": 2.616040955631399e-06, + "loss": 1.2575, + "mean_token_accuracy": 0.6937162548303604, + "num_tokens": 586391682.0, + "step": 4510 + }, + { + "entropy": 1.759765625, + "epoch": 0.7481585698915832, + "grad_norm": 2.3971071240982185, + "learning_rate": 2.5989761092150173e-06, + "loss": 1.2617, + "mean_token_accuracy": 0.695154020190239, + "num_tokens": 587693126.0, + "step": 4520 + }, + { + "entropy": 1.74453125, + "epoch": 0.7498137879665646, + "grad_norm": 2.481216562732887, + "learning_rate": 2.581911262798635e-06, + "loss": 1.2323, + "mean_token_accuracy": 0.7014958649873734, + "num_tokens": 588995123.0, + "step": 4530 + }, + { + "entropy": 1.753125, + "epoch": 0.751469006041546, + "grad_norm": 2.4286454976552756, + "learning_rate": 2.5648464163822526e-06, + "loss": 1.2409, + "mean_token_accuracy": 0.7000439822673797, + "num_tokens": 590295241.0, + "step": 4540 + }, + { + "entropy": 1.754296875, + "epoch": 0.7531242241165274, + "grad_norm": 2.6241311020576705, + "learning_rate": 2.5477815699658703e-06, + "loss": 1.2487, + "mean_token_accuracy": 0.6975617289543152, + "num_tokens": 591593030.0, + "step": 4550 + }, + { + "entropy": 1.7203125, + "epoch": 0.7547794421915087, + "grad_norm": 2.248123149216047, + "learning_rate": 2.5307167235494884e-06, + "loss": 1.2254, + "mean_token_accuracy": 0.7026310443878174, + "num_tokens": 592893147.0, + "step": 4560 + }, + { + "entropy": 1.749609375, + "epoch": 0.7564346602664901, + "grad_norm": 2.344497387555351, + "learning_rate": 2.5136518771331057e-06, + "loss": 1.2352, + "mean_token_accuracy": 0.7007616192102433, + "num_tokens": 594193212.0, + "step": 4570 + }, + { + "entropy": 1.75703125, + "epoch": 0.7580898783414715, + "grad_norm": 2.5143023839512404, + "learning_rate": 2.496587030716724e-06, + "loss": 1.277, + "mean_token_accuracy": 0.6920879483222961, + "num_tokens": 595492832.0, + "step": 4580 + }, + { + "entropy": 1.744140625, + "epoch": 0.7597450964164528, + "grad_norm": 2.616947952252582, + "learning_rate": 2.4795221843003415e-06, + "loss": 1.2571, + "mean_token_accuracy": 0.6941735863685607, + "num_tokens": 596793582.0, + "step": 4590 + }, + { + "entropy": 1.744921875, + "epoch": 0.7614003144914342, + "grad_norm": 3.142140767125069, + "learning_rate": 2.462457337883959e-06, + "loss": 1.2256, + "mean_token_accuracy": 0.701491180062294, + "num_tokens": 598095276.0, + "step": 4600 + }, + { + "epoch": 0.7614003144914342, + "eval_entropy": 1.7176339285714286, + "eval_loss": 1.1813000440597534, + "eval_mean_token_accuracy": 0.7109173195702689, + "eval_num_tokens": 598095276.0, + "eval_runtime": 4.649, + "eval_samples_per_second": 94.213, + "eval_steps_per_second": 1.506, + "step": 4600 + }, + { + "entropy": 1.741796875, + "epoch": 0.7630555325664157, + "grad_norm": 2.4778943375543436, + "learning_rate": 2.445392491467577e-06, + "loss": 1.2458, + "mean_token_accuracy": 0.6977786660194397, + "num_tokens": 599396996.0, + "step": 4610 + }, + { + "entropy": 1.722265625, + "epoch": 0.764710750641397, + "grad_norm": 2.5147907725884324, + "learning_rate": 2.4283276450511946e-06, + "loss": 1.224, + "mean_token_accuracy": 0.702233475446701, + "num_tokens": 600698777.0, + "step": 4620 + }, + { + "entropy": 1.7328125, + "epoch": 0.7663659687163784, + "grad_norm": 2.6239489587931306, + "learning_rate": 2.4112627986348127e-06, + "loss": 1.2269, + "mean_token_accuracy": 0.7043916165828705, + "num_tokens": 602000853.0, + "step": 4630 + }, + { + "entropy": 1.74375, + "epoch": 0.7680211867913598, + "grad_norm": 2.3874629037925237, + "learning_rate": 2.3941979522184304e-06, + "loss": 1.2641, + "mean_token_accuracy": 0.694331356883049, + "num_tokens": 603300386.0, + "step": 4640 + }, + { + "entropy": 1.734765625, + "epoch": 0.7696764048663411, + "grad_norm": 2.4354900075205435, + "learning_rate": 2.377133105802048e-06, + "loss": 1.2205, + "mean_token_accuracy": 0.7044494062662124, + "num_tokens": 604599542.0, + "step": 4650 + }, + { + "entropy": 1.741015625, + "epoch": 0.7713316229413225, + "grad_norm": 2.652100570389199, + "learning_rate": 2.3600682593856658e-06, + "loss": 1.219, + "mean_token_accuracy": 0.7054125666618347, + "num_tokens": 605902216.0, + "step": 4660 + }, + { + "entropy": 1.73984375, + "epoch": 0.7729868410163039, + "grad_norm": 2.581315697912295, + "learning_rate": 2.3430034129692835e-06, + "loss": 1.2428, + "mean_token_accuracy": 0.7006408751010895, + "num_tokens": 607203127.0, + "step": 4670 + }, + { + "entropy": 1.728515625, + "epoch": 0.7746420590912853, + "grad_norm": 2.3698535637016844, + "learning_rate": 2.325938566552901e-06, + "loss": 1.206, + "mean_token_accuracy": 0.7037875890731812, + "num_tokens": 608504076.0, + "step": 4680 + }, + { + "entropy": 1.731640625, + "epoch": 0.7762972771662666, + "grad_norm": 2.415911545350536, + "learning_rate": 2.308873720136519e-06, + "loss": 1.2221, + "mean_token_accuracy": 0.7028371393680573, + "num_tokens": 609804563.0, + "step": 4690 + }, + { + "entropy": 1.734765625, + "epoch": 0.777952495241248, + "grad_norm": 2.487078386506121, + "learning_rate": 2.2918088737201365e-06, + "loss": 1.2222, + "mean_token_accuracy": 0.703727638721466, + "num_tokens": 611105768.0, + "step": 4700 + }, + { + "epoch": 0.777952495241248, + "eval_entropy": 1.7321428571428572, + "eval_loss": 1.1747254133224487, + "eval_mean_token_accuracy": 0.7124179686818805, + "eval_num_tokens": 611105768.0, + "eval_runtime": 4.626, + "eval_samples_per_second": 94.683, + "eval_steps_per_second": 1.513, + "step": 4700 + }, + { + "entropy": 1.74140625, + "epoch": 0.7796077133162295, + "grad_norm": 2.501756118440677, + "learning_rate": 2.2747440273037542e-06, + "loss": 1.2184, + "mean_token_accuracy": 0.702805009484291, + "num_tokens": 612406499.0, + "step": 4710 + }, + { + "entropy": 1.730859375, + "epoch": 0.7812629313912108, + "grad_norm": 2.492791390583526, + "learning_rate": 2.2576791808873723e-06, + "loss": 1.2343, + "mean_token_accuracy": 0.703232204914093, + "num_tokens": 613706312.0, + "step": 4720 + }, + { + "entropy": 1.73046875, + "epoch": 0.7829181494661922, + "grad_norm": 2.5509425664152334, + "learning_rate": 2.24061433447099e-06, + "loss": 1.2196, + "mean_token_accuracy": 0.7046588331460952, + "num_tokens": 615007089.0, + "step": 4730 + }, + { + "entropy": 1.745703125, + "epoch": 0.7845733675411736, + "grad_norm": 2.582099344558271, + "learning_rate": 2.2235494880546077e-06, + "loss": 1.2245, + "mean_token_accuracy": 0.7022117048501968, + "num_tokens": 616306265.0, + "step": 4740 + }, + { + "entropy": 1.713671875, + "epoch": 0.7862285856161549, + "grad_norm": 2.541672472885552, + "learning_rate": 2.2064846416382254e-06, + "loss": 1.2521, + "mean_token_accuracy": 0.6983891487121582, + "num_tokens": 617607878.0, + "step": 4750 + }, + { + "entropy": 1.73046875, + "epoch": 0.7878838036911363, + "grad_norm": 2.710141164620272, + "learning_rate": 2.189419795221843e-06, + "loss": 1.2492, + "mean_token_accuracy": 0.7002591222524643, + "num_tokens": 618907062.0, + "step": 4760 + }, + { + "entropy": 1.720703125, + "epoch": 0.7895390217661177, + "grad_norm": 2.2998213547627047, + "learning_rate": 2.172354948805461e-06, + "loss": 1.2259, + "mean_token_accuracy": 0.7028292834758758, + "num_tokens": 620209236.0, + "step": 4770 + }, + { + "entropy": 1.740234375, + "epoch": 0.791194239841099, + "grad_norm": 2.4147755521948953, + "learning_rate": 2.155290102389079e-06, + "loss": 1.2495, + "mean_token_accuracy": 0.6991397470235825, + "num_tokens": 621510109.0, + "step": 4780 + }, + { + "entropy": 1.73359375, + "epoch": 0.7928494579160804, + "grad_norm": 2.481650515244591, + "learning_rate": 2.1382252559726966e-06, + "loss": 1.2281, + "mean_token_accuracy": 0.7010512501001358, + "num_tokens": 622810129.0, + "step": 4790 + }, + { + "entropy": 1.728125, + "epoch": 0.7945046759910618, + "grad_norm": 2.5965323241884777, + "learning_rate": 2.1211604095563143e-06, + "loss": 1.2347, + "mean_token_accuracy": 0.701545363664627, + "num_tokens": 624108203.0, + "step": 4800 + }, + { + "epoch": 0.7945046759910618, + "eval_entropy": 1.7220982142857142, + "eval_loss": 1.1706337928771973, + "eval_mean_token_accuracy": 0.7139174086706979, + "eval_num_tokens": 624108203.0, + "eval_runtime": 4.6471, + "eval_samples_per_second": 94.252, + "eval_steps_per_second": 1.506, + "step": 4800 + }, + { + "entropy": 1.72890625, + "epoch": 0.7961598940660432, + "grad_norm": 2.6834320181687823, + "learning_rate": 2.104095563139932e-06, + "loss": 1.2033, + "mean_token_accuracy": 0.706162941455841, + "num_tokens": 625409752.0, + "step": 4810 + }, + { + "entropy": 1.714453125, + "epoch": 0.7978151121410246, + "grad_norm": 2.7656742398218874, + "learning_rate": 2.0870307167235496e-06, + "loss": 1.2326, + "mean_token_accuracy": 0.6992395788431167, + "num_tokens": 626710727.0, + "step": 4820 + }, + { + "entropy": 1.7296875, + "epoch": 0.799470330216006, + "grad_norm": 2.417483829515833, + "learning_rate": 2.0699658703071673e-06, + "loss": 1.2238, + "mean_token_accuracy": 0.7033618688583374, + "num_tokens": 628008519.0, + "step": 4830 + }, + { + "entropy": 1.712109375, + "epoch": 0.8011255482909874, + "grad_norm": 2.7983458595582738, + "learning_rate": 2.052901023890785e-06, + "loss": 1.2226, + "mean_token_accuracy": 0.7046162158250808, + "num_tokens": 629309542.0, + "step": 4840 + }, + { + "entropy": 1.735546875, + "epoch": 0.8027807663659687, + "grad_norm": 2.461124399269518, + "learning_rate": 2.0358361774744027e-06, + "loss": 1.2449, + "mean_token_accuracy": 0.6981534391641617, + "num_tokens": 630607012.0, + "step": 4850 + }, + { + "entropy": 1.705078125, + "epoch": 0.8044359844409501, + "grad_norm": 2.30547900633082, + "learning_rate": 2.018771331058021e-06, + "loss": 1.2117, + "mean_token_accuracy": 0.705395695567131, + "num_tokens": 631909663.0, + "step": 4860 + }, + { + "entropy": 1.73359375, + "epoch": 0.8060912025159315, + "grad_norm": 2.3784927817676933, + "learning_rate": 2.0017064846416385e-06, + "loss": 1.2392, + "mean_token_accuracy": 0.6994657307863236, + "num_tokens": 633212385.0, + "step": 4870 + }, + { + "entropy": 1.737109375, + "epoch": 0.8077464205909128, + "grad_norm": 3.0219781141560476, + "learning_rate": 1.984641638225256e-06, + "loss": 1.2156, + "mean_token_accuracy": 0.706381943821907, + "num_tokens": 634512002.0, + "step": 4880 + }, + { + "entropy": 1.73125, + "epoch": 0.8094016386658942, + "grad_norm": 2.419740782497377, + "learning_rate": 1.967576791808874e-06, + "loss": 1.2206, + "mean_token_accuracy": 0.7045120120048523, + "num_tokens": 635811494.0, + "step": 4890 + }, + { + "entropy": 1.719140625, + "epoch": 0.8110568567408756, + "grad_norm": 2.7057503497275484, + "learning_rate": 1.9505119453924916e-06, + "loss": 1.2357, + "mean_token_accuracy": 0.700121134519577, + "num_tokens": 637115547.0, + "step": 4900 + }, + { + "epoch": 0.8110568567408756, + "eval_entropy": 1.6886160714285714, + "eval_loss": 1.167832374572754, + "eval_mean_token_accuracy": 0.7145262275423322, + "eval_num_tokens": 637115547.0, + "eval_runtime": 4.6417, + "eval_samples_per_second": 94.362, + "eval_steps_per_second": 1.508, + "step": 4900 + }, + { + "entropy": 1.690234375, + "epoch": 0.8127120748158569, + "grad_norm": 2.416264356515718, + "learning_rate": 1.9334470989761097e-06, + "loss": 1.2073, + "mean_token_accuracy": 0.7066628962755204, + "num_tokens": 638414864.0, + "step": 4910 + }, + { + "entropy": 1.713671875, + "epoch": 0.8143672928908384, + "grad_norm": 2.278990325527126, + "learning_rate": 1.916382252559727e-06, + "loss": 1.1944, + "mean_token_accuracy": 0.7096973121166229, + "num_tokens": 639716654.0, + "step": 4920 + }, + { + "entropy": 1.725390625, + "epoch": 0.8160225109658198, + "grad_norm": 2.28597480961386, + "learning_rate": 1.8993174061433447e-06, + "loss": 1.2208, + "mean_token_accuracy": 0.7041647166013718, + "num_tokens": 641014901.0, + "step": 4930 + }, + { + "entropy": 1.7171875, + "epoch": 0.8176777290408012, + "grad_norm": 2.4992518766179654, + "learning_rate": 1.8822525597269626e-06, + "loss": 1.2228, + "mean_token_accuracy": 0.7048165917396545, + "num_tokens": 642313130.0, + "step": 4940 + }, + { + "entropy": 1.72734375, + "epoch": 0.8193329471157825, + "grad_norm": 2.380047184338535, + "learning_rate": 1.8651877133105803e-06, + "loss": 1.2131, + "mean_token_accuracy": 0.7051455497741699, + "num_tokens": 643613932.0, + "step": 4950 + }, + { + "entropy": 1.726953125, + "epoch": 0.8209881651907639, + "grad_norm": 2.52794062489713, + "learning_rate": 1.8481228668941982e-06, + "loss": 1.1828, + "mean_token_accuracy": 0.7140951514244079, + "num_tokens": 644915495.0, + "step": 4960 + }, + { + "entropy": 1.705859375, + "epoch": 0.8226433832657453, + "grad_norm": 2.400013866562432, + "learning_rate": 1.8310580204778158e-06, + "loss": 1.2235, + "mean_token_accuracy": 0.7036207258701325, + "num_tokens": 646214772.0, + "step": 4970 + }, + { + "entropy": 1.7328125, + "epoch": 0.8242986013407266, + "grad_norm": 2.4970560324849544, + "learning_rate": 1.8139931740614335e-06, + "loss": 1.2254, + "mean_token_accuracy": 0.701385423541069, + "num_tokens": 647514673.0, + "step": 4980 + }, + { + "entropy": 1.723828125, + "epoch": 0.825953819415708, + "grad_norm": 2.409134715019309, + "learning_rate": 1.7969283276450514e-06, + "loss": 1.2296, + "mean_token_accuracy": 0.7022872388362884, + "num_tokens": 648813144.0, + "step": 4990 + }, + { + "entropy": 1.701171875, + "epoch": 0.8276090374906894, + "grad_norm": 2.4503447697111866, + "learning_rate": 1.7798634812286691e-06, + "loss": 1.227, + "mean_token_accuracy": 0.7024197041988373, + "num_tokens": 650112866.0, + "step": 5000 + }, + { + "epoch": 0.8276090374906894, + "eval_entropy": 1.6953125, + "eval_loss": 1.1605145931243896, + "eval_mean_token_accuracy": 0.7159572754587445, + "eval_num_tokens": 650112866.0, + "eval_runtime": 4.6405, + "eval_samples_per_second": 94.387, + "eval_steps_per_second": 1.508, + "step": 5000 + }, + { + "entropy": 1.72890625, + "epoch": 0.8292642555656707, + "grad_norm": 2.1638407195235927, + "learning_rate": 1.7627986348122868e-06, + "loss": 1.2627, + "mean_token_accuracy": 0.6967393845319748, + "num_tokens": 651413556.0, + "step": 5010 + }, + { + "entropy": 1.709765625, + "epoch": 0.8309194736406521, + "grad_norm": 2.529792864108116, + "learning_rate": 1.7457337883959047e-06, + "loss": 1.217, + "mean_token_accuracy": 0.7071486830711364, + "num_tokens": 652710944.0, + "step": 5020 + }, + { + "entropy": 1.719921875, + "epoch": 0.8325746917156336, + "grad_norm": 2.591470838857573, + "learning_rate": 1.7286689419795224e-06, + "loss": 1.2107, + "mean_token_accuracy": 0.7071384876966477, + "num_tokens": 654009687.0, + "step": 5030 + }, + { + "entropy": 1.704296875, + "epoch": 0.834229909790615, + "grad_norm": 2.5631166223280637, + "learning_rate": 1.7116040955631399e-06, + "loss": 1.2242, + "mean_token_accuracy": 0.7026595771312714, + "num_tokens": 655310906.0, + "step": 5040 + }, + { + "entropy": 1.69453125, + "epoch": 0.8358851278655963, + "grad_norm": 2.3894758934971008, + "learning_rate": 1.6945392491467578e-06, + "loss": 1.2176, + "mean_token_accuracy": 0.7043740272521972, + "num_tokens": 656611720.0, + "step": 5050 + }, + { + "entropy": 1.69296875, + "epoch": 0.8375403459405777, + "grad_norm": 2.5881645623780765, + "learning_rate": 1.6774744027303755e-06, + "loss": 1.2239, + "mean_token_accuracy": 0.7036783367395401, + "num_tokens": 657911624.0, + "step": 5060 + }, + { + "entropy": 1.691015625, + "epoch": 0.8391955640155591, + "grad_norm": 2.5356112174019256, + "learning_rate": 1.6604095563139932e-06, + "loss": 1.1898, + "mean_token_accuracy": 0.7108400166034698, + "num_tokens": 659211922.0, + "step": 5070 + }, + { + "entropy": 1.685546875, + "epoch": 0.8408507820905404, + "grad_norm": 2.400561107357791, + "learning_rate": 1.643344709897611e-06, + "loss": 1.1333, + "mean_token_accuracy": 0.7243988543748856, + "num_tokens": 660513061.0, + "step": 5080 + }, + { + "entropy": 1.680859375, + "epoch": 0.8425060001655218, + "grad_norm": 2.5202817630150607, + "learning_rate": 1.6262798634812288e-06, + "loss": 1.2197, + "mean_token_accuracy": 0.7054201334714889, + "num_tokens": 661814769.0, + "step": 5090 + }, + { + "entropy": 1.691015625, + "epoch": 0.8441612182405032, + "grad_norm": 2.7664016221318612, + "learning_rate": 1.6092150170648467e-06, + "loss": 1.1947, + "mean_token_accuracy": 0.7094619840383529, + "num_tokens": 663113259.0, + "step": 5100 + }, + { + "epoch": 0.8441612182405032, + "eval_entropy": 1.6729910714285714, + "eval_loss": 1.15461003780365, + "eval_mean_token_accuracy": 0.7179374694824219, + "eval_num_tokens": 663113259.0, + "eval_runtime": 4.6423, + "eval_samples_per_second": 94.349, + "eval_steps_per_second": 1.508, + "step": 5100 + }, + { + "entropy": 1.694140625, + "epoch": 0.8458164363154845, + "grad_norm": 2.2798493021498207, + "learning_rate": 1.5921501706484643e-06, + "loss": 1.225, + "mean_token_accuracy": 0.7036926746368408, + "num_tokens": 664414174.0, + "step": 5110 + }, + { + "entropy": 1.70625, + "epoch": 0.8474716543904659, + "grad_norm": 2.756116817162809, + "learning_rate": 1.575085324232082e-06, + "loss": 1.2088, + "mean_token_accuracy": 0.7079380184412003, + "num_tokens": 665710504.0, + "step": 5120 + }, + { + "entropy": 1.700390625, + "epoch": 0.8491268724654473, + "grad_norm": 2.318916721458488, + "learning_rate": 1.5580204778157e-06, + "loss": 1.205, + "mean_token_accuracy": 0.709128561615944, + "num_tokens": 667011851.0, + "step": 5130 + }, + { + "entropy": 1.687109375, + "epoch": 0.8507820905404287, + "grad_norm": 2.7269481450974413, + "learning_rate": 1.5409556313993176e-06, + "loss": 1.1811, + "mean_token_accuracy": 0.7131611227989196, + "num_tokens": 668313934.0, + "step": 5140 + }, + { + "entropy": 1.705859375, + "epoch": 0.8524373086154101, + "grad_norm": 2.76621621730755, + "learning_rate": 1.5238907849829353e-06, + "loss": 1.1732, + "mean_token_accuracy": 0.7159410178661346, + "num_tokens": 669613159.0, + "step": 5150 + }, + { + "entropy": 1.719140625, + "epoch": 0.8540925266903915, + "grad_norm": 2.6368995413749436, + "learning_rate": 1.506825938566553e-06, + "loss": 1.2193, + "mean_token_accuracy": 0.7045050859451294, + "num_tokens": 670913348.0, + "step": 5160 + }, + { + "entropy": 1.69921875, + "epoch": 0.8557477447653729, + "grad_norm": 2.138280212177874, + "learning_rate": 1.4897610921501707e-06, + "loss": 1.2362, + "mean_token_accuracy": 0.698490560054779, + "num_tokens": 672212688.0, + "step": 5170 + }, + { + "entropy": 1.70234375, + "epoch": 0.8574029628403542, + "grad_norm": 2.522957403410208, + "learning_rate": 1.4726962457337884e-06, + "loss": 1.2308, + "mean_token_accuracy": 0.7012133836746216, + "num_tokens": 673512670.0, + "step": 5180 + }, + { + "entropy": 1.691796875, + "epoch": 0.8590581809153356, + "grad_norm": 2.917567216391627, + "learning_rate": 1.4556313993174063e-06, + "loss": 1.2063, + "mean_token_accuracy": 0.7076902210712432, + "num_tokens": 674812581.0, + "step": 5190 + }, + { + "entropy": 1.692578125, + "epoch": 0.860713398990317, + "grad_norm": 2.4919985625651924, + "learning_rate": 1.438566552901024e-06, + "loss": 1.2444, + "mean_token_accuracy": 0.700436320900917, + "num_tokens": 676114800.0, + "step": 5200 + }, + { + "epoch": 0.860713398990317, + "eval_entropy": 1.6819196428571428, + "eval_loss": 1.1496624946594238, + "eval_mean_token_accuracy": 0.7184418865612575, + "eval_num_tokens": 676114800.0, + "eval_runtime": 4.646, + "eval_samples_per_second": 94.276, + "eval_steps_per_second": 1.507, + "step": 5200 + }, + { + "entropy": 1.678125, + "epoch": 0.8623686170652983, + "grad_norm": 2.3833175550797385, + "learning_rate": 1.4215017064846417e-06, + "loss": 1.1491, + "mean_token_accuracy": 0.7202961862087249, + "num_tokens": 677414259.0, + "step": 5210 + }, + { + "entropy": 1.6921875, + "epoch": 0.8640238351402797, + "grad_norm": 2.5481108128284093, + "learning_rate": 1.4044368600682596e-06, + "loss": 1.2256, + "mean_token_accuracy": 0.703664293885231, + "num_tokens": 678714417.0, + "step": 5220 + }, + { + "entropy": 1.690625, + "epoch": 0.8656790532152611, + "grad_norm": 2.3684152925456194, + "learning_rate": 1.3873720136518773e-06, + "loss": 1.1862, + "mean_token_accuracy": 0.7128602772951126, + "num_tokens": 680014191.0, + "step": 5230 + }, + { + "entropy": 1.7078125, + "epoch": 0.8673342712902425, + "grad_norm": 2.499886809970028, + "learning_rate": 1.3703071672354952e-06, + "loss": 1.2224, + "mean_token_accuracy": 0.705528923869133, + "num_tokens": 681313657.0, + "step": 5240 + }, + { + "entropy": 1.68984375, + "epoch": 0.8689894893652239, + "grad_norm": 2.353981546209193, + "learning_rate": 1.3532423208191128e-06, + "loss": 1.2043, + "mean_token_accuracy": 0.7095526695251465, + "num_tokens": 682613040.0, + "step": 5250 + }, + { + "entropy": 1.686328125, + "epoch": 0.8706447074402053, + "grad_norm": 2.614771698528443, + "learning_rate": 1.3361774744027305e-06, + "loss": 1.2051, + "mean_token_accuracy": 0.7094178676605225, + "num_tokens": 683911575.0, + "step": 5260 + }, + { + "entropy": 1.675390625, + "epoch": 0.8722999255151866, + "grad_norm": 2.9462784675839626, + "learning_rate": 1.319112627986348e-06, + "loss": 1.2066, + "mean_token_accuracy": 0.7074761331081391, + "num_tokens": 685211271.0, + "step": 5270 + }, + { + "entropy": 1.695703125, + "epoch": 0.873955143590168, + "grad_norm": 2.411434225266009, + "learning_rate": 1.302047781569966e-06, + "loss": 1.2364, + "mean_token_accuracy": 0.7027457624673843, + "num_tokens": 686510348.0, + "step": 5280 + }, + { + "entropy": 1.698046875, + "epoch": 0.8756103616651494, + "grad_norm": 2.2853766185993014, + "learning_rate": 1.2849829351535836e-06, + "loss": 1.2143, + "mean_token_accuracy": 0.7067865848541259, + "num_tokens": 687810713.0, + "step": 5290 + }, + { + "entropy": 1.6859375, + "epoch": 0.8772655797401308, + "grad_norm": 2.570431897229414, + "learning_rate": 1.2679180887372015e-06, + "loss": 1.1931, + "mean_token_accuracy": 0.7100887477397919, + "num_tokens": 689111678.0, + "step": 5300 + }, + { + "epoch": 0.8772655797401308, + "eval_entropy": 1.6785714285714286, + "eval_loss": 1.1478396654129028, + "eval_mean_token_accuracy": 0.7197196653911045, + "eval_num_tokens": 689111678.0, + "eval_runtime": 4.6181, + "eval_samples_per_second": 94.844, + "eval_steps_per_second": 1.516, + "step": 5300 + }, + { + "entropy": 1.67578125, + "epoch": 0.8789207978151121, + "grad_norm": 2.698575489814039, + "learning_rate": 1.2508532423208192e-06, + "loss": 1.184, + "mean_token_accuracy": 0.711807781457901, + "num_tokens": 690412847.0, + "step": 5310 + }, + { + "entropy": 1.6921875, + "epoch": 0.8805760158900935, + "grad_norm": 2.421671194302912, + "learning_rate": 1.2337883959044369e-06, + "loss": 1.1932, + "mean_token_accuracy": 0.7122453153133392, + "num_tokens": 691714133.0, + "step": 5320 + }, + { + "entropy": 1.677734375, + "epoch": 0.8822312339650749, + "grad_norm": 2.385132349329902, + "learning_rate": 1.2167235494880548e-06, + "loss": 1.18, + "mean_token_accuracy": 0.714538985490799, + "num_tokens": 693013876.0, + "step": 5330 + }, + { + "entropy": 1.6828125, + "epoch": 0.8838864520400562, + "grad_norm": 2.3782746884336334, + "learning_rate": 1.1996587030716725e-06, + "loss": 1.1985, + "mean_token_accuracy": 0.7107744961977005, + "num_tokens": 694312806.0, + "step": 5340 + }, + { + "entropy": 1.67890625, + "epoch": 0.8855416701150377, + "grad_norm": 2.920423131853787, + "learning_rate": 1.1825938566552902e-06, + "loss": 1.1985, + "mean_token_accuracy": 0.7071870237588882, + "num_tokens": 695612328.0, + "step": 5350 + }, + { + "entropy": 1.695703125, + "epoch": 0.8871968881900191, + "grad_norm": 2.4713626707821645, + "learning_rate": 1.1655290102389079e-06, + "loss": 1.2157, + "mean_token_accuracy": 0.7056057065725326, + "num_tokens": 696910923.0, + "step": 5360 + }, + { + "entropy": 1.67109375, + "epoch": 0.8888521062650004, + "grad_norm": 2.2977277753908503, + "learning_rate": 1.1484641638225255e-06, + "loss": 1.1761, + "mean_token_accuracy": 0.7166738420724869, + "num_tokens": 698209705.0, + "step": 5370 + }, + { + "entropy": 1.685546875, + "epoch": 0.8905073243399818, + "grad_norm": 2.4680006449525083, + "learning_rate": 1.1313993174061434e-06, + "loss": 1.2151, + "mean_token_accuracy": 0.7052205175161361, + "num_tokens": 699509801.0, + "step": 5380 + }, + { + "entropy": 1.679296875, + "epoch": 0.8921625424149632, + "grad_norm": 2.565968581210806, + "learning_rate": 1.1143344709897611e-06, + "loss": 1.2122, + "mean_token_accuracy": 0.706723740696907, + "num_tokens": 700809957.0, + "step": 5390 + }, + { + "entropy": 1.6890625, + "epoch": 0.8938177604899445, + "grad_norm": 2.4398144892964106, + "learning_rate": 1.097269624573379e-06, + "loss": 1.2412, + "mean_token_accuracy": 0.6996538907289505, + "num_tokens": 702112697.0, + "step": 5400 + }, + { + "epoch": 0.8938177604899445, + "eval_entropy": 1.6729910714285714, + "eval_loss": 1.144997239112854, + "eval_mean_token_accuracy": 0.7196730375289917, + "eval_num_tokens": 702112697.0, + "eval_runtime": 4.6462, + "eval_samples_per_second": 94.27, + "eval_steps_per_second": 1.507, + "step": 5400 + }, + { + "entropy": 1.66640625, + "epoch": 0.8954729785649259, + "grad_norm": 2.557031084350994, + "learning_rate": 1.0802047781569967e-06, + "loss": 1.1855, + "mean_token_accuracy": 0.7125361770391464, + "num_tokens": 703411890.0, + "step": 5410 + }, + { + "entropy": 1.665625, + "epoch": 0.8971281966399073, + "grad_norm": 2.4943303377638166, + "learning_rate": 1.0631399317406144e-06, + "loss": 1.1971, + "mean_token_accuracy": 0.711174088716507, + "num_tokens": 704713790.0, + "step": 5420 + }, + { + "entropy": 1.66796875, + "epoch": 0.8987834147148887, + "grad_norm": 2.3834680814107525, + "learning_rate": 1.0460750853242321e-06, + "loss": 1.2222, + "mean_token_accuracy": 0.7055434554815292, + "num_tokens": 706016302.0, + "step": 5430 + }, + { + "entropy": 1.676171875, + "epoch": 0.90043863278987, + "grad_norm": 2.5571130593211824, + "learning_rate": 1.0290102389078498e-06, + "loss": 1.2123, + "mean_token_accuracy": 0.7071451842784882, + "num_tokens": 707318961.0, + "step": 5440 + }, + { + "entropy": 1.658203125, + "epoch": 0.9020938508648515, + "grad_norm": 2.4939858212847112, + "learning_rate": 1.0119453924914677e-06, + "loss": 1.1631, + "mean_token_accuracy": 0.7174129098653793, + "num_tokens": 708619849.0, + "step": 5450 + }, + { + "entropy": 1.68125, + "epoch": 0.9037490689398329, + "grad_norm": 2.184266414666622, + "learning_rate": 9.948805460750854e-07, + "loss": 1.2222, + "mean_token_accuracy": 0.7045280992984772, + "num_tokens": 709919592.0, + "step": 5460 + }, + { + "entropy": 1.6671875, + "epoch": 0.9054042870148142, + "grad_norm": 2.417296974257445, + "learning_rate": 9.778156996587033e-07, + "loss": 1.1784, + "mean_token_accuracy": 0.7142821103334427, + "num_tokens": 711216648.0, + "step": 5470 + }, + { + "entropy": 1.65234375, + "epoch": 0.9070595050897956, + "grad_norm": 2.247466531729344, + "learning_rate": 9.607508532423208e-07, + "loss": 1.1887, + "mean_token_accuracy": 0.712055966258049, + "num_tokens": 712516899.0, + "step": 5480 + }, + { + "entropy": 1.64921875, + "epoch": 0.908714723164777, + "grad_norm": 2.658573458765412, + "learning_rate": 9.436860068259386e-07, + "loss": 1.1539, + "mean_token_accuracy": 0.7208747625350952, + "num_tokens": 713818398.0, + "step": 5490 + }, + { + "entropy": 1.659765625, + "epoch": 0.9103699412397583, + "grad_norm": 2.5386346558634507, + "learning_rate": 9.266211604095564e-07, + "loss": 1.1911, + "mean_token_accuracy": 0.7138865172863007, + "num_tokens": 715117372.0, + "step": 5500 + }, + { + "epoch": 0.9103699412397583, + "eval_entropy": 1.6618303571428572, + "eval_loss": 1.1425745487213135, + "eval_mean_token_accuracy": 0.7208817856652396, + "eval_num_tokens": 715117372.0, + "eval_runtime": 4.6407, + "eval_samples_per_second": 94.383, + "eval_steps_per_second": 1.508, + "step": 5500 + }, + { + "entropy": 1.656640625, + "epoch": 0.9120251593147397, + "grad_norm": 2.895852691716332, + "learning_rate": 9.095563139931742e-07, + "loss": 1.1955, + "mean_token_accuracy": 0.7099819570779801, + "num_tokens": 716417012.0, + "step": 5510 + }, + { + "entropy": 1.6734375, + "epoch": 0.9136803773897211, + "grad_norm": 2.2437368498516377, + "learning_rate": 8.92491467576792e-07, + "loss": 1.1983, + "mean_token_accuracy": 0.709630960226059, + "num_tokens": 717718573.0, + "step": 5520 + }, + { + "entropy": 1.665234375, + "epoch": 0.9153355954647024, + "grad_norm": 2.4838833841380734, + "learning_rate": 8.754266211604096e-07, + "loss": 1.1915, + "mean_token_accuracy": 0.7114917218685151, + "num_tokens": 719019853.0, + "step": 5530 + }, + { + "entropy": 1.677734375, + "epoch": 0.9169908135396838, + "grad_norm": 2.457415147343755, + "learning_rate": 8.583617747440273e-07, + "loss": 1.2145, + "mean_token_accuracy": 0.7078872859477997, + "num_tokens": 720320006.0, + "step": 5540 + }, + { + "entropy": 1.6765625, + "epoch": 0.9186460316146652, + "grad_norm": 3.0463433351851674, + "learning_rate": 8.412969283276451e-07, + "loss": 1.1731, + "mean_token_accuracy": 0.7164380788803101, + "num_tokens": 721621340.0, + "step": 5550 + }, + { + "entropy": 1.667578125, + "epoch": 0.9203012496896467, + "grad_norm": 2.5822118867864714, + "learning_rate": 8.242320819112628e-07, + "loss": 1.2071, + "mean_token_accuracy": 0.7079561799764633, + "num_tokens": 722922997.0, + "step": 5560 + }, + { + "entropy": 1.640625, + "epoch": 0.921956467764628, + "grad_norm": 2.4540700770867048, + "learning_rate": 8.071672354948806e-07, + "loss": 1.153, + "mean_token_accuracy": 0.7199074774980545, + "num_tokens": 724223348.0, + "step": 5570 + }, + { + "entropy": 1.629296875, + "epoch": 0.9236116858396094, + "grad_norm": 2.4616370036160076, + "learning_rate": 7.901023890784984e-07, + "loss": 1.1231, + "mean_token_accuracy": 0.7277600914239883, + "num_tokens": 725524790.0, + "step": 5580 + }, + { + "entropy": 1.62265625, + "epoch": 0.9252669039145908, + "grad_norm": 2.5084303516945523, + "learning_rate": 7.730375426621162e-07, + "loss": 1.142, + "mean_token_accuracy": 0.7232432901859284, + "num_tokens": 726819162.0, + "step": 5590 + }, + { + "entropy": 1.670703125, + "epoch": 0.9269221219895721, + "grad_norm": 2.5794560357783696, + "learning_rate": 7.559726962457338e-07, + "loss": 1.2282, + "mean_token_accuracy": 0.704438054561615, + "num_tokens": 728120893.0, + "step": 5600 + }, + { + "epoch": 0.9269221219895721, + "eval_entropy": 1.6473214285714286, + "eval_loss": 1.136872410774231, + "eval_mean_token_accuracy": 0.7222055196762085, + "eval_num_tokens": 728120893.0, + "eval_runtime": 4.6504, + "eval_samples_per_second": 94.185, + "eval_steps_per_second": 1.505, + "step": 5600 + }, + { + "entropy": 1.642578125, + "epoch": 0.9285773400645535, + "grad_norm": 2.2009271518126954, + "learning_rate": 7.389078498293516e-07, + "loss": 1.1934, + "mean_token_accuracy": 0.7113809257745742, + "num_tokens": 729421041.0, + "step": 5610 + }, + { + "entropy": 1.644921875, + "epoch": 0.9302325581395349, + "grad_norm": 2.2886697458388356, + "learning_rate": 7.218430034129694e-07, + "loss": 1.2046, + "mean_token_accuracy": 0.709241172671318, + "num_tokens": 730723105.0, + "step": 5620 + }, + { + "entropy": 1.644140625, + "epoch": 0.9318877762145162, + "grad_norm": 2.4850189866562076, + "learning_rate": 7.047781569965871e-07, + "loss": 1.1897, + "mean_token_accuracy": 0.7128428429365158, + "num_tokens": 732021737.0, + "step": 5630 + }, + { + "entropy": 1.66171875, + "epoch": 0.9335429942894976, + "grad_norm": 2.4045895924966048, + "learning_rate": 6.877133105802049e-07, + "loss": 1.2301, + "mean_token_accuracy": 0.7039463728666305, + "num_tokens": 733320278.0, + "step": 5640 + }, + { + "entropy": 1.663671875, + "epoch": 0.935198212364479, + "grad_norm": 2.6816354388672874, + "learning_rate": 6.706484641638227e-07, + "loss": 1.1826, + "mean_token_accuracy": 0.7122929930686951, + "num_tokens": 734621413.0, + "step": 5650 + }, + { + "entropy": 1.6453125, + "epoch": 0.9368534304394603, + "grad_norm": 2.5788443835611776, + "learning_rate": 6.535836177474402e-07, + "loss": 1.1753, + "mean_token_accuracy": 0.7148543536663056, + "num_tokens": 735922333.0, + "step": 5660 + }, + { + "entropy": 1.6328125, + "epoch": 0.9385086485144418, + "grad_norm": 2.4703523025753293, + "learning_rate": 6.36518771331058e-07, + "loss": 1.2287, + "mean_token_accuracy": 0.7038672745227814, + "num_tokens": 737218938.0, + "step": 5670 + }, + { + "entropy": 1.6328125, + "epoch": 0.9401638665894232, + "grad_norm": 2.5075409963096247, + "learning_rate": 6.194539249146758e-07, + "loss": 1.1624, + "mean_token_accuracy": 0.7173958837985992, + "num_tokens": 738520444.0, + "step": 5680 + }, + { + "entropy": 1.66328125, + "epoch": 0.9418190846644046, + "grad_norm": 2.3297081179765224, + "learning_rate": 6.023890784982936e-07, + "loss": 1.2454, + "mean_token_accuracy": 0.7009893357753754, + "num_tokens": 739820678.0, + "step": 5690 + }, + { + "entropy": 1.656640625, + "epoch": 0.9434743027393859, + "grad_norm": 2.4040198902942156, + "learning_rate": 5.853242320819113e-07, + "loss": 1.1952, + "mean_token_accuracy": 0.7101381480693817, + "num_tokens": 741121190.0, + "step": 5700 + }, + { + "epoch": 0.9434743027393859, + "eval_entropy": 1.6339285714285714, + "eval_loss": 1.134825348854065, + "eval_mean_token_accuracy": 0.7231063331876483, + "eval_num_tokens": 741121190.0, + "eval_runtime": 4.6288, + "eval_samples_per_second": 94.624, + "eval_steps_per_second": 1.512, + "step": 5700 + }, + { + "entropy": 1.635546875, + "epoch": 0.9451295208143673, + "grad_norm": 2.6217005974731964, + "learning_rate": 5.68259385665529e-07, + "loss": 1.1424, + "mean_token_accuracy": 0.7224426507949829, + "num_tokens": 742421633.0, + "step": 5710 + }, + { + "entropy": 1.632421875, + "epoch": 0.9467847388893487, + "grad_norm": 2.7564822538303084, + "learning_rate": 5.511945392491468e-07, + "loss": 1.1866, + "mean_token_accuracy": 0.712186062335968, + "num_tokens": 743722235.0, + "step": 5720 + }, + { + "entropy": 1.625390625, + "epoch": 0.94843995696433, + "grad_norm": 2.4559536220276614, + "learning_rate": 5.341296928327645e-07, + "loss": 1.1783, + "mean_token_accuracy": 0.7159310281276703, + "num_tokens": 745023298.0, + "step": 5730 + }, + { + "entropy": 1.65078125, + "epoch": 0.9500951750393114, + "grad_norm": 2.4745924072342635, + "learning_rate": 5.170648464163823e-07, + "loss": 1.1706, + "mean_token_accuracy": 0.7162785291671753, + "num_tokens": 746322688.0, + "step": 5740 + }, + { + "entropy": 1.645703125, + "epoch": 0.9517503931142928, + "grad_norm": 2.5071244166442304, + "learning_rate": 5.000000000000001e-07, + "loss": 1.1735, + "mean_token_accuracy": 0.7163868635892868, + "num_tokens": 747622841.0, + "step": 5750 + }, + { + "entropy": 1.653125, + "epoch": 0.9534056111892741, + "grad_norm": 2.241756215031614, + "learning_rate": 4.829351535836178e-07, + "loss": 1.2018, + "mean_token_accuracy": 0.7105491161346436, + "num_tokens": 748921795.0, + "step": 5760 + }, + { + "entropy": 1.66484375, + "epoch": 0.9550608292642556, + "grad_norm": 2.3271931005217343, + "learning_rate": 4.658703071672355e-07, + "loss": 1.1893, + "mean_token_accuracy": 0.7114069998264313, + "num_tokens": 750219671.0, + "step": 5770 + }, + { + "entropy": 1.6609375, + "epoch": 0.956716047339237, + "grad_norm": 2.365768371619488, + "learning_rate": 4.488054607508533e-07, + "loss": 1.2535, + "mean_token_accuracy": 0.6983337104320526, + "num_tokens": 751520550.0, + "step": 5780 + }, + { + "entropy": 1.65078125, + "epoch": 0.9583712654142184, + "grad_norm": 2.5625969504715553, + "learning_rate": 4.31740614334471e-07, + "loss": 1.2235, + "mean_token_accuracy": 0.7040059089660644, + "num_tokens": 752821538.0, + "step": 5790 + }, + { + "entropy": 1.64296875, + "epoch": 0.9600264834891997, + "grad_norm": 2.2330055474219694, + "learning_rate": 4.146757679180888e-07, + "loss": 1.1653, + "mean_token_accuracy": 0.7181041747331619, + "num_tokens": 754121836.0, + "step": 5800 + }, + { + "epoch": 0.9600264834891997, + "eval_entropy": 1.6350446428571428, + "eval_loss": 1.1316940784454346, + "eval_mean_token_accuracy": 0.7235601459230695, + "eval_num_tokens": 754121836.0, + "eval_runtime": 4.6339, + "eval_samples_per_second": 94.522, + "eval_steps_per_second": 1.511, + "step": 5800 + }, + { + "entropy": 1.647265625, + "epoch": 0.9616817015641811, + "grad_norm": 2.70355121466036, + "learning_rate": 3.9761092150170654e-07, + "loss": 1.1869, + "mean_token_accuracy": 0.7108889371156693, + "num_tokens": 755421282.0, + "step": 5810 + }, + { + "entropy": 1.628125, + "epoch": 0.9633369196391625, + "grad_norm": 2.350089987375434, + "learning_rate": 3.8054607508532423e-07, + "loss": 1.1714, + "mean_token_accuracy": 0.7173035323619843, + "num_tokens": 756722744.0, + "step": 5820 + }, + { + "entropy": 1.634375, + "epoch": 0.9649921377141438, + "grad_norm": 2.5019797266418533, + "learning_rate": 3.63481228668942e-07, + "loss": 1.182, + "mean_token_accuracy": 0.7134859770536423, + "num_tokens": 758025051.0, + "step": 5830 + }, + { + "entropy": 1.640234375, + "epoch": 0.9666473557891252, + "grad_norm": 2.483565757009998, + "learning_rate": 3.4641638225255977e-07, + "loss": 1.1889, + "mean_token_accuracy": 0.7113577246665954, + "num_tokens": 759322572.0, + "step": 5840 + }, + { + "entropy": 1.637109375, + "epoch": 0.9683025738641066, + "grad_norm": 2.5593633654877133, + "learning_rate": 3.293515358361775e-07, + "loss": 1.1644, + "mean_token_accuracy": 0.7168270587921143, + "num_tokens": 760624149.0, + "step": 5850 + }, + { + "entropy": 1.644140625, + "epoch": 0.9699577919390879, + "grad_norm": 2.195177118040664, + "learning_rate": 3.1228668941979525e-07, + "loss": 1.1707, + "mean_token_accuracy": 0.7175578981637954, + "num_tokens": 761926491.0, + "step": 5860 + }, + { + "entropy": 1.6359375, + "epoch": 0.9716130100140693, + "grad_norm": 2.2298551085331444, + "learning_rate": 2.95221843003413e-07, + "loss": 1.1929, + "mean_token_accuracy": 0.7100267529487609, + "num_tokens": 763227867.0, + "step": 5870 + }, + { + "entropy": 1.634375, + "epoch": 0.9732682280890508, + "grad_norm": 2.513728323196809, + "learning_rate": 2.7815699658703074e-07, + "loss": 1.1827, + "mean_token_accuracy": 0.712663471698761, + "num_tokens": 764526779.0, + "step": 5880 + }, + { + "entropy": 1.6234375, + "epoch": 0.9749234461640321, + "grad_norm": 2.836739733339177, + "learning_rate": 2.610921501706485e-07, + "loss": 1.1529, + "mean_token_accuracy": 0.7203582406044007, + "num_tokens": 765829812.0, + "step": 5890 + }, + { + "entropy": 1.64296875, + "epoch": 0.9765786642390135, + "grad_norm": 2.674999278426445, + "learning_rate": 2.440273037542662e-07, + "loss": 1.2116, + "mean_token_accuracy": 0.7050048023462295, + "num_tokens": 767131193.0, + "step": 5900 + }, + { + "epoch": 0.9765786642390135, + "eval_entropy": 1.6316964285714286, + "eval_loss": 1.1298809051513672, + "eval_mean_token_accuracy": 0.7243056637900216, + "eval_num_tokens": 767131193.0, + "eval_runtime": 4.6497, + "eval_samples_per_second": 94.199, + "eval_steps_per_second": 1.505, + "step": 5900 + }, + { + "entropy": 1.651953125, + "epoch": 0.9782338823139949, + "grad_norm": 2.3939513308111033, + "learning_rate": 2.2696245733788397e-07, + "loss": 1.1859, + "mean_token_accuracy": 0.7112313896417618, + "num_tokens": 768430806.0, + "step": 5910 + }, + { + "entropy": 1.6328125, + "epoch": 0.9798891003889763, + "grad_norm": 2.4696719057885597, + "learning_rate": 2.0989761092150173e-07, + "loss": 1.1841, + "mean_token_accuracy": 0.7151529520750046, + "num_tokens": 769730178.0, + "step": 5920 + }, + { + "entropy": 1.63125, + "epoch": 0.9815443184639576, + "grad_norm": 2.6030460463634726, + "learning_rate": 1.9283276450511948e-07, + "loss": 1.1693, + "mean_token_accuracy": 0.7164003252983093, + "num_tokens": 771029307.0, + "step": 5930 + }, + { + "entropy": 1.61328125, + "epoch": 0.983199536538939, + "grad_norm": 2.4936941203781764, + "learning_rate": 1.7576791808873722e-07, + "loss": 1.1656, + "mean_token_accuracy": 0.718073970079422, + "num_tokens": 772330558.0, + "step": 5940 + }, + { + "entropy": 1.625390625, + "epoch": 0.9848547546139204, + "grad_norm": 2.229901883140784, + "learning_rate": 1.58703071672355e-07, + "loss": 1.1632, + "mean_token_accuracy": 0.7166268914937973, + "num_tokens": 773631359.0, + "step": 5950 + }, + { + "entropy": 1.61796875, + "epoch": 0.9865099726889017, + "grad_norm": 2.4145776487526054, + "learning_rate": 1.416382252559727e-07, + "loss": 1.2082, + "mean_token_accuracy": 0.708984300494194, + "num_tokens": 774931172.0, + "step": 5960 + }, + { + "entropy": 1.6171875, + "epoch": 0.9881651907638831, + "grad_norm": 2.412672345461028, + "learning_rate": 1.2457337883959045e-07, + "loss": 1.1641, + "mean_token_accuracy": 0.7202441304922104, + "num_tokens": 776226600.0, + "step": 5970 + }, + { + "entropy": 1.620703125, + "epoch": 0.9898204088388646, + "grad_norm": 2.3038156529091727, + "learning_rate": 1.075085324232082e-07, + "loss": 1.2059, + "mean_token_accuracy": 0.7096736639738083, + "num_tokens": 777528566.0, + "step": 5980 + }, + { + "entropy": 1.626953125, + "epoch": 0.9914756269138459, + "grad_norm": 2.6039170428559015, + "learning_rate": 9.044368600682596e-08, + "loss": 1.1356, + "mean_token_accuracy": 0.7243529796600342, + "num_tokens": 778829598.0, + "step": 5990 + }, + { + "entropy": 1.62265625, + "epoch": 0.9931308449888273, + "grad_norm": 2.551964961765006, + "learning_rate": 7.33788395904437e-08, + "loss": 1.1997, + "mean_token_accuracy": 0.7105666846036911, + "num_tokens": 780130050.0, + "step": 6000 + }, + { + "epoch": 0.9931308449888273, + "eval_entropy": 1.6227678571428572, + "eval_loss": 1.1272960901260376, + "eval_mean_token_accuracy": 0.7249177268573216, + "eval_num_tokens": 780130050.0, + "eval_runtime": 4.6376, + "eval_samples_per_second": 94.445, + "eval_steps_per_second": 1.509, + "step": 6000 + }, + { + "entropy": 1.6390625, + "epoch": 0.9947860630638087, + "grad_norm": 2.2748934348033525, + "learning_rate": 5.631399317406144e-08, + "loss": 1.1892, + "mean_token_accuracy": 0.7133197635412216, + "num_tokens": 781430866.0, + "step": 6010 + }, + { + "entropy": 1.631640625, + "epoch": 0.99644128113879, + "grad_norm": 2.4049449953563653, + "learning_rate": 3.924914675767919e-08, + "loss": 1.1836, + "mean_token_accuracy": 0.7134488612413407, + "num_tokens": 782730858.0, + "step": 6020 + }, + { + "entropy": 1.634765625, + "epoch": 0.9980964992137714, + "grad_norm": 2.3426996944917398, + "learning_rate": 2.2184300341296933e-08, + "loss": 1.1882, + "mean_token_accuracy": 0.7128017842769623, + "num_tokens": 784029792.0, + "step": 6030 + }, + { + "entropy": 1.629296875, + "epoch": 0.9997517172887528, + "grad_norm": 2.4433507450092615, + "learning_rate": 5.119453924914676e-09, + "loss": 1.1769, + "mean_token_accuracy": 0.7147614687681199, + "num_tokens": 785330202.0, + "step": 6040 + }, + { + "entropy": 1.6328125, + "epoch": 1.0, + "mean_token_accuracy": 0.7037371794382731, + "num_tokens": 785524633.0, + "step": 6042, + "total_flos": 1254725330468864.0, + "train_loss": 1.377876113716714, + "train_runtime": 31760.1021, + "train_samples_per_second": 24.347, + "train_steps_per_second": 0.19 + } + ], + "logging_steps": 10, + "max_steps": 6042, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 605, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1254725330468864.0, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..55e67d4 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2eec86ebec25714cdc0d29343fd117b3be946d696e58d51b4b9035b6104f929 +size 11064