commit 3f15326763be00f5d1419aac72c3099bc68b546b Author: ModelHub XC Date: Wed Jul 15 19:30:10 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: modaserMoj/csc415-phase1-0.5b-fast Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..63d574e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,39 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-3058/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..a2a0e60 --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +--- +base_model: Qwen/Qwen2.5-0.5B +library_name: transformers +model_name: phase1_0.5b_fast +tags: +- generated_from_trainer +- trl +- grpo +licence: license +--- + +# Model Card for phase1_0.5b_fast + +This model is a fine-tuned version of [Qwen/Qwen2.5-0.5B](https://huggingface.co/Qwen/Qwen2.5-0.5B). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + + + + + +This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300). + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.3.0 +- Pytorch: 2.10.0+cu128 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + +Cite GRPO as: + +```bibtex +@article{shao2024deepseekmath, + title = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}}, + author = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo}, + year = 2024, + eprint = {arXiv:2402.03300}, +} + +``` + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..28028c0 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-2000/chat_template.jinja b/checkpoint-2000/chat_template.jinja new file mode 100644 index 0000000..28028c0 --- /dev/null +++ b/checkpoint-2000/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-2000/config.json b/checkpoint-2000/config.json new file mode 100644 index 0000000..08e4041 --- /dev/null +++ b/checkpoint-2000/config.json @@ -0,0 +1,58 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.3.0", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-2000/generation_config.json b/checkpoint-2000/generation_config.json new file mode 100644 index 0000000..8da9fd6 --- /dev/null +++ b/checkpoint-2000/generation_config.json @@ -0,0 +1,9 @@ +{ + "do_sample": false, + "eos_token_id": [ + 151643 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.3.0" +} diff --git a/checkpoint-2000/model.safetensors b/checkpoint-2000/model.safetensors new file mode 100644 index 0000000..5a5aa3c --- /dev/null +++ b/checkpoint-2000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:152e0ae2c441347b1ab293b4464c644cf609b85f1add42363801bcb765e38b70 +size 1976163472 diff --git a/checkpoint-2000/optimizer.pt b/checkpoint-2000/optimizer.pt new file mode 100644 index 0000000..bdbee0e --- /dev/null +++ b/checkpoint-2000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec2ea3cc5a18bb6bccd85e662ac0d000f9b89300c7ee878f01ee790cb920a2fc +size 3952509771 diff --git a/checkpoint-2000/rng_state.pth b/checkpoint-2000/rng_state.pth new file mode 100644 index 0000000..b6dcae0 --- /dev/null +++ b/checkpoint-2000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3af6887480e38197707c80ca84155fa9e7b4158269b9347c8d5d80fb362de4ee +size 14645 diff --git a/checkpoint-2000/scheduler.pt b/checkpoint-2000/scheduler.pt new file mode 100644 index 0000000..91ac8bb --- /dev/null +++ b/checkpoint-2000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ea52f9167ac34de1463a7d9d8892503924829652cdaaf117f45d84d2549a9b8 +size 1465 diff --git a/checkpoint-2000/tokenizer.json b/checkpoint-2000/tokenizer.json new file mode 100644 index 0000000..3c72934 --- /dev/null +++ b/checkpoint-2000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f96da3a872b5e901575b2067c744ad336c3a3d77a21584d20024557b1bd7f0 +size 11422059 diff --git a/checkpoint-2000/tokenizer_config.json b/checkpoint-2000/tokenizer_config.json new file mode 100644 index 0000000..db25db0 --- /dev/null +++ b/checkpoint-2000/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "truncation_side": "left", + "unk_token": null +} diff --git a/checkpoint-2000/trainer_state.json b/checkpoint-2000/trainer_state.json new file mode 100644 index 0000000..9be4ec9 --- /dev/null +++ b/checkpoint-2000/trainer_state.json @@ -0,0 +1,5634 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.3080444735120995, + "eval_steps": 500, + "global_step": 2000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.49000000953674316, + "completions/max_length": 384.0, + "completions/max_terminated_length": 372.8, + "completions/mean_length": 291.7250061035156, + "completions/mean_terminated_length": 204.1207077026367, + "completions/min_length": 15.7, + "completions/min_terminated_length": 15.7, + "entropy": 1.5640255227684974, + "epoch": 0.006540222367560497, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5129233002662659, + "kl": 0.0005587212668615394, + "learning_rate": 4.985284499672989e-07, + "loss": -0.04552017748355865, + "num_tokens": 324400.0, + "reward": -2.107620294111712e-06, + "reward_std": 0.9998070120811462, + "rewards/rnd_reward/mean": -2.107620250058062e-06, + "rewards/rnd_reward/std": 0.9998070061206817, + "step": 10, + "step_time": 9.577712386706844 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5312500178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 374.2, + "completions/mean_length": 293.2887481689453, + "completions/mean_terminated_length": 190.75759887695312, + "completions/min_length": 18.8, + "completions/min_terminated_length": 18.8, + "entropy": 1.451838418841362, + "epoch": 0.013080444735120994, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.47466912865638733, + "kl": 0.000643831241177395, + "learning_rate": 4.968933943754087e-07, + "loss": 0.00032075121998786926, + "num_tokens": 647161.0, + "reward": -1.490712192975252e-06, + "reward_std": 0.9998168766498565, + "rewards/rnd_reward/mean": -1.4907121759222264e-06, + "rewards/rnd_reward/std": 0.9998168766498565, + "step": 20, + "step_time": 9.337269740481862 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5450000017881393, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.6, + "completions/mean_length": 298.80250549316406, + "completions/mean_terminated_length": 196.99718322753907, + "completions/min_length": 20.3, + "completions/min_terminated_length": 20.3, + "entropy": 1.5065956979990005, + "epoch": 0.01962066710268149, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5856828689575195, + "kl": 0.0006675069424090907, + "learning_rate": 4.952583387835186e-07, + "loss": 0.02223682850599289, + "num_tokens": 982698.0, + "reward": -1.2338162438396694e-07, + "reward_std": 0.9998676717281342, + "rewards/rnd_reward/mean": -1.233816414369926e-07, + "rewards/rnd_reward/std": 0.9998676896095275, + "step": 30, + "step_time": 9.97233030446805 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5125000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 371.2, + "completions/mean_length": 287.2925048828125, + "completions/mean_terminated_length": 185.24396667480468, + "completions/min_length": 10.5, + "completions/min_terminated_length": 10.5, + "entropy": 1.6533239781856537, + "epoch": 0.026160889470241987, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5665533542633057, + "kl": 0.0007774221347062848, + "learning_rate": 4.936232831916285e-07, + "loss": 0.06135546565055847, + "num_tokens": 1303862.0, + "reward": -6.012618625206301e-07, + "reward_std": 0.9998966813087463, + "rewards/rnd_reward/mean": -6.012618571915596e-07, + "rewards/rnd_reward/std": 0.9998966932296753, + "step": 40, + "step_time": 9.489675145596266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46375001072883604, + "completions/max_length": 384.0, + "completions/max_terminated_length": 377.3, + "completions/mean_length": 269.5550064086914, + "completions/mean_terminated_length": 170.72807769775392, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 1.7108511567115783, + "epoch": 0.032701111837802485, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7200465798377991, + "kl": 0.0023742157864035107, + "learning_rate": 4.919882275997384e-07, + "loss": 0.11103466749191285, + "num_tokens": 1618921.0, + "reward": -8.426606612488285e-08, + "reward_std": 0.9999245047569275, + "rewards/rnd_reward/mean": -8.426606257216918e-08, + "rewards/rnd_reward/std": 0.999924510717392, + "step": 50, + "step_time": 10.167311429185792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4012500047683716, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.1, + "completions/mean_length": 223.41500549316407, + "completions/mean_terminated_length": 115.63493576049805, + "completions/min_length": 2.8, + "completions/min_terminated_length": 2.8, + "entropy": 2.521006014943123, + "epoch": 0.03924133420536298, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.70113205909729, + "kl": 0.020581040543038397, + "learning_rate": 4.903531720078482e-07, + "loss": 0.17053321599960328, + "num_tokens": 1894883.0, + "reward": -1.1384487108578867e-07, + "reward_std": 0.9999224483966828, + "rewards/rnd_reward/mean": -1.1384487108578867e-07, + "rewards/rnd_reward/std": 0.9999224364757537, + "step": 60, + "step_time": 9.91588868314866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3012500047683716, + "completions/max_length": 384.0, + "completions/max_terminated_length": 373.8, + "completions/mean_length": 183.6437515258789, + "completions/mean_terminated_length": 97.22235260009765, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7396299123764036, + "epoch": 0.04578155657292348, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7759609222412109, + "kl": 0.0463468745816499, + "learning_rate": 4.887181164159581e-07, + "loss": 0.14934378862380981, + "num_tokens": 2138653.0, + "reward": -4.6938660425155375e-08, + "reward_std": 0.9998835921287537, + "rewards/rnd_reward/mean": -4.6938658648798536e-08, + "rewards/rnd_reward/std": 0.9998836040496826, + "step": 70, + "step_time": 10.02495912197046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2825000062584877, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.8, + "completions/mean_length": 176.12750549316405, + "completions/mean_terminated_length": 94.6318603515625, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7325789600610735, + "epoch": 0.052321778940483975, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8319016695022583, + "kl": 0.04594853101298213, + "learning_rate": 4.87083060824068e-07, + "loss": 0.018015310168266296, + "num_tokens": 2376420.0, + "reward": -4.187226378959963e-08, + "reward_std": 0.9998487293720245, + "rewards/rnd_reward/mean": -4.1872263079056894e-08, + "rewards/rnd_reward/std": 0.9998487293720245, + "step": 80, + "step_time": 9.701193272857926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34750000238418577, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.4, + "completions/mean_length": 207.52375335693358, + "completions/mean_terminated_length": 114.01394119262696, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.342811721563339, + "epoch": 0.058862001308044476, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5935761332511902, + "kl": 0.019986469263676553, + "learning_rate": 4.854480052321779e-07, + "loss": 0.010216768831014633, + "num_tokens": 2641029.0, + "reward": -2.0414591972439666e-08, + "reward_std": 0.9998277425765991, + "rewards/rnd_reward/mean": -2.0414591972439666e-08, + "rewards/rnd_reward/std": 0.9998277485370636, + "step": 90, + "step_time": 10.468820211617276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4150000065565109, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.7, + "completions/mean_length": 237.4625045776367, + "completions/mean_terminated_length": 134.2795196533203, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "entropy": 2.1418308436870577, + "epoch": 0.06540222367560497, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5613327622413635, + "kl": 0.006808469316456467, + "learning_rate": 4.838129496402878e-07, + "loss": 0.04351484477519989, + "num_tokens": 2930639.0, + "reward": 8.493661918240747e-09, + "reward_std": 0.9997802138328552, + "rewards/rnd_reward/mean": 8.493660852426644e-09, + "rewards/rnd_reward/std": 0.9997801959514618, + "step": 100, + "step_time": 10.307341906265355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.38875000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 370.8, + "completions/mean_length": 234.60625610351562, + "completions/mean_terminated_length": 139.61359405517578, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.0046547770500185, + "epoch": 0.07194244604316546, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7055774927139282, + "kl": 0.006759516580495983, + "learning_rate": 4.821778940483976e-07, + "loss": 0.0377700001001358, + "num_tokens": 3215169.0, + "reward": -2.920627686364696e-08, + "reward_std": 0.9997233927249909, + "rewards/rnd_reward/mean": -2.9206276153104228e-08, + "rewards/rnd_reward/std": 0.9997233927249909, + "step": 110, + "step_time": 9.7759924543323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.38250001072883605, + "completions/max_length": 384.0, + "completions/max_terminated_length": 364.8, + "completions/mean_length": 221.19375457763672, + "completions/mean_terminated_length": 120.60059127807617, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.0983517467975616, + "epoch": 0.07848266841072596, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5727905631065369, + "kl": 0.009989026037510484, + "learning_rate": 4.805428384565075e-07, + "loss": 0.09626840949058532, + "num_tokens": 3496559.0, + "reward": 1.654028882747127e-08, + "reward_std": 0.9997243106365203, + "rewards/rnd_reward/mean": 1.654028882747127e-08, + "rewards/rnd_reward/std": 0.9997243225574494, + "step": 120, + "step_time": 9.981053173542023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34750000536441805, + "completions/max_length": 384.0, + "completions/max_terminated_length": 370.4, + "completions/mean_length": 207.67625274658204, + "completions/mean_terminated_length": 114.74080200195313, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2568650126457213, + "epoch": 0.08502289077828647, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8493079543113708, + "kl": 0.014032764779403805, + "learning_rate": 4.789077828646174e-07, + "loss": -0.015352001786231995, + "num_tokens": 3759245.0, + "reward": -5.06639494801675e-09, + "reward_std": 0.9996613085269928, + "rewards/rnd_reward/mean": -5.066393526931279e-09, + "rewards/rnd_reward/std": 0.9996613144874573, + "step": 130, + "step_time": 10.069910727906972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3137500062584877, + "completions/max_length": 384.0, + "completions/max_terminated_length": 373.4, + "completions/mean_length": 196.95000305175782, + "completions/mean_terminated_length": 111.34912643432617, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.305853408575058, + "epoch": 0.09156311314584696, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7982010245323181, + "kl": 0.016896341042593122, + "learning_rate": 4.772727272727273e-07, + "loss": 0.02038821578025818, + "num_tokens": 4010260.0, + "reward": -6.914138950264714e-08, + "reward_std": 0.9996093094348908, + "rewards/rnd_reward/mean": -6.914138808156168e-08, + "rewards/rnd_reward/std": 0.9996093094348908, + "step": 140, + "step_time": 9.98037786844652 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3700000047683716, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.3, + "completions/mean_length": 212.5687515258789, + "completions/mean_terminated_length": 111.88678207397462, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2448606371879576, + "epoch": 0.09810333551340746, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6012561321258545, + "kl": 0.01302192104049027, + "learning_rate": 4.756376716808371e-07, + "loss": 0.041692107915878296, + "num_tokens": 4270360.0, + "reward": -6.854534007061375e-09, + "reward_std": 0.9995533764362335, + "rewards/rnd_reward/mean": -6.854535428146846e-09, + "rewards/rnd_reward/std": 0.9995533883571625, + "step": 150, + "step_time": 9.479559903079643 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3687500059604645, + "completions/max_length": 384.0, + "completions/max_terminated_length": 373.0, + "completions/mean_length": 221.59500274658203, + "completions/mean_terminated_length": 127.51908645629882, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2110244154930117, + "epoch": 0.10464355788096795, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7059744000434875, + "kl": 0.009931368473917245, + "learning_rate": 4.74002616088947e-07, + "loss": 0.0420710027217865, + "num_tokens": 4538671.0, + "reward": -6.467104074481256e-08, + "reward_std": 0.9995002686977387, + "rewards/rnd_reward/mean": -6.467103932372709e-08, + "rewards/rnd_reward/std": 0.9995002567768096, + "step": 160, + "step_time": 9.463879713765346 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.38750000596046447, + "completions/max_length": 384.0, + "completions/max_terminated_length": 366.5, + "completions/mean_length": 224.09500274658203, + "completions/mean_terminated_length": 123.22678070068359, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.105703830718994, + "epoch": 0.11118378024852844, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6353176236152649, + "kl": 0.009572444634977728, + "learning_rate": 4.723675604970569e-07, + "loss": 0.057880669832229614, + "num_tokens": 4810167.0, + "reward": -1.3113021868349506e-08, + "reward_std": 0.9995246767997742, + "rewards/rnd_reward/mean": -1.3113020802535403e-08, + "rewards/rnd_reward/std": 0.9995246648788452, + "step": 170, + "step_time": 9.689974679402075 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3425000041723251, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.8, + "completions/mean_length": 207.19125213623047, + "completions/mean_terminated_length": 115.52461242675781, + "completions/min_length": 2.7, + "completions/min_terminated_length": 2.7, + "entropy": 2.3119001001119615, + "epoch": 0.11772400261608895, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6543518900871277, + "kl": 0.01470447750762105, + "learning_rate": 4.7073250490516674e-07, + "loss": 0.05609138607978821, + "num_tokens": 5069705.0, + "reward": -3.21865089580875e-08, + "reward_std": 0.9993573248386383, + "rewards/rnd_reward/mean": -3.218650753700203e-08, + "rewards/rnd_reward/std": 0.9993573546409606, + "step": 180, + "step_time": 9.860351232904941 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.29625000804662704, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.4, + "completions/mean_length": 186.85875396728517, + "completions/mean_terminated_length": 103.74458618164063, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.3917055457830427, + "epoch": 0.12426422498364945, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6832460165023804, + "kl": 0.02189517344813794, + "learning_rate": 4.690974493132766e-07, + "loss": 0.0940218448638916, + "num_tokens": 5313337.0, + "reward": -2.801418297337932e-08, + "reward_std": 0.9993866741657257, + "rewards/rnd_reward/mean": -2.801418386155774e-08, + "rewards/rnd_reward/std": 0.9993866682052612, + "step": 190, + "step_time": 10.32580083089415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3087500035762787, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.0, + "completions/mean_length": 195.6250030517578, + "completions/mean_terminated_length": 111.06491088867188, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.365246319770813, + "epoch": 0.13080444735120994, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7736798524856567, + "kl": 0.018843610235489906, + "learning_rate": 4.674623937213865e-07, + "loss": 0.09085838794708252, + "num_tokens": 5564832.0, + "reward": -3.784895152847412e-08, + "reward_std": 0.9993082165718079, + "rewards/rnd_reward/mean": -3.7848948331031805e-08, + "rewards/rnd_reward/std": 0.9993082165718079, + "step": 200, + "step_time": 10.074973361729644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2800000086426735, + "completions/max_length": 384.0, + "completions/max_terminated_length": 364.2, + "completions/mean_length": 183.75125274658203, + "completions/mean_terminated_length": 105.59480438232421, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3340355485677717, + "epoch": 0.13734466971877043, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7310492992401123, + "kl": 0.019957996485754848, + "learning_rate": 4.6582733812949637e-07, + "loss": 0.09393368363380432, + "num_tokens": 5808593.0, + "reward": -7.003545956507651e-08, + "reward_std": 0.9992451012134552, + "rewards/rnd_reward/mean": -7.003545814399103e-08, + "rewards/rnd_reward/std": 0.9992450892925262, + "step": 210, + "step_time": 9.864037699718029 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2600000038743019, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.5, + "completions/mean_length": 182.30500335693358, + "completions/mean_terminated_length": 111.45323791503907, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3039004147052764, + "epoch": 0.14388489208633093, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7140376567840576, + "kl": 0.020338369254022836, + "learning_rate": 4.641922825376063e-07, + "loss": 0.03314664363861084, + "num_tokens": 6039847.0, + "reward": -4.202127490415819e-08, + "reward_std": 0.9992108285427094, + "rewards/rnd_reward/mean": -4.202127419361546e-08, + "rewards/rnd_reward/std": 0.9992108345031738, + "step": 220, + "step_time": 9.367073794314638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2750000059604645, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.2, + "completions/mean_length": 176.20750274658204, + "completions/mean_terminated_length": 97.27564849853516, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.2686150133609773, + "epoch": 0.15042511445389142, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8464763760566711, + "kl": 0.021972918952815234, + "learning_rate": 4.6255722694571613e-07, + "loss": 0.04784287512302399, + "num_tokens": 6270078.0, + "reward": -1.1757016302738065e-07, + "reward_std": 0.9992310464382171, + "rewards/rnd_reward/mean": -1.1757016231683792e-07, + "rewards/rnd_reward/std": 0.9992310464382171, + "step": 230, + "step_time": 9.406287361658178 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23750000298023224, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.0, + "completions/mean_length": 164.8237548828125, + "completions/mean_terminated_length": 96.10623779296876, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4642169535160066, + "epoch": 0.15696533682145192, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.874134361743927, + "kl": 0.027495546033605933, + "learning_rate": 4.60922171353826e-07, + "loss": 0.09257912039756774, + "num_tokens": 6507582.0, + "reward": -2.2351741080228748e-08, + "reward_std": 0.9989596605300903, + "rewards/rnd_reward/mean": -2.2351743211856957e-08, + "rewards/rnd_reward/std": 0.9989596605300903, + "step": 240, + "step_time": 10.88162840725854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.26375000476837157, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.7, + "completions/mean_length": 179.21375274658203, + "completions/mean_terminated_length": 105.97551345825195, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.3090430706739427, + "epoch": 0.16350555918901244, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7480731010437012, + "kl": 0.022477348684333265, + "learning_rate": 4.592871157619359e-07, + "loss": 0.0661897599697113, + "num_tokens": 6746233.0, + "reward": -8.285045609568442e-08, + "reward_std": 0.9989501237869263, + "rewards/rnd_reward/mean": -8.285045680622715e-08, + "rewards/rnd_reward/std": 0.9989501178264618, + "step": 250, + "step_time": 9.837229801248759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23000000417232513, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.9, + "completions/mean_length": 166.79125366210937, + "completions/mean_terminated_length": 101.81034469604492, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3430103093385695, + "epoch": 0.17004578155657293, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7986056804656982, + "kl": 0.026151024643331765, + "learning_rate": 4.5765206017004576e-07, + "loss": 0.1205091118812561, + "num_tokens": 6976091.0, + "reward": -7.808208817294826e-08, + "reward_std": 0.9988620162010193, + "rewards/rnd_reward/mean": -7.808208604132006e-08, + "rewards/rnd_reward/std": 0.9988620281219482, + "step": 260, + "step_time": 9.811681813630276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23125000298023224, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.4, + "completions/mean_length": 162.19500274658202, + "completions/mean_terminated_length": 95.31229019165039, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2510120779275895, + "epoch": 0.17658600392413343, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6842794418334961, + "kl": 0.025852510519325734, + "learning_rate": 4.5601700457815564e-07, + "loss": 0.07729284763336182, + "num_tokens": 7199527.0, + "reward": -7.122755136812487e-08, + "reward_std": 0.9989409029483796, + "rewards/rnd_reward/mean": -7.122754994703939e-08, + "rewards/rnd_reward/std": 0.9989409029483796, + "step": 270, + "step_time": 9.695072391931898 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.24750000536441802, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.6, + "completions/mean_length": 165.32125244140624, + "completions/mean_terminated_length": 93.36570816040039, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2813161820173264, + "epoch": 0.18312622629169392, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8117495775222778, + "kl": 0.02381342255976051, + "learning_rate": 4.543819489862655e-07, + "loss": 0.08756870031356812, + "num_tokens": 7423314.0, + "reward": -3.010034372863402e-08, + "reward_std": 0.9987568318843841, + "rewards/rnd_reward/mean": -3.0100344261541065e-08, + "rewards/rnd_reward/std": 0.9987568318843841, + "step": 280, + "step_time": 9.538212020602078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.27250000536441804, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.5, + "completions/mean_length": 171.89125366210936, + "completions/mean_terminated_length": 91.7330307006836, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.2805801182985306, + "epoch": 0.18966644865925442, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7356550693511963, + "kl": 0.021292086993344127, + "learning_rate": 4.527468933943754e-07, + "loss": 0.0787746787071228, + "num_tokens": 7649017.0, + "reward": -2.2798775312082853e-08, + "reward_std": 0.9987796723842621, + "rewards/rnd_reward/mean": -2.279877513444717e-08, + "rewards/rnd_reward/std": 0.99877969622612, + "step": 290, + "step_time": 9.4448504867265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.4, + "completions/mean_length": 172.06375274658203, + "completions/mean_terminated_length": 101.12049560546875, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.062102970480919, + "epoch": 0.1962066710268149, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.678175151348114, + "kl": 0.021230081818066536, + "learning_rate": 4.511118378024852e-07, + "loss": 0.10422745943069459, + "num_tokens": 7893248.0, + "reward": 1.7285347131412722e-08, + "reward_std": 0.9987858593463897, + "rewards/rnd_reward/mean": 1.7285347841955455e-08, + "rewards/rnd_reward/std": 0.9987858593463897, + "step": 300, + "step_time": 11.542911725770683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23125, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.3, + "completions/mean_length": 155.94375305175782, + "completions/mean_terminated_length": 87.63837699890136, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.110625982284546, + "epoch": 0.2027468933943754, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8758037090301514, + "kl": 0.025008385512046517, + "learning_rate": 4.4947678221059515e-07, + "loss": 0.0913781225681305, + "num_tokens": 8105838.0, + "reward": -6.72042411054008e-08, + "reward_std": 0.9986489951610565, + "rewards/rnd_reward/mean": -6.720423826322985e-08, + "rewards/rnd_reward/std": 0.9986490130424499, + "step": 310, + "step_time": 9.397380503430032 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19500000327825545, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.1, + "completions/mean_length": 141.38000259399413, + "completions/mean_terminated_length": 82.59085083007812, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.311973440647125, + "epoch": 0.2092871157619359, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9461565613746643, + "kl": 0.03633355665951967, + "learning_rate": 4.4784172661870503e-07, + "loss": 0.08147850632667542, + "num_tokens": 8325122.0, + "reward": -9.238719300697084e-09, + "reward_std": 0.9986236810684204, + "rewards/rnd_reward/mean": -9.238719300697084e-09, + "rewards/rnd_reward/std": 0.9986236929893494, + "step": 320, + "step_time": 11.082472172076814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.20000000447034835, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.7, + "completions/mean_length": 141.80000228881835, + "completions/mean_terminated_length": 81.46792144775391, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3696460843086244, + "epoch": 0.2158273381294964, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8129741549491882, + "kl": 0.034535441920161246, + "learning_rate": 4.462066710268149e-07, + "loss": 0.06801753044128418, + "num_tokens": 8535267.0, + "reward": -8.24779284869237e-08, + "reward_std": 0.9986066579818725, + "rewards/rnd_reward/mean": -8.24779284869237e-08, + "rewards/rnd_reward/std": 0.9986066579818725, + "step": 330, + "step_time": 9.98545232352335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2100000038743019, + "completions/max_length": 384.0, + "completions/max_terminated_length": 364.4, + "completions/mean_length": 141.52500228881837, + "completions/mean_terminated_length": 77.51832160949706, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3277006804943086, + "epoch": 0.2223675604970569, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8912848234176636, + "kl": 0.0404030725825578, + "learning_rate": 4.445716154349248e-07, + "loss": 0.09486417174339294, + "num_tokens": 8741817.0, + "reward": -1.6838312300038183e-08, + "reward_std": 0.9984602987766266, + "rewards/rnd_reward/mean": -1.683831301058092e-08, + "rewards/rnd_reward/std": 0.9984602928161621, + "step": 340, + "step_time": 9.806188964121976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1912500038743019, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.6, + "completions/mean_length": 139.37500076293946, + "completions/mean_terminated_length": 81.56527061462403, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.3621277123689652, + "epoch": 0.2289077828646174, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9253888130187988, + "kl": 0.03642263817600906, + "learning_rate": 4.429365598430346e-07, + "loss": 0.09143019318580628, + "num_tokens": 8945182.0, + "reward": -2.4735924242236252e-08, + "reward_std": 0.9984571993350982, + "rewards/rnd_reward/mean": -2.4735929926578137e-08, + "rewards/rnd_reward/std": 0.9984571933746338, + "step": 350, + "step_time": 9.939818266802467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18750000447034837, + "completions/max_length": 384.0, + "completions/max_terminated_length": 371.7, + "completions/mean_length": 138.01250076293945, + "completions/mean_terminated_length": 81.37003707885742, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.270884335041046, + "epoch": 0.2354480052321779, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8366574645042419, + "kl": 0.03530883879866451, + "learning_rate": 4.4130150425114454e-07, + "loss": 0.10644701719284058, + "num_tokens": 9147587.0, + "reward": -1.15185978266652e-07, + "reward_std": 0.9983633041381836, + "rewards/rnd_reward/mean": -1.1518597684556652e-07, + "rewards/rnd_reward/std": 0.9983633100986481, + "step": 360, + "step_time": 9.722514500608668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17000000029802323, + "completions/max_length": 384.0, + "completions/max_terminated_length": 366.5, + "completions/mean_length": 130.59125213623048, + "completions/mean_terminated_length": 79.17214775085449, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3677036464214325, + "epoch": 0.2419882275997384, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.964614987373352, + "kl": 0.043067421624436976, + "learning_rate": 4.3966644865925437e-07, + "loss": 0.08349081873893738, + "num_tokens": 9340010.0, + "reward": -3.0696392450124675e-08, + "reward_std": 0.9982355415821076, + "rewards/rnd_reward/mean": -3.0696389607953736e-08, + "rewards/rnd_reward/std": 0.9982355415821076, + "step": 370, + "step_time": 9.710164245427586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1887500010430813, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.2, + "completions/mean_length": 138.41875305175782, + "completions/mean_terminated_length": 81.47324066162109, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.258895403146744, + "epoch": 0.2485284499672989, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7839703559875488, + "kl": 0.039375259960070255, + "learning_rate": 4.3803139306736425e-07, + "loss": 0.08618925213813781, + "num_tokens": 9545205.0, + "reward": -1.0460615040841503e-07, + "reward_std": 0.998303747177124, + "rewards/rnd_reward/mean": -1.046061518295005e-07, + "rewards/rnd_reward/std": 0.9983037531375885, + "step": 380, + "step_time": 9.776164471404627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1625000037252903, + "completions/max_length": 384.0, + "completions/max_terminated_length": 342.9, + "completions/mean_length": 123.14125137329101, + "completions/mean_terminated_length": 72.26187210083008, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2043879717588424, + "epoch": 0.25506867233485936, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.013533353805542, + "kl": 0.045199727965518834, + "learning_rate": 4.363963374754742e-07, + "loss": 0.08801217675209046, + "num_tokens": 9732323.0, + "reward": -1.3418495639072603e-07, + "reward_std": 0.9981010854244232, + "rewards/rnd_reward/mean": -1.3418495496964057e-07, + "rewards/rnd_reward/std": 0.9981010735034943, + "step": 390, + "step_time": 9.470399480173365 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16500000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.9, + "completions/mean_length": 126.54125213623047, + "completions/mean_terminated_length": 75.62271652221679, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.1946812689304354, + "epoch": 0.2616088947024199, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9289289712905884, + "kl": 0.043896416388452056, + "learning_rate": 4.34761281883584e-07, + "loss": 0.06013514399528504, + "num_tokens": 9923251.0, + "reward": -1.1950731622079046e-07, + "reward_std": 0.9980400562286377, + "rewards/rnd_reward/mean": -1.19507314799705e-07, + "rewards/rnd_reward/std": 0.9980400741100312, + "step": 400, + "step_time": 9.578276086249389 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19000000283122062, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.4, + "completions/mean_length": 142.2825019836426, + "completions/mean_terminated_length": 85.19309997558594, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3042476952075956, + "epoch": 0.2681491170699804, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8114489316940308, + "kl": 0.039571146154776214, + "learning_rate": 4.331262262916939e-07, + "loss": 0.08236328959465027, + "num_tokens": 10132682.0, + "reward": -1.4752151500374566e-08, + "reward_std": 0.9982209146022797, + "rewards/rnd_reward/mean": -1.4752152210917303e-08, + "rewards/rnd_reward/std": 0.9982209086418152, + "step": 410, + "step_time": 9.807740885205567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.20625000149011613, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 149.2062530517578, + "completions/mean_terminated_length": 88.13671951293945, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.38392159640789, + "epoch": 0.27468933943754087, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7539530396461487, + "kl": 0.03540583737194538, + "learning_rate": 4.3149117069980376e-07, + "loss": 0.08869684934616089, + "num_tokens": 10345522.0, + "reward": -1.3530254712179611e-07, + "reward_std": 0.9981609523296356, + "rewards/rnd_reward/mean": -1.3530254356908243e-07, + "rewards/rnd_reward/std": 0.9981609761714936, + "step": 420, + "step_time": 9.619695667061023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18750000223517418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.4, + "completions/mean_length": 139.9700035095215, + "completions/mean_terminated_length": 83.52344436645508, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.225811779499054, + "epoch": 0.2812295618051014, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7882169485092163, + "kl": 0.031479011243209246, + "learning_rate": 4.2985611510791364e-07, + "loss": 0.09766495227813721, + "num_tokens": 10563093.0, + "reward": -1.4021993024471158e-07, + "reward_std": 0.998052579164505, + "rewards/rnd_reward/mean": -1.4021992598145516e-07, + "rewards/rnd_reward/std": 0.9980525553226471, + "step": 430, + "step_time": 11.129851599340327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18875000178813933, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.8, + "completions/mean_length": 143.57625198364258, + "completions/mean_terminated_length": 87.31694107055664, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.1123261958360673, + "epoch": 0.28776978417266186, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7737316489219666, + "kl": 0.0280194781254977, + "learning_rate": 4.2822105951602357e-07, + "loss": 0.10439527034759521, + "num_tokens": 10766159.0, + "reward": -4.2766332430943524e-08, + "reward_std": 0.998040771484375, + "rewards/rnd_reward/mean": -4.276633118749373e-08, + "rewards/rnd_reward/std": 0.998040747642517, + "step": 440, + "step_time": 9.481722956942395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16625000312924385, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.1, + "completions/mean_length": 131.25125427246093, + "completions/mean_terminated_length": 80.89927139282227, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2791686803102493, + "epoch": 0.2943100065402224, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.834428608417511, + "kl": 0.04166741594672203, + "learning_rate": 4.265860039241334e-07, + "loss": 0.07646016478538513, + "num_tokens": 10965315.0, + "reward": -1.0013580258316779e-07, + "reward_std": 0.998044753074646, + "rewards/rnd_reward/mean": -1.0013580116208232e-07, + "rewards/rnd_reward/std": 0.9980447709560394, + "step": 450, + "step_time": 10.756261567212642 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13750000298023224, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.9, + "completions/mean_length": 118.16625137329102, + "completions/mean_terminated_length": 75.56828384399414, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3162699341773987, + "epoch": 0.30085022890778285, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1823770999908447, + "kl": 0.04914104142226279, + "learning_rate": 4.249509483322433e-07, + "loss": 0.09161773324012756, + "num_tokens": 11151658.0, + "reward": 1.4454126073815132e-08, + "reward_std": 0.9979428768157959, + "rewards/rnd_reward/mean": 1.4454126073815132e-08, + "rewards/rnd_reward/std": 0.9979428887367249, + "step": 460, + "step_time": 9.690541778854094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625000596046448, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.0, + "completions/mean_length": 129.17875442504882, + "completions/mean_terminated_length": 81.90925521850586, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.250484657287598, + "epoch": 0.30739045127534337, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7849432229995728, + "kl": 0.044866825267672536, + "learning_rate": 4.2331589274035315e-07, + "loss": 0.1366190195083618, + "num_tokens": 11350266.0, + "reward": -5.528331037396583e-08, + "reward_std": 0.997910988330841, + "rewards/rnd_reward/mean": -5.528330859760899e-08, + "rewards/rnd_reward/std": 0.997911012172699, + "step": 470, + "step_time": 10.094013441796415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14375000149011613, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.1, + "completions/mean_length": 121.7000015258789, + "completions/mean_terminated_length": 77.64777450561523, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.356913906335831, + "epoch": 0.31393067364290383, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7464061975479126, + "kl": 0.047594004729762676, + "learning_rate": 4.2168083714846303e-07, + "loss": 0.093714839220047, + "num_tokens": 11546191.0, + "reward": -2.1904707381281696e-08, + "reward_std": 0.9980335652828216, + "rewards/rnd_reward/mean": -2.1904707381281696e-08, + "rewards/rnd_reward/std": 0.9980335593223572, + "step": 480, + "step_time": 10.49183724487666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1312500022351742, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.0, + "completions/mean_length": 110.71375274658203, + "completions/mean_terminated_length": 69.09722747802735, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.3989109843969345, + "epoch": 0.32047089601046436, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9596973061561584, + "kl": 0.059090607427060605, + "learning_rate": 4.200457815565729e-07, + "loss": 0.061995261907577516, + "num_tokens": 11729672.0, + "reward": -8.441507906020718e-08, + "reward_std": 0.9977342307567596, + "rewards/rnd_reward/mean": -8.441507870493581e-08, + "rewards/rnd_reward/std": 0.9977342247962951, + "step": 490, + "step_time": 9.824798973021098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000193715094, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.2, + "completions/mean_length": 110.2125015258789, + "completions/mean_terminated_length": 69.26982688903809, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.335902637243271, + "epoch": 0.3270111183780249, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0347245931625366, + "kl": 0.06009262367151678, + "learning_rate": 4.184107259646828e-07, + "loss": 0.011750234663486481, + "num_tokens": 11908552.0, + "reward": -7.688999374977356e-08, + "reward_std": 0.9977168381214142, + "rewards/rnd_reward/mean": -7.688999019705988e-08, + "rewards/rnd_reward/std": 0.9977168321609498, + "step": 500, + "step_time": 9.731353901792318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1450000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.2, + "completions/mean_length": 118.08375244140625, + "completions/mean_terminated_length": 73.00995483398438, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.557803976535797, + "epoch": 0.33355134074558535, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9074205160140991, + "kl": 0.06632269411347806, + "learning_rate": 4.1677567037279267e-07, + "loss": 0.08373542428016663, + "num_tokens": 12103254.0, + "reward": -1.1697411828048843e-07, + "reward_std": 0.9978465795516968, + "rewards/rnd_reward/mean": -1.1697411828048843e-07, + "rewards/rnd_reward/std": 0.9978465795516968, + "step": 510, + "step_time": 10.207280423282645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15875000208616258, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.2, + "completions/mean_length": 123.88500137329102, + "completions/mean_terminated_length": 74.63431396484376, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.437187620997429, + "epoch": 0.34009156311314587, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8857874870300293, + "kl": 0.06048318287357688, + "learning_rate": 4.151406147809025e-07, + "loss": 0.037069684267044066, + "num_tokens": 12302092.0, + "reward": -3.874301874873254e-08, + "reward_std": 0.9979513227939606, + "rewards/rnd_reward/mean": -3.874301874873254e-08, + "rewards/rnd_reward/std": 0.9979513168334961, + "step": 520, + "step_time": 10.34260532106273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17000000253319741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.1, + "completions/mean_length": 132.52125396728516, + "completions/mean_terminated_length": 80.95349197387695, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4997961223125458, + "epoch": 0.34663178548070633, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9028640985488892, + "kl": 0.05214450396597385, + "learning_rate": 4.135055591890124e-07, + "loss": 0.07458721995353698, + "num_tokens": 12499434.0, + "reward": -2.2247433655309124e-07, + "reward_std": 0.9978251278400421, + "rewards/rnd_reward/mean": -2.2247433655309124e-07, + "rewards/rnd_reward/std": 0.9978251218795776, + "step": 530, + "step_time": 9.664499558508396 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15125000178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.4, + "completions/mean_length": 120.7087516784668, + "completions/mean_terminated_length": 73.79461898803712, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.547806602716446, + "epoch": 0.35317200784826686, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0128388404846191, + "kl": 0.05931666204705834, + "learning_rate": 4.118705035971223e-07, + "loss": 0.11692253351211548, + "num_tokens": 12683096.0, + "reward": -8.344649238267721e-09, + "reward_std": 0.9977494418621063, + "rewards/rnd_reward/mean": -8.344651369895927e-09, + "rewards/rnd_reward/std": 0.9977494418621063, + "step": 540, + "step_time": 9.593502512201667 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13500000387430192, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.6, + "completions/mean_length": 111.18750305175782, + "completions/mean_terminated_length": 68.47658958435059, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.681893822550774, + "epoch": 0.3597122302158273, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1282111406326294, + "kl": 0.06902076229453087, + "learning_rate": 4.1023544800523213e-07, + "loss": 0.023469969630241394, + "num_tokens": 12862906.0, + "reward": -5.677342400645102e-08, + "reward_std": 0.9978136360645294, + "rewards/rnd_reward/mean": -5.6773422585365554e-08, + "rewards/rnd_reward/std": 0.997813630104065, + "step": 550, + "step_time": 9.778036265238189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12000000327825547, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.0, + "completions/mean_length": 104.53875045776367, + "completions/mean_terminated_length": 66.3183536529541, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5951172381639482, + "epoch": 0.36625245258338784, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1439512968063354, + "kl": 0.07057485273107886, + "learning_rate": 4.0860039241334206e-07, + "loss": 0.1278893232345581, + "num_tokens": 13049132.0, + "reward": -1.1786818543413347e-07, + "reward_std": 0.9977734804153442, + "rewards/rnd_reward/mean": -1.17868184013048e-07, + "rewards/rnd_reward/std": 0.9977734982967377, + "step": 560, + "step_time": 10.389736549765804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10500000044703484, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.3, + "completions/mean_length": 99.66875076293945, + "completions/mean_terminated_length": 66.29649963378907, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6503134578466416, + "epoch": 0.3727926749509483, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0748413801193237, + "kl": 0.08892364026978612, + "learning_rate": 4.069653368214519e-07, + "loss": 0.13109040260314941, + "num_tokens": 13221157.0, + "reward": -1.4342367933295462e-07, + "reward_std": 0.997740238904953, + "rewards/rnd_reward/mean": -1.4342367791186916e-07, + "rewards/rnd_reward/std": 0.9977402448654175, + "step": 570, + "step_time": 9.71618521134369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12250000312924385, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 110.21125030517578, + "completions/mean_terminated_length": 71.826904296875, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.702208098769188, + "epoch": 0.37933289731850883, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0691598653793335, + "kl": 0.07306001400575042, + "learning_rate": 4.053302812295618e-07, + "loss": 0.09580587744712829, + "num_tokens": 13409746.0, + "reward": -1.6450882269758438e-07, + "reward_std": 0.9975189864635468, + "rewards/rnd_reward/mean": -1.6450882198704164e-07, + "rewards/rnd_reward/std": 0.9975189983844757, + "step": 580, + "step_time": 10.602098648622633 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13250000216066837, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.2, + "completions/mean_length": 118.48125228881835, + "completions/mean_terminated_length": 77.98408432006836, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5411072075366974, + "epoch": 0.3858731196860693, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8738215565681458, + "kl": 0.06685408288612962, + "learning_rate": 4.0369522563767164e-07, + "loss": 0.0531582236289978, + "num_tokens": 13603691.0, + "reward": -8.791685381925874e-08, + "reward_std": 0.9976041257381439, + "rewards/rnd_reward/mean": -8.791684955600231e-08, + "rewards/rnd_reward/std": 0.9976041257381439, + "step": 590, + "step_time": 10.606248078867793 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13375000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.1, + "completions/mean_length": 120.9775001525879, + "completions/mean_terminated_length": 80.35308418273925, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.454717567563057, + "epoch": 0.3924133420536298, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8597258925437927, + "kl": 0.06251584179699421, + "learning_rate": 4.020601700457815e-07, + "loss": 0.10086486339569092, + "num_tokens": 13793088.0, + "reward": -4.872679788547885e-08, + "reward_std": 0.997677344083786, + "rewards/rnd_reward/mean": -4.872679859602158e-08, + "rewards/rnd_reward/std": 0.997677332162857, + "step": 600, + "step_time": 9.838808755297213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13875000327825546, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.2, + "completions/mean_length": 112.62250213623047, + "completions/mean_terminated_length": 69.04776878356934, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5064212560653685, + "epoch": 0.39895356442119034, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0397368669509888, + "kl": 0.06775042302906513, + "learning_rate": 4.0042511445389145e-07, + "loss": 0.14560239315032958, + "num_tokens": 13975906.0, + "reward": -1.70022253698221e-07, + "reward_std": 0.997452187538147, + "rewards/rnd_reward/mean": -1.7002224872442184e-07, + "rewards/rnd_reward/std": 0.9974521696567535, + "step": 610, + "step_time": 9.711915594828316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11000000387430191, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.5, + "completions/mean_length": 105.6062515258789, + "completions/mean_terminated_length": 71.07123832702636, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.531564822793007, + "epoch": 0.4054937867887508, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8839622139930725, + "kl": 0.07682965183630586, + "learning_rate": 3.987900588620013e-07, + "loss": 0.06944795846939086, + "num_tokens": 14154656.0, + "reward": 5.662443069986978e-09, + "reward_std": 0.9975684463977814, + "rewards/rnd_reward/mean": 5.662442181808558e-09, + "rewards/rnd_reward/std": 0.9975684404373169, + "step": 620, + "step_time": 10.306970553193242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10375000201165677, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.6, + "completions/mean_length": 106.93500137329102, + "completions/mean_terminated_length": 74.97318267822266, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.859010672569275, + "epoch": 0.41203400915631133, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.103442907333374, + "kl": 0.08963548736646772, + "learning_rate": 3.9715500327011115e-07, + "loss": 0.033435261249542235, + "num_tokens": 14327599.0, + "reward": -1.2174248684715394e-07, + "reward_std": 0.9974590480327606, + "rewards/rnd_reward/mean": -1.2174248613661122e-07, + "rewards/rnd_reward/std": 0.9974590301513672, + "step": 630, + "step_time": 9.607878714823164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12625000327825547, + "completions/max_length": 384.0, + "completions/max_terminated_length": 343.2, + "completions/mean_length": 108.95125045776368, + "completions/mean_terminated_length": 69.31592407226563, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.823798879981041, + "epoch": 0.4185742315238718, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1582435369491577, + "kl": 0.09182370994240045, + "learning_rate": 3.9551994767822103e-07, + "loss": 0.06608177423477173, + "num_tokens": 14513650.0, + "reward": -1.3239681819765537e-07, + "reward_std": 0.9976047217845917, + "rewards/rnd_reward/mean": -1.3239681961874084e-07, + "rewards/rnd_reward/std": 0.9976047277450562, + "step": 640, + "step_time": 10.307288937154226 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11000000163912774, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.8, + "completions/mean_length": 107.63500213623047, + "completions/mean_terminated_length": 73.39098167419434, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.695296034216881, + "epoch": 0.4251144538914323, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0280959606170654, + "kl": 0.09326308779418468, + "learning_rate": 3.938848920863309e-07, + "loss": 0.030906683206558226, + "num_tokens": 14693328.0, + "reward": -1.2964010291938167e-07, + "reward_std": 0.9975219249725342, + "rewards/rnd_reward/mean": -1.2964010327465302e-07, + "rewards/rnd_reward/std": 0.9975219309329987, + "step": 650, + "step_time": 10.01973620983772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1512500040233135, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.3, + "completions/mean_length": 129.62250213623048, + "completions/mean_terminated_length": 84.16030578613281, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.613438582420349, + "epoch": 0.4316546762589928, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9445599317550659, + "kl": 0.0699499910697341, + "learning_rate": 3.922498364944408e-07, + "loss": 0.08974806666374206, + "num_tokens": 14892686.0, + "reward": -4.194676717084178e-08, + "reward_std": 0.9975471794605255, + "rewards/rnd_reward/mean": -4.1946770057421645e-08, + "rewards/rnd_reward/std": 0.99754718542099, + "step": 660, + "step_time": 10.161354359728284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14750000312924386, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.7, + "completions/mean_length": 126.59500045776367, + "completions/mean_terminated_length": 81.99441680908203, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.7322494119405745, + "epoch": 0.4381948986265533, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9810017943382263, + "kl": 0.08053934266790748, + "learning_rate": 3.9061478090255067e-07, + "loss": 0.14365124702453613, + "num_tokens": 15083047.0, + "reward": -7.979571803318208e-08, + "reward_std": 0.9972860395908356, + "rewards/rnd_reward/mean": -7.979571803318208e-08, + "rewards/rnd_reward/std": 0.9972860395908356, + "step": 670, + "step_time": 9.683341883006506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14250000119209288, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.5, + "completions/mean_length": 125.95000381469727, + "completions/mean_terminated_length": 83.18684883117676, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6392508417367937, + "epoch": 0.4447351209941138, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9357602596282959, + "kl": 0.07438107710331679, + "learning_rate": 3.8897972531066054e-07, + "loss": 0.047136351466178894, + "num_tokens": 15272992.0, + "reward": -1.0043382800972722e-07, + "reward_std": 0.9973943531513214, + "rewards/rnd_reward/mean": -1.0043382658864175e-07, + "rewards/rnd_reward/std": 0.997394347190857, + "step": 680, + "step_time": 9.536431295378133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14375000223517417, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.2, + "completions/mean_length": 125.81125106811524, + "completions/mean_terminated_length": 82.36017532348633, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.2503296345472337, + "epoch": 0.4512753433616743, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7484872937202454, + "kl": 0.0611060356721282, + "learning_rate": 3.873446697187704e-07, + "loss": 0.09799213409423828, + "num_tokens": 15463906.0, + "reward": -6.347894991876046e-08, + "reward_std": 0.9972564399242401, + "rewards/rnd_reward/mean": -6.34789495634891e-08, + "rewards/rnd_reward/std": 0.9972564518451691, + "step": 690, + "step_time": 9.653547045495362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15500000342726708, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.6, + "completions/mean_length": 133.82500076293945, + "completions/mean_terminated_length": 87.98081855773925, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5280281245708465, + "epoch": 0.4578155657292348, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9658915996551514, + "kl": 0.059800021536648276, + "learning_rate": 3.857096141268803e-07, + "loss": 0.12680201530456542, + "num_tokens": 15667301.0, + "reward": -1.5869737026719122e-07, + "reward_std": 0.9973293662071228, + "rewards/rnd_reward/mean": -1.5869737026719122e-07, + "rewards/rnd_reward/std": 0.9973293721675873, + "step": 700, + "step_time": 9.966963570658118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10875000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.1, + "completions/mean_length": 107.25000228881837, + "completions/mean_terminated_length": 73.4811897277832, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4395824640989305, + "epoch": 0.4643557880967953, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.049780011177063, + "kl": 0.08286946956068278, + "learning_rate": 3.840745585349902e-07, + "loss": 0.15116369724273682, + "num_tokens": 15847671.0, + "reward": -1.005828406341891e-07, + "reward_std": 0.9973811268806457, + "rewards/rnd_reward/mean": -1.0058283921310362e-07, + "rewards/rnd_reward/std": 0.9973811089992524, + "step": 710, + "step_time": 9.890902339830063 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09000000115484, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.2, + "completions/mean_length": 93.01625213623046, + "completions/mean_terminated_length": 64.09797096252441, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8066621690988542, + "epoch": 0.4708960104643558, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34576416015625, + "kl": 0.10443261573091149, + "learning_rate": 3.8243950294310006e-07, + "loss": 0.10605430603027344, + "num_tokens": 16021059.0, + "reward": -1.0505319210096787e-07, + "reward_std": 0.9972734034061432, + "rewards/rnd_reward/mean": -1.0505318357445503e-07, + "rewards/rnd_reward/std": 0.9972733795642853, + "step": 720, + "step_time": 10.098007511859759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000074505807, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 94.07875137329101, + "completions/mean_terminated_length": 66.15557479858398, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6466055154800414, + "epoch": 0.4774362328319163, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0772963762283325, + "kl": 0.10096871191635728, + "learning_rate": 3.8080444735120994e-07, + "loss": 0.09338142871856689, + "num_tokens": 16191222.0, + "reward": -1.777708558137192e-07, + "reward_std": 0.9971938729286194, + "rewards/rnd_reward/mean": -1.7777085012937732e-07, + "rewards/rnd_reward/std": 0.9971938788890838, + "step": 730, + "step_time": 10.299412568542175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13375000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.1, + "completions/mean_length": 126.26250228881835, + "completions/mean_terminated_length": 86.43918380737304, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.173070195317268, + "epoch": 0.4839764551994768, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7416156530380249, + "kl": 0.053471417538821696, + "learning_rate": 3.7916939175931976e-07, + "loss": 0.07696372270584106, + "num_tokens": 16393972.0, + "reward": -1.7151236602064303e-07, + "reward_std": 0.9972269594669342, + "rewards/rnd_reward/mean": -1.715123674417285e-07, + "rewards/rnd_reward/std": 0.9972269415855408, + "step": 740, + "step_time": 10.502406498114578 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.148750002682209, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 139.12875213623047, + "completions/mean_terminated_length": 96.77134628295899, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.325762075185776, + "epoch": 0.49051667756703726, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7352995872497559, + "kl": 0.05052672061137855, + "learning_rate": 3.775343361674297e-07, + "loss": 0.09521990418434143, + "num_tokens": 16596670.0, + "reward": -2.0787120362797397e-07, + "reward_std": 0.9972488164901734, + "rewards/rnd_reward/mean": -2.0787119723308933e-07, + "rewards/rnd_reward/std": 0.9972488105297088, + "step": 750, + "step_time": 9.698242208594456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14750000238418579, + "completions/max_length": 384.0, + "completions/max_terminated_length": 371.2, + "completions/mean_length": 127.9612533569336, + "completions/mean_terminated_length": 83.97822761535645, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.2959645807743074, + "epoch": 0.4970568999345978, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8263393044471741, + "kl": 0.05697637004777789, + "learning_rate": 3.7589928057553957e-07, + "loss": 0.10516003370285035, + "num_tokens": 16792629.0, + "reward": -1.6942620391091622e-07, + "reward_std": 0.9972022175788879, + "rewards/rnd_reward/mean": -1.6942620106874529e-07, + "rewards/rnd_reward/std": 0.9972022175788879, + "step": 760, + "step_time": 10.044187037739903 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10875000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 339.2, + "completions/mean_length": 112.1150016784668, + "completions/mean_terminated_length": 79.13344192504883, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7552184253931045, + "epoch": 0.5035971223021583, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.029379963874817, + "kl": 0.07228619959205389, + "learning_rate": 3.742642249836494e-07, + "loss": 0.03081640601158142, + "num_tokens": 16971246.0, + "reward": -8.299946501200139e-08, + "reward_std": 0.9970763862133026, + "rewards/rnd_reward/mean": -8.299946785417234e-08, + "rewards/rnd_reward/std": 0.9970763862133026, + "step": 770, + "step_time": 9.667120404168964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13125000298023223, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.9, + "completions/mean_length": 124.3275032043457, + "completions/mean_terminated_length": 85.28903274536133, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.8021608620882033, + "epoch": 0.5101373446697187, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0165283679962158, + "kl": 0.07025760738179088, + "learning_rate": 3.7262916939175933e-07, + "loss": 0.03996403813362122, + "num_tokens": 17162308.0, + "reward": -5.170702825907369e-08, + "reward_std": 0.9971189498901367, + "rewards/rnd_reward/mean": -5.170703110124464e-08, + "rewards/rnd_reward/std": 0.9971189439296723, + "step": 780, + "step_time": 9.969513631938025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13375000283122063, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.5, + "completions/mean_length": 127.03500213623047, + "completions/mean_terminated_length": 87.38848876953125, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.637522208690643, + "epoch": 0.5166775670372793, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9886835813522339, + "kl": 0.06816753121092915, + "learning_rate": 3.7099411379986915e-07, + "loss": 0.07478119730949402, + "num_tokens": 17356536.0, + "reward": -1.1831521575800253e-07, + "reward_std": 0.9971611678600312, + "rewards/rnd_reward/mean": -1.1831521575800253e-07, + "rewards/rnd_reward/std": 0.997161191701889, + "step": 790, + "step_time": 9.748413591040299 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12625000327825547, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.3, + "completions/mean_length": 120.86375045776367, + "completions/mean_terminated_length": 83.05458145141601, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6706619918346406, + "epoch": 0.5232177894048398, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8250897526741028, + "kl": 0.07074365848675371, + "learning_rate": 3.693590582079791e-07, + "loss": 0.05061581134796143, + "num_tokens": 17545417.0, + "reward": -6.467104096685716e-08, + "reward_std": 0.9972768127918243, + "rewards/rnd_reward/mean": -6.467104096685716e-08, + "rewards/rnd_reward/std": 0.9972768068313599, + "step": 800, + "step_time": 9.678878564713523 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12500000298023223, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.5, + "completions/mean_length": 121.30500183105468, + "completions/mean_terminated_length": 83.6633544921875, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.795678195357323, + "epoch": 0.5297580117724002, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0277215242385864, + "kl": 0.06990557787939906, + "learning_rate": 3.677240026160889e-07, + "loss": 0.059202158451080324, + "num_tokens": 17737161.0, + "reward": -1.2487173535191686e-07, + "reward_std": 0.9970629572868347, + "rewards/rnd_reward/mean": -1.2487172682540403e-07, + "rewards/rnd_reward/std": 0.9970629572868347, + "step": 810, + "step_time": 10.510143173905089 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.4, + "completions/mean_length": 119.66500091552734, + "completions/mean_terminated_length": 80.23575859069824, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8059179961681364, + "epoch": 0.5362982341399608, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8309988379478455, + "kl": 0.08165676584467292, + "learning_rate": 3.660889470241988e-07, + "loss": 0.12505731582641602, + "num_tokens": 17927903.0, + "reward": -2.3901462826714236e-07, + "reward_std": 0.997013247013092, + "rewards/rnd_reward/mean": -2.3901462400388593e-07, + "rewards/rnd_reward/std": 0.9970132112503052, + "step": 820, + "step_time": 10.190648046904244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14875000417232515, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.9, + "completions/mean_length": 126.17125396728515, + "completions/mean_terminated_length": 80.60145225524903, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.938824012875557, + "epoch": 0.5428384565075213, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1022851467132568, + "kl": 0.07984373616054655, + "learning_rate": 3.644538914323087e-07, + "loss": 0.023773609101772307, + "num_tokens": 18123110.0, + "reward": -2.2649766062698973e-08, + "reward_std": 0.9969716727733612, + "rewards/rnd_reward/mean": -2.2649762509985294e-08, + "rewards/rnd_reward/std": 0.9969716787338256, + "step": 830, + "step_time": 9.89921972535085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14375000447034836, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.7, + "completions/mean_length": 127.39125366210938, + "completions/mean_terminated_length": 84.2758415222168, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8030223518610002, + "epoch": 0.5493786788750817, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9427273273468018, + "kl": 0.0753337662667036, + "learning_rate": 3.6281883584041854e-07, + "loss": 0.05915579795837402, + "num_tokens": 18319518.0, + "reward": -1.1295080284412507e-07, + "reward_std": 0.9969306707382202, + "rewards/rnd_reward/mean": -1.1295080142303959e-07, + "rewards/rnd_reward/std": 0.9969306766986847, + "step": 840, + "step_time": 9.87428822228685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15750000178813933, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.2, + "completions/mean_length": 137.7587516784668, + "completions/mean_terminated_length": 91.88029098510742, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.810399681329727, + "epoch": 0.5559189012426422, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8602397441864014, + "kl": 0.0653490168042481, + "learning_rate": 3.611837802485284e-07, + "loss": 0.03035195469856262, + "num_tokens": 18526310.0, + "reward": -1.1295080284412507e-07, + "reward_std": 0.9969462454319, + "rewards/rnd_reward/mean": -1.1295080426521054e-07, + "rewards/rnd_reward/std": 0.9969462633132935, + "step": 850, + "step_time": 10.37288966330234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15250000283122062, + "completions/max_length": 384.0, + "completions/max_terminated_length": 343.9, + "completions/mean_length": 127.73250350952148, + "completions/mean_terminated_length": 81.79599456787109, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.83120773434639, + "epoch": 0.5624591236102028, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0732090473175049, + "kl": 0.08459939705207944, + "learning_rate": 3.595487246566383e-07, + "loss": 0.03491010665893555, + "num_tokens": 18724316.0, + "reward": -1.4752149617436318e-07, + "reward_std": 0.9969268858432769, + "rewards/rnd_reward/mean": -1.4752149759544864e-07, + "rewards/rnd_reward/std": 0.9969269037246704, + "step": 860, + "step_time": 10.52987401576247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14000000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.0, + "completions/mean_length": 131.95375213623046, + "completions/mean_terminated_length": 90.77078781127929, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.9562632203102113, + "epoch": 0.5689993459777632, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.969403862953186, + "kl": 0.08435474634170533, + "learning_rate": 3.579136690647482e-07, + "loss": 0.04807090759277344, + "num_tokens": 18926489.0, + "reward": -1.3545155894689742e-07, + "reward_std": 0.9968080937862396, + "rewards/rnd_reward/mean": -1.3545156178906837e-07, + "rewards/rnd_reward/std": 0.9968081057071686, + "step": 870, + "step_time": 9.85819350676611 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16750000193715095, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.2, + "completions/mean_length": 137.6675033569336, + "completions/mean_terminated_length": 88.34969863891601, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7675389796495438, + "epoch": 0.5755395683453237, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.828363835811615, + "kl": 0.07512732800096274, + "learning_rate": 3.5627861347285806e-07, + "loss": 0.010637739300727844, + "num_tokens": 19126833.0, + "reward": -7.003545690054125e-08, + "reward_std": 0.9967847049236298, + "rewards/rnd_reward/mean": -7.003545903216945e-08, + "rewards/rnd_reward/std": 0.9967847049236298, + "step": 880, + "step_time": 9.748134864587337 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19125000312924384, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.5, + "completions/mean_length": 148.54750366210936, + "completions/mean_terminated_length": 93.13098068237305, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.6605105638504027, + "epoch": 0.5820797907128843, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8364513516426086, + "kl": 0.06730008013546467, + "learning_rate": 3.5464355788096794e-07, + "loss": 0.0366318553686142, + "num_tokens": 19338336.0, + "reward": -1.0654330893089536e-07, + "reward_std": 0.9968020260334015, + "rewards/rnd_reward/mean": -1.0654329685166886e-07, + "rewards/rnd_reward/std": 0.9968020260334015, + "step": 890, + "step_time": 9.872815124248154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16625000312924385, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.5, + "completions/mean_length": 149.7400032043457, + "completions/mean_terminated_length": 103.11838226318359, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.535254544019699, + "epoch": 0.5886200130804448, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7181042432785034, + "kl": 0.05683262627571821, + "learning_rate": 3.530085022890778e-07, + "loss": 0.08454052209854127, + "num_tokens": 19557363.0, + "reward": -1.737475452401327e-07, + "reward_std": 0.9967290103435517, + "rewards/rnd_reward/mean": -1.737475338714489e-07, + "rewards/rnd_reward/std": 0.9967289924621582, + "step": 900, + "step_time": 10.189786232402549 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1912500038743019, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 157.67875213623046, + "completions/mean_terminated_length": 104.52761840820312, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5941266864538193, + "epoch": 0.5951602354480052, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9426244497299194, + "kl": 0.050556823750957845, + "learning_rate": 3.513734466971877e-07, + "loss": 0.07448753714561462, + "num_tokens": 19775536.0, + "reward": -2.162158537544201e-07, + "reward_std": 0.9968621492385864, + "rewards/rnd_reward/mean": -2.1621584735953548e-07, + "rewards/rnd_reward/std": 0.9968621671199799, + "step": 910, + "step_time": 9.74868165422231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1662500023841858, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.0, + "completions/mean_length": 151.89000396728517, + "completions/mean_terminated_length": 105.61829071044922, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.508600893616676, + "epoch": 0.6017004578155657, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7371200323104858, + "kl": 0.04880232191644609, + "learning_rate": 3.4973839110529757e-07, + "loss": 0.12514564990997315, + "num_tokens": 19990298.0, + "reward": -1.93715107066339e-08, + "reward_std": 0.9970090925693512, + "rewards/rnd_reward/mean": -1.937150893027706e-08, + "rewards/rnd_reward/std": 0.9970090866088868, + "step": 920, + "step_time": 9.937099263281562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1575000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.5, + "completions/mean_length": 138.63375244140624, + "completions/mean_terminated_length": 92.52680130004883, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4236575067043304, + "epoch": 0.6082406801831263, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9167628884315491, + "kl": 0.05787700763903558, + "learning_rate": 3.4810333551340745e-07, + "loss": 0.10835990905761719, + "num_tokens": 20187745.0, + "reward": -7.212161943215278e-08, + "reward_std": 0.996803218126297, + "rewards/rnd_reward/mean": -7.212161801106731e-08, + "rewards/rnd_reward/std": 0.9968032300472259, + "step": 930, + "step_time": 9.689079960295931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1337500013411045, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.4, + "completions/mean_length": 129.22500305175782, + "completions/mean_terminated_length": 89.93161087036133, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.6605860769748686, + "epoch": 0.6147809025506867, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0838011503219604, + "kl": 0.06689512394368649, + "learning_rate": 3.4646827992151733e-07, + "loss": 0.07303244471549988, + "num_tokens": 20383990.0, + "reward": -1.1324883786301143e-08, + "reward_std": 0.9967760264873504, + "rewards/rnd_reward/mean": -1.1324883786301143e-08, + "rewards/rnd_reward/std": 0.9967760384082794, + "step": 940, + "step_time": 9.85847494918853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16125000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 364.1, + "completions/mean_length": 134.65625534057617, + "completions/mean_terminated_length": 86.58066329956054, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7146200090646744, + "epoch": 0.6213211249182472, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9745354652404785, + "kl": 0.06564742000773549, + "learning_rate": 3.448332243296272e-07, + "loss": 0.10424284934997559, + "num_tokens": 20593190.0, + "reward": -1.0967255157368072e-07, + "reward_std": 0.9968843460083008, + "rewards/rnd_reward/mean": -1.096725451787961e-07, + "rewards/rnd_reward/std": 0.9968843400478363, + "step": 950, + "step_time": 10.862376277497969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10875000022351741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.1, + "completions/mean_length": 113.23000411987304, + "completions/mean_terminated_length": 80.14122467041015, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8509408473968505, + "epoch": 0.6278613472858077, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9323987364768982, + "kl": 0.08510139184072614, + "learning_rate": 3.4319816873773703e-07, + "loss": 0.0554282009601593, + "num_tokens": 20776164.0, + "reward": -7.957220322651893e-08, + "reward_std": 0.9968249201774597, + "rewards/rnd_reward/mean": -7.957220180543345e-08, + "rewards/rnd_reward/std": 0.9968249082565308, + "step": 960, + "step_time": 9.989999298867769 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10750000327825546, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.2, + "completions/mean_length": 112.88375091552734, + "completions/mean_terminated_length": 80.20179443359375, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.88864361345768, + "epoch": 0.6344015696533682, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8851020932197571, + "kl": 0.0875691649504006, + "learning_rate": 3.4156311314584696e-07, + "loss": 0.10126043558120727, + "num_tokens": 20958826.0, + "reward": -1.6622245713193706e-07, + "reward_std": 0.9968612968921662, + "rewards/rnd_reward/mean": -1.6622244931596696e-07, + "rewards/rnd_reward/std": 0.9968612730503082, + "step": 970, + "step_time": 10.018015099223703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11125000156462192, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.6, + "completions/mean_length": 106.1625, + "completions/mean_terminated_length": 71.38758811950683, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9656033277511598, + "epoch": 0.6409417920209287, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9377958178520203, + "kl": 0.09664760064333677, + "learning_rate": 3.3992805755395684e-07, + "loss": 0.046259531378746034, + "num_tokens": 21147866.0, + "reward": -2.499669804478799e-07, + "reward_std": 0.9968202471733093, + "rewards/rnd_reward/mean": -2.499669751188094e-07, + "rewards/rnd_reward/std": 0.9968202352523804, + "step": 980, + "step_time": 11.508573367865756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1262500025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.6, + "completions/mean_length": 122.56625289916992, + "completions/mean_terminated_length": 84.71166687011718, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.80318078994751, + "epoch": 0.6474820143884892, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9525420069694519, + "kl": 0.07310699988156558, + "learning_rate": 3.3829300196206667e-07, + "loss": 0.02322930097579956, + "num_tokens": 21338124.0, + "reward": -9.76771161731449e-08, + "reward_std": 0.9966861188411713, + "rewards/rnd_reward/mean": -9.7677110488803e-08, + "rewards/rnd_reward/std": 0.9966861248016358, + "step": 990, + "step_time": 9.876657200045884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15375000201165676, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.8, + "completions/mean_length": 137.66875076293945, + "completions/mean_terminated_length": 92.6232063293457, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9049280107021334, + "epoch": 0.6540222367560498, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8244759440422058, + "kl": 0.061428536847233775, + "learning_rate": 3.366579463701766e-07, + "loss": 0.05219428539276123, + "num_tokens": 21539579.0, + "reward": -1.6495585271059098e-07, + "reward_std": 0.9966946303844452, + "rewards/rnd_reward/mean": -1.6495585271059098e-07, + "rewards/rnd_reward/std": 0.9966946005821228, + "step": 1000, + "step_time": 9.666829459602013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18250000178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.9, + "completions/mean_length": 146.93875427246093, + "completions/mean_terminated_length": 93.79228134155274, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8016369342803955, + "epoch": 0.6605624591236102, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.766015350818634, + "kl": 0.05675441958010197, + "learning_rate": 3.350228907782864e-07, + "loss": 0.08666471242904664, + "num_tokens": 21754125.0, + "reward": -2.2858381782953073e-07, + "reward_std": 0.996807062625885, + "rewards/rnd_reward/mean": -2.2858381640844527e-07, + "rewards/rnd_reward/std": 0.9968070745468139, + "step": 1010, + "step_time": 10.005978721287102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13875000253319741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.3, + "completions/mean_length": 136.35500259399413, + "completions/mean_terminated_length": 96.46944885253906, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8755707025527952, + "epoch": 0.6671026814911707, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0061216354370117, + "kl": 0.05596274421550333, + "learning_rate": 3.3338783518639635e-07, + "loss": 0.09954084157943725, + "num_tokens": 21957634.0, + "reward": -1.9520521661320344e-07, + "reward_std": 0.9966324806213379, + "rewards/rnd_reward/mean": -1.9520520950777608e-07, + "rewards/rnd_reward/std": 0.9966325044631958, + "step": 1020, + "step_time": 10.040163300000131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15875000432133674, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.5, + "completions/mean_length": 136.7437530517578, + "completions/mean_terminated_length": 90.20843048095703, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7692076802253722, + "epoch": 0.6736429038587312, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8324238061904907, + "kl": 0.057948453910648824, + "learning_rate": 3.317527795945062e-07, + "loss": 0.07627512216567993, + "num_tokens": 22158654.0, + "reward": -3.7699937749380295e-08, + "reward_std": 0.9966008186340332, + "rewards/rnd_reward/mean": -3.769993632829482e-08, + "rewards/rnd_reward/std": 0.9966008424758911, + "step": 1030, + "step_time": 10.1797529961681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16250000298023223, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.7, + "completions/mean_length": 139.57500534057618, + "completions/mean_terminated_length": 92.21996688842773, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.8831230252981186, + "epoch": 0.6801831262262917, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8833721876144409, + "kl": 0.060850747767835854, + "learning_rate": 3.3011772400261606e-07, + "loss": 0.10484603643417359, + "num_tokens": 22366559.0, + "reward": -1.0505318579490108e-07, + "reward_std": 0.996761429309845, + "rewards/rnd_reward/mean": -1.0505318588371893e-07, + "rewards/rnd_reward/std": 0.9967614591121674, + "step": 1040, + "step_time": 10.081386449630372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1625000037252903, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.9, + "completions/mean_length": 137.33875198364257, + "completions/mean_terminated_length": 89.72673568725585, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.885282647609711, + "epoch": 0.6867233485938522, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9477590918540955, + "kl": 0.05932593122124672, + "learning_rate": 3.28482668410726e-07, + "loss": 0.024539683759212495, + "num_tokens": 22571575.0, + "reward": 4.9620866704458424e-08, + "reward_std": 0.996524053812027, + "rewards/rnd_reward/mean": 4.9620866704458424e-08, + "rewards/rnd_reward/std": 0.9965240597724915, + "step": 1050, + "step_time": 10.535549363144673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16250000447034835, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.5, + "completions/mean_length": 138.3237533569336, + "completions/mean_terminated_length": 90.70624694824218, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.875243663787842, + "epoch": 0.6932635709614127, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9407637119293213, + "kl": 0.06043377108871937, + "learning_rate": 3.268476128188358e-07, + "loss": 0.06624546647071838, + "num_tokens": 22771314.0, + "reward": -5.781650713743147e-08, + "reward_std": 0.9966591358184814, + "rewards/rnd_reward/mean": -5.7816505716346e-08, + "rewards/rnd_reward/std": 0.9966591358184814, + "step": 1060, + "step_time": 10.021701269014738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1400000039488077, + "completions/max_length": 384.0, + "completions/max_terminated_length": 342.1, + "completions/mean_length": 127.3300048828125, + "completions/mean_terminated_length": 85.42830429077148, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.927751249074936, + "epoch": 0.6998037933289731, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8508753180503845, + "kl": 0.0620661674067378, + "learning_rate": 3.252125572269457e-07, + "loss": 0.12821773290634156, + "num_tokens": 22965548.0, + "reward": -1.0244548391824537e-07, + "reward_std": 0.9966984510421752, + "rewards/rnd_reward/mean": -1.0244548391824537e-07, + "rewards/rnd_reward/std": 0.9966984808444976, + "step": 1070, + "step_time": 9.767576195369475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14500000327825546, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.6, + "completions/mean_length": 136.3862518310547, + "completions/mean_terminated_length": 94.67120056152343, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.6616650968790054, + "epoch": 0.7063440156965337, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.789601743221283, + "kl": 0.0545062692835927, + "learning_rate": 3.2357750163505557e-07, + "loss": 0.11497254371643066, + "num_tokens": 23170007.0, + "reward": 1.226365569095833e-07, + "reward_std": 0.9966248035430908, + "rewards/rnd_reward/mean": 1.2263655548849784e-07, + "rewards/rnd_reward/std": 0.9966248333454132, + "step": 1080, + "step_time": 9.882125865970739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14250000566244125, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.8, + "completions/mean_length": 130.01000289916993, + "completions/mean_terminated_length": 87.69976119995117, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.890427288413048, + "epoch": 0.7128842380640942, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8021807670593262, + "kl": 0.05592884449288249, + "learning_rate": 3.2194244604316545e-07, + "loss": 0.09672708511352539, + "num_tokens": 23366425.0, + "reward": 6.273389110411643e-08, + "reward_std": 0.9965825915336609, + "rewards/rnd_reward/mean": 6.273389181465916e-08, + "rewards/rnd_reward/std": 0.9965825736522674, + "step": 1090, + "step_time": 9.854898347100242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14500000178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 336.9, + "completions/mean_length": 131.98875274658204, + "completions/mean_terminated_length": 89.18458480834961, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7609634190797805, + "epoch": 0.7194244604316546, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0092402696609497, + "kl": 0.05501702083274722, + "learning_rate": 3.2030739045127533e-07, + "loss": 0.08695750832557678, + "num_tokens": 23567981.0, + "reward": -1.3828277736216422e-07, + "reward_std": 0.9966061592102051, + "rewards/rnd_reward/mean": -1.3828277451999326e-07, + "rewards/rnd_reward/std": 0.9966061592102051, + "step": 1100, + "step_time": 10.145819194451906 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1212500013411045, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.9, + "completions/mean_length": 121.10875091552734, + "completions/mean_terminated_length": 84.89219970703125, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8415472239255903, + "epoch": 0.7259646827992152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8949585556983948, + "kl": 0.06347150038927793, + "learning_rate": 3.186723348593852e-07, + "loss": 0.08753393888473511, + "num_tokens": 23758168.0, + "reward": -1.1354684570363816e-07, + "reward_std": 0.9964610040187836, + "rewards/rnd_reward/mean": -1.1354684286146721e-07, + "rewards/rnd_reward/std": 0.9964609861373901, + "step": 1110, + "step_time": 10.006089114071802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12125000208616257, + "completions/max_length": 384.0, + "completions/max_terminated_length": 369.7, + "completions/mean_length": 121.20000228881835, + "completions/mean_terminated_length": 84.96716842651367, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.9164877772331237, + "epoch": 0.7325049051667757, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1227521896362305, + "kl": 0.06196983400732279, + "learning_rate": 3.170372792674951e-07, + "loss": 0.07237975597381592, + "num_tokens": 23955023.0, + "reward": -5.5432321488524396e-08, + "reward_std": 0.9964903295040131, + "rewards/rnd_reward/mean": -5.543231935689619e-08, + "rewards/rnd_reward/std": 0.9964903354644775, + "step": 1120, + "step_time": 10.375506636733189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11875000298023224, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.2, + "completions/mean_length": 117.09375, + "completions/mean_terminated_length": 81.17416534423828, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.7361535727977753, + "epoch": 0.7390451275343362, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8529885411262512, + "kl": 0.06673006983473898, + "learning_rate": 3.1540222367560496e-07, + "loss": 0.09247955083847045, + "num_tokens": 24140203.0, + "reward": -2.6300548938706926e-08, + "reward_std": 0.9966336786746979, + "rewards/rnd_reward/mean": -2.6300551070335132e-08, + "rewards/rnd_reward/std": 0.9966336965560914, + "step": 1130, + "step_time": 10.012891574040987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11500000357627868, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.7, + "completions/mean_length": 119.6712516784668, + "completions/mean_terminated_length": 85.35453567504882, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6402025282382966, + "epoch": 0.7455853499018966, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9655517935752869, + "kl": 0.06339178960770368, + "learning_rate": 3.1376716808371484e-07, + "loss": 0.012396740168333054, + "num_tokens": 24333505.0, + "reward": -1.3798475424486866e-07, + "reward_std": 0.9963988780975341, + "rewards/rnd_reward/mean": -1.379847538895973e-07, + "rewards/rnd_reward/std": 0.9963988780975341, + "step": 1140, + "step_time": 10.81275618087966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12750000208616258, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.4, + "completions/mean_length": 130.15375289916992, + "completions/mean_terminated_length": 93.11451568603516, + "completions/min_length": 2.6, + "completions/min_terminated_length": 2.6, + "entropy": 2.7988665878772734, + "epoch": 0.7521255722694572, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9594887495040894, + "kl": 0.057864805683493614, + "learning_rate": 3.121321124918247e-07, + "loss": 0.05015655755996704, + "num_tokens": 24526988.0, + "reward": -2.7045605088815705e-08, + "reward_std": 0.9962463498115539, + "rewards/rnd_reward/mean": -2.7045605088815705e-08, + "rewards/rnd_reward/std": 0.9962463617324829, + "step": 1150, + "step_time": 9.847500793170184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000305473805, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.1, + "completions/mean_length": 119.13750076293945, + "completions/mean_terminated_length": 79.57122192382812, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.799566090106964, + "epoch": 0.7586657946370177, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3144031763076782, + "kl": 0.06464958414435387, + "learning_rate": 3.104970568999346e-07, + "loss": 0.08081794381141663, + "num_tokens": 24712093.0, + "reward": 2.2947787847726885e-08, + "reward_std": 0.9963953137397766, + "rewards/rnd_reward/mean": 2.2947789268812358e-08, + "rewards/rnd_reward/std": 0.9963953375816346, + "step": 1160, + "step_time": 9.831479465053416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 366.1, + "completions/mean_length": 128.67500228881835, + "completions/mean_terminated_length": 90.44079208374023, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.758173832297325, + "epoch": 0.7652060170045781, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8444108963012695, + "kl": 0.061153271794319154, + "learning_rate": 3.088620013080445e-07, + "loss": 0.06105284690856934, + "num_tokens": 24905938.0, + "reward": -9.223818580039733e-08, + "reward_std": 0.9962478995323181, + "rewards/rnd_reward/mean": -9.22381850898546e-08, + "rewards/rnd_reward/std": 0.9962479054927826, + "step": 1170, + "step_time": 9.715631593717262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13125000149011612, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.7, + "completions/mean_length": 132.65000381469727, + "completions/mean_terminated_length": 94.60303192138672, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.652452102303505, + "epoch": 0.7717462393721386, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8246821761131287, + "kl": 0.05974919479340315, + "learning_rate": 3.072269457161543e-07, + "loss": 0.07230232954025269, + "num_tokens": 25111683.0, + "reward": -1.4752150079289095e-08, + "reward_std": 0.9964145243167877, + "rewards/rnd_reward/mean": -1.4752149368746359e-08, + "rewards/rnd_reward/std": 0.9964145421981812, + "step": 1180, + "step_time": 10.24143796269782 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1437500037252903, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 133.8912498474121, + "completions/mean_terminated_length": 92.00172958374023, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.534428471326828, + "epoch": 0.7782864617396992, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7489418387413025, + "kl": 0.05822144625708461, + "learning_rate": 3.0559189012426423e-07, + "loss": 0.07474024295806884, + "num_tokens": 25319036.0, + "reward": -8.940696432091499e-08, + "reward_std": 0.9965973794460297, + "rewards/rnd_reward/mean": -8.940696574200047e-08, + "rewards/rnd_reward/std": 0.9965973973274231, + "step": 1190, + "step_time": 10.355807177536189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1450000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.6, + "completions/mean_length": 138.9825012207031, + "completions/mean_terminated_length": 97.18752059936523, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8433078229427338, + "epoch": 0.7848266841072596, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8156785368919373, + "kl": 0.06143994340673089, + "learning_rate": 3.039568345323741e-07, + "loss": 0.08815957307815551, + "num_tokens": 25526837.0, + "reward": -1.0192394315211572e-07, + "reward_std": 0.996451473236084, + "rewards/rnd_reward/mean": -1.0192393888885931e-07, + "rewards/rnd_reward/std": 0.996451485157013, + "step": 1200, + "step_time": 10.298059746343643 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14000000283122063, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.0, + "completions/mean_length": 127.81375274658203, + "completions/mean_terminated_length": 85.73782577514649, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7779837220907213, + "epoch": 0.7913669064748201, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.937714695930481, + "kl": 0.06527232667431235, + "learning_rate": 3.0232177894048394e-07, + "loss": 0.08186920881271362, + "num_tokens": 25721938.0, + "reward": -1.0982156268823929e-07, + "reward_std": 0.9962611317634582, + "rewards/rnd_reward/mean": -1.0982156055661107e-07, + "rewards/rnd_reward/std": 0.9962611377239228, + "step": 1210, + "step_time": 9.923247428331525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13625000193715095, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.9, + "completions/mean_length": 128.51625213623046, + "completions/mean_terminated_length": 88.076513671875, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.661186295747757, + "epoch": 0.7979071288423807, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9877737164497375, + "kl": 0.06118576750159264, + "learning_rate": 3.0068672334859387e-07, + "loss": 0.0857498049736023, + "num_tokens": 25916611.0, + "reward": -9.454786980001017e-08, + "reward_std": 0.9962615430355072, + "rewards/rnd_reward/mean": -9.454786553675376e-08, + "rewards/rnd_reward/std": 0.9962615609169007, + "step": 1220, + "step_time": 10.026552183739842 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13875000402331353, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.7, + "completions/mean_length": 127.41250381469726, + "completions/mean_terminated_length": 86.15239715576172, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.597662454843521, + "epoch": 0.8044473512099412, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9214726090431213, + "kl": 0.06120588518679142, + "learning_rate": 2.990516677567037e-07, + "loss": 0.08460624814033509, + "num_tokens": 26120156.0, + "reward": -1.5325844344715732e-07, + "reward_std": 0.9961971819400788, + "rewards/rnd_reward/mean": -1.5325843989444364e-07, + "rewards/rnd_reward/std": 0.9961971819400788, + "step": 1230, + "step_time": 10.459681713976897 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12875000089406968, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.9, + "completions/mean_length": 121.05875167846679, + "completions/mean_terminated_length": 82.47761840820313, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.899898463487625, + "epoch": 0.8109875735775016, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9018652439117432, + "kl": 0.06602321406826377, + "learning_rate": 2.9741661216481357e-07, + "loss": 0.05030661225318909, + "num_tokens": 26307033.0, + "reward": -1.496076620810527e-07, + "reward_std": 0.996212637424469, + "rewards/rnd_reward/mean": -1.4960765781779628e-07, + "rewards/rnd_reward/std": 0.9962126433849334, + "step": 1240, + "step_time": 9.930500056478195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13875000253319741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.1, + "completions/mean_length": 125.87625274658203, + "completions/mean_terminated_length": 84.06948699951172, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.7101810783147813, + "epoch": 0.8175277959450621, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9933876991271973, + "kl": 0.06315780784934759, + "learning_rate": 2.9578155657292345e-07, + "loss": 0.14283275604248047, + "num_tokens": 26506699.0, + "reward": -4.701316604904804e-08, + "reward_std": 0.996234393119812, + "rewards/rnd_reward/mean": -4.7013166404319404e-08, + "rewards/rnd_reward/std": 0.9962343990802764, + "step": 1250, + "step_time": 11.001938613550738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14875000342726707, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.5, + "completions/mean_length": 131.21625289916992, + "completions/mean_terminated_length": 87.06982650756837, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.695519116520882, + "epoch": 0.8240680183126227, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8424022793769836, + "kl": 0.06466643353924155, + "learning_rate": 2.9414650098103333e-07, + "loss": 0.08369569182395935, + "num_tokens": 26703352.0, + "reward": -2.6673080633088375e-08, + "reward_std": 0.9962631404399872, + "rewards/rnd_reward/mean": -2.6673077790917433e-08, + "rewards/rnd_reward/std": 0.9962631464004517, + "step": 1260, + "step_time": 9.866059662331827 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16375000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.6, + "completions/mean_length": 132.26625137329103, + "completions/mean_terminated_length": 83.00667343139648, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7214665055274962, + "epoch": 0.8306082406801831, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.061651349067688, + "kl": 0.060832435730844736, + "learning_rate": 2.9251144538914326e-07, + "loss": 0.011636599898338318, + "num_tokens": 26905195.0, + "reward": -1.28149986178272e-08, + "reward_std": 0.996177339553833, + "rewards/rnd_reward/mean": -1.2814998262555832e-08, + "rewards/rnd_reward/std": 0.9961773693561554, + "step": 1270, + "step_time": 10.226913845306262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1462500013411045, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.4, + "completions/mean_length": 135.36000289916993, + "completions/mean_terminated_length": 92.69313278198243, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5201783299446108, + "epoch": 0.8371484630477436, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9056648015975952, + "kl": 0.05058280224911869, + "learning_rate": 2.908763897972531e-07, + "loss": 0.08019465208053589, + "num_tokens": 27106543.0, + "reward": -1.9341707613307335e-07, + "reward_std": 0.9962323784828186, + "rewards/rnd_reward/mean": -1.9341706973818874e-07, + "rewards/rnd_reward/std": 0.9962323844432831, + "step": 1280, + "step_time": 10.01526041161269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1225000001490116, + "completions/max_length": 384.0, + "completions/max_terminated_length": 373.1, + "completions/mean_length": 130.23250198364258, + "completions/mean_terminated_length": 94.88236541748047, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.623024606704712, + "epoch": 0.8436886854153042, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.783699095249176, + "kl": 0.05370175880379975, + "learning_rate": 2.8924133420536296e-07, + "loss": 0.11414051055908203, + "num_tokens": 27305554.0, + "reward": -9.253621175986382e-08, + "reward_std": 0.9961644470691681, + "rewards/rnd_reward/mean": -9.253620731897172e-08, + "rewards/rnd_reward/std": 0.9961644232273101, + "step": 1290, + "step_time": 10.230407970934175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14875000193715096, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.6, + "completions/mean_length": 136.5575035095215, + "completions/mean_terminated_length": 93.4459114074707, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.676475775241852, + "epoch": 0.8502289077828646, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0168393850326538, + "kl": 0.052173000667244195, + "learning_rate": 2.8760627861347284e-07, + "loss": 0.049798119068145755, + "num_tokens": 27507145.0, + "reward": -1.576542894232702e-07, + "reward_std": 0.9962258338928223, + "rewards/rnd_reward/mean": -1.5765429013381292e-07, + "rewards/rnd_reward/std": 0.9962258338928223, + "step": 1300, + "step_time": 9.98130206335336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13500000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.1, + "completions/mean_length": 128.2087516784668, + "completions/mean_terminated_length": 88.18164749145508, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.640226128697395, + "epoch": 0.8567691301504251, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8835275769233704, + "kl": 0.05669898777268827, + "learning_rate": 2.859712230215827e-07, + "loss": 0.10237141847610473, + "num_tokens": 27707452.0, + "reward": -6.869435509315736e-08, + "reward_std": 0.9961890399456024, + "rewards/rnd_reward/mean": -6.869434940881546e-08, + "rewards/rnd_reward/std": 0.99618901014328, + "step": 1310, + "step_time": 10.37953436721582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13625000268220902, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.5, + "completions/mean_length": 128.80750274658203, + "completions/mean_terminated_length": 88.5141227722168, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.4644729048013687, + "epoch": 0.8633093525179856, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8116011619567871, + "kl": 0.05524417916312814, + "learning_rate": 2.843361674296926e-07, + "loss": 0.1121416449546814, + "num_tokens": 27900098.0, + "reward": -5.230307502657183e-08, + "reward_std": 0.9961800217628479, + "rewards/rnd_reward/mean": -5.230307715820004e-08, + "rewards/rnd_reward/std": 0.9961800217628479, + "step": 1320, + "step_time": 9.958397940779104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11625000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.3, + "completions/mean_length": 122.20125198364258, + "completions/mean_terminated_length": 87.82691497802735, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.619413223862648, + "epoch": 0.8698495748855462, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1175379753112793, + "kl": 0.06109289322048426, + "learning_rate": 2.827011118378025e-07, + "loss": 0.05299858450889587, + "num_tokens": 28083799.0, + "reward": -9.030103811369373e-08, + "reward_std": 0.9959750473499298, + "rewards/rnd_reward/mean": -9.030103793605804e-08, + "rewards/rnd_reward/std": 0.9959750413894654, + "step": 1330, + "step_time": 9.82245781556703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11625000312924386, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.4, + "completions/mean_length": 122.17500152587891, + "completions/mean_terminated_length": 87.70087432861328, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.900624880194664, + "epoch": 0.8763897972531066, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.942399263381958, + "kl": 0.0663422117009759, + "learning_rate": 2.8106605624591235e-07, + "loss": 0.07738288044929505, + "num_tokens": 28270204.0, + "reward": -1.4573336244438905e-07, + "reward_std": 0.9961058497428894, + "rewards/rnd_reward/mean": -1.4573335960221812e-07, + "rewards/rnd_reward/std": 0.9961058616638183, + "step": 1340, + "step_time": 9.953787335986272 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1312500022351742, + "completions/max_length": 384.0, + "completions/max_terminated_length": 370.4, + "completions/mean_length": 120.04500045776368, + "completions/mean_terminated_length": 80.18856277465821, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.87275829911232, + "epoch": 0.8829300196206671, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0761396884918213, + "kl": 0.07089657634496689, + "learning_rate": 2.794310006540222e-07, + "loss": 0.13924288749694824, + "num_tokens": 28461485.0, + "reward": 2.965331162840812e-08, + "reward_std": 0.9960719764232635, + "rewards/rnd_reward/mean": 2.965331162840812e-08, + "rewards/rnd_reward/std": 0.996071970462799, + "step": 1350, + "step_time": 10.60484689238947 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12875000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.8, + "completions/mean_length": 115.14375228881836, + "completions/mean_terminated_length": 75.8135986328125, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 3.0420612663030626, + "epoch": 0.8894702419882276, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1144483089447021, + "kl": 0.07124942755326628, + "learning_rate": 2.777959450621321e-07, + "loss": 0.08705209493637085, + "num_tokens": 28652205.0, + "reward": -1.2695789215477759e-07, + "reward_std": 0.9961645841598511, + "rewards/rnd_reward/mean": -1.2695789215477759e-07, + "rewards/rnd_reward/std": 0.99616459608078, + "step": 1360, + "step_time": 10.374728590669111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12250000275671483, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.6, + "completions/mean_length": 116.32750015258789, + "completions/mean_terminated_length": 78.6882926940918, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8743550151586534, + "epoch": 0.8960104643557881, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.038732886314392, + "kl": 0.07000278439372778, + "learning_rate": 2.76160889470242e-07, + "loss": 0.11522959470748902, + "num_tokens": 28836007.0, + "reward": -2.1174550361990897e-07, + "reward_std": 0.9960576713085174, + "rewards/rnd_reward/mean": -2.117455032646376e-07, + "rewards/rnd_reward/std": 0.9960576832294464, + "step": 1370, + "step_time": 10.044307312183083 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10750000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.4, + "completions/mean_length": 112.57625274658203, + "completions/mean_terminated_length": 79.90474357604981, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.814165809750557, + "epoch": 0.9025506867233486, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.853209912776947, + "kl": 0.07179404897615313, + "learning_rate": 2.7452583387835187e-07, + "loss": 0.11325899362564087, + "num_tokens": 29020798.0, + "reward": -4.172328260665381e-09, + "reward_std": 0.995877069234848, + "rewards/rnd_reward/mean": -4.172324707951702e-09, + "rewards/rnd_reward/std": 0.9958770573139191, + "step": 1380, + "step_time": 10.452752491505816 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09500000216066837, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.1, + "completions/mean_length": 97.97000122070312, + "completions/mean_terminated_length": 68.24239921569824, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.9521768242120743, + "epoch": 0.9090909090909091, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2329504489898682, + "kl": 0.09182704174891114, + "learning_rate": 2.7289077828646175e-07, + "loss": 0.09825693368911743, + "num_tokens": 29195334.0, + "reward": 9.715557460765468e-08, + "reward_std": 0.9959101438522339, + "rewards/rnd_reward/mean": 9.715556679168457e-08, + "rewards/rnd_reward/std": 0.9959101438522339, + "step": 1390, + "step_time": 10.322968539246357 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10125000234693289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.9, + "completions/mean_length": 103.2087516784668, + "completions/mean_terminated_length": 71.93345756530762, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 3.0182015657424928, + "epoch": 0.9156311314584696, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4850752353668213, + "kl": 0.09229131015017629, + "learning_rate": 2.7125572269457157e-07, + "loss": 0.07991544604301452, + "num_tokens": 29371486.0, + "reward": -1.5191734150654668e-07, + "reward_std": 0.995727401971817, + "rewards/rnd_reward/mean": -1.5191734008546122e-07, + "rewards/rnd_reward/std": 0.9957274079322815, + "step": 1400, + "step_time": 10.564200303750113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11875000409781933, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.6, + "completions/mean_length": 109.10750198364258, + "completions/mean_terminated_length": 72.09171180725097, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9613120049238204, + "epoch": 0.9221713538260301, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9964513778686523, + "kl": 0.0881183447316289, + "learning_rate": 2.696206671026815e-07, + "loss": 0.05779829025268555, + "num_tokens": 29547567.0, + "reward": -2.881884620276054e-07, + "reward_std": 0.9955705881118775, + "rewards/rnd_reward/mean": -2.881884591854345e-07, + "rewards/rnd_reward/std": 0.9955705881118775, + "step": 1410, + "step_time": 10.214018813963047 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000156462193, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.2, + "completions/mean_length": 102.86875228881836, + "completions/mean_terminated_length": 76.34417495727538, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.703101697564125, + "epoch": 0.9287115761935906, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8821516633033752, + "kl": 0.08714565662667155, + "learning_rate": 2.679856115107914e-07, + "loss": 0.017349694669246674, + "num_tokens": 29720227.0, + "reward": -8.821487531562867e-08, + "reward_std": 0.9958377420902252, + "rewards/rnd_reward/mean": -8.821487389454319e-08, + "rewards/rnd_reward/std": 0.9958377361297608, + "step": 1420, + "step_time": 10.117785238358191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000044703483, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.4, + "completions/mean_length": 115.6462516784668, + "completions/mean_terminated_length": 75.65214309692382, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.66856093108654, + "epoch": 0.935251798561151, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8306900858879089, + "kl": 0.07128150118514895, + "learning_rate": 2.663505559189012e-07, + "loss": 0.07572194337844848, + "num_tokens": 29907299.0, + "reward": -1.2844801631217705e-07, + "reward_std": 0.995860880613327, + "rewards/rnd_reward/mean": -1.2844801062783516e-07, + "rewards/rnd_reward/std": 0.9958608686923981, + "step": 1430, + "step_time": 10.222553351335227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12500000223517418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.7, + "completions/mean_length": 121.33125076293945, + "completions/mean_terminated_length": 83.87310562133788, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.7864604860544206, + "epoch": 0.9417920209287116, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.97917640209198, + "kl": 0.06842925064265729, + "learning_rate": 2.6471550032701114e-07, + "loss": 0.11840918064117431, + "num_tokens": 30100959.0, + "reward": -1.5377998749954713e-07, + "reward_std": 0.9958663046360016, + "rewards/rnd_reward/mean": -1.5377998110466252e-07, + "rewards/rnd_reward/std": 0.9958662986755371, + "step": 1440, + "step_time": 10.766110814665444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14125000238418578, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.3, + "completions/mean_length": 124.87875289916992, + "completions/mean_terminated_length": 82.31183624267578, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.699194425344467, + "epoch": 0.9483322432962721, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8881458640098572, + "kl": 0.06711579142138362, + "learning_rate": 2.6308044473512096e-07, + "loss": 0.10254174470901489, + "num_tokens": 30297562.0, + "reward": -6.161630330403512e-08, + "reward_std": 0.9957123577594758, + "rewards/rnd_reward/mean": -6.16162979749646e-08, + "rewards/rnd_reward/std": 0.9957123517990112, + "step": 1450, + "step_time": 10.593307360098697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1200000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.1, + "completions/mean_length": 116.06000213623047, + "completions/mean_terminated_length": 79.77308044433593, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.995149564743042, + "epoch": 0.9548724656638325, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0435981750488281, + "kl": 0.07414040835574269, + "learning_rate": 2.6144538914323084e-07, + "loss": 0.08301687836647034, + "num_tokens": 30483565.0, + "reward": -1.0631978923925089e-07, + "reward_std": 0.9958245873451232, + "rewards/rnd_reward/mean": -1.0631978462072311e-07, + "rewards/rnd_reward/std": 0.9958245933055878, + "step": 1460, + "step_time": 10.52764433112461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12750000283122062, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.3, + "completions/mean_length": 111.42625198364257, + "completions/mean_terminated_length": 71.7352523803711, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8058162331581116, + "epoch": 0.961412688031393, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9467556476593018, + "kl": 0.07855136888101696, + "learning_rate": 2.598103335513407e-07, + "loss": 0.10095235109329223, + "num_tokens": 30671516.0, + "reward": -1.529604193084566e-07, + "reward_std": 0.9958255529403687, + "rewards/rnd_reward/mean": -1.5296041966372798e-07, + "rewards/rnd_reward/std": 0.9958255529403687, + "step": 1470, + "step_time": 10.889759617857635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11125000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.2, + "completions/mean_length": 105.6962516784668, + "completions/mean_terminated_length": 70.84101905822754, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9369604349136353, + "epoch": 0.9679529103989536, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9438734650611877, + "kl": 0.0841084063053131, + "learning_rate": 2.581752779594506e-07, + "loss": 0.06790788173675537, + "num_tokens": 30852283.0, + "reward": -1.862645504502325e-08, + "reward_std": 0.9955705404281616, + "rewards/rnd_reward/mean": -1.8626451847580937e-08, + "rewards/rnd_reward/std": 0.9955705344676972, + "step": 1480, + "step_time": 10.544715784140863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11250000223517417, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.8, + "completions/mean_length": 111.51875228881836, + "completions/mean_terminated_length": 77.10519866943359, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.871400186419487, + "epoch": 0.9744931327665141, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0710206031799316, + "kl": 0.07257421473041177, + "learning_rate": 2.5654022236756053e-07, + "loss": 0.10279649496078491, + "num_tokens": 31039368.0, + "reward": -3.039836773410798e-08, + "reward_std": 0.9956327259540558, + "rewards/rnd_reward/mean": -3.039836915519345e-08, + "rewards/rnd_reward/std": 0.9956327199935913, + "step": 1490, + "step_time": 11.004371847352013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11750000305473804, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.9, + "completions/mean_length": 110.3587516784668, + "completions/mean_terminated_length": 74.05159416198731, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8570073008537293, + "epoch": 0.9810333551340745, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9674124717712402, + "kl": 0.07076722700148821, + "learning_rate": 2.5490516677567035e-07, + "loss": 0.09354864358901978, + "num_tokens": 31229300.0, + "reward": -7.458031374696361e-08, + "reward_std": 0.9957621216773986, + "rewards/rnd_reward/mean": -7.458031019424993e-08, + "rewards/rnd_reward/std": 0.9957621157169342, + "step": 1500, + "step_time": 10.672748254612088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09250000156462193, + "completions/max_length": 384.0, + "completions/max_terminated_length": 366.3, + "completions/mean_length": 109.77875213623047, + "completions/mean_terminated_length": 81.85236892700195, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8984388679265978, + "epoch": 0.9875735775016351, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.144245982170105, + "kl": 0.07264794735237956, + "learning_rate": 2.5327011118378023e-07, + "loss": 0.10363882780075073, + "num_tokens": 31410018.0, + "reward": 3.874301679474002e-09, + "reward_std": 0.9956199109554291, + "rewards/rnd_reward/mean": 3.874301679474002e-09, + "rewards/rnd_reward/std": 0.9956199288368225, + "step": 1510, + "step_time": 10.57373680251185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11250000223517417, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.4, + "completions/mean_length": 107.2912498474121, + "completions/mean_terminated_length": 71.87631187438964, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.835093083977699, + "epoch": 0.9941137998691956, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9730135798454285, + "kl": 0.06768459901213646, + "learning_rate": 2.516350555918901e-07, + "loss": 0.04406432807445526, + "num_tokens": 31584581.0, + "reward": -6.169080961626605e-08, + "reward_std": 0.9956278324127197, + "rewards/rnd_reward/mean": -6.169080748463785e-08, + "rewards/rnd_reward/std": 0.9956278324127197, + "step": 1520, + "step_time": 9.995071636443026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10000000149011612, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.7, + "completions/mean_length": 105.59500198364258, + "completions/mean_terminated_length": 74.4354133605957, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.818147099018097, + "epoch": 1.0006540222367561, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0657150745391846, + "kl": 0.0751381286419928, + "learning_rate": 2.5e-07, + "loss": 0.14666662216186524, + "num_tokens": 31762082.0, + "reward": -7.718801313671974e-08, + "reward_std": 0.9956592619419098, + "rewards/rnd_reward/mean": -7.718801242617701e-08, + "rewards/rnd_reward/std": 0.9956592619419098, + "step": 1530, + "step_time": 10.494197542965413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.0, + "completions/mean_length": 100.37250137329102, + "completions/mean_terminated_length": 73.99925155639649, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.850705623626709, + "epoch": 1.0071942446043165, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8915496468544006, + "kl": 0.08450272670015693, + "learning_rate": 2.4836494440810987e-07, + "loss": 0.10265235900878907, + "num_tokens": 31935920.0, + "reward": -7.78958216152148e-08, + "reward_std": 0.9955736041069031, + "rewards/rnd_reward/mean": -7.789581664141565e-08, + "rewards/rnd_reward/std": 0.995573616027832, + "step": 1540, + "step_time": 10.282078485726379 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09625000357627869, + "completions/max_length": 384.0, + "completions/max_terminated_length": 339.5, + "completions/mean_length": 100.54375, + "completions/mean_terminated_length": 70.20137939453124, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7832202523946763, + "epoch": 1.013734466971877, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9670330882072449, + "kl": 0.07673759749159217, + "learning_rate": 2.4672988881621975e-07, + "loss": 0.053245991468429565, + "num_tokens": 32105575.0, + "reward": -7.98702268323126e-08, + "reward_std": 0.9954982161521911, + "rewards/rnd_reward/mean": -7.987022256905619e-08, + "rewards/rnd_reward/std": 0.9954982340335846, + "step": 1550, + "step_time": 10.094654931733384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09625000134110451, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.6, + "completions/mean_length": 105.8862518310547, + "completions/mean_terminated_length": 76.32862396240235, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.806140884757042, + "epoch": 1.0202746893394374, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9716780185699463, + "kl": 0.07651977436617016, + "learning_rate": 2.450948332243296e-07, + "loss": 0.07588104605674743, + "num_tokens": 32282264.0, + "reward": -1.2878328945475914e-07, + "reward_std": 0.9953814685344696, + "rewards/rnd_reward/mean": -1.2878328448096e-07, + "rewards/rnd_reward/std": 0.9953814685344696, + "step": 1560, + "step_time": 10.249592515034601 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09250000044703484, + "completions/max_length": 384.0, + "completions/max_terminated_length": 370.2, + "completions/mean_length": 101.42500152587891, + "completions/mean_terminated_length": 72.58811225891114, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.706285873055458, + "epoch": 1.026814911706998, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2053457498550415, + "kl": 0.07990584680810571, + "learning_rate": 2.434597776324395e-07, + "loss": 0.08478202223777771, + "num_tokens": 32458489.0, + "reward": -1.197308309830447e-07, + "reward_std": 0.9958563566207885, + "rewards/rnd_reward/mean": -1.1973082814087377e-07, + "rewards/rnd_reward/std": 0.9958563566207885, + "step": 1570, + "step_time": 10.385891513410025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000223517418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.3, + "completions/mean_length": 92.8375015258789, + "completions/mean_terminated_length": 65.02403182983399, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.832634910941124, + "epoch": 1.0333551340745586, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2888636589050293, + "kl": 0.08601820273324848, + "learning_rate": 2.418247220405494e-07, + "loss": 0.024559563398361205, + "num_tokens": 32626059.0, + "reward": -1.628696985278566e-07, + "reward_std": 0.9954473912715912, + "rewards/rnd_reward/mean": -1.6286969000134378e-07, + "rewards/rnd_reward/std": 0.9954474031925201, + "step": 1580, + "step_time": 10.754275933234021 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09125000201165676, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.7, + "completions/mean_length": 99.23750076293945, + "completions/mean_terminated_length": 70.4692756652832, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.9075894236564634, + "epoch": 1.039895356442119, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0340582132339478, + "kl": 0.0824001532047987, + "learning_rate": 2.4018966644865926e-07, + "loss": 0.04327432513236999, + "num_tokens": 32797839.0, + "reward": 6.616115513224941e-08, + "reward_std": 0.9953638434410095, + "rewards/rnd_reward/mean": 6.61611530006212e-08, + "rewards/rnd_reward/std": 0.9953638732433319, + "step": 1590, + "step_time": 10.39529764095787 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1075000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.2, + "completions/mean_length": 105.15875091552735, + "completions/mean_terminated_length": 71.44011192321777, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 3.068614384531975, + "epoch": 1.0464355788096795, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9280741810798645, + "kl": 0.08188259806483984, + "learning_rate": 2.3855461085676914e-07, + "loss": 0.07503924369812012, + "num_tokens": 32970486.0, + "reward": 2.354383532576776e-08, + "reward_std": 0.9955627143383026, + "rewards/rnd_reward/mean": 2.3543834259953656e-08, + "rewards/rnd_reward/std": 0.9955627143383026, + "step": 1600, + "step_time": 10.102484514610842 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12375000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.0, + "completions/mean_length": 108.67875137329102, + "completions/mean_terminated_length": 69.76623840332032, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.7116469889879227, + "epoch": 1.05297580117724, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8361508846282959, + "kl": 0.07135752588510513, + "learning_rate": 2.36919555264879e-07, + "loss": 0.08789730072021484, + "num_tokens": 33155644.0, + "reward": -7.331371434382561e-08, + "reward_std": 0.9953298509120941, + "rewards/rnd_reward/mean": -7.331371363328287e-08, + "rewards/rnd_reward/std": 0.9953298389911651, + "step": 1610, + "step_time": 10.373006676253862 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10250000208616257, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.6, + "completions/mean_length": 107.08625106811523, + "completions/mean_terminated_length": 75.19281349182128, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.767005005478859, + "epoch": 1.0595160235448005, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8207447528839111, + "kl": 0.07195558175444602, + "learning_rate": 2.352844996729889e-07, + "loss": 0.07792651653289795, + "num_tokens": 33333863.0, + "reward": -7.919967437430842e-08, + "reward_std": 0.9952020287513733, + "rewards/rnd_reward/mean": -7.919966940050927e-08, + "rewards/rnd_reward/std": 0.9952020525932312, + "step": 1620, + "step_time": 10.309986177436077 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12875000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.7, + "completions/mean_length": 117.29250259399414, + "completions/mean_terminated_length": 78.07871360778809, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8041924834251404, + "epoch": 1.066056245912361, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7970576882362366, + "kl": 0.07171947872266174, + "learning_rate": 2.3364944408109875e-07, + "loss": 0.07321782112121582, + "num_tokens": 33529112.0, + "reward": -7.54743808784042e-08, + "reward_std": 0.9954242706298828, + "rewards/rnd_reward/mean": -7.54743808784042e-08, + "rewards/rnd_reward/std": 0.9954242765903473, + "step": 1630, + "step_time": 11.020952883246355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10375000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.8, + "completions/mean_length": 109.85750122070313, + "completions/mean_terminated_length": 78.08732643127442, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7948483556509016, + "epoch": 1.0725964682799216, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0839210748672485, + "kl": 0.07083125235512852, + "learning_rate": 2.3201438848920862e-07, + "loss": 0.07660450935363769, + "num_tokens": 33711328.0, + "reward": -4.068016963287846e-08, + "reward_std": 0.9954971015453339, + "rewards/rnd_reward/mean": -4.068016963287846e-08, + "rewards/rnd_reward/std": 0.9954971194267273, + "step": 1640, + "step_time": 10.339221865893341 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09125000033527612, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.3, + "completions/mean_length": 100.59000167846679, + "completions/mean_terminated_length": 72.11472930908204, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7371392965316774, + "epoch": 1.079136690647482, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9994949698448181, + "kl": 0.08674983121454716, + "learning_rate": 2.303793328973185e-07, + "loss": 0.14760398864746094, + "num_tokens": 33894090.0, + "reward": -4.775822315394862e-08, + "reward_std": 0.9951541960239411, + "rewards/rnd_reward/mean": -4.7758222088134514e-08, + "rewards/rnd_reward/std": 0.9951542019844055, + "step": 1650, + "step_time": 11.004630160145462 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09750000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.6, + "completions/mean_length": 103.39125213623046, + "completions/mean_terminated_length": 73.03910675048829, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.719507083296776, + "epoch": 1.0856769130150425, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0636616945266724, + "kl": 0.07622466497123241, + "learning_rate": 2.2874427730542835e-07, + "loss": 0.11726703643798828, + "num_tokens": 34065638.0, + "reward": -6.273389079325397e-08, + "reward_std": 0.9954430878162384, + "rewards/rnd_reward/mean": -6.273388937216851e-08, + "rewards/rnd_reward/std": 0.9954430878162384, + "step": 1660, + "step_time": 10.205868328805082 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11125000230967999, + "completions/max_length": 384.0, + "completions/max_terminated_length": 337.9, + "completions/mean_length": 103.16000251770019, + "completions/mean_terminated_length": 68.0124439239502, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.555436444282532, + "epoch": 1.0922171353826031, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4149969816207886, + "kl": 0.08045686418190598, + "learning_rate": 2.2710922171353826e-07, + "loss": 0.18388016223907472, + "num_tokens": 34243911.0, + "reward": -6.049871625890546e-08, + "reward_std": 0.995277750492096, + "rewards/rnd_reward/mean": -6.049871181801336e-08, + "rewards/rnd_reward/std": 0.9952777206897736, + "step": 1670, + "step_time": 10.405935624614358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09250000230967999, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.5, + "completions/mean_length": 98.8687515258789, + "completions/mean_terminated_length": 69.8191291809082, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.679945033788681, + "epoch": 1.0987573577501635, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1418001651763916, + "kl": 0.07972979731857777, + "learning_rate": 2.2547416612164814e-07, + "loss": 0.08271819949150086, + "num_tokens": 34422086.0, + "reward": -2.2441148974650105e-07, + "reward_std": 0.9950048804283143, + "rewards/rnd_reward/mean": -2.2441148654905875e-07, + "rewards/rnd_reward/std": 0.9950048506259919, + "step": 1680, + "step_time": 10.670063342596404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000141561032, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.9, + "completions/mean_length": 95.35000228881836, + "completions/mean_terminated_length": 67.31705818176269, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6938273787498472, + "epoch": 1.105297580117724, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0555897951126099, + "kl": 0.0881148618645966, + "learning_rate": 2.2383911052975802e-07, + "loss": 0.057183104753494265, + "num_tokens": 34592661.0, + "reward": -5.260109574578564e-08, + "reward_std": 0.9953494489192962, + "rewards/rnd_reward/mean": -5.260109787741385e-08, + "rewards/rnd_reward/std": 0.9953494608402252, + "step": 1690, + "step_time": 10.320013692067004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09500000197440386, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 96.38625259399414, + "completions/mean_terminated_length": 66.25210113525391, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8528232872486115, + "epoch": 1.1118378024852844, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1842913627624512, + "kl": 0.09175272611901164, + "learning_rate": 2.2220405493786787e-07, + "loss": 0.11493265628814697, + "num_tokens": 34767800.0, + "reward": -3.5762789174498266e-08, + "reward_std": 0.9952230334281922, + "rewards/rnd_reward/mean": -3.5762784911241854e-08, + "rewards/rnd_reward/std": 0.9952230513095855, + "step": 1700, + "step_time": 11.20805806266144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06625000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.7, + "completions/mean_length": 88.85, + "completions/mean_terminated_length": 67.81800193786621, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7997246354818346, + "epoch": 1.118378024852845, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3541666269302368, + "kl": 0.0883994361385703, + "learning_rate": 2.2056899934597775e-07, + "loss": 0.10645198822021484, + "num_tokens": 34940290.0, + "reward": -9.074807572062582e-08, + "reward_std": 0.9952512562274933, + "rewards/rnd_reward/mean": -9.074807181264077e-08, + "rewards/rnd_reward/std": 0.9952512562274933, + "step": 1710, + "step_time": 11.268909463495948 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08375000078231096, + "completions/max_length": 384.0, + "completions/max_terminated_length": 341.1, + "completions/mean_length": 92.34500198364258, + "completions/mean_terminated_length": 65.63225555419922, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5688311845064162, + "epoch": 1.1249182472204056, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.086549997329712, + "kl": 0.08725747494027018, + "learning_rate": 2.1893394375408762e-07, + "loss": 0.09306548833847046, + "num_tokens": 35107316.0, + "reward": -1.1682510461241691e-07, + "reward_std": 0.9951877236366272, + "rewards/rnd_reward/mean": -1.1682510461241691e-07, + "rewards/rnd_reward/std": 0.9951877236366272, + "step": 1720, + "step_time": 10.925348741887138 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000104308128, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.1, + "completions/mean_length": 97.91875076293945, + "completions/mean_terminated_length": 70.1454303741455, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.690840741991997, + "epoch": 1.131458469587966, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0729917287826538, + "kl": 0.08789877109229564, + "learning_rate": 2.172988881621975e-07, + "loss": 0.10979057550430298, + "num_tokens": 35281881.0, + "reward": -1.907348732288483e-08, + "reward_std": 0.995078319311142, + "rewards/rnd_reward/mean": -1.907348732288483e-08, + "rewards/rnd_reward/std": 0.9950783312320709, + "step": 1730, + "step_time": 10.632008279883303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000223517418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.8, + "completions/mean_length": 96.91125183105468, + "completions/mean_terminated_length": 69.48632164001465, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.681867742538452, + "epoch": 1.1379986919555265, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0131276845932007, + "kl": 0.08944809632375836, + "learning_rate": 2.1566383257030738e-07, + "loss": 0.10518434047698974, + "num_tokens": 35461970.0, + "reward": 2.6673077968553116e-08, + "reward_std": 0.9949174404144288, + "rewards/rnd_reward/mean": 2.6673081521266795e-08, + "rewards/rnd_reward/std": 0.9949174463748932, + "step": 1740, + "step_time": 11.253412028984167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0787500012665987, + "completions/max_length": 384.0, + "completions/max_terminated_length": 338.7, + "completions/mean_length": 84.14750061035156, + "completions/mean_terminated_length": 58.63842086791992, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.567233794927597, + "epoch": 1.144538914323087, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1535600423812866, + "kl": 0.09054145393893123, + "learning_rate": 2.1402877697841726e-07, + "loss": 0.10667283535003662, + "num_tokens": 35622023.0, + "reward": -5.97536580215774e-08, + "reward_std": 0.9952092111110687, + "rewards/rnd_reward/mean": -5.975365517940645e-08, + "rewards/rnd_reward/std": 0.9952091991901397, + "step": 1750, + "step_time": 10.509924068162217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07000000029802322, + "completions/max_length": 384.0, + "completions/max_terminated_length": 331.8, + "completions/mean_length": 86.03000106811524, + "completions/mean_terminated_length": 63.5996639251709, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.5665056079626085, + "epoch": 1.1510791366906474, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1171236038208008, + "kl": 0.08619214734062552, + "learning_rate": 2.1239372138652714e-07, + "loss": 0.07680293321609497, + "num_tokens": 35780972.0, + "reward": -4.529952919440916e-08, + "reward_std": 0.9950806498527527, + "rewards/rnd_reward/mean": -4.529952919440916e-08, + "rewards/rnd_reward/std": 0.9950806617736816, + "step": 1760, + "step_time": 10.277455605636352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06875000074505806, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.5, + "completions/mean_length": 89.81625213623047, + "completions/mean_terminated_length": 68.19218559265137, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.563973236083984, + "epoch": 1.157619359058208, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1521550416946411, + "kl": 0.0870651968754828, + "learning_rate": 2.10758665794637e-07, + "loss": 0.1115831971168518, + "num_tokens": 35948090.0, + "reward": 1.3858081859829951e-08, + "reward_std": 0.9948914706707, + "rewards/rnd_reward/mean": 1.38580813269229e-08, + "rewards/rnd_reward/std": 0.9948914825916291, + "step": 1770, + "step_time": 10.7476754687028 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07750000115483999, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.2, + "completions/mean_length": 85.87875022888184, + "completions/mean_terminated_length": 60.88022689819336, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.646288427710533, + "epoch": 1.1641595814257686, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9658340811729431, + "kl": 0.09920121757313609, + "learning_rate": 2.091236102027469e-07, + "loss": 0.08330439329147339, + "num_tokens": 36114108.0, + "reward": -7.227063285597523e-08, + "reward_std": 0.9949531733989716, + "rewards/rnd_reward/mean": -7.227063250070387e-08, + "rewards/rnd_reward/std": 0.994953167438507, + "step": 1780, + "step_time": 11.134026414644904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06250000018626452, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.1, + "completions/mean_length": 84.51375122070313, + "completions/mean_terminated_length": 64.53133659362793, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.704926884174347, + "epoch": 1.170699803793329, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2755341529846191, + "kl": 0.10716334227472543, + "learning_rate": 2.0748855461085677e-07, + "loss": 0.13019881248474122, + "num_tokens": 36271504.0, + "reward": -1.305341750246214e-07, + "reward_std": 0.9949664771556854, + "rewards/rnd_reward/mean": -1.305341714719077e-07, + "rewards/rnd_reward/std": 0.9949664771556854, + "step": 1790, + "step_time": 10.397847580048255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000175088644, + "completions/max_length": 384.0, + "completions/max_terminated_length": 326.3, + "completions/mean_length": 87.63249969482422, + "completions/mean_terminated_length": 59.75216636657715, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.593474331498146, + "epoch": 1.1772400261608895, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.155380129814148, + "kl": 0.09049692396074534, + "learning_rate": 2.0585349901896665e-07, + "loss": 0.12421066761016845, + "num_tokens": 36439860.0, + "reward": 1.50501714202278e-08, + "reward_std": 0.9950001060962677, + "rewards/rnd_reward/mean": 1.505017284131327e-08, + "rewards/rnd_reward/std": 0.9950001001358032, + "step": 1800, + "step_time": 10.84767303718254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06250000158324838, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.8, + "completions/mean_length": 91.63375244140624, + "completions/mean_terminated_length": 72.32135162353515, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5027853429317473, + "epoch": 1.1837802485284499, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0669174194335938, + "kl": 0.08057954236865043, + "learning_rate": 2.042184434270765e-07, + "loss": 0.09477431774139404, + "num_tokens": 36602592.0, + "reward": -1.505017284131327e-07, + "reward_std": 0.994869589805603, + "rewards/rnd_reward/mean": -1.5050172876840407e-07, + "rewards/rnd_reward/std": 0.9948696017265319, + "step": 1810, + "step_time": 10.333302733465098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000216066838, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.4, + "completions/mean_length": 92.49750137329102, + "completions/mean_terminated_length": 68.43786659240723, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.487256759405136, + "epoch": 1.1903204708960105, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.864107608795166, + "kl": 0.07850290769711137, + "learning_rate": 2.0258338783518638e-07, + "loss": 0.09753584861755371, + "num_tokens": 36768390.0, + "reward": -1.2189149920516229e-07, + "reward_std": 0.9947888195514679, + "rewards/rnd_reward/mean": -1.2189150062624775e-07, + "rewards/rnd_reward/std": 0.9947888076305389, + "step": 1820, + "step_time": 10.366325606242754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07500000111758709, + "completions/max_length": 384.0, + "completions/max_terminated_length": 327.2, + "completions/mean_length": 87.97500152587891, + "completions/mean_terminated_length": 64.1132999420166, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5121534377336503, + "epoch": 1.196860693263571, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8458340167999268, + "kl": 0.08256649980321526, + "learning_rate": 2.0094833224329626e-07, + "loss": 0.06835322976112365, + "num_tokens": 36933960.0, + "reward": -1.531839394885992e-07, + "reward_std": 0.9951117813587189, + "rewards/rnd_reward/mean": -1.5318393664642826e-07, + "rewards/rnd_reward/std": 0.9951117634773254, + "step": 1830, + "step_time": 11.365050849993713 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07375000137835741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.0, + "completions/mean_length": 94.00125045776367, + "completions/mean_terminated_length": 71.0650505065918, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4178095728158953, + "epoch": 1.2034009156311314, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1388354301452637, + "kl": 0.0803013733588159, + "learning_rate": 1.9931327665140614e-07, + "loss": 0.12810555696487427, + "num_tokens": 37099636.0, + "reward": -4.909932687091611e-08, + "reward_std": 0.9949400007724762, + "rewards/rnd_reward/mean": -4.909932758145885e-08, + "rewards/rnd_reward/std": 0.9949400067329407, + "step": 1840, + "step_time": 10.505639822757804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06875000093132258, + "completions/max_length": 384.0, + "completions/max_terminated_length": 341.5, + "completions/mean_length": 91.8550018310547, + "completions/mean_terminated_length": 70.10141868591309, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.4157417356967925, + "epoch": 1.209941137998692, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0894784927368164, + "kl": 0.08061180328950286, + "learning_rate": 1.9767822105951601e-07, + "loss": 0.09716765880584717, + "num_tokens": 37269225.0, + "reward": -4.3064359189770585e-08, + "reward_std": 0.9946663022041321, + "rewards/rnd_reward/mean": -4.306435528178554e-08, + "rewards/rnd_reward/std": 0.9946663022041321, + "step": 1850, + "step_time": 11.422960984380916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000074505807, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.2, + "completions/mean_length": 93.6987518310547, + "completions/mean_terminated_length": 65.78671073913574, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5770903497934343, + "epoch": 1.2164813603662525, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9160525798797607, + "kl": 0.08047085385769606, + "learning_rate": 1.960431654676259e-07, + "loss": 0.05968161225318909, + "num_tokens": 37440344.0, + "reward": 2.6524067830990816e-08, + "reward_std": 0.9945189356803894, + "rewards/rnd_reward/mean": 2.65240635677344e-08, + "rewards/rnd_reward/std": 0.9945189356803894, + "step": 1860, + "step_time": 10.725451961229556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000063329935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 335.1, + "completions/mean_length": 95.60625076293945, + "completions/mean_terminated_length": 68.25989952087403, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.4541951596736906, + "epoch": 1.223021582733813, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.999136209487915, + "kl": 0.07508358927443623, + "learning_rate": 1.9440810987573577e-07, + "loss": 0.13460888862609863, + "num_tokens": 37611004.0, + "reward": -1.23679642705099e-07, + "reward_std": 0.9949141681194306, + "rewards/rnd_reward/mean": -1.236796327575007e-07, + "rewards/rnd_reward/std": 0.9949141621589661, + "step": 1870, + "step_time": 10.482190745323896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 328.5, + "completions/mean_length": 89.38125152587891, + "completions/mean_terminated_length": 65.09402084350586, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.567149502038956, + "epoch": 1.2295618051013735, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2019909620285034, + "kl": 0.08660045554861426, + "learning_rate": 1.9277305428384562e-07, + "loss": 0.20026679039001466, + "num_tokens": 37786329.0, + "reward": -8.866190821521514e-08, + "reward_std": 0.9948227822780609, + "rewards/rnd_reward/mean": -8.866190999157198e-08, + "rewards/rnd_reward/std": 0.9948227941989899, + "step": 1880, + "step_time": 11.85091487034224 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06375000011175871, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.9, + "completions/mean_length": 89.55750122070313, + "completions/mean_terminated_length": 69.49982872009278, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8322397589683534, + "epoch": 1.236102027468934, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1517750024795532, + "kl": 0.09399051470682025, + "learning_rate": 1.9113799869195553e-07, + "loss": 0.17799322605133056, + "num_tokens": 37947720.0, + "reward": -1.0848045732814171e-07, + "reward_std": 0.9948072552680969, + "rewards/rnd_reward/mean": -1.0848045448597076e-07, + "rewards/rnd_reward/std": 0.9948072493076324, + "step": 1890, + "step_time": 10.414563778741286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06625000014901161, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.4, + "completions/mean_length": 86.39500045776367, + "completions/mean_terminated_length": 65.30856475830078, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7242774665355682, + "epoch": 1.2426422498364944, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3484736680984497, + "kl": 0.09410189539194107, + "learning_rate": 1.895029431000654e-07, + "loss": 0.0743591845035553, + "num_tokens": 38111876.0, + "reward": -6.467104114449285e-08, + "reward_std": 0.9947612941265106, + "rewards/rnd_reward/mean": -6.467103901286464e-08, + "rewards/rnd_reward/std": 0.9947613418102265, + "step": 1900, + "step_time": 10.527994227781893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06625000108033419, + "completions/max_length": 384.0, + "completions/max_terminated_length": 338.0, + "completions/mean_length": 84.77500114440917, + "completions/mean_terminated_length": 63.568695068359375, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.954171320796013, + "epoch": 1.249182472204055, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9760697484016418, + "kl": 0.10021719867363572, + "learning_rate": 1.8786788750817528e-07, + "loss": 0.02971080243587494, + "num_tokens": 38279376.0, + "reward": -1.5616416861874428e-07, + "reward_std": 0.9947503983974457, + "rewards/rnd_reward/mean": -1.5616416790820153e-07, + "rewards/rnd_reward/std": 0.9947503864765167, + "step": 1910, + "step_time": 11.339212405891157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08000000193715096, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.5, + "completions/mean_length": 88.47500076293946, + "completions/mean_terminated_length": 62.80438385009766, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.846560522913933, + "epoch": 1.2557226945716153, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2370659112930298, + "kl": 0.09515976402908563, + "learning_rate": 1.8623283191628514e-07, + "loss": 0.10529450178146363, + "num_tokens": 38449416.0, + "reward": -3.65078278719011e-09, + "reward_std": 0.9944445490837097, + "rewards/rnd_reward/mean": -3.6507834977328456e-09, + "rewards/rnd_reward/std": 0.9944445431232453, + "step": 1920, + "step_time": 10.698268468677998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000160187483, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.8, + "completions/mean_length": 87.2925018310547, + "completions/mean_terminated_length": 62.797546768188475, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.9005165725946425, + "epoch": 1.262262916939176, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3234843015670776, + "kl": 0.10119491461664439, + "learning_rate": 1.8459777632439501e-07, + "loss": 0.014341549575328827, + "num_tokens": 38616700.0, + "reward": -4.380941049930698e-08, + "reward_std": 0.9944035768508911, + "rewards/rnd_reward/mean": -4.3809414407292026e-08, + "rewards/rnd_reward/std": 0.994403600692749, + "step": 1930, + "step_time": 11.375351972319185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000111758709, + "completions/max_length": 384.0, + "completions/max_terminated_length": 334.7, + "completions/mean_length": 96.3550018310547, + "completions/mean_terminated_length": 68.85774459838868, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.9528768658638, + "epoch": 1.2688031393067365, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1305567026138306, + "kl": 0.09469359386712313, + "learning_rate": 1.829627207325049e-07, + "loss": 0.0362447589635849, + "num_tokens": 38788569.0, + "reward": -5.45382517813664e-08, + "reward_std": 0.9948436379432678, + "rewards/rnd_reward/mean": -5.4538249649738193e-08, + "rewards/rnd_reward/std": 0.9948436439037323, + "step": 1940, + "step_time": 10.781422062939964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10000000298023223, + "completions/max_length": 384.0, + "completions/max_terminated_length": 342.1, + "completions/mean_length": 93.3875015258789, + "completions/mean_terminated_length": 61.06837921142578, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7504739254713058, + "epoch": 1.2753433616742968, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.153893232345581, + "kl": 0.08643652414903044, + "learning_rate": 1.8132766514061477e-07, + "loss": 0.10290535688400268, + "num_tokens": 38954489.0, + "reward": -1.2144446595030445e-07, + "reward_std": 0.9947971343994141, + "rewards/rnd_reward/mean": -1.2144446346340487e-07, + "rewards/rnd_reward/std": 0.994797146320343, + "step": 1950, + "step_time": 10.527900594798847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0950000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 329.3, + "completions/mean_length": 100.58875122070313, + "completions/mean_terminated_length": 70.79409141540528, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.7141620010137557, + "epoch": 1.2818835840418574, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0054807662963867, + "kl": 0.08213893175125123, + "learning_rate": 1.7969260954872465e-07, + "loss": 0.07241884469985962, + "num_tokens": 39126695.0, + "reward": -1.028180136586343e-08, + "reward_std": 0.9948840856552124, + "rewards/rnd_reward/mean": -1.0281802786948902e-08, + "rewards/rnd_reward/std": 0.9948840975761414, + "step": 1960, + "step_time": 10.461326698656194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06750000081956387, + "completions/max_length": 383.2, + "completions/max_terminated_length": 350.0, + "completions/mean_length": 88.08000106811524, + "completions/mean_terminated_length": 66.5671230316162, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.7966362595558167, + "epoch": 1.2884238064094178, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1359318494796753, + "kl": 0.09009936284273863, + "learning_rate": 1.7805755395683453e-07, + "loss": 0.12549155950546265, + "num_tokens": 39294724.0, + "reward": 4.082918199088681e-08, + "reward_std": 0.9944938480854034, + "rewards/rnd_reward/mean": 4.0829182701429545e-08, + "rewards/rnd_reward/std": 0.9944938659667969, + "step": 1970, + "step_time": 10.983743568998761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.4, + "completions/mean_length": 95.8025016784668, + "completions/mean_terminated_length": 71.99091606140136, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7956566989421843, + "epoch": 1.2949640287769784, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3064550161361694, + "kl": 0.08204388581216335, + "learning_rate": 1.764224983649444e-07, + "loss": 0.06261605024337769, + "num_tokens": 39465981.0, + "reward": -7.681548792604076e-08, + "reward_std": 0.9945617258548737, + "rewards/rnd_reward/mean": -7.681548792604076e-08, + "rewards/rnd_reward/std": 0.9945617258548737, + "step": 1980, + "step_time": 10.856887764390558 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10500000119209289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 336.7, + "completions/mean_length": 99.39125137329101, + "completions/mean_terminated_length": 66.14606056213378, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.809012603759766, + "epoch": 1.301504251144539, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0052663087844849, + "kl": 0.08238375652581453, + "learning_rate": 1.7478744277305426e-07, + "loss": 0.04824514389038086, + "num_tokens": 39642064.0, + "reward": -2.3305416743113483e-07, + "reward_std": 0.9944523334503174, + "rewards/rnd_reward/mean": -2.3305416174679294e-07, + "rewards/rnd_reward/std": 0.9944523453712464, + "step": 1990, + "step_time": 10.668248945730738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06875000093132258, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.9, + "completions/mean_length": 94.09000167846679, + "completions/mean_terminated_length": 72.67349357604981, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.782809841632843, + "epoch": 1.3080444735120995, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.088478684425354, + "kl": 0.08519633868709206, + "learning_rate": 1.7315238718116416e-07, + "loss": 0.12635375261306764, + "num_tokens": 39807401.0, + "reward": -9.67830406706227e-08, + "reward_std": 0.9944651544094085, + "rewards/rnd_reward/mean": -9.67830406706227e-08, + "rewards/rnd_reward/std": 0.9944651544094085, + "step": 2000, + "step_time": 10.477193391043693 + } + ], + "logging_steps": 10, + "max_steps": 3058, + "num_input_tokens_seen": 39807401, + "num_train_epochs": 2, + "save_steps": 1000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 20, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-2000/training_args.bin b/checkpoint-2000/training_args.bin new file mode 100644 index 0000000..4c0ba1a --- /dev/null +++ b/checkpoint-2000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f7744bfe8fe7e43ca27c8620e9670a669a73959e4583e2dccebcee526f09ef4 +size 6929 diff --git a/checkpoint-3000/chat_template.jinja b/checkpoint-3000/chat_template.jinja new file mode 100644 index 0000000..28028c0 --- /dev/null +++ b/checkpoint-3000/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-3000/config.json b/checkpoint-3000/config.json new file mode 100644 index 0000000..08e4041 --- /dev/null +++ b/checkpoint-3000/config.json @@ -0,0 +1,58 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.3.0", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-3000/generation_config.json b/checkpoint-3000/generation_config.json new file mode 100644 index 0000000..8da9fd6 --- /dev/null +++ b/checkpoint-3000/generation_config.json @@ -0,0 +1,9 @@ +{ + "do_sample": false, + "eos_token_id": [ + 151643 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.3.0" +} diff --git a/checkpoint-3000/model.safetensors b/checkpoint-3000/model.safetensors new file mode 100644 index 0000000..4833ebf --- /dev/null +++ b/checkpoint-3000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c8fcb43d4668b505d3a884a777bfbf1a5ee826d17c96e855631df035a1c9c83 +size 1976163472 diff --git a/checkpoint-3000/optimizer.pt b/checkpoint-3000/optimizer.pt new file mode 100644 index 0000000..59cbdd9 --- /dev/null +++ b/checkpoint-3000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccab96731ac5d64484e4e109223c184b3fa3874f3dff26a619d5bb7daf74be7a +size 3952509771 diff --git a/checkpoint-3000/rng_state.pth b/checkpoint-3000/rng_state.pth new file mode 100644 index 0000000..c50393d --- /dev/null +++ b/checkpoint-3000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2680912076ed1a4e55c50136740b88b6c7df8c773714f74b373e1eec19d42ca0 +size 14645 diff --git a/checkpoint-3000/scheduler.pt b/checkpoint-3000/scheduler.pt new file mode 100644 index 0000000..6b3255f --- /dev/null +++ b/checkpoint-3000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f84fe6c6ea89520bcd2adfdb4903ed1a1968be9639f09eb7ec05d2acab619a70 +size 1465 diff --git a/checkpoint-3000/tokenizer.json b/checkpoint-3000/tokenizer.json new file mode 100644 index 0000000..3c72934 --- /dev/null +++ b/checkpoint-3000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f96da3a872b5e901575b2067c744ad336c3a3d77a21584d20024557b1bd7f0 +size 11422059 diff --git a/checkpoint-3000/tokenizer_config.json b/checkpoint-3000/tokenizer_config.json new file mode 100644 index 0000000..db25db0 --- /dev/null +++ b/checkpoint-3000/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "truncation_side": "left", + "unk_token": null +} diff --git a/checkpoint-3000/trainer_state.json b/checkpoint-3000/trainer_state.json new file mode 100644 index 0000000..b03855c --- /dev/null +++ b/checkpoint-3000/trainer_state.json @@ -0,0 +1,8434 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.9620667102681493, + "eval_steps": 500, + "global_step": 3000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.49000000953674316, + "completions/max_length": 384.0, + "completions/max_terminated_length": 372.8, + "completions/mean_length": 291.7250061035156, + "completions/mean_terminated_length": 204.1207077026367, + "completions/min_length": 15.7, + "completions/min_terminated_length": 15.7, + "entropy": 1.5640255227684974, + "epoch": 0.006540222367560497, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5129233002662659, + "kl": 0.0005587212668615394, + "learning_rate": 4.985284499672989e-07, + "loss": -0.04552017748355865, + "num_tokens": 324400.0, + "reward": -2.107620294111712e-06, + "reward_std": 0.9998070120811462, + "rewards/rnd_reward/mean": -2.107620250058062e-06, + "rewards/rnd_reward/std": 0.9998070061206817, + "step": 10, + "step_time": 9.577712386706844 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5312500178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 374.2, + "completions/mean_length": 293.2887481689453, + "completions/mean_terminated_length": 190.75759887695312, + "completions/min_length": 18.8, + "completions/min_terminated_length": 18.8, + "entropy": 1.451838418841362, + "epoch": 0.013080444735120994, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.47466912865638733, + "kl": 0.000643831241177395, + "learning_rate": 4.968933943754087e-07, + "loss": 0.00032075121998786926, + "num_tokens": 647161.0, + "reward": -1.490712192975252e-06, + "reward_std": 0.9998168766498565, + "rewards/rnd_reward/mean": -1.4907121759222264e-06, + "rewards/rnd_reward/std": 0.9998168766498565, + "step": 20, + "step_time": 9.337269740481862 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5450000017881393, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.6, + "completions/mean_length": 298.80250549316406, + "completions/mean_terminated_length": 196.99718322753907, + "completions/min_length": 20.3, + "completions/min_terminated_length": 20.3, + "entropy": 1.5065956979990005, + "epoch": 0.01962066710268149, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5856828689575195, + "kl": 0.0006675069424090907, + "learning_rate": 4.952583387835186e-07, + "loss": 0.02223682850599289, + "num_tokens": 982698.0, + "reward": -1.2338162438396694e-07, + "reward_std": 0.9998676717281342, + "rewards/rnd_reward/mean": -1.233816414369926e-07, + "rewards/rnd_reward/std": 0.9998676896095275, + "step": 30, + "step_time": 9.97233030446805 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5125000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 371.2, + "completions/mean_length": 287.2925048828125, + "completions/mean_terminated_length": 185.24396667480468, + "completions/min_length": 10.5, + "completions/min_terminated_length": 10.5, + "entropy": 1.6533239781856537, + "epoch": 0.026160889470241987, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5665533542633057, + "kl": 0.0007774221347062848, + "learning_rate": 4.936232831916285e-07, + "loss": 0.06135546565055847, + "num_tokens": 1303862.0, + "reward": -6.012618625206301e-07, + "reward_std": 0.9998966813087463, + "rewards/rnd_reward/mean": -6.012618571915596e-07, + "rewards/rnd_reward/std": 0.9998966932296753, + "step": 40, + "step_time": 9.489675145596266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46375001072883604, + "completions/max_length": 384.0, + "completions/max_terminated_length": 377.3, + "completions/mean_length": 269.5550064086914, + "completions/mean_terminated_length": 170.72807769775392, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 1.7108511567115783, + "epoch": 0.032701111837802485, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7200465798377991, + "kl": 0.0023742157864035107, + "learning_rate": 4.919882275997384e-07, + "loss": 0.11103466749191285, + "num_tokens": 1618921.0, + "reward": -8.426606612488285e-08, + "reward_std": 0.9999245047569275, + "rewards/rnd_reward/mean": -8.426606257216918e-08, + "rewards/rnd_reward/std": 0.999924510717392, + "step": 50, + "step_time": 10.167311429185792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4012500047683716, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.1, + "completions/mean_length": 223.41500549316407, + "completions/mean_terminated_length": 115.63493576049805, + "completions/min_length": 2.8, + "completions/min_terminated_length": 2.8, + "entropy": 2.521006014943123, + "epoch": 0.03924133420536298, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.70113205909729, + "kl": 0.020581040543038397, + "learning_rate": 4.903531720078482e-07, + "loss": 0.17053321599960328, + "num_tokens": 1894883.0, + "reward": -1.1384487108578867e-07, + "reward_std": 0.9999224483966828, + "rewards/rnd_reward/mean": -1.1384487108578867e-07, + "rewards/rnd_reward/std": 0.9999224364757537, + "step": 60, + "step_time": 9.91588868314866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3012500047683716, + "completions/max_length": 384.0, + "completions/max_terminated_length": 373.8, + "completions/mean_length": 183.6437515258789, + "completions/mean_terminated_length": 97.22235260009765, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7396299123764036, + "epoch": 0.04578155657292348, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7759609222412109, + "kl": 0.0463468745816499, + "learning_rate": 4.887181164159581e-07, + "loss": 0.14934378862380981, + "num_tokens": 2138653.0, + "reward": -4.6938660425155375e-08, + "reward_std": 0.9998835921287537, + "rewards/rnd_reward/mean": -4.6938658648798536e-08, + "rewards/rnd_reward/std": 0.9998836040496826, + "step": 70, + "step_time": 10.02495912197046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2825000062584877, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.8, + "completions/mean_length": 176.12750549316405, + "completions/mean_terminated_length": 94.6318603515625, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7325789600610735, + "epoch": 0.052321778940483975, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8319016695022583, + "kl": 0.04594853101298213, + "learning_rate": 4.87083060824068e-07, + "loss": 0.018015310168266296, + "num_tokens": 2376420.0, + "reward": -4.187226378959963e-08, + "reward_std": 0.9998487293720245, + "rewards/rnd_reward/mean": -4.1872263079056894e-08, + "rewards/rnd_reward/std": 0.9998487293720245, + "step": 80, + "step_time": 9.701193272857926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34750000238418577, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.4, + "completions/mean_length": 207.52375335693358, + "completions/mean_terminated_length": 114.01394119262696, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.342811721563339, + "epoch": 0.058862001308044476, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5935761332511902, + "kl": 0.019986469263676553, + "learning_rate": 4.854480052321779e-07, + "loss": 0.010216768831014633, + "num_tokens": 2641029.0, + "reward": -2.0414591972439666e-08, + "reward_std": 0.9998277425765991, + "rewards/rnd_reward/mean": -2.0414591972439666e-08, + "rewards/rnd_reward/std": 0.9998277485370636, + "step": 90, + "step_time": 10.468820211617276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4150000065565109, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.7, + "completions/mean_length": 237.4625045776367, + "completions/mean_terminated_length": 134.2795196533203, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "entropy": 2.1418308436870577, + "epoch": 0.06540222367560497, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5613327622413635, + "kl": 0.006808469316456467, + "learning_rate": 4.838129496402878e-07, + "loss": 0.04351484477519989, + "num_tokens": 2930639.0, + "reward": 8.493661918240747e-09, + "reward_std": 0.9997802138328552, + "rewards/rnd_reward/mean": 8.493660852426644e-09, + "rewards/rnd_reward/std": 0.9997801959514618, + "step": 100, + "step_time": 10.307341906265355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.38875000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 370.8, + "completions/mean_length": 234.60625610351562, + "completions/mean_terminated_length": 139.61359405517578, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.0046547770500185, + "epoch": 0.07194244604316546, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7055774927139282, + "kl": 0.006759516580495983, + "learning_rate": 4.821778940483976e-07, + "loss": 0.0377700001001358, + "num_tokens": 3215169.0, + "reward": -2.920627686364696e-08, + "reward_std": 0.9997233927249909, + "rewards/rnd_reward/mean": -2.9206276153104228e-08, + "rewards/rnd_reward/std": 0.9997233927249909, + "step": 110, + "step_time": 9.7759924543323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.38250001072883605, + "completions/max_length": 384.0, + "completions/max_terminated_length": 364.8, + "completions/mean_length": 221.19375457763672, + "completions/mean_terminated_length": 120.60059127807617, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.0983517467975616, + "epoch": 0.07848266841072596, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5727905631065369, + "kl": 0.009989026037510484, + "learning_rate": 4.805428384565075e-07, + "loss": 0.09626840949058532, + "num_tokens": 3496559.0, + "reward": 1.654028882747127e-08, + "reward_std": 0.9997243106365203, + "rewards/rnd_reward/mean": 1.654028882747127e-08, + "rewards/rnd_reward/std": 0.9997243225574494, + "step": 120, + "step_time": 9.981053173542023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34750000536441805, + "completions/max_length": 384.0, + "completions/max_terminated_length": 370.4, + "completions/mean_length": 207.67625274658204, + "completions/mean_terminated_length": 114.74080200195313, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2568650126457213, + "epoch": 0.08502289077828647, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8493079543113708, + "kl": 0.014032764779403805, + "learning_rate": 4.789077828646174e-07, + "loss": -0.015352001786231995, + "num_tokens": 3759245.0, + "reward": -5.06639494801675e-09, + "reward_std": 0.9996613085269928, + "rewards/rnd_reward/mean": -5.066393526931279e-09, + "rewards/rnd_reward/std": 0.9996613144874573, + "step": 130, + "step_time": 10.069910727906972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3137500062584877, + "completions/max_length": 384.0, + "completions/max_terminated_length": 373.4, + "completions/mean_length": 196.95000305175782, + "completions/mean_terminated_length": 111.34912643432617, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.305853408575058, + "epoch": 0.09156311314584696, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7982010245323181, + "kl": 0.016896341042593122, + "learning_rate": 4.772727272727273e-07, + "loss": 0.02038821578025818, + "num_tokens": 4010260.0, + "reward": -6.914138950264714e-08, + "reward_std": 0.9996093094348908, + "rewards/rnd_reward/mean": -6.914138808156168e-08, + "rewards/rnd_reward/std": 0.9996093094348908, + "step": 140, + "step_time": 9.98037786844652 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3700000047683716, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.3, + "completions/mean_length": 212.5687515258789, + "completions/mean_terminated_length": 111.88678207397462, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2448606371879576, + "epoch": 0.09810333551340746, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6012561321258545, + "kl": 0.01302192104049027, + "learning_rate": 4.756376716808371e-07, + "loss": 0.041692107915878296, + "num_tokens": 4270360.0, + "reward": -6.854534007061375e-09, + "reward_std": 0.9995533764362335, + "rewards/rnd_reward/mean": -6.854535428146846e-09, + "rewards/rnd_reward/std": 0.9995533883571625, + "step": 150, + "step_time": 9.479559903079643 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3687500059604645, + "completions/max_length": 384.0, + "completions/max_terminated_length": 373.0, + "completions/mean_length": 221.59500274658203, + "completions/mean_terminated_length": 127.51908645629882, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2110244154930117, + "epoch": 0.10464355788096795, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7059744000434875, + "kl": 0.009931368473917245, + "learning_rate": 4.74002616088947e-07, + "loss": 0.0420710027217865, + "num_tokens": 4538671.0, + "reward": -6.467104074481256e-08, + "reward_std": 0.9995002686977387, + "rewards/rnd_reward/mean": -6.467103932372709e-08, + "rewards/rnd_reward/std": 0.9995002567768096, + "step": 160, + "step_time": 9.463879713765346 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.38750000596046447, + "completions/max_length": 384.0, + "completions/max_terminated_length": 366.5, + "completions/mean_length": 224.09500274658203, + "completions/mean_terminated_length": 123.22678070068359, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.105703830718994, + "epoch": 0.11118378024852844, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6353176236152649, + "kl": 0.009572444634977728, + "learning_rate": 4.723675604970569e-07, + "loss": 0.057880669832229614, + "num_tokens": 4810167.0, + "reward": -1.3113021868349506e-08, + "reward_std": 0.9995246767997742, + "rewards/rnd_reward/mean": -1.3113020802535403e-08, + "rewards/rnd_reward/std": 0.9995246648788452, + "step": 170, + "step_time": 9.689974679402075 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3425000041723251, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.8, + "completions/mean_length": 207.19125213623047, + "completions/mean_terminated_length": 115.52461242675781, + "completions/min_length": 2.7, + "completions/min_terminated_length": 2.7, + "entropy": 2.3119001001119615, + "epoch": 0.11772400261608895, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6543518900871277, + "kl": 0.01470447750762105, + "learning_rate": 4.7073250490516674e-07, + "loss": 0.05609138607978821, + "num_tokens": 5069705.0, + "reward": -3.21865089580875e-08, + "reward_std": 0.9993573248386383, + "rewards/rnd_reward/mean": -3.218650753700203e-08, + "rewards/rnd_reward/std": 0.9993573546409606, + "step": 180, + "step_time": 9.860351232904941 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.29625000804662704, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.4, + "completions/mean_length": 186.85875396728517, + "completions/mean_terminated_length": 103.74458618164063, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.3917055457830427, + "epoch": 0.12426422498364945, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6832460165023804, + "kl": 0.02189517344813794, + "learning_rate": 4.690974493132766e-07, + "loss": 0.0940218448638916, + "num_tokens": 5313337.0, + "reward": -2.801418297337932e-08, + "reward_std": 0.9993866741657257, + "rewards/rnd_reward/mean": -2.801418386155774e-08, + "rewards/rnd_reward/std": 0.9993866682052612, + "step": 190, + "step_time": 10.32580083089415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3087500035762787, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.0, + "completions/mean_length": 195.6250030517578, + "completions/mean_terminated_length": 111.06491088867188, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.365246319770813, + "epoch": 0.13080444735120994, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7736798524856567, + "kl": 0.018843610235489906, + "learning_rate": 4.674623937213865e-07, + "loss": 0.09085838794708252, + "num_tokens": 5564832.0, + "reward": -3.784895152847412e-08, + "reward_std": 0.9993082165718079, + "rewards/rnd_reward/mean": -3.7848948331031805e-08, + "rewards/rnd_reward/std": 0.9993082165718079, + "step": 200, + "step_time": 10.074973361729644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2800000086426735, + "completions/max_length": 384.0, + "completions/max_terminated_length": 364.2, + "completions/mean_length": 183.75125274658203, + "completions/mean_terminated_length": 105.59480438232421, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3340355485677717, + "epoch": 0.13734466971877043, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7310492992401123, + "kl": 0.019957996485754848, + "learning_rate": 4.6582733812949637e-07, + "loss": 0.09393368363380432, + "num_tokens": 5808593.0, + "reward": -7.003545956507651e-08, + "reward_std": 0.9992451012134552, + "rewards/rnd_reward/mean": -7.003545814399103e-08, + "rewards/rnd_reward/std": 0.9992450892925262, + "step": 210, + "step_time": 9.864037699718029 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2600000038743019, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.5, + "completions/mean_length": 182.30500335693358, + "completions/mean_terminated_length": 111.45323791503907, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3039004147052764, + "epoch": 0.14388489208633093, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7140376567840576, + "kl": 0.020338369254022836, + "learning_rate": 4.641922825376063e-07, + "loss": 0.03314664363861084, + "num_tokens": 6039847.0, + "reward": -4.202127490415819e-08, + "reward_std": 0.9992108285427094, + "rewards/rnd_reward/mean": -4.202127419361546e-08, + "rewards/rnd_reward/std": 0.9992108345031738, + "step": 220, + "step_time": 9.367073794314638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2750000059604645, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.2, + "completions/mean_length": 176.20750274658204, + "completions/mean_terminated_length": 97.27564849853516, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.2686150133609773, + "epoch": 0.15042511445389142, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8464763760566711, + "kl": 0.021972918952815234, + "learning_rate": 4.6255722694571613e-07, + "loss": 0.04784287512302399, + "num_tokens": 6270078.0, + "reward": -1.1757016302738065e-07, + "reward_std": 0.9992310464382171, + "rewards/rnd_reward/mean": -1.1757016231683792e-07, + "rewards/rnd_reward/std": 0.9992310464382171, + "step": 230, + "step_time": 9.406287361658178 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23750000298023224, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.0, + "completions/mean_length": 164.8237548828125, + "completions/mean_terminated_length": 96.10623779296876, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4642169535160066, + "epoch": 0.15696533682145192, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.874134361743927, + "kl": 0.027495546033605933, + "learning_rate": 4.60922171353826e-07, + "loss": 0.09257912039756774, + "num_tokens": 6507582.0, + "reward": -2.2351741080228748e-08, + "reward_std": 0.9989596605300903, + "rewards/rnd_reward/mean": -2.2351743211856957e-08, + "rewards/rnd_reward/std": 0.9989596605300903, + "step": 240, + "step_time": 10.88162840725854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.26375000476837157, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.7, + "completions/mean_length": 179.21375274658203, + "completions/mean_terminated_length": 105.97551345825195, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.3090430706739427, + "epoch": 0.16350555918901244, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7480731010437012, + "kl": 0.022477348684333265, + "learning_rate": 4.592871157619359e-07, + "loss": 0.0661897599697113, + "num_tokens": 6746233.0, + "reward": -8.285045609568442e-08, + "reward_std": 0.9989501237869263, + "rewards/rnd_reward/mean": -8.285045680622715e-08, + "rewards/rnd_reward/std": 0.9989501178264618, + "step": 250, + "step_time": 9.837229801248759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23000000417232513, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.9, + "completions/mean_length": 166.79125366210937, + "completions/mean_terminated_length": 101.81034469604492, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3430103093385695, + "epoch": 0.17004578155657293, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7986056804656982, + "kl": 0.026151024643331765, + "learning_rate": 4.5765206017004576e-07, + "loss": 0.1205091118812561, + "num_tokens": 6976091.0, + "reward": -7.808208817294826e-08, + "reward_std": 0.9988620162010193, + "rewards/rnd_reward/mean": -7.808208604132006e-08, + "rewards/rnd_reward/std": 0.9988620281219482, + "step": 260, + "step_time": 9.811681813630276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23125000298023224, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.4, + "completions/mean_length": 162.19500274658202, + "completions/mean_terminated_length": 95.31229019165039, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2510120779275895, + "epoch": 0.17658600392413343, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6842794418334961, + "kl": 0.025852510519325734, + "learning_rate": 4.5601700457815564e-07, + "loss": 0.07729284763336182, + "num_tokens": 7199527.0, + "reward": -7.122755136812487e-08, + "reward_std": 0.9989409029483796, + "rewards/rnd_reward/mean": -7.122754994703939e-08, + "rewards/rnd_reward/std": 0.9989409029483796, + "step": 270, + "step_time": 9.695072391931898 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.24750000536441802, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.6, + "completions/mean_length": 165.32125244140624, + "completions/mean_terminated_length": 93.36570816040039, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2813161820173264, + "epoch": 0.18312622629169392, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8117495775222778, + "kl": 0.02381342255976051, + "learning_rate": 4.543819489862655e-07, + "loss": 0.08756870031356812, + "num_tokens": 7423314.0, + "reward": -3.010034372863402e-08, + "reward_std": 0.9987568318843841, + "rewards/rnd_reward/mean": -3.0100344261541065e-08, + "rewards/rnd_reward/std": 0.9987568318843841, + "step": 280, + "step_time": 9.538212020602078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.27250000536441804, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.5, + "completions/mean_length": 171.89125366210936, + "completions/mean_terminated_length": 91.7330307006836, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.2805801182985306, + "epoch": 0.18966644865925442, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7356550693511963, + "kl": 0.021292086993344127, + "learning_rate": 4.527468933943754e-07, + "loss": 0.0787746787071228, + "num_tokens": 7649017.0, + "reward": -2.2798775312082853e-08, + "reward_std": 0.9987796723842621, + "rewards/rnd_reward/mean": -2.279877513444717e-08, + "rewards/rnd_reward/std": 0.99877969622612, + "step": 290, + "step_time": 9.4448504867265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.4, + "completions/mean_length": 172.06375274658203, + "completions/mean_terminated_length": 101.12049560546875, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.062102970480919, + "epoch": 0.1962066710268149, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.678175151348114, + "kl": 0.021230081818066536, + "learning_rate": 4.511118378024852e-07, + "loss": 0.10422745943069459, + "num_tokens": 7893248.0, + "reward": 1.7285347131412722e-08, + "reward_std": 0.9987858593463897, + "rewards/rnd_reward/mean": 1.7285347841955455e-08, + "rewards/rnd_reward/std": 0.9987858593463897, + "step": 300, + "step_time": 11.542911725770683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23125, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.3, + "completions/mean_length": 155.94375305175782, + "completions/mean_terminated_length": 87.63837699890136, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.110625982284546, + "epoch": 0.2027468933943754, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8758037090301514, + "kl": 0.025008385512046517, + "learning_rate": 4.4947678221059515e-07, + "loss": 0.0913781225681305, + "num_tokens": 8105838.0, + "reward": -6.72042411054008e-08, + "reward_std": 0.9986489951610565, + "rewards/rnd_reward/mean": -6.720423826322985e-08, + "rewards/rnd_reward/std": 0.9986490130424499, + "step": 310, + "step_time": 9.397380503430032 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19500000327825545, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.1, + "completions/mean_length": 141.38000259399413, + "completions/mean_terminated_length": 82.59085083007812, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.311973440647125, + "epoch": 0.2092871157619359, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9461565613746643, + "kl": 0.03633355665951967, + "learning_rate": 4.4784172661870503e-07, + "loss": 0.08147850632667542, + "num_tokens": 8325122.0, + "reward": -9.238719300697084e-09, + "reward_std": 0.9986236810684204, + "rewards/rnd_reward/mean": -9.238719300697084e-09, + "rewards/rnd_reward/std": 0.9986236929893494, + "step": 320, + "step_time": 11.082472172076814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.20000000447034835, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.7, + "completions/mean_length": 141.80000228881835, + "completions/mean_terminated_length": 81.46792144775391, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3696460843086244, + "epoch": 0.2158273381294964, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8129741549491882, + "kl": 0.034535441920161246, + "learning_rate": 4.462066710268149e-07, + "loss": 0.06801753044128418, + "num_tokens": 8535267.0, + "reward": -8.24779284869237e-08, + "reward_std": 0.9986066579818725, + "rewards/rnd_reward/mean": -8.24779284869237e-08, + "rewards/rnd_reward/std": 0.9986066579818725, + "step": 330, + "step_time": 9.98545232352335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2100000038743019, + "completions/max_length": 384.0, + "completions/max_terminated_length": 364.4, + "completions/mean_length": 141.52500228881837, + "completions/mean_terminated_length": 77.51832160949706, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3277006804943086, + "epoch": 0.2223675604970569, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8912848234176636, + "kl": 0.0404030725825578, + "learning_rate": 4.445716154349248e-07, + "loss": 0.09486417174339294, + "num_tokens": 8741817.0, + "reward": -1.6838312300038183e-08, + "reward_std": 0.9984602987766266, + "rewards/rnd_reward/mean": -1.683831301058092e-08, + "rewards/rnd_reward/std": 0.9984602928161621, + "step": 340, + "step_time": 9.806188964121976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1912500038743019, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.6, + "completions/mean_length": 139.37500076293946, + "completions/mean_terminated_length": 81.56527061462403, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.3621277123689652, + "epoch": 0.2289077828646174, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9253888130187988, + "kl": 0.03642263817600906, + "learning_rate": 4.429365598430346e-07, + "loss": 0.09143019318580628, + "num_tokens": 8945182.0, + "reward": -2.4735924242236252e-08, + "reward_std": 0.9984571993350982, + "rewards/rnd_reward/mean": -2.4735929926578137e-08, + "rewards/rnd_reward/std": 0.9984571933746338, + "step": 350, + "step_time": 9.939818266802467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18750000447034837, + "completions/max_length": 384.0, + "completions/max_terminated_length": 371.7, + "completions/mean_length": 138.01250076293945, + "completions/mean_terminated_length": 81.37003707885742, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.270884335041046, + "epoch": 0.2354480052321779, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8366574645042419, + "kl": 0.03530883879866451, + "learning_rate": 4.4130150425114454e-07, + "loss": 0.10644701719284058, + "num_tokens": 9147587.0, + "reward": -1.15185978266652e-07, + "reward_std": 0.9983633041381836, + "rewards/rnd_reward/mean": -1.1518597684556652e-07, + "rewards/rnd_reward/std": 0.9983633100986481, + "step": 360, + "step_time": 9.722514500608668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17000000029802323, + "completions/max_length": 384.0, + "completions/max_terminated_length": 366.5, + "completions/mean_length": 130.59125213623048, + "completions/mean_terminated_length": 79.17214775085449, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3677036464214325, + "epoch": 0.2419882275997384, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.964614987373352, + "kl": 0.043067421624436976, + "learning_rate": 4.3966644865925437e-07, + "loss": 0.08349081873893738, + "num_tokens": 9340010.0, + "reward": -3.0696392450124675e-08, + "reward_std": 0.9982355415821076, + "rewards/rnd_reward/mean": -3.0696389607953736e-08, + "rewards/rnd_reward/std": 0.9982355415821076, + "step": 370, + "step_time": 9.710164245427586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1887500010430813, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.2, + "completions/mean_length": 138.41875305175782, + "completions/mean_terminated_length": 81.47324066162109, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.258895403146744, + "epoch": 0.2485284499672989, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7839703559875488, + "kl": 0.039375259960070255, + "learning_rate": 4.3803139306736425e-07, + "loss": 0.08618925213813781, + "num_tokens": 9545205.0, + "reward": -1.0460615040841503e-07, + "reward_std": 0.998303747177124, + "rewards/rnd_reward/mean": -1.046061518295005e-07, + "rewards/rnd_reward/std": 0.9983037531375885, + "step": 380, + "step_time": 9.776164471404627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1625000037252903, + "completions/max_length": 384.0, + "completions/max_terminated_length": 342.9, + "completions/mean_length": 123.14125137329101, + "completions/mean_terminated_length": 72.26187210083008, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2043879717588424, + "epoch": 0.25506867233485936, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.013533353805542, + "kl": 0.045199727965518834, + "learning_rate": 4.363963374754742e-07, + "loss": 0.08801217675209046, + "num_tokens": 9732323.0, + "reward": -1.3418495639072603e-07, + "reward_std": 0.9981010854244232, + "rewards/rnd_reward/mean": -1.3418495496964057e-07, + "rewards/rnd_reward/std": 0.9981010735034943, + "step": 390, + "step_time": 9.470399480173365 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16500000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.9, + "completions/mean_length": 126.54125213623047, + "completions/mean_terminated_length": 75.62271652221679, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.1946812689304354, + "epoch": 0.2616088947024199, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9289289712905884, + "kl": 0.043896416388452056, + "learning_rate": 4.34761281883584e-07, + "loss": 0.06013514399528504, + "num_tokens": 9923251.0, + "reward": -1.1950731622079046e-07, + "reward_std": 0.9980400562286377, + "rewards/rnd_reward/mean": -1.19507314799705e-07, + "rewards/rnd_reward/std": 0.9980400741100312, + "step": 400, + "step_time": 9.578276086249389 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19000000283122062, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.4, + "completions/mean_length": 142.2825019836426, + "completions/mean_terminated_length": 85.19309997558594, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3042476952075956, + "epoch": 0.2681491170699804, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8114489316940308, + "kl": 0.039571146154776214, + "learning_rate": 4.331262262916939e-07, + "loss": 0.08236328959465027, + "num_tokens": 10132682.0, + "reward": -1.4752151500374566e-08, + "reward_std": 0.9982209146022797, + "rewards/rnd_reward/mean": -1.4752152210917303e-08, + "rewards/rnd_reward/std": 0.9982209086418152, + "step": 410, + "step_time": 9.807740885205567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.20625000149011613, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 149.2062530517578, + "completions/mean_terminated_length": 88.13671951293945, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.38392159640789, + "epoch": 0.27468933943754087, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7539530396461487, + "kl": 0.03540583737194538, + "learning_rate": 4.3149117069980376e-07, + "loss": 0.08869684934616089, + "num_tokens": 10345522.0, + "reward": -1.3530254712179611e-07, + "reward_std": 0.9981609523296356, + "rewards/rnd_reward/mean": -1.3530254356908243e-07, + "rewards/rnd_reward/std": 0.9981609761714936, + "step": 420, + "step_time": 9.619695667061023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18750000223517418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.4, + "completions/mean_length": 139.9700035095215, + "completions/mean_terminated_length": 83.52344436645508, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.225811779499054, + "epoch": 0.2812295618051014, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7882169485092163, + "kl": 0.031479011243209246, + "learning_rate": 4.2985611510791364e-07, + "loss": 0.09766495227813721, + "num_tokens": 10563093.0, + "reward": -1.4021993024471158e-07, + "reward_std": 0.998052579164505, + "rewards/rnd_reward/mean": -1.4021992598145516e-07, + "rewards/rnd_reward/std": 0.9980525553226471, + "step": 430, + "step_time": 11.129851599340327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18875000178813933, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.8, + "completions/mean_length": 143.57625198364258, + "completions/mean_terminated_length": 87.31694107055664, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.1123261958360673, + "epoch": 0.28776978417266186, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7737316489219666, + "kl": 0.0280194781254977, + "learning_rate": 4.2822105951602357e-07, + "loss": 0.10439527034759521, + "num_tokens": 10766159.0, + "reward": -4.2766332430943524e-08, + "reward_std": 0.998040771484375, + "rewards/rnd_reward/mean": -4.276633118749373e-08, + "rewards/rnd_reward/std": 0.998040747642517, + "step": 440, + "step_time": 9.481722956942395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16625000312924385, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.1, + "completions/mean_length": 131.25125427246093, + "completions/mean_terminated_length": 80.89927139282227, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2791686803102493, + "epoch": 0.2943100065402224, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.834428608417511, + "kl": 0.04166741594672203, + "learning_rate": 4.265860039241334e-07, + "loss": 0.07646016478538513, + "num_tokens": 10965315.0, + "reward": -1.0013580258316779e-07, + "reward_std": 0.998044753074646, + "rewards/rnd_reward/mean": -1.0013580116208232e-07, + "rewards/rnd_reward/std": 0.9980447709560394, + "step": 450, + "step_time": 10.756261567212642 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13750000298023224, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.9, + "completions/mean_length": 118.16625137329102, + "completions/mean_terminated_length": 75.56828384399414, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3162699341773987, + "epoch": 0.30085022890778285, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1823770999908447, + "kl": 0.04914104142226279, + "learning_rate": 4.249509483322433e-07, + "loss": 0.09161773324012756, + "num_tokens": 11151658.0, + "reward": 1.4454126073815132e-08, + "reward_std": 0.9979428768157959, + "rewards/rnd_reward/mean": 1.4454126073815132e-08, + "rewards/rnd_reward/std": 0.9979428887367249, + "step": 460, + "step_time": 9.690541778854094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625000596046448, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.0, + "completions/mean_length": 129.17875442504882, + "completions/mean_terminated_length": 81.90925521850586, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.250484657287598, + "epoch": 0.30739045127534337, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7849432229995728, + "kl": 0.044866825267672536, + "learning_rate": 4.2331589274035315e-07, + "loss": 0.1366190195083618, + "num_tokens": 11350266.0, + "reward": -5.528331037396583e-08, + "reward_std": 0.997910988330841, + "rewards/rnd_reward/mean": -5.528330859760899e-08, + "rewards/rnd_reward/std": 0.997911012172699, + "step": 470, + "step_time": 10.094013441796415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14375000149011613, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.1, + "completions/mean_length": 121.7000015258789, + "completions/mean_terminated_length": 77.64777450561523, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.356913906335831, + "epoch": 0.31393067364290383, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7464061975479126, + "kl": 0.047594004729762676, + "learning_rate": 4.2168083714846303e-07, + "loss": 0.093714839220047, + "num_tokens": 11546191.0, + "reward": -2.1904707381281696e-08, + "reward_std": 0.9980335652828216, + "rewards/rnd_reward/mean": -2.1904707381281696e-08, + "rewards/rnd_reward/std": 0.9980335593223572, + "step": 480, + "step_time": 10.49183724487666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1312500022351742, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.0, + "completions/mean_length": 110.71375274658203, + "completions/mean_terminated_length": 69.09722747802735, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.3989109843969345, + "epoch": 0.32047089601046436, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9596973061561584, + "kl": 0.059090607427060605, + "learning_rate": 4.200457815565729e-07, + "loss": 0.061995261907577516, + "num_tokens": 11729672.0, + "reward": -8.441507906020718e-08, + "reward_std": 0.9977342307567596, + "rewards/rnd_reward/mean": -8.441507870493581e-08, + "rewards/rnd_reward/std": 0.9977342247962951, + "step": 490, + "step_time": 9.824798973021098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000193715094, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.2, + "completions/mean_length": 110.2125015258789, + "completions/mean_terminated_length": 69.26982688903809, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.335902637243271, + "epoch": 0.3270111183780249, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0347245931625366, + "kl": 0.06009262367151678, + "learning_rate": 4.184107259646828e-07, + "loss": 0.011750234663486481, + "num_tokens": 11908552.0, + "reward": -7.688999374977356e-08, + "reward_std": 0.9977168381214142, + "rewards/rnd_reward/mean": -7.688999019705988e-08, + "rewards/rnd_reward/std": 0.9977168321609498, + "step": 500, + "step_time": 9.731353901792318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1450000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.2, + "completions/mean_length": 118.08375244140625, + "completions/mean_terminated_length": 73.00995483398438, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.557803976535797, + "epoch": 0.33355134074558535, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9074205160140991, + "kl": 0.06632269411347806, + "learning_rate": 4.1677567037279267e-07, + "loss": 0.08373542428016663, + "num_tokens": 12103254.0, + "reward": -1.1697411828048843e-07, + "reward_std": 0.9978465795516968, + "rewards/rnd_reward/mean": -1.1697411828048843e-07, + "rewards/rnd_reward/std": 0.9978465795516968, + "step": 510, + "step_time": 10.207280423282645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15875000208616258, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.2, + "completions/mean_length": 123.88500137329102, + "completions/mean_terminated_length": 74.63431396484376, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.437187620997429, + "epoch": 0.34009156311314587, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8857874870300293, + "kl": 0.06048318287357688, + "learning_rate": 4.151406147809025e-07, + "loss": 0.037069684267044066, + "num_tokens": 12302092.0, + "reward": -3.874301874873254e-08, + "reward_std": 0.9979513227939606, + "rewards/rnd_reward/mean": -3.874301874873254e-08, + "rewards/rnd_reward/std": 0.9979513168334961, + "step": 520, + "step_time": 10.34260532106273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17000000253319741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.1, + "completions/mean_length": 132.52125396728516, + "completions/mean_terminated_length": 80.95349197387695, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4997961223125458, + "epoch": 0.34663178548070633, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9028640985488892, + "kl": 0.05214450396597385, + "learning_rate": 4.135055591890124e-07, + "loss": 0.07458721995353698, + "num_tokens": 12499434.0, + "reward": -2.2247433655309124e-07, + "reward_std": 0.9978251278400421, + "rewards/rnd_reward/mean": -2.2247433655309124e-07, + "rewards/rnd_reward/std": 0.9978251218795776, + "step": 530, + "step_time": 9.664499558508396 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15125000178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.4, + "completions/mean_length": 120.7087516784668, + "completions/mean_terminated_length": 73.79461898803712, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.547806602716446, + "epoch": 0.35317200784826686, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0128388404846191, + "kl": 0.05931666204705834, + "learning_rate": 4.118705035971223e-07, + "loss": 0.11692253351211548, + "num_tokens": 12683096.0, + "reward": -8.344649238267721e-09, + "reward_std": 0.9977494418621063, + "rewards/rnd_reward/mean": -8.344651369895927e-09, + "rewards/rnd_reward/std": 0.9977494418621063, + "step": 540, + "step_time": 9.593502512201667 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13500000387430192, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.6, + "completions/mean_length": 111.18750305175782, + "completions/mean_terminated_length": 68.47658958435059, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.681893822550774, + "epoch": 0.3597122302158273, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1282111406326294, + "kl": 0.06902076229453087, + "learning_rate": 4.1023544800523213e-07, + "loss": 0.023469969630241394, + "num_tokens": 12862906.0, + "reward": -5.677342400645102e-08, + "reward_std": 0.9978136360645294, + "rewards/rnd_reward/mean": -5.6773422585365554e-08, + "rewards/rnd_reward/std": 0.997813630104065, + "step": 550, + "step_time": 9.778036265238189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12000000327825547, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.0, + "completions/mean_length": 104.53875045776367, + "completions/mean_terminated_length": 66.3183536529541, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5951172381639482, + "epoch": 0.36625245258338784, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1439512968063354, + "kl": 0.07057485273107886, + "learning_rate": 4.0860039241334206e-07, + "loss": 0.1278893232345581, + "num_tokens": 13049132.0, + "reward": -1.1786818543413347e-07, + "reward_std": 0.9977734804153442, + "rewards/rnd_reward/mean": -1.17868184013048e-07, + "rewards/rnd_reward/std": 0.9977734982967377, + "step": 560, + "step_time": 10.389736549765804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10500000044703484, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.3, + "completions/mean_length": 99.66875076293945, + "completions/mean_terminated_length": 66.29649963378907, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6503134578466416, + "epoch": 0.3727926749509483, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0748413801193237, + "kl": 0.08892364026978612, + "learning_rate": 4.069653368214519e-07, + "loss": 0.13109040260314941, + "num_tokens": 13221157.0, + "reward": -1.4342367933295462e-07, + "reward_std": 0.997740238904953, + "rewards/rnd_reward/mean": -1.4342367791186916e-07, + "rewards/rnd_reward/std": 0.9977402448654175, + "step": 570, + "step_time": 9.71618521134369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12250000312924385, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 110.21125030517578, + "completions/mean_terminated_length": 71.826904296875, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.702208098769188, + "epoch": 0.37933289731850883, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0691598653793335, + "kl": 0.07306001400575042, + "learning_rate": 4.053302812295618e-07, + "loss": 0.09580587744712829, + "num_tokens": 13409746.0, + "reward": -1.6450882269758438e-07, + "reward_std": 0.9975189864635468, + "rewards/rnd_reward/mean": -1.6450882198704164e-07, + "rewards/rnd_reward/std": 0.9975189983844757, + "step": 580, + "step_time": 10.602098648622633 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13250000216066837, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.2, + "completions/mean_length": 118.48125228881835, + "completions/mean_terminated_length": 77.98408432006836, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5411072075366974, + "epoch": 0.3858731196860693, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8738215565681458, + "kl": 0.06685408288612962, + "learning_rate": 4.0369522563767164e-07, + "loss": 0.0531582236289978, + "num_tokens": 13603691.0, + "reward": -8.791685381925874e-08, + "reward_std": 0.9976041257381439, + "rewards/rnd_reward/mean": -8.791684955600231e-08, + "rewards/rnd_reward/std": 0.9976041257381439, + "step": 590, + "step_time": 10.606248078867793 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13375000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.1, + "completions/mean_length": 120.9775001525879, + "completions/mean_terminated_length": 80.35308418273925, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.454717567563057, + "epoch": 0.3924133420536298, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8597258925437927, + "kl": 0.06251584179699421, + "learning_rate": 4.020601700457815e-07, + "loss": 0.10086486339569092, + "num_tokens": 13793088.0, + "reward": -4.872679788547885e-08, + "reward_std": 0.997677344083786, + "rewards/rnd_reward/mean": -4.872679859602158e-08, + "rewards/rnd_reward/std": 0.997677332162857, + "step": 600, + "step_time": 9.838808755297213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13875000327825546, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.2, + "completions/mean_length": 112.62250213623047, + "completions/mean_terminated_length": 69.04776878356934, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5064212560653685, + "epoch": 0.39895356442119034, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0397368669509888, + "kl": 0.06775042302906513, + "learning_rate": 4.0042511445389145e-07, + "loss": 0.14560239315032958, + "num_tokens": 13975906.0, + "reward": -1.70022253698221e-07, + "reward_std": 0.997452187538147, + "rewards/rnd_reward/mean": -1.7002224872442184e-07, + "rewards/rnd_reward/std": 0.9974521696567535, + "step": 610, + "step_time": 9.711915594828316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11000000387430191, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.5, + "completions/mean_length": 105.6062515258789, + "completions/mean_terminated_length": 71.07123832702636, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.531564822793007, + "epoch": 0.4054937867887508, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8839622139930725, + "kl": 0.07682965183630586, + "learning_rate": 3.987900588620013e-07, + "loss": 0.06944795846939086, + "num_tokens": 14154656.0, + "reward": 5.662443069986978e-09, + "reward_std": 0.9975684463977814, + "rewards/rnd_reward/mean": 5.662442181808558e-09, + "rewards/rnd_reward/std": 0.9975684404373169, + "step": 620, + "step_time": 10.306970553193242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10375000201165677, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.6, + "completions/mean_length": 106.93500137329102, + "completions/mean_terminated_length": 74.97318267822266, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.859010672569275, + "epoch": 0.41203400915631133, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.103442907333374, + "kl": 0.08963548736646772, + "learning_rate": 3.9715500327011115e-07, + "loss": 0.033435261249542235, + "num_tokens": 14327599.0, + "reward": -1.2174248684715394e-07, + "reward_std": 0.9974590480327606, + "rewards/rnd_reward/mean": -1.2174248613661122e-07, + "rewards/rnd_reward/std": 0.9974590301513672, + "step": 630, + "step_time": 9.607878714823164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12625000327825547, + "completions/max_length": 384.0, + "completions/max_terminated_length": 343.2, + "completions/mean_length": 108.95125045776368, + "completions/mean_terminated_length": 69.31592407226563, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.823798879981041, + "epoch": 0.4185742315238718, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1582435369491577, + "kl": 0.09182370994240045, + "learning_rate": 3.9551994767822103e-07, + "loss": 0.06608177423477173, + "num_tokens": 14513650.0, + "reward": -1.3239681819765537e-07, + "reward_std": 0.9976047217845917, + "rewards/rnd_reward/mean": -1.3239681961874084e-07, + "rewards/rnd_reward/std": 0.9976047277450562, + "step": 640, + "step_time": 10.307288937154226 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11000000163912774, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.8, + "completions/mean_length": 107.63500213623047, + "completions/mean_terminated_length": 73.39098167419434, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.695296034216881, + "epoch": 0.4251144538914323, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0280959606170654, + "kl": 0.09326308779418468, + "learning_rate": 3.938848920863309e-07, + "loss": 0.030906683206558226, + "num_tokens": 14693328.0, + "reward": -1.2964010291938167e-07, + "reward_std": 0.9975219249725342, + "rewards/rnd_reward/mean": -1.2964010327465302e-07, + "rewards/rnd_reward/std": 0.9975219309329987, + "step": 650, + "step_time": 10.01973620983772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1512500040233135, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.3, + "completions/mean_length": 129.62250213623048, + "completions/mean_terminated_length": 84.16030578613281, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.613438582420349, + "epoch": 0.4316546762589928, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9445599317550659, + "kl": 0.0699499910697341, + "learning_rate": 3.922498364944408e-07, + "loss": 0.08974806666374206, + "num_tokens": 14892686.0, + "reward": -4.194676717084178e-08, + "reward_std": 0.9975471794605255, + "rewards/rnd_reward/mean": -4.1946770057421645e-08, + "rewards/rnd_reward/std": 0.99754718542099, + "step": 660, + "step_time": 10.161354359728284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14750000312924386, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.7, + "completions/mean_length": 126.59500045776367, + "completions/mean_terminated_length": 81.99441680908203, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.7322494119405745, + "epoch": 0.4381948986265533, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9810017943382263, + "kl": 0.08053934266790748, + "learning_rate": 3.9061478090255067e-07, + "loss": 0.14365124702453613, + "num_tokens": 15083047.0, + "reward": -7.979571803318208e-08, + "reward_std": 0.9972860395908356, + "rewards/rnd_reward/mean": -7.979571803318208e-08, + "rewards/rnd_reward/std": 0.9972860395908356, + "step": 670, + "step_time": 9.683341883006506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14250000119209288, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.5, + "completions/mean_length": 125.95000381469727, + "completions/mean_terminated_length": 83.18684883117676, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6392508417367937, + "epoch": 0.4447351209941138, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9357602596282959, + "kl": 0.07438107710331679, + "learning_rate": 3.8897972531066054e-07, + "loss": 0.047136351466178894, + "num_tokens": 15272992.0, + "reward": -1.0043382800972722e-07, + "reward_std": 0.9973943531513214, + "rewards/rnd_reward/mean": -1.0043382658864175e-07, + "rewards/rnd_reward/std": 0.997394347190857, + "step": 680, + "step_time": 9.536431295378133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14375000223517417, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.2, + "completions/mean_length": 125.81125106811524, + "completions/mean_terminated_length": 82.36017532348633, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.2503296345472337, + "epoch": 0.4512753433616743, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7484872937202454, + "kl": 0.0611060356721282, + "learning_rate": 3.873446697187704e-07, + "loss": 0.09799213409423828, + "num_tokens": 15463906.0, + "reward": -6.347894991876046e-08, + "reward_std": 0.9972564399242401, + "rewards/rnd_reward/mean": -6.34789495634891e-08, + "rewards/rnd_reward/std": 0.9972564518451691, + "step": 690, + "step_time": 9.653547045495362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15500000342726708, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.6, + "completions/mean_length": 133.82500076293945, + "completions/mean_terminated_length": 87.98081855773925, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5280281245708465, + "epoch": 0.4578155657292348, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9658915996551514, + "kl": 0.059800021536648276, + "learning_rate": 3.857096141268803e-07, + "loss": 0.12680201530456542, + "num_tokens": 15667301.0, + "reward": -1.5869737026719122e-07, + "reward_std": 0.9973293662071228, + "rewards/rnd_reward/mean": -1.5869737026719122e-07, + "rewards/rnd_reward/std": 0.9973293721675873, + "step": 700, + "step_time": 9.966963570658118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10875000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.1, + "completions/mean_length": 107.25000228881837, + "completions/mean_terminated_length": 73.4811897277832, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4395824640989305, + "epoch": 0.4643557880967953, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.049780011177063, + "kl": 0.08286946956068278, + "learning_rate": 3.840745585349902e-07, + "loss": 0.15116369724273682, + "num_tokens": 15847671.0, + "reward": -1.005828406341891e-07, + "reward_std": 0.9973811268806457, + "rewards/rnd_reward/mean": -1.0058283921310362e-07, + "rewards/rnd_reward/std": 0.9973811089992524, + "step": 710, + "step_time": 9.890902339830063 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09000000115484, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.2, + "completions/mean_length": 93.01625213623046, + "completions/mean_terminated_length": 64.09797096252441, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8066621690988542, + "epoch": 0.4708960104643558, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34576416015625, + "kl": 0.10443261573091149, + "learning_rate": 3.8243950294310006e-07, + "loss": 0.10605430603027344, + "num_tokens": 16021059.0, + "reward": -1.0505319210096787e-07, + "reward_std": 0.9972734034061432, + "rewards/rnd_reward/mean": -1.0505318357445503e-07, + "rewards/rnd_reward/std": 0.9972733795642853, + "step": 720, + "step_time": 10.098007511859759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000074505807, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 94.07875137329101, + "completions/mean_terminated_length": 66.15557479858398, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6466055154800414, + "epoch": 0.4774362328319163, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0772963762283325, + "kl": 0.10096871191635728, + "learning_rate": 3.8080444735120994e-07, + "loss": 0.09338142871856689, + "num_tokens": 16191222.0, + "reward": -1.777708558137192e-07, + "reward_std": 0.9971938729286194, + "rewards/rnd_reward/mean": -1.7777085012937732e-07, + "rewards/rnd_reward/std": 0.9971938788890838, + "step": 730, + "step_time": 10.299412568542175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13375000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.1, + "completions/mean_length": 126.26250228881835, + "completions/mean_terminated_length": 86.43918380737304, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.173070195317268, + "epoch": 0.4839764551994768, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7416156530380249, + "kl": 0.053471417538821696, + "learning_rate": 3.7916939175931976e-07, + "loss": 0.07696372270584106, + "num_tokens": 16393972.0, + "reward": -1.7151236602064303e-07, + "reward_std": 0.9972269594669342, + "rewards/rnd_reward/mean": -1.715123674417285e-07, + "rewards/rnd_reward/std": 0.9972269415855408, + "step": 740, + "step_time": 10.502406498114578 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.148750002682209, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 139.12875213623047, + "completions/mean_terminated_length": 96.77134628295899, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.325762075185776, + "epoch": 0.49051667756703726, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7352995872497559, + "kl": 0.05052672061137855, + "learning_rate": 3.775343361674297e-07, + "loss": 0.09521990418434143, + "num_tokens": 16596670.0, + "reward": -2.0787120362797397e-07, + "reward_std": 0.9972488164901734, + "rewards/rnd_reward/mean": -2.0787119723308933e-07, + "rewards/rnd_reward/std": 0.9972488105297088, + "step": 750, + "step_time": 9.698242208594456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14750000238418579, + "completions/max_length": 384.0, + "completions/max_terminated_length": 371.2, + "completions/mean_length": 127.9612533569336, + "completions/mean_terminated_length": 83.97822761535645, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.2959645807743074, + "epoch": 0.4970568999345978, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8263393044471741, + "kl": 0.05697637004777789, + "learning_rate": 3.7589928057553957e-07, + "loss": 0.10516003370285035, + "num_tokens": 16792629.0, + "reward": -1.6942620391091622e-07, + "reward_std": 0.9972022175788879, + "rewards/rnd_reward/mean": -1.6942620106874529e-07, + "rewards/rnd_reward/std": 0.9972022175788879, + "step": 760, + "step_time": 10.044187037739903 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10875000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 339.2, + "completions/mean_length": 112.1150016784668, + "completions/mean_terminated_length": 79.13344192504883, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7552184253931045, + "epoch": 0.5035971223021583, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.029379963874817, + "kl": 0.07228619959205389, + "learning_rate": 3.742642249836494e-07, + "loss": 0.03081640601158142, + "num_tokens": 16971246.0, + "reward": -8.299946501200139e-08, + "reward_std": 0.9970763862133026, + "rewards/rnd_reward/mean": -8.299946785417234e-08, + "rewards/rnd_reward/std": 0.9970763862133026, + "step": 770, + "step_time": 9.667120404168964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13125000298023223, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.9, + "completions/mean_length": 124.3275032043457, + "completions/mean_terminated_length": 85.28903274536133, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.8021608620882033, + "epoch": 0.5101373446697187, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0165283679962158, + "kl": 0.07025760738179088, + "learning_rate": 3.7262916939175933e-07, + "loss": 0.03996403813362122, + "num_tokens": 17162308.0, + "reward": -5.170702825907369e-08, + "reward_std": 0.9971189498901367, + "rewards/rnd_reward/mean": -5.170703110124464e-08, + "rewards/rnd_reward/std": 0.9971189439296723, + "step": 780, + "step_time": 9.969513631938025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13375000283122063, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.5, + "completions/mean_length": 127.03500213623047, + "completions/mean_terminated_length": 87.38848876953125, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.637522208690643, + "epoch": 0.5166775670372793, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9886835813522339, + "kl": 0.06816753121092915, + "learning_rate": 3.7099411379986915e-07, + "loss": 0.07478119730949402, + "num_tokens": 17356536.0, + "reward": -1.1831521575800253e-07, + "reward_std": 0.9971611678600312, + "rewards/rnd_reward/mean": -1.1831521575800253e-07, + "rewards/rnd_reward/std": 0.997161191701889, + "step": 790, + "step_time": 9.748413591040299 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12625000327825547, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.3, + "completions/mean_length": 120.86375045776367, + "completions/mean_terminated_length": 83.05458145141601, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6706619918346406, + "epoch": 0.5232177894048398, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8250897526741028, + "kl": 0.07074365848675371, + "learning_rate": 3.693590582079791e-07, + "loss": 0.05061581134796143, + "num_tokens": 17545417.0, + "reward": -6.467104096685716e-08, + "reward_std": 0.9972768127918243, + "rewards/rnd_reward/mean": -6.467104096685716e-08, + "rewards/rnd_reward/std": 0.9972768068313599, + "step": 800, + "step_time": 9.678878564713523 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12500000298023223, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.5, + "completions/mean_length": 121.30500183105468, + "completions/mean_terminated_length": 83.6633544921875, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.795678195357323, + "epoch": 0.5297580117724002, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0277215242385864, + "kl": 0.06990557787939906, + "learning_rate": 3.677240026160889e-07, + "loss": 0.059202158451080324, + "num_tokens": 17737161.0, + "reward": -1.2487173535191686e-07, + "reward_std": 0.9970629572868347, + "rewards/rnd_reward/mean": -1.2487172682540403e-07, + "rewards/rnd_reward/std": 0.9970629572868347, + "step": 810, + "step_time": 10.510143173905089 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.4, + "completions/mean_length": 119.66500091552734, + "completions/mean_terminated_length": 80.23575859069824, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8059179961681364, + "epoch": 0.5362982341399608, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8309988379478455, + "kl": 0.08165676584467292, + "learning_rate": 3.660889470241988e-07, + "loss": 0.12505731582641602, + "num_tokens": 17927903.0, + "reward": -2.3901462826714236e-07, + "reward_std": 0.997013247013092, + "rewards/rnd_reward/mean": -2.3901462400388593e-07, + "rewards/rnd_reward/std": 0.9970132112503052, + "step": 820, + "step_time": 10.190648046904244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14875000417232515, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.9, + "completions/mean_length": 126.17125396728515, + "completions/mean_terminated_length": 80.60145225524903, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.938824012875557, + "epoch": 0.5428384565075213, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1022851467132568, + "kl": 0.07984373616054655, + "learning_rate": 3.644538914323087e-07, + "loss": 0.023773609101772307, + "num_tokens": 18123110.0, + "reward": -2.2649766062698973e-08, + "reward_std": 0.9969716727733612, + "rewards/rnd_reward/mean": -2.2649762509985294e-08, + "rewards/rnd_reward/std": 0.9969716787338256, + "step": 830, + "step_time": 9.89921972535085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14375000447034836, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.7, + "completions/mean_length": 127.39125366210938, + "completions/mean_terminated_length": 84.2758415222168, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8030223518610002, + "epoch": 0.5493786788750817, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9427273273468018, + "kl": 0.0753337662667036, + "learning_rate": 3.6281883584041854e-07, + "loss": 0.05915579795837402, + "num_tokens": 18319518.0, + "reward": -1.1295080284412507e-07, + "reward_std": 0.9969306707382202, + "rewards/rnd_reward/mean": -1.1295080142303959e-07, + "rewards/rnd_reward/std": 0.9969306766986847, + "step": 840, + "step_time": 9.87428822228685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15750000178813933, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.2, + "completions/mean_length": 137.7587516784668, + "completions/mean_terminated_length": 91.88029098510742, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.810399681329727, + "epoch": 0.5559189012426422, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8602397441864014, + "kl": 0.0653490168042481, + "learning_rate": 3.611837802485284e-07, + "loss": 0.03035195469856262, + "num_tokens": 18526310.0, + "reward": -1.1295080284412507e-07, + "reward_std": 0.9969462454319, + "rewards/rnd_reward/mean": -1.1295080426521054e-07, + "rewards/rnd_reward/std": 0.9969462633132935, + "step": 850, + "step_time": 10.37288966330234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15250000283122062, + "completions/max_length": 384.0, + "completions/max_terminated_length": 343.9, + "completions/mean_length": 127.73250350952148, + "completions/mean_terminated_length": 81.79599456787109, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.83120773434639, + "epoch": 0.5624591236102028, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0732090473175049, + "kl": 0.08459939705207944, + "learning_rate": 3.595487246566383e-07, + "loss": 0.03491010665893555, + "num_tokens": 18724316.0, + "reward": -1.4752149617436318e-07, + "reward_std": 0.9969268858432769, + "rewards/rnd_reward/mean": -1.4752149759544864e-07, + "rewards/rnd_reward/std": 0.9969269037246704, + "step": 860, + "step_time": 10.52987401576247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14000000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.0, + "completions/mean_length": 131.95375213623046, + "completions/mean_terminated_length": 90.77078781127929, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.9562632203102113, + "epoch": 0.5689993459777632, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.969403862953186, + "kl": 0.08435474634170533, + "learning_rate": 3.579136690647482e-07, + "loss": 0.04807090759277344, + "num_tokens": 18926489.0, + "reward": -1.3545155894689742e-07, + "reward_std": 0.9968080937862396, + "rewards/rnd_reward/mean": -1.3545156178906837e-07, + "rewards/rnd_reward/std": 0.9968081057071686, + "step": 870, + "step_time": 9.85819350676611 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16750000193715095, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.2, + "completions/mean_length": 137.6675033569336, + "completions/mean_terminated_length": 88.34969863891601, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7675389796495438, + "epoch": 0.5755395683453237, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.828363835811615, + "kl": 0.07512732800096274, + "learning_rate": 3.5627861347285806e-07, + "loss": 0.010637739300727844, + "num_tokens": 19126833.0, + "reward": -7.003545690054125e-08, + "reward_std": 0.9967847049236298, + "rewards/rnd_reward/mean": -7.003545903216945e-08, + "rewards/rnd_reward/std": 0.9967847049236298, + "step": 880, + "step_time": 9.748134864587337 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19125000312924384, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.5, + "completions/mean_length": 148.54750366210936, + "completions/mean_terminated_length": 93.13098068237305, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.6605105638504027, + "epoch": 0.5820797907128843, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8364513516426086, + "kl": 0.06730008013546467, + "learning_rate": 3.5464355788096794e-07, + "loss": 0.0366318553686142, + "num_tokens": 19338336.0, + "reward": -1.0654330893089536e-07, + "reward_std": 0.9968020260334015, + "rewards/rnd_reward/mean": -1.0654329685166886e-07, + "rewards/rnd_reward/std": 0.9968020260334015, + "step": 890, + "step_time": 9.872815124248154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16625000312924385, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.5, + "completions/mean_length": 149.7400032043457, + "completions/mean_terminated_length": 103.11838226318359, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.535254544019699, + "epoch": 0.5886200130804448, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7181042432785034, + "kl": 0.05683262627571821, + "learning_rate": 3.530085022890778e-07, + "loss": 0.08454052209854127, + "num_tokens": 19557363.0, + "reward": -1.737475452401327e-07, + "reward_std": 0.9967290103435517, + "rewards/rnd_reward/mean": -1.737475338714489e-07, + "rewards/rnd_reward/std": 0.9967289924621582, + "step": 900, + "step_time": 10.189786232402549 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1912500038743019, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 157.67875213623046, + "completions/mean_terminated_length": 104.52761840820312, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5941266864538193, + "epoch": 0.5951602354480052, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9426244497299194, + "kl": 0.050556823750957845, + "learning_rate": 3.513734466971877e-07, + "loss": 0.07448753714561462, + "num_tokens": 19775536.0, + "reward": -2.162158537544201e-07, + "reward_std": 0.9968621492385864, + "rewards/rnd_reward/mean": -2.1621584735953548e-07, + "rewards/rnd_reward/std": 0.9968621671199799, + "step": 910, + "step_time": 9.74868165422231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1662500023841858, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.0, + "completions/mean_length": 151.89000396728517, + "completions/mean_terminated_length": 105.61829071044922, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.508600893616676, + "epoch": 0.6017004578155657, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7371200323104858, + "kl": 0.04880232191644609, + "learning_rate": 3.4973839110529757e-07, + "loss": 0.12514564990997315, + "num_tokens": 19990298.0, + "reward": -1.93715107066339e-08, + "reward_std": 0.9970090925693512, + "rewards/rnd_reward/mean": -1.937150893027706e-08, + "rewards/rnd_reward/std": 0.9970090866088868, + "step": 920, + "step_time": 9.937099263281562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1575000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.5, + "completions/mean_length": 138.63375244140624, + "completions/mean_terminated_length": 92.52680130004883, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4236575067043304, + "epoch": 0.6082406801831263, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9167628884315491, + "kl": 0.05787700763903558, + "learning_rate": 3.4810333551340745e-07, + "loss": 0.10835990905761719, + "num_tokens": 20187745.0, + "reward": -7.212161943215278e-08, + "reward_std": 0.996803218126297, + "rewards/rnd_reward/mean": -7.212161801106731e-08, + "rewards/rnd_reward/std": 0.9968032300472259, + "step": 930, + "step_time": 9.689079960295931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1337500013411045, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.4, + "completions/mean_length": 129.22500305175782, + "completions/mean_terminated_length": 89.93161087036133, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.6605860769748686, + "epoch": 0.6147809025506867, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0838011503219604, + "kl": 0.06689512394368649, + "learning_rate": 3.4646827992151733e-07, + "loss": 0.07303244471549988, + "num_tokens": 20383990.0, + "reward": -1.1324883786301143e-08, + "reward_std": 0.9967760264873504, + "rewards/rnd_reward/mean": -1.1324883786301143e-08, + "rewards/rnd_reward/std": 0.9967760384082794, + "step": 940, + "step_time": 9.85847494918853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16125000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 364.1, + "completions/mean_length": 134.65625534057617, + "completions/mean_terminated_length": 86.58066329956054, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7146200090646744, + "epoch": 0.6213211249182472, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9745354652404785, + "kl": 0.06564742000773549, + "learning_rate": 3.448332243296272e-07, + "loss": 0.10424284934997559, + "num_tokens": 20593190.0, + "reward": -1.0967255157368072e-07, + "reward_std": 0.9968843460083008, + "rewards/rnd_reward/mean": -1.096725451787961e-07, + "rewards/rnd_reward/std": 0.9968843400478363, + "step": 950, + "step_time": 10.862376277497969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10875000022351741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.1, + "completions/mean_length": 113.23000411987304, + "completions/mean_terminated_length": 80.14122467041015, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8509408473968505, + "epoch": 0.6278613472858077, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9323987364768982, + "kl": 0.08510139184072614, + "learning_rate": 3.4319816873773703e-07, + "loss": 0.0554282009601593, + "num_tokens": 20776164.0, + "reward": -7.957220322651893e-08, + "reward_std": 0.9968249201774597, + "rewards/rnd_reward/mean": -7.957220180543345e-08, + "rewards/rnd_reward/std": 0.9968249082565308, + "step": 960, + "step_time": 9.989999298867769 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10750000327825546, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.2, + "completions/mean_length": 112.88375091552734, + "completions/mean_terminated_length": 80.20179443359375, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.88864361345768, + "epoch": 0.6344015696533682, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8851020932197571, + "kl": 0.0875691649504006, + "learning_rate": 3.4156311314584696e-07, + "loss": 0.10126043558120727, + "num_tokens": 20958826.0, + "reward": -1.6622245713193706e-07, + "reward_std": 0.9968612968921662, + "rewards/rnd_reward/mean": -1.6622244931596696e-07, + "rewards/rnd_reward/std": 0.9968612730503082, + "step": 970, + "step_time": 10.018015099223703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11125000156462192, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.6, + "completions/mean_length": 106.1625, + "completions/mean_terminated_length": 71.38758811950683, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9656033277511598, + "epoch": 0.6409417920209287, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9377958178520203, + "kl": 0.09664760064333677, + "learning_rate": 3.3992805755395684e-07, + "loss": 0.046259531378746034, + "num_tokens": 21147866.0, + "reward": -2.499669804478799e-07, + "reward_std": 0.9968202471733093, + "rewards/rnd_reward/mean": -2.499669751188094e-07, + "rewards/rnd_reward/std": 0.9968202352523804, + "step": 980, + "step_time": 11.508573367865756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1262500025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.6, + "completions/mean_length": 122.56625289916992, + "completions/mean_terminated_length": 84.71166687011718, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.80318078994751, + "epoch": 0.6474820143884892, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9525420069694519, + "kl": 0.07310699988156558, + "learning_rate": 3.3829300196206667e-07, + "loss": 0.02322930097579956, + "num_tokens": 21338124.0, + "reward": -9.76771161731449e-08, + "reward_std": 0.9966861188411713, + "rewards/rnd_reward/mean": -9.7677110488803e-08, + "rewards/rnd_reward/std": 0.9966861248016358, + "step": 990, + "step_time": 9.876657200045884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15375000201165676, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.8, + "completions/mean_length": 137.66875076293945, + "completions/mean_terminated_length": 92.6232063293457, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9049280107021334, + "epoch": 0.6540222367560498, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8244759440422058, + "kl": 0.061428536847233775, + "learning_rate": 3.366579463701766e-07, + "loss": 0.05219428539276123, + "num_tokens": 21539579.0, + "reward": -1.6495585271059098e-07, + "reward_std": 0.9966946303844452, + "rewards/rnd_reward/mean": -1.6495585271059098e-07, + "rewards/rnd_reward/std": 0.9966946005821228, + "step": 1000, + "step_time": 9.666829459602013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18250000178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.9, + "completions/mean_length": 146.93875427246093, + "completions/mean_terminated_length": 93.79228134155274, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8016369342803955, + "epoch": 0.6605624591236102, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.766015350818634, + "kl": 0.05675441958010197, + "learning_rate": 3.350228907782864e-07, + "loss": 0.08666471242904664, + "num_tokens": 21754125.0, + "reward": -2.2858381782953073e-07, + "reward_std": 0.996807062625885, + "rewards/rnd_reward/mean": -2.2858381640844527e-07, + "rewards/rnd_reward/std": 0.9968070745468139, + "step": 1010, + "step_time": 10.005978721287102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13875000253319741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.3, + "completions/mean_length": 136.35500259399413, + "completions/mean_terminated_length": 96.46944885253906, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8755707025527952, + "epoch": 0.6671026814911707, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0061216354370117, + "kl": 0.05596274421550333, + "learning_rate": 3.3338783518639635e-07, + "loss": 0.09954084157943725, + "num_tokens": 21957634.0, + "reward": -1.9520521661320344e-07, + "reward_std": 0.9966324806213379, + "rewards/rnd_reward/mean": -1.9520520950777608e-07, + "rewards/rnd_reward/std": 0.9966325044631958, + "step": 1020, + "step_time": 10.040163300000131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15875000432133674, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.5, + "completions/mean_length": 136.7437530517578, + "completions/mean_terminated_length": 90.20843048095703, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7692076802253722, + "epoch": 0.6736429038587312, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8324238061904907, + "kl": 0.057948453910648824, + "learning_rate": 3.317527795945062e-07, + "loss": 0.07627512216567993, + "num_tokens": 22158654.0, + "reward": -3.7699937749380295e-08, + "reward_std": 0.9966008186340332, + "rewards/rnd_reward/mean": -3.769993632829482e-08, + "rewards/rnd_reward/std": 0.9966008424758911, + "step": 1030, + "step_time": 10.1797529961681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16250000298023223, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.7, + "completions/mean_length": 139.57500534057618, + "completions/mean_terminated_length": 92.21996688842773, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.8831230252981186, + "epoch": 0.6801831262262917, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8833721876144409, + "kl": 0.060850747767835854, + "learning_rate": 3.3011772400261606e-07, + "loss": 0.10484603643417359, + "num_tokens": 22366559.0, + "reward": -1.0505318579490108e-07, + "reward_std": 0.996761429309845, + "rewards/rnd_reward/mean": -1.0505318588371893e-07, + "rewards/rnd_reward/std": 0.9967614591121674, + "step": 1040, + "step_time": 10.081386449630372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1625000037252903, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.9, + "completions/mean_length": 137.33875198364257, + "completions/mean_terminated_length": 89.72673568725585, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.885282647609711, + "epoch": 0.6867233485938522, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9477590918540955, + "kl": 0.05932593122124672, + "learning_rate": 3.28482668410726e-07, + "loss": 0.024539683759212495, + "num_tokens": 22571575.0, + "reward": 4.9620866704458424e-08, + "reward_std": 0.996524053812027, + "rewards/rnd_reward/mean": 4.9620866704458424e-08, + "rewards/rnd_reward/std": 0.9965240597724915, + "step": 1050, + "step_time": 10.535549363144673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16250000447034835, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.5, + "completions/mean_length": 138.3237533569336, + "completions/mean_terminated_length": 90.70624694824218, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.875243663787842, + "epoch": 0.6932635709614127, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9407637119293213, + "kl": 0.06043377108871937, + "learning_rate": 3.268476128188358e-07, + "loss": 0.06624546647071838, + "num_tokens": 22771314.0, + "reward": -5.781650713743147e-08, + "reward_std": 0.9966591358184814, + "rewards/rnd_reward/mean": -5.7816505716346e-08, + "rewards/rnd_reward/std": 0.9966591358184814, + "step": 1060, + "step_time": 10.021701269014738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1400000039488077, + "completions/max_length": 384.0, + "completions/max_terminated_length": 342.1, + "completions/mean_length": 127.3300048828125, + "completions/mean_terminated_length": 85.42830429077148, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.927751249074936, + "epoch": 0.6998037933289731, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8508753180503845, + "kl": 0.0620661674067378, + "learning_rate": 3.252125572269457e-07, + "loss": 0.12821773290634156, + "num_tokens": 22965548.0, + "reward": -1.0244548391824537e-07, + "reward_std": 0.9966984510421752, + "rewards/rnd_reward/mean": -1.0244548391824537e-07, + "rewards/rnd_reward/std": 0.9966984808444976, + "step": 1070, + "step_time": 9.767576195369475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14500000327825546, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.6, + "completions/mean_length": 136.3862518310547, + "completions/mean_terminated_length": 94.67120056152343, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.6616650968790054, + "epoch": 0.7063440156965337, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.789601743221283, + "kl": 0.0545062692835927, + "learning_rate": 3.2357750163505557e-07, + "loss": 0.11497254371643066, + "num_tokens": 23170007.0, + "reward": 1.226365569095833e-07, + "reward_std": 0.9966248035430908, + "rewards/rnd_reward/mean": 1.2263655548849784e-07, + "rewards/rnd_reward/std": 0.9966248333454132, + "step": 1080, + "step_time": 9.882125865970739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14250000566244125, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.8, + "completions/mean_length": 130.01000289916993, + "completions/mean_terminated_length": 87.69976119995117, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.890427288413048, + "epoch": 0.7128842380640942, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8021807670593262, + "kl": 0.05592884449288249, + "learning_rate": 3.2194244604316545e-07, + "loss": 0.09672708511352539, + "num_tokens": 23366425.0, + "reward": 6.273389110411643e-08, + "reward_std": 0.9965825915336609, + "rewards/rnd_reward/mean": 6.273389181465916e-08, + "rewards/rnd_reward/std": 0.9965825736522674, + "step": 1090, + "step_time": 9.854898347100242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14500000178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 336.9, + "completions/mean_length": 131.98875274658204, + "completions/mean_terminated_length": 89.18458480834961, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7609634190797805, + "epoch": 0.7194244604316546, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0092402696609497, + "kl": 0.05501702083274722, + "learning_rate": 3.2030739045127533e-07, + "loss": 0.08695750832557678, + "num_tokens": 23567981.0, + "reward": -1.3828277736216422e-07, + "reward_std": 0.9966061592102051, + "rewards/rnd_reward/mean": -1.3828277451999326e-07, + "rewards/rnd_reward/std": 0.9966061592102051, + "step": 1100, + "step_time": 10.145819194451906 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1212500013411045, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.9, + "completions/mean_length": 121.10875091552734, + "completions/mean_terminated_length": 84.89219970703125, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8415472239255903, + "epoch": 0.7259646827992152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8949585556983948, + "kl": 0.06347150038927793, + "learning_rate": 3.186723348593852e-07, + "loss": 0.08753393888473511, + "num_tokens": 23758168.0, + "reward": -1.1354684570363816e-07, + "reward_std": 0.9964610040187836, + "rewards/rnd_reward/mean": -1.1354684286146721e-07, + "rewards/rnd_reward/std": 0.9964609861373901, + "step": 1110, + "step_time": 10.006089114071802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12125000208616257, + "completions/max_length": 384.0, + "completions/max_terminated_length": 369.7, + "completions/mean_length": 121.20000228881835, + "completions/mean_terminated_length": 84.96716842651367, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.9164877772331237, + "epoch": 0.7325049051667757, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1227521896362305, + "kl": 0.06196983400732279, + "learning_rate": 3.170372792674951e-07, + "loss": 0.07237975597381592, + "num_tokens": 23955023.0, + "reward": -5.5432321488524396e-08, + "reward_std": 0.9964903295040131, + "rewards/rnd_reward/mean": -5.543231935689619e-08, + "rewards/rnd_reward/std": 0.9964903354644775, + "step": 1120, + "step_time": 10.375506636733189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11875000298023224, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.2, + "completions/mean_length": 117.09375, + "completions/mean_terminated_length": 81.17416534423828, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.7361535727977753, + "epoch": 0.7390451275343362, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8529885411262512, + "kl": 0.06673006983473898, + "learning_rate": 3.1540222367560496e-07, + "loss": 0.09247955083847045, + "num_tokens": 24140203.0, + "reward": -2.6300548938706926e-08, + "reward_std": 0.9966336786746979, + "rewards/rnd_reward/mean": -2.6300551070335132e-08, + "rewards/rnd_reward/std": 0.9966336965560914, + "step": 1130, + "step_time": 10.012891574040987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11500000357627868, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.7, + "completions/mean_length": 119.6712516784668, + "completions/mean_terminated_length": 85.35453567504882, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6402025282382966, + "epoch": 0.7455853499018966, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9655517935752869, + "kl": 0.06339178960770368, + "learning_rate": 3.1376716808371484e-07, + "loss": 0.012396740168333054, + "num_tokens": 24333505.0, + "reward": -1.3798475424486866e-07, + "reward_std": 0.9963988780975341, + "rewards/rnd_reward/mean": -1.379847538895973e-07, + "rewards/rnd_reward/std": 0.9963988780975341, + "step": 1140, + "step_time": 10.81275618087966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12750000208616258, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.4, + "completions/mean_length": 130.15375289916992, + "completions/mean_terminated_length": 93.11451568603516, + "completions/min_length": 2.6, + "completions/min_terminated_length": 2.6, + "entropy": 2.7988665878772734, + "epoch": 0.7521255722694572, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9594887495040894, + "kl": 0.057864805683493614, + "learning_rate": 3.121321124918247e-07, + "loss": 0.05015655755996704, + "num_tokens": 24526988.0, + "reward": -2.7045605088815705e-08, + "reward_std": 0.9962463498115539, + "rewards/rnd_reward/mean": -2.7045605088815705e-08, + "rewards/rnd_reward/std": 0.9962463617324829, + "step": 1150, + "step_time": 9.847500793170184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000305473805, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.1, + "completions/mean_length": 119.13750076293945, + "completions/mean_terminated_length": 79.57122192382812, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.799566090106964, + "epoch": 0.7586657946370177, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3144031763076782, + "kl": 0.06464958414435387, + "learning_rate": 3.104970568999346e-07, + "loss": 0.08081794381141663, + "num_tokens": 24712093.0, + "reward": 2.2947787847726885e-08, + "reward_std": 0.9963953137397766, + "rewards/rnd_reward/mean": 2.2947789268812358e-08, + "rewards/rnd_reward/std": 0.9963953375816346, + "step": 1160, + "step_time": 9.831479465053416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 366.1, + "completions/mean_length": 128.67500228881835, + "completions/mean_terminated_length": 90.44079208374023, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.758173832297325, + "epoch": 0.7652060170045781, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8444108963012695, + "kl": 0.061153271794319154, + "learning_rate": 3.088620013080445e-07, + "loss": 0.06105284690856934, + "num_tokens": 24905938.0, + "reward": -9.223818580039733e-08, + "reward_std": 0.9962478995323181, + "rewards/rnd_reward/mean": -9.22381850898546e-08, + "rewards/rnd_reward/std": 0.9962479054927826, + "step": 1170, + "step_time": 9.715631593717262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13125000149011612, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.7, + "completions/mean_length": 132.65000381469727, + "completions/mean_terminated_length": 94.60303192138672, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.652452102303505, + "epoch": 0.7717462393721386, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8246821761131287, + "kl": 0.05974919479340315, + "learning_rate": 3.072269457161543e-07, + "loss": 0.07230232954025269, + "num_tokens": 25111683.0, + "reward": -1.4752150079289095e-08, + "reward_std": 0.9964145243167877, + "rewards/rnd_reward/mean": -1.4752149368746359e-08, + "rewards/rnd_reward/std": 0.9964145421981812, + "step": 1180, + "step_time": 10.24143796269782 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1437500037252903, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 133.8912498474121, + "completions/mean_terminated_length": 92.00172958374023, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.534428471326828, + "epoch": 0.7782864617396992, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7489418387413025, + "kl": 0.05822144625708461, + "learning_rate": 3.0559189012426423e-07, + "loss": 0.07474024295806884, + "num_tokens": 25319036.0, + "reward": -8.940696432091499e-08, + "reward_std": 0.9965973794460297, + "rewards/rnd_reward/mean": -8.940696574200047e-08, + "rewards/rnd_reward/std": 0.9965973973274231, + "step": 1190, + "step_time": 10.355807177536189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1450000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.6, + "completions/mean_length": 138.9825012207031, + "completions/mean_terminated_length": 97.18752059936523, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8433078229427338, + "epoch": 0.7848266841072596, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8156785368919373, + "kl": 0.06143994340673089, + "learning_rate": 3.039568345323741e-07, + "loss": 0.08815957307815551, + "num_tokens": 25526837.0, + "reward": -1.0192394315211572e-07, + "reward_std": 0.996451473236084, + "rewards/rnd_reward/mean": -1.0192393888885931e-07, + "rewards/rnd_reward/std": 0.996451485157013, + "step": 1200, + "step_time": 10.298059746343643 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14000000283122063, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.0, + "completions/mean_length": 127.81375274658203, + "completions/mean_terminated_length": 85.73782577514649, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7779837220907213, + "epoch": 0.7913669064748201, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.937714695930481, + "kl": 0.06527232667431235, + "learning_rate": 3.0232177894048394e-07, + "loss": 0.08186920881271362, + "num_tokens": 25721938.0, + "reward": -1.0982156268823929e-07, + "reward_std": 0.9962611317634582, + "rewards/rnd_reward/mean": -1.0982156055661107e-07, + "rewards/rnd_reward/std": 0.9962611377239228, + "step": 1210, + "step_time": 9.923247428331525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13625000193715095, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.9, + "completions/mean_length": 128.51625213623046, + "completions/mean_terminated_length": 88.076513671875, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.661186295747757, + "epoch": 0.7979071288423807, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9877737164497375, + "kl": 0.06118576750159264, + "learning_rate": 3.0068672334859387e-07, + "loss": 0.0857498049736023, + "num_tokens": 25916611.0, + "reward": -9.454786980001017e-08, + "reward_std": 0.9962615430355072, + "rewards/rnd_reward/mean": -9.454786553675376e-08, + "rewards/rnd_reward/std": 0.9962615609169007, + "step": 1220, + "step_time": 10.026552183739842 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13875000402331353, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.7, + "completions/mean_length": 127.41250381469726, + "completions/mean_terminated_length": 86.15239715576172, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.597662454843521, + "epoch": 0.8044473512099412, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9214726090431213, + "kl": 0.06120588518679142, + "learning_rate": 2.990516677567037e-07, + "loss": 0.08460624814033509, + "num_tokens": 26120156.0, + "reward": -1.5325844344715732e-07, + "reward_std": 0.9961971819400788, + "rewards/rnd_reward/mean": -1.5325843989444364e-07, + "rewards/rnd_reward/std": 0.9961971819400788, + "step": 1230, + "step_time": 10.459681713976897 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12875000089406968, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.9, + "completions/mean_length": 121.05875167846679, + "completions/mean_terminated_length": 82.47761840820313, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.899898463487625, + "epoch": 0.8109875735775016, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9018652439117432, + "kl": 0.06602321406826377, + "learning_rate": 2.9741661216481357e-07, + "loss": 0.05030661225318909, + "num_tokens": 26307033.0, + "reward": -1.496076620810527e-07, + "reward_std": 0.996212637424469, + "rewards/rnd_reward/mean": -1.4960765781779628e-07, + "rewards/rnd_reward/std": 0.9962126433849334, + "step": 1240, + "step_time": 9.930500056478195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13875000253319741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.1, + "completions/mean_length": 125.87625274658203, + "completions/mean_terminated_length": 84.06948699951172, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.7101810783147813, + "epoch": 0.8175277959450621, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9933876991271973, + "kl": 0.06315780784934759, + "learning_rate": 2.9578155657292345e-07, + "loss": 0.14283275604248047, + "num_tokens": 26506699.0, + "reward": -4.701316604904804e-08, + "reward_std": 0.996234393119812, + "rewards/rnd_reward/mean": -4.7013166404319404e-08, + "rewards/rnd_reward/std": 0.9962343990802764, + "step": 1250, + "step_time": 11.001938613550738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14875000342726707, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.5, + "completions/mean_length": 131.21625289916992, + "completions/mean_terminated_length": 87.06982650756837, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.695519116520882, + "epoch": 0.8240680183126227, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8424022793769836, + "kl": 0.06466643353924155, + "learning_rate": 2.9414650098103333e-07, + "loss": 0.08369569182395935, + "num_tokens": 26703352.0, + "reward": -2.6673080633088375e-08, + "reward_std": 0.9962631404399872, + "rewards/rnd_reward/mean": -2.6673077790917433e-08, + "rewards/rnd_reward/std": 0.9962631464004517, + "step": 1260, + "step_time": 9.866059662331827 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16375000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.6, + "completions/mean_length": 132.26625137329103, + "completions/mean_terminated_length": 83.00667343139648, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7214665055274962, + "epoch": 0.8306082406801831, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.061651349067688, + "kl": 0.060832435730844736, + "learning_rate": 2.9251144538914326e-07, + "loss": 0.011636599898338318, + "num_tokens": 26905195.0, + "reward": -1.28149986178272e-08, + "reward_std": 0.996177339553833, + "rewards/rnd_reward/mean": -1.2814998262555832e-08, + "rewards/rnd_reward/std": 0.9961773693561554, + "step": 1270, + "step_time": 10.226913845306262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1462500013411045, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.4, + "completions/mean_length": 135.36000289916993, + "completions/mean_terminated_length": 92.69313278198243, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5201783299446108, + "epoch": 0.8371484630477436, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9056648015975952, + "kl": 0.05058280224911869, + "learning_rate": 2.908763897972531e-07, + "loss": 0.08019465208053589, + "num_tokens": 27106543.0, + "reward": -1.9341707613307335e-07, + "reward_std": 0.9962323784828186, + "rewards/rnd_reward/mean": -1.9341706973818874e-07, + "rewards/rnd_reward/std": 0.9962323844432831, + "step": 1280, + "step_time": 10.01526041161269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1225000001490116, + "completions/max_length": 384.0, + "completions/max_terminated_length": 373.1, + "completions/mean_length": 130.23250198364258, + "completions/mean_terminated_length": 94.88236541748047, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.623024606704712, + "epoch": 0.8436886854153042, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.783699095249176, + "kl": 0.05370175880379975, + "learning_rate": 2.8924133420536296e-07, + "loss": 0.11414051055908203, + "num_tokens": 27305554.0, + "reward": -9.253621175986382e-08, + "reward_std": 0.9961644470691681, + "rewards/rnd_reward/mean": -9.253620731897172e-08, + "rewards/rnd_reward/std": 0.9961644232273101, + "step": 1290, + "step_time": 10.230407970934175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14875000193715096, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.6, + "completions/mean_length": 136.5575035095215, + "completions/mean_terminated_length": 93.4459114074707, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.676475775241852, + "epoch": 0.8502289077828646, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0168393850326538, + "kl": 0.052173000667244195, + "learning_rate": 2.8760627861347284e-07, + "loss": 0.049798119068145755, + "num_tokens": 27507145.0, + "reward": -1.576542894232702e-07, + "reward_std": 0.9962258338928223, + "rewards/rnd_reward/mean": -1.5765429013381292e-07, + "rewards/rnd_reward/std": 0.9962258338928223, + "step": 1300, + "step_time": 9.98130206335336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13500000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.1, + "completions/mean_length": 128.2087516784668, + "completions/mean_terminated_length": 88.18164749145508, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.640226128697395, + "epoch": 0.8567691301504251, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8835275769233704, + "kl": 0.05669898777268827, + "learning_rate": 2.859712230215827e-07, + "loss": 0.10237141847610473, + "num_tokens": 27707452.0, + "reward": -6.869435509315736e-08, + "reward_std": 0.9961890399456024, + "rewards/rnd_reward/mean": -6.869434940881546e-08, + "rewards/rnd_reward/std": 0.99618901014328, + "step": 1310, + "step_time": 10.37953436721582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13625000268220902, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.5, + "completions/mean_length": 128.80750274658203, + "completions/mean_terminated_length": 88.5141227722168, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.4644729048013687, + "epoch": 0.8633093525179856, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8116011619567871, + "kl": 0.05524417916312814, + "learning_rate": 2.843361674296926e-07, + "loss": 0.1121416449546814, + "num_tokens": 27900098.0, + "reward": -5.230307502657183e-08, + "reward_std": 0.9961800217628479, + "rewards/rnd_reward/mean": -5.230307715820004e-08, + "rewards/rnd_reward/std": 0.9961800217628479, + "step": 1320, + "step_time": 9.958397940779104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11625000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.3, + "completions/mean_length": 122.20125198364258, + "completions/mean_terminated_length": 87.82691497802735, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.619413223862648, + "epoch": 0.8698495748855462, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1175379753112793, + "kl": 0.06109289322048426, + "learning_rate": 2.827011118378025e-07, + "loss": 0.05299858450889587, + "num_tokens": 28083799.0, + "reward": -9.030103811369373e-08, + "reward_std": 0.9959750473499298, + "rewards/rnd_reward/mean": -9.030103793605804e-08, + "rewards/rnd_reward/std": 0.9959750413894654, + "step": 1330, + "step_time": 9.82245781556703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11625000312924386, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.4, + "completions/mean_length": 122.17500152587891, + "completions/mean_terminated_length": 87.70087432861328, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.900624880194664, + "epoch": 0.8763897972531066, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.942399263381958, + "kl": 0.0663422117009759, + "learning_rate": 2.8106605624591235e-07, + "loss": 0.07738288044929505, + "num_tokens": 28270204.0, + "reward": -1.4573336244438905e-07, + "reward_std": 0.9961058497428894, + "rewards/rnd_reward/mean": -1.4573335960221812e-07, + "rewards/rnd_reward/std": 0.9961058616638183, + "step": 1340, + "step_time": 9.953787335986272 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1312500022351742, + "completions/max_length": 384.0, + "completions/max_terminated_length": 370.4, + "completions/mean_length": 120.04500045776368, + "completions/mean_terminated_length": 80.18856277465821, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.87275829911232, + "epoch": 0.8829300196206671, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0761396884918213, + "kl": 0.07089657634496689, + "learning_rate": 2.794310006540222e-07, + "loss": 0.13924288749694824, + "num_tokens": 28461485.0, + "reward": 2.965331162840812e-08, + "reward_std": 0.9960719764232635, + "rewards/rnd_reward/mean": 2.965331162840812e-08, + "rewards/rnd_reward/std": 0.996071970462799, + "step": 1350, + "step_time": 10.60484689238947 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12875000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.8, + "completions/mean_length": 115.14375228881836, + "completions/mean_terminated_length": 75.8135986328125, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 3.0420612663030626, + "epoch": 0.8894702419882276, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1144483089447021, + "kl": 0.07124942755326628, + "learning_rate": 2.777959450621321e-07, + "loss": 0.08705209493637085, + "num_tokens": 28652205.0, + "reward": -1.2695789215477759e-07, + "reward_std": 0.9961645841598511, + "rewards/rnd_reward/mean": -1.2695789215477759e-07, + "rewards/rnd_reward/std": 0.99616459608078, + "step": 1360, + "step_time": 10.374728590669111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12250000275671483, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.6, + "completions/mean_length": 116.32750015258789, + "completions/mean_terminated_length": 78.6882926940918, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8743550151586534, + "epoch": 0.8960104643557881, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.038732886314392, + "kl": 0.07000278439372778, + "learning_rate": 2.76160889470242e-07, + "loss": 0.11522959470748902, + "num_tokens": 28836007.0, + "reward": -2.1174550361990897e-07, + "reward_std": 0.9960576713085174, + "rewards/rnd_reward/mean": -2.117455032646376e-07, + "rewards/rnd_reward/std": 0.9960576832294464, + "step": 1370, + "step_time": 10.044307312183083 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10750000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.4, + "completions/mean_length": 112.57625274658203, + "completions/mean_terminated_length": 79.90474357604981, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.814165809750557, + "epoch": 0.9025506867233486, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.853209912776947, + "kl": 0.07179404897615313, + "learning_rate": 2.7452583387835187e-07, + "loss": 0.11325899362564087, + "num_tokens": 29020798.0, + "reward": -4.172328260665381e-09, + "reward_std": 0.995877069234848, + "rewards/rnd_reward/mean": -4.172324707951702e-09, + "rewards/rnd_reward/std": 0.9958770573139191, + "step": 1380, + "step_time": 10.452752491505816 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09500000216066837, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.1, + "completions/mean_length": 97.97000122070312, + "completions/mean_terminated_length": 68.24239921569824, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.9521768242120743, + "epoch": 0.9090909090909091, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2329504489898682, + "kl": 0.09182704174891114, + "learning_rate": 2.7289077828646175e-07, + "loss": 0.09825693368911743, + "num_tokens": 29195334.0, + "reward": 9.715557460765468e-08, + "reward_std": 0.9959101438522339, + "rewards/rnd_reward/mean": 9.715556679168457e-08, + "rewards/rnd_reward/std": 0.9959101438522339, + "step": 1390, + "step_time": 10.322968539246357 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10125000234693289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.9, + "completions/mean_length": 103.2087516784668, + "completions/mean_terminated_length": 71.93345756530762, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 3.0182015657424928, + "epoch": 0.9156311314584696, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4850752353668213, + "kl": 0.09229131015017629, + "learning_rate": 2.7125572269457157e-07, + "loss": 0.07991544604301452, + "num_tokens": 29371486.0, + "reward": -1.5191734150654668e-07, + "reward_std": 0.995727401971817, + "rewards/rnd_reward/mean": -1.5191734008546122e-07, + "rewards/rnd_reward/std": 0.9957274079322815, + "step": 1400, + "step_time": 10.564200303750113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11875000409781933, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.6, + "completions/mean_length": 109.10750198364258, + "completions/mean_terminated_length": 72.09171180725097, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9613120049238204, + "epoch": 0.9221713538260301, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9964513778686523, + "kl": 0.0881183447316289, + "learning_rate": 2.696206671026815e-07, + "loss": 0.05779829025268555, + "num_tokens": 29547567.0, + "reward": -2.881884620276054e-07, + "reward_std": 0.9955705881118775, + "rewards/rnd_reward/mean": -2.881884591854345e-07, + "rewards/rnd_reward/std": 0.9955705881118775, + "step": 1410, + "step_time": 10.214018813963047 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000156462193, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.2, + "completions/mean_length": 102.86875228881836, + "completions/mean_terminated_length": 76.34417495727538, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.703101697564125, + "epoch": 0.9287115761935906, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8821516633033752, + "kl": 0.08714565662667155, + "learning_rate": 2.679856115107914e-07, + "loss": 0.017349694669246674, + "num_tokens": 29720227.0, + "reward": -8.821487531562867e-08, + "reward_std": 0.9958377420902252, + "rewards/rnd_reward/mean": -8.821487389454319e-08, + "rewards/rnd_reward/std": 0.9958377361297608, + "step": 1420, + "step_time": 10.117785238358191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000044703483, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.4, + "completions/mean_length": 115.6462516784668, + "completions/mean_terminated_length": 75.65214309692382, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.66856093108654, + "epoch": 0.935251798561151, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8306900858879089, + "kl": 0.07128150118514895, + "learning_rate": 2.663505559189012e-07, + "loss": 0.07572194337844848, + "num_tokens": 29907299.0, + "reward": -1.2844801631217705e-07, + "reward_std": 0.995860880613327, + "rewards/rnd_reward/mean": -1.2844801062783516e-07, + "rewards/rnd_reward/std": 0.9958608686923981, + "step": 1430, + "step_time": 10.222553351335227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12500000223517418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.7, + "completions/mean_length": 121.33125076293945, + "completions/mean_terminated_length": 83.87310562133788, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.7864604860544206, + "epoch": 0.9417920209287116, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.97917640209198, + "kl": 0.06842925064265729, + "learning_rate": 2.6471550032701114e-07, + "loss": 0.11840918064117431, + "num_tokens": 30100959.0, + "reward": -1.5377998749954713e-07, + "reward_std": 0.9958663046360016, + "rewards/rnd_reward/mean": -1.5377998110466252e-07, + "rewards/rnd_reward/std": 0.9958662986755371, + "step": 1440, + "step_time": 10.766110814665444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14125000238418578, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.3, + "completions/mean_length": 124.87875289916992, + "completions/mean_terminated_length": 82.31183624267578, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.699194425344467, + "epoch": 0.9483322432962721, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8881458640098572, + "kl": 0.06711579142138362, + "learning_rate": 2.6308044473512096e-07, + "loss": 0.10254174470901489, + "num_tokens": 30297562.0, + "reward": -6.161630330403512e-08, + "reward_std": 0.9957123577594758, + "rewards/rnd_reward/mean": -6.16162979749646e-08, + "rewards/rnd_reward/std": 0.9957123517990112, + "step": 1450, + "step_time": 10.593307360098697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1200000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.1, + "completions/mean_length": 116.06000213623047, + "completions/mean_terminated_length": 79.77308044433593, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.995149564743042, + "epoch": 0.9548724656638325, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0435981750488281, + "kl": 0.07414040835574269, + "learning_rate": 2.6144538914323084e-07, + "loss": 0.08301687836647034, + "num_tokens": 30483565.0, + "reward": -1.0631978923925089e-07, + "reward_std": 0.9958245873451232, + "rewards/rnd_reward/mean": -1.0631978462072311e-07, + "rewards/rnd_reward/std": 0.9958245933055878, + "step": 1460, + "step_time": 10.52764433112461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12750000283122062, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.3, + "completions/mean_length": 111.42625198364257, + "completions/mean_terminated_length": 71.7352523803711, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8058162331581116, + "epoch": 0.961412688031393, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9467556476593018, + "kl": 0.07855136888101696, + "learning_rate": 2.598103335513407e-07, + "loss": 0.10095235109329223, + "num_tokens": 30671516.0, + "reward": -1.529604193084566e-07, + "reward_std": 0.9958255529403687, + "rewards/rnd_reward/mean": -1.5296041966372798e-07, + "rewards/rnd_reward/std": 0.9958255529403687, + "step": 1470, + "step_time": 10.889759617857635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11125000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.2, + "completions/mean_length": 105.6962516784668, + "completions/mean_terminated_length": 70.84101905822754, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9369604349136353, + "epoch": 0.9679529103989536, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9438734650611877, + "kl": 0.0841084063053131, + "learning_rate": 2.581752779594506e-07, + "loss": 0.06790788173675537, + "num_tokens": 30852283.0, + "reward": -1.862645504502325e-08, + "reward_std": 0.9955705404281616, + "rewards/rnd_reward/mean": -1.8626451847580937e-08, + "rewards/rnd_reward/std": 0.9955705344676972, + "step": 1480, + "step_time": 10.544715784140863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11250000223517417, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.8, + "completions/mean_length": 111.51875228881836, + "completions/mean_terminated_length": 77.10519866943359, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.871400186419487, + "epoch": 0.9744931327665141, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0710206031799316, + "kl": 0.07257421473041177, + "learning_rate": 2.5654022236756053e-07, + "loss": 0.10279649496078491, + "num_tokens": 31039368.0, + "reward": -3.039836773410798e-08, + "reward_std": 0.9956327259540558, + "rewards/rnd_reward/mean": -3.039836915519345e-08, + "rewards/rnd_reward/std": 0.9956327199935913, + "step": 1490, + "step_time": 11.004371847352013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11750000305473804, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.9, + "completions/mean_length": 110.3587516784668, + "completions/mean_terminated_length": 74.05159416198731, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8570073008537293, + "epoch": 0.9810333551340745, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9674124717712402, + "kl": 0.07076722700148821, + "learning_rate": 2.5490516677567035e-07, + "loss": 0.09354864358901978, + "num_tokens": 31229300.0, + "reward": -7.458031374696361e-08, + "reward_std": 0.9957621216773986, + "rewards/rnd_reward/mean": -7.458031019424993e-08, + "rewards/rnd_reward/std": 0.9957621157169342, + "step": 1500, + "step_time": 10.672748254612088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09250000156462193, + "completions/max_length": 384.0, + "completions/max_terminated_length": 366.3, + "completions/mean_length": 109.77875213623047, + "completions/mean_terminated_length": 81.85236892700195, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8984388679265978, + "epoch": 0.9875735775016351, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.144245982170105, + "kl": 0.07264794735237956, + "learning_rate": 2.5327011118378023e-07, + "loss": 0.10363882780075073, + "num_tokens": 31410018.0, + "reward": 3.874301679474002e-09, + "reward_std": 0.9956199109554291, + "rewards/rnd_reward/mean": 3.874301679474002e-09, + "rewards/rnd_reward/std": 0.9956199288368225, + "step": 1510, + "step_time": 10.57373680251185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11250000223517417, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.4, + "completions/mean_length": 107.2912498474121, + "completions/mean_terminated_length": 71.87631187438964, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.835093083977699, + "epoch": 0.9941137998691956, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9730135798454285, + "kl": 0.06768459901213646, + "learning_rate": 2.516350555918901e-07, + "loss": 0.04406432807445526, + "num_tokens": 31584581.0, + "reward": -6.169080961626605e-08, + "reward_std": 0.9956278324127197, + "rewards/rnd_reward/mean": -6.169080748463785e-08, + "rewards/rnd_reward/std": 0.9956278324127197, + "step": 1520, + "step_time": 9.995071636443026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10000000149011612, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.7, + "completions/mean_length": 105.59500198364258, + "completions/mean_terminated_length": 74.4354133605957, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.818147099018097, + "epoch": 1.0006540222367561, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0657150745391846, + "kl": 0.0751381286419928, + "learning_rate": 2.5e-07, + "loss": 0.14666662216186524, + "num_tokens": 31762082.0, + "reward": -7.718801313671974e-08, + "reward_std": 0.9956592619419098, + "rewards/rnd_reward/mean": -7.718801242617701e-08, + "rewards/rnd_reward/std": 0.9956592619419098, + "step": 1530, + "step_time": 10.494197542965413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.0, + "completions/mean_length": 100.37250137329102, + "completions/mean_terminated_length": 73.99925155639649, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.850705623626709, + "epoch": 1.0071942446043165, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8915496468544006, + "kl": 0.08450272670015693, + "learning_rate": 2.4836494440810987e-07, + "loss": 0.10265235900878907, + "num_tokens": 31935920.0, + "reward": -7.78958216152148e-08, + "reward_std": 0.9955736041069031, + "rewards/rnd_reward/mean": -7.789581664141565e-08, + "rewards/rnd_reward/std": 0.995573616027832, + "step": 1540, + "step_time": 10.282078485726379 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09625000357627869, + "completions/max_length": 384.0, + "completions/max_terminated_length": 339.5, + "completions/mean_length": 100.54375, + "completions/mean_terminated_length": 70.20137939453124, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7832202523946763, + "epoch": 1.013734466971877, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9670330882072449, + "kl": 0.07673759749159217, + "learning_rate": 2.4672988881621975e-07, + "loss": 0.053245991468429565, + "num_tokens": 32105575.0, + "reward": -7.98702268323126e-08, + "reward_std": 0.9954982161521911, + "rewards/rnd_reward/mean": -7.987022256905619e-08, + "rewards/rnd_reward/std": 0.9954982340335846, + "step": 1550, + "step_time": 10.094654931733384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09625000134110451, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.6, + "completions/mean_length": 105.8862518310547, + "completions/mean_terminated_length": 76.32862396240235, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.806140884757042, + "epoch": 1.0202746893394374, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9716780185699463, + "kl": 0.07651977436617016, + "learning_rate": 2.450948332243296e-07, + "loss": 0.07588104605674743, + "num_tokens": 32282264.0, + "reward": -1.2878328945475914e-07, + "reward_std": 0.9953814685344696, + "rewards/rnd_reward/mean": -1.2878328448096e-07, + "rewards/rnd_reward/std": 0.9953814685344696, + "step": 1560, + "step_time": 10.249592515034601 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09250000044703484, + "completions/max_length": 384.0, + "completions/max_terminated_length": 370.2, + "completions/mean_length": 101.42500152587891, + "completions/mean_terminated_length": 72.58811225891114, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.706285873055458, + "epoch": 1.026814911706998, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2053457498550415, + "kl": 0.07990584680810571, + "learning_rate": 2.434597776324395e-07, + "loss": 0.08478202223777771, + "num_tokens": 32458489.0, + "reward": -1.197308309830447e-07, + "reward_std": 0.9958563566207885, + "rewards/rnd_reward/mean": -1.1973082814087377e-07, + "rewards/rnd_reward/std": 0.9958563566207885, + "step": 1570, + "step_time": 10.385891513410025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000223517418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.3, + "completions/mean_length": 92.8375015258789, + "completions/mean_terminated_length": 65.02403182983399, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.832634910941124, + "epoch": 1.0333551340745586, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2888636589050293, + "kl": 0.08601820273324848, + "learning_rate": 2.418247220405494e-07, + "loss": 0.024559563398361205, + "num_tokens": 32626059.0, + "reward": -1.628696985278566e-07, + "reward_std": 0.9954473912715912, + "rewards/rnd_reward/mean": -1.6286969000134378e-07, + "rewards/rnd_reward/std": 0.9954474031925201, + "step": 1580, + "step_time": 10.754275933234021 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09125000201165676, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.7, + "completions/mean_length": 99.23750076293945, + "completions/mean_terminated_length": 70.4692756652832, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.9075894236564634, + "epoch": 1.039895356442119, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0340582132339478, + "kl": 0.0824001532047987, + "learning_rate": 2.4018966644865926e-07, + "loss": 0.04327432513236999, + "num_tokens": 32797839.0, + "reward": 6.616115513224941e-08, + "reward_std": 0.9953638434410095, + "rewards/rnd_reward/mean": 6.61611530006212e-08, + "rewards/rnd_reward/std": 0.9953638732433319, + "step": 1590, + "step_time": 10.39529764095787 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1075000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.2, + "completions/mean_length": 105.15875091552735, + "completions/mean_terminated_length": 71.44011192321777, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 3.068614384531975, + "epoch": 1.0464355788096795, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9280741810798645, + "kl": 0.08188259806483984, + "learning_rate": 2.3855461085676914e-07, + "loss": 0.07503924369812012, + "num_tokens": 32970486.0, + "reward": 2.354383532576776e-08, + "reward_std": 0.9955627143383026, + "rewards/rnd_reward/mean": 2.3543834259953656e-08, + "rewards/rnd_reward/std": 0.9955627143383026, + "step": 1600, + "step_time": 10.102484514610842 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12375000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.0, + "completions/mean_length": 108.67875137329102, + "completions/mean_terminated_length": 69.76623840332032, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.7116469889879227, + "epoch": 1.05297580117724, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8361508846282959, + "kl": 0.07135752588510513, + "learning_rate": 2.36919555264879e-07, + "loss": 0.08789730072021484, + "num_tokens": 33155644.0, + "reward": -7.331371434382561e-08, + "reward_std": 0.9953298509120941, + "rewards/rnd_reward/mean": -7.331371363328287e-08, + "rewards/rnd_reward/std": 0.9953298389911651, + "step": 1610, + "step_time": 10.373006676253862 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10250000208616257, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.6, + "completions/mean_length": 107.08625106811523, + "completions/mean_terminated_length": 75.19281349182128, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.767005005478859, + "epoch": 1.0595160235448005, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8207447528839111, + "kl": 0.07195558175444602, + "learning_rate": 2.352844996729889e-07, + "loss": 0.07792651653289795, + "num_tokens": 33333863.0, + "reward": -7.919967437430842e-08, + "reward_std": 0.9952020287513733, + "rewards/rnd_reward/mean": -7.919966940050927e-08, + "rewards/rnd_reward/std": 0.9952020525932312, + "step": 1620, + "step_time": 10.309986177436077 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12875000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.7, + "completions/mean_length": 117.29250259399414, + "completions/mean_terminated_length": 78.07871360778809, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8041924834251404, + "epoch": 1.066056245912361, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7970576882362366, + "kl": 0.07171947872266174, + "learning_rate": 2.3364944408109875e-07, + "loss": 0.07321782112121582, + "num_tokens": 33529112.0, + "reward": -7.54743808784042e-08, + "reward_std": 0.9954242706298828, + "rewards/rnd_reward/mean": -7.54743808784042e-08, + "rewards/rnd_reward/std": 0.9954242765903473, + "step": 1630, + "step_time": 11.020952883246355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10375000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.8, + "completions/mean_length": 109.85750122070313, + "completions/mean_terminated_length": 78.08732643127442, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7948483556509016, + "epoch": 1.0725964682799216, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0839210748672485, + "kl": 0.07083125235512852, + "learning_rate": 2.3201438848920862e-07, + "loss": 0.07660450935363769, + "num_tokens": 33711328.0, + "reward": -4.068016963287846e-08, + "reward_std": 0.9954971015453339, + "rewards/rnd_reward/mean": -4.068016963287846e-08, + "rewards/rnd_reward/std": 0.9954971194267273, + "step": 1640, + "step_time": 10.339221865893341 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09125000033527612, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.3, + "completions/mean_length": 100.59000167846679, + "completions/mean_terminated_length": 72.11472930908204, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7371392965316774, + "epoch": 1.079136690647482, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9994949698448181, + "kl": 0.08674983121454716, + "learning_rate": 2.303793328973185e-07, + "loss": 0.14760398864746094, + "num_tokens": 33894090.0, + "reward": -4.775822315394862e-08, + "reward_std": 0.9951541960239411, + "rewards/rnd_reward/mean": -4.7758222088134514e-08, + "rewards/rnd_reward/std": 0.9951542019844055, + "step": 1650, + "step_time": 11.004630160145462 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09750000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.6, + "completions/mean_length": 103.39125213623046, + "completions/mean_terminated_length": 73.03910675048829, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.719507083296776, + "epoch": 1.0856769130150425, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0636616945266724, + "kl": 0.07622466497123241, + "learning_rate": 2.2874427730542835e-07, + "loss": 0.11726703643798828, + "num_tokens": 34065638.0, + "reward": -6.273389079325397e-08, + "reward_std": 0.9954430878162384, + "rewards/rnd_reward/mean": -6.273388937216851e-08, + "rewards/rnd_reward/std": 0.9954430878162384, + "step": 1660, + "step_time": 10.205868328805082 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11125000230967999, + "completions/max_length": 384.0, + "completions/max_terminated_length": 337.9, + "completions/mean_length": 103.16000251770019, + "completions/mean_terminated_length": 68.0124439239502, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.555436444282532, + "epoch": 1.0922171353826031, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4149969816207886, + "kl": 0.08045686418190598, + "learning_rate": 2.2710922171353826e-07, + "loss": 0.18388016223907472, + "num_tokens": 34243911.0, + "reward": -6.049871625890546e-08, + "reward_std": 0.995277750492096, + "rewards/rnd_reward/mean": -6.049871181801336e-08, + "rewards/rnd_reward/std": 0.9952777206897736, + "step": 1670, + "step_time": 10.405935624614358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09250000230967999, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.5, + "completions/mean_length": 98.8687515258789, + "completions/mean_terminated_length": 69.8191291809082, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.679945033788681, + "epoch": 1.0987573577501635, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1418001651763916, + "kl": 0.07972979731857777, + "learning_rate": 2.2547416612164814e-07, + "loss": 0.08271819949150086, + "num_tokens": 34422086.0, + "reward": -2.2441148974650105e-07, + "reward_std": 0.9950048804283143, + "rewards/rnd_reward/mean": -2.2441148654905875e-07, + "rewards/rnd_reward/std": 0.9950048506259919, + "step": 1680, + "step_time": 10.670063342596404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000141561032, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.9, + "completions/mean_length": 95.35000228881836, + "completions/mean_terminated_length": 67.31705818176269, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6938273787498472, + "epoch": 1.105297580117724, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0555897951126099, + "kl": 0.0881148618645966, + "learning_rate": 2.2383911052975802e-07, + "loss": 0.057183104753494265, + "num_tokens": 34592661.0, + "reward": -5.260109574578564e-08, + "reward_std": 0.9953494489192962, + "rewards/rnd_reward/mean": -5.260109787741385e-08, + "rewards/rnd_reward/std": 0.9953494608402252, + "step": 1690, + "step_time": 10.320013692067004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09500000197440386, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 96.38625259399414, + "completions/mean_terminated_length": 66.25210113525391, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8528232872486115, + "epoch": 1.1118378024852844, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1842913627624512, + "kl": 0.09175272611901164, + "learning_rate": 2.2220405493786787e-07, + "loss": 0.11493265628814697, + "num_tokens": 34767800.0, + "reward": -3.5762789174498266e-08, + "reward_std": 0.9952230334281922, + "rewards/rnd_reward/mean": -3.5762784911241854e-08, + "rewards/rnd_reward/std": 0.9952230513095855, + "step": 1700, + "step_time": 11.20805806266144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06625000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.7, + "completions/mean_length": 88.85, + "completions/mean_terminated_length": 67.81800193786621, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7997246354818346, + "epoch": 1.118378024852845, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3541666269302368, + "kl": 0.0883994361385703, + "learning_rate": 2.2056899934597775e-07, + "loss": 0.10645198822021484, + "num_tokens": 34940290.0, + "reward": -9.074807572062582e-08, + "reward_std": 0.9952512562274933, + "rewards/rnd_reward/mean": -9.074807181264077e-08, + "rewards/rnd_reward/std": 0.9952512562274933, + "step": 1710, + "step_time": 11.268909463495948 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08375000078231096, + "completions/max_length": 384.0, + "completions/max_terminated_length": 341.1, + "completions/mean_length": 92.34500198364258, + "completions/mean_terminated_length": 65.63225555419922, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5688311845064162, + "epoch": 1.1249182472204056, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.086549997329712, + "kl": 0.08725747494027018, + "learning_rate": 2.1893394375408762e-07, + "loss": 0.09306548833847046, + "num_tokens": 35107316.0, + "reward": -1.1682510461241691e-07, + "reward_std": 0.9951877236366272, + "rewards/rnd_reward/mean": -1.1682510461241691e-07, + "rewards/rnd_reward/std": 0.9951877236366272, + "step": 1720, + "step_time": 10.925348741887138 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000104308128, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.1, + "completions/mean_length": 97.91875076293945, + "completions/mean_terminated_length": 70.1454303741455, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.690840741991997, + "epoch": 1.131458469587966, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0729917287826538, + "kl": 0.08789877109229564, + "learning_rate": 2.172988881621975e-07, + "loss": 0.10979057550430298, + "num_tokens": 35281881.0, + "reward": -1.907348732288483e-08, + "reward_std": 0.995078319311142, + "rewards/rnd_reward/mean": -1.907348732288483e-08, + "rewards/rnd_reward/std": 0.9950783312320709, + "step": 1730, + "step_time": 10.632008279883303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000223517418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.8, + "completions/mean_length": 96.91125183105468, + "completions/mean_terminated_length": 69.48632164001465, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.681867742538452, + "epoch": 1.1379986919555265, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0131276845932007, + "kl": 0.08944809632375836, + "learning_rate": 2.1566383257030738e-07, + "loss": 0.10518434047698974, + "num_tokens": 35461970.0, + "reward": 2.6673077968553116e-08, + "reward_std": 0.9949174404144288, + "rewards/rnd_reward/mean": 2.6673081521266795e-08, + "rewards/rnd_reward/std": 0.9949174463748932, + "step": 1740, + "step_time": 11.253412028984167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0787500012665987, + "completions/max_length": 384.0, + "completions/max_terminated_length": 338.7, + "completions/mean_length": 84.14750061035156, + "completions/mean_terminated_length": 58.63842086791992, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.567233794927597, + "epoch": 1.144538914323087, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1535600423812866, + "kl": 0.09054145393893123, + "learning_rate": 2.1402877697841726e-07, + "loss": 0.10667283535003662, + "num_tokens": 35622023.0, + "reward": -5.97536580215774e-08, + "reward_std": 0.9952092111110687, + "rewards/rnd_reward/mean": -5.975365517940645e-08, + "rewards/rnd_reward/std": 0.9952091991901397, + "step": 1750, + "step_time": 10.509924068162217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07000000029802322, + "completions/max_length": 384.0, + "completions/max_terminated_length": 331.8, + "completions/mean_length": 86.03000106811524, + "completions/mean_terminated_length": 63.5996639251709, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.5665056079626085, + "epoch": 1.1510791366906474, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1171236038208008, + "kl": 0.08619214734062552, + "learning_rate": 2.1239372138652714e-07, + "loss": 0.07680293321609497, + "num_tokens": 35780972.0, + "reward": -4.529952919440916e-08, + "reward_std": 0.9950806498527527, + "rewards/rnd_reward/mean": -4.529952919440916e-08, + "rewards/rnd_reward/std": 0.9950806617736816, + "step": 1760, + "step_time": 10.277455605636352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06875000074505806, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.5, + "completions/mean_length": 89.81625213623047, + "completions/mean_terminated_length": 68.19218559265137, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.563973236083984, + "epoch": 1.157619359058208, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1521550416946411, + "kl": 0.0870651968754828, + "learning_rate": 2.10758665794637e-07, + "loss": 0.1115831971168518, + "num_tokens": 35948090.0, + "reward": 1.3858081859829951e-08, + "reward_std": 0.9948914706707, + "rewards/rnd_reward/mean": 1.38580813269229e-08, + "rewards/rnd_reward/std": 0.9948914825916291, + "step": 1770, + "step_time": 10.7476754687028 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07750000115483999, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.2, + "completions/mean_length": 85.87875022888184, + "completions/mean_terminated_length": 60.88022689819336, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.646288427710533, + "epoch": 1.1641595814257686, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9658340811729431, + "kl": 0.09920121757313609, + "learning_rate": 2.091236102027469e-07, + "loss": 0.08330439329147339, + "num_tokens": 36114108.0, + "reward": -7.227063285597523e-08, + "reward_std": 0.9949531733989716, + "rewards/rnd_reward/mean": -7.227063250070387e-08, + "rewards/rnd_reward/std": 0.994953167438507, + "step": 1780, + "step_time": 11.134026414644904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06250000018626452, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.1, + "completions/mean_length": 84.51375122070313, + "completions/mean_terminated_length": 64.53133659362793, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.704926884174347, + "epoch": 1.170699803793329, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2755341529846191, + "kl": 0.10716334227472543, + "learning_rate": 2.0748855461085677e-07, + "loss": 0.13019881248474122, + "num_tokens": 36271504.0, + "reward": -1.305341750246214e-07, + "reward_std": 0.9949664771556854, + "rewards/rnd_reward/mean": -1.305341714719077e-07, + "rewards/rnd_reward/std": 0.9949664771556854, + "step": 1790, + "step_time": 10.397847580048255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000175088644, + "completions/max_length": 384.0, + "completions/max_terminated_length": 326.3, + "completions/mean_length": 87.63249969482422, + "completions/mean_terminated_length": 59.75216636657715, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.593474331498146, + "epoch": 1.1772400261608895, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.155380129814148, + "kl": 0.09049692396074534, + "learning_rate": 2.0585349901896665e-07, + "loss": 0.12421066761016845, + "num_tokens": 36439860.0, + "reward": 1.50501714202278e-08, + "reward_std": 0.9950001060962677, + "rewards/rnd_reward/mean": 1.505017284131327e-08, + "rewards/rnd_reward/std": 0.9950001001358032, + "step": 1800, + "step_time": 10.84767303718254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06250000158324838, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.8, + "completions/mean_length": 91.63375244140624, + "completions/mean_terminated_length": 72.32135162353515, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5027853429317473, + "epoch": 1.1837802485284499, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0669174194335938, + "kl": 0.08057954236865043, + "learning_rate": 2.042184434270765e-07, + "loss": 0.09477431774139404, + "num_tokens": 36602592.0, + "reward": -1.505017284131327e-07, + "reward_std": 0.994869589805603, + "rewards/rnd_reward/mean": -1.5050172876840407e-07, + "rewards/rnd_reward/std": 0.9948696017265319, + "step": 1810, + "step_time": 10.333302733465098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000216066838, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.4, + "completions/mean_length": 92.49750137329102, + "completions/mean_terminated_length": 68.43786659240723, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.487256759405136, + "epoch": 1.1903204708960105, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.864107608795166, + "kl": 0.07850290769711137, + "learning_rate": 2.0258338783518638e-07, + "loss": 0.09753584861755371, + "num_tokens": 36768390.0, + "reward": -1.2189149920516229e-07, + "reward_std": 0.9947888195514679, + "rewards/rnd_reward/mean": -1.2189150062624775e-07, + "rewards/rnd_reward/std": 0.9947888076305389, + "step": 1820, + "step_time": 10.366325606242754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07500000111758709, + "completions/max_length": 384.0, + "completions/max_terminated_length": 327.2, + "completions/mean_length": 87.97500152587891, + "completions/mean_terminated_length": 64.1132999420166, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5121534377336503, + "epoch": 1.196860693263571, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8458340167999268, + "kl": 0.08256649980321526, + "learning_rate": 2.0094833224329626e-07, + "loss": 0.06835322976112365, + "num_tokens": 36933960.0, + "reward": -1.531839394885992e-07, + "reward_std": 0.9951117813587189, + "rewards/rnd_reward/mean": -1.5318393664642826e-07, + "rewards/rnd_reward/std": 0.9951117634773254, + "step": 1830, + "step_time": 11.365050849993713 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07375000137835741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.0, + "completions/mean_length": 94.00125045776367, + "completions/mean_terminated_length": 71.0650505065918, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4178095728158953, + "epoch": 1.2034009156311314, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1388354301452637, + "kl": 0.0803013733588159, + "learning_rate": 1.9931327665140614e-07, + "loss": 0.12810555696487427, + "num_tokens": 37099636.0, + "reward": -4.909932687091611e-08, + "reward_std": 0.9949400007724762, + "rewards/rnd_reward/mean": -4.909932758145885e-08, + "rewards/rnd_reward/std": 0.9949400067329407, + "step": 1840, + "step_time": 10.505639822757804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06875000093132258, + "completions/max_length": 384.0, + "completions/max_terminated_length": 341.5, + "completions/mean_length": 91.8550018310547, + "completions/mean_terminated_length": 70.10141868591309, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.4157417356967925, + "epoch": 1.209941137998692, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0894784927368164, + "kl": 0.08061180328950286, + "learning_rate": 1.9767822105951601e-07, + "loss": 0.09716765880584717, + "num_tokens": 37269225.0, + "reward": -4.3064359189770585e-08, + "reward_std": 0.9946663022041321, + "rewards/rnd_reward/mean": -4.306435528178554e-08, + "rewards/rnd_reward/std": 0.9946663022041321, + "step": 1850, + "step_time": 11.422960984380916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000074505807, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.2, + "completions/mean_length": 93.6987518310547, + "completions/mean_terminated_length": 65.78671073913574, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5770903497934343, + "epoch": 1.2164813603662525, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9160525798797607, + "kl": 0.08047085385769606, + "learning_rate": 1.960431654676259e-07, + "loss": 0.05968161225318909, + "num_tokens": 37440344.0, + "reward": 2.6524067830990816e-08, + "reward_std": 0.9945189356803894, + "rewards/rnd_reward/mean": 2.65240635677344e-08, + "rewards/rnd_reward/std": 0.9945189356803894, + "step": 1860, + "step_time": 10.725451961229556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000063329935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 335.1, + "completions/mean_length": 95.60625076293945, + "completions/mean_terminated_length": 68.25989952087403, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.4541951596736906, + "epoch": 1.223021582733813, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.999136209487915, + "kl": 0.07508358927443623, + "learning_rate": 1.9440810987573577e-07, + "loss": 0.13460888862609863, + "num_tokens": 37611004.0, + "reward": -1.23679642705099e-07, + "reward_std": 0.9949141681194306, + "rewards/rnd_reward/mean": -1.236796327575007e-07, + "rewards/rnd_reward/std": 0.9949141621589661, + "step": 1870, + "step_time": 10.482190745323896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 328.5, + "completions/mean_length": 89.38125152587891, + "completions/mean_terminated_length": 65.09402084350586, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.567149502038956, + "epoch": 1.2295618051013735, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2019909620285034, + "kl": 0.08660045554861426, + "learning_rate": 1.9277305428384562e-07, + "loss": 0.20026679039001466, + "num_tokens": 37786329.0, + "reward": -8.866190821521514e-08, + "reward_std": 0.9948227822780609, + "rewards/rnd_reward/mean": -8.866190999157198e-08, + "rewards/rnd_reward/std": 0.9948227941989899, + "step": 1880, + "step_time": 11.85091487034224 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06375000011175871, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.9, + "completions/mean_length": 89.55750122070313, + "completions/mean_terminated_length": 69.49982872009278, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8322397589683534, + "epoch": 1.236102027468934, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1517750024795532, + "kl": 0.09399051470682025, + "learning_rate": 1.9113799869195553e-07, + "loss": 0.17799322605133056, + "num_tokens": 37947720.0, + "reward": -1.0848045732814171e-07, + "reward_std": 0.9948072552680969, + "rewards/rnd_reward/mean": -1.0848045448597076e-07, + "rewards/rnd_reward/std": 0.9948072493076324, + "step": 1890, + "step_time": 10.414563778741286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06625000014901161, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.4, + "completions/mean_length": 86.39500045776367, + "completions/mean_terminated_length": 65.30856475830078, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7242774665355682, + "epoch": 1.2426422498364944, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3484736680984497, + "kl": 0.09410189539194107, + "learning_rate": 1.895029431000654e-07, + "loss": 0.0743591845035553, + "num_tokens": 38111876.0, + "reward": -6.467104114449285e-08, + "reward_std": 0.9947612941265106, + "rewards/rnd_reward/mean": -6.467103901286464e-08, + "rewards/rnd_reward/std": 0.9947613418102265, + "step": 1900, + "step_time": 10.527994227781893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06625000108033419, + "completions/max_length": 384.0, + "completions/max_terminated_length": 338.0, + "completions/mean_length": 84.77500114440917, + "completions/mean_terminated_length": 63.568695068359375, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.954171320796013, + "epoch": 1.249182472204055, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9760697484016418, + "kl": 0.10021719867363572, + "learning_rate": 1.8786788750817528e-07, + "loss": 0.02971080243587494, + "num_tokens": 38279376.0, + "reward": -1.5616416861874428e-07, + "reward_std": 0.9947503983974457, + "rewards/rnd_reward/mean": -1.5616416790820153e-07, + "rewards/rnd_reward/std": 0.9947503864765167, + "step": 1910, + "step_time": 11.339212405891157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08000000193715096, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.5, + "completions/mean_length": 88.47500076293946, + "completions/mean_terminated_length": 62.80438385009766, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.846560522913933, + "epoch": 1.2557226945716153, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2370659112930298, + "kl": 0.09515976402908563, + "learning_rate": 1.8623283191628514e-07, + "loss": 0.10529450178146363, + "num_tokens": 38449416.0, + "reward": -3.65078278719011e-09, + "reward_std": 0.9944445490837097, + "rewards/rnd_reward/mean": -3.6507834977328456e-09, + "rewards/rnd_reward/std": 0.9944445431232453, + "step": 1920, + "step_time": 10.698268468677998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000160187483, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.8, + "completions/mean_length": 87.2925018310547, + "completions/mean_terminated_length": 62.797546768188475, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.9005165725946425, + "epoch": 1.262262916939176, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3234843015670776, + "kl": 0.10119491461664439, + "learning_rate": 1.8459777632439501e-07, + "loss": 0.014341549575328827, + "num_tokens": 38616700.0, + "reward": -4.380941049930698e-08, + "reward_std": 0.9944035768508911, + "rewards/rnd_reward/mean": -4.3809414407292026e-08, + "rewards/rnd_reward/std": 0.994403600692749, + "step": 1930, + "step_time": 11.375351972319185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000111758709, + "completions/max_length": 384.0, + "completions/max_terminated_length": 334.7, + "completions/mean_length": 96.3550018310547, + "completions/mean_terminated_length": 68.85774459838868, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.9528768658638, + "epoch": 1.2688031393067365, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1305567026138306, + "kl": 0.09469359386712313, + "learning_rate": 1.829627207325049e-07, + "loss": 0.0362447589635849, + "num_tokens": 38788569.0, + "reward": -5.45382517813664e-08, + "reward_std": 0.9948436379432678, + "rewards/rnd_reward/mean": -5.4538249649738193e-08, + "rewards/rnd_reward/std": 0.9948436439037323, + "step": 1940, + "step_time": 10.781422062939964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10000000298023223, + "completions/max_length": 384.0, + "completions/max_terminated_length": 342.1, + "completions/mean_length": 93.3875015258789, + "completions/mean_terminated_length": 61.06837921142578, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7504739254713058, + "epoch": 1.2753433616742968, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.153893232345581, + "kl": 0.08643652414903044, + "learning_rate": 1.8132766514061477e-07, + "loss": 0.10290535688400268, + "num_tokens": 38954489.0, + "reward": -1.2144446595030445e-07, + "reward_std": 0.9947971343994141, + "rewards/rnd_reward/mean": -1.2144446346340487e-07, + "rewards/rnd_reward/std": 0.994797146320343, + "step": 1950, + "step_time": 10.527900594798847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0950000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 329.3, + "completions/mean_length": 100.58875122070313, + "completions/mean_terminated_length": 70.79409141540528, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.7141620010137557, + "epoch": 1.2818835840418574, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0054807662963867, + "kl": 0.08213893175125123, + "learning_rate": 1.7969260954872465e-07, + "loss": 0.07241884469985962, + "num_tokens": 39126695.0, + "reward": -1.028180136586343e-08, + "reward_std": 0.9948840856552124, + "rewards/rnd_reward/mean": -1.0281802786948902e-08, + "rewards/rnd_reward/std": 0.9948840975761414, + "step": 1960, + "step_time": 10.461326698656194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06750000081956387, + "completions/max_length": 383.2, + "completions/max_terminated_length": 350.0, + "completions/mean_length": 88.08000106811524, + "completions/mean_terminated_length": 66.5671230316162, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.7966362595558167, + "epoch": 1.2884238064094178, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1359318494796753, + "kl": 0.09009936284273863, + "learning_rate": 1.7805755395683453e-07, + "loss": 0.12549155950546265, + "num_tokens": 39294724.0, + "reward": 4.082918199088681e-08, + "reward_std": 0.9944938480854034, + "rewards/rnd_reward/mean": 4.0829182701429545e-08, + "rewards/rnd_reward/std": 0.9944938659667969, + "step": 1970, + "step_time": 10.983743568998761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.4, + "completions/mean_length": 95.8025016784668, + "completions/mean_terminated_length": 71.99091606140136, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7956566989421843, + "epoch": 1.2949640287769784, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3064550161361694, + "kl": 0.08204388581216335, + "learning_rate": 1.764224983649444e-07, + "loss": 0.06261605024337769, + "num_tokens": 39465981.0, + "reward": -7.681548792604076e-08, + "reward_std": 0.9945617258548737, + "rewards/rnd_reward/mean": -7.681548792604076e-08, + "rewards/rnd_reward/std": 0.9945617258548737, + "step": 1980, + "step_time": 10.856887764390558 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10500000119209289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 336.7, + "completions/mean_length": 99.39125137329101, + "completions/mean_terminated_length": 66.14606056213378, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.809012603759766, + "epoch": 1.301504251144539, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0052663087844849, + "kl": 0.08238375652581453, + "learning_rate": 1.7478744277305426e-07, + "loss": 0.04824514389038086, + "num_tokens": 39642064.0, + "reward": -2.3305416743113483e-07, + "reward_std": 0.9944523334503174, + "rewards/rnd_reward/mean": -2.3305416174679294e-07, + "rewards/rnd_reward/std": 0.9944523453712464, + "step": 1990, + "step_time": 10.668248945730738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06875000093132258, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.9, + "completions/mean_length": 94.09000167846679, + "completions/mean_terminated_length": 72.67349357604981, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.782809841632843, + "epoch": 1.3080444735120995, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.088478684425354, + "kl": 0.08519633868709206, + "learning_rate": 1.7315238718116416e-07, + "loss": 0.12635375261306764, + "num_tokens": 39807401.0, + "reward": -9.67830406706227e-08, + "reward_std": 0.9944651544094085, + "rewards/rnd_reward/mean": -9.67830406706227e-08, + "rewards/rnd_reward/std": 0.9944651544094085, + "step": 2000, + "step_time": 10.477193391043693 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08125000074505806, + "completions/max_length": 384.0, + "completions/max_terminated_length": 333.7, + "completions/mean_length": 97.57875061035156, + "completions/mean_terminated_length": 72.30035781860352, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.851186656951904, + "epoch": 1.3145846958796599, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0262951850891113, + "kl": 0.08686133781448006, + "learning_rate": 1.7151733158927404e-07, + "loss": 0.16627882719039916, + "num_tokens": 39979409.0, + "reward": -1.8477440306696734e-07, + "reward_std": 0.9941576004028321, + "rewards/rnd_reward/mean": -1.8477439738262548e-07, + "rewards/rnd_reward/std": 0.994157612323761, + "step": 2010, + "step_time": 10.494085661577992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07875000070780516, + "completions/max_length": 384.0, + "completions/max_terminated_length": 332.3, + "completions/mean_length": 98.56625061035156, + "completions/mean_terminated_length": 74.01468200683594, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.6910068154335023, + "epoch": 1.3211249182472204, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.180172085762024, + "kl": 0.08328518448397518, + "learning_rate": 1.6988227599738392e-07, + "loss": 0.1285926580429077, + "num_tokens": 40151192.0, + "reward": -2.4214387792653726e-07, + "reward_std": 0.9945690989494324, + "rewards/rnd_reward/mean": -2.421438694000244e-07, + "rewards/rnd_reward/std": 0.9945691049098968, + "step": 2020, + "step_time": 10.840997534058989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06375000085681677, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.1, + "completions/mean_length": 86.02375068664551, + "completions/mean_terminated_length": 65.85984230041504, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.954036074876785, + "epoch": 1.3276651406147808, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1177207231521606, + "kl": 0.09944681767374278, + "learning_rate": 1.6824722040549377e-07, + "loss": 0.07238306999206542, + "num_tokens": 40313136.0, + "reward": -2.1941960497429137e-07, + "reward_std": 0.9945330858230591, + "rewards/rnd_reward/mean": -2.1941959928994948e-07, + "rewards/rnd_reward/std": 0.9945330798625946, + "step": 2030, + "step_time": 10.803823693329468 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11000000201165676, + "completions/max_length": 384.0, + "completions/max_terminated_length": 326.8, + "completions/mean_length": 100.16625137329102, + "completions/mean_terminated_length": 64.80138359069824, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 3.0097627967596052, + "epoch": 1.3342053629823414, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1276546716690063, + "kl": 0.08885867483913898, + "learning_rate": 1.6661216481360365e-07, + "loss": 0.010317666083574295, + "num_tokens": 40489459.0, + "reward": -1.4454126961993552e-07, + "reward_std": 0.9944145083427429, + "rewards/rnd_reward/mean": -1.4454125896179448e-07, + "rewards/rnd_reward/std": 0.994414496421814, + "step": 2040, + "step_time": 11.046867836569437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09500000067055225, + "completions/max_length": 384.0, + "completions/max_terminated_length": 329.5, + "completions/mean_length": 96.87375183105469, + "completions/mean_terminated_length": 66.7010440826416, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.999319186806679, + "epoch": 1.340745585349902, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0046088695526123, + "kl": 0.09102731477469206, + "learning_rate": 1.6497710922171353e-07, + "loss": 0.0822763979434967, + "num_tokens": 40655648.0, + "reward": -1.0967255033023094e-07, + "reward_std": 0.9943422853946686, + "rewards/rnd_reward/mean": -1.0967254464588904e-07, + "rewards/rnd_reward/std": 0.9943422734737396, + "step": 2050, + "step_time": 10.422167311538942 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0912500012665987, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.1, + "completions/mean_length": 97.65625228881837, + "completions/mean_terminated_length": 68.92426071166992, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8394038140773774, + "epoch": 1.3472858077174623, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9094915986061096, + "kl": 0.08663466572761536, + "learning_rate": 1.633420536298234e-07, + "loss": 0.020597906410694124, + "num_tokens": 40822473.0, + "reward": 1.2516974123855107e-08, + "reward_std": 0.9942900657653808, + "rewards/rnd_reward/mean": 1.251697430149079e-08, + "rewards/rnd_reward/std": 0.9942900657653808, + "step": 2060, + "step_time": 10.357499032304622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09500000141561031, + "completions/max_length": 384.0, + "completions/max_terminated_length": 323.5, + "completions/mean_length": 97.2625015258789, + "completions/mean_terminated_length": 67.25451889038087, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.869212657213211, + "epoch": 1.353826030085023, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9482806921005249, + "kl": 0.08622422777116298, + "learning_rate": 1.6170699803793328e-07, + "loss": 0.05238085389137268, + "num_tokens": 40987073.0, + "reward": -1.0728836219442428e-07, + "reward_std": 0.9939788460731507, + "rewards/rnd_reward/mean": -1.0728835935225333e-07, + "rewards/rnd_reward/std": 0.9939788401126861, + "step": 2070, + "step_time": 10.501573320990428 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08250000290572643, + "completions/max_length": 384.0, + "completions/max_terminated_length": 334.2, + "completions/mean_length": 91.6250015258789, + "completions/mean_terminated_length": 65.58112106323242, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9047418594360352, + "epoch": 1.3603662524525835, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.018664002418518, + "kl": 0.08782740281894803, + "learning_rate": 1.6007194244604316e-07, + "loss": 0.15487767457962037, + "num_tokens": 41152698.0, + "reward": -1.256167863061819e-07, + "reward_std": 0.9939099371433258, + "rewards/rnd_reward/mean": -1.2561678914835283e-07, + "rewards/rnd_reward/std": 0.9939099371433258, + "step": 2080, + "step_time": 10.78773567241151 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09625000096857547, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.6, + "completions/mean_length": 100.0337516784668, + "completions/mean_terminated_length": 69.91814079284669, + "completions/min_length": 1.9, + "completions/min_terminated_length": 1.9, + "entropy": 2.7196634620428086, + "epoch": 1.3669064748201438, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0979881286621094, + "kl": 0.0808198869228363, + "learning_rate": 1.5843688685415304e-07, + "loss": 0.131181538105011, + "num_tokens": 41330705.0, + "reward": -1.9848346672901583e-07, + "reward_std": 0.9943111777305603, + "rewards/rnd_reward/mean": -1.9848346672901583e-07, + "rewards/rnd_reward/std": 0.9943111896514892, + "step": 2090, + "step_time": 11.131318410113455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11250000149011612, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.0, + "completions/mean_length": 100.51125030517578, + "completions/mean_terminated_length": 64.72891502380371, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.9781338214874267, + "epoch": 1.3734466971877044, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0100359916687012, + "kl": 0.08852830519899726, + "learning_rate": 1.568018312622629e-07, + "loss": 0.1010061264038086, + "num_tokens": 41512969.0, + "reward": -2.2500751484244574e-08, + "reward_std": 0.9940204381942749, + "rewards/rnd_reward/mean": -2.2500754326415517e-08, + "rewards/rnd_reward/std": 0.9940204560756684, + "step": 2100, + "step_time": 11.608714979188516 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09250000165775418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.9, + "completions/mean_length": 99.50000114440918, + "completions/mean_terminated_length": 70.67034187316895, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.7862779945135117, + "epoch": 1.379986919555265, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.939647376537323, + "kl": 0.08671271968632936, + "learning_rate": 1.551667756703728e-07, + "loss": 0.07131583094596863, + "num_tokens": 41688714.0, + "reward": -5.654990502534929e-08, + "reward_std": 0.9939040839672089, + "rewards/rnd_reward/mean": -5.654991070969118e-08, + "rewards/rnd_reward/std": 0.9939040839672089, + "step": 2110, + "step_time": 10.6546359580243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.2, + "completions/mean_length": 105.17625045776367, + "completions/mean_terminated_length": 79.25297164916992, + "completions/min_length": 1.9, + "completions/min_terminated_length": 1.9, + "entropy": 2.7606303304433824, + "epoch": 1.3865271419228253, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.05146062374115, + "kl": 0.07692439574748278, + "learning_rate": 1.5353172007848268e-07, + "loss": 0.08600187301635742, + "num_tokens": 41867675.0, + "reward": -2.2687018805811476e-07, + "reward_std": 0.9938721656799316, + "rewards/rnd_reward/mean": -2.2687017953160193e-07, + "rewards/rnd_reward/std": 0.9938721656799316, + "step": 2120, + "step_time": 10.522320462344215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.0, + "completions/mean_length": 96.24625244140626, + "completions/mean_terminated_length": 67.99528999328614, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.848968583345413, + "epoch": 1.393067364290386, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0902374982833862, + "kl": 0.08595932517200708, + "learning_rate": 1.5189666448659253e-07, + "loss": 0.10983504056930542, + "num_tokens": 42040322.0, + "reward": -1.4990568644179803e-07, + "reward_std": 0.9940501570701599, + "rewards/rnd_reward/mean": -1.4990567649419971e-07, + "rewards/rnd_reward/std": 0.9940501391887665, + "step": 2130, + "step_time": 11.159733151923865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000098720192, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.4, + "completions/mean_length": 102.9712516784668, + "completions/mean_terminated_length": 77.02480850219726, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.790306252241135, + "epoch": 1.3996075866579463, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9154292941093445, + "kl": 0.077430235221982, + "learning_rate": 1.502616088947024e-07, + "loss": 0.06642908453941346, + "num_tokens": 42217629.0, + "reward": -9.521842088133781e-08, + "reward_std": 0.9938531458377838, + "rewards/rnd_reward/mean": -9.521841803916686e-08, + "rewards/rnd_reward/std": 0.9938531517982483, + "step": 2140, + "step_time": 11.171463684784248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000145286321, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.3, + "completions/mean_length": 102.10125198364258, + "completions/mean_terminated_length": 75.82226257324218, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.675621399283409, + "epoch": 1.4061478090255068, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0319130420684814, + "kl": 0.07909353226423263, + "learning_rate": 1.4862655330281228e-07, + "loss": 0.0862727165222168, + "num_tokens": 42394325.0, + "reward": -1.949071898987853e-07, + "reward_std": 0.9938407003879547, + "rewards/rnd_reward/mean": -1.9490719025405667e-07, + "rewards/rnd_reward/std": 0.9938406765460968, + "step": 2150, + "step_time": 11.259053491754457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07500000111758709, + "completions/max_length": 384.0, + "completions/max_terminated_length": 324.9, + "completions/mean_length": 85.76750183105469, + "completions/mean_terminated_length": 61.72400894165039, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.564385971426964, + "epoch": 1.4126880313930674, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0090827941894531, + "kl": 0.08957632025703788, + "learning_rate": 1.4699149771092216e-07, + "loss": 0.05871064066886902, + "num_tokens": 42561024.0, + "reward": -4.068017274150293e-08, + "reward_std": 0.9942023098468781, + "rewards/rnd_reward/mean": -4.068016918878925e-08, + "rewards/rnd_reward/std": 0.9942023098468781, + "step": 2160, + "step_time": 11.197773828869686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000052154064, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.5, + "completions/mean_length": 94.29875259399414, + "completions/mean_terminated_length": 67.34189720153809, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.5985243558883666, + "epoch": 1.4192282537606278, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9199637174606323, + "kl": 0.08871069457381964, + "learning_rate": 1.4535644211903204e-07, + "loss": 0.14661363363265992, + "num_tokens": 42731253.0, + "reward": -9.38773203174037e-08, + "reward_std": 0.9938823819160462, + "rewards/rnd_reward/mean": -9.387731640941865e-08, + "rewards/rnd_reward/std": 0.9938823938369751, + "step": 2170, + "step_time": 10.55815609311685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0600000012665987, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.3, + "completions/mean_length": 83.71750259399414, + "completions/mean_terminated_length": 64.72726974487304, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.617268776893616, + "epoch": 1.4257684761281884, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0482478141784668, + "kl": 0.0882584748789668, + "learning_rate": 1.4372138652714192e-07, + "loss": 0.04273238778114319, + "num_tokens": 42891912.0, + "reward": -2.068281148126516e-07, + "reward_std": 0.9936731696128845, + "rewards/rnd_reward/mean": -2.0682811765482257e-07, + "rewards/rnd_reward/std": 0.9936731576919555, + "step": 2180, + "step_time": 10.923749357438647 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06250000074505806, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.3, + "completions/mean_length": 86.99125213623047, + "completions/mean_terminated_length": 67.24082832336425, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.775143876671791, + "epoch": 1.432308698495749, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1199995279312134, + "kl": 0.08594764936715364, + "learning_rate": 1.420863309352518e-07, + "loss": 0.09395463466644287, + "num_tokens": 43055435.0, + "reward": -7.629394431774017e-08, + "reward_std": 0.9939565777778625, + "rewards/rnd_reward/mean": -7.629394431774017e-08, + "rewards/rnd_reward/std": 0.9939565896987915, + "step": 2190, + "step_time": 10.720726100169122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05625000055879355, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.9, + "completions/mean_length": 77.54250106811523, + "completions/mean_terminated_length": 59.23452491760254, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7800998747348786, + "epoch": 1.4388489208633093, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2252848148345947, + "kl": 0.09403526708483696, + "learning_rate": 1.4045127534336168e-07, + "loss": 0.072505122423172, + "num_tokens": 43214484.0, + "reward": -2.1472573372705028e-07, + "reward_std": 0.9937775075435639, + "rewards/rnd_reward/mean": -2.1472573088487935e-07, + "rewards/rnd_reward/std": 0.9937775015830994, + "step": 2200, + "step_time": 11.38782194799278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08250000104308128, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.3, + "completions/mean_length": 93.60000152587891, + "completions/mean_terminated_length": 67.61266975402832, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5929481953382494, + "epoch": 1.4453891432308699, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0342575311660767, + "kl": 0.08299258435145021, + "learning_rate": 1.3881621975147153e-07, + "loss": 0.1498832106590271, + "num_tokens": 43386444.0, + "reward": -1.038610928816297e-07, + "reward_std": 0.9938294231891632, + "rewards/rnd_reward/mean": -1.0386109572380064e-07, + "rewards/rnd_reward/std": 0.9938294172286988, + "step": 2210, + "step_time": 11.334175962558948 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08000000193715096, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.8, + "completions/mean_length": 88.12125167846679, + "completions/mean_terminated_length": 62.357414627075194, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.737210324406624, + "epoch": 1.4519293655984304, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2599236965179443, + "kl": 0.08750931257382036, + "learning_rate": 1.3718116415958143e-07, + "loss": 0.10528864860534667, + "num_tokens": 43548366.0, + "reward": -9.566545777772717e-08, + "reward_std": 0.9939428031444549, + "rewards/rnd_reward/mean": -9.566545600137033e-08, + "rewards/rnd_reward/std": 0.993942791223526, + "step": 2220, + "step_time": 10.802253967593424 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060000000335276124, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.9, + "completions/mean_length": 83.26000137329102, + "completions/mean_terminated_length": 64.11358032226562, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.655692085623741, + "epoch": 1.4584695879659908, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0874786376953125, + "kl": 0.09471626300364733, + "learning_rate": 1.355461085676913e-07, + "loss": 0.0698157787322998, + "num_tokens": 43704804.0, + "reward": -1.7568469061757242e-07, + "reward_std": 0.9932361245155334, + "rewards/rnd_reward/mean": -1.7568469061757242e-07, + "rewards/rnd_reward/std": 0.993236118555069, + "step": 2230, + "step_time": 10.654912683716976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07375000119209289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 333.5, + "completions/mean_length": 86.09000091552734, + "completions/mean_terminated_length": 62.54329261779785, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.6805451244115828, + "epoch": 1.4650098103335514, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2531136274337769, + "kl": 0.09066678332164883, + "learning_rate": 1.3391105297580116e-07, + "loss": 0.1158557415008545, + "num_tokens": 43870226.0, + "reward": -5.9008597474985435e-08, + "reward_std": 0.9934276759624481, + "rewards/rnd_reward/mean": -5.900859711971407e-08, + "rewards/rnd_reward/std": 0.9934276819229126, + "step": 2240, + "step_time": 10.659809662820772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06875000232830644, + "completions/max_length": 384.0, + "completions/max_terminated_length": 319.8, + "completions/mean_length": 86.3887508392334, + "completions/mean_terminated_length": 64.96298866271972, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.90979445874691, + "epoch": 1.4715500327011117, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8808140754699707, + "kl": 0.10609434265643358, + "learning_rate": 1.3227599738391104e-07, + "loss": 0.051298785209655764, + "num_tokens": 44036937.0, + "reward": -1.4841556961187052e-07, + "reward_std": 0.9932375848293304, + "rewards/rnd_reward/mean": -1.4841556605915684e-07, + "rewards/rnd_reward/std": 0.9932375907897949, + "step": 2250, + "step_time": 11.205043429811486 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07375000193715095, + "completions/max_length": 384.0, + "completions/max_terminated_length": 318.5, + "completions/mean_length": 82.19750175476074, + "completions/mean_terminated_length": 58.15153121948242, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 3.074227809906006, + "epoch": 1.4780902550686723, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.282700538635254, + "kl": 0.11512546790763736, + "learning_rate": 1.3064094179202092e-07, + "loss": 0.0818709373474121, + "num_tokens": 44197515.0, + "reward": -5.573034247419173e-08, + "reward_std": 0.9930948257446289, + "rewards/rnd_reward/mean": -5.573034211892036e-08, + "rewards/rnd_reward/std": 0.9930948317050934, + "step": 2260, + "step_time": 10.878628938994371 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06750000081956387, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.7, + "completions/mean_length": 84.10250320434571, + "completions/mean_terminated_length": 62.43845558166504, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.938316199183464, + "epoch": 1.4846304774362329, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2573261260986328, + "kl": 0.10300196381285787, + "learning_rate": 1.290058862001308e-07, + "loss": 0.0356877863407135, + "num_tokens": 44362002.0, + "reward": -1.7605722284486093e-07, + "reward_std": 0.9934640884399414, + "rewards/rnd_reward/mean": -1.7605722000268997e-07, + "rewards/rnd_reward/std": 0.9934640944004058, + "step": 2270, + "step_time": 11.115813568769955 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07750000040978193, + "completions/max_length": 384.0, + "completions/max_terminated_length": 343.9, + "completions/mean_length": 87.04125137329102, + "completions/mean_terminated_length": 62.220666122436526, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 3.1251493275165556, + "epoch": 1.4911706998037935, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3937524557113647, + "kl": 0.10200166283175349, + "learning_rate": 1.2737083060824068e-07, + "loss": 0.08271963596343994, + "num_tokens": 44524310.0, + "reward": -8.717179245110174e-08, + "reward_std": 0.9933235943317413, + "rewards/rnd_reward/mean": -8.717178818784533e-08, + "rewards/rnd_reward/std": 0.9933236181735993, + "step": 2280, + "step_time": 10.662292190524749 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06750000100582838, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.0, + "completions/mean_length": 84.15750160217286, + "completions/mean_terminated_length": 62.54970436096191, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 3.019434627890587, + "epoch": 1.4977109221713538, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0537996292114258, + "kl": 0.10465592611581087, + "learning_rate": 1.2573577501635055e-07, + "loss": 0.15651319026947022, + "num_tokens": 44686211.0, + "reward": -7.629394449537585e-08, + "reward_std": 0.9937636077404022, + "rewards/rnd_reward/mean": -7.629394520591859e-08, + "rewards/rnd_reward/std": 0.9937636256217957, + "step": 2290, + "step_time": 10.87905588184949 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07375000063329935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 337.0, + "completions/mean_length": 91.2612518310547, + "completions/mean_terminated_length": 67.97848358154297, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7947313100099564, + "epoch": 1.5042511445389142, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9707646369934082, + "kl": 0.0921105288900435, + "learning_rate": 1.2410071942446043e-07, + "loss": 0.037941274046897885, + "num_tokens": 44854255.0, + "reward": -3.1441446779467697e-08, + "reward_std": 0.9936448276042938, + "rewards/rnd_reward/mean": -3.144145281908095e-08, + "rewards/rnd_reward/std": 0.9936448395252228, + "step": 2300, + "step_time": 10.841944112465717 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05875000171363354, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.4, + "completions/mean_length": 82.65125198364258, + "completions/mean_terminated_length": 63.85377426147461, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.9727181673049925, + "epoch": 1.5107913669064748, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.044342279434204, + "kl": 0.10308242114260793, + "learning_rate": 1.224656638325703e-07, + "loss": 0.08985200524330139, + "num_tokens": 45021761.0, + "reward": -2.3201108554360416e-07, + "reward_std": 0.9937818229198456, + "rewards/rnd_reward/mean": -2.3201108447779007e-07, + "rewards/rnd_reward/std": 0.9937818348407745, + "step": 2310, + "step_time": 12.239735689153894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07500000195577741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 327.8, + "completions/mean_length": 90.46125106811523, + "completions/mean_terminated_length": 66.89172592163087, + "completions/min_length": 2.6, + "completions/min_terminated_length": 2.6, + "entropy": 2.8864229738712313, + "epoch": 1.5173315892740353, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1288195848464966, + "kl": 0.08688412019982934, + "learning_rate": 1.2083060824068016e-07, + "loss": 0.029066413640975952, + "num_tokens": 45186095.0, + "reward": 5.736946953049937e-08, + "reward_std": 0.99327272772789, + "rewards/rnd_reward/mean": 5.736947450429852e-08, + "rewards/rnd_reward/std": 0.9932727217674255, + "step": 2320, + "step_time": 10.734421823220327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07000000132247805, + "completions/max_length": 384.0, + "completions/max_terminated_length": 339.1, + "completions/mean_length": 92.6050018310547, + "completions/mean_terminated_length": 70.5366439819336, + "completions/min_length": 1.9, + "completions/min_terminated_length": 1.9, + "entropy": 2.7233670204877853, + "epoch": 1.523871811641596, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.24017333984375, + "kl": 0.08950559962540865, + "learning_rate": 1.1919555264879005e-07, + "loss": 0.07465057373046875, + "num_tokens": 45354024.0, + "reward": -2.0265579721012726e-07, + "reward_std": 0.993443387746811, + "rewards/rnd_reward/mean": -2.026557901046999e-07, + "rewards/rnd_reward/std": 0.9934433996677399, + "step": 2330, + "step_time": 10.924078989424743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06375000141561031, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.3, + "completions/mean_length": 85.09000129699707, + "completions/mean_terminated_length": 64.81442031860351, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.803052818775177, + "epoch": 1.5304120340091563, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.400153636932373, + "kl": 0.09231640296056867, + "learning_rate": 1.1756049705689992e-07, + "loss": 0.126143217086792, + "num_tokens": 45510991.0, + "reward": -1.0177493525720394e-07, + "reward_std": 0.9933503866195679, + "rewards/rnd_reward/mean": -1.0177492921759068e-07, + "rewards/rnd_reward/std": 0.9933504104614258, + "step": 2340, + "step_time": 10.554431772092357 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08000000081956386, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.3, + "completions/mean_length": 88.79000091552734, + "completions/mean_terminated_length": 62.90113410949707, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7890906423330306, + "epoch": 1.5369522563767168, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1739593744277954, + "kl": 0.08973503690212965, + "learning_rate": 1.1592544146500981e-07, + "loss": 0.016262634098529814, + "num_tokens": 45673063.0, + "reward": -2.1778047649689823e-07, + "reward_std": 0.9933681070804596, + "rewards/rnd_reward/mean": -2.177804736547273e-07, + "rewards/rnd_reward/std": 0.9933681011199951, + "step": 2350, + "step_time": 10.583909913594834 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09750000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.9, + "completions/mean_length": 96.30625228881836, + "completions/mean_terminated_length": 65.25630912780761, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7480629831552505, + "epoch": 1.5434924787442772, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.265565037727356, + "kl": 0.08257783968001604, + "learning_rate": 1.1429038587311968e-07, + "loss": 0.09627790451049804, + "num_tokens": 45842103.0, + "reward": -8.836388873945111e-08, + "reward_std": 0.9934152483940124, + "rewards/rnd_reward/mean": -8.836388589728018e-08, + "rewards/rnd_reward/std": 0.993415230512619, + "step": 2360, + "step_time": 10.685579545726068 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08375000134110451, + "completions/max_length": 384.0, + "completions/max_terminated_length": 333.4, + "completions/mean_length": 91.16250152587891, + "completions/mean_terminated_length": 64.41967430114747, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.663706639409065, + "epoch": 1.5500327011118378, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5513228178024292, + "kl": 0.08946019792929291, + "learning_rate": 1.1265533028122955e-07, + "loss": 0.03821712434291839, + "num_tokens": 46015743.0, + "reward": -2.2679567930339316e-07, + "reward_std": 0.993412846326828, + "rewards/rnd_reward/mean": -2.2679567575067948e-07, + "rewards/rnd_reward/std": 0.9934128284454345, + "step": 2370, + "step_time": 11.590211745630949 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06125000081956387, + "completions/max_length": 383.4, + "completions/max_terminated_length": 346.7, + "completions/mean_length": 89.28625183105468, + "completions/mean_terminated_length": 69.96004066467285, + "completions/min_length": 2.7, + "completions/min_terminated_length": 2.7, + "entropy": 3.0011739432811737, + "epoch": 1.5565729234793984, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1020563840866089, + "kl": 0.09448451111093163, + "learning_rate": 1.1102027468933943e-07, + "loss": 0.13594624996185303, + "num_tokens": 46183567.0, + "reward": 6.988644907224284e-08, + "reward_std": 0.9929902851581573, + "rewards/rnd_reward/mean": 6.98864462300719e-08, + "rewards/rnd_reward/std": 0.9929902911186218, + "step": 2380, + "step_time": 11.415522536402568 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0750000024214387, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.0, + "completions/mean_length": 90.28625106811523, + "completions/mean_terminated_length": 66.44939155578614, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.8806687116622927, + "epoch": 1.563113145846959, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0451889038085938, + "kl": 0.09008068647235631, + "learning_rate": 1.0938521909744931e-07, + "loss": 0.047099918127059937, + "num_tokens": 46343361.0, + "reward": -5.662440694109705e-09, + "reward_std": 0.9929801285266876, + "rewards/rnd_reward/mean": -5.6624399835669695e-09, + "rewards/rnd_reward/std": 0.9929801285266876, + "step": 2390, + "step_time": 10.55025691683404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08250000234693289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 333.8, + "completions/mean_length": 92.99375228881836, + "completions/mean_terminated_length": 66.80701446533203, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8602111876010894, + "epoch": 1.5696533682145193, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2736488580703735, + "kl": 0.09589690109714866, + "learning_rate": 1.0775016350555919e-07, + "loss": 0.09055531024932861, + "num_tokens": 46510141.0, + "reward": -1.1608004477636768e-07, + "reward_std": 0.9930357456207275, + "rewards/rnd_reward/mean": -1.1608004477636768e-07, + "rewards/rnd_reward/std": 0.9930357277393341, + "step": 2400, + "step_time": 10.752315263729542 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000150874257, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.3, + "completions/mean_length": 99.20625152587891, + "completions/mean_terminated_length": 75.86554489135742, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.8277246713638307, + "epoch": 1.5761935905820796, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0789973735809326, + "kl": 0.08739803759381175, + "learning_rate": 1.0611510791366907e-07, + "loss": 0.06712510585784912, + "num_tokens": 46679921.0, + "reward": -9.715557247602646e-08, + "reward_std": 0.9933283686637878, + "rewards/rnd_reward/mean": -9.715556821277005e-08, + "rewards/rnd_reward/std": 0.9933283627033234, + "step": 2410, + "step_time": 10.51516141500324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000186264514, + "completions/max_length": 384.0, + "completions/max_terminated_length": 335.9, + "completions/mean_length": 99.28125076293945, + "completions/mean_terminated_length": 72.07235450744629, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.6956777185201646, + "epoch": 1.5827338129496402, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.215997338294983, + "kl": 0.08504411997273564, + "learning_rate": 1.0448005232177893e-07, + "loss": 0.06522908210754394, + "num_tokens": 46848921.0, + "reward": -4.142523053474179e-08, + "reward_std": 0.9928943634033203, + "rewards/rnd_reward/mean": -4.142522982419905e-08, + "rewards/rnd_reward/std": 0.9928943693637848, + "step": 2420, + "step_time": 10.540731458202936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06875000121071935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.5, + "completions/mean_length": 85.0125015258789, + "completions/mean_terminated_length": 62.73664093017578, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 3.0628849148750303, + "epoch": 1.5892740353172008, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.336714267730713, + "kl": 0.10201989021152258, + "learning_rate": 1.0284499672988881e-07, + "loss": 0.05233617424964905, + "num_tokens": 47004126.0, + "reward": -1.521408613314179e-07, + "reward_std": 0.9932492971420288, + "rewards/rnd_reward/mean": -1.5214085848924697e-07, + "rewards/rnd_reward/std": 0.9932492971420288, + "step": 2430, + "step_time": 10.492817461281083 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07500000074505805, + "completions/max_length": 384.0, + "completions/max_terminated_length": 332.8, + "completions/mean_length": 86.80125122070312, + "completions/mean_terminated_length": 62.760490036010744, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.799528333544731, + "epoch": 1.5958142576847614, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4625661373138428, + "kl": 0.10000391742214561, + "learning_rate": 1.0120994113799869e-07, + "loss": 0.031586483120918274, + "num_tokens": 47168777.0, + "reward": -7.808208586368437e-08, + "reward_std": 0.9931057214736938, + "rewards/rnd_reward/mean": -7.80820865742271e-08, + "rewards/rnd_reward/std": 0.9931057274341584, + "step": 2440, + "step_time": 10.696252883039415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07875000163912774, + "completions/max_length": 384.0, + "completions/max_terminated_length": 334.2, + "completions/mean_length": 88.19125213623047, + "completions/mean_terminated_length": 62.81879196166992, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.742514914274216, + "epoch": 1.6023544800523217, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1427501440048218, + "kl": 0.09996775221079587, + "learning_rate": 9.957488554610855e-08, + "loss": 0.041766366362571715, + "num_tokens": 47331300.0, + "reward": 5.722046019229765e-08, + "reward_std": 0.9931129157543183, + "rewards/rnd_reward/mean": 5.7220459481754915e-08, + "rewards/rnd_reward/std": 0.9931129038333892, + "step": 2450, + "step_time": 10.62438986178022 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07874999996274709, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.3, + "completions/mean_length": 90.64625091552735, + "completions/mean_terminated_length": 65.4418701171875, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.8206116437911986, + "epoch": 1.6088947024198823, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2679322957992554, + "kl": 0.0997009389102459, + "learning_rate": 9.793982995421845e-08, + "loss": 0.08149058222770691, + "num_tokens": 47497532.0, + "reward": 1.2382865293147917e-07, + "reward_std": 0.9925880253314971, + "rewards/rnd_reward/mean": 1.238286500893082e-07, + "rewards/rnd_reward/std": 0.9925880193710327, + "step": 2460, + "step_time": 10.703694239328616 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000111758709, + "completions/max_length": 384.0, + "completions/max_terminated_length": 341.8, + "completions/mean_length": 95.64250106811524, + "completions/mean_terminated_length": 67.96361427307129, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.671072342991829, + "epoch": 1.6154349247874427, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.008203387260437, + "kl": 0.09101189225912094, + "learning_rate": 9.630477436232831e-08, + "loss": 0.02300444096326828, + "num_tokens": 47673241.0, + "reward": -1.6734004162355376e-07, + "reward_std": 0.9930606245994568, + "rewards/rnd_reward/mean": -1.6734004304463923e-07, + "rewards/rnd_reward/std": 0.9930606305599212, + "step": 2470, + "step_time": 11.121605953085236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000104308128, + "completions/max_length": 384.0, + "completions/max_terminated_length": 342.3, + "completions/mean_length": 91.83125076293945, + "completions/mean_terminated_length": 63.43943824768066, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.891758808493614, + "epoch": 1.6219751471550032, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0772875547409058, + "kl": 0.09280837709084153, + "learning_rate": 9.466971877043819e-08, + "loss": 0.032535853981971743, + "num_tokens": 47836976.0, + "reward": -1.448392879410676e-07, + "reward_std": 0.9926415860652924, + "rewards/rnd_reward/mean": -1.4483928936215307e-07, + "rewards/rnd_reward/std": 0.9926415801048278, + "step": 2480, + "step_time": 10.584856062568724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07875000201165676, + "completions/max_length": 384.0, + "completions/max_terminated_length": 343.0, + "completions/mean_length": 91.06500015258788, + "completions/mean_terminated_length": 65.96636085510254, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.72680883705616, + "epoch": 1.6285153695225638, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3315449953079224, + "kl": 0.09668447291478514, + "learning_rate": 9.303466317854807e-08, + "loss": 0.1025187611579895, + "num_tokens": 48003343.0, + "reward": -8.106231996762858e-08, + "reward_std": 0.9925062835216523, + "rewards/rnd_reward/mean": -8.10623164149149e-08, + "rewards/rnd_reward/std": 0.9925062835216523, + "step": 2490, + "step_time": 10.71222500174772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07000000141561032, + "completions/max_length": 384.0, + "completions/max_terminated_length": 318.3, + "completions/mean_length": 85.66875228881835, + "completions/mean_terminated_length": 63.251688385009764, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.662171345949173, + "epoch": 1.6350555918901244, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3252993822097778, + "kl": 0.09045366756618023, + "learning_rate": 9.139960758665795e-08, + "loss": 0.0439204216003418, + "num_tokens": 48168473.0, + "reward": -3.5464765701931354e-08, + "reward_std": 0.9928713321685791, + "rewards/rnd_reward/mean": -3.5464758596503997e-08, + "rewards/rnd_reward/std": 0.9928713619709015, + "step": 2500, + "step_time": 10.784380649239756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.8, + "completions/mean_length": 96.40625228881837, + "completions/mean_terminated_length": 68.52096328735351, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.7209132075309754, + "epoch": 1.6415958142576847, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0920087099075317, + "kl": 0.08929237592965364, + "learning_rate": 8.976455199476782e-08, + "loss": 0.07013883590698242, + "num_tokens": 48335753.0, + "reward": -1.1816621103832858e-07, + "reward_std": 0.9925974249839783, + "rewards/rnd_reward/mean": -1.1816620961724312e-07, + "rewards/rnd_reward/std": 0.9925974488258362, + "step": 2510, + "step_time": 10.549463000241667 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000067055225, + "completions/max_length": 384.0, + "completions/max_terminated_length": 335.7, + "completions/mean_length": 94.45250015258789, + "completions/mean_terminated_length": 66.36481285095215, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.6327116012573244, + "epoch": 1.648136036625245, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.945324182510376, + "kl": 0.08747830325737596, + "learning_rate": 8.812949640287769e-08, + "loss": 0.13749951124191284, + "num_tokens": 48500460.0, + "reward": -7.227063427706071e-08, + "reward_std": 0.9929073750972748, + "rewards/rnd_reward/mean": -7.227063250070387e-08, + "rewards/rnd_reward/std": 0.9929073631763459, + "step": 2520, + "step_time": 10.486474628048018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000104308129, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.3, + "completions/mean_length": 92.80625228881836, + "completions/mean_terminated_length": 68.86259307861329, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.5589567482471467, + "epoch": 1.6546762589928057, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.173561692237854, + "kl": 0.08464234415441751, + "learning_rate": 8.649444081098757e-08, + "loss": 0.07577463388442993, + "num_tokens": 48664250.0, + "reward": -1.436471933402572e-07, + "reward_std": 0.9925457060337066, + "rewards/rnd_reward/mean": -1.436471933402572e-07, + "rewards/rnd_reward/std": 0.9925457000732422, + "step": 2530, + "step_time": 10.735207921243273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0800000000745058, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.6, + "completions/mean_length": 93.55000152587891, + "completions/mean_terminated_length": 68.25583038330078, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6400635212659838, + "epoch": 1.6612164813603663, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0104870796203613, + "kl": 0.08781723519787192, + "learning_rate": 8.485938521909745e-08, + "loss": 0.03668081760406494, + "num_tokens": 48831920.0, + "reward": -4.6491619976052335e-08, + "reward_std": 0.9925249576568603, + "rewards/rnd_reward/mean": -4.6491622818223274e-08, + "rewards/rnd_reward/std": 0.9925249457359314, + "step": 2540, + "step_time": 10.869395459746011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10750000104308129, + "completions/max_length": 384.0, + "completions/max_terminated_length": 332.1, + "completions/mean_length": 100.57750244140625, + "completions/mean_terminated_length": 66.49176979064941, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7832937359809877, + "epoch": 1.6677567037279268, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0521481037139893, + "kl": 0.08043702868744732, + "learning_rate": 8.322432962720732e-08, + "loss": 0.05087646245956421, + "num_tokens": 49003242.0, + "reward": -6.422400495864622e-08, + "reward_std": 0.9929387688636779, + "rewards/rnd_reward/mean": -6.422400566918895e-08, + "rewards/rnd_reward/std": 0.9929387867450714, + "step": 2550, + "step_time": 10.56416127383709 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000104308128, + "completions/max_length": 384.0, + "completions/max_terminated_length": 337.2, + "completions/mean_length": 98.30000152587891, + "completions/mean_terminated_length": 70.51053009033203, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.768531382083893, + "epoch": 1.6742969260954872, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9573103785514832, + "kl": 0.08502876814454793, + "learning_rate": 8.158927403531719e-08, + "loss": 0.0772208571434021, + "num_tokens": 49174652.0, + "reward": -1.469254531372144e-07, + "reward_std": 0.9924641728401185, + "rewards/rnd_reward/mean": -1.4692545029504346e-07, + "rewards/rnd_reward/std": 0.9924641907215118, + "step": 2560, + "step_time": 10.583414485724642 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.7, + "completions/mean_length": 96.5462516784668, + "completions/mean_terminated_length": 69.8155361175537, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8294411927461622, + "epoch": 1.6808371484630478, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9854137301445007, + "kl": 0.09185233255848288, + "learning_rate": 7.995421844342708e-08, + "loss": 0.1261582612991333, + "num_tokens": 49354119.0, + "reward": 8.67247596403331e-08, + "reward_std": 0.9922890484333038, + "rewards/rnd_reward/mean": 8.67247596403331e-08, + "rewards/rnd_reward/std": 0.9922890365123749, + "step": 2570, + "step_time": 11.603049716376699 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07875000052154064, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.7, + "completions/mean_length": 92.55000152587891, + "completions/mean_terminated_length": 67.6265712738037, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7690848499536513, + "epoch": 1.6873773708306081, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1284137964248657, + "kl": 0.09411861123517155, + "learning_rate": 7.831916285153695e-08, + "loss": 0.13598878383636476, + "num_tokens": 49524444.0, + "reward": 6.571412010103473e-08, + "reward_std": 0.9924658596515655, + "rewards/rnd_reward/mean": 6.571412010103473e-08, + "rewards/rnd_reward/std": 0.9924658596515655, + "step": 2580, + "step_time": 11.369293246162124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09000000208616257, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.5, + "completions/mean_length": 100.7300018310547, + "completions/mean_terminated_length": 72.78359603881836, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.656350103020668, + "epoch": 1.6939175931981687, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1325582265853882, + "kl": 0.08164091287180782, + "learning_rate": 7.668410725964682e-08, + "loss": 0.09759321212768554, + "num_tokens": 49702873.0, + "reward": -8.478760804564445e-08, + "reward_std": 0.9928931176662446, + "rewards/rnd_reward/mean": -8.478760946672991e-08, + "rewards/rnd_reward/std": 0.9928931057453155, + "step": 2590, + "step_time": 10.856351627549156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07000000141561032, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.6, + "completions/mean_length": 91.03375091552735, + "completions/mean_terminated_length": 69.03646697998047, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.887327644228935, + "epoch": 1.7004578155657293, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3039114475250244, + "kl": 0.0926243552006781, + "learning_rate": 7.50490516677567e-08, + "loss": 0.0819834053516388, + "num_tokens": 49876500.0, + "reward": 1.3411066479562806e-09, + "reward_std": 0.9926319658756256, + "rewards/rnd_reward/mean": 1.3411066479562806e-09, + "rewards/rnd_reward/std": 0.9926319539546966, + "step": 2600, + "step_time": 11.05162076470442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08125000149011612, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.7, + "completions/mean_length": 89.97250061035156, + "completions/mean_terminated_length": 63.9521728515625, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.807651326060295, + "epoch": 1.7069980379332899, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0977989435195923, + "kl": 0.08759319856762886, + "learning_rate": 7.341399607586658e-08, + "loss": 0.13040846586227417, + "num_tokens": 50044628.0, + "reward": -6.228685363041109e-08, + "reward_std": 0.9921288013458252, + "rewards/rnd_reward/mean": -6.228685291986836e-08, + "rewards/rnd_reward/std": 0.9921288132667542, + "step": 2610, + "step_time": 11.296212270553223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08375000227242708, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.6, + "completions/mean_length": 91.39750061035156, + "completions/mean_terminated_length": 64.60456695556641, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.771796461939812, + "epoch": 1.7135382603008502, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.101976990699768, + "kl": 0.09053975995630026, + "learning_rate": 7.177894048397645e-08, + "loss": 0.07479513287544251, + "num_tokens": 50204246.0, + "reward": -1.3120472530658844e-07, + "reward_std": 0.9925267994403839, + "rewards/rnd_reward/mean": -1.3120472530658844e-07, + "rewards/rnd_reward/std": 0.9925268173217774, + "step": 2620, + "step_time": 10.44964153978508 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375000074505806, + "completions/max_length": 384.0, + "completions/max_terminated_length": 325.7, + "completions/mean_length": 95.06375122070312, + "completions/mean_terminated_length": 65.27039031982422, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6531850188970565, + "epoch": 1.7200784826684106, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.112268090248108, + "kl": 0.08289336860179901, + "learning_rate": 7.014388489208632e-08, + "loss": 0.08899658918380737, + "num_tokens": 50373747.0, + "reward": -2.539157884839938e-07, + "reward_std": 0.9920955657958984, + "rewards/rnd_reward/mean": -2.539157849312801e-07, + "rewards/rnd_reward/std": 0.9920955717563629, + "step": 2630, + "step_time": 10.710641134064645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07875000201165676, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.8, + "completions/mean_length": 93.92000198364258, + "completions/mean_terminated_length": 69.12318840026856, + "completions/min_length": 2.7, + "completions/min_terminated_length": 2.7, + "entropy": 2.5591167956590652, + "epoch": 1.7266187050359711, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0098210573196411, + "kl": 0.08759762533009052, + "learning_rate": 6.85088293001962e-08, + "loss": 0.11666052341461182, + "num_tokens": 50542968.0, + "reward": -2.8014184039193423e-08, + "reward_std": 0.9922697126865387, + "rewards/rnd_reward/mean": -2.8014186881364366e-08, + "rewards/rnd_reward/std": 0.9922697246074677, + "step": 2640, + "step_time": 10.810134377889336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09000000171363354, + "completions/max_length": 384.0, + "completions/max_terminated_length": 327.5, + "completions/mean_length": 90.80375213623047, + "completions/mean_terminated_length": 61.77821655273438, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.779536947607994, + "epoch": 1.7331589274035317, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1324180364608765, + "kl": 0.08631603391841054, + "learning_rate": 6.687377370830608e-08, + "loss": 0.09856376647949219, + "num_tokens": 50707726.0, + "reward": -5.237758227139011e-08, + "reward_std": 0.992433100938797, + "rewards/rnd_reward/mean": -5.237758085030464e-08, + "rewards/rnd_reward/std": 0.9924331188201905, + "step": 2650, + "step_time": 10.956776090711354 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375000186264515, + "completions/max_length": 384.0, + "completions/max_terminated_length": 334.8, + "completions/mean_length": 97.68875045776367, + "completions/mean_terminated_length": 68.05043869018554, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.7804177552461624, + "epoch": 1.7396991497710923, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0591152906417847, + "kl": 0.09146134741604328, + "learning_rate": 6.523871811641596e-08, + "loss": 0.04577620327472687, + "num_tokens": 50878792.0, + "reward": -7.227059839465255e-09, + "reward_std": 0.9924111425876617, + "rewards/rnd_reward/mean": -7.227060550007991e-09, + "rewards/rnd_reward/std": 0.9924111247062684, + "step": 2660, + "step_time": 10.708624366554432 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000108033419, + "completions/max_length": 384.0, + "completions/max_terminated_length": 330.3, + "completions/mean_length": 95.15875205993652, + "completions/mean_terminated_length": 68.52624740600587, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.626550641655922, + "epoch": 1.7462393721386529, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0052024126052856, + "kl": 0.09057780019938946, + "learning_rate": 6.360366252452582e-08, + "loss": 0.03838871717453003, + "num_tokens": 51047709.0, + "reward": -1.60261992654398e-07, + "reward_std": 0.9925138711929321, + "rewards/rnd_reward/mean": -1.6026198892404864e-07, + "rewards/rnd_reward/std": 0.9925138831138611, + "step": 2670, + "step_time": 10.687237256928347 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07000000113621355, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.8, + "completions/mean_length": 91.47625198364258, + "completions/mean_terminated_length": 69.65823402404786, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6517731666564943, + "epoch": 1.7527795945062132, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.085845947265625, + "kl": 0.08733204193413258, + "learning_rate": 6.19686069326357e-08, + "loss": 0.126353120803833, + "num_tokens": 51221270.0, + "reward": -1.2561679199052378e-07, + "reward_std": 0.9924490749835968, + "rewards/rnd_reward/mean": -1.2561678275346821e-07, + "rewards/rnd_reward/std": 0.9924490511417389, + "step": 2680, + "step_time": 11.201374626043252 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08250000150874257, + "completions/max_length": 384.0, + "completions/max_terminated_length": 339.3, + "completions/mean_length": 97.07750091552734, + "completions/mean_terminated_length": 71.1569839477539, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.534248262643814, + "epoch": 1.7593198168737736, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1865549087524414, + "kl": 0.07945491755381226, + "learning_rate": 6.033355134074558e-08, + "loss": 0.023079755902290344, + "num_tokens": 51391307.0, + "reward": 1.2665985660298417e-08, + "reward_std": 0.9924781501293183, + "rewards/rnd_reward/mean": 1.2665986370841154e-08, + "rewards/rnd_reward/std": 0.9924781501293183, + "step": 2690, + "step_time": 10.600427453825251 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10375000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.4, + "completions/mean_length": 102.50500183105468, + "completions/mean_terminated_length": 69.94977340698242, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5867824822664263, + "epoch": 1.7658600392413342, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9612109661102295, + "kl": 0.08055433174595236, + "learning_rate": 5.869849574885546e-08, + "loss": 0.09532687664031983, + "num_tokens": 51566461.0, + "reward": -4.351139040181806e-08, + "reward_std": 0.9921834826469421, + "rewards/rnd_reward/mean": -4.351139040181806e-08, + "rewards/rnd_reward/std": 0.9921834707260132, + "step": 2700, + "step_time": 10.596421872614883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09625000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.4, + "completions/mean_length": 103.48625259399414, + "completions/mean_terminated_length": 73.66226654052734, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.574134039878845, + "epoch": 1.7724002616088947, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8851132392883301, + "kl": 0.07895892215892672, + "learning_rate": 5.706344015696534e-08, + "loss": 0.03208762109279632, + "num_tokens": 51739420.0, + "reward": 1.4901161127234275e-08, + "reward_std": 0.9922367691993713, + "rewards/rnd_reward/mean": 1.4901161482505642e-08, + "rewards/rnd_reward/std": 0.992236751317978, + "step": 2710, + "step_time": 10.52075885424856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07750000096857548, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.7, + "completions/mean_length": 98.13000183105468, + "completions/mean_terminated_length": 74.1987533569336, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7050154358148575, + "epoch": 1.7789404839764553, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2265753746032715, + "kl": 0.08297120407223701, + "learning_rate": 5.542838456507521e-08, + "loss": 0.07181986570358276, + "num_tokens": 51906604.0, + "reward": 2.1010632700324548e-08, + "reward_std": 0.9923449456691742, + "rewards/rnd_reward/mean": 2.101064406900832e-08, + "rewards/rnd_reward/std": 0.9923449575901031, + "step": 2720, + "step_time": 10.726289883395657 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10125000029802322, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.8, + "completions/mean_length": 97.52250137329102, + "completions/mean_terminated_length": 65.36152725219726, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5555239796638487, + "epoch": 1.7854807063440157, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.344201922416687, + "kl": 0.07681396808475256, + "learning_rate": 5.379332897318509e-08, + "loss": 0.09209707379341125, + "num_tokens": 52078602.0, + "reward": 7.450558392463336e-10, + "reward_std": 0.991978645324707, + "rewards/rnd_reward/mean": 7.450586814172766e-10, + "rewards/rnd_reward/std": 0.9919786393642426, + "step": 2730, + "step_time": 10.693091615103185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09500000141561031, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.1, + "completions/mean_length": 99.98125228881835, + "completions/mean_terminated_length": 70.13334350585937, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.684102329611778, + "epoch": 1.792020928711576, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1477556228637695, + "kl": 0.08274123426526785, + "learning_rate": 5.2158273381294966e-08, + "loss": 0.025458133220672606, + "num_tokens": 52245797.0, + "reward": 3.457069563950199e-08, + "reward_std": 0.9920007467269898, + "rewards/rnd_reward/mean": 3.457069741585883e-08, + "rewards/rnd_reward/std": 0.9920007586479187, + "step": 2740, + "step_time": 10.481961290817708 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07750000078231097, + "completions/max_length": 384.0, + "completions/max_terminated_length": 339.5, + "completions/mean_length": 91.90375061035157, + "completions/mean_terminated_length": 67.26799354553222, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.7344156444072723, + "epoch": 1.7985611510791366, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1605439186096191, + "kl": 0.08739866856485605, + "learning_rate": 5.052321778940484e-08, + "loss": 0.0825818121433258, + "num_tokens": 52417430.0, + "reward": -5.275011112360062e-08, + "reward_std": 0.9921901881694793, + "rewards/rnd_reward/mean": -5.275011254468609e-08, + "rewards/rnd_reward/std": 0.992190170288086, + "step": 2750, + "step_time": 11.562560918554663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000126659871, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.7, + "completions/mean_length": 95.38625106811523, + "completions/mean_terminated_length": 68.69163131713867, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.508667880296707, + "epoch": 1.8051013734466972, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1222717761993408, + "kl": 0.08437787899747491, + "learning_rate": 4.8888162197514716e-08, + "loss": 0.08072326779365539, + "num_tokens": 52587249.0, + "reward": -3.2708050046181825e-08, + "reward_std": 0.9921342968940735, + "rewards/rnd_reward/mean": -3.270804720401088e-08, + "rewards/rnd_reward/std": 0.9921342968940735, + "step": 2760, + "step_time": 10.81694066869095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09125000275671483, + "completions/max_length": 384.0, + "completions/max_terminated_length": 342.4, + "completions/mean_length": 100.94500198364258, + "completions/mean_terminated_length": 72.59214401245117, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.762883481383324, + "epoch": 1.8116415958142578, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3192421197891235, + "kl": 0.08027938418090344, + "learning_rate": 4.725310660562459e-08, + "loss": 0.10753175020217895, + "num_tokens": 52769960.0, + "reward": 5.5879356897214906e-08, + "reward_std": 0.9917623281478882, + "rewards/rnd_reward/mean": 5.587935405504396e-08, + "rewards/rnd_reward/std": 0.9917623460292816, + "step": 2770, + "step_time": 11.582827594131231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0787500012665987, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.5, + "completions/mean_length": 95.13875122070313, + "completions/mean_terminated_length": 70.49329643249511, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6798209667205812, + "epoch": 1.8181818181818183, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1080927848815918, + "kl": 0.08364078365266323, + "learning_rate": 4.5618051013734466e-08, + "loss": 0.04997023940086365, + "num_tokens": 52940211.0, + "reward": -9.596347978479969e-08, + "reward_std": 0.9920522153377533, + "rewards/rnd_reward/mean": -9.596347978479969e-08, + "rewards/rnd_reward/std": 0.9920521914958954, + "step": 2780, + "step_time": 10.742116263648494 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000044703483, + "completions/max_length": 384.0, + "completions/max_terminated_length": 326.3, + "completions/mean_length": 93.41375198364258, + "completions/mean_terminated_length": 65.94217224121094, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.7876433610916136, + "epoch": 1.8247220405493787, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2520111799240112, + "kl": 0.08399344785138965, + "learning_rate": 4.398299542184434e-08, + "loss": 0.11648783683776856, + "num_tokens": 53106412.0, + "reward": -1.3023615004215116e-07, + "reward_std": 0.9917174458503724, + "rewards/rnd_reward/mean": -1.3023615146323665e-07, + "rewards/rnd_reward/std": 0.9917174398899078, + "step": 2790, + "step_time": 10.729628002620302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000029802323, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.1, + "completions/mean_length": 95.42125091552734, + "completions/mean_terminated_length": 67.35246047973632, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.751756873726845, + "epoch": 1.831262262916939, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2754117250442505, + "kl": 0.08136667888611555, + "learning_rate": 4.2347939829954215e-08, + "loss": 0.10360376834869385, + "num_tokens": 53273129.0, + "reward": -8.925795818015558e-08, + "reward_std": 0.992232620716095, + "rewards/rnd_reward/mean": -8.92579546274419e-08, + "rewards/rnd_reward/std": 0.9922326266765594, + "step": 2800, + "step_time": 10.579858994716778 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08250000178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.3, + "completions/mean_length": 101.0425018310547, + "completions/mean_terminated_length": 75.79266548156738, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.514050582051277, + "epoch": 1.8378024852844996, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.292152762413025, + "kl": 0.07886797543615102, + "learning_rate": 4.0712884238064094e-08, + "loss": 0.07606931924819946, + "num_tokens": 53448303.0, + "reward": -2.5182961227443456e-08, + "reward_std": 0.9920957505702972, + "rewards/rnd_reward/mean": -2.5182961227443456e-08, + "rewards/rnd_reward/std": 0.9920957386493683, + "step": 2810, + "step_time": 11.144761186954565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000126659871, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.3, + "completions/mean_length": 99.48500213623046, + "completions/mean_terminated_length": 73.15060997009277, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.75241442322731, + "epoch": 1.8443427076520602, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0998656749725342, + "kl": 0.07965973112732172, + "learning_rate": 3.9077828646173965e-08, + "loss": 0.08108996152877808, + "num_tokens": 53620201.0, + "reward": 1.832842806415158e-08, + "reward_std": 0.9918050169944763, + "rewards/rnd_reward/mean": 1.832843197213663e-08, + "rewards/rnd_reward/std": 0.9918050229549408, + "step": 2820, + "step_time": 11.469240892818197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000119209289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.9, + "completions/mean_length": 97.51125106811523, + "completions/mean_terminated_length": 70.54906845092773, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.549181741476059, + "epoch": 1.8508829300196208, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0469340085983276, + "kl": 0.0786884649656713, + "learning_rate": 3.7442773054283844e-08, + "loss": 0.05131007432937622, + "num_tokens": 53801605.0, + "reward": -5.766749477942312e-08, + "reward_std": 0.9918078780174255, + "rewards/rnd_reward/mean": -5.766749477942312e-08, + "rewards/rnd_reward/std": 0.9918078899383544, + "step": 2830, + "step_time": 11.47003021549899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07375000119209289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.9, + "completions/mean_length": 91.74375228881836, + "completions/mean_terminated_length": 68.46907119750976, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.741053342819214, + "epoch": 1.8574231523871811, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0931239128112793, + "kl": 0.0858668725937605, + "learning_rate": 3.580771746239372e-08, + "loss": 0.01960263103246689, + "num_tokens": 53967620.0, + "reward": -8.866190732703672e-08, + "reward_std": 0.9914042472839355, + "rewards/rnd_reward/mean": -8.866190661649398e-08, + "rewards/rnd_reward/std": 0.9914042532444001, + "step": 2840, + "step_time": 11.055267255497165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07875000052154064, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 94.5212516784668, + "completions/mean_terminated_length": 69.70959434509277, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.616148290038109, + "epoch": 1.8639633747547415, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1507196426391602, + "kl": 0.07959493407979608, + "learning_rate": 3.41726618705036e-08, + "loss": 0.09600520730018616, + "num_tokens": 54134747.0, + "reward": -1.1116266360744475e-07, + "reward_std": 0.9913456678390503, + "rewards/rnd_reward/mean": -1.1116266360744475e-07, + "rewards/rnd_reward/std": 0.9913456499576568, + "step": 2850, + "step_time": 10.70933379684575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08000000175088644, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.4, + "completions/mean_length": 92.95874938964843, + "completions/mean_terminated_length": 67.40721321105957, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.561130478978157, + "epoch": 1.870503597122302, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9997308254241943, + "kl": 0.08147254046052695, + "learning_rate": 3.253760627861347e-08, + "loss": 0.09602594375610352, + "num_tokens": 54297779.0, + "reward": -1.2561679305633788e-07, + "reward_std": 0.9915251851081848, + "rewards/rnd_reward/mean": -1.256167838192823e-07, + "rewards/rnd_reward/std": 0.9915251791477203, + "step": 2860, + "step_time": 10.617276117694564 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000048428773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 324.1, + "completions/mean_length": 94.39250106811524, + "completions/mean_terminated_length": 70.46714973449707, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5941164553165437, + "epoch": 1.8770438194898627, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0238882303237915, + "kl": 0.08611528370529413, + "learning_rate": 3.090255068672335e-08, + "loss": 0.1147921085357666, + "num_tokens": 54467228.0, + "reward": -1.1593103739215849e-07, + "reward_std": 0.9916765570640564, + "rewards/rnd_reward/mean": -1.159310317078166e-07, + "rewards/rnd_reward/std": 0.9916765511035919, + "step": 2870, + "step_time": 11.231190301850438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375000149011611, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.1, + "completions/mean_length": 96.7587516784668, + "completions/mean_terminated_length": 67.1300651550293, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5091202437877653, + "epoch": 1.8835840418574232, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9646898508071899, + "kl": 0.08239567633718252, + "learning_rate": 2.9267495094833225e-08, + "loss": 0.110014808177948, + "num_tokens": 54638260.0, + "reward": -1.0073185103820493e-07, + "reward_std": 0.9914380252361298, + "rewards/rnd_reward/mean": -1.0073185103820493e-07, + "rewards/rnd_reward/std": 0.9914380311965942, + "step": 2880, + "step_time": 10.927705601369961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1087500024586916, + "completions/max_length": 384.0, + "completions/max_terminated_length": 336.5, + "completions/mean_length": 104.54500198364258, + "completions/mean_terminated_length": 70.47102432250976, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.605569988489151, + "epoch": 1.8901242642249838, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9842224717140198, + "kl": 0.07593777822330594, + "learning_rate": 2.7632439502943097e-08, + "loss": 0.09672788381576539, + "num_tokens": 54823046.0, + "reward": 4.0531157274870114e-08, + "reward_std": 0.9914953529834747, + "rewards/rnd_reward/mean": 4.053116011704105e-08, + "rewards/rnd_reward/std": 0.9914953589439393, + "step": 2890, + "step_time": 12.156373349227943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0912500012665987, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.0, + "completions/mean_length": 101.96250228881836, + "completions/mean_terminated_length": 73.72331771850585, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.55685573220253, + "epoch": 1.8966644865925442, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0599300861358643, + "kl": 0.07744721919298173, + "learning_rate": 2.5997383911052975e-08, + "loss": 0.07260736227035522, + "num_tokens": 54993921.0, + "reward": -7.45057977535879e-09, + "reward_std": 0.9911764979362487, + "rewards/rnd_reward/mean": -7.450580485901526e-09, + "rewards/rnd_reward/std": 0.9911765158176422, + "step": 2900, + "step_time": 10.672014983766712 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000160187482, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.5, + "completions/mean_length": 97.90375137329102, + "completions/mean_terminated_length": 70.14430389404296, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.7193702697753905, + "epoch": 1.9032047089601045, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4999836683273315, + "kl": 0.08308390835300088, + "learning_rate": 2.436232831916285e-08, + "loss": 0.07153651714324952, + "num_tokens": 55168059.0, + "reward": -1.7181038884928058e-07, + "reward_std": 0.991384220123291, + "rewards/rnd_reward/mean": -1.7181038458602416e-07, + "rewards/rnd_reward/std": 0.9913842082023621, + "step": 2910, + "step_time": 11.116872487938963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09875000230967998, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.8, + "completions/mean_length": 100.51500244140625, + "completions/mean_terminated_length": 69.35774002075195, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.603737047314644, + "epoch": 1.909744931327665, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0500236749649048, + "kl": 0.08141210982576012, + "learning_rate": 2.2727272727272725e-08, + "loss": 0.02487606555223465, + "num_tokens": 55350021.0, + "reward": -4.3809413696749286e-08, + "reward_std": 0.9915971457958221, + "rewards/rnd_reward/mean": -4.3809414407292026e-08, + "rewards/rnd_reward/std": 0.9915971517562866, + "step": 2920, + "step_time": 11.766088206716812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.9, + "completions/mean_length": 96.75125122070312, + "completions/mean_terminated_length": 70.08793716430664, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.6659322917461394, + "epoch": 1.9162851536952257, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1256399154663086, + "kl": 0.08670818526297808, + "learning_rate": 2.1092217135382604e-08, + "loss": 0.09936992526054382, + "num_tokens": 55522567.0, + "reward": -4.485250073571478e-08, + "reward_std": 0.9913349151611328, + "rewards/rnd_reward/mean": -4.485249220920195e-08, + "rewards/rnd_reward/std": 0.9913349032402039, + "step": 2930, + "step_time": 11.034189373115078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10750000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 319.1, + "completions/mean_length": 99.32625198364258, + "completions/mean_terminated_length": 65.10275955200196, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.6324418902397158, + "epoch": 1.9228253760627863, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9896228313446045, + "kl": 0.08653121832758189, + "learning_rate": 1.9457161543492475e-08, + "loss": 0.0832364022731781, + "num_tokens": 55692093.0, + "reward": -1.2382865079985095e-07, + "reward_std": 0.9916486382484436, + "rewards/rnd_reward/mean": -1.238286500893082e-07, + "rewards/rnd_reward/std": 0.9916486382484436, + "step": 2940, + "step_time": 11.120865160599351 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08250000048428774, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.6, + "completions/mean_length": 100.11875228881836, + "completions/mean_terminated_length": 74.31617698669433, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.6869781494140623, + "epoch": 1.9293655984303466, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.051353931427002, + "kl": 0.0818368756212294, + "learning_rate": 1.7822105951602354e-08, + "loss": 0.11521185636520385, + "num_tokens": 55868653.0, + "reward": -9.492039936276342e-08, + "reward_std": 0.9913798630237579, + "rewards/rnd_reward/mean": -9.492039865222068e-08, + "rewards/rnd_reward/std": 0.9913798570632935, + "step": 2950, + "step_time": 11.270131665398367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000141561032, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.1, + "completions/mean_length": 97.49125289916992, + "completions/mean_terminated_length": 69.66969985961914, + "completions/min_length": 2.6, + "completions/min_terminated_length": 2.6, + "entropy": 2.753763663768768, + "epoch": 1.9359058207979072, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8812336325645447, + "kl": 0.09095769878476859, + "learning_rate": 1.618705035971223e-08, + "loss": 0.043346887826919554, + "num_tokens": 56043276.0, + "reward": -6.645918113612481e-08, + "reward_std": 0.9914931774139404, + "rewards/rnd_reward/mean": -6.645917722813977e-08, + "rewards/rnd_reward/std": 0.991493159532547, + "step": 2960, + "step_time": 11.296948407590389 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07750000078231097, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.9, + "completions/mean_length": 89.08500251770019, + "completions/mean_terminated_length": 64.43022918701172, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.546912333369255, + "epoch": 1.9424460431654675, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2673183679580688, + "kl": 0.09239747198298573, + "learning_rate": 1.4551994767822105e-08, + "loss": 0.05778201222419739, + "num_tokens": 56208799.0, + "reward": -1.1771917272085374e-07, + "reward_std": 0.9914692223072052, + "rewards/rnd_reward/mean": -1.1771917272085374e-07, + "rewards/rnd_reward/std": 0.9914692461490631, + "step": 2970, + "step_time": 10.944724940578453 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000119209289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.3, + "completions/mean_length": 94.3025016784668, + "completions/mean_terminated_length": 67.02933235168457, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.585839459300041, + "epoch": 1.9489862655330281, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1477481126785278, + "kl": 0.08655477333813906, + "learning_rate": 1.291693917593198e-08, + "loss": 0.04644548296928406, + "num_tokens": 56386666.0, + "reward": -1.7434358667856031e-07, + "reward_std": 0.9915457606315613, + "rewards/rnd_reward/mean": -1.7434358596801757e-07, + "rewards/rnd_reward/std": 0.9915457487106323, + "step": 2980, + "step_time": 12.27661042441614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000275671482, + "completions/max_length": 384.0, + "completions/max_terminated_length": 332.0, + "completions/mean_length": 92.81125259399414, + "completions/mean_terminated_length": 65.69409408569337, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.7046052366495132, + "epoch": 1.9555264879005887, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1466176509857178, + "kl": 0.08695422541350126, + "learning_rate": 1.1281883584041857e-08, + "loss": 0.10825624465942382, + "num_tokens": 56555065.0, + "reward": -2.5480985499370944e-08, + "reward_std": 0.9915042757987976, + "rewards/rnd_reward/mean": -2.5480984078285475e-08, + "rewards/rnd_reward/std": 0.9915042877197265, + "step": 2990, + "step_time": 11.024760208884254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09250000044703484, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.3, + "completions/mean_length": 92.64375152587891, + "completions/mean_terminated_length": 62.92685241699219, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.619092810153961, + "epoch": 1.9620667102681493, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9762617349624634, + "kl": 0.09417410604655743, + "learning_rate": 9.646827992151732e-09, + "loss": 0.11771528720855713, + "num_tokens": 56719745.0, + "reward": -4.336237964253087e-08, + "reward_std": 0.9909454762935639, + "rewards/rnd_reward/mean": -4.336238106361634e-08, + "rewards/rnd_reward/std": 0.9909454941749573, + "step": 3000, + "step_time": 10.862733572837897 + } + ], + "logging_steps": 10, + "max_steps": 3058, + "num_input_tokens_seen": 56719745, + "num_train_epochs": 2, + "save_steps": 1000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 20, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-3000/training_args.bin b/checkpoint-3000/training_args.bin new file mode 100644 index 0000000..4c0ba1a --- /dev/null +++ b/checkpoint-3000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f7744bfe8fe7e43ca27c8620e9670a669a73959e4583e2dccebcee526f09ef4 +size 6929 diff --git a/checkpoint-3058/chat_template.jinja b/checkpoint-3058/chat_template.jinja new file mode 100644 index 0000000..28028c0 --- /dev/null +++ b/checkpoint-3058/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-3058/config.json b/checkpoint-3058/config.json new file mode 100644 index 0000000..08e4041 --- /dev/null +++ b/checkpoint-3058/config.json @@ -0,0 +1,58 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.3.0", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-3058/generation_config.json b/checkpoint-3058/generation_config.json new file mode 100644 index 0000000..8da9fd6 --- /dev/null +++ b/checkpoint-3058/generation_config.json @@ -0,0 +1,9 @@ +{ + "do_sample": false, + "eos_token_id": [ + 151643 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.3.0" +} diff --git a/checkpoint-3058/model.safetensors b/checkpoint-3058/model.safetensors new file mode 100644 index 0000000..d4314af --- /dev/null +++ b/checkpoint-3058/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bba02ff6673562d1ae9767f7aed880522c023b1bd034ca25c1c1e21b03db7846 +size 1976163472 diff --git a/checkpoint-3058/optimizer.pt b/checkpoint-3058/optimizer.pt new file mode 100644 index 0000000..589f831 --- /dev/null +++ b/checkpoint-3058/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4645458aa6bfe6553b5811bbfe5626bbfb6159a462e676cb94a87cb5c6178ab7 +size 3952509771 diff --git a/checkpoint-3058/rng_state.pth b/checkpoint-3058/rng_state.pth new file mode 100644 index 0000000..9b9ef17 --- /dev/null +++ b/checkpoint-3058/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b224aa317f06048440fdd92bab06b4c4161453e22a17e8c2818939a938e1f34 +size 14645 diff --git a/checkpoint-3058/scheduler.pt b/checkpoint-3058/scheduler.pt new file mode 100644 index 0000000..89da9c0 --- /dev/null +++ b/checkpoint-3058/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36855fd940941606d1958b822b8626509c130a869e44087571168d528d4b7743 +size 1465 diff --git a/checkpoint-3058/tokenizer.json b/checkpoint-3058/tokenizer.json new file mode 100644 index 0000000..3c72934 --- /dev/null +++ b/checkpoint-3058/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f96da3a872b5e901575b2067c744ad336c3a3d77a21584d20024557b1bd7f0 +size 11422059 diff --git a/checkpoint-3058/tokenizer_config.json b/checkpoint-3058/tokenizer_config.json new file mode 100644 index 0000000..db25db0 --- /dev/null +++ b/checkpoint-3058/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "truncation_side": "left", + "unk_token": null +} diff --git a/checkpoint-3058/trainer_state.json b/checkpoint-3058/trainer_state.json new file mode 100644 index 0000000..54f4c8a --- /dev/null +++ b/checkpoint-3058/trainer_state.json @@ -0,0 +1,8574 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 3058, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.49000000953674316, + "completions/max_length": 384.0, + "completions/max_terminated_length": 372.8, + "completions/mean_length": 291.7250061035156, + "completions/mean_terminated_length": 204.1207077026367, + "completions/min_length": 15.7, + "completions/min_terminated_length": 15.7, + "entropy": 1.5640255227684974, + "epoch": 0.006540222367560497, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5129233002662659, + "kl": 0.0005587212668615394, + "learning_rate": 4.985284499672989e-07, + "loss": -0.04552017748355865, + "num_tokens": 324400.0, + "reward": -2.107620294111712e-06, + "reward_std": 0.9998070120811462, + "rewards/rnd_reward/mean": -2.107620250058062e-06, + "rewards/rnd_reward/std": 0.9998070061206817, + "step": 10, + "step_time": 9.577712386706844 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5312500178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 374.2, + "completions/mean_length": 293.2887481689453, + "completions/mean_terminated_length": 190.75759887695312, + "completions/min_length": 18.8, + "completions/min_terminated_length": 18.8, + "entropy": 1.451838418841362, + "epoch": 0.013080444735120994, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.47466912865638733, + "kl": 0.000643831241177395, + "learning_rate": 4.968933943754087e-07, + "loss": 0.00032075121998786926, + "num_tokens": 647161.0, + "reward": -1.490712192975252e-06, + "reward_std": 0.9998168766498565, + "rewards/rnd_reward/mean": -1.4907121759222264e-06, + "rewards/rnd_reward/std": 0.9998168766498565, + "step": 20, + "step_time": 9.337269740481862 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5450000017881393, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.6, + "completions/mean_length": 298.80250549316406, + "completions/mean_terminated_length": 196.99718322753907, + "completions/min_length": 20.3, + "completions/min_terminated_length": 20.3, + "entropy": 1.5065956979990005, + "epoch": 0.01962066710268149, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5856828689575195, + "kl": 0.0006675069424090907, + "learning_rate": 4.952583387835186e-07, + "loss": 0.02223682850599289, + "num_tokens": 982698.0, + "reward": -1.2338162438396694e-07, + "reward_std": 0.9998676717281342, + "rewards/rnd_reward/mean": -1.233816414369926e-07, + "rewards/rnd_reward/std": 0.9998676896095275, + "step": 30, + "step_time": 9.97233030446805 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5125000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 371.2, + "completions/mean_length": 287.2925048828125, + "completions/mean_terminated_length": 185.24396667480468, + "completions/min_length": 10.5, + "completions/min_terminated_length": 10.5, + "entropy": 1.6533239781856537, + "epoch": 0.026160889470241987, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5665533542633057, + "kl": 0.0007774221347062848, + "learning_rate": 4.936232831916285e-07, + "loss": 0.06135546565055847, + "num_tokens": 1303862.0, + "reward": -6.012618625206301e-07, + "reward_std": 0.9998966813087463, + "rewards/rnd_reward/mean": -6.012618571915596e-07, + "rewards/rnd_reward/std": 0.9998966932296753, + "step": 40, + "step_time": 9.489675145596266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46375001072883604, + "completions/max_length": 384.0, + "completions/max_terminated_length": 377.3, + "completions/mean_length": 269.5550064086914, + "completions/mean_terminated_length": 170.72807769775392, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 1.7108511567115783, + "epoch": 0.032701111837802485, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7200465798377991, + "kl": 0.0023742157864035107, + "learning_rate": 4.919882275997384e-07, + "loss": 0.11103466749191285, + "num_tokens": 1618921.0, + "reward": -8.426606612488285e-08, + "reward_std": 0.9999245047569275, + "rewards/rnd_reward/mean": -8.426606257216918e-08, + "rewards/rnd_reward/std": 0.999924510717392, + "step": 50, + "step_time": 10.167311429185792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4012500047683716, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.1, + "completions/mean_length": 223.41500549316407, + "completions/mean_terminated_length": 115.63493576049805, + "completions/min_length": 2.8, + "completions/min_terminated_length": 2.8, + "entropy": 2.521006014943123, + "epoch": 0.03924133420536298, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.70113205909729, + "kl": 0.020581040543038397, + "learning_rate": 4.903531720078482e-07, + "loss": 0.17053321599960328, + "num_tokens": 1894883.0, + "reward": -1.1384487108578867e-07, + "reward_std": 0.9999224483966828, + "rewards/rnd_reward/mean": -1.1384487108578867e-07, + "rewards/rnd_reward/std": 0.9999224364757537, + "step": 60, + "step_time": 9.91588868314866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3012500047683716, + "completions/max_length": 384.0, + "completions/max_terminated_length": 373.8, + "completions/mean_length": 183.6437515258789, + "completions/mean_terminated_length": 97.22235260009765, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7396299123764036, + "epoch": 0.04578155657292348, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7759609222412109, + "kl": 0.0463468745816499, + "learning_rate": 4.887181164159581e-07, + "loss": 0.14934378862380981, + "num_tokens": 2138653.0, + "reward": -4.6938660425155375e-08, + "reward_std": 0.9998835921287537, + "rewards/rnd_reward/mean": -4.6938658648798536e-08, + "rewards/rnd_reward/std": 0.9998836040496826, + "step": 70, + "step_time": 10.02495912197046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2825000062584877, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.8, + "completions/mean_length": 176.12750549316405, + "completions/mean_terminated_length": 94.6318603515625, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7325789600610735, + "epoch": 0.052321778940483975, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8319016695022583, + "kl": 0.04594853101298213, + "learning_rate": 4.87083060824068e-07, + "loss": 0.018015310168266296, + "num_tokens": 2376420.0, + "reward": -4.187226378959963e-08, + "reward_std": 0.9998487293720245, + "rewards/rnd_reward/mean": -4.1872263079056894e-08, + "rewards/rnd_reward/std": 0.9998487293720245, + "step": 80, + "step_time": 9.701193272857926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34750000238418577, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.4, + "completions/mean_length": 207.52375335693358, + "completions/mean_terminated_length": 114.01394119262696, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.342811721563339, + "epoch": 0.058862001308044476, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5935761332511902, + "kl": 0.019986469263676553, + "learning_rate": 4.854480052321779e-07, + "loss": 0.010216768831014633, + "num_tokens": 2641029.0, + "reward": -2.0414591972439666e-08, + "reward_std": 0.9998277425765991, + "rewards/rnd_reward/mean": -2.0414591972439666e-08, + "rewards/rnd_reward/std": 0.9998277485370636, + "step": 90, + "step_time": 10.468820211617276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4150000065565109, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.7, + "completions/mean_length": 237.4625045776367, + "completions/mean_terminated_length": 134.2795196533203, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "entropy": 2.1418308436870577, + "epoch": 0.06540222367560497, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5613327622413635, + "kl": 0.006808469316456467, + "learning_rate": 4.838129496402878e-07, + "loss": 0.04351484477519989, + "num_tokens": 2930639.0, + "reward": 8.493661918240747e-09, + "reward_std": 0.9997802138328552, + "rewards/rnd_reward/mean": 8.493660852426644e-09, + "rewards/rnd_reward/std": 0.9997801959514618, + "step": 100, + "step_time": 10.307341906265355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.38875000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 370.8, + "completions/mean_length": 234.60625610351562, + "completions/mean_terminated_length": 139.61359405517578, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.0046547770500185, + "epoch": 0.07194244604316546, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7055774927139282, + "kl": 0.006759516580495983, + "learning_rate": 4.821778940483976e-07, + "loss": 0.0377700001001358, + "num_tokens": 3215169.0, + "reward": -2.920627686364696e-08, + "reward_std": 0.9997233927249909, + "rewards/rnd_reward/mean": -2.9206276153104228e-08, + "rewards/rnd_reward/std": 0.9997233927249909, + "step": 110, + "step_time": 9.7759924543323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.38250001072883605, + "completions/max_length": 384.0, + "completions/max_terminated_length": 364.8, + "completions/mean_length": 221.19375457763672, + "completions/mean_terminated_length": 120.60059127807617, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.0983517467975616, + "epoch": 0.07848266841072596, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5727905631065369, + "kl": 0.009989026037510484, + "learning_rate": 4.805428384565075e-07, + "loss": 0.09626840949058532, + "num_tokens": 3496559.0, + "reward": 1.654028882747127e-08, + "reward_std": 0.9997243106365203, + "rewards/rnd_reward/mean": 1.654028882747127e-08, + "rewards/rnd_reward/std": 0.9997243225574494, + "step": 120, + "step_time": 9.981053173542023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34750000536441805, + "completions/max_length": 384.0, + "completions/max_terminated_length": 370.4, + "completions/mean_length": 207.67625274658204, + "completions/mean_terminated_length": 114.74080200195313, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2568650126457213, + "epoch": 0.08502289077828647, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8493079543113708, + "kl": 0.014032764779403805, + "learning_rate": 4.789077828646174e-07, + "loss": -0.015352001786231995, + "num_tokens": 3759245.0, + "reward": -5.06639494801675e-09, + "reward_std": 0.9996613085269928, + "rewards/rnd_reward/mean": -5.066393526931279e-09, + "rewards/rnd_reward/std": 0.9996613144874573, + "step": 130, + "step_time": 10.069910727906972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3137500062584877, + "completions/max_length": 384.0, + "completions/max_terminated_length": 373.4, + "completions/mean_length": 196.95000305175782, + "completions/mean_terminated_length": 111.34912643432617, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.305853408575058, + "epoch": 0.09156311314584696, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7982010245323181, + "kl": 0.016896341042593122, + "learning_rate": 4.772727272727273e-07, + "loss": 0.02038821578025818, + "num_tokens": 4010260.0, + "reward": -6.914138950264714e-08, + "reward_std": 0.9996093094348908, + "rewards/rnd_reward/mean": -6.914138808156168e-08, + "rewards/rnd_reward/std": 0.9996093094348908, + "step": 140, + "step_time": 9.98037786844652 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3700000047683716, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.3, + "completions/mean_length": 212.5687515258789, + "completions/mean_terminated_length": 111.88678207397462, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2448606371879576, + "epoch": 0.09810333551340746, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6012561321258545, + "kl": 0.01302192104049027, + "learning_rate": 4.756376716808371e-07, + "loss": 0.041692107915878296, + "num_tokens": 4270360.0, + "reward": -6.854534007061375e-09, + "reward_std": 0.9995533764362335, + "rewards/rnd_reward/mean": -6.854535428146846e-09, + "rewards/rnd_reward/std": 0.9995533883571625, + "step": 150, + "step_time": 9.479559903079643 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3687500059604645, + "completions/max_length": 384.0, + "completions/max_terminated_length": 373.0, + "completions/mean_length": 221.59500274658203, + "completions/mean_terminated_length": 127.51908645629882, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2110244154930117, + "epoch": 0.10464355788096795, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7059744000434875, + "kl": 0.009931368473917245, + "learning_rate": 4.74002616088947e-07, + "loss": 0.0420710027217865, + "num_tokens": 4538671.0, + "reward": -6.467104074481256e-08, + "reward_std": 0.9995002686977387, + "rewards/rnd_reward/mean": -6.467103932372709e-08, + "rewards/rnd_reward/std": 0.9995002567768096, + "step": 160, + "step_time": 9.463879713765346 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.38750000596046447, + "completions/max_length": 384.0, + "completions/max_terminated_length": 366.5, + "completions/mean_length": 224.09500274658203, + "completions/mean_terminated_length": 123.22678070068359, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.105703830718994, + "epoch": 0.11118378024852844, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6353176236152649, + "kl": 0.009572444634977728, + "learning_rate": 4.723675604970569e-07, + "loss": 0.057880669832229614, + "num_tokens": 4810167.0, + "reward": -1.3113021868349506e-08, + "reward_std": 0.9995246767997742, + "rewards/rnd_reward/mean": -1.3113020802535403e-08, + "rewards/rnd_reward/std": 0.9995246648788452, + "step": 170, + "step_time": 9.689974679402075 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3425000041723251, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.8, + "completions/mean_length": 207.19125213623047, + "completions/mean_terminated_length": 115.52461242675781, + "completions/min_length": 2.7, + "completions/min_terminated_length": 2.7, + "entropy": 2.3119001001119615, + "epoch": 0.11772400261608895, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6543518900871277, + "kl": 0.01470447750762105, + "learning_rate": 4.7073250490516674e-07, + "loss": 0.05609138607978821, + "num_tokens": 5069705.0, + "reward": -3.21865089580875e-08, + "reward_std": 0.9993573248386383, + "rewards/rnd_reward/mean": -3.218650753700203e-08, + "rewards/rnd_reward/std": 0.9993573546409606, + "step": 180, + "step_time": 9.860351232904941 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.29625000804662704, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.4, + "completions/mean_length": 186.85875396728517, + "completions/mean_terminated_length": 103.74458618164063, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.3917055457830427, + "epoch": 0.12426422498364945, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6832460165023804, + "kl": 0.02189517344813794, + "learning_rate": 4.690974493132766e-07, + "loss": 0.0940218448638916, + "num_tokens": 5313337.0, + "reward": -2.801418297337932e-08, + "reward_std": 0.9993866741657257, + "rewards/rnd_reward/mean": -2.801418386155774e-08, + "rewards/rnd_reward/std": 0.9993866682052612, + "step": 190, + "step_time": 10.32580083089415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3087500035762787, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.0, + "completions/mean_length": 195.6250030517578, + "completions/mean_terminated_length": 111.06491088867188, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.365246319770813, + "epoch": 0.13080444735120994, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7736798524856567, + "kl": 0.018843610235489906, + "learning_rate": 4.674623937213865e-07, + "loss": 0.09085838794708252, + "num_tokens": 5564832.0, + "reward": -3.784895152847412e-08, + "reward_std": 0.9993082165718079, + "rewards/rnd_reward/mean": -3.7848948331031805e-08, + "rewards/rnd_reward/std": 0.9993082165718079, + "step": 200, + "step_time": 10.074973361729644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2800000086426735, + "completions/max_length": 384.0, + "completions/max_terminated_length": 364.2, + "completions/mean_length": 183.75125274658203, + "completions/mean_terminated_length": 105.59480438232421, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3340355485677717, + "epoch": 0.13734466971877043, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7310492992401123, + "kl": 0.019957996485754848, + "learning_rate": 4.6582733812949637e-07, + "loss": 0.09393368363380432, + "num_tokens": 5808593.0, + "reward": -7.003545956507651e-08, + "reward_std": 0.9992451012134552, + "rewards/rnd_reward/mean": -7.003545814399103e-08, + "rewards/rnd_reward/std": 0.9992450892925262, + "step": 210, + "step_time": 9.864037699718029 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2600000038743019, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.5, + "completions/mean_length": 182.30500335693358, + "completions/mean_terminated_length": 111.45323791503907, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3039004147052764, + "epoch": 0.14388489208633093, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7140376567840576, + "kl": 0.020338369254022836, + "learning_rate": 4.641922825376063e-07, + "loss": 0.03314664363861084, + "num_tokens": 6039847.0, + "reward": -4.202127490415819e-08, + "reward_std": 0.9992108285427094, + "rewards/rnd_reward/mean": -4.202127419361546e-08, + "rewards/rnd_reward/std": 0.9992108345031738, + "step": 220, + "step_time": 9.367073794314638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2750000059604645, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.2, + "completions/mean_length": 176.20750274658204, + "completions/mean_terminated_length": 97.27564849853516, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.2686150133609773, + "epoch": 0.15042511445389142, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8464763760566711, + "kl": 0.021972918952815234, + "learning_rate": 4.6255722694571613e-07, + "loss": 0.04784287512302399, + "num_tokens": 6270078.0, + "reward": -1.1757016302738065e-07, + "reward_std": 0.9992310464382171, + "rewards/rnd_reward/mean": -1.1757016231683792e-07, + "rewards/rnd_reward/std": 0.9992310464382171, + "step": 230, + "step_time": 9.406287361658178 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23750000298023224, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.0, + "completions/mean_length": 164.8237548828125, + "completions/mean_terminated_length": 96.10623779296876, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4642169535160066, + "epoch": 0.15696533682145192, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.874134361743927, + "kl": 0.027495546033605933, + "learning_rate": 4.60922171353826e-07, + "loss": 0.09257912039756774, + "num_tokens": 6507582.0, + "reward": -2.2351741080228748e-08, + "reward_std": 0.9989596605300903, + "rewards/rnd_reward/mean": -2.2351743211856957e-08, + "rewards/rnd_reward/std": 0.9989596605300903, + "step": 240, + "step_time": 10.88162840725854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.26375000476837157, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.7, + "completions/mean_length": 179.21375274658203, + "completions/mean_terminated_length": 105.97551345825195, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.3090430706739427, + "epoch": 0.16350555918901244, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7480731010437012, + "kl": 0.022477348684333265, + "learning_rate": 4.592871157619359e-07, + "loss": 0.0661897599697113, + "num_tokens": 6746233.0, + "reward": -8.285045609568442e-08, + "reward_std": 0.9989501237869263, + "rewards/rnd_reward/mean": -8.285045680622715e-08, + "rewards/rnd_reward/std": 0.9989501178264618, + "step": 250, + "step_time": 9.837229801248759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23000000417232513, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.9, + "completions/mean_length": 166.79125366210937, + "completions/mean_terminated_length": 101.81034469604492, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3430103093385695, + "epoch": 0.17004578155657293, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7986056804656982, + "kl": 0.026151024643331765, + "learning_rate": 4.5765206017004576e-07, + "loss": 0.1205091118812561, + "num_tokens": 6976091.0, + "reward": -7.808208817294826e-08, + "reward_std": 0.9988620162010193, + "rewards/rnd_reward/mean": -7.808208604132006e-08, + "rewards/rnd_reward/std": 0.9988620281219482, + "step": 260, + "step_time": 9.811681813630276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23125000298023224, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.4, + "completions/mean_length": 162.19500274658202, + "completions/mean_terminated_length": 95.31229019165039, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2510120779275895, + "epoch": 0.17658600392413343, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6842794418334961, + "kl": 0.025852510519325734, + "learning_rate": 4.5601700457815564e-07, + "loss": 0.07729284763336182, + "num_tokens": 7199527.0, + "reward": -7.122755136812487e-08, + "reward_std": 0.9989409029483796, + "rewards/rnd_reward/mean": -7.122754994703939e-08, + "rewards/rnd_reward/std": 0.9989409029483796, + "step": 270, + "step_time": 9.695072391931898 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.24750000536441802, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.6, + "completions/mean_length": 165.32125244140624, + "completions/mean_terminated_length": 93.36570816040039, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2813161820173264, + "epoch": 0.18312622629169392, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8117495775222778, + "kl": 0.02381342255976051, + "learning_rate": 4.543819489862655e-07, + "loss": 0.08756870031356812, + "num_tokens": 7423314.0, + "reward": -3.010034372863402e-08, + "reward_std": 0.9987568318843841, + "rewards/rnd_reward/mean": -3.0100344261541065e-08, + "rewards/rnd_reward/std": 0.9987568318843841, + "step": 280, + "step_time": 9.538212020602078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.27250000536441804, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.5, + "completions/mean_length": 171.89125366210936, + "completions/mean_terminated_length": 91.7330307006836, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.2805801182985306, + "epoch": 0.18966644865925442, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7356550693511963, + "kl": 0.021292086993344127, + "learning_rate": 4.527468933943754e-07, + "loss": 0.0787746787071228, + "num_tokens": 7649017.0, + "reward": -2.2798775312082853e-08, + "reward_std": 0.9987796723842621, + "rewards/rnd_reward/mean": -2.279877513444717e-08, + "rewards/rnd_reward/std": 0.99877969622612, + "step": 290, + "step_time": 9.4448504867265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.4, + "completions/mean_length": 172.06375274658203, + "completions/mean_terminated_length": 101.12049560546875, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.062102970480919, + "epoch": 0.1962066710268149, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.678175151348114, + "kl": 0.021230081818066536, + "learning_rate": 4.511118378024852e-07, + "loss": 0.10422745943069459, + "num_tokens": 7893248.0, + "reward": 1.7285347131412722e-08, + "reward_std": 0.9987858593463897, + "rewards/rnd_reward/mean": 1.7285347841955455e-08, + "rewards/rnd_reward/std": 0.9987858593463897, + "step": 300, + "step_time": 11.542911725770683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23125, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.3, + "completions/mean_length": 155.94375305175782, + "completions/mean_terminated_length": 87.63837699890136, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.110625982284546, + "epoch": 0.2027468933943754, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8758037090301514, + "kl": 0.025008385512046517, + "learning_rate": 4.4947678221059515e-07, + "loss": 0.0913781225681305, + "num_tokens": 8105838.0, + "reward": -6.72042411054008e-08, + "reward_std": 0.9986489951610565, + "rewards/rnd_reward/mean": -6.720423826322985e-08, + "rewards/rnd_reward/std": 0.9986490130424499, + "step": 310, + "step_time": 9.397380503430032 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19500000327825545, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.1, + "completions/mean_length": 141.38000259399413, + "completions/mean_terminated_length": 82.59085083007812, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.311973440647125, + "epoch": 0.2092871157619359, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9461565613746643, + "kl": 0.03633355665951967, + "learning_rate": 4.4784172661870503e-07, + "loss": 0.08147850632667542, + "num_tokens": 8325122.0, + "reward": -9.238719300697084e-09, + "reward_std": 0.9986236810684204, + "rewards/rnd_reward/mean": -9.238719300697084e-09, + "rewards/rnd_reward/std": 0.9986236929893494, + "step": 320, + "step_time": 11.082472172076814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.20000000447034835, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.7, + "completions/mean_length": 141.80000228881835, + "completions/mean_terminated_length": 81.46792144775391, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3696460843086244, + "epoch": 0.2158273381294964, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8129741549491882, + "kl": 0.034535441920161246, + "learning_rate": 4.462066710268149e-07, + "loss": 0.06801753044128418, + "num_tokens": 8535267.0, + "reward": -8.24779284869237e-08, + "reward_std": 0.9986066579818725, + "rewards/rnd_reward/mean": -8.24779284869237e-08, + "rewards/rnd_reward/std": 0.9986066579818725, + "step": 330, + "step_time": 9.98545232352335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2100000038743019, + "completions/max_length": 384.0, + "completions/max_terminated_length": 364.4, + "completions/mean_length": 141.52500228881837, + "completions/mean_terminated_length": 77.51832160949706, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3277006804943086, + "epoch": 0.2223675604970569, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8912848234176636, + "kl": 0.0404030725825578, + "learning_rate": 4.445716154349248e-07, + "loss": 0.09486417174339294, + "num_tokens": 8741817.0, + "reward": -1.6838312300038183e-08, + "reward_std": 0.9984602987766266, + "rewards/rnd_reward/mean": -1.683831301058092e-08, + "rewards/rnd_reward/std": 0.9984602928161621, + "step": 340, + "step_time": 9.806188964121976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1912500038743019, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.6, + "completions/mean_length": 139.37500076293946, + "completions/mean_terminated_length": 81.56527061462403, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.3621277123689652, + "epoch": 0.2289077828646174, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9253888130187988, + "kl": 0.03642263817600906, + "learning_rate": 4.429365598430346e-07, + "loss": 0.09143019318580628, + "num_tokens": 8945182.0, + "reward": -2.4735924242236252e-08, + "reward_std": 0.9984571993350982, + "rewards/rnd_reward/mean": -2.4735929926578137e-08, + "rewards/rnd_reward/std": 0.9984571933746338, + "step": 350, + "step_time": 9.939818266802467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18750000447034837, + "completions/max_length": 384.0, + "completions/max_terminated_length": 371.7, + "completions/mean_length": 138.01250076293945, + "completions/mean_terminated_length": 81.37003707885742, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.270884335041046, + "epoch": 0.2354480052321779, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8366574645042419, + "kl": 0.03530883879866451, + "learning_rate": 4.4130150425114454e-07, + "loss": 0.10644701719284058, + "num_tokens": 9147587.0, + "reward": -1.15185978266652e-07, + "reward_std": 0.9983633041381836, + "rewards/rnd_reward/mean": -1.1518597684556652e-07, + "rewards/rnd_reward/std": 0.9983633100986481, + "step": 360, + "step_time": 9.722514500608668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17000000029802323, + "completions/max_length": 384.0, + "completions/max_terminated_length": 366.5, + "completions/mean_length": 130.59125213623048, + "completions/mean_terminated_length": 79.17214775085449, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3677036464214325, + "epoch": 0.2419882275997384, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.964614987373352, + "kl": 0.043067421624436976, + "learning_rate": 4.3966644865925437e-07, + "loss": 0.08349081873893738, + "num_tokens": 9340010.0, + "reward": -3.0696392450124675e-08, + "reward_std": 0.9982355415821076, + "rewards/rnd_reward/mean": -3.0696389607953736e-08, + "rewards/rnd_reward/std": 0.9982355415821076, + "step": 370, + "step_time": 9.710164245427586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1887500010430813, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.2, + "completions/mean_length": 138.41875305175782, + "completions/mean_terminated_length": 81.47324066162109, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.258895403146744, + "epoch": 0.2485284499672989, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7839703559875488, + "kl": 0.039375259960070255, + "learning_rate": 4.3803139306736425e-07, + "loss": 0.08618925213813781, + "num_tokens": 9545205.0, + "reward": -1.0460615040841503e-07, + "reward_std": 0.998303747177124, + "rewards/rnd_reward/mean": -1.046061518295005e-07, + "rewards/rnd_reward/std": 0.9983037531375885, + "step": 380, + "step_time": 9.776164471404627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1625000037252903, + "completions/max_length": 384.0, + "completions/max_terminated_length": 342.9, + "completions/mean_length": 123.14125137329101, + "completions/mean_terminated_length": 72.26187210083008, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2043879717588424, + "epoch": 0.25506867233485936, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.013533353805542, + "kl": 0.045199727965518834, + "learning_rate": 4.363963374754742e-07, + "loss": 0.08801217675209046, + "num_tokens": 9732323.0, + "reward": -1.3418495639072603e-07, + "reward_std": 0.9981010854244232, + "rewards/rnd_reward/mean": -1.3418495496964057e-07, + "rewards/rnd_reward/std": 0.9981010735034943, + "step": 390, + "step_time": 9.470399480173365 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16500000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.9, + "completions/mean_length": 126.54125213623047, + "completions/mean_terminated_length": 75.62271652221679, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.1946812689304354, + "epoch": 0.2616088947024199, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9289289712905884, + "kl": 0.043896416388452056, + "learning_rate": 4.34761281883584e-07, + "loss": 0.06013514399528504, + "num_tokens": 9923251.0, + "reward": -1.1950731622079046e-07, + "reward_std": 0.9980400562286377, + "rewards/rnd_reward/mean": -1.19507314799705e-07, + "rewards/rnd_reward/std": 0.9980400741100312, + "step": 400, + "step_time": 9.578276086249389 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19000000283122062, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.4, + "completions/mean_length": 142.2825019836426, + "completions/mean_terminated_length": 85.19309997558594, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3042476952075956, + "epoch": 0.2681491170699804, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8114489316940308, + "kl": 0.039571146154776214, + "learning_rate": 4.331262262916939e-07, + "loss": 0.08236328959465027, + "num_tokens": 10132682.0, + "reward": -1.4752151500374566e-08, + "reward_std": 0.9982209146022797, + "rewards/rnd_reward/mean": -1.4752152210917303e-08, + "rewards/rnd_reward/std": 0.9982209086418152, + "step": 410, + "step_time": 9.807740885205567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.20625000149011613, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 149.2062530517578, + "completions/mean_terminated_length": 88.13671951293945, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.38392159640789, + "epoch": 0.27468933943754087, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7539530396461487, + "kl": 0.03540583737194538, + "learning_rate": 4.3149117069980376e-07, + "loss": 0.08869684934616089, + "num_tokens": 10345522.0, + "reward": -1.3530254712179611e-07, + "reward_std": 0.9981609523296356, + "rewards/rnd_reward/mean": -1.3530254356908243e-07, + "rewards/rnd_reward/std": 0.9981609761714936, + "step": 420, + "step_time": 9.619695667061023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18750000223517418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.4, + "completions/mean_length": 139.9700035095215, + "completions/mean_terminated_length": 83.52344436645508, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.225811779499054, + "epoch": 0.2812295618051014, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7882169485092163, + "kl": 0.031479011243209246, + "learning_rate": 4.2985611510791364e-07, + "loss": 0.09766495227813721, + "num_tokens": 10563093.0, + "reward": -1.4021993024471158e-07, + "reward_std": 0.998052579164505, + "rewards/rnd_reward/mean": -1.4021992598145516e-07, + "rewards/rnd_reward/std": 0.9980525553226471, + "step": 430, + "step_time": 11.129851599340327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18875000178813933, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.8, + "completions/mean_length": 143.57625198364258, + "completions/mean_terminated_length": 87.31694107055664, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.1123261958360673, + "epoch": 0.28776978417266186, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7737316489219666, + "kl": 0.0280194781254977, + "learning_rate": 4.2822105951602357e-07, + "loss": 0.10439527034759521, + "num_tokens": 10766159.0, + "reward": -4.2766332430943524e-08, + "reward_std": 0.998040771484375, + "rewards/rnd_reward/mean": -4.276633118749373e-08, + "rewards/rnd_reward/std": 0.998040747642517, + "step": 440, + "step_time": 9.481722956942395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16625000312924385, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.1, + "completions/mean_length": 131.25125427246093, + "completions/mean_terminated_length": 80.89927139282227, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.2791686803102493, + "epoch": 0.2943100065402224, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.834428608417511, + "kl": 0.04166741594672203, + "learning_rate": 4.265860039241334e-07, + "loss": 0.07646016478538513, + "num_tokens": 10965315.0, + "reward": -1.0013580258316779e-07, + "reward_std": 0.998044753074646, + "rewards/rnd_reward/mean": -1.0013580116208232e-07, + "rewards/rnd_reward/std": 0.9980447709560394, + "step": 450, + "step_time": 10.756261567212642 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13750000298023224, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.9, + "completions/mean_length": 118.16625137329102, + "completions/mean_terminated_length": 75.56828384399414, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.3162699341773987, + "epoch": 0.30085022890778285, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1823770999908447, + "kl": 0.04914104142226279, + "learning_rate": 4.249509483322433e-07, + "loss": 0.09161773324012756, + "num_tokens": 11151658.0, + "reward": 1.4454126073815132e-08, + "reward_std": 0.9979428768157959, + "rewards/rnd_reward/mean": 1.4454126073815132e-08, + "rewards/rnd_reward/std": 0.9979428887367249, + "step": 460, + "step_time": 9.690541778854094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625000596046448, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.0, + "completions/mean_length": 129.17875442504882, + "completions/mean_terminated_length": 81.90925521850586, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.250484657287598, + "epoch": 0.30739045127534337, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7849432229995728, + "kl": 0.044866825267672536, + "learning_rate": 4.2331589274035315e-07, + "loss": 0.1366190195083618, + "num_tokens": 11350266.0, + "reward": -5.528331037396583e-08, + "reward_std": 0.997910988330841, + "rewards/rnd_reward/mean": -5.528330859760899e-08, + "rewards/rnd_reward/std": 0.997911012172699, + "step": 470, + "step_time": 10.094013441796415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14375000149011613, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.1, + "completions/mean_length": 121.7000015258789, + "completions/mean_terminated_length": 77.64777450561523, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.356913906335831, + "epoch": 0.31393067364290383, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7464061975479126, + "kl": 0.047594004729762676, + "learning_rate": 4.2168083714846303e-07, + "loss": 0.093714839220047, + "num_tokens": 11546191.0, + "reward": -2.1904707381281696e-08, + "reward_std": 0.9980335652828216, + "rewards/rnd_reward/mean": -2.1904707381281696e-08, + "rewards/rnd_reward/std": 0.9980335593223572, + "step": 480, + "step_time": 10.49183724487666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1312500022351742, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.0, + "completions/mean_length": 110.71375274658203, + "completions/mean_terminated_length": 69.09722747802735, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.3989109843969345, + "epoch": 0.32047089601046436, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9596973061561584, + "kl": 0.059090607427060605, + "learning_rate": 4.200457815565729e-07, + "loss": 0.061995261907577516, + "num_tokens": 11729672.0, + "reward": -8.441507906020718e-08, + "reward_std": 0.9977342307567596, + "rewards/rnd_reward/mean": -8.441507870493581e-08, + "rewards/rnd_reward/std": 0.9977342247962951, + "step": 490, + "step_time": 9.824798973021098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000193715094, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.2, + "completions/mean_length": 110.2125015258789, + "completions/mean_terminated_length": 69.26982688903809, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.335902637243271, + "epoch": 0.3270111183780249, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0347245931625366, + "kl": 0.06009262367151678, + "learning_rate": 4.184107259646828e-07, + "loss": 0.011750234663486481, + "num_tokens": 11908552.0, + "reward": -7.688999374977356e-08, + "reward_std": 0.9977168381214142, + "rewards/rnd_reward/mean": -7.688999019705988e-08, + "rewards/rnd_reward/std": 0.9977168321609498, + "step": 500, + "step_time": 9.731353901792318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1450000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.2, + "completions/mean_length": 118.08375244140625, + "completions/mean_terminated_length": 73.00995483398438, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.557803976535797, + "epoch": 0.33355134074558535, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9074205160140991, + "kl": 0.06632269411347806, + "learning_rate": 4.1677567037279267e-07, + "loss": 0.08373542428016663, + "num_tokens": 12103254.0, + "reward": -1.1697411828048843e-07, + "reward_std": 0.9978465795516968, + "rewards/rnd_reward/mean": -1.1697411828048843e-07, + "rewards/rnd_reward/std": 0.9978465795516968, + "step": 510, + "step_time": 10.207280423282645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15875000208616258, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.2, + "completions/mean_length": 123.88500137329102, + "completions/mean_terminated_length": 74.63431396484376, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.437187620997429, + "epoch": 0.34009156311314587, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8857874870300293, + "kl": 0.06048318287357688, + "learning_rate": 4.151406147809025e-07, + "loss": 0.037069684267044066, + "num_tokens": 12302092.0, + "reward": -3.874301874873254e-08, + "reward_std": 0.9979513227939606, + "rewards/rnd_reward/mean": -3.874301874873254e-08, + "rewards/rnd_reward/std": 0.9979513168334961, + "step": 520, + "step_time": 10.34260532106273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17000000253319741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.1, + "completions/mean_length": 132.52125396728516, + "completions/mean_terminated_length": 80.95349197387695, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4997961223125458, + "epoch": 0.34663178548070633, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9028640985488892, + "kl": 0.05214450396597385, + "learning_rate": 4.135055591890124e-07, + "loss": 0.07458721995353698, + "num_tokens": 12499434.0, + "reward": -2.2247433655309124e-07, + "reward_std": 0.9978251278400421, + "rewards/rnd_reward/mean": -2.2247433655309124e-07, + "rewards/rnd_reward/std": 0.9978251218795776, + "step": 530, + "step_time": 9.664499558508396 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15125000178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.4, + "completions/mean_length": 120.7087516784668, + "completions/mean_terminated_length": 73.79461898803712, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.547806602716446, + "epoch": 0.35317200784826686, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0128388404846191, + "kl": 0.05931666204705834, + "learning_rate": 4.118705035971223e-07, + "loss": 0.11692253351211548, + "num_tokens": 12683096.0, + "reward": -8.344649238267721e-09, + "reward_std": 0.9977494418621063, + "rewards/rnd_reward/mean": -8.344651369895927e-09, + "rewards/rnd_reward/std": 0.9977494418621063, + "step": 540, + "step_time": 9.593502512201667 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13500000387430192, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.6, + "completions/mean_length": 111.18750305175782, + "completions/mean_terminated_length": 68.47658958435059, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.681893822550774, + "epoch": 0.3597122302158273, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1282111406326294, + "kl": 0.06902076229453087, + "learning_rate": 4.1023544800523213e-07, + "loss": 0.023469969630241394, + "num_tokens": 12862906.0, + "reward": -5.677342400645102e-08, + "reward_std": 0.9978136360645294, + "rewards/rnd_reward/mean": -5.6773422585365554e-08, + "rewards/rnd_reward/std": 0.997813630104065, + "step": 550, + "step_time": 9.778036265238189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12000000327825547, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.0, + "completions/mean_length": 104.53875045776367, + "completions/mean_terminated_length": 66.3183536529541, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5951172381639482, + "epoch": 0.36625245258338784, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1439512968063354, + "kl": 0.07057485273107886, + "learning_rate": 4.0860039241334206e-07, + "loss": 0.1278893232345581, + "num_tokens": 13049132.0, + "reward": -1.1786818543413347e-07, + "reward_std": 0.9977734804153442, + "rewards/rnd_reward/mean": -1.17868184013048e-07, + "rewards/rnd_reward/std": 0.9977734982967377, + "step": 560, + "step_time": 10.389736549765804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10500000044703484, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.3, + "completions/mean_length": 99.66875076293945, + "completions/mean_terminated_length": 66.29649963378907, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6503134578466416, + "epoch": 0.3727926749509483, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0748413801193237, + "kl": 0.08892364026978612, + "learning_rate": 4.069653368214519e-07, + "loss": 0.13109040260314941, + "num_tokens": 13221157.0, + "reward": -1.4342367933295462e-07, + "reward_std": 0.997740238904953, + "rewards/rnd_reward/mean": -1.4342367791186916e-07, + "rewards/rnd_reward/std": 0.9977402448654175, + "step": 570, + "step_time": 9.71618521134369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12250000312924385, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 110.21125030517578, + "completions/mean_terminated_length": 71.826904296875, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.702208098769188, + "epoch": 0.37933289731850883, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0691598653793335, + "kl": 0.07306001400575042, + "learning_rate": 4.053302812295618e-07, + "loss": 0.09580587744712829, + "num_tokens": 13409746.0, + "reward": -1.6450882269758438e-07, + "reward_std": 0.9975189864635468, + "rewards/rnd_reward/mean": -1.6450882198704164e-07, + "rewards/rnd_reward/std": 0.9975189983844757, + "step": 580, + "step_time": 10.602098648622633 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13250000216066837, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.2, + "completions/mean_length": 118.48125228881835, + "completions/mean_terminated_length": 77.98408432006836, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5411072075366974, + "epoch": 0.3858731196860693, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8738215565681458, + "kl": 0.06685408288612962, + "learning_rate": 4.0369522563767164e-07, + "loss": 0.0531582236289978, + "num_tokens": 13603691.0, + "reward": -8.791685381925874e-08, + "reward_std": 0.9976041257381439, + "rewards/rnd_reward/mean": -8.791684955600231e-08, + "rewards/rnd_reward/std": 0.9976041257381439, + "step": 590, + "step_time": 10.606248078867793 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13375000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.1, + "completions/mean_length": 120.9775001525879, + "completions/mean_terminated_length": 80.35308418273925, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.454717567563057, + "epoch": 0.3924133420536298, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8597258925437927, + "kl": 0.06251584179699421, + "learning_rate": 4.020601700457815e-07, + "loss": 0.10086486339569092, + "num_tokens": 13793088.0, + "reward": -4.872679788547885e-08, + "reward_std": 0.997677344083786, + "rewards/rnd_reward/mean": -4.872679859602158e-08, + "rewards/rnd_reward/std": 0.997677332162857, + "step": 600, + "step_time": 9.838808755297213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13875000327825546, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.2, + "completions/mean_length": 112.62250213623047, + "completions/mean_terminated_length": 69.04776878356934, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5064212560653685, + "epoch": 0.39895356442119034, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0397368669509888, + "kl": 0.06775042302906513, + "learning_rate": 4.0042511445389145e-07, + "loss": 0.14560239315032958, + "num_tokens": 13975906.0, + "reward": -1.70022253698221e-07, + "reward_std": 0.997452187538147, + "rewards/rnd_reward/mean": -1.7002224872442184e-07, + "rewards/rnd_reward/std": 0.9974521696567535, + "step": 610, + "step_time": 9.711915594828316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11000000387430191, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.5, + "completions/mean_length": 105.6062515258789, + "completions/mean_terminated_length": 71.07123832702636, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.531564822793007, + "epoch": 0.4054937867887508, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8839622139930725, + "kl": 0.07682965183630586, + "learning_rate": 3.987900588620013e-07, + "loss": 0.06944795846939086, + "num_tokens": 14154656.0, + "reward": 5.662443069986978e-09, + "reward_std": 0.9975684463977814, + "rewards/rnd_reward/mean": 5.662442181808558e-09, + "rewards/rnd_reward/std": 0.9975684404373169, + "step": 620, + "step_time": 10.306970553193242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10375000201165677, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.6, + "completions/mean_length": 106.93500137329102, + "completions/mean_terminated_length": 74.97318267822266, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.859010672569275, + "epoch": 0.41203400915631133, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.103442907333374, + "kl": 0.08963548736646772, + "learning_rate": 3.9715500327011115e-07, + "loss": 0.033435261249542235, + "num_tokens": 14327599.0, + "reward": -1.2174248684715394e-07, + "reward_std": 0.9974590480327606, + "rewards/rnd_reward/mean": -1.2174248613661122e-07, + "rewards/rnd_reward/std": 0.9974590301513672, + "step": 630, + "step_time": 9.607878714823164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12625000327825547, + "completions/max_length": 384.0, + "completions/max_terminated_length": 343.2, + "completions/mean_length": 108.95125045776368, + "completions/mean_terminated_length": 69.31592407226563, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.823798879981041, + "epoch": 0.4185742315238718, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1582435369491577, + "kl": 0.09182370994240045, + "learning_rate": 3.9551994767822103e-07, + "loss": 0.06608177423477173, + "num_tokens": 14513650.0, + "reward": -1.3239681819765537e-07, + "reward_std": 0.9976047217845917, + "rewards/rnd_reward/mean": -1.3239681961874084e-07, + "rewards/rnd_reward/std": 0.9976047277450562, + "step": 640, + "step_time": 10.307288937154226 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11000000163912774, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.8, + "completions/mean_length": 107.63500213623047, + "completions/mean_terminated_length": 73.39098167419434, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.695296034216881, + "epoch": 0.4251144538914323, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0280959606170654, + "kl": 0.09326308779418468, + "learning_rate": 3.938848920863309e-07, + "loss": 0.030906683206558226, + "num_tokens": 14693328.0, + "reward": -1.2964010291938167e-07, + "reward_std": 0.9975219249725342, + "rewards/rnd_reward/mean": -1.2964010327465302e-07, + "rewards/rnd_reward/std": 0.9975219309329987, + "step": 650, + "step_time": 10.01973620983772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1512500040233135, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.3, + "completions/mean_length": 129.62250213623048, + "completions/mean_terminated_length": 84.16030578613281, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.613438582420349, + "epoch": 0.4316546762589928, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9445599317550659, + "kl": 0.0699499910697341, + "learning_rate": 3.922498364944408e-07, + "loss": 0.08974806666374206, + "num_tokens": 14892686.0, + "reward": -4.194676717084178e-08, + "reward_std": 0.9975471794605255, + "rewards/rnd_reward/mean": -4.1946770057421645e-08, + "rewards/rnd_reward/std": 0.99754718542099, + "step": 660, + "step_time": 10.161354359728284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14750000312924386, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.7, + "completions/mean_length": 126.59500045776367, + "completions/mean_terminated_length": 81.99441680908203, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.7322494119405745, + "epoch": 0.4381948986265533, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9810017943382263, + "kl": 0.08053934266790748, + "learning_rate": 3.9061478090255067e-07, + "loss": 0.14365124702453613, + "num_tokens": 15083047.0, + "reward": -7.979571803318208e-08, + "reward_std": 0.9972860395908356, + "rewards/rnd_reward/mean": -7.979571803318208e-08, + "rewards/rnd_reward/std": 0.9972860395908356, + "step": 670, + "step_time": 9.683341883006506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14250000119209288, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.5, + "completions/mean_length": 125.95000381469727, + "completions/mean_terminated_length": 83.18684883117676, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6392508417367937, + "epoch": 0.4447351209941138, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9357602596282959, + "kl": 0.07438107710331679, + "learning_rate": 3.8897972531066054e-07, + "loss": 0.047136351466178894, + "num_tokens": 15272992.0, + "reward": -1.0043382800972722e-07, + "reward_std": 0.9973943531513214, + "rewards/rnd_reward/mean": -1.0043382658864175e-07, + "rewards/rnd_reward/std": 0.997394347190857, + "step": 680, + "step_time": 9.536431295378133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14375000223517417, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.2, + "completions/mean_length": 125.81125106811524, + "completions/mean_terminated_length": 82.36017532348633, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.2503296345472337, + "epoch": 0.4512753433616743, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7484872937202454, + "kl": 0.0611060356721282, + "learning_rate": 3.873446697187704e-07, + "loss": 0.09799213409423828, + "num_tokens": 15463906.0, + "reward": -6.347894991876046e-08, + "reward_std": 0.9972564399242401, + "rewards/rnd_reward/mean": -6.34789495634891e-08, + "rewards/rnd_reward/std": 0.9972564518451691, + "step": 690, + "step_time": 9.653547045495362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15500000342726708, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.6, + "completions/mean_length": 133.82500076293945, + "completions/mean_terminated_length": 87.98081855773925, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5280281245708465, + "epoch": 0.4578155657292348, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9658915996551514, + "kl": 0.059800021536648276, + "learning_rate": 3.857096141268803e-07, + "loss": 0.12680201530456542, + "num_tokens": 15667301.0, + "reward": -1.5869737026719122e-07, + "reward_std": 0.9973293662071228, + "rewards/rnd_reward/mean": -1.5869737026719122e-07, + "rewards/rnd_reward/std": 0.9973293721675873, + "step": 700, + "step_time": 9.966963570658118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10875000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.1, + "completions/mean_length": 107.25000228881837, + "completions/mean_terminated_length": 73.4811897277832, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4395824640989305, + "epoch": 0.4643557880967953, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.049780011177063, + "kl": 0.08286946956068278, + "learning_rate": 3.840745585349902e-07, + "loss": 0.15116369724273682, + "num_tokens": 15847671.0, + "reward": -1.005828406341891e-07, + "reward_std": 0.9973811268806457, + "rewards/rnd_reward/mean": -1.0058283921310362e-07, + "rewards/rnd_reward/std": 0.9973811089992524, + "step": 710, + "step_time": 9.890902339830063 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09000000115484, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.2, + "completions/mean_length": 93.01625213623046, + "completions/mean_terminated_length": 64.09797096252441, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8066621690988542, + "epoch": 0.4708960104643558, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34576416015625, + "kl": 0.10443261573091149, + "learning_rate": 3.8243950294310006e-07, + "loss": 0.10605430603027344, + "num_tokens": 16021059.0, + "reward": -1.0505319210096787e-07, + "reward_std": 0.9972734034061432, + "rewards/rnd_reward/mean": -1.0505318357445503e-07, + "rewards/rnd_reward/std": 0.9972733795642853, + "step": 720, + "step_time": 10.098007511859759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000074505807, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 94.07875137329101, + "completions/mean_terminated_length": 66.15557479858398, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6466055154800414, + "epoch": 0.4774362328319163, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0772963762283325, + "kl": 0.10096871191635728, + "learning_rate": 3.8080444735120994e-07, + "loss": 0.09338142871856689, + "num_tokens": 16191222.0, + "reward": -1.777708558137192e-07, + "reward_std": 0.9971938729286194, + "rewards/rnd_reward/mean": -1.7777085012937732e-07, + "rewards/rnd_reward/std": 0.9971938788890838, + "step": 730, + "step_time": 10.299412568542175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13375000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.1, + "completions/mean_length": 126.26250228881835, + "completions/mean_terminated_length": 86.43918380737304, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.173070195317268, + "epoch": 0.4839764551994768, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7416156530380249, + "kl": 0.053471417538821696, + "learning_rate": 3.7916939175931976e-07, + "loss": 0.07696372270584106, + "num_tokens": 16393972.0, + "reward": -1.7151236602064303e-07, + "reward_std": 0.9972269594669342, + "rewards/rnd_reward/mean": -1.715123674417285e-07, + "rewards/rnd_reward/std": 0.9972269415855408, + "step": 740, + "step_time": 10.502406498114578 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.148750002682209, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 139.12875213623047, + "completions/mean_terminated_length": 96.77134628295899, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.325762075185776, + "epoch": 0.49051667756703726, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7352995872497559, + "kl": 0.05052672061137855, + "learning_rate": 3.775343361674297e-07, + "loss": 0.09521990418434143, + "num_tokens": 16596670.0, + "reward": -2.0787120362797397e-07, + "reward_std": 0.9972488164901734, + "rewards/rnd_reward/mean": -2.0787119723308933e-07, + "rewards/rnd_reward/std": 0.9972488105297088, + "step": 750, + "step_time": 9.698242208594456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14750000238418579, + "completions/max_length": 384.0, + "completions/max_terminated_length": 371.2, + "completions/mean_length": 127.9612533569336, + "completions/mean_terminated_length": 83.97822761535645, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.2959645807743074, + "epoch": 0.4970568999345978, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8263393044471741, + "kl": 0.05697637004777789, + "learning_rate": 3.7589928057553957e-07, + "loss": 0.10516003370285035, + "num_tokens": 16792629.0, + "reward": -1.6942620391091622e-07, + "reward_std": 0.9972022175788879, + "rewards/rnd_reward/mean": -1.6942620106874529e-07, + "rewards/rnd_reward/std": 0.9972022175788879, + "step": 760, + "step_time": 10.044187037739903 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10875000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 339.2, + "completions/mean_length": 112.1150016784668, + "completions/mean_terminated_length": 79.13344192504883, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7552184253931045, + "epoch": 0.5035971223021583, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.029379963874817, + "kl": 0.07228619959205389, + "learning_rate": 3.742642249836494e-07, + "loss": 0.03081640601158142, + "num_tokens": 16971246.0, + "reward": -8.299946501200139e-08, + "reward_std": 0.9970763862133026, + "rewards/rnd_reward/mean": -8.299946785417234e-08, + "rewards/rnd_reward/std": 0.9970763862133026, + "step": 770, + "step_time": 9.667120404168964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13125000298023223, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.9, + "completions/mean_length": 124.3275032043457, + "completions/mean_terminated_length": 85.28903274536133, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.8021608620882033, + "epoch": 0.5101373446697187, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0165283679962158, + "kl": 0.07025760738179088, + "learning_rate": 3.7262916939175933e-07, + "loss": 0.03996403813362122, + "num_tokens": 17162308.0, + "reward": -5.170702825907369e-08, + "reward_std": 0.9971189498901367, + "rewards/rnd_reward/mean": -5.170703110124464e-08, + "rewards/rnd_reward/std": 0.9971189439296723, + "step": 780, + "step_time": 9.969513631938025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13375000283122063, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.5, + "completions/mean_length": 127.03500213623047, + "completions/mean_terminated_length": 87.38848876953125, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.637522208690643, + "epoch": 0.5166775670372793, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9886835813522339, + "kl": 0.06816753121092915, + "learning_rate": 3.7099411379986915e-07, + "loss": 0.07478119730949402, + "num_tokens": 17356536.0, + "reward": -1.1831521575800253e-07, + "reward_std": 0.9971611678600312, + "rewards/rnd_reward/mean": -1.1831521575800253e-07, + "rewards/rnd_reward/std": 0.997161191701889, + "step": 790, + "step_time": 9.748413591040299 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12625000327825547, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.3, + "completions/mean_length": 120.86375045776367, + "completions/mean_terminated_length": 83.05458145141601, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6706619918346406, + "epoch": 0.5232177894048398, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8250897526741028, + "kl": 0.07074365848675371, + "learning_rate": 3.693590582079791e-07, + "loss": 0.05061581134796143, + "num_tokens": 17545417.0, + "reward": -6.467104096685716e-08, + "reward_std": 0.9972768127918243, + "rewards/rnd_reward/mean": -6.467104096685716e-08, + "rewards/rnd_reward/std": 0.9972768068313599, + "step": 800, + "step_time": 9.678878564713523 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12500000298023223, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.5, + "completions/mean_length": 121.30500183105468, + "completions/mean_terminated_length": 83.6633544921875, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.795678195357323, + "epoch": 0.5297580117724002, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0277215242385864, + "kl": 0.06990557787939906, + "learning_rate": 3.677240026160889e-07, + "loss": 0.059202158451080324, + "num_tokens": 17737161.0, + "reward": -1.2487173535191686e-07, + "reward_std": 0.9970629572868347, + "rewards/rnd_reward/mean": -1.2487172682540403e-07, + "rewards/rnd_reward/std": 0.9970629572868347, + "step": 810, + "step_time": 10.510143173905089 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.4, + "completions/mean_length": 119.66500091552734, + "completions/mean_terminated_length": 80.23575859069824, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8059179961681364, + "epoch": 0.5362982341399608, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8309988379478455, + "kl": 0.08165676584467292, + "learning_rate": 3.660889470241988e-07, + "loss": 0.12505731582641602, + "num_tokens": 17927903.0, + "reward": -2.3901462826714236e-07, + "reward_std": 0.997013247013092, + "rewards/rnd_reward/mean": -2.3901462400388593e-07, + "rewards/rnd_reward/std": 0.9970132112503052, + "step": 820, + "step_time": 10.190648046904244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14875000417232515, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.9, + "completions/mean_length": 126.17125396728515, + "completions/mean_terminated_length": 80.60145225524903, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.938824012875557, + "epoch": 0.5428384565075213, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1022851467132568, + "kl": 0.07984373616054655, + "learning_rate": 3.644538914323087e-07, + "loss": 0.023773609101772307, + "num_tokens": 18123110.0, + "reward": -2.2649766062698973e-08, + "reward_std": 0.9969716727733612, + "rewards/rnd_reward/mean": -2.2649762509985294e-08, + "rewards/rnd_reward/std": 0.9969716787338256, + "step": 830, + "step_time": 9.89921972535085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14375000447034836, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.7, + "completions/mean_length": 127.39125366210938, + "completions/mean_terminated_length": 84.2758415222168, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8030223518610002, + "epoch": 0.5493786788750817, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9427273273468018, + "kl": 0.0753337662667036, + "learning_rate": 3.6281883584041854e-07, + "loss": 0.05915579795837402, + "num_tokens": 18319518.0, + "reward": -1.1295080284412507e-07, + "reward_std": 0.9969306707382202, + "rewards/rnd_reward/mean": -1.1295080142303959e-07, + "rewards/rnd_reward/std": 0.9969306766986847, + "step": 840, + "step_time": 9.87428822228685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15750000178813933, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.2, + "completions/mean_length": 137.7587516784668, + "completions/mean_terminated_length": 91.88029098510742, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.810399681329727, + "epoch": 0.5559189012426422, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8602397441864014, + "kl": 0.0653490168042481, + "learning_rate": 3.611837802485284e-07, + "loss": 0.03035195469856262, + "num_tokens": 18526310.0, + "reward": -1.1295080284412507e-07, + "reward_std": 0.9969462454319, + "rewards/rnd_reward/mean": -1.1295080426521054e-07, + "rewards/rnd_reward/std": 0.9969462633132935, + "step": 850, + "step_time": 10.37288966330234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15250000283122062, + "completions/max_length": 384.0, + "completions/max_terminated_length": 343.9, + "completions/mean_length": 127.73250350952148, + "completions/mean_terminated_length": 81.79599456787109, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.83120773434639, + "epoch": 0.5624591236102028, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0732090473175049, + "kl": 0.08459939705207944, + "learning_rate": 3.595487246566383e-07, + "loss": 0.03491010665893555, + "num_tokens": 18724316.0, + "reward": -1.4752149617436318e-07, + "reward_std": 0.9969268858432769, + "rewards/rnd_reward/mean": -1.4752149759544864e-07, + "rewards/rnd_reward/std": 0.9969269037246704, + "step": 860, + "step_time": 10.52987401576247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14000000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.0, + "completions/mean_length": 131.95375213623046, + "completions/mean_terminated_length": 90.77078781127929, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.9562632203102113, + "epoch": 0.5689993459777632, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.969403862953186, + "kl": 0.08435474634170533, + "learning_rate": 3.579136690647482e-07, + "loss": 0.04807090759277344, + "num_tokens": 18926489.0, + "reward": -1.3545155894689742e-07, + "reward_std": 0.9968080937862396, + "rewards/rnd_reward/mean": -1.3545156178906837e-07, + "rewards/rnd_reward/std": 0.9968081057071686, + "step": 870, + "step_time": 9.85819350676611 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16750000193715095, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.2, + "completions/mean_length": 137.6675033569336, + "completions/mean_terminated_length": 88.34969863891601, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7675389796495438, + "epoch": 0.5755395683453237, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.828363835811615, + "kl": 0.07512732800096274, + "learning_rate": 3.5627861347285806e-07, + "loss": 0.010637739300727844, + "num_tokens": 19126833.0, + "reward": -7.003545690054125e-08, + "reward_std": 0.9967847049236298, + "rewards/rnd_reward/mean": -7.003545903216945e-08, + "rewards/rnd_reward/std": 0.9967847049236298, + "step": 880, + "step_time": 9.748134864587337 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19125000312924384, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.5, + "completions/mean_length": 148.54750366210936, + "completions/mean_terminated_length": 93.13098068237305, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.6605105638504027, + "epoch": 0.5820797907128843, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8364513516426086, + "kl": 0.06730008013546467, + "learning_rate": 3.5464355788096794e-07, + "loss": 0.0366318553686142, + "num_tokens": 19338336.0, + "reward": -1.0654330893089536e-07, + "reward_std": 0.9968020260334015, + "rewards/rnd_reward/mean": -1.0654329685166886e-07, + "rewards/rnd_reward/std": 0.9968020260334015, + "step": 890, + "step_time": 9.872815124248154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16625000312924385, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.5, + "completions/mean_length": 149.7400032043457, + "completions/mean_terminated_length": 103.11838226318359, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.535254544019699, + "epoch": 0.5886200130804448, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7181042432785034, + "kl": 0.05683262627571821, + "learning_rate": 3.530085022890778e-07, + "loss": 0.08454052209854127, + "num_tokens": 19557363.0, + "reward": -1.737475452401327e-07, + "reward_std": 0.9967290103435517, + "rewards/rnd_reward/mean": -1.737475338714489e-07, + "rewards/rnd_reward/std": 0.9967289924621582, + "step": 900, + "step_time": 10.189786232402549 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1912500038743019, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 157.67875213623046, + "completions/mean_terminated_length": 104.52761840820312, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5941266864538193, + "epoch": 0.5951602354480052, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9426244497299194, + "kl": 0.050556823750957845, + "learning_rate": 3.513734466971877e-07, + "loss": 0.07448753714561462, + "num_tokens": 19775536.0, + "reward": -2.162158537544201e-07, + "reward_std": 0.9968621492385864, + "rewards/rnd_reward/mean": -2.1621584735953548e-07, + "rewards/rnd_reward/std": 0.9968621671199799, + "step": 910, + "step_time": 9.74868165422231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1662500023841858, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.0, + "completions/mean_length": 151.89000396728517, + "completions/mean_terminated_length": 105.61829071044922, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.508600893616676, + "epoch": 0.6017004578155657, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7371200323104858, + "kl": 0.04880232191644609, + "learning_rate": 3.4973839110529757e-07, + "loss": 0.12514564990997315, + "num_tokens": 19990298.0, + "reward": -1.93715107066339e-08, + "reward_std": 0.9970090925693512, + "rewards/rnd_reward/mean": -1.937150893027706e-08, + "rewards/rnd_reward/std": 0.9970090866088868, + "step": 920, + "step_time": 9.937099263281562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1575000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.5, + "completions/mean_length": 138.63375244140624, + "completions/mean_terminated_length": 92.52680130004883, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4236575067043304, + "epoch": 0.6082406801831263, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9167628884315491, + "kl": 0.05787700763903558, + "learning_rate": 3.4810333551340745e-07, + "loss": 0.10835990905761719, + "num_tokens": 20187745.0, + "reward": -7.212161943215278e-08, + "reward_std": 0.996803218126297, + "rewards/rnd_reward/mean": -7.212161801106731e-08, + "rewards/rnd_reward/std": 0.9968032300472259, + "step": 930, + "step_time": 9.689079960295931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1337500013411045, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.4, + "completions/mean_length": 129.22500305175782, + "completions/mean_terminated_length": 89.93161087036133, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.6605860769748686, + "epoch": 0.6147809025506867, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0838011503219604, + "kl": 0.06689512394368649, + "learning_rate": 3.4646827992151733e-07, + "loss": 0.07303244471549988, + "num_tokens": 20383990.0, + "reward": -1.1324883786301143e-08, + "reward_std": 0.9967760264873504, + "rewards/rnd_reward/mean": -1.1324883786301143e-08, + "rewards/rnd_reward/std": 0.9967760384082794, + "step": 940, + "step_time": 9.85847494918853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16125000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 364.1, + "completions/mean_length": 134.65625534057617, + "completions/mean_terminated_length": 86.58066329956054, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7146200090646744, + "epoch": 0.6213211249182472, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9745354652404785, + "kl": 0.06564742000773549, + "learning_rate": 3.448332243296272e-07, + "loss": 0.10424284934997559, + "num_tokens": 20593190.0, + "reward": -1.0967255157368072e-07, + "reward_std": 0.9968843460083008, + "rewards/rnd_reward/mean": -1.096725451787961e-07, + "rewards/rnd_reward/std": 0.9968843400478363, + "step": 950, + "step_time": 10.862376277497969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10875000022351741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.1, + "completions/mean_length": 113.23000411987304, + "completions/mean_terminated_length": 80.14122467041015, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8509408473968505, + "epoch": 0.6278613472858077, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9323987364768982, + "kl": 0.08510139184072614, + "learning_rate": 3.4319816873773703e-07, + "loss": 0.0554282009601593, + "num_tokens": 20776164.0, + "reward": -7.957220322651893e-08, + "reward_std": 0.9968249201774597, + "rewards/rnd_reward/mean": -7.957220180543345e-08, + "rewards/rnd_reward/std": 0.9968249082565308, + "step": 960, + "step_time": 9.989999298867769 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10750000327825546, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.2, + "completions/mean_length": 112.88375091552734, + "completions/mean_terminated_length": 80.20179443359375, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.88864361345768, + "epoch": 0.6344015696533682, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8851020932197571, + "kl": 0.0875691649504006, + "learning_rate": 3.4156311314584696e-07, + "loss": 0.10126043558120727, + "num_tokens": 20958826.0, + "reward": -1.6622245713193706e-07, + "reward_std": 0.9968612968921662, + "rewards/rnd_reward/mean": -1.6622244931596696e-07, + "rewards/rnd_reward/std": 0.9968612730503082, + "step": 970, + "step_time": 10.018015099223703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11125000156462192, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.6, + "completions/mean_length": 106.1625, + "completions/mean_terminated_length": 71.38758811950683, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9656033277511598, + "epoch": 0.6409417920209287, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9377958178520203, + "kl": 0.09664760064333677, + "learning_rate": 3.3992805755395684e-07, + "loss": 0.046259531378746034, + "num_tokens": 21147866.0, + "reward": -2.499669804478799e-07, + "reward_std": 0.9968202471733093, + "rewards/rnd_reward/mean": -2.499669751188094e-07, + "rewards/rnd_reward/std": 0.9968202352523804, + "step": 980, + "step_time": 11.508573367865756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1262500025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.6, + "completions/mean_length": 122.56625289916992, + "completions/mean_terminated_length": 84.71166687011718, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.80318078994751, + "epoch": 0.6474820143884892, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9525420069694519, + "kl": 0.07310699988156558, + "learning_rate": 3.3829300196206667e-07, + "loss": 0.02322930097579956, + "num_tokens": 21338124.0, + "reward": -9.76771161731449e-08, + "reward_std": 0.9966861188411713, + "rewards/rnd_reward/mean": -9.7677110488803e-08, + "rewards/rnd_reward/std": 0.9966861248016358, + "step": 990, + "step_time": 9.876657200045884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15375000201165676, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.8, + "completions/mean_length": 137.66875076293945, + "completions/mean_terminated_length": 92.6232063293457, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9049280107021334, + "epoch": 0.6540222367560498, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8244759440422058, + "kl": 0.061428536847233775, + "learning_rate": 3.366579463701766e-07, + "loss": 0.05219428539276123, + "num_tokens": 21539579.0, + "reward": -1.6495585271059098e-07, + "reward_std": 0.9966946303844452, + "rewards/rnd_reward/mean": -1.6495585271059098e-07, + "rewards/rnd_reward/std": 0.9966946005821228, + "step": 1000, + "step_time": 9.666829459602013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18250000178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.9, + "completions/mean_length": 146.93875427246093, + "completions/mean_terminated_length": 93.79228134155274, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8016369342803955, + "epoch": 0.6605624591236102, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.766015350818634, + "kl": 0.05675441958010197, + "learning_rate": 3.350228907782864e-07, + "loss": 0.08666471242904664, + "num_tokens": 21754125.0, + "reward": -2.2858381782953073e-07, + "reward_std": 0.996807062625885, + "rewards/rnd_reward/mean": -2.2858381640844527e-07, + "rewards/rnd_reward/std": 0.9968070745468139, + "step": 1010, + "step_time": 10.005978721287102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13875000253319741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.3, + "completions/mean_length": 136.35500259399413, + "completions/mean_terminated_length": 96.46944885253906, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8755707025527952, + "epoch": 0.6671026814911707, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0061216354370117, + "kl": 0.05596274421550333, + "learning_rate": 3.3338783518639635e-07, + "loss": 0.09954084157943725, + "num_tokens": 21957634.0, + "reward": -1.9520521661320344e-07, + "reward_std": 0.9966324806213379, + "rewards/rnd_reward/mean": -1.9520520950777608e-07, + "rewards/rnd_reward/std": 0.9966325044631958, + "step": 1020, + "step_time": 10.040163300000131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15875000432133674, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.5, + "completions/mean_length": 136.7437530517578, + "completions/mean_terminated_length": 90.20843048095703, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7692076802253722, + "epoch": 0.6736429038587312, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8324238061904907, + "kl": 0.057948453910648824, + "learning_rate": 3.317527795945062e-07, + "loss": 0.07627512216567993, + "num_tokens": 22158654.0, + "reward": -3.7699937749380295e-08, + "reward_std": 0.9966008186340332, + "rewards/rnd_reward/mean": -3.769993632829482e-08, + "rewards/rnd_reward/std": 0.9966008424758911, + "step": 1030, + "step_time": 10.1797529961681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16250000298023223, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.7, + "completions/mean_length": 139.57500534057618, + "completions/mean_terminated_length": 92.21996688842773, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.8831230252981186, + "epoch": 0.6801831262262917, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8833721876144409, + "kl": 0.060850747767835854, + "learning_rate": 3.3011772400261606e-07, + "loss": 0.10484603643417359, + "num_tokens": 22366559.0, + "reward": -1.0505318579490108e-07, + "reward_std": 0.996761429309845, + "rewards/rnd_reward/mean": -1.0505318588371893e-07, + "rewards/rnd_reward/std": 0.9967614591121674, + "step": 1040, + "step_time": 10.081386449630372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1625000037252903, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.9, + "completions/mean_length": 137.33875198364257, + "completions/mean_terminated_length": 89.72673568725585, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.885282647609711, + "epoch": 0.6867233485938522, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9477590918540955, + "kl": 0.05932593122124672, + "learning_rate": 3.28482668410726e-07, + "loss": 0.024539683759212495, + "num_tokens": 22571575.0, + "reward": 4.9620866704458424e-08, + "reward_std": 0.996524053812027, + "rewards/rnd_reward/mean": 4.9620866704458424e-08, + "rewards/rnd_reward/std": 0.9965240597724915, + "step": 1050, + "step_time": 10.535549363144673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16250000447034835, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.5, + "completions/mean_length": 138.3237533569336, + "completions/mean_terminated_length": 90.70624694824218, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.875243663787842, + "epoch": 0.6932635709614127, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9407637119293213, + "kl": 0.06043377108871937, + "learning_rate": 3.268476128188358e-07, + "loss": 0.06624546647071838, + "num_tokens": 22771314.0, + "reward": -5.781650713743147e-08, + "reward_std": 0.9966591358184814, + "rewards/rnd_reward/mean": -5.7816505716346e-08, + "rewards/rnd_reward/std": 0.9966591358184814, + "step": 1060, + "step_time": 10.021701269014738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1400000039488077, + "completions/max_length": 384.0, + "completions/max_terminated_length": 342.1, + "completions/mean_length": 127.3300048828125, + "completions/mean_terminated_length": 85.42830429077148, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.927751249074936, + "epoch": 0.6998037933289731, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8508753180503845, + "kl": 0.0620661674067378, + "learning_rate": 3.252125572269457e-07, + "loss": 0.12821773290634156, + "num_tokens": 22965548.0, + "reward": -1.0244548391824537e-07, + "reward_std": 0.9966984510421752, + "rewards/rnd_reward/mean": -1.0244548391824537e-07, + "rewards/rnd_reward/std": 0.9966984808444976, + "step": 1070, + "step_time": 9.767576195369475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14500000327825546, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.6, + "completions/mean_length": 136.3862518310547, + "completions/mean_terminated_length": 94.67120056152343, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.6616650968790054, + "epoch": 0.7063440156965337, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.789601743221283, + "kl": 0.0545062692835927, + "learning_rate": 3.2357750163505557e-07, + "loss": 0.11497254371643066, + "num_tokens": 23170007.0, + "reward": 1.226365569095833e-07, + "reward_std": 0.9966248035430908, + "rewards/rnd_reward/mean": 1.2263655548849784e-07, + "rewards/rnd_reward/std": 0.9966248333454132, + "step": 1080, + "step_time": 9.882125865970739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14250000566244125, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.8, + "completions/mean_length": 130.01000289916993, + "completions/mean_terminated_length": 87.69976119995117, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.890427288413048, + "epoch": 0.7128842380640942, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8021807670593262, + "kl": 0.05592884449288249, + "learning_rate": 3.2194244604316545e-07, + "loss": 0.09672708511352539, + "num_tokens": 23366425.0, + "reward": 6.273389110411643e-08, + "reward_std": 0.9965825915336609, + "rewards/rnd_reward/mean": 6.273389181465916e-08, + "rewards/rnd_reward/std": 0.9965825736522674, + "step": 1090, + "step_time": 9.854898347100242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14500000178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 336.9, + "completions/mean_length": 131.98875274658204, + "completions/mean_terminated_length": 89.18458480834961, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7609634190797805, + "epoch": 0.7194244604316546, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0092402696609497, + "kl": 0.05501702083274722, + "learning_rate": 3.2030739045127533e-07, + "loss": 0.08695750832557678, + "num_tokens": 23567981.0, + "reward": -1.3828277736216422e-07, + "reward_std": 0.9966061592102051, + "rewards/rnd_reward/mean": -1.3828277451999326e-07, + "rewards/rnd_reward/std": 0.9966061592102051, + "step": 1100, + "step_time": 10.145819194451906 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1212500013411045, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.9, + "completions/mean_length": 121.10875091552734, + "completions/mean_terminated_length": 84.89219970703125, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8415472239255903, + "epoch": 0.7259646827992152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8949585556983948, + "kl": 0.06347150038927793, + "learning_rate": 3.186723348593852e-07, + "loss": 0.08753393888473511, + "num_tokens": 23758168.0, + "reward": -1.1354684570363816e-07, + "reward_std": 0.9964610040187836, + "rewards/rnd_reward/mean": -1.1354684286146721e-07, + "rewards/rnd_reward/std": 0.9964609861373901, + "step": 1110, + "step_time": 10.006089114071802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12125000208616257, + "completions/max_length": 384.0, + "completions/max_terminated_length": 369.7, + "completions/mean_length": 121.20000228881835, + "completions/mean_terminated_length": 84.96716842651367, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.9164877772331237, + "epoch": 0.7325049051667757, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1227521896362305, + "kl": 0.06196983400732279, + "learning_rate": 3.170372792674951e-07, + "loss": 0.07237975597381592, + "num_tokens": 23955023.0, + "reward": -5.5432321488524396e-08, + "reward_std": 0.9964903295040131, + "rewards/rnd_reward/mean": -5.543231935689619e-08, + "rewards/rnd_reward/std": 0.9964903354644775, + "step": 1120, + "step_time": 10.375506636733189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11875000298023224, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.2, + "completions/mean_length": 117.09375, + "completions/mean_terminated_length": 81.17416534423828, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.7361535727977753, + "epoch": 0.7390451275343362, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8529885411262512, + "kl": 0.06673006983473898, + "learning_rate": 3.1540222367560496e-07, + "loss": 0.09247955083847045, + "num_tokens": 24140203.0, + "reward": -2.6300548938706926e-08, + "reward_std": 0.9966336786746979, + "rewards/rnd_reward/mean": -2.6300551070335132e-08, + "rewards/rnd_reward/std": 0.9966336965560914, + "step": 1130, + "step_time": 10.012891574040987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11500000357627868, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.7, + "completions/mean_length": 119.6712516784668, + "completions/mean_terminated_length": 85.35453567504882, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6402025282382966, + "epoch": 0.7455853499018966, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9655517935752869, + "kl": 0.06339178960770368, + "learning_rate": 3.1376716808371484e-07, + "loss": 0.012396740168333054, + "num_tokens": 24333505.0, + "reward": -1.3798475424486866e-07, + "reward_std": 0.9963988780975341, + "rewards/rnd_reward/mean": -1.379847538895973e-07, + "rewards/rnd_reward/std": 0.9963988780975341, + "step": 1140, + "step_time": 10.81275618087966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12750000208616258, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.4, + "completions/mean_length": 130.15375289916992, + "completions/mean_terminated_length": 93.11451568603516, + "completions/min_length": 2.6, + "completions/min_terminated_length": 2.6, + "entropy": 2.7988665878772734, + "epoch": 0.7521255722694572, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9594887495040894, + "kl": 0.057864805683493614, + "learning_rate": 3.121321124918247e-07, + "loss": 0.05015655755996704, + "num_tokens": 24526988.0, + "reward": -2.7045605088815705e-08, + "reward_std": 0.9962463498115539, + "rewards/rnd_reward/mean": -2.7045605088815705e-08, + "rewards/rnd_reward/std": 0.9962463617324829, + "step": 1150, + "step_time": 9.847500793170184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000305473805, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.1, + "completions/mean_length": 119.13750076293945, + "completions/mean_terminated_length": 79.57122192382812, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.799566090106964, + "epoch": 0.7586657946370177, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3144031763076782, + "kl": 0.06464958414435387, + "learning_rate": 3.104970568999346e-07, + "loss": 0.08081794381141663, + "num_tokens": 24712093.0, + "reward": 2.2947787847726885e-08, + "reward_std": 0.9963953137397766, + "rewards/rnd_reward/mean": 2.2947789268812358e-08, + "rewards/rnd_reward/std": 0.9963953375816346, + "step": 1160, + "step_time": 9.831479465053416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 366.1, + "completions/mean_length": 128.67500228881835, + "completions/mean_terminated_length": 90.44079208374023, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.758173832297325, + "epoch": 0.7652060170045781, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8444108963012695, + "kl": 0.061153271794319154, + "learning_rate": 3.088620013080445e-07, + "loss": 0.06105284690856934, + "num_tokens": 24905938.0, + "reward": -9.223818580039733e-08, + "reward_std": 0.9962478995323181, + "rewards/rnd_reward/mean": -9.22381850898546e-08, + "rewards/rnd_reward/std": 0.9962479054927826, + "step": 1170, + "step_time": 9.715631593717262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13125000149011612, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.7, + "completions/mean_length": 132.65000381469727, + "completions/mean_terminated_length": 94.60303192138672, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.652452102303505, + "epoch": 0.7717462393721386, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8246821761131287, + "kl": 0.05974919479340315, + "learning_rate": 3.072269457161543e-07, + "loss": 0.07230232954025269, + "num_tokens": 25111683.0, + "reward": -1.4752150079289095e-08, + "reward_std": 0.9964145243167877, + "rewards/rnd_reward/mean": -1.4752149368746359e-08, + "rewards/rnd_reward/std": 0.9964145421981812, + "step": 1180, + "step_time": 10.24143796269782 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1437500037252903, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 133.8912498474121, + "completions/mean_terminated_length": 92.00172958374023, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.534428471326828, + "epoch": 0.7782864617396992, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7489418387413025, + "kl": 0.05822144625708461, + "learning_rate": 3.0559189012426423e-07, + "loss": 0.07474024295806884, + "num_tokens": 25319036.0, + "reward": -8.940696432091499e-08, + "reward_std": 0.9965973794460297, + "rewards/rnd_reward/mean": -8.940696574200047e-08, + "rewards/rnd_reward/std": 0.9965973973274231, + "step": 1190, + "step_time": 10.355807177536189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1450000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.6, + "completions/mean_length": 138.9825012207031, + "completions/mean_terminated_length": 97.18752059936523, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8433078229427338, + "epoch": 0.7848266841072596, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8156785368919373, + "kl": 0.06143994340673089, + "learning_rate": 3.039568345323741e-07, + "loss": 0.08815957307815551, + "num_tokens": 25526837.0, + "reward": -1.0192394315211572e-07, + "reward_std": 0.996451473236084, + "rewards/rnd_reward/mean": -1.0192393888885931e-07, + "rewards/rnd_reward/std": 0.996451485157013, + "step": 1200, + "step_time": 10.298059746343643 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14000000283122063, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.0, + "completions/mean_length": 127.81375274658203, + "completions/mean_terminated_length": 85.73782577514649, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7779837220907213, + "epoch": 0.7913669064748201, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.937714695930481, + "kl": 0.06527232667431235, + "learning_rate": 3.0232177894048394e-07, + "loss": 0.08186920881271362, + "num_tokens": 25721938.0, + "reward": -1.0982156268823929e-07, + "reward_std": 0.9962611317634582, + "rewards/rnd_reward/mean": -1.0982156055661107e-07, + "rewards/rnd_reward/std": 0.9962611377239228, + "step": 1210, + "step_time": 9.923247428331525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13625000193715095, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.9, + "completions/mean_length": 128.51625213623046, + "completions/mean_terminated_length": 88.076513671875, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.661186295747757, + "epoch": 0.7979071288423807, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9877737164497375, + "kl": 0.06118576750159264, + "learning_rate": 3.0068672334859387e-07, + "loss": 0.0857498049736023, + "num_tokens": 25916611.0, + "reward": -9.454786980001017e-08, + "reward_std": 0.9962615430355072, + "rewards/rnd_reward/mean": -9.454786553675376e-08, + "rewards/rnd_reward/std": 0.9962615609169007, + "step": 1220, + "step_time": 10.026552183739842 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13875000402331353, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.7, + "completions/mean_length": 127.41250381469726, + "completions/mean_terminated_length": 86.15239715576172, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.597662454843521, + "epoch": 0.8044473512099412, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9214726090431213, + "kl": 0.06120588518679142, + "learning_rate": 2.990516677567037e-07, + "loss": 0.08460624814033509, + "num_tokens": 26120156.0, + "reward": -1.5325844344715732e-07, + "reward_std": 0.9961971819400788, + "rewards/rnd_reward/mean": -1.5325843989444364e-07, + "rewards/rnd_reward/std": 0.9961971819400788, + "step": 1230, + "step_time": 10.459681713976897 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12875000089406968, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.9, + "completions/mean_length": 121.05875167846679, + "completions/mean_terminated_length": 82.47761840820313, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.899898463487625, + "epoch": 0.8109875735775016, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9018652439117432, + "kl": 0.06602321406826377, + "learning_rate": 2.9741661216481357e-07, + "loss": 0.05030661225318909, + "num_tokens": 26307033.0, + "reward": -1.496076620810527e-07, + "reward_std": 0.996212637424469, + "rewards/rnd_reward/mean": -1.4960765781779628e-07, + "rewards/rnd_reward/std": 0.9962126433849334, + "step": 1240, + "step_time": 9.930500056478195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13875000253319741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.1, + "completions/mean_length": 125.87625274658203, + "completions/mean_terminated_length": 84.06948699951172, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.7101810783147813, + "epoch": 0.8175277959450621, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9933876991271973, + "kl": 0.06315780784934759, + "learning_rate": 2.9578155657292345e-07, + "loss": 0.14283275604248047, + "num_tokens": 26506699.0, + "reward": -4.701316604904804e-08, + "reward_std": 0.996234393119812, + "rewards/rnd_reward/mean": -4.7013166404319404e-08, + "rewards/rnd_reward/std": 0.9962343990802764, + "step": 1250, + "step_time": 11.001938613550738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14875000342726707, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.5, + "completions/mean_length": 131.21625289916992, + "completions/mean_terminated_length": 87.06982650756837, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.695519116520882, + "epoch": 0.8240680183126227, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8424022793769836, + "kl": 0.06466643353924155, + "learning_rate": 2.9414650098103333e-07, + "loss": 0.08369569182395935, + "num_tokens": 26703352.0, + "reward": -2.6673080633088375e-08, + "reward_std": 0.9962631404399872, + "rewards/rnd_reward/mean": -2.6673077790917433e-08, + "rewards/rnd_reward/std": 0.9962631464004517, + "step": 1260, + "step_time": 9.866059662331827 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16375000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 368.6, + "completions/mean_length": 132.26625137329103, + "completions/mean_terminated_length": 83.00667343139648, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7214665055274962, + "epoch": 0.8306082406801831, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.061651349067688, + "kl": 0.060832435730844736, + "learning_rate": 2.9251144538914326e-07, + "loss": 0.011636599898338318, + "num_tokens": 26905195.0, + "reward": -1.28149986178272e-08, + "reward_std": 0.996177339553833, + "rewards/rnd_reward/mean": -1.2814998262555832e-08, + "rewards/rnd_reward/std": 0.9961773693561554, + "step": 1270, + "step_time": 10.226913845306262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1462500013411045, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.4, + "completions/mean_length": 135.36000289916993, + "completions/mean_terminated_length": 92.69313278198243, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5201783299446108, + "epoch": 0.8371484630477436, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9056648015975952, + "kl": 0.05058280224911869, + "learning_rate": 2.908763897972531e-07, + "loss": 0.08019465208053589, + "num_tokens": 27106543.0, + "reward": -1.9341707613307335e-07, + "reward_std": 0.9962323784828186, + "rewards/rnd_reward/mean": -1.9341706973818874e-07, + "rewards/rnd_reward/std": 0.9962323844432831, + "step": 1280, + "step_time": 10.01526041161269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1225000001490116, + "completions/max_length": 384.0, + "completions/max_terminated_length": 373.1, + "completions/mean_length": 130.23250198364258, + "completions/mean_terminated_length": 94.88236541748047, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.623024606704712, + "epoch": 0.8436886854153042, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.783699095249176, + "kl": 0.05370175880379975, + "learning_rate": 2.8924133420536296e-07, + "loss": 0.11414051055908203, + "num_tokens": 27305554.0, + "reward": -9.253621175986382e-08, + "reward_std": 0.9961644470691681, + "rewards/rnd_reward/mean": -9.253620731897172e-08, + "rewards/rnd_reward/std": 0.9961644232273101, + "step": 1290, + "step_time": 10.230407970934175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14875000193715096, + "completions/max_length": 384.0, + "completions/max_terminated_length": 365.6, + "completions/mean_length": 136.5575035095215, + "completions/mean_terminated_length": 93.4459114074707, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.676475775241852, + "epoch": 0.8502289077828646, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0168393850326538, + "kl": 0.052173000667244195, + "learning_rate": 2.8760627861347284e-07, + "loss": 0.049798119068145755, + "num_tokens": 27507145.0, + "reward": -1.576542894232702e-07, + "reward_std": 0.9962258338928223, + "rewards/rnd_reward/mean": -1.5765429013381292e-07, + "rewards/rnd_reward/std": 0.9962258338928223, + "step": 1300, + "step_time": 9.98130206335336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13500000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.1, + "completions/mean_length": 128.2087516784668, + "completions/mean_terminated_length": 88.18164749145508, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.640226128697395, + "epoch": 0.8567691301504251, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8835275769233704, + "kl": 0.05669898777268827, + "learning_rate": 2.859712230215827e-07, + "loss": 0.10237141847610473, + "num_tokens": 27707452.0, + "reward": -6.869435509315736e-08, + "reward_std": 0.9961890399456024, + "rewards/rnd_reward/mean": -6.869434940881546e-08, + "rewards/rnd_reward/std": 0.99618901014328, + "step": 1310, + "step_time": 10.37953436721582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13625000268220902, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.5, + "completions/mean_length": 128.80750274658203, + "completions/mean_terminated_length": 88.5141227722168, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.4644729048013687, + "epoch": 0.8633093525179856, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8116011619567871, + "kl": 0.05524417916312814, + "learning_rate": 2.843361674296926e-07, + "loss": 0.1121416449546814, + "num_tokens": 27900098.0, + "reward": -5.230307502657183e-08, + "reward_std": 0.9961800217628479, + "rewards/rnd_reward/mean": -5.230307715820004e-08, + "rewards/rnd_reward/std": 0.9961800217628479, + "step": 1320, + "step_time": 9.958397940779104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11625000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.3, + "completions/mean_length": 122.20125198364258, + "completions/mean_terminated_length": 87.82691497802735, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.619413223862648, + "epoch": 0.8698495748855462, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1175379753112793, + "kl": 0.06109289322048426, + "learning_rate": 2.827011118378025e-07, + "loss": 0.05299858450889587, + "num_tokens": 28083799.0, + "reward": -9.030103811369373e-08, + "reward_std": 0.9959750473499298, + "rewards/rnd_reward/mean": -9.030103793605804e-08, + "rewards/rnd_reward/std": 0.9959750413894654, + "step": 1330, + "step_time": 9.82245781556703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11625000312924386, + "completions/max_length": 384.0, + "completions/max_terminated_length": 361.4, + "completions/mean_length": 122.17500152587891, + "completions/mean_terminated_length": 87.70087432861328, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.900624880194664, + "epoch": 0.8763897972531066, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.942399263381958, + "kl": 0.0663422117009759, + "learning_rate": 2.8106605624591235e-07, + "loss": 0.07738288044929505, + "num_tokens": 28270204.0, + "reward": -1.4573336244438905e-07, + "reward_std": 0.9961058497428894, + "rewards/rnd_reward/mean": -1.4573335960221812e-07, + "rewards/rnd_reward/std": 0.9961058616638183, + "step": 1340, + "step_time": 9.953787335986272 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1312500022351742, + "completions/max_length": 384.0, + "completions/max_terminated_length": 370.4, + "completions/mean_length": 120.04500045776368, + "completions/mean_terminated_length": 80.18856277465821, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.87275829911232, + "epoch": 0.8829300196206671, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0761396884918213, + "kl": 0.07089657634496689, + "learning_rate": 2.794310006540222e-07, + "loss": 0.13924288749694824, + "num_tokens": 28461485.0, + "reward": 2.965331162840812e-08, + "reward_std": 0.9960719764232635, + "rewards/rnd_reward/mean": 2.965331162840812e-08, + "rewards/rnd_reward/std": 0.996071970462799, + "step": 1350, + "step_time": 10.60484689238947 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12875000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.8, + "completions/mean_length": 115.14375228881836, + "completions/mean_terminated_length": 75.8135986328125, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 3.0420612663030626, + "epoch": 0.8894702419882276, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1144483089447021, + "kl": 0.07124942755326628, + "learning_rate": 2.777959450621321e-07, + "loss": 0.08705209493637085, + "num_tokens": 28652205.0, + "reward": -1.2695789215477759e-07, + "reward_std": 0.9961645841598511, + "rewards/rnd_reward/mean": -1.2695789215477759e-07, + "rewards/rnd_reward/std": 0.99616459608078, + "step": 1360, + "step_time": 10.374728590669111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12250000275671483, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.6, + "completions/mean_length": 116.32750015258789, + "completions/mean_terminated_length": 78.6882926940918, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8743550151586534, + "epoch": 0.8960104643557881, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.038732886314392, + "kl": 0.07000278439372778, + "learning_rate": 2.76160889470242e-07, + "loss": 0.11522959470748902, + "num_tokens": 28836007.0, + "reward": -2.1174550361990897e-07, + "reward_std": 0.9960576713085174, + "rewards/rnd_reward/mean": -2.117455032646376e-07, + "rewards/rnd_reward/std": 0.9960576832294464, + "step": 1370, + "step_time": 10.044307312183083 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10750000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.4, + "completions/mean_length": 112.57625274658203, + "completions/mean_terminated_length": 79.90474357604981, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.814165809750557, + "epoch": 0.9025506867233486, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.853209912776947, + "kl": 0.07179404897615313, + "learning_rate": 2.7452583387835187e-07, + "loss": 0.11325899362564087, + "num_tokens": 29020798.0, + "reward": -4.172328260665381e-09, + "reward_std": 0.995877069234848, + "rewards/rnd_reward/mean": -4.172324707951702e-09, + "rewards/rnd_reward/std": 0.9958770573139191, + "step": 1380, + "step_time": 10.452752491505816 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09500000216066837, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.1, + "completions/mean_length": 97.97000122070312, + "completions/mean_terminated_length": 68.24239921569824, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.9521768242120743, + "epoch": 0.9090909090909091, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2329504489898682, + "kl": 0.09182704174891114, + "learning_rate": 2.7289077828646175e-07, + "loss": 0.09825693368911743, + "num_tokens": 29195334.0, + "reward": 9.715557460765468e-08, + "reward_std": 0.9959101438522339, + "rewards/rnd_reward/mean": 9.715556679168457e-08, + "rewards/rnd_reward/std": 0.9959101438522339, + "step": 1390, + "step_time": 10.322968539246357 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10125000234693289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.9, + "completions/mean_length": 103.2087516784668, + "completions/mean_terminated_length": 71.93345756530762, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 3.0182015657424928, + "epoch": 0.9156311314584696, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4850752353668213, + "kl": 0.09229131015017629, + "learning_rate": 2.7125572269457157e-07, + "loss": 0.07991544604301452, + "num_tokens": 29371486.0, + "reward": -1.5191734150654668e-07, + "reward_std": 0.995727401971817, + "rewards/rnd_reward/mean": -1.5191734008546122e-07, + "rewards/rnd_reward/std": 0.9957274079322815, + "step": 1400, + "step_time": 10.564200303750113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11875000409781933, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.6, + "completions/mean_length": 109.10750198364258, + "completions/mean_terminated_length": 72.09171180725097, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9613120049238204, + "epoch": 0.9221713538260301, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9964513778686523, + "kl": 0.0881183447316289, + "learning_rate": 2.696206671026815e-07, + "loss": 0.05779829025268555, + "num_tokens": 29547567.0, + "reward": -2.881884620276054e-07, + "reward_std": 0.9955705881118775, + "rewards/rnd_reward/mean": -2.881884591854345e-07, + "rewards/rnd_reward/std": 0.9955705881118775, + "step": 1410, + "step_time": 10.214018813963047 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000156462193, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.2, + "completions/mean_length": 102.86875228881836, + "completions/mean_terminated_length": 76.34417495727538, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.703101697564125, + "epoch": 0.9287115761935906, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8821516633033752, + "kl": 0.08714565662667155, + "learning_rate": 2.679856115107914e-07, + "loss": 0.017349694669246674, + "num_tokens": 29720227.0, + "reward": -8.821487531562867e-08, + "reward_std": 0.9958377420902252, + "rewards/rnd_reward/mean": -8.821487389454319e-08, + "rewards/rnd_reward/std": 0.9958377361297608, + "step": 1420, + "step_time": 10.117785238358191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13000000044703483, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.4, + "completions/mean_length": 115.6462516784668, + "completions/mean_terminated_length": 75.65214309692382, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.66856093108654, + "epoch": 0.935251798561151, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8306900858879089, + "kl": 0.07128150118514895, + "learning_rate": 2.663505559189012e-07, + "loss": 0.07572194337844848, + "num_tokens": 29907299.0, + "reward": -1.2844801631217705e-07, + "reward_std": 0.995860880613327, + "rewards/rnd_reward/mean": -1.2844801062783516e-07, + "rewards/rnd_reward/std": 0.9958608686923981, + "step": 1430, + "step_time": 10.222553351335227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12500000223517418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.7, + "completions/mean_length": 121.33125076293945, + "completions/mean_terminated_length": 83.87310562133788, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.7864604860544206, + "epoch": 0.9417920209287116, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.97917640209198, + "kl": 0.06842925064265729, + "learning_rate": 2.6471550032701114e-07, + "loss": 0.11840918064117431, + "num_tokens": 30100959.0, + "reward": -1.5377998749954713e-07, + "reward_std": 0.9958663046360016, + "rewards/rnd_reward/mean": -1.5377998110466252e-07, + "rewards/rnd_reward/std": 0.9958662986755371, + "step": 1440, + "step_time": 10.766110814665444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14125000238418578, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.3, + "completions/mean_length": 124.87875289916992, + "completions/mean_terminated_length": 82.31183624267578, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.699194425344467, + "epoch": 0.9483322432962721, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8881458640098572, + "kl": 0.06711579142138362, + "learning_rate": 2.6308044473512096e-07, + "loss": 0.10254174470901489, + "num_tokens": 30297562.0, + "reward": -6.161630330403512e-08, + "reward_std": 0.9957123577594758, + "rewards/rnd_reward/mean": -6.16162979749646e-08, + "rewards/rnd_reward/std": 0.9957123517990112, + "step": 1450, + "step_time": 10.593307360098697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1200000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.1, + "completions/mean_length": 116.06000213623047, + "completions/mean_terminated_length": 79.77308044433593, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.995149564743042, + "epoch": 0.9548724656638325, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0435981750488281, + "kl": 0.07414040835574269, + "learning_rate": 2.6144538914323084e-07, + "loss": 0.08301687836647034, + "num_tokens": 30483565.0, + "reward": -1.0631978923925089e-07, + "reward_std": 0.9958245873451232, + "rewards/rnd_reward/mean": -1.0631978462072311e-07, + "rewards/rnd_reward/std": 0.9958245933055878, + "step": 1460, + "step_time": 10.52764433112461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12750000283122062, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.3, + "completions/mean_length": 111.42625198364257, + "completions/mean_terminated_length": 71.7352523803711, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8058162331581116, + "epoch": 0.961412688031393, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9467556476593018, + "kl": 0.07855136888101696, + "learning_rate": 2.598103335513407e-07, + "loss": 0.10095235109329223, + "num_tokens": 30671516.0, + "reward": -1.529604193084566e-07, + "reward_std": 0.9958255529403687, + "rewards/rnd_reward/mean": -1.5296041966372798e-07, + "rewards/rnd_reward/std": 0.9958255529403687, + "step": 1470, + "step_time": 10.889759617857635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11125000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.2, + "completions/mean_length": 105.6962516784668, + "completions/mean_terminated_length": 70.84101905822754, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9369604349136353, + "epoch": 0.9679529103989536, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9438734650611877, + "kl": 0.0841084063053131, + "learning_rate": 2.581752779594506e-07, + "loss": 0.06790788173675537, + "num_tokens": 30852283.0, + "reward": -1.862645504502325e-08, + "reward_std": 0.9955705404281616, + "rewards/rnd_reward/mean": -1.8626451847580937e-08, + "rewards/rnd_reward/std": 0.9955705344676972, + "step": 1480, + "step_time": 10.544715784140863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11250000223517417, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.8, + "completions/mean_length": 111.51875228881836, + "completions/mean_terminated_length": 77.10519866943359, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.871400186419487, + "epoch": 0.9744931327665141, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0710206031799316, + "kl": 0.07257421473041177, + "learning_rate": 2.5654022236756053e-07, + "loss": 0.10279649496078491, + "num_tokens": 31039368.0, + "reward": -3.039836773410798e-08, + "reward_std": 0.9956327259540558, + "rewards/rnd_reward/mean": -3.039836915519345e-08, + "rewards/rnd_reward/std": 0.9956327199935913, + "step": 1490, + "step_time": 11.004371847352013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11750000305473804, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.9, + "completions/mean_length": 110.3587516784668, + "completions/mean_terminated_length": 74.05159416198731, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8570073008537293, + "epoch": 0.9810333551340745, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9674124717712402, + "kl": 0.07076722700148821, + "learning_rate": 2.5490516677567035e-07, + "loss": 0.09354864358901978, + "num_tokens": 31229300.0, + "reward": -7.458031374696361e-08, + "reward_std": 0.9957621216773986, + "rewards/rnd_reward/mean": -7.458031019424993e-08, + "rewards/rnd_reward/std": 0.9957621157169342, + "step": 1500, + "step_time": 10.672748254612088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09250000156462193, + "completions/max_length": 384.0, + "completions/max_terminated_length": 366.3, + "completions/mean_length": 109.77875213623047, + "completions/mean_terminated_length": 81.85236892700195, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8984388679265978, + "epoch": 0.9875735775016351, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.144245982170105, + "kl": 0.07264794735237956, + "learning_rate": 2.5327011118378023e-07, + "loss": 0.10363882780075073, + "num_tokens": 31410018.0, + "reward": 3.874301679474002e-09, + "reward_std": 0.9956199109554291, + "rewards/rnd_reward/mean": 3.874301679474002e-09, + "rewards/rnd_reward/std": 0.9956199288368225, + "step": 1510, + "step_time": 10.57373680251185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11250000223517417, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.4, + "completions/mean_length": 107.2912498474121, + "completions/mean_terminated_length": 71.87631187438964, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.835093083977699, + "epoch": 0.9941137998691956, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9730135798454285, + "kl": 0.06768459901213646, + "learning_rate": 2.516350555918901e-07, + "loss": 0.04406432807445526, + "num_tokens": 31584581.0, + "reward": -6.169080961626605e-08, + "reward_std": 0.9956278324127197, + "rewards/rnd_reward/mean": -6.169080748463785e-08, + "rewards/rnd_reward/std": 0.9956278324127197, + "step": 1520, + "step_time": 9.995071636443026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10000000149011612, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.7, + "completions/mean_length": 105.59500198364258, + "completions/mean_terminated_length": 74.4354133605957, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.818147099018097, + "epoch": 1.0006540222367561, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0657150745391846, + "kl": 0.0751381286419928, + "learning_rate": 2.5e-07, + "loss": 0.14666662216186524, + "num_tokens": 31762082.0, + "reward": -7.718801313671974e-08, + "reward_std": 0.9956592619419098, + "rewards/rnd_reward/mean": -7.718801242617701e-08, + "rewards/rnd_reward/std": 0.9956592619419098, + "step": 1530, + "step_time": 10.494197542965413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.0, + "completions/mean_length": 100.37250137329102, + "completions/mean_terminated_length": 73.99925155639649, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.850705623626709, + "epoch": 1.0071942446043165, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8915496468544006, + "kl": 0.08450272670015693, + "learning_rate": 2.4836494440810987e-07, + "loss": 0.10265235900878907, + "num_tokens": 31935920.0, + "reward": -7.78958216152148e-08, + "reward_std": 0.9955736041069031, + "rewards/rnd_reward/mean": -7.789581664141565e-08, + "rewards/rnd_reward/std": 0.995573616027832, + "step": 1540, + "step_time": 10.282078485726379 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09625000357627869, + "completions/max_length": 384.0, + "completions/max_terminated_length": 339.5, + "completions/mean_length": 100.54375, + "completions/mean_terminated_length": 70.20137939453124, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7832202523946763, + "epoch": 1.013734466971877, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9670330882072449, + "kl": 0.07673759749159217, + "learning_rate": 2.4672988881621975e-07, + "loss": 0.053245991468429565, + "num_tokens": 32105575.0, + "reward": -7.98702268323126e-08, + "reward_std": 0.9954982161521911, + "rewards/rnd_reward/mean": -7.987022256905619e-08, + "rewards/rnd_reward/std": 0.9954982340335846, + "step": 1550, + "step_time": 10.094654931733384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09625000134110451, + "completions/max_length": 384.0, + "completions/max_terminated_length": 356.6, + "completions/mean_length": 105.8862518310547, + "completions/mean_terminated_length": 76.32862396240235, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.806140884757042, + "epoch": 1.0202746893394374, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9716780185699463, + "kl": 0.07651977436617016, + "learning_rate": 2.450948332243296e-07, + "loss": 0.07588104605674743, + "num_tokens": 32282264.0, + "reward": -1.2878328945475914e-07, + "reward_std": 0.9953814685344696, + "rewards/rnd_reward/mean": -1.2878328448096e-07, + "rewards/rnd_reward/std": 0.9953814685344696, + "step": 1560, + "step_time": 10.249592515034601 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09250000044703484, + "completions/max_length": 384.0, + "completions/max_terminated_length": 370.2, + "completions/mean_length": 101.42500152587891, + "completions/mean_terminated_length": 72.58811225891114, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.706285873055458, + "epoch": 1.026814911706998, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2053457498550415, + "kl": 0.07990584680810571, + "learning_rate": 2.434597776324395e-07, + "loss": 0.08478202223777771, + "num_tokens": 32458489.0, + "reward": -1.197308309830447e-07, + "reward_std": 0.9958563566207885, + "rewards/rnd_reward/mean": -1.1973082814087377e-07, + "rewards/rnd_reward/std": 0.9958563566207885, + "step": 1570, + "step_time": 10.385891513410025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000223517418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.3, + "completions/mean_length": 92.8375015258789, + "completions/mean_terminated_length": 65.02403182983399, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.832634910941124, + "epoch": 1.0333551340745586, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2888636589050293, + "kl": 0.08601820273324848, + "learning_rate": 2.418247220405494e-07, + "loss": 0.024559563398361205, + "num_tokens": 32626059.0, + "reward": -1.628696985278566e-07, + "reward_std": 0.9954473912715912, + "rewards/rnd_reward/mean": -1.6286969000134378e-07, + "rewards/rnd_reward/std": 0.9954474031925201, + "step": 1580, + "step_time": 10.754275933234021 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09125000201165676, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.7, + "completions/mean_length": 99.23750076293945, + "completions/mean_terminated_length": 70.4692756652832, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.9075894236564634, + "epoch": 1.039895356442119, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0340582132339478, + "kl": 0.0824001532047987, + "learning_rate": 2.4018966644865926e-07, + "loss": 0.04327432513236999, + "num_tokens": 32797839.0, + "reward": 6.616115513224941e-08, + "reward_std": 0.9953638434410095, + "rewards/rnd_reward/mean": 6.61611530006212e-08, + "rewards/rnd_reward/std": 0.9953638732433319, + "step": 1590, + "step_time": 10.39529764095787 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1075000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.2, + "completions/mean_length": 105.15875091552735, + "completions/mean_terminated_length": 71.44011192321777, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 3.068614384531975, + "epoch": 1.0464355788096795, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9280741810798645, + "kl": 0.08188259806483984, + "learning_rate": 2.3855461085676914e-07, + "loss": 0.07503924369812012, + "num_tokens": 32970486.0, + "reward": 2.354383532576776e-08, + "reward_std": 0.9955627143383026, + "rewards/rnd_reward/mean": 2.3543834259953656e-08, + "rewards/rnd_reward/std": 0.9955627143383026, + "step": 1600, + "step_time": 10.102484514610842 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12375000268220901, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.0, + "completions/mean_length": 108.67875137329102, + "completions/mean_terminated_length": 69.76623840332032, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.7116469889879227, + "epoch": 1.05297580117724, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8361508846282959, + "kl": 0.07135752588510513, + "learning_rate": 2.36919555264879e-07, + "loss": 0.08789730072021484, + "num_tokens": 33155644.0, + "reward": -7.331371434382561e-08, + "reward_std": 0.9953298509120941, + "rewards/rnd_reward/mean": -7.331371363328287e-08, + "rewards/rnd_reward/std": 0.9953298389911651, + "step": 1610, + "step_time": 10.373006676253862 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10250000208616257, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.6, + "completions/mean_length": 107.08625106811523, + "completions/mean_terminated_length": 75.19281349182128, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.767005005478859, + "epoch": 1.0595160235448005, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8207447528839111, + "kl": 0.07195558175444602, + "learning_rate": 2.352844996729889e-07, + "loss": 0.07792651653289795, + "num_tokens": 33333863.0, + "reward": -7.919967437430842e-08, + "reward_std": 0.9952020287513733, + "rewards/rnd_reward/mean": -7.919966940050927e-08, + "rewards/rnd_reward/std": 0.9952020525932312, + "step": 1620, + "step_time": 10.309986177436077 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12875000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.7, + "completions/mean_length": 117.29250259399414, + "completions/mean_terminated_length": 78.07871360778809, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8041924834251404, + "epoch": 1.066056245912361, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7970576882362366, + "kl": 0.07171947872266174, + "learning_rate": 2.3364944408109875e-07, + "loss": 0.07321782112121582, + "num_tokens": 33529112.0, + "reward": -7.54743808784042e-08, + "reward_std": 0.9954242706298828, + "rewards/rnd_reward/mean": -7.54743808784042e-08, + "rewards/rnd_reward/std": 0.9954242765903473, + "step": 1630, + "step_time": 11.020952883246355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10375000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 367.8, + "completions/mean_length": 109.85750122070313, + "completions/mean_terminated_length": 78.08732643127442, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7948483556509016, + "epoch": 1.0725964682799216, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0839210748672485, + "kl": 0.07083125235512852, + "learning_rate": 2.3201438848920862e-07, + "loss": 0.07660450935363769, + "num_tokens": 33711328.0, + "reward": -4.068016963287846e-08, + "reward_std": 0.9954971015453339, + "rewards/rnd_reward/mean": -4.068016963287846e-08, + "rewards/rnd_reward/std": 0.9954971194267273, + "step": 1640, + "step_time": 10.339221865893341 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09125000033527612, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.3, + "completions/mean_length": 100.59000167846679, + "completions/mean_terminated_length": 72.11472930908204, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7371392965316774, + "epoch": 1.079136690647482, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9994949698448181, + "kl": 0.08674983121454716, + "learning_rate": 2.303793328973185e-07, + "loss": 0.14760398864746094, + "num_tokens": 33894090.0, + "reward": -4.775822315394862e-08, + "reward_std": 0.9951541960239411, + "rewards/rnd_reward/mean": -4.7758222088134514e-08, + "rewards/rnd_reward/std": 0.9951542019844055, + "step": 1650, + "step_time": 11.004630160145462 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09750000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.6, + "completions/mean_length": 103.39125213623046, + "completions/mean_terminated_length": 73.03910675048829, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.719507083296776, + "epoch": 1.0856769130150425, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0636616945266724, + "kl": 0.07622466497123241, + "learning_rate": 2.2874427730542835e-07, + "loss": 0.11726703643798828, + "num_tokens": 34065638.0, + "reward": -6.273389079325397e-08, + "reward_std": 0.9954430878162384, + "rewards/rnd_reward/mean": -6.273388937216851e-08, + "rewards/rnd_reward/std": 0.9954430878162384, + "step": 1660, + "step_time": 10.205868328805082 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11125000230967999, + "completions/max_length": 384.0, + "completions/max_terminated_length": 337.9, + "completions/mean_length": 103.16000251770019, + "completions/mean_terminated_length": 68.0124439239502, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.555436444282532, + "epoch": 1.0922171353826031, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4149969816207886, + "kl": 0.08045686418190598, + "learning_rate": 2.2710922171353826e-07, + "loss": 0.18388016223907472, + "num_tokens": 34243911.0, + "reward": -6.049871625890546e-08, + "reward_std": 0.995277750492096, + "rewards/rnd_reward/mean": -6.049871181801336e-08, + "rewards/rnd_reward/std": 0.9952777206897736, + "step": 1670, + "step_time": 10.405935624614358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09250000230967999, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.5, + "completions/mean_length": 98.8687515258789, + "completions/mean_terminated_length": 69.8191291809082, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.679945033788681, + "epoch": 1.0987573577501635, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1418001651763916, + "kl": 0.07972979731857777, + "learning_rate": 2.2547416612164814e-07, + "loss": 0.08271819949150086, + "num_tokens": 34422086.0, + "reward": -2.2441148974650105e-07, + "reward_std": 0.9950048804283143, + "rewards/rnd_reward/mean": -2.2441148654905875e-07, + "rewards/rnd_reward/std": 0.9950048506259919, + "step": 1680, + "step_time": 10.670063342596404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000141561032, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.9, + "completions/mean_length": 95.35000228881836, + "completions/mean_terminated_length": 67.31705818176269, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6938273787498472, + "epoch": 1.105297580117724, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0555897951126099, + "kl": 0.0881148618645966, + "learning_rate": 2.2383911052975802e-07, + "loss": 0.057183104753494265, + "num_tokens": 34592661.0, + "reward": -5.260109574578564e-08, + "reward_std": 0.9953494489192962, + "rewards/rnd_reward/mean": -5.260109787741385e-08, + "rewards/rnd_reward/std": 0.9953494608402252, + "step": 1690, + "step_time": 10.320013692067004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09500000197440386, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 96.38625259399414, + "completions/mean_terminated_length": 66.25210113525391, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8528232872486115, + "epoch": 1.1118378024852844, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1842913627624512, + "kl": 0.09175272611901164, + "learning_rate": 2.2220405493786787e-07, + "loss": 0.11493265628814697, + "num_tokens": 34767800.0, + "reward": -3.5762789174498266e-08, + "reward_std": 0.9952230334281922, + "rewards/rnd_reward/mean": -3.5762784911241854e-08, + "rewards/rnd_reward/std": 0.9952230513095855, + "step": 1700, + "step_time": 11.20805806266144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06625000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.7, + "completions/mean_length": 88.85, + "completions/mean_terminated_length": 67.81800193786621, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7997246354818346, + "epoch": 1.118378024852845, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3541666269302368, + "kl": 0.0883994361385703, + "learning_rate": 2.2056899934597775e-07, + "loss": 0.10645198822021484, + "num_tokens": 34940290.0, + "reward": -9.074807572062582e-08, + "reward_std": 0.9952512562274933, + "rewards/rnd_reward/mean": -9.074807181264077e-08, + "rewards/rnd_reward/std": 0.9952512562274933, + "step": 1710, + "step_time": 11.268909463495948 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08375000078231096, + "completions/max_length": 384.0, + "completions/max_terminated_length": 341.1, + "completions/mean_length": 92.34500198364258, + "completions/mean_terminated_length": 65.63225555419922, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5688311845064162, + "epoch": 1.1249182472204056, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.086549997329712, + "kl": 0.08725747494027018, + "learning_rate": 2.1893394375408762e-07, + "loss": 0.09306548833847046, + "num_tokens": 35107316.0, + "reward": -1.1682510461241691e-07, + "reward_std": 0.9951877236366272, + "rewards/rnd_reward/mean": -1.1682510461241691e-07, + "rewards/rnd_reward/std": 0.9951877236366272, + "step": 1720, + "step_time": 10.925348741887138 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000104308128, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.1, + "completions/mean_length": 97.91875076293945, + "completions/mean_terminated_length": 70.1454303741455, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.690840741991997, + "epoch": 1.131458469587966, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0729917287826538, + "kl": 0.08789877109229564, + "learning_rate": 2.172988881621975e-07, + "loss": 0.10979057550430298, + "num_tokens": 35281881.0, + "reward": -1.907348732288483e-08, + "reward_std": 0.995078319311142, + "rewards/rnd_reward/mean": -1.907348732288483e-08, + "rewards/rnd_reward/std": 0.9950783312320709, + "step": 1730, + "step_time": 10.632008279883303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000223517418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.8, + "completions/mean_length": 96.91125183105468, + "completions/mean_terminated_length": 69.48632164001465, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.681867742538452, + "epoch": 1.1379986919555265, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0131276845932007, + "kl": 0.08944809632375836, + "learning_rate": 2.1566383257030738e-07, + "loss": 0.10518434047698974, + "num_tokens": 35461970.0, + "reward": 2.6673077968553116e-08, + "reward_std": 0.9949174404144288, + "rewards/rnd_reward/mean": 2.6673081521266795e-08, + "rewards/rnd_reward/std": 0.9949174463748932, + "step": 1740, + "step_time": 11.253412028984167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0787500012665987, + "completions/max_length": 384.0, + "completions/max_terminated_length": 338.7, + "completions/mean_length": 84.14750061035156, + "completions/mean_terminated_length": 58.63842086791992, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.567233794927597, + "epoch": 1.144538914323087, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1535600423812866, + "kl": 0.09054145393893123, + "learning_rate": 2.1402877697841726e-07, + "loss": 0.10667283535003662, + "num_tokens": 35622023.0, + "reward": -5.97536580215774e-08, + "reward_std": 0.9952092111110687, + "rewards/rnd_reward/mean": -5.975365517940645e-08, + "rewards/rnd_reward/std": 0.9952091991901397, + "step": 1750, + "step_time": 10.509924068162217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07000000029802322, + "completions/max_length": 384.0, + "completions/max_terminated_length": 331.8, + "completions/mean_length": 86.03000106811524, + "completions/mean_terminated_length": 63.5996639251709, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.5665056079626085, + "epoch": 1.1510791366906474, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1171236038208008, + "kl": 0.08619214734062552, + "learning_rate": 2.1239372138652714e-07, + "loss": 0.07680293321609497, + "num_tokens": 35780972.0, + "reward": -4.529952919440916e-08, + "reward_std": 0.9950806498527527, + "rewards/rnd_reward/mean": -4.529952919440916e-08, + "rewards/rnd_reward/std": 0.9950806617736816, + "step": 1760, + "step_time": 10.277455605636352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06875000074505806, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.5, + "completions/mean_length": 89.81625213623047, + "completions/mean_terminated_length": 68.19218559265137, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.563973236083984, + "epoch": 1.157619359058208, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1521550416946411, + "kl": 0.0870651968754828, + "learning_rate": 2.10758665794637e-07, + "loss": 0.1115831971168518, + "num_tokens": 35948090.0, + "reward": 1.3858081859829951e-08, + "reward_std": 0.9948914706707, + "rewards/rnd_reward/mean": 1.38580813269229e-08, + "rewards/rnd_reward/std": 0.9948914825916291, + "step": 1770, + "step_time": 10.7476754687028 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07750000115483999, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.2, + "completions/mean_length": 85.87875022888184, + "completions/mean_terminated_length": 60.88022689819336, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.646288427710533, + "epoch": 1.1641595814257686, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9658340811729431, + "kl": 0.09920121757313609, + "learning_rate": 2.091236102027469e-07, + "loss": 0.08330439329147339, + "num_tokens": 36114108.0, + "reward": -7.227063285597523e-08, + "reward_std": 0.9949531733989716, + "rewards/rnd_reward/mean": -7.227063250070387e-08, + "rewards/rnd_reward/std": 0.994953167438507, + "step": 1780, + "step_time": 11.134026414644904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06250000018626452, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.1, + "completions/mean_length": 84.51375122070313, + "completions/mean_terminated_length": 64.53133659362793, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.704926884174347, + "epoch": 1.170699803793329, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2755341529846191, + "kl": 0.10716334227472543, + "learning_rate": 2.0748855461085677e-07, + "loss": 0.13019881248474122, + "num_tokens": 36271504.0, + "reward": -1.305341750246214e-07, + "reward_std": 0.9949664771556854, + "rewards/rnd_reward/mean": -1.305341714719077e-07, + "rewards/rnd_reward/std": 0.9949664771556854, + "step": 1790, + "step_time": 10.397847580048255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000175088644, + "completions/max_length": 384.0, + "completions/max_terminated_length": 326.3, + "completions/mean_length": 87.63249969482422, + "completions/mean_terminated_length": 59.75216636657715, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.593474331498146, + "epoch": 1.1772400261608895, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.155380129814148, + "kl": 0.09049692396074534, + "learning_rate": 2.0585349901896665e-07, + "loss": 0.12421066761016845, + "num_tokens": 36439860.0, + "reward": 1.50501714202278e-08, + "reward_std": 0.9950001060962677, + "rewards/rnd_reward/mean": 1.505017284131327e-08, + "rewards/rnd_reward/std": 0.9950001001358032, + "step": 1800, + "step_time": 10.84767303718254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06250000158324838, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.8, + "completions/mean_length": 91.63375244140624, + "completions/mean_terminated_length": 72.32135162353515, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5027853429317473, + "epoch": 1.1837802485284499, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0669174194335938, + "kl": 0.08057954236865043, + "learning_rate": 2.042184434270765e-07, + "loss": 0.09477431774139404, + "num_tokens": 36602592.0, + "reward": -1.505017284131327e-07, + "reward_std": 0.994869589805603, + "rewards/rnd_reward/mean": -1.5050172876840407e-07, + "rewards/rnd_reward/std": 0.9948696017265319, + "step": 1810, + "step_time": 10.333302733465098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000216066838, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.4, + "completions/mean_length": 92.49750137329102, + "completions/mean_terminated_length": 68.43786659240723, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.487256759405136, + "epoch": 1.1903204708960105, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.864107608795166, + "kl": 0.07850290769711137, + "learning_rate": 2.0258338783518638e-07, + "loss": 0.09753584861755371, + "num_tokens": 36768390.0, + "reward": -1.2189149920516229e-07, + "reward_std": 0.9947888195514679, + "rewards/rnd_reward/mean": -1.2189150062624775e-07, + "rewards/rnd_reward/std": 0.9947888076305389, + "step": 1820, + "step_time": 10.366325606242754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07500000111758709, + "completions/max_length": 384.0, + "completions/max_terminated_length": 327.2, + "completions/mean_length": 87.97500152587891, + "completions/mean_terminated_length": 64.1132999420166, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5121534377336503, + "epoch": 1.196860693263571, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8458340167999268, + "kl": 0.08256649980321526, + "learning_rate": 2.0094833224329626e-07, + "loss": 0.06835322976112365, + "num_tokens": 36933960.0, + "reward": -1.531839394885992e-07, + "reward_std": 0.9951117813587189, + "rewards/rnd_reward/mean": -1.5318393664642826e-07, + "rewards/rnd_reward/std": 0.9951117634773254, + "step": 1830, + "step_time": 11.365050849993713 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07375000137835741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.0, + "completions/mean_length": 94.00125045776367, + "completions/mean_terminated_length": 71.0650505065918, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.4178095728158953, + "epoch": 1.2034009156311314, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1388354301452637, + "kl": 0.0803013733588159, + "learning_rate": 1.9931327665140614e-07, + "loss": 0.12810555696487427, + "num_tokens": 37099636.0, + "reward": -4.909932687091611e-08, + "reward_std": 0.9949400007724762, + "rewards/rnd_reward/mean": -4.909932758145885e-08, + "rewards/rnd_reward/std": 0.9949400067329407, + "step": 1840, + "step_time": 10.505639822757804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06875000093132258, + "completions/max_length": 384.0, + "completions/max_terminated_length": 341.5, + "completions/mean_length": 91.8550018310547, + "completions/mean_terminated_length": 70.10141868591309, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.4157417356967925, + "epoch": 1.209941137998692, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0894784927368164, + "kl": 0.08061180328950286, + "learning_rate": 1.9767822105951601e-07, + "loss": 0.09716765880584717, + "num_tokens": 37269225.0, + "reward": -4.3064359189770585e-08, + "reward_std": 0.9946663022041321, + "rewards/rnd_reward/mean": -4.306435528178554e-08, + "rewards/rnd_reward/std": 0.9946663022041321, + "step": 1850, + "step_time": 11.422960984380916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000074505807, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.2, + "completions/mean_length": 93.6987518310547, + "completions/mean_terminated_length": 65.78671073913574, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5770903497934343, + "epoch": 1.2164813603662525, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9160525798797607, + "kl": 0.08047085385769606, + "learning_rate": 1.960431654676259e-07, + "loss": 0.05968161225318909, + "num_tokens": 37440344.0, + "reward": 2.6524067830990816e-08, + "reward_std": 0.9945189356803894, + "rewards/rnd_reward/mean": 2.65240635677344e-08, + "rewards/rnd_reward/std": 0.9945189356803894, + "step": 1860, + "step_time": 10.725451961229556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000063329935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 335.1, + "completions/mean_length": 95.60625076293945, + "completions/mean_terminated_length": 68.25989952087403, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.4541951596736906, + "epoch": 1.223021582733813, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.999136209487915, + "kl": 0.07508358927443623, + "learning_rate": 1.9440810987573577e-07, + "loss": 0.13460888862609863, + "num_tokens": 37611004.0, + "reward": -1.23679642705099e-07, + "reward_std": 0.9949141681194306, + "rewards/rnd_reward/mean": -1.236796327575007e-07, + "rewards/rnd_reward/std": 0.9949141621589661, + "step": 1870, + "step_time": 10.482190745323896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 328.5, + "completions/mean_length": 89.38125152587891, + "completions/mean_terminated_length": 65.09402084350586, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.567149502038956, + "epoch": 1.2295618051013735, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2019909620285034, + "kl": 0.08660045554861426, + "learning_rate": 1.9277305428384562e-07, + "loss": 0.20026679039001466, + "num_tokens": 37786329.0, + "reward": -8.866190821521514e-08, + "reward_std": 0.9948227822780609, + "rewards/rnd_reward/mean": -8.866190999157198e-08, + "rewards/rnd_reward/std": 0.9948227941989899, + "step": 1880, + "step_time": 11.85091487034224 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06375000011175871, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.9, + "completions/mean_length": 89.55750122070313, + "completions/mean_terminated_length": 69.49982872009278, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8322397589683534, + "epoch": 1.236102027468934, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1517750024795532, + "kl": 0.09399051470682025, + "learning_rate": 1.9113799869195553e-07, + "loss": 0.17799322605133056, + "num_tokens": 37947720.0, + "reward": -1.0848045732814171e-07, + "reward_std": 0.9948072552680969, + "rewards/rnd_reward/mean": -1.0848045448597076e-07, + "rewards/rnd_reward/std": 0.9948072493076324, + "step": 1890, + "step_time": 10.414563778741286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06625000014901161, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.4, + "completions/mean_length": 86.39500045776367, + "completions/mean_terminated_length": 65.30856475830078, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7242774665355682, + "epoch": 1.2426422498364944, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3484736680984497, + "kl": 0.09410189539194107, + "learning_rate": 1.895029431000654e-07, + "loss": 0.0743591845035553, + "num_tokens": 38111876.0, + "reward": -6.467104114449285e-08, + "reward_std": 0.9947612941265106, + "rewards/rnd_reward/mean": -6.467103901286464e-08, + "rewards/rnd_reward/std": 0.9947613418102265, + "step": 1900, + "step_time": 10.527994227781893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06625000108033419, + "completions/max_length": 384.0, + "completions/max_terminated_length": 338.0, + "completions/mean_length": 84.77500114440917, + "completions/mean_terminated_length": 63.568695068359375, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.954171320796013, + "epoch": 1.249182472204055, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9760697484016418, + "kl": 0.10021719867363572, + "learning_rate": 1.8786788750817528e-07, + "loss": 0.02971080243587494, + "num_tokens": 38279376.0, + "reward": -1.5616416861874428e-07, + "reward_std": 0.9947503983974457, + "rewards/rnd_reward/mean": -1.5616416790820153e-07, + "rewards/rnd_reward/std": 0.9947503864765167, + "step": 1910, + "step_time": 11.339212405891157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08000000193715096, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.5, + "completions/mean_length": 88.47500076293946, + "completions/mean_terminated_length": 62.80438385009766, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.846560522913933, + "epoch": 1.2557226945716153, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2370659112930298, + "kl": 0.09515976402908563, + "learning_rate": 1.8623283191628514e-07, + "loss": 0.10529450178146363, + "num_tokens": 38449416.0, + "reward": -3.65078278719011e-09, + "reward_std": 0.9944445490837097, + "rewards/rnd_reward/mean": -3.6507834977328456e-09, + "rewards/rnd_reward/std": 0.9944445431232453, + "step": 1920, + "step_time": 10.698268468677998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000160187483, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.8, + "completions/mean_length": 87.2925018310547, + "completions/mean_terminated_length": 62.797546768188475, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.9005165725946425, + "epoch": 1.262262916939176, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3234843015670776, + "kl": 0.10119491461664439, + "learning_rate": 1.8459777632439501e-07, + "loss": 0.014341549575328827, + "num_tokens": 38616700.0, + "reward": -4.380941049930698e-08, + "reward_std": 0.9944035768508911, + "rewards/rnd_reward/mean": -4.3809414407292026e-08, + "rewards/rnd_reward/std": 0.994403600692749, + "step": 1930, + "step_time": 11.375351972319185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000111758709, + "completions/max_length": 384.0, + "completions/max_terminated_length": 334.7, + "completions/mean_length": 96.3550018310547, + "completions/mean_terminated_length": 68.85774459838868, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.9528768658638, + "epoch": 1.2688031393067365, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1305567026138306, + "kl": 0.09469359386712313, + "learning_rate": 1.829627207325049e-07, + "loss": 0.0362447589635849, + "num_tokens": 38788569.0, + "reward": -5.45382517813664e-08, + "reward_std": 0.9948436379432678, + "rewards/rnd_reward/mean": -5.4538249649738193e-08, + "rewards/rnd_reward/std": 0.9948436439037323, + "step": 1940, + "step_time": 10.781422062939964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10000000298023223, + "completions/max_length": 384.0, + "completions/max_terminated_length": 342.1, + "completions/mean_length": 93.3875015258789, + "completions/mean_terminated_length": 61.06837921142578, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7504739254713058, + "epoch": 1.2753433616742968, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.153893232345581, + "kl": 0.08643652414903044, + "learning_rate": 1.8132766514061477e-07, + "loss": 0.10290535688400268, + "num_tokens": 38954489.0, + "reward": -1.2144446595030445e-07, + "reward_std": 0.9947971343994141, + "rewards/rnd_reward/mean": -1.2144446346340487e-07, + "rewards/rnd_reward/std": 0.994797146320343, + "step": 1950, + "step_time": 10.527900594798847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0950000025331974, + "completions/max_length": 384.0, + "completions/max_terminated_length": 329.3, + "completions/mean_length": 100.58875122070313, + "completions/mean_terminated_length": 70.79409141540528, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.7141620010137557, + "epoch": 1.2818835840418574, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0054807662963867, + "kl": 0.08213893175125123, + "learning_rate": 1.7969260954872465e-07, + "loss": 0.07241884469985962, + "num_tokens": 39126695.0, + "reward": -1.028180136586343e-08, + "reward_std": 0.9948840856552124, + "rewards/rnd_reward/mean": -1.0281802786948902e-08, + "rewards/rnd_reward/std": 0.9948840975761414, + "step": 1960, + "step_time": 10.461326698656194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06750000081956387, + "completions/max_length": 383.2, + "completions/max_terminated_length": 350.0, + "completions/mean_length": 88.08000106811524, + "completions/mean_terminated_length": 66.5671230316162, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.7966362595558167, + "epoch": 1.2884238064094178, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1359318494796753, + "kl": 0.09009936284273863, + "learning_rate": 1.7805755395683453e-07, + "loss": 0.12549155950546265, + "num_tokens": 39294724.0, + "reward": 4.082918199088681e-08, + "reward_std": 0.9944938480854034, + "rewards/rnd_reward/mean": 4.0829182701429545e-08, + "rewards/rnd_reward/std": 0.9944938659667969, + "step": 1970, + "step_time": 10.983743568998761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.4, + "completions/mean_length": 95.8025016784668, + "completions/mean_terminated_length": 71.99091606140136, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7956566989421843, + "epoch": 1.2949640287769784, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3064550161361694, + "kl": 0.08204388581216335, + "learning_rate": 1.764224983649444e-07, + "loss": 0.06261605024337769, + "num_tokens": 39465981.0, + "reward": -7.681548792604076e-08, + "reward_std": 0.9945617258548737, + "rewards/rnd_reward/mean": -7.681548792604076e-08, + "rewards/rnd_reward/std": 0.9945617258548737, + "step": 1980, + "step_time": 10.856887764390558 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10500000119209289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 336.7, + "completions/mean_length": 99.39125137329101, + "completions/mean_terminated_length": 66.14606056213378, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.809012603759766, + "epoch": 1.301504251144539, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0052663087844849, + "kl": 0.08238375652581453, + "learning_rate": 1.7478744277305426e-07, + "loss": 0.04824514389038086, + "num_tokens": 39642064.0, + "reward": -2.3305416743113483e-07, + "reward_std": 0.9944523334503174, + "rewards/rnd_reward/mean": -2.3305416174679294e-07, + "rewards/rnd_reward/std": 0.9944523453712464, + "step": 1990, + "step_time": 10.668248945730738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06875000093132258, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.9, + "completions/mean_length": 94.09000167846679, + "completions/mean_terminated_length": 72.67349357604981, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.782809841632843, + "epoch": 1.3080444735120995, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.088478684425354, + "kl": 0.08519633868709206, + "learning_rate": 1.7315238718116416e-07, + "loss": 0.12635375261306764, + "num_tokens": 39807401.0, + "reward": -9.67830406706227e-08, + "reward_std": 0.9944651544094085, + "rewards/rnd_reward/mean": -9.67830406706227e-08, + "rewards/rnd_reward/std": 0.9944651544094085, + "step": 2000, + "step_time": 10.477193391043693 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08125000074505806, + "completions/max_length": 384.0, + "completions/max_terminated_length": 333.7, + "completions/mean_length": 97.57875061035156, + "completions/mean_terminated_length": 72.30035781860352, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.851186656951904, + "epoch": 1.3145846958796599, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0262951850891113, + "kl": 0.08686133781448006, + "learning_rate": 1.7151733158927404e-07, + "loss": 0.16627882719039916, + "num_tokens": 39979409.0, + "reward": -1.8477440306696734e-07, + "reward_std": 0.9941576004028321, + "rewards/rnd_reward/mean": -1.8477439738262548e-07, + "rewards/rnd_reward/std": 0.994157612323761, + "step": 2010, + "step_time": 10.494085661577992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07875000070780516, + "completions/max_length": 384.0, + "completions/max_terminated_length": 332.3, + "completions/mean_length": 98.56625061035156, + "completions/mean_terminated_length": 74.01468200683594, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.6910068154335023, + "epoch": 1.3211249182472204, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.180172085762024, + "kl": 0.08328518448397518, + "learning_rate": 1.6988227599738392e-07, + "loss": 0.1285926580429077, + "num_tokens": 40151192.0, + "reward": -2.4214387792653726e-07, + "reward_std": 0.9945690989494324, + "rewards/rnd_reward/mean": -2.421438694000244e-07, + "rewards/rnd_reward/std": 0.9945691049098968, + "step": 2020, + "step_time": 10.840997534058989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06375000085681677, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.1, + "completions/mean_length": 86.02375068664551, + "completions/mean_terminated_length": 65.85984230041504, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.954036074876785, + "epoch": 1.3276651406147808, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1177207231521606, + "kl": 0.09944681767374278, + "learning_rate": 1.6824722040549377e-07, + "loss": 0.07238306999206542, + "num_tokens": 40313136.0, + "reward": -2.1941960497429137e-07, + "reward_std": 0.9945330858230591, + "rewards/rnd_reward/mean": -2.1941959928994948e-07, + "rewards/rnd_reward/std": 0.9945330798625946, + "step": 2030, + "step_time": 10.803823693329468 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11000000201165676, + "completions/max_length": 384.0, + "completions/max_terminated_length": 326.8, + "completions/mean_length": 100.16625137329102, + "completions/mean_terminated_length": 64.80138359069824, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 3.0097627967596052, + "epoch": 1.3342053629823414, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1276546716690063, + "kl": 0.08885867483913898, + "learning_rate": 1.6661216481360365e-07, + "loss": 0.010317666083574295, + "num_tokens": 40489459.0, + "reward": -1.4454126961993552e-07, + "reward_std": 0.9944145083427429, + "rewards/rnd_reward/mean": -1.4454125896179448e-07, + "rewards/rnd_reward/std": 0.994414496421814, + "step": 2040, + "step_time": 11.046867836569437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09500000067055225, + "completions/max_length": 384.0, + "completions/max_terminated_length": 329.5, + "completions/mean_length": 96.87375183105469, + "completions/mean_terminated_length": 66.7010440826416, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.999319186806679, + "epoch": 1.340745585349902, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0046088695526123, + "kl": 0.09102731477469206, + "learning_rate": 1.6497710922171353e-07, + "loss": 0.0822763979434967, + "num_tokens": 40655648.0, + "reward": -1.0967255033023094e-07, + "reward_std": 0.9943422853946686, + "rewards/rnd_reward/mean": -1.0967254464588904e-07, + "rewards/rnd_reward/std": 0.9943422734737396, + "step": 2050, + "step_time": 10.422167311538942 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0912500012665987, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.1, + "completions/mean_length": 97.65625228881837, + "completions/mean_terminated_length": 68.92426071166992, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8394038140773774, + "epoch": 1.3472858077174623, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9094915986061096, + "kl": 0.08663466572761536, + "learning_rate": 1.633420536298234e-07, + "loss": 0.020597906410694124, + "num_tokens": 40822473.0, + "reward": 1.2516974123855107e-08, + "reward_std": 0.9942900657653808, + "rewards/rnd_reward/mean": 1.251697430149079e-08, + "rewards/rnd_reward/std": 0.9942900657653808, + "step": 2060, + "step_time": 10.357499032304622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09500000141561031, + "completions/max_length": 384.0, + "completions/max_terminated_length": 323.5, + "completions/mean_length": 97.2625015258789, + "completions/mean_terminated_length": 67.25451889038087, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.869212657213211, + "epoch": 1.353826030085023, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9482806921005249, + "kl": 0.08622422777116298, + "learning_rate": 1.6170699803793328e-07, + "loss": 0.05238085389137268, + "num_tokens": 40987073.0, + "reward": -1.0728836219442428e-07, + "reward_std": 0.9939788460731507, + "rewards/rnd_reward/mean": -1.0728835935225333e-07, + "rewards/rnd_reward/std": 0.9939788401126861, + "step": 2070, + "step_time": 10.501573320990428 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08250000290572643, + "completions/max_length": 384.0, + "completions/max_terminated_length": 334.2, + "completions/mean_length": 91.6250015258789, + "completions/mean_terminated_length": 65.58112106323242, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.9047418594360352, + "epoch": 1.3603662524525835, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.018664002418518, + "kl": 0.08782740281894803, + "learning_rate": 1.6007194244604316e-07, + "loss": 0.15487767457962037, + "num_tokens": 41152698.0, + "reward": -1.256167863061819e-07, + "reward_std": 0.9939099371433258, + "rewards/rnd_reward/mean": -1.2561678914835283e-07, + "rewards/rnd_reward/std": 0.9939099371433258, + "step": 2080, + "step_time": 10.78773567241151 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09625000096857547, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.6, + "completions/mean_length": 100.0337516784668, + "completions/mean_terminated_length": 69.91814079284669, + "completions/min_length": 1.9, + "completions/min_terminated_length": 1.9, + "entropy": 2.7196634620428086, + "epoch": 1.3669064748201438, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0979881286621094, + "kl": 0.0808198869228363, + "learning_rate": 1.5843688685415304e-07, + "loss": 0.131181538105011, + "num_tokens": 41330705.0, + "reward": -1.9848346672901583e-07, + "reward_std": 0.9943111777305603, + "rewards/rnd_reward/mean": -1.9848346672901583e-07, + "rewards/rnd_reward/std": 0.9943111896514892, + "step": 2090, + "step_time": 11.131318410113455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11250000149011612, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.0, + "completions/mean_length": 100.51125030517578, + "completions/mean_terminated_length": 64.72891502380371, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.9781338214874267, + "epoch": 1.3734466971877044, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0100359916687012, + "kl": 0.08852830519899726, + "learning_rate": 1.568018312622629e-07, + "loss": 0.1010061264038086, + "num_tokens": 41512969.0, + "reward": -2.2500751484244574e-08, + "reward_std": 0.9940204381942749, + "rewards/rnd_reward/mean": -2.2500754326415517e-08, + "rewards/rnd_reward/std": 0.9940204560756684, + "step": 2100, + "step_time": 11.608714979188516 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09250000165775418, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.9, + "completions/mean_length": 99.50000114440918, + "completions/mean_terminated_length": 70.67034187316895, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.7862779945135117, + "epoch": 1.379986919555265, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.939647376537323, + "kl": 0.08671271968632936, + "learning_rate": 1.551667756703728e-07, + "loss": 0.07131583094596863, + "num_tokens": 41688714.0, + "reward": -5.654990502534929e-08, + "reward_std": 0.9939040839672089, + "rewards/rnd_reward/mean": -5.654991070969118e-08, + "rewards/rnd_reward/std": 0.9939040839672089, + "step": 2110, + "step_time": 10.6546359580243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.2, + "completions/mean_length": 105.17625045776367, + "completions/mean_terminated_length": 79.25297164916992, + "completions/min_length": 1.9, + "completions/min_terminated_length": 1.9, + "entropy": 2.7606303304433824, + "epoch": 1.3865271419228253, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.05146062374115, + "kl": 0.07692439574748278, + "learning_rate": 1.5353172007848268e-07, + "loss": 0.08600187301635742, + "num_tokens": 41867675.0, + "reward": -2.2687018805811476e-07, + "reward_std": 0.9938721656799316, + "rewards/rnd_reward/mean": -2.2687017953160193e-07, + "rewards/rnd_reward/std": 0.9938721656799316, + "step": 2120, + "step_time": 10.522320462344215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.0, + "completions/mean_length": 96.24625244140626, + "completions/mean_terminated_length": 67.99528999328614, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.848968583345413, + "epoch": 1.393067364290386, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0902374982833862, + "kl": 0.08595932517200708, + "learning_rate": 1.5189666448659253e-07, + "loss": 0.10983504056930542, + "num_tokens": 42040322.0, + "reward": -1.4990568644179803e-07, + "reward_std": 0.9940501570701599, + "rewards/rnd_reward/mean": -1.4990567649419971e-07, + "rewards/rnd_reward/std": 0.9940501391887665, + "step": 2130, + "step_time": 11.159733151923865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000098720192, + "completions/max_length": 384.0, + "completions/max_terminated_length": 359.4, + "completions/mean_length": 102.9712516784668, + "completions/mean_terminated_length": 77.02480850219726, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.790306252241135, + "epoch": 1.3996075866579463, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9154292941093445, + "kl": 0.077430235221982, + "learning_rate": 1.502616088947024e-07, + "loss": 0.06642908453941346, + "num_tokens": 42217629.0, + "reward": -9.521842088133781e-08, + "reward_std": 0.9938531458377838, + "rewards/rnd_reward/mean": -9.521841803916686e-08, + "rewards/rnd_reward/std": 0.9938531517982483, + "step": 2140, + "step_time": 11.171463684784248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000145286321, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.3, + "completions/mean_length": 102.10125198364258, + "completions/mean_terminated_length": 75.82226257324218, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.675621399283409, + "epoch": 1.4061478090255068, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0319130420684814, + "kl": 0.07909353226423263, + "learning_rate": 1.4862655330281228e-07, + "loss": 0.0862727165222168, + "num_tokens": 42394325.0, + "reward": -1.949071898987853e-07, + "reward_std": 0.9938407003879547, + "rewards/rnd_reward/mean": -1.9490719025405667e-07, + "rewards/rnd_reward/std": 0.9938406765460968, + "step": 2150, + "step_time": 11.259053491754457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07500000111758709, + "completions/max_length": 384.0, + "completions/max_terminated_length": 324.9, + "completions/mean_length": 85.76750183105469, + "completions/mean_terminated_length": 61.72400894165039, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.564385971426964, + "epoch": 1.4126880313930674, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0090827941894531, + "kl": 0.08957632025703788, + "learning_rate": 1.4699149771092216e-07, + "loss": 0.05871064066886902, + "num_tokens": 42561024.0, + "reward": -4.068017274150293e-08, + "reward_std": 0.9942023098468781, + "rewards/rnd_reward/mean": -4.068016918878925e-08, + "rewards/rnd_reward/std": 0.9942023098468781, + "step": 2160, + "step_time": 11.197773828869686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000052154064, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.5, + "completions/mean_length": 94.29875259399414, + "completions/mean_terminated_length": 67.34189720153809, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.5985243558883666, + "epoch": 1.4192282537606278, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9199637174606323, + "kl": 0.08871069457381964, + "learning_rate": 1.4535644211903204e-07, + "loss": 0.14661363363265992, + "num_tokens": 42731253.0, + "reward": -9.38773203174037e-08, + "reward_std": 0.9938823819160462, + "rewards/rnd_reward/mean": -9.387731640941865e-08, + "rewards/rnd_reward/std": 0.9938823938369751, + "step": 2170, + "step_time": 10.55815609311685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0600000012665987, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.3, + "completions/mean_length": 83.71750259399414, + "completions/mean_terminated_length": 64.72726974487304, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.617268776893616, + "epoch": 1.4257684761281884, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0482478141784668, + "kl": 0.0882584748789668, + "learning_rate": 1.4372138652714192e-07, + "loss": 0.04273238778114319, + "num_tokens": 42891912.0, + "reward": -2.068281148126516e-07, + "reward_std": 0.9936731696128845, + "rewards/rnd_reward/mean": -2.0682811765482257e-07, + "rewards/rnd_reward/std": 0.9936731576919555, + "step": 2180, + "step_time": 10.923749357438647 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06250000074505806, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.3, + "completions/mean_length": 86.99125213623047, + "completions/mean_terminated_length": 67.24082832336425, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.775143876671791, + "epoch": 1.432308698495749, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1199995279312134, + "kl": 0.08594764936715364, + "learning_rate": 1.420863309352518e-07, + "loss": 0.09395463466644287, + "num_tokens": 43055435.0, + "reward": -7.629394431774017e-08, + "reward_std": 0.9939565777778625, + "rewards/rnd_reward/mean": -7.629394431774017e-08, + "rewards/rnd_reward/std": 0.9939565896987915, + "step": 2190, + "step_time": 10.720726100169122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05625000055879355, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.9, + "completions/mean_length": 77.54250106811523, + "completions/mean_terminated_length": 59.23452491760254, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7800998747348786, + "epoch": 1.4388489208633093, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2252848148345947, + "kl": 0.09403526708483696, + "learning_rate": 1.4045127534336168e-07, + "loss": 0.072505122423172, + "num_tokens": 43214484.0, + "reward": -2.1472573372705028e-07, + "reward_std": 0.9937775075435639, + "rewards/rnd_reward/mean": -2.1472573088487935e-07, + "rewards/rnd_reward/std": 0.9937775015830994, + "step": 2200, + "step_time": 11.38782194799278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08250000104308128, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.3, + "completions/mean_length": 93.60000152587891, + "completions/mean_terminated_length": 67.61266975402832, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.5929481953382494, + "epoch": 1.4453891432308699, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0342575311660767, + "kl": 0.08299258435145021, + "learning_rate": 1.3881621975147153e-07, + "loss": 0.1498832106590271, + "num_tokens": 43386444.0, + "reward": -1.038610928816297e-07, + "reward_std": 0.9938294231891632, + "rewards/rnd_reward/mean": -1.0386109572380064e-07, + "rewards/rnd_reward/std": 0.9938294172286988, + "step": 2210, + "step_time": 11.334175962558948 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08000000193715096, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.8, + "completions/mean_length": 88.12125167846679, + "completions/mean_terminated_length": 62.357414627075194, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.737210324406624, + "epoch": 1.4519293655984304, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2599236965179443, + "kl": 0.08750931257382036, + "learning_rate": 1.3718116415958143e-07, + "loss": 0.10528864860534667, + "num_tokens": 43548366.0, + "reward": -9.566545777772717e-08, + "reward_std": 0.9939428031444549, + "rewards/rnd_reward/mean": -9.566545600137033e-08, + "rewards/rnd_reward/std": 0.993942791223526, + "step": 2220, + "step_time": 10.802253967593424 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060000000335276124, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.9, + "completions/mean_length": 83.26000137329102, + "completions/mean_terminated_length": 64.11358032226562, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.655692085623741, + "epoch": 1.4584695879659908, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0874786376953125, + "kl": 0.09471626300364733, + "learning_rate": 1.355461085676913e-07, + "loss": 0.0698157787322998, + "num_tokens": 43704804.0, + "reward": -1.7568469061757242e-07, + "reward_std": 0.9932361245155334, + "rewards/rnd_reward/mean": -1.7568469061757242e-07, + "rewards/rnd_reward/std": 0.993236118555069, + "step": 2230, + "step_time": 10.654912683716976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07375000119209289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 333.5, + "completions/mean_length": 86.09000091552734, + "completions/mean_terminated_length": 62.54329261779785, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.6805451244115828, + "epoch": 1.4650098103335514, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2531136274337769, + "kl": 0.09066678332164883, + "learning_rate": 1.3391105297580116e-07, + "loss": 0.1158557415008545, + "num_tokens": 43870226.0, + "reward": -5.9008597474985435e-08, + "reward_std": 0.9934276759624481, + "rewards/rnd_reward/mean": -5.900859711971407e-08, + "rewards/rnd_reward/std": 0.9934276819229126, + "step": 2240, + "step_time": 10.659809662820772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06875000232830644, + "completions/max_length": 384.0, + "completions/max_terminated_length": 319.8, + "completions/mean_length": 86.3887508392334, + "completions/mean_terminated_length": 64.96298866271972, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.90979445874691, + "epoch": 1.4715500327011117, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8808140754699707, + "kl": 0.10609434265643358, + "learning_rate": 1.3227599738391104e-07, + "loss": 0.051298785209655764, + "num_tokens": 44036937.0, + "reward": -1.4841556961187052e-07, + "reward_std": 0.9932375848293304, + "rewards/rnd_reward/mean": -1.4841556605915684e-07, + "rewards/rnd_reward/std": 0.9932375907897949, + "step": 2250, + "step_time": 11.205043429811486 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07375000193715095, + "completions/max_length": 384.0, + "completions/max_terminated_length": 318.5, + "completions/mean_length": 82.19750175476074, + "completions/mean_terminated_length": 58.15153121948242, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 3.074227809906006, + "epoch": 1.4780902550686723, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.282700538635254, + "kl": 0.11512546790763736, + "learning_rate": 1.3064094179202092e-07, + "loss": 0.0818709373474121, + "num_tokens": 44197515.0, + "reward": -5.573034247419173e-08, + "reward_std": 0.9930948257446289, + "rewards/rnd_reward/mean": -5.573034211892036e-08, + "rewards/rnd_reward/std": 0.9930948317050934, + "step": 2260, + "step_time": 10.878628938994371 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06750000081956387, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.7, + "completions/mean_length": 84.10250320434571, + "completions/mean_terminated_length": 62.43845558166504, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.938316199183464, + "epoch": 1.4846304774362329, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2573261260986328, + "kl": 0.10300196381285787, + "learning_rate": 1.290058862001308e-07, + "loss": 0.0356877863407135, + "num_tokens": 44362002.0, + "reward": -1.7605722284486093e-07, + "reward_std": 0.9934640884399414, + "rewards/rnd_reward/mean": -1.7605722000268997e-07, + "rewards/rnd_reward/std": 0.9934640944004058, + "step": 2270, + "step_time": 11.115813568769955 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07750000040978193, + "completions/max_length": 384.0, + "completions/max_terminated_length": 343.9, + "completions/mean_length": 87.04125137329102, + "completions/mean_terminated_length": 62.220666122436526, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 3.1251493275165556, + "epoch": 1.4911706998037935, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3937524557113647, + "kl": 0.10200166283175349, + "learning_rate": 1.2737083060824068e-07, + "loss": 0.08271963596343994, + "num_tokens": 44524310.0, + "reward": -8.717179245110174e-08, + "reward_std": 0.9933235943317413, + "rewards/rnd_reward/mean": -8.717178818784533e-08, + "rewards/rnd_reward/std": 0.9933236181735993, + "step": 2280, + "step_time": 10.662292190524749 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06750000100582838, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.0, + "completions/mean_length": 84.15750160217286, + "completions/mean_terminated_length": 62.54970436096191, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 3.019434627890587, + "epoch": 1.4977109221713538, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0537996292114258, + "kl": 0.10465592611581087, + "learning_rate": 1.2573577501635055e-07, + "loss": 0.15651319026947022, + "num_tokens": 44686211.0, + "reward": -7.629394449537585e-08, + "reward_std": 0.9937636077404022, + "rewards/rnd_reward/mean": -7.629394520591859e-08, + "rewards/rnd_reward/std": 0.9937636256217957, + "step": 2290, + "step_time": 10.87905588184949 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07375000063329935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 337.0, + "completions/mean_length": 91.2612518310547, + "completions/mean_terminated_length": 67.97848358154297, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7947313100099564, + "epoch": 1.5042511445389142, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9707646369934082, + "kl": 0.0921105288900435, + "learning_rate": 1.2410071942446043e-07, + "loss": 0.037941274046897885, + "num_tokens": 44854255.0, + "reward": -3.1441446779467697e-08, + "reward_std": 0.9936448276042938, + "rewards/rnd_reward/mean": -3.144145281908095e-08, + "rewards/rnd_reward/std": 0.9936448395252228, + "step": 2300, + "step_time": 10.841944112465717 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05875000171363354, + "completions/max_length": 384.0, + "completions/max_terminated_length": 353.4, + "completions/mean_length": 82.65125198364258, + "completions/mean_terminated_length": 63.85377426147461, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.9727181673049925, + "epoch": 1.5107913669064748, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.044342279434204, + "kl": 0.10308242114260793, + "learning_rate": 1.224656638325703e-07, + "loss": 0.08985200524330139, + "num_tokens": 45021761.0, + "reward": -2.3201108554360416e-07, + "reward_std": 0.9937818229198456, + "rewards/rnd_reward/mean": -2.3201108447779007e-07, + "rewards/rnd_reward/std": 0.9937818348407745, + "step": 2310, + "step_time": 12.239735689153894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07500000195577741, + "completions/max_length": 384.0, + "completions/max_terminated_length": 327.8, + "completions/mean_length": 90.46125106811523, + "completions/mean_terminated_length": 66.89172592163087, + "completions/min_length": 2.6, + "completions/min_terminated_length": 2.6, + "entropy": 2.8864229738712313, + "epoch": 1.5173315892740353, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1288195848464966, + "kl": 0.08688412019982934, + "learning_rate": 1.2083060824068016e-07, + "loss": 0.029066413640975952, + "num_tokens": 45186095.0, + "reward": 5.736946953049937e-08, + "reward_std": 0.99327272772789, + "rewards/rnd_reward/mean": 5.736947450429852e-08, + "rewards/rnd_reward/std": 0.9932727217674255, + "step": 2320, + "step_time": 10.734421823220327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07000000132247805, + "completions/max_length": 384.0, + "completions/max_terminated_length": 339.1, + "completions/mean_length": 92.6050018310547, + "completions/mean_terminated_length": 70.5366439819336, + "completions/min_length": 1.9, + "completions/min_terminated_length": 1.9, + "entropy": 2.7233670204877853, + "epoch": 1.523871811641596, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.24017333984375, + "kl": 0.08950559962540865, + "learning_rate": 1.1919555264879005e-07, + "loss": 0.07465057373046875, + "num_tokens": 45354024.0, + "reward": -2.0265579721012726e-07, + "reward_std": 0.993443387746811, + "rewards/rnd_reward/mean": -2.026557901046999e-07, + "rewards/rnd_reward/std": 0.9934433996677399, + "step": 2330, + "step_time": 10.924078989424743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06375000141561031, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.3, + "completions/mean_length": 85.09000129699707, + "completions/mean_terminated_length": 64.81442031860351, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.803052818775177, + "epoch": 1.5304120340091563, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.400153636932373, + "kl": 0.09231640296056867, + "learning_rate": 1.1756049705689992e-07, + "loss": 0.126143217086792, + "num_tokens": 45510991.0, + "reward": -1.0177493525720394e-07, + "reward_std": 0.9933503866195679, + "rewards/rnd_reward/mean": -1.0177492921759068e-07, + "rewards/rnd_reward/std": 0.9933504104614258, + "step": 2340, + "step_time": 10.554431772092357 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08000000081956386, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.3, + "completions/mean_length": 88.79000091552734, + "completions/mean_terminated_length": 62.90113410949707, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7890906423330306, + "epoch": 1.5369522563767168, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1739593744277954, + "kl": 0.08973503690212965, + "learning_rate": 1.1592544146500981e-07, + "loss": 0.016262634098529814, + "num_tokens": 45673063.0, + "reward": -2.1778047649689823e-07, + "reward_std": 0.9933681070804596, + "rewards/rnd_reward/mean": -2.177804736547273e-07, + "rewards/rnd_reward/std": 0.9933681011199951, + "step": 2350, + "step_time": 10.583909913594834 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09750000089406967, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.9, + "completions/mean_length": 96.30625228881836, + "completions/mean_terminated_length": 65.25630912780761, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7480629831552505, + "epoch": 1.5434924787442772, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.265565037727356, + "kl": 0.08257783968001604, + "learning_rate": 1.1429038587311968e-07, + "loss": 0.09627790451049804, + "num_tokens": 45842103.0, + "reward": -8.836388873945111e-08, + "reward_std": 0.9934152483940124, + "rewards/rnd_reward/mean": -8.836388589728018e-08, + "rewards/rnd_reward/std": 0.993415230512619, + "step": 2360, + "step_time": 10.685579545726068 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08375000134110451, + "completions/max_length": 384.0, + "completions/max_terminated_length": 333.4, + "completions/mean_length": 91.16250152587891, + "completions/mean_terminated_length": 64.41967430114747, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.663706639409065, + "epoch": 1.5500327011118378, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5513228178024292, + "kl": 0.08946019792929291, + "learning_rate": 1.1265533028122955e-07, + "loss": 0.03821712434291839, + "num_tokens": 46015743.0, + "reward": -2.2679567930339316e-07, + "reward_std": 0.993412846326828, + "rewards/rnd_reward/mean": -2.2679567575067948e-07, + "rewards/rnd_reward/std": 0.9934128284454345, + "step": 2370, + "step_time": 11.590211745630949 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06125000081956387, + "completions/max_length": 383.4, + "completions/max_terminated_length": 346.7, + "completions/mean_length": 89.28625183105468, + "completions/mean_terminated_length": 69.96004066467285, + "completions/min_length": 2.7, + "completions/min_terminated_length": 2.7, + "entropy": 3.0011739432811737, + "epoch": 1.5565729234793984, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1020563840866089, + "kl": 0.09448451111093163, + "learning_rate": 1.1102027468933943e-07, + "loss": 0.13594624996185303, + "num_tokens": 46183567.0, + "reward": 6.988644907224284e-08, + "reward_std": 0.9929902851581573, + "rewards/rnd_reward/mean": 6.98864462300719e-08, + "rewards/rnd_reward/std": 0.9929902911186218, + "step": 2380, + "step_time": 11.415522536402568 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0750000024214387, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.0, + "completions/mean_length": 90.28625106811523, + "completions/mean_terminated_length": 66.44939155578614, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.8806687116622927, + "epoch": 1.563113145846959, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0451889038085938, + "kl": 0.09008068647235631, + "learning_rate": 1.0938521909744931e-07, + "loss": 0.047099918127059937, + "num_tokens": 46343361.0, + "reward": -5.662440694109705e-09, + "reward_std": 0.9929801285266876, + "rewards/rnd_reward/mean": -5.6624399835669695e-09, + "rewards/rnd_reward/std": 0.9929801285266876, + "step": 2390, + "step_time": 10.55025691683404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08250000234693289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 333.8, + "completions/mean_length": 92.99375228881836, + "completions/mean_terminated_length": 66.80701446533203, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.8602111876010894, + "epoch": 1.5696533682145193, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2736488580703735, + "kl": 0.09589690109714866, + "learning_rate": 1.0775016350555919e-07, + "loss": 0.09055531024932861, + "num_tokens": 46510141.0, + "reward": -1.1608004477636768e-07, + "reward_std": 0.9930357456207275, + "rewards/rnd_reward/mean": -1.1608004477636768e-07, + "rewards/rnd_reward/std": 0.9930357277393341, + "step": 2400, + "step_time": 10.752315263729542 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000150874257, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.3, + "completions/mean_length": 99.20625152587891, + "completions/mean_terminated_length": 75.86554489135742, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.8277246713638307, + "epoch": 1.5761935905820796, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0789973735809326, + "kl": 0.08739803759381175, + "learning_rate": 1.0611510791366907e-07, + "loss": 0.06712510585784912, + "num_tokens": 46679921.0, + "reward": -9.715557247602646e-08, + "reward_std": 0.9933283686637878, + "rewards/rnd_reward/mean": -9.715556821277005e-08, + "rewards/rnd_reward/std": 0.9933283627033234, + "step": 2410, + "step_time": 10.51516141500324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000186264514, + "completions/max_length": 384.0, + "completions/max_terminated_length": 335.9, + "completions/mean_length": 99.28125076293945, + "completions/mean_terminated_length": 72.07235450744629, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.6956777185201646, + "epoch": 1.5827338129496402, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.215997338294983, + "kl": 0.08504411997273564, + "learning_rate": 1.0448005232177893e-07, + "loss": 0.06522908210754394, + "num_tokens": 46848921.0, + "reward": -4.142523053474179e-08, + "reward_std": 0.9928943634033203, + "rewards/rnd_reward/mean": -4.142522982419905e-08, + "rewards/rnd_reward/std": 0.9928943693637848, + "step": 2420, + "step_time": 10.540731458202936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06875000121071935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.5, + "completions/mean_length": 85.0125015258789, + "completions/mean_terminated_length": 62.73664093017578, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 3.0628849148750303, + "epoch": 1.5892740353172008, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.336714267730713, + "kl": 0.10201989021152258, + "learning_rate": 1.0284499672988881e-07, + "loss": 0.05233617424964905, + "num_tokens": 47004126.0, + "reward": -1.521408613314179e-07, + "reward_std": 0.9932492971420288, + "rewards/rnd_reward/mean": -1.5214085848924697e-07, + "rewards/rnd_reward/std": 0.9932492971420288, + "step": 2430, + "step_time": 10.492817461281083 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07500000074505805, + "completions/max_length": 384.0, + "completions/max_terminated_length": 332.8, + "completions/mean_length": 86.80125122070312, + "completions/mean_terminated_length": 62.760490036010744, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.799528333544731, + "epoch": 1.5958142576847614, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4625661373138428, + "kl": 0.10000391742214561, + "learning_rate": 1.0120994113799869e-07, + "loss": 0.031586483120918274, + "num_tokens": 47168777.0, + "reward": -7.808208586368437e-08, + "reward_std": 0.9931057214736938, + "rewards/rnd_reward/mean": -7.80820865742271e-08, + "rewards/rnd_reward/std": 0.9931057274341584, + "step": 2440, + "step_time": 10.696252883039415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07875000163912774, + "completions/max_length": 384.0, + "completions/max_terminated_length": 334.2, + "completions/mean_length": 88.19125213623047, + "completions/mean_terminated_length": 62.81879196166992, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.742514914274216, + "epoch": 1.6023544800523217, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1427501440048218, + "kl": 0.09996775221079587, + "learning_rate": 9.957488554610855e-08, + "loss": 0.041766366362571715, + "num_tokens": 47331300.0, + "reward": 5.722046019229765e-08, + "reward_std": 0.9931129157543183, + "rewards/rnd_reward/mean": 5.7220459481754915e-08, + "rewards/rnd_reward/std": 0.9931129038333892, + "step": 2450, + "step_time": 10.62438986178022 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07874999996274709, + "completions/max_length": 384.0, + "completions/max_terminated_length": 347.3, + "completions/mean_length": 90.64625091552735, + "completions/mean_terminated_length": 65.4418701171875, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.8206116437911986, + "epoch": 1.6088947024198823, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2679322957992554, + "kl": 0.0997009389102459, + "learning_rate": 9.793982995421845e-08, + "loss": 0.08149058222770691, + "num_tokens": 47497532.0, + "reward": 1.2382865293147917e-07, + "reward_std": 0.9925880253314971, + "rewards/rnd_reward/mean": 1.238286500893082e-07, + "rewards/rnd_reward/std": 0.9925880193710327, + "step": 2460, + "step_time": 10.703694239328616 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08750000111758709, + "completions/max_length": 384.0, + "completions/max_terminated_length": 341.8, + "completions/mean_length": 95.64250106811524, + "completions/mean_terminated_length": 67.96361427307129, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.671072342991829, + "epoch": 1.6154349247874427, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.008203387260437, + "kl": 0.09101189225912094, + "learning_rate": 9.630477436232831e-08, + "loss": 0.02300444096326828, + "num_tokens": 47673241.0, + "reward": -1.6734004162355376e-07, + "reward_std": 0.9930606245994568, + "rewards/rnd_reward/mean": -1.6734004304463923e-07, + "rewards/rnd_reward/std": 0.9930606305599212, + "step": 2470, + "step_time": 11.121605953085236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000104308128, + "completions/max_length": 384.0, + "completions/max_terminated_length": 342.3, + "completions/mean_length": 91.83125076293945, + "completions/mean_terminated_length": 63.43943824768066, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.891758808493614, + "epoch": 1.6219751471550032, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0772875547409058, + "kl": 0.09280837709084153, + "learning_rate": 9.466971877043819e-08, + "loss": 0.032535853981971743, + "num_tokens": 47836976.0, + "reward": -1.448392879410676e-07, + "reward_std": 0.9926415860652924, + "rewards/rnd_reward/mean": -1.4483928936215307e-07, + "rewards/rnd_reward/std": 0.9926415801048278, + "step": 2480, + "step_time": 10.584856062568724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07875000201165676, + "completions/max_length": 384.0, + "completions/max_terminated_length": 343.0, + "completions/mean_length": 91.06500015258788, + "completions/mean_terminated_length": 65.96636085510254, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.72680883705616, + "epoch": 1.6285153695225638, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3315449953079224, + "kl": 0.09668447291478514, + "learning_rate": 9.303466317854807e-08, + "loss": 0.1025187611579895, + "num_tokens": 48003343.0, + "reward": -8.106231996762858e-08, + "reward_std": 0.9925062835216523, + "rewards/rnd_reward/mean": -8.10623164149149e-08, + "rewards/rnd_reward/std": 0.9925062835216523, + "step": 2490, + "step_time": 10.71222500174772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07000000141561032, + "completions/max_length": 384.0, + "completions/max_terminated_length": 318.3, + "completions/mean_length": 85.66875228881835, + "completions/mean_terminated_length": 63.251688385009764, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.662171345949173, + "epoch": 1.6350555918901244, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3252993822097778, + "kl": 0.09045366756618023, + "learning_rate": 9.139960758665795e-08, + "loss": 0.0439204216003418, + "num_tokens": 48168473.0, + "reward": -3.5464765701931354e-08, + "reward_std": 0.9928713321685791, + "rewards/rnd_reward/mean": -3.5464758596503997e-08, + "rewards/rnd_reward/std": 0.9928713619709015, + "step": 2500, + "step_time": 10.784380649239756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.8, + "completions/mean_length": 96.40625228881837, + "completions/mean_terminated_length": 68.52096328735351, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.7209132075309754, + "epoch": 1.6415958142576847, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0920087099075317, + "kl": 0.08929237592965364, + "learning_rate": 8.976455199476782e-08, + "loss": 0.07013883590698242, + "num_tokens": 48335753.0, + "reward": -1.1816621103832858e-07, + "reward_std": 0.9925974249839783, + "rewards/rnd_reward/mean": -1.1816620961724312e-07, + "rewards/rnd_reward/std": 0.9925974488258362, + "step": 2510, + "step_time": 10.549463000241667 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000067055225, + "completions/max_length": 384.0, + "completions/max_terminated_length": 335.7, + "completions/mean_length": 94.45250015258789, + "completions/mean_terminated_length": 66.36481285095215, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.6327116012573244, + "epoch": 1.648136036625245, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.945324182510376, + "kl": 0.08747830325737596, + "learning_rate": 8.812949640287769e-08, + "loss": 0.13749951124191284, + "num_tokens": 48500460.0, + "reward": -7.227063427706071e-08, + "reward_std": 0.9929073750972748, + "rewards/rnd_reward/mean": -7.227063250070387e-08, + "rewards/rnd_reward/std": 0.9929073631763459, + "step": 2520, + "step_time": 10.486474628048018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000104308129, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.3, + "completions/mean_length": 92.80625228881836, + "completions/mean_terminated_length": 68.86259307861329, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.5589567482471467, + "epoch": 1.6546762589928057, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.173561692237854, + "kl": 0.08464234415441751, + "learning_rate": 8.649444081098757e-08, + "loss": 0.07577463388442993, + "num_tokens": 48664250.0, + "reward": -1.436471933402572e-07, + "reward_std": 0.9925457060337066, + "rewards/rnd_reward/mean": -1.436471933402572e-07, + "rewards/rnd_reward/std": 0.9925457000732422, + "step": 2530, + "step_time": 10.735207921243273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0800000000745058, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.6, + "completions/mean_length": 93.55000152587891, + "completions/mean_terminated_length": 68.25583038330078, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6400635212659838, + "epoch": 1.6612164813603663, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0104870796203613, + "kl": 0.08781723519787192, + "learning_rate": 8.485938521909745e-08, + "loss": 0.03668081760406494, + "num_tokens": 48831920.0, + "reward": -4.6491619976052335e-08, + "reward_std": 0.9925249576568603, + "rewards/rnd_reward/mean": -4.6491622818223274e-08, + "rewards/rnd_reward/std": 0.9925249457359314, + "step": 2540, + "step_time": 10.869395459746011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10750000104308129, + "completions/max_length": 384.0, + "completions/max_terminated_length": 332.1, + "completions/mean_length": 100.57750244140625, + "completions/mean_terminated_length": 66.49176979064941, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.7832937359809877, + "epoch": 1.6677567037279268, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0521481037139893, + "kl": 0.08043702868744732, + "learning_rate": 8.322432962720732e-08, + "loss": 0.05087646245956421, + "num_tokens": 49003242.0, + "reward": -6.422400495864622e-08, + "reward_std": 0.9929387688636779, + "rewards/rnd_reward/mean": -6.422400566918895e-08, + "rewards/rnd_reward/std": 0.9929387867450714, + "step": 2550, + "step_time": 10.56416127383709 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000104308128, + "completions/max_length": 384.0, + "completions/max_terminated_length": 337.2, + "completions/mean_length": 98.30000152587891, + "completions/mean_terminated_length": 70.51053009033203, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.768531382083893, + "epoch": 1.6742969260954872, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9573103785514832, + "kl": 0.08502876814454793, + "learning_rate": 8.158927403531719e-08, + "loss": 0.0772208571434021, + "num_tokens": 49174652.0, + "reward": -1.469254531372144e-07, + "reward_std": 0.9924641728401185, + "rewards/rnd_reward/mean": -1.4692545029504346e-07, + "rewards/rnd_reward/std": 0.9924641907215118, + "step": 2560, + "step_time": 10.583414485724642 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.7, + "completions/mean_length": 96.5462516784668, + "completions/mean_terminated_length": 69.8155361175537, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.8294411927461622, + "epoch": 1.6808371484630478, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9854137301445007, + "kl": 0.09185233255848288, + "learning_rate": 7.995421844342708e-08, + "loss": 0.1261582612991333, + "num_tokens": 49354119.0, + "reward": 8.67247596403331e-08, + "reward_std": 0.9922890484333038, + "rewards/rnd_reward/mean": 8.67247596403331e-08, + "rewards/rnd_reward/std": 0.9922890365123749, + "step": 2570, + "step_time": 11.603049716376699 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07875000052154064, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.7, + "completions/mean_length": 92.55000152587891, + "completions/mean_terminated_length": 67.6265712738037, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7690848499536513, + "epoch": 1.6873773708306081, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1284137964248657, + "kl": 0.09411861123517155, + "learning_rate": 7.831916285153695e-08, + "loss": 0.13598878383636476, + "num_tokens": 49524444.0, + "reward": 6.571412010103473e-08, + "reward_std": 0.9924658596515655, + "rewards/rnd_reward/mean": 6.571412010103473e-08, + "rewards/rnd_reward/std": 0.9924658596515655, + "step": 2580, + "step_time": 11.369293246162124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09000000208616257, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.5, + "completions/mean_length": 100.7300018310547, + "completions/mean_terminated_length": 72.78359603881836, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.656350103020668, + "epoch": 1.6939175931981687, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1325582265853882, + "kl": 0.08164091287180782, + "learning_rate": 7.668410725964682e-08, + "loss": 0.09759321212768554, + "num_tokens": 49702873.0, + "reward": -8.478760804564445e-08, + "reward_std": 0.9928931176662446, + "rewards/rnd_reward/mean": -8.478760946672991e-08, + "rewards/rnd_reward/std": 0.9928931057453155, + "step": 2590, + "step_time": 10.856351627549156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07000000141561032, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.6, + "completions/mean_length": 91.03375091552735, + "completions/mean_terminated_length": 69.03646697998047, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.887327644228935, + "epoch": 1.7004578155657293, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3039114475250244, + "kl": 0.0926243552006781, + "learning_rate": 7.50490516677567e-08, + "loss": 0.0819834053516388, + "num_tokens": 49876500.0, + "reward": 1.3411066479562806e-09, + "reward_std": 0.9926319658756256, + "rewards/rnd_reward/mean": 1.3411066479562806e-09, + "rewards/rnd_reward/std": 0.9926319539546966, + "step": 2600, + "step_time": 11.05162076470442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08125000149011612, + "completions/max_length": 384.0, + "completions/max_terminated_length": 354.7, + "completions/mean_length": 89.97250061035156, + "completions/mean_terminated_length": 63.9521728515625, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.807651326060295, + "epoch": 1.7069980379332899, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0977989435195923, + "kl": 0.08759319856762886, + "learning_rate": 7.341399607586658e-08, + "loss": 0.13040846586227417, + "num_tokens": 50044628.0, + "reward": -6.228685363041109e-08, + "reward_std": 0.9921288013458252, + "rewards/rnd_reward/mean": -6.228685291986836e-08, + "rewards/rnd_reward/std": 0.9921288132667542, + "step": 2610, + "step_time": 11.296212270553223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08375000227242708, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.6, + "completions/mean_length": 91.39750061035156, + "completions/mean_terminated_length": 64.60456695556641, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.771796461939812, + "epoch": 1.7135382603008502, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.101976990699768, + "kl": 0.09053975995630026, + "learning_rate": 7.177894048397645e-08, + "loss": 0.07479513287544251, + "num_tokens": 50204246.0, + "reward": -1.3120472530658844e-07, + "reward_std": 0.9925267994403839, + "rewards/rnd_reward/mean": -1.3120472530658844e-07, + "rewards/rnd_reward/std": 0.9925268173217774, + "step": 2620, + "step_time": 10.44964153978508 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375000074505806, + "completions/max_length": 384.0, + "completions/max_terminated_length": 325.7, + "completions/mean_length": 95.06375122070312, + "completions/mean_terminated_length": 65.27039031982422, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6531850188970565, + "epoch": 1.7200784826684106, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.112268090248108, + "kl": 0.08289336860179901, + "learning_rate": 7.014388489208632e-08, + "loss": 0.08899658918380737, + "num_tokens": 50373747.0, + "reward": -2.539157884839938e-07, + "reward_std": 0.9920955657958984, + "rewards/rnd_reward/mean": -2.539157849312801e-07, + "rewards/rnd_reward/std": 0.9920955717563629, + "step": 2630, + "step_time": 10.710641134064645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07875000201165676, + "completions/max_length": 384.0, + "completions/max_terminated_length": 357.8, + "completions/mean_length": 93.92000198364258, + "completions/mean_terminated_length": 69.12318840026856, + "completions/min_length": 2.7, + "completions/min_terminated_length": 2.7, + "entropy": 2.5591167956590652, + "epoch": 1.7266187050359711, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0098210573196411, + "kl": 0.08759762533009052, + "learning_rate": 6.85088293001962e-08, + "loss": 0.11666052341461182, + "num_tokens": 50542968.0, + "reward": -2.8014184039193423e-08, + "reward_std": 0.9922697126865387, + "rewards/rnd_reward/mean": -2.8014186881364366e-08, + "rewards/rnd_reward/std": 0.9922697246074677, + "step": 2640, + "step_time": 10.810134377889336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09000000171363354, + "completions/max_length": 384.0, + "completions/max_terminated_length": 327.5, + "completions/mean_length": 90.80375213623047, + "completions/mean_terminated_length": 61.77821655273438, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.779536947607994, + "epoch": 1.7331589274035317, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1324180364608765, + "kl": 0.08631603391841054, + "learning_rate": 6.687377370830608e-08, + "loss": 0.09856376647949219, + "num_tokens": 50707726.0, + "reward": -5.237758227139011e-08, + "reward_std": 0.992433100938797, + "rewards/rnd_reward/mean": -5.237758085030464e-08, + "rewards/rnd_reward/std": 0.9924331188201905, + "step": 2650, + "step_time": 10.956776090711354 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375000186264515, + "completions/max_length": 384.0, + "completions/max_terminated_length": 334.8, + "completions/mean_length": 97.68875045776367, + "completions/mean_terminated_length": 68.05043869018554, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.7804177552461624, + "epoch": 1.7396991497710923, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0591152906417847, + "kl": 0.09146134741604328, + "learning_rate": 6.523871811641596e-08, + "loss": 0.04577620327472687, + "num_tokens": 50878792.0, + "reward": -7.227059839465255e-09, + "reward_std": 0.9924111425876617, + "rewards/rnd_reward/mean": -7.227060550007991e-09, + "rewards/rnd_reward/std": 0.9924111247062684, + "step": 2660, + "step_time": 10.708624366554432 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000108033419, + "completions/max_length": 384.0, + "completions/max_terminated_length": 330.3, + "completions/mean_length": 95.15875205993652, + "completions/mean_terminated_length": 68.52624740600587, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.626550641655922, + "epoch": 1.7462393721386529, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0052024126052856, + "kl": 0.09057780019938946, + "learning_rate": 6.360366252452582e-08, + "loss": 0.03838871717453003, + "num_tokens": 51047709.0, + "reward": -1.60261992654398e-07, + "reward_std": 0.9925138711929321, + "rewards/rnd_reward/mean": -1.6026198892404864e-07, + "rewards/rnd_reward/std": 0.9925138831138611, + "step": 2670, + "step_time": 10.687237256928347 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07000000113621355, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.8, + "completions/mean_length": 91.47625198364258, + "completions/mean_terminated_length": 69.65823402404786, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6517731666564943, + "epoch": 1.7527795945062132, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.085845947265625, + "kl": 0.08733204193413258, + "learning_rate": 6.19686069326357e-08, + "loss": 0.126353120803833, + "num_tokens": 51221270.0, + "reward": -1.2561679199052378e-07, + "reward_std": 0.9924490749835968, + "rewards/rnd_reward/mean": -1.2561678275346821e-07, + "rewards/rnd_reward/std": 0.9924490511417389, + "step": 2680, + "step_time": 11.201374626043252 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08250000150874257, + "completions/max_length": 384.0, + "completions/max_terminated_length": 339.3, + "completions/mean_length": 97.07750091552734, + "completions/mean_terminated_length": 71.1569839477539, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.534248262643814, + "epoch": 1.7593198168737736, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1865549087524414, + "kl": 0.07945491755381226, + "learning_rate": 6.033355134074558e-08, + "loss": 0.023079755902290344, + "num_tokens": 51391307.0, + "reward": 1.2665985660298417e-08, + "reward_std": 0.9924781501293183, + "rewards/rnd_reward/mean": 1.2665986370841154e-08, + "rewards/rnd_reward/std": 0.9924781501293183, + "step": 2690, + "step_time": 10.600427453825251 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10375000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.4, + "completions/mean_length": 102.50500183105468, + "completions/mean_terminated_length": 69.94977340698242, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5867824822664263, + "epoch": 1.7658600392413342, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9612109661102295, + "kl": 0.08055433174595236, + "learning_rate": 5.869849574885546e-08, + "loss": 0.09532687664031983, + "num_tokens": 51566461.0, + "reward": -4.351139040181806e-08, + "reward_std": 0.9921834826469421, + "rewards/rnd_reward/mean": -4.351139040181806e-08, + "rewards/rnd_reward/std": 0.9921834707260132, + "step": 2700, + "step_time": 10.596421872614883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09625000208616256, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.4, + "completions/mean_length": 103.48625259399414, + "completions/mean_terminated_length": 73.66226654052734, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.574134039878845, + "epoch": 1.7724002616088947, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8851132392883301, + "kl": 0.07895892215892672, + "learning_rate": 5.706344015696534e-08, + "loss": 0.03208762109279632, + "num_tokens": 51739420.0, + "reward": 1.4901161127234275e-08, + "reward_std": 0.9922367691993713, + "rewards/rnd_reward/mean": 1.4901161482505642e-08, + "rewards/rnd_reward/std": 0.992236751317978, + "step": 2710, + "step_time": 10.52075885424856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07750000096857548, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.7, + "completions/mean_length": 98.13000183105468, + "completions/mean_terminated_length": 74.1987533569336, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7050154358148575, + "epoch": 1.7789404839764553, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2265753746032715, + "kl": 0.08297120407223701, + "learning_rate": 5.542838456507521e-08, + "loss": 0.07181986570358276, + "num_tokens": 51906604.0, + "reward": 2.1010632700324548e-08, + "reward_std": 0.9923449456691742, + "rewards/rnd_reward/mean": 2.101064406900832e-08, + "rewards/rnd_reward/std": 0.9923449575901031, + "step": 2720, + "step_time": 10.726289883395657 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10125000029802322, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.8, + "completions/mean_length": 97.52250137329102, + "completions/mean_terminated_length": 65.36152725219726, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5555239796638487, + "epoch": 1.7854807063440157, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.344201922416687, + "kl": 0.07681396808475256, + "learning_rate": 5.379332897318509e-08, + "loss": 0.09209707379341125, + "num_tokens": 52078602.0, + "reward": 7.450558392463336e-10, + "reward_std": 0.991978645324707, + "rewards/rnd_reward/mean": 7.450586814172766e-10, + "rewards/rnd_reward/std": 0.9919786393642426, + "step": 2730, + "step_time": 10.693091615103185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09500000141561031, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.1, + "completions/mean_length": 99.98125228881835, + "completions/mean_terminated_length": 70.13334350585937, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.684102329611778, + "epoch": 1.792020928711576, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1477556228637695, + "kl": 0.08274123426526785, + "learning_rate": 5.2158273381294966e-08, + "loss": 0.025458133220672606, + "num_tokens": 52245797.0, + "reward": 3.457069563950199e-08, + "reward_std": 0.9920007467269898, + "rewards/rnd_reward/mean": 3.457069741585883e-08, + "rewards/rnd_reward/std": 0.9920007586479187, + "step": 2740, + "step_time": 10.481961290817708 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07750000078231097, + "completions/max_length": 384.0, + "completions/max_terminated_length": 339.5, + "completions/mean_length": 91.90375061035157, + "completions/mean_terminated_length": 67.26799354553222, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.7344156444072723, + "epoch": 1.7985611510791366, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1605439186096191, + "kl": 0.08739866856485605, + "learning_rate": 5.052321778940484e-08, + "loss": 0.0825818121433258, + "num_tokens": 52417430.0, + "reward": -5.275011112360062e-08, + "reward_std": 0.9921901881694793, + "rewards/rnd_reward/mean": -5.275011254468609e-08, + "rewards/rnd_reward/std": 0.992190170288086, + "step": 2750, + "step_time": 11.562560918554663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000126659871, + "completions/max_length": 384.0, + "completions/max_terminated_length": 355.7, + "completions/mean_length": 95.38625106811523, + "completions/mean_terminated_length": 68.69163131713867, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.508667880296707, + "epoch": 1.8051013734466972, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1222717761993408, + "kl": 0.08437787899747491, + "learning_rate": 4.8888162197514716e-08, + "loss": 0.08072326779365539, + "num_tokens": 52587249.0, + "reward": -3.2708050046181825e-08, + "reward_std": 0.9921342968940735, + "rewards/rnd_reward/mean": -3.270804720401088e-08, + "rewards/rnd_reward/std": 0.9921342968940735, + "step": 2760, + "step_time": 10.81694066869095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09125000275671483, + "completions/max_length": 384.0, + "completions/max_terminated_length": 342.4, + "completions/mean_length": 100.94500198364258, + "completions/mean_terminated_length": 72.59214401245117, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.762883481383324, + "epoch": 1.8116415958142578, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3192421197891235, + "kl": 0.08027938418090344, + "learning_rate": 4.725310660562459e-08, + "loss": 0.10753175020217895, + "num_tokens": 52769960.0, + "reward": 5.5879356897214906e-08, + "reward_std": 0.9917623281478882, + "rewards/rnd_reward/mean": 5.587935405504396e-08, + "rewards/rnd_reward/std": 0.9917623460292816, + "step": 2770, + "step_time": 11.582827594131231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0787500012665987, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.5, + "completions/mean_length": 95.13875122070313, + "completions/mean_terminated_length": 70.49329643249511, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.6798209667205812, + "epoch": 1.8181818181818183, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1080927848815918, + "kl": 0.08364078365266323, + "learning_rate": 4.5618051013734466e-08, + "loss": 0.04997023940086365, + "num_tokens": 52940211.0, + "reward": -9.596347978479969e-08, + "reward_std": 0.9920522153377533, + "rewards/rnd_reward/mean": -9.596347978479969e-08, + "rewards/rnd_reward/std": 0.9920521914958954, + "step": 2780, + "step_time": 10.742116263648494 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000044703483, + "completions/max_length": 384.0, + "completions/max_terminated_length": 326.3, + "completions/mean_length": 93.41375198364258, + "completions/mean_terminated_length": 65.94217224121094, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.7876433610916136, + "epoch": 1.8247220405493787, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2520111799240112, + "kl": 0.08399344785138965, + "learning_rate": 4.398299542184434e-08, + "loss": 0.11648783683776856, + "num_tokens": 53106412.0, + "reward": -1.3023615004215116e-07, + "reward_std": 0.9917174458503724, + "rewards/rnd_reward/mean": -1.3023615146323665e-07, + "rewards/rnd_reward/std": 0.9917174398899078, + "step": 2790, + "step_time": 10.729628002620302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000029802323, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.1, + "completions/mean_length": 95.42125091552734, + "completions/mean_terminated_length": 67.35246047973632, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.751756873726845, + "epoch": 1.831262262916939, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2754117250442505, + "kl": 0.08136667888611555, + "learning_rate": 4.2347939829954215e-08, + "loss": 0.10360376834869385, + "num_tokens": 53273129.0, + "reward": -8.925795818015558e-08, + "reward_std": 0.992232620716095, + "rewards/rnd_reward/mean": -8.92579546274419e-08, + "rewards/rnd_reward/std": 0.9922326266765594, + "step": 2800, + "step_time": 10.579858994716778 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08250000178813935, + "completions/max_length": 384.0, + "completions/max_terminated_length": 349.3, + "completions/mean_length": 101.0425018310547, + "completions/mean_terminated_length": 75.79266548156738, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.514050582051277, + "epoch": 1.8378024852844996, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.292152762413025, + "kl": 0.07886797543615102, + "learning_rate": 4.0712884238064094e-08, + "loss": 0.07606931924819946, + "num_tokens": 53448303.0, + "reward": -2.5182961227443456e-08, + "reward_std": 0.9920957505702972, + "rewards/rnd_reward/mean": -2.5182961227443456e-08, + "rewards/rnd_reward/std": 0.9920957386493683, + "step": 2810, + "step_time": 11.144761186954565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000126659871, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.3, + "completions/mean_length": 99.48500213623046, + "completions/mean_terminated_length": 73.15060997009277, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.75241442322731, + "epoch": 1.8443427076520602, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0998656749725342, + "kl": 0.07965973112732172, + "learning_rate": 3.9077828646173965e-08, + "loss": 0.08108996152877808, + "num_tokens": 53620201.0, + "reward": 1.832842806415158e-08, + "reward_std": 0.9918050169944763, + "rewards/rnd_reward/mean": 1.832843197213663e-08, + "rewards/rnd_reward/std": 0.9918050229549408, + "step": 2820, + "step_time": 11.469240892818197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000119209289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 346.9, + "completions/mean_length": 97.51125106811523, + "completions/mean_terminated_length": 70.54906845092773, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.549181741476059, + "epoch": 1.8508829300196208, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0469340085983276, + "kl": 0.0786884649656713, + "learning_rate": 3.7442773054283844e-08, + "loss": 0.05131007432937622, + "num_tokens": 53801605.0, + "reward": -5.766749477942312e-08, + "reward_std": 0.9918078780174255, + "rewards/rnd_reward/mean": -5.766749477942312e-08, + "rewards/rnd_reward/std": 0.9918078899383544, + "step": 2830, + "step_time": 11.47003021549899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07375000119209289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.9, + "completions/mean_length": 91.74375228881836, + "completions/mean_terminated_length": 68.46907119750976, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.741053342819214, + "epoch": 1.8574231523871811, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0931239128112793, + "kl": 0.0858668725937605, + "learning_rate": 3.580771746239372e-08, + "loss": 0.01960263103246689, + "num_tokens": 53967620.0, + "reward": -8.866190732703672e-08, + "reward_std": 0.9914042472839355, + "rewards/rnd_reward/mean": -8.866190661649398e-08, + "rewards/rnd_reward/std": 0.9914042532444001, + "step": 2840, + "step_time": 11.055267255497165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07875000052154064, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.8, + "completions/mean_length": 94.5212516784668, + "completions/mean_terminated_length": 69.70959434509277, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.616148290038109, + "epoch": 1.8639633747547415, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1507196426391602, + "kl": 0.07959493407979608, + "learning_rate": 3.41726618705036e-08, + "loss": 0.09600520730018616, + "num_tokens": 54134747.0, + "reward": -1.1116266360744475e-07, + "reward_std": 0.9913456678390503, + "rewards/rnd_reward/mean": -1.1116266360744475e-07, + "rewards/rnd_reward/std": 0.9913456499576568, + "step": 2850, + "step_time": 10.70933379684575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08000000175088644, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.4, + "completions/mean_length": 92.95874938964843, + "completions/mean_terminated_length": 67.40721321105957, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.561130478978157, + "epoch": 1.870503597122302, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9997308254241943, + "kl": 0.08147254046052695, + "learning_rate": 3.253760627861347e-08, + "loss": 0.09602594375610352, + "num_tokens": 54297779.0, + "reward": -1.2561679305633788e-07, + "reward_std": 0.9915251851081848, + "rewards/rnd_reward/mean": -1.256167838192823e-07, + "rewards/rnd_reward/std": 0.9915251791477203, + "step": 2860, + "step_time": 10.617276117694564 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000048428773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 324.1, + "completions/mean_length": 94.39250106811524, + "completions/mean_terminated_length": 70.46714973449707, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5941164553165437, + "epoch": 1.8770438194898627, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0238882303237915, + "kl": 0.08611528370529413, + "learning_rate": 3.090255068672335e-08, + "loss": 0.1147921085357666, + "num_tokens": 54467228.0, + "reward": -1.1593103739215849e-07, + "reward_std": 0.9916765570640564, + "rewards/rnd_reward/mean": -1.159310317078166e-07, + "rewards/rnd_reward/std": 0.9916765511035919, + "step": 2870, + "step_time": 11.231190301850438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375000149011611, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.1, + "completions/mean_length": 96.7587516784668, + "completions/mean_terminated_length": 67.1300651550293, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5091202437877653, + "epoch": 1.8835840418574232, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9646898508071899, + "kl": 0.08239567633718252, + "learning_rate": 2.9267495094833225e-08, + "loss": 0.110014808177948, + "num_tokens": 54638260.0, + "reward": -1.0073185103820493e-07, + "reward_std": 0.9914380252361298, + "rewards/rnd_reward/mean": -1.0073185103820493e-07, + "rewards/rnd_reward/std": 0.9914380311965942, + "step": 2880, + "step_time": 10.927705601369961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1087500024586916, + "completions/max_length": 384.0, + "completions/max_terminated_length": 336.5, + "completions/mean_length": 104.54500198364258, + "completions/mean_terminated_length": 70.47102432250976, + "completions/min_length": 2.3, + "completions/min_terminated_length": 2.3, + "entropy": 2.605569988489151, + "epoch": 1.8901242642249838, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9842224717140198, + "kl": 0.07593777822330594, + "learning_rate": 2.7632439502943097e-08, + "loss": 0.09672788381576539, + "num_tokens": 54823046.0, + "reward": 4.0531157274870114e-08, + "reward_std": 0.9914953529834747, + "rewards/rnd_reward/mean": 4.053116011704105e-08, + "rewards/rnd_reward/std": 0.9914953589439393, + "step": 2890, + "step_time": 12.156373349227943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0912500012665987, + "completions/max_length": 384.0, + "completions/max_terminated_length": 362.0, + "completions/mean_length": 101.96250228881836, + "completions/mean_terminated_length": 73.72331771850585, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.55685573220253, + "epoch": 1.8966644865925442, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0599300861358643, + "kl": 0.07744721919298173, + "learning_rate": 2.5997383911052975e-08, + "loss": 0.07260736227035522, + "num_tokens": 54993921.0, + "reward": -7.45057977535879e-09, + "reward_std": 0.9911764979362487, + "rewards/rnd_reward/mean": -7.450580485901526e-09, + "rewards/rnd_reward/std": 0.9911765158176422, + "step": 2900, + "step_time": 10.672014983766712 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000160187482, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.5, + "completions/mean_length": 97.90375137329102, + "completions/mean_terminated_length": 70.14430389404296, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.7193702697753905, + "epoch": 1.9032047089601045, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4999836683273315, + "kl": 0.08308390835300088, + "learning_rate": 2.436232831916285e-08, + "loss": 0.07153651714324952, + "num_tokens": 55168059.0, + "reward": -1.7181038884928058e-07, + "reward_std": 0.991384220123291, + "rewards/rnd_reward/mean": -1.7181038458602416e-07, + "rewards/rnd_reward/std": 0.9913842082023621, + "step": 2910, + "step_time": 11.116872487938963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09875000230967998, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.8, + "completions/mean_length": 100.51500244140625, + "completions/mean_terminated_length": 69.35774002075195, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.603737047314644, + "epoch": 1.909744931327665, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0500236749649048, + "kl": 0.08141210982576012, + "learning_rate": 2.2727272727272725e-08, + "loss": 0.02487606555223465, + "num_tokens": 55350021.0, + "reward": -4.3809413696749286e-08, + "reward_std": 0.9915971457958221, + "rewards/rnd_reward/mean": -4.3809414407292026e-08, + "rewards/rnd_reward/std": 0.9915971517562866, + "step": 2920, + "step_time": 11.766088206716812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000163912773, + "completions/max_length": 384.0, + "completions/max_terminated_length": 345.9, + "completions/mean_length": 96.75125122070312, + "completions/mean_terminated_length": 70.08793716430664, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.6659322917461394, + "epoch": 1.9162851536952257, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1256399154663086, + "kl": 0.08670818526297808, + "learning_rate": 2.1092217135382604e-08, + "loss": 0.09936992526054382, + "num_tokens": 55522567.0, + "reward": -4.485250073571478e-08, + "reward_std": 0.9913349151611328, + "rewards/rnd_reward/mean": -4.485249220920195e-08, + "rewards/rnd_reward/std": 0.9913349032402039, + "step": 2930, + "step_time": 11.034189373115078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10750000178813934, + "completions/max_length": 384.0, + "completions/max_terminated_length": 319.1, + "completions/mean_length": 99.32625198364258, + "completions/mean_terminated_length": 65.10275955200196, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.6324418902397158, + "epoch": 1.9228253760627863, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9896228313446045, + "kl": 0.08653121832758189, + "learning_rate": 1.9457161543492475e-08, + "loss": 0.0832364022731781, + "num_tokens": 55692093.0, + "reward": -1.2382865079985095e-07, + "reward_std": 0.9916486382484436, + "rewards/rnd_reward/mean": -1.238286500893082e-07, + "rewards/rnd_reward/std": 0.9916486382484436, + "step": 2940, + "step_time": 11.120865160599351 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08250000048428774, + "completions/max_length": 384.0, + "completions/max_terminated_length": 360.6, + "completions/mean_length": 100.11875228881836, + "completions/mean_terminated_length": 74.31617698669433, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.6869781494140623, + "epoch": 1.9293655984303466, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.051353931427002, + "kl": 0.0818368756212294, + "learning_rate": 1.7822105951602354e-08, + "loss": 0.11521185636520385, + "num_tokens": 55868653.0, + "reward": -9.492039936276342e-08, + "reward_std": 0.9913798630237579, + "rewards/rnd_reward/mean": -9.492039865222068e-08, + "rewards/rnd_reward/std": 0.9913798570632935, + "step": 2950, + "step_time": 11.270131665398367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000141561032, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.1, + "completions/mean_length": 97.49125289916992, + "completions/mean_terminated_length": 69.66969985961914, + "completions/min_length": 2.6, + "completions/min_terminated_length": 2.6, + "entropy": 2.753763663768768, + "epoch": 1.9359058207979072, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8812336325645447, + "kl": 0.09095769878476859, + "learning_rate": 1.618705035971223e-08, + "loss": 0.043346887826919554, + "num_tokens": 56043276.0, + "reward": -6.645918113612481e-08, + "reward_std": 0.9914931774139404, + "rewards/rnd_reward/mean": -6.645917722813977e-08, + "rewards/rnd_reward/std": 0.991493159532547, + "step": 2960, + "step_time": 11.296948407590389 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07750000078231097, + "completions/max_length": 384.0, + "completions/max_terminated_length": 363.9, + "completions/mean_length": 89.08500251770019, + "completions/mean_terminated_length": 64.43022918701172, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.546912333369255, + "epoch": 1.9424460431654675, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2673183679580688, + "kl": 0.09239747198298573, + "learning_rate": 1.4551994767822105e-08, + "loss": 0.05778201222419739, + "num_tokens": 56208799.0, + "reward": -1.1771917272085374e-07, + "reward_std": 0.9914692223072052, + "rewards/rnd_reward/mean": -1.1771917272085374e-07, + "rewards/rnd_reward/std": 0.9914692461490631, + "step": 2970, + "step_time": 10.944724940578453 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08625000119209289, + "completions/max_length": 384.0, + "completions/max_terminated_length": 344.3, + "completions/mean_length": 94.3025016784668, + "completions/mean_terminated_length": 67.02933235168457, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.585839459300041, + "epoch": 1.9489862655330281, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1477481126785278, + "kl": 0.08655477333813906, + "learning_rate": 1.291693917593198e-08, + "loss": 0.04644548296928406, + "num_tokens": 56386666.0, + "reward": -1.7434358667856031e-07, + "reward_std": 0.9915457606315613, + "rewards/rnd_reward/mean": -1.7434358596801757e-07, + "rewards/rnd_reward/std": 0.9915457487106323, + "step": 2980, + "step_time": 12.27661042441614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08500000275671482, + "completions/max_length": 384.0, + "completions/max_terminated_length": 332.0, + "completions/mean_length": 92.81125259399414, + "completions/mean_terminated_length": 65.69409408569337, + "completions/min_length": 2.5, + "completions/min_terminated_length": 2.5, + "entropy": 2.7046052366495132, + "epoch": 1.9555264879005887, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1466176509857178, + "kl": 0.08695422541350126, + "learning_rate": 1.1281883584041857e-08, + "loss": 0.10825624465942382, + "num_tokens": 56555065.0, + "reward": -2.5480985499370944e-08, + "reward_std": 0.9915042757987976, + "rewards/rnd_reward/mean": -2.5480984078285475e-08, + "rewards/rnd_reward/std": 0.9915042877197265, + "step": 2990, + "step_time": 11.024760208884254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09250000044703484, + "completions/max_length": 384.0, + "completions/max_terminated_length": 358.3, + "completions/mean_length": 92.64375152587891, + "completions/mean_terminated_length": 62.92685241699219, + "completions/min_length": 2.4, + "completions/min_terminated_length": 2.4, + "entropy": 2.619092810153961, + "epoch": 1.9620667102681493, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9762617349624634, + "kl": 0.09417410604655743, + "learning_rate": 9.646827992151732e-09, + "loss": 0.11771528720855713, + "num_tokens": 56719745.0, + "reward": -4.336237964253087e-08, + "reward_std": 0.9909454762935639, + "rewards/rnd_reward/mean": -4.336238106361634e-08, + "rewards/rnd_reward/std": 0.9909454941749573, + "step": 3000, + "step_time": 10.862733572837897 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07625000216066838, + "completions/max_length": 384.0, + "completions/max_terminated_length": 340.9, + "completions/mean_length": 97.29875106811524, + "completions/mean_terminated_length": 73.6241340637207, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 2.5855335265398027, + "epoch": 1.9686069326357096, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9564973711967468, + "kl": 0.08265417153015733, + "learning_rate": 8.011772400261609e-09, + "loss": 0.1359064221382141, + "num_tokens": 56895629.0, + "reward": -1.6182661193298032e-07, + "reward_std": 0.9910951733589173, + "rewards/rnd_reward/mean": -1.6182661051189483e-07, + "rewards/rnd_reward/std": 0.9910951614379883, + "step": 3010, + "step_time": 11.971274512028321 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07750000096857548, + "completions/max_length": 384.0, + "completions/max_terminated_length": 352.7, + "completions/mean_length": 98.39375152587891, + "completions/mean_terminated_length": 74.40588836669922, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.7866202265024187, + "epoch": 1.97514715500327, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1531956195831299, + "kl": 0.08200438115745783, + "learning_rate": 6.376716808371484e-09, + "loss": 0.06191434860229492, + "num_tokens": 57066964.0, + "reward": 2.905726255164609e-08, + "reward_std": 0.9912680804729461, + "rewards/rnd_reward/mean": 2.90572646832743e-08, + "rewards/rnd_reward/std": 0.9912680625915528, + "step": 3020, + "step_time": 10.92601122239139 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08875000271946192, + "completions/max_length": 384.0, + "completions/max_terminated_length": 350.2, + "completions/mean_length": 97.73500289916993, + "completions/mean_terminated_length": 69.95395584106446, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.3795560866594316, + "epoch": 1.9816873773708306, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.179951786994934, + "kl": 0.08247894970700145, + "learning_rate": 4.74166121648136e-09, + "loss": -0.005165333300828934, + "num_tokens": 57234217.0, + "reward": -1.201778673021181e-07, + "reward_std": 0.990892642736435, + "rewards/rnd_reward/mean": -1.2017786552576126e-07, + "rewards/rnd_reward/std": 0.9908926308155059, + "step": 3030, + "step_time": 10.846299196965992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10375000238418579, + "completions/max_length": 384.0, + "completions/max_terminated_length": 348.1, + "completions/mean_length": 106.65250244140626, + "completions/mean_terminated_length": 74.42479286193847, + "completions/min_length": 2.1, + "completions/min_terminated_length": 2.1, + "entropy": 2.6479441046714784, + "epoch": 1.9882275997383911, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0753920078277588, + "kl": 0.07422810150310397, + "learning_rate": 3.106605624591236e-09, + "loss": 0.10241245031356812, + "num_tokens": 57419159.0, + "reward": -2.192705938597328e-07, + "reward_std": 0.9911743998527527, + "rewards/rnd_reward/mean": -2.1927058213577766e-07, + "rewards/rnd_reward/std": 0.9911744236946106, + "step": 3040, + "step_time": 11.45431538640987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0987500011920929, + "completions/max_length": 384.0, + "completions/max_terminated_length": 351.0, + "completions/mean_length": 104.60750122070313, + "completions/mean_terminated_length": 73.98390045166016, + "completions/min_length": 2.2, + "completions/min_terminated_length": 2.2, + "entropy": 2.5245404928922652, + "epoch": 1.9947678221059517, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9773614406585693, + "kl": 0.07916937218979, + "learning_rate": 1.4715500327011118e-09, + "loss": 0.06593776941299438, + "num_tokens": 57600290.0, + "reward": -4.723668549644344e-08, + "reward_std": 0.9912211298942566, + "rewards/rnd_reward/mean": -4.723667839101608e-08, + "rewards/rnd_reward/std": 0.991221135854721, + "step": 3050, + "step_time": 11.087068415922113 + } + ], + "logging_steps": 10, + "max_steps": 3058, + "num_input_tokens_seen": 57735163, + "num_train_epochs": 2, + "save_steps": 1000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 20, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-3058/training_args.bin b/checkpoint-3058/training_args.bin new file mode 100644 index 0000000..4c0ba1a --- /dev/null +++ b/checkpoint-3058/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f7744bfe8fe7e43ca27c8620e9670a669a73959e4583e2dccebcee526f09ef4 +size 6929 diff --git a/config.json b/config.json new file mode 100644 index 0000000..08e4041 --- /dev/null +++ b/config.json @@ -0,0 +1,58 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.3.0", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..8da9fd6 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "do_sample": false, + "eos_token_id": [ + 151643 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.3.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..d4314af --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bba02ff6673562d1ae9767f7aed880522c023b1bd034ca25c1c1e21b03db7846 +size 1976163472 diff --git a/rnd_checkpoint.pt b/rnd_checkpoint.pt new file mode 100644 index 0000000..5cf46fb --- /dev/null +++ b/rnd_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:398591fc033c6afe522285abde1509b3ca2276f82f7ad681563aeab27b525c90 +size 3166277 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..3c72934 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f96da3a872b5e901575b2067c744ad336c3a3d77a21584d20024557b1bd7f0 +size 11422059 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..db25db0 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "truncation_side": "left", + "unk_token": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..4c0ba1a --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f7744bfe8fe7e43ca27c8620e9670a669a73959e4583e2dccebcee526f09ef4 +size 6929