From 9dee0c327e4e02bcb220ffca626754e8f40e94ed Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 13 Jun 2026 10:41:18 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: smjain/qwen25-coder-bash-agent-grpo Source: Original Platform --- .gitattributes | 37 + README.md | 68 + chat_template.jinja | 54 + config.json | 57 + generation_config.json | 13 + last-checkpoint/chat_template.jinja | 54 + last-checkpoint/config.json | 57 + last-checkpoint/generation_config.json | 13 + last-checkpoint/model.safetensors | 3 + last-checkpoint/optimizer.pt | 3 + last-checkpoint/rng_state.pth | 3 + last-checkpoint/scheduler.pt | 3 + last-checkpoint/tokenizer.json | 3 + last-checkpoint/tokenizer_config.json | 29 + last-checkpoint/trainer_state.json | 10294 +++++++++++++++++++++++ last-checkpoint/training_args.bin | 3 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 29 + training_args.bin | 3 + 20 files changed, 10732 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 last-checkpoint/chat_template.jinja create mode 100644 last-checkpoint/config.json create mode 100644 last-checkpoint/generation_config.json create mode 100644 last-checkpoint/model.safetensors create mode 100644 last-checkpoint/optimizer.pt create mode 100644 last-checkpoint/rng_state.pth create mode 100644 last-checkpoint/scheduler.pt create mode 100644 last-checkpoint/tokenizer.json create mode 100644 last-checkpoint/tokenizer_config.json create mode 100644 last-checkpoint/trainer_state.json create mode 100644 last-checkpoint/training_args.bin create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b6153b4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +last-checkpoint/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..1276521 --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +--- +base_model: Qwen/Qwen2.5-Coder-0.5B-Instruct +library_name: transformers +model_name: qwen25-coder-bash-agent-grpo +tags: +- generated_from_trainer +- grpo +- trl +licence: license +--- + +# Model Card for qwen25-coder-bash-agent-grpo + +This model is a fine-tuned version of [Qwen/Qwen2.5-Coder-0.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B-Instruct). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="smjain/qwen25-coder-bash-agent-grpo", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + + + + + +This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300). + +### Framework versions + +- TRL: 0.29.1 +- Transformers: 5.3.0 +- Pytorch: 2.10.0 +- Datasets: 4.3.0 +- Tokenizers: 0.22.2 + +## Citations + +Cite GRPO as: + +```bibtex +@article{shao2024deepseekmath, + title = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}}, + author = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo}, + year = 2024, + eprint = {arXiv:2402.03300}, +} + +``` + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..1c42d76 --- /dev/null +++ b/config.json @@ -0,0 +1,57 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.3.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..622cff8 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.3.0" +} diff --git a/last-checkpoint/chat_template.jinja b/last-checkpoint/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/last-checkpoint/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/last-checkpoint/config.json b/last-checkpoint/config.json new file mode 100644 index 0000000..1c42d76 --- /dev/null +++ b/last-checkpoint/config.json @@ -0,0 +1,57 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.3.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/last-checkpoint/generation_config.json b/last-checkpoint/generation_config.json new file mode 100644 index 0000000..622cff8 --- /dev/null +++ b/last-checkpoint/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.3.0" +} diff --git a/last-checkpoint/model.safetensors b/last-checkpoint/model.safetensors new file mode 100644 index 0000000..84b5918 --- /dev/null +++ b/last-checkpoint/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65193256d5ccb30438cef11eb6ad7674f1d2fcf09ea0393e49b1d87a3d515dd8 +size 988097824 diff --git a/last-checkpoint/optimizer.pt b/last-checkpoint/optimizer.pt new file mode 100644 index 0000000..251fa45 --- /dev/null +++ b/last-checkpoint/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5b2f08bda4c77244e802f619cac412b8568eb56d7991981ab37eefad3475063 +size 1816758411 diff --git a/last-checkpoint/rng_state.pth b/last-checkpoint/rng_state.pth new file mode 100644 index 0000000..15de497 --- /dev/null +++ b/last-checkpoint/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:923d9ca6ba1dfaf2c4704fadd30fd7c899d9dd816934e038458e278bd4a9300f +size 14645 diff --git a/last-checkpoint/scheduler.pt b/last-checkpoint/scheduler.pt new file mode 100644 index 0000000..0cebb62 --- /dev/null +++ b/last-checkpoint/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a72c39316057401dd6b864fda9ee9359b8cb559abc0fe3f44858724c1e7e7457 +size 1465 diff --git a/last-checkpoint/tokenizer.json b/last-checkpoint/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/last-checkpoint/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/last-checkpoint/tokenizer_config.json b/last-checkpoint/tokenizer_config.json new file mode 100644 index 0000000..e4fb8c3 --- /dev/null +++ b/last-checkpoint/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 32768, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/last-checkpoint/trainer_state.json b/last-checkpoint/trainer_state.json new file mode 100644 index 0000000..bcc0735 --- /dev/null +++ b/last-checkpoint/trainer_state.json @@ -0,0 +1,10294 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.9587628865979383, + "eval_steps": 500, + "global_step": 1900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 24.8, + "completions/max_terminated_length": 24.8, + "completions/mean_length": 15.05, + "completions/mean_terminated_length": 15.05, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 1.0464817345142365, + "epoch": 0.005154639175257732, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 1.0309278350515463e-08, + "loss": 0.14594812393188478, + "num_tokens": 1329.0, + "reward": -0.675, + "reward_std": 0.22320507764816283, + "rewards/reward_fn/mean": -0.675, + "rewards/reward_fn/std": 0.22320507764816283, + "step": 5, + "step_time": 0.7463470116257668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 25.6, + "completions/max_terminated_length": 25.6, + "completions/mean_length": 14.95, + "completions/mean_terminated_length": 14.95, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 0.8166205495595932, + "epoch": 0.010309278350515464, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 2.3195876288659794e-08, + "loss": 0.0769703209400177, + "num_tokens": 2696.0, + "reward": -0.65, + "reward_std": 0.15773502588272095, + "rewards/reward_fn/mean": -0.65, + "rewards/reward_fn/std": 0.15773502588272095, + "step": 10, + "step_time": 0.6073689106851816 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.4, + "completions/max_terminated_length": 15.4, + "completions/mean_length": 12.2, + "completions/mean_terminated_length": 12.2, + "completions/min_length": 9.4, + "completions/min_terminated_length": 9.4, + "entropy": 0.9185687482357026, + "epoch": 0.015463917525773196, + "frac_reward_zero_std": 0.6, + "grad_norm": 13.9375, + "learning_rate": 3.608247422680412e-08, + "loss": -0.0014619171619415284, + "num_tokens": 3944.0, + "reward": -0.6100000023841858, + "reward_std": 0.15754843950271608, + "rewards/reward_fn/mean": -0.6100000023841858, + "rewards/reward_fn/std": 0.15754843950271608, + "step": 15, + "step_time": 0.449109182972461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.4, + "completions/max_terminated_length": 15.4, + "completions/mean_length": 12.0, + "completions/mean_terminated_length": 12.0, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 0.6480183869600296, + "epoch": 0.020618556701030927, + "frac_reward_zero_std": 0.0, + "grad_norm": 13.875, + "learning_rate": 4.896907216494845e-08, + "loss": 0.11619714498519898, + "num_tokens": 5188.0, + "reward": -0.6175000011920929, + "reward_std": 0.36337884664535525, + "rewards/reward_fn/mean": -0.6175000011920929, + "rewards/reward_fn/std": 0.36337885856628416, + "step": 20, + "step_time": 0.452333062607795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.2, + "completions/max_terminated_length": 15.2, + "completions/mean_length": 9.4, + "completions/mean_terminated_length": 9.4, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 0.9010713428258896, + "epoch": 0.02577319587628866, + "frac_reward_zero_std": 0.2, + "grad_norm": 34.75, + "learning_rate": 6.185567010309278e-08, + "loss": 0.044125625491142274, + "num_tokens": 6388.0, + "reward": -0.2300000011920929, + "reward_std": 0.43011106848716735, + "rewards/reward_fn/mean": -0.2300000011920929, + "rewards/reward_fn/std": 0.43011106848716735, + "step": 25, + "step_time": 0.44755604499951007 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.4, + "completions/max_terminated_length": 11.4, + "completions/mean_length": 9.0, + "completions/mean_terminated_length": 9.0, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.8676799893379211, + "epoch": 0.030927835051546393, + "frac_reward_zero_std": 0.2, + "grad_norm": 20.75, + "learning_rate": 7.474226804123711e-08, + "loss": 0.1554401397705078, + "num_tokens": 7568.0, + "reward": -0.602999997138977, + "reward_std": 0.2935142517089844, + "rewards/reward_fn/mean": -0.602999997138977, + "rewards/reward_fn/std": 0.29351426362991334, + "step": 30, + "step_time": 0.39083454087376596 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 27.4, + "completions/max_terminated_length": 27.4, + "completions/mean_length": 14.8, + "completions/mean_terminated_length": 14.8, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "entropy": 0.8659799456596374, + "epoch": 0.03608247422680412, + "frac_reward_zero_std": 0.2, + "grad_norm": 24.5, + "learning_rate": 8.762886597938143e-08, + "loss": 0.1145447850227356, + "num_tokens": 8864.0, + "reward": -0.75, + "reward_std": 0.2154700517654419, + "rewards/reward_fn/mean": -0.75, + "rewards/reward_fn/std": 0.2154700517654419, + "step": 35, + "step_time": 0.7107938446104527 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.2, + "completions/max_terminated_length": 19.2, + "completions/mean_length": 13.3, + "completions/mean_terminated_length": 13.3, + "completions/min_length": 9.2, + "completions/min_terminated_length": 9.2, + "entropy": 0.8932873725891113, + "epoch": 0.041237113402061855, + "frac_reward_zero_std": 0.4, + "grad_norm": 22.125, + "learning_rate": 1.0051546391752578e-07, + "loss": 0.11950116157531739, + "num_tokens": 10174.0, + "reward": -0.24749999344348908, + "reward_std": 0.3299496531486511, + "rewards/reward_fn/mean": -0.24749999344348908, + "rewards/reward_fn/std": 0.32994966506958007, + "step": 40, + "step_time": 0.5169275973923504 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.6, + "completions/max_terminated_length": 15.6, + "completions/mean_length": 11.5, + "completions/mean_terminated_length": 11.5, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.8041223675012589, + "epoch": 0.04639175257731959, + "frac_reward_zero_std": 0.0, + "grad_norm": 17.5, + "learning_rate": 1.134020618556701e-07, + "loss": 0.1752654194831848, + "num_tokens": 11420.0, + "reward": -0.5862500011920929, + "reward_std": 0.37825352549552915, + "rewards/reward_fn/mean": -0.5862500011920929, + "rewards/reward_fn/std": 0.3782535195350647, + "step": 45, + "step_time": 0.45717077497392894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.4, + "completions/max_terminated_length": 15.4, + "completions/mean_length": 9.8, + "completions/mean_terminated_length": 9.8, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.6957661122083664, + "epoch": 0.05154639175257732, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 1.2628865979381443e-07, + "loss": 0.1835160255432129, + "num_tokens": 12616.0, + "reward": -0.525, + "reward_std": 0.2809401035308838, + "rewards/reward_fn/mean": -0.525, + "rewards/reward_fn/std": 0.2809401035308838, + "step": 50, + "step_time": 0.45142662366852165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 23.6, + "completions/max_terminated_length": 23.6, + "completions/mean_length": 11.7, + "completions/mean_terminated_length": 11.7, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 0.9917734563350677, + "epoch": 0.05670103092783505, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 1.3917525773195876e-07, + "loss": 0.14120221138000488, + "num_tokens": 13862.0, + "reward": -0.43666666746139526, + "reward_std": 0.3740913510322571, + "rewards/reward_fn/mean": -0.43666666746139526, + "rewards/reward_fn/std": 0.3740913510322571, + "step": 55, + "step_time": 0.6059751262888312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 24.4, + "completions/max_terminated_length": 24.4, + "completions/mean_length": 17.15, + "completions/mean_terminated_length": 17.15, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "entropy": 0.8903216406702995, + "epoch": 0.061855670103092786, + "frac_reward_zero_std": 0.2, + "grad_norm": 5.6875, + "learning_rate": 1.5206185567010307e-07, + "loss": 0.07498554587364196, + "num_tokens": 15237.0, + "reward": -0.6100000023841858, + "reward_std": 0.2532050788402557, + "rewards/reward_fn/mean": -0.6100000023841858, + "rewards/reward_fn/std": 0.2532050788402557, + "step": 60, + "step_time": 0.6272518557496369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 34.4, + "completions/max_terminated_length": 34.4, + "completions/mean_length": 16.65, + "completions/mean_terminated_length": 16.65, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.894698689877987, + "epoch": 0.06701030927835051, + "frac_reward_zero_std": 0.2, + "grad_norm": 18.0, + "learning_rate": 1.649484536082474e-07, + "loss": 0.3684789419174194, + "num_tokens": 16570.0, + "reward": -0.675, + "reward_std": 0.22320507764816283, + "rewards/reward_fn/mean": -0.675, + "rewards/reward_fn/std": 0.22320507764816283, + "step": 65, + "step_time": 0.7892749737016856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.2, + "completions/max_terminated_length": 12.2, + "completions/mean_length": 9.45, + "completions/mean_terminated_length": 9.45, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.9464951276779174, + "epoch": 0.07216494845360824, + "frac_reward_zero_std": 0.6, + "grad_norm": 15.3125, + "learning_rate": 1.7783505154639176e-07, + "loss": 0.05676301717758179, + "num_tokens": 17791.0, + "reward": -0.625, + "reward_std": 0.10773502588272095, + "rewards/reward_fn/mean": -0.625, + "rewards/reward_fn/std": 0.10773502588272095, + "step": 70, + "step_time": 0.45811321027576923 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 27.4, + "completions/max_terminated_length": 27.4, + "completions/mean_length": 17.65, + "completions/mean_terminated_length": 17.65, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 1.323281478881836, + "epoch": 0.07731958762886598, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 1.9072164948453607e-07, + "loss": 0.08197730779647827, + "num_tokens": 19168.0, + "reward": -0.4675000011920929, + "reward_std": 0.16499999761581421, + "rewards/reward_fn/mean": -0.4675000011920929, + "rewards/reward_fn/std": 0.16499999761581421, + "step": 75, + "step_time": 0.6762679857201874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.8, + "completions/max_terminated_length": 12.8, + "completions/mean_length": 9.3, + "completions/mean_terminated_length": 9.3, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 0.6954251408576966, + "epoch": 0.08247422680412371, + "frac_reward_zero_std": 0.2, + "grad_norm": 17.375, + "learning_rate": 2.036082474226804e-07, + "loss": 0.1398395538330078, + "num_tokens": 20382.0, + "reward": -0.65, + "reward_std": 0.2, + "rewards/reward_fn/mean": -0.65, + "rewards/reward_fn/std": 0.2, + "step": 80, + "step_time": 0.5250002114102245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.2, + "completions/max_terminated_length": 20.2, + "completions/mean_length": 13.85, + "completions/mean_terminated_length": 13.85, + "completions/min_length": 8.4, + "completions/min_terminated_length": 8.4, + "entropy": 1.0841705203056335, + "epoch": 0.08762886597938144, + "frac_reward_zero_std": 0.2, + "grad_norm": 23.5, + "learning_rate": 2.1649484536082473e-07, + "loss": 0.06400776505470276, + "num_tokens": 21667.0, + "reward": -0.5949999988079071, + "reward_std": 0.3001110672950745, + "rewards/reward_fn/mean": -0.5949999988079071, + "rewards/reward_fn/std": 0.3001110672950745, + "step": 85, + "step_time": 0.5649234197102487 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 51.2, + "completions/max_terminated_length": 51.2, + "completions/mean_length": 20.95, + "completions/mean_terminated_length": 20.95, + "completions/min_length": 8.8, + "completions/min_terminated_length": 8.8, + "entropy": 0.9970528572797775, + "epoch": 0.09278350515463918, + "frac_reward_zero_std": 0.2, + "grad_norm": 13.4375, + "learning_rate": 2.2938144329896904e-07, + "loss": 0.30556085109710696, + "num_tokens": 23110.0, + "reward": -0.5733333349227905, + "reward_std": 0.2945683240890503, + "rewards/reward_fn/mean": -0.5733333349227905, + "rewards/reward_fn/std": 0.2945683240890503, + "step": 90, + "step_time": 1.0468046895228327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 24.0, + "completions/max_terminated_length": 24.0, + "completions/mean_length": 10.55, + "completions/mean_terminated_length": 10.55, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "entropy": 0.6387317299842834, + "epoch": 0.0979381443298969, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 2.4226804123711337e-07, + "loss": 0.18993494510650635, + "num_tokens": 24341.0, + "reward": -0.3, + "reward_std": 0.1154700517654419, + "rewards/reward_fn/mean": -0.3, + "rewards/reward_fn/std": 0.1154700517654419, + "step": 95, + "step_time": 0.6111652076244354 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05, + "completions/max_length": 101.8, + "completions/max_terminated_length": 55.8, + "completions/mean_length": 37.95, + "completions/mean_terminated_length": 26.15, + "completions/min_length": 12.4, + "completions/min_terminated_length": 12.4, + "entropy": 1.1228386163711548, + "epoch": 0.10309278350515463, + "frac_reward_zero_std": 0.2, + "grad_norm": 8.6875, + "learning_rate": 2.551546391752577e-07, + "loss": 0.16065646409988404, + "num_tokens": 26128.0, + "reward": -0.675, + "reward_std": 0.22320507764816283, + "rewards/reward_fn/mean": -0.675, + "rewards/reward_fn/std": 0.22320507764816283, + "step": 100, + "step_time": 1.852026186324656 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 27.0, + "completions/max_terminated_length": 27.0, + "completions/mean_length": 16.9, + "completions/mean_terminated_length": 16.9, + "completions/min_length": 10.6, + "completions/min_terminated_length": 10.6, + "entropy": 0.9643263280391693, + "epoch": 0.10824742268041238, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 2.6804123711340204e-07, + "loss": 0.07265600562095642, + "num_tokens": 27494.0, + "reward": -0.575, + "reward_std": 0.15, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.15, + "step": 105, + "step_time": 0.6847881611436606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.2, + "completions/max_terminated_length": 19.2, + "completions/mean_length": 13.1, + "completions/mean_terminated_length": 13.1, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.5918209165334701, + "epoch": 0.1134020618556701, + "frac_reward_zero_std": 0.4, + "grad_norm": 8.75, + "learning_rate": 2.809278350515464e-07, + "loss": 0.14053068161010743, + "num_tokens": 28792.0, + "reward": -0.725, + "reward_std": 0.15, + "rewards/reward_fn/mean": -0.725, + "rewards/reward_fn/std": 0.15, + "step": 110, + "step_time": 0.5405681782402099 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.8, + "completions/max_terminated_length": 20.8, + "completions/mean_length": 12.4, + "completions/mean_terminated_length": 12.4, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 1.0056458711624146, + "epoch": 0.11855670103092783, + "frac_reward_zero_std": 0.0, + "grad_norm": 16.625, + "learning_rate": 2.938144329896907e-07, + "loss": 0.23783767223358154, + "num_tokens": 30064.0, + "reward": -0.4850000023841858, + "reward_std": 0.4591261982917786, + "rewards/reward_fn/mean": -0.4850000023841858, + "rewards/reward_fn/std": 0.4591261982917786, + "step": 115, + "step_time": 0.5577491442672908 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.4, + "completions/max_terminated_length": 13.4, + "completions/mean_length": 10.5, + "completions/mean_terminated_length": 10.5, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.6822600662708282, + "epoch": 0.12371134020618557, + "frac_reward_zero_std": 0.0, + "grad_norm": 12.75, + "learning_rate": 3.0670103092783506e-07, + "loss": 0.11178064346313477, + "num_tokens": 31290.0, + "reward": -0.3266666650772095, + "reward_std": 0.4108335912227631, + "rewards/reward_fn/mean": -0.3266666650772095, + "rewards/reward_fn/std": 0.4108336091041565, + "step": 120, + "step_time": 0.5232813514769077 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 23.6, + "completions/max_terminated_length": 23.6, + "completions/mean_length": 14.2, + "completions/mean_terminated_length": 14.2, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 0.8330166220664978, + "epoch": 0.12886597938144329, + "frac_reward_zero_std": 0.0, + "grad_norm": 17.5, + "learning_rate": 3.195876288659793e-07, + "loss": 0.236272931098938, + "num_tokens": 32594.0, + "reward": -0.725, + "reward_std": 0.25, + "rewards/reward_fn/mean": -0.725, + "rewards/reward_fn/std": 0.25, + "step": 125, + "step_time": 0.5966341249644757 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.2, + "completions/max_terminated_length": 18.2, + "completions/mean_length": 13.4, + "completions/mean_terminated_length": 13.4, + "completions/min_length": 9.2, + "completions/min_terminated_length": 9.2, + "entropy": 1.0293640315532684, + "epoch": 0.13402061855670103, + "frac_reward_zero_std": 0.2, + "grad_norm": 15.5625, + "learning_rate": 3.324742268041237e-07, + "loss": 0.03524231314659119, + "num_tokens": 33878.0, + "reward": -0.65, + "reward_std": 0.2, + "rewards/reward_fn/mean": -0.65, + "rewards/reward_fn/std": 0.2, + "step": 130, + "step_time": 0.5198140442371368 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.6, + "completions/max_terminated_length": 17.6, + "completions/mean_length": 12.4, + "completions/mean_terminated_length": 12.4, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "entropy": 0.8266386479139328, + "epoch": 0.13917525773195877, + "frac_reward_zero_std": 0.0, + "grad_norm": 15.625, + "learning_rate": 3.4536082474226804e-07, + "loss": 0.019967517256736754, + "num_tokens": 35158.0, + "reward": -0.6100000023841858, + "reward_std": 0.315283465385437, + "rewards/reward_fn/mean": -0.6100000023841858, + "rewards/reward_fn/std": 0.315283465385437, + "step": 135, + "step_time": 0.5171223360113799 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.6, + "completions/max_terminated_length": 19.6, + "completions/mean_length": 14.5, + "completions/mean_terminated_length": 14.5, + "completions/min_length": 10.6, + "completions/min_terminated_length": 10.6, + "entropy": 0.8061805367469788, + "epoch": 0.14432989690721648, + "frac_reward_zero_std": 0.0, + "grad_norm": 13.5625, + "learning_rate": 3.5824742268041234e-07, + "loss": 0.05468950271606445, + "num_tokens": 36480.0, + "reward": -0.4533333331346512, + "reward_std": 0.3666622221469879, + "rewards/reward_fn/mean": -0.4533333331346512, + "rewards/reward_fn/std": 0.36666223406791687, + "step": 140, + "step_time": 0.5422550996765494 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.4, + "completions/max_terminated_length": 12.4, + "completions/mean_length": 10.05, + "completions/mean_terminated_length": 10.05, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 0.6229674249887467, + "epoch": 0.14948453608247422, + "frac_reward_zero_std": 0.4, + "grad_norm": 20.5, + "learning_rate": 3.711340206185567e-07, + "loss": 0.09494972825050355, + "num_tokens": 37697.0, + "reward": -0.5850000023841858, + "reward_std": 0.23048137426376342, + "rewards/reward_fn/mean": -0.5850000023841858, + "rewards/reward_fn/std": 0.23048137426376342, + "step": 145, + "step_time": 0.4210976084694266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.4, + "completions/max_terminated_length": 17.4, + "completions/mean_length": 11.35, + "completions/mean_terminated_length": 11.35, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 0.8502504467964173, + "epoch": 0.15463917525773196, + "frac_reward_zero_std": 0.0, + "grad_norm": 14.0, + "learning_rate": 3.8402061855670106e-07, + "loss": 0.16331512928009034, + "num_tokens": 38932.0, + "reward": -0.625, + "reward_std": 0.25, + "rewards/reward_fn/mean": -0.625, + "rewards/reward_fn/std": 0.25, + "step": 150, + "step_time": 0.5016824883408845 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.8, + "completions/max_terminated_length": 17.8, + "completions/mean_length": 11.7, + "completions/mean_terminated_length": 11.7, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.8068040907382965, + "epoch": 0.15979381443298968, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 3.969072164948453e-07, + "loss": 0.046058404445648196, + "num_tokens": 40186.0, + "reward": -0.5083333313465118, + "reward_std": 0.22301849126815795, + "rewards/reward_fn/mean": -0.5083333313465118, + "rewards/reward_fn/std": 0.22301849126815795, + "step": 155, + "step_time": 0.6020563900470733 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 25.2, + "completions/max_terminated_length": 25.2, + "completions/mean_length": 13.3, + "completions/mean_terminated_length": 13.3, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.9016699820756913, + "epoch": 0.16494845360824742, + "frac_reward_zero_std": 0.2, + "grad_norm": 15.1875, + "learning_rate": 4.097938144329897e-07, + "loss": 0.1976318597793579, + "num_tokens": 41464.0, + "reward": -0.675, + "reward_std": 0.20773502588272094, + "rewards/reward_fn/mean": -0.675, + "rewards/reward_fn/std": 0.20773502588272094, + "step": 160, + "step_time": 0.6755657859146595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.4, + "completions/max_terminated_length": 15.4, + "completions/mean_length": 11.25, + "completions/mean_terminated_length": 11.25, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "entropy": 0.7308270364999772, + "epoch": 0.17010309278350516, + "frac_reward_zero_std": 0.2, + "grad_norm": 12.4375, + "learning_rate": 4.22680412371134e-07, + "loss": 0.10954289436340332, + "num_tokens": 42701.0, + "reward": -0.725, + "reward_std": 0.22320507764816283, + "rewards/reward_fn/mean": -0.725, + "rewards/reward_fn/std": 0.22320507764816283, + "step": 165, + "step_time": 0.49519333997741344 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.8, + "completions/max_terminated_length": 21.8, + "completions/mean_length": 12.55, + "completions/mean_terminated_length": 12.55, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.746524891257286, + "epoch": 0.17525773195876287, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.125, + "learning_rate": 4.3556701030927834e-07, + "loss": 0.14766517877578736, + "num_tokens": 43944.0, + "reward": -0.75, + "reward_std": 0.25773502588272096, + "rewards/reward_fn/mean": -0.75, + "rewards/reward_fn/std": 0.25773502588272096, + "step": 170, + "step_time": 0.6326937489211559 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 27.2, + "completions/max_terminated_length": 27.2, + "completions/mean_length": 16.4, + "completions/mean_terminated_length": 16.4, + "completions/min_length": 10.2, + "completions/min_terminated_length": 10.2, + "entropy": 0.5598460659384727, + "epoch": 0.18041237113402062, + "frac_reward_zero_std": 0.4, + "grad_norm": 7.34375, + "learning_rate": 4.484536082474227e-07, + "loss": 0.05735980272293091, + "num_tokens": 45308.0, + "reward": -0.7, + "reward_std": 0.2732050776481628, + "rewards/reward_fn/mean": -0.7, + "rewards/reward_fn/std": 0.2732050776481628, + "step": 175, + "step_time": 0.7125312600284814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.0, + "completions/max_terminated_length": 11.0, + "completions/mean_length": 8.25, + "completions/mean_terminated_length": 8.25, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 0.5958581089973449, + "epoch": 0.18556701030927836, + "frac_reward_zero_std": 0.4, + "grad_norm": 16.625, + "learning_rate": 4.61340206185567e-07, + "loss": 0.11873382329940796, + "num_tokens": 46469.0, + "reward": -0.6, + "reward_std": 0.15773502588272095, + "rewards/reward_fn/mean": -0.6, + "rewards/reward_fn/std": 0.15773502588272095, + "step": 180, + "step_time": 0.4346679733134806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 23.6, + "completions/max_terminated_length": 23.6, + "completions/mean_length": 13.75, + "completions/mean_terminated_length": 13.75, + "completions/min_length": 7.4, + "completions/min_terminated_length": 7.4, + "entropy": 0.8451726168394089, + "epoch": 0.19072164948453607, + "frac_reward_zero_std": 0.0, + "grad_norm": 21.5, + "learning_rate": 4.742268041237113e-07, + "loss": 0.058754390478134154, + "num_tokens": 47752.0, + "reward": -0.5240000009536743, + "reward_std": 0.387283456325531, + "rewards/reward_fn/mean": -0.5240000009536743, + "rewards/reward_fn/std": 0.3872834801673889, + "step": 185, + "step_time": 0.6394024872221052 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.6, + "completions/max_terminated_length": 20.6, + "completions/mean_length": 13.35, + "completions/mean_terminated_length": 13.35, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.8434844225645065, + "epoch": 0.1958762886597938, + "frac_reward_zero_std": 0.0, + "grad_norm": 34.75, + "learning_rate": 4.871134020618556e-07, + "loss": 0.23021700382232665, + "num_tokens": 49043.0, + "reward": -0.5949999988079071, + "reward_std": 0.30011107325553893, + "rewards/reward_fn/mean": -0.5949999988079071, + "rewards/reward_fn/std": 0.30011107325553893, + "step": 190, + "step_time": 0.5879216003231704 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.2, + "completions/max_terminated_length": 14.2, + "completions/mean_length": 11.25, + "completions/mean_terminated_length": 11.25, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "entropy": 0.900922417640686, + "epoch": 0.20103092783505155, + "frac_reward_zero_std": 0.2, + "grad_norm": 21.5, + "learning_rate": 5e-07, + "loss": 0.017963612079620363, + "num_tokens": 50296.0, + "reward": -0.4800000011920929, + "reward_std": 0.2878461003303528, + "rewards/reward_fn/mean": -0.4800000011920929, + "rewards/reward_fn/std": 0.2878461003303528, + "step": 195, + "step_time": 0.47086463402956724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 32.8, + "completions/max_terminated_length": 32.8, + "completions/mean_length": 15.8, + "completions/mean_terminated_length": 15.8, + "completions/min_length": 8.4, + "completions/min_terminated_length": 8.4, + "entropy": 0.8438230156898499, + "epoch": 0.20618556701030927, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 4.999898828460382e-07, + "loss": 0.05334873795509339, + "num_tokens": 51648.0, + "reward": -0.4050000011920929, + "reward_std": 0.43887237906455995, + "rewards/reward_fn/mean": -0.4050000011920929, + "rewards/reward_fn/std": 0.43887237906455995, + "step": 200, + "step_time": 0.820699253398925 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.4, + "completions/max_terminated_length": 20.4, + "completions/mean_length": 13.0, + "completions/mean_terminated_length": 13.0, + "completions/min_length": 8.8, + "completions/min_terminated_length": 8.8, + "entropy": 0.703792467713356, + "epoch": 0.211340206185567, + "frac_reward_zero_std": 0.2, + "grad_norm": 18.75, + "learning_rate": 4.999595322030074e-07, + "loss": 0.05575525760650635, + "num_tokens": 52944.0, + "reward": -0.675, + "reward_std": 0.22320507764816283, + "rewards/reward_fn/mean": -0.675, + "rewards/reward_fn/std": 0.22320507764816283, + "step": 205, + "step_time": 0.599746462609619 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.2, + "completions/max_terminated_length": 18.2, + "completions/mean_length": 12.1, + "completions/mean_terminated_length": 12.1, + "completions/min_length": 8.4, + "completions/min_terminated_length": 8.4, + "entropy": 0.696284830570221, + "epoch": 0.21649484536082475, + "frac_reward_zero_std": 0.0, + "grad_norm": 14.8125, + "learning_rate": 4.999089505274044e-07, + "loss": 0.17524465322494506, + "num_tokens": 54234.0, + "reward": -0.55, + "reward_std": 0.5196152329444885, + "rewards/reward_fn/mean": -0.55, + "rewards/reward_fn/std": 0.5196152329444885, + "step": 210, + "step_time": 0.5528348212130367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.0, + "completions/max_terminated_length": 20.0, + "completions/mean_length": 15.9, + "completions/mean_terminated_length": 15.9, + "completions/min_length": 8.8, + "completions/min_terminated_length": 8.8, + "entropy": 0.6460077106952667, + "epoch": 0.22164948453608246, + "frac_reward_zero_std": 0.2, + "grad_norm": 17.125, + "learning_rate": 4.998381419131701e-07, + "loss": -0.008495497703552245, + "num_tokens": 55584.0, + "reward": -0.7425000011920929, + "reward_std": 0.3133788466453552, + "rewards/reward_fn/mean": -0.7425000011920929, + "rewards/reward_fn/std": 0.31337885856628417, + "step": 215, + "step_time": 0.5730495001189411 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 59.4, + "completions/max_terminated_length": 59.4, + "completions/mean_length": 24.85, + "completions/mean_terminated_length": 24.85, + "completions/min_length": 9.2, + "completions/min_terminated_length": 9.2, + "entropy": 0.6805280774831772, + "epoch": 0.2268041237113402, + "frac_reward_zero_std": 0.2, + "grad_norm": 9.6875, + "learning_rate": 4.997471120913578e-07, + "loss": 0.09315776824951172, + "num_tokens": 57113.0, + "reward": -0.5025000005960465, + "reward_std": 0.29997222423553466, + "rewards/reward_fn/mean": -0.5025000005960465, + "rewards/reward_fn/std": 0.2999722123146057, + "step": 220, + "step_time": 1.246410696581006 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.0, + "completions/max_terminated_length": 21.0, + "completions/mean_length": 13.35, + "completions/mean_terminated_length": 13.35, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.5927552968263626, + "epoch": 0.23195876288659795, + "frac_reward_zero_std": 0.2, + "grad_norm": 14.9375, + "learning_rate": 4.996358684296693e-07, + "loss": 0.1518040895462036, + "num_tokens": 58416.0, + "reward": -0.23333333134651185, + "reward_std": 0.5039585947990417, + "rewards/reward_fn/mean": -0.23333333134651185, + "rewards/reward_fn/std": 0.5039585947990417, + "step": 225, + "step_time": 0.6047702097333968 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.4, + "completions/max_terminated_length": 13.4, + "completions/mean_length": 8.75, + "completions/mean_terminated_length": 8.75, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 0.7510967075824737, + "epoch": 0.23711340206185566, + "frac_reward_zero_std": 0.2, + "grad_norm": 21.5, + "learning_rate": 4.995044199318583e-07, + "loss": 0.1716856002807617, + "num_tokens": 59591.0, + "reward": -0.40666666328907014, + "reward_std": 0.3867302894592285, + "rewards/reward_fn/mean": -0.40666666328907014, + "rewards/reward_fn/std": 0.386730295419693, + "step": 230, + "step_time": 0.4744772708974779 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10.0, + "completions/max_terminated_length": 10.0, + "completions/mean_length": 7.55, + "completions/mean_terminated_length": 7.55, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 0.6909678518772125, + "epoch": 0.2422680412371134, + "frac_reward_zero_std": 0.4, + "grad_norm": 40.75, + "learning_rate": 4.993527772370027e-07, + "loss": 0.10667741298675537, + "num_tokens": 60754.0, + "reward": -0.4350000023841858, + "reward_std": 0.23279056549072266, + "rewards/reward_fn/mean": -0.4350000023841858, + "rewards/reward_fn/std": 0.23279056549072266, + "step": 235, + "step_time": 0.40216352241113784 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.4, + "completions/max_terminated_length": 16.4, + "completions/mean_length": 10.7, + "completions/mean_terminated_length": 10.7, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.7458656638860702, + "epoch": 0.24742268041237114, + "frac_reward_zero_std": 0.0, + "grad_norm": 16.125, + "learning_rate": 4.991809526186423e-07, + "loss": -0.0752179503440857, + "num_tokens": 62004.0, + "reward": -0.40208333283662795, + "reward_std": 0.4155647039413452, + "rewards/reward_fn/mean": -0.40208333283662795, + "rewards/reward_fn/std": 0.4155647039413452, + "step": 240, + "step_time": 0.5043572745285928 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.6, + "completions/max_terminated_length": 14.6, + "completions/mean_length": 9.65, + "completions/mean_terminated_length": 9.65, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.7298850357532501, + "epoch": 0.25257731958762886, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.989889599837861e-07, + "loss": 0.137645161151886, + "num_tokens": 63209.0, + "reward": -0.5, + "reward_std": 0.25773502588272096, + "rewards/reward_fn/mean": -0.5, + "rewards/reward_fn/std": 0.25773502588272096, + "step": 245, + "step_time": 0.5419826946221292 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.6, + "completions/max_terminated_length": 13.6, + "completions/mean_length": 8.1, + "completions/mean_terminated_length": 8.1, + "completions/min_length": 4.6, + "completions/min_terminated_length": 4.6, + "entropy": 0.6799002349376678, + "epoch": 0.25773195876288657, + "frac_reward_zero_std": 0.0, + "grad_norm": 17.375, + "learning_rate": 4.987768148717863e-07, + "loss": 0.14280350208282472, + "num_tokens": 64371.0, + "reward": -0.4350000023841858, + "reward_std": 0.5459514260292053, + "rewards/reward_fn/mean": -0.4350000023841858, + "rewards/reward_fn/std": 0.5459514260292053, + "step": 250, + "step_time": 0.46002390459179876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.4, + "completions/max_terminated_length": 17.4, + "completions/mean_length": 12.05, + "completions/mean_terminated_length": 12.05, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.6803009271621704, + "epoch": 0.26288659793814434, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.98544534453081e-07, + "loss": 0.14994003772735595, + "num_tokens": 65652.0, + "reward": -0.625, + "reward_std": 0.15, + "rewards/reward_fn/mean": -0.625, + "rewards/reward_fn/std": 0.15, + "step": 255, + "step_time": 0.5426725285127759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.8, + "completions/max_terminated_length": 14.8, + "completions/mean_length": 13.1, + "completions/mean_terminated_length": 13.1, + "completions/min_length": 10.6, + "completions/min_terminated_length": 10.6, + "entropy": 0.6927774518728256, + "epoch": 0.26804123711340205, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 4.982921375278046e-07, + "loss": 0.05132720470428467, + "num_tokens": 66962.0, + "reward": -0.575, + "reward_std": 0.30773502588272095, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.30773502588272095, + "step": 260, + "step_time": 0.49349935110658405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 24.4, + "completions/max_terminated_length": 24.4, + "completions/mean_length": 14.55, + "completions/mean_terminated_length": 14.55, + "completions/min_length": 9.4, + "completions/min_terminated_length": 9.4, + "entropy": 0.6729653716087342, + "epoch": 0.27319587628865977, + "frac_reward_zero_std": 0.2, + "grad_norm": 11.5625, + "learning_rate": 4.98019644524265e-07, + "loss": 0.1055484414100647, + "num_tokens": 68277.0, + "reward": -0.7, + "reward_std": 0.2, + "rewards/reward_fn/mean": -0.7, + "rewards/reward_fn/std": 0.2, + "step": 265, + "step_time": 0.6585672344081104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.4, + "completions/max_terminated_length": 17.4, + "completions/mean_length": 11.15, + "completions/mean_terminated_length": 11.15, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 1.0371261090040207, + "epoch": 0.27835051546391754, + "frac_reward_zero_std": 0.2, + "grad_norm": 17.875, + "learning_rate": 4.97727077497292e-07, + "loss": 0.13463952541351318, + "num_tokens": 69496.0, + "reward": -0.625, + "reward_std": 0.20773502588272094, + "rewards/reward_fn/mean": -0.625, + "rewards/reward_fn/std": 0.20773502588272094, + "step": 270, + "step_time": 0.5468919139355422 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.0, + "completions/max_terminated_length": 15.0, + "completions/mean_length": 9.3, + "completions/mean_terminated_length": 9.3, + "completions/min_length": 5.4, + "completions/min_terminated_length": 5.4, + "entropy": 0.7886780798435211, + "epoch": 0.28350515463917525, + "frac_reward_zero_std": 0.0, + "grad_norm": 16.875, + "learning_rate": 4.974144601264506e-07, + "loss": 0.21641788482666016, + "num_tokens": 70690.0, + "reward": -0.41000000238418577, + "reward_std": 0.4559956431388855, + "rewards/reward_fn/mean": -0.41000000238418577, + "rewards/reward_fn/std": 0.4559956431388855, + "step": 275, + "step_time": 0.5083554971963167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.2, + "completions/max_terminated_length": 18.2, + "completions/mean_length": 11.9, + "completions/mean_terminated_length": 11.9, + "completions/min_length": 7.4, + "completions/min_terminated_length": 7.4, + "entropy": 0.9502978086471557, + "epoch": 0.28865979381443296, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.970818177141255e-07, + "loss": 0.03687924444675446, + "num_tokens": 71972.0, + "reward": -0.4033333331346512, + "reward_std": 0.3089271903038025, + "rewards/reward_fn/mean": -0.4033333331346512, + "rewards/reward_fn/std": 0.30892720222473147, + "step": 280, + "step_time": 0.5869207398965954 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 24.0, + "completions/max_terminated_length": 24.0, + "completions/mean_length": 14.7, + "completions/mean_terminated_length": 14.7, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 1.0687203228473663, + "epoch": 0.29381443298969073, + "frac_reward_zero_std": 0.2, + "grad_norm": 19.0, + "learning_rate": 4.967291771834726e-07, + "loss": 0.006063053011894226, + "num_tokens": 73274.0, + "reward": -0.5199999988079071, + "reward_std": 0.4233161449432373, + "rewards/reward_fn/mean": -0.5199999988079071, + "rewards/reward_fn/std": 0.4233161449432373, + "step": 285, + "step_time": 0.6561609494499863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.8, + "completions/max_terminated_length": 11.8, + "completions/mean_length": 8.55, + "completions/mean_terminated_length": 8.55, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 0.5886041432619095, + "epoch": 0.29896907216494845, + "frac_reward_zero_std": 0.0, + "grad_norm": 16.5, + "learning_rate": 4.963565670762404e-07, + "loss": 0.20434038639068602, + "num_tokens": 74445.0, + "reward": -0.3962500035762787, + "reward_std": 0.48144287467002866, + "rewards/reward_fn/mean": -0.3962500035762787, + "rewards/reward_fn/std": 0.48144287467002866, + "step": 290, + "step_time": 0.4511735229752958 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.8, + "completions/max_terminated_length": 13.8, + "completions/mean_length": 10.25, + "completions/mean_terminated_length": 10.25, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.8686697453260421, + "epoch": 0.30412371134020616, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 4.959640175504593e-07, + "loss": 0.0394548237323761, + "num_tokens": 75694.0, + "reward": -0.4675000011920929, + "reward_std": 0.16499999761581421, + "rewards/reward_fn/mean": -0.4675000011920929, + "rewards/reward_fn/std": 0.16499999761581421, + "step": 295, + "step_time": 0.474055802822113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.0, + "completions/max_terminated_length": 15.0, + "completions/mean_length": 9.85, + "completions/mean_terminated_length": 9.85, + "completions/min_length": 5.4, + "completions/min_terminated_length": 5.4, + "entropy": 0.7599256873130799, + "epoch": 0.30927835051546393, + "frac_reward_zero_std": 0.2, + "grad_norm": 13.25, + "learning_rate": 4.955515603780013e-07, + "loss": 0.16931414604187012, + "num_tokens": 76915.0, + "reward": -0.49083333611488345, + "reward_std": 0.35166221857070923, + "rewards/reward_fn/mean": -0.49083333611488345, + "rewards/reward_fn/std": 0.35166221857070923, + "step": 300, + "step_time": 0.4883337885141373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 29.6, + "completions/max_terminated_length": 29.6, + "completions/mean_length": 14.55, + "completions/mean_terminated_length": 14.55, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.8846446335315704, + "epoch": 0.31443298969072164, + "frac_reward_zero_std": 0.4, + "grad_norm": 8.375, + "learning_rate": 4.951192289420081e-07, + "loss": 0.11423228979110718, + "num_tokens": 78226.0, + "reward": -0.5699999988079071, + "reward_std": 0.192376047372818, + "rewards/reward_fn/mean": -0.5699999988079071, + "rewards/reward_fn/std": 0.192376047372818, + "step": 305, + "step_time": 0.7256460329517722 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 35.4, + "completions/max_terminated_length": 35.4, + "completions/mean_length": 17.4, + "completions/mean_terminated_length": 17.4, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 1.0954357445240022, + "epoch": 0.31958762886597936, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 4.946670582341894e-07, + "loss": 0.2573416709899902, + "num_tokens": 79606.0, + "reward": -0.675, + "reward_std": 0.20773502588272094, + "rewards/reward_fn/mean": -0.675, + "rewards/reward_fn/std": 0.20773502588272094, + "step": 310, + "step_time": 0.8166594183072448 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 28.4, + "completions/max_terminated_length": 28.4, + "completions/mean_length": 14.6, + "completions/mean_terminated_length": 14.6, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.6663665235042572, + "epoch": 0.3247422680412371, + "frac_reward_zero_std": 0.6, + "grad_norm": 10.75, + "learning_rate": 4.941950848519903e-07, + "loss": 0.12805309295654296, + "num_tokens": 80894.0, + "reward": -0.41166666746139524, + "reward_std": 0.21860073804855346, + "rewards/reward_fn/mean": -0.41166666746139524, + "rewards/reward_fn/std": 0.21860073804855346, + "step": 315, + "step_time": 0.691332861687988 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 44.8, + "completions/max_terminated_length": 44.8, + "completions/mean_length": 18.1, + "completions/mean_terminated_length": 18.1, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 1.0570011407136917, + "epoch": 0.32989690721649484, + "frac_reward_zero_std": 0.4, + "grad_norm": 17.5, + "learning_rate": 4.937033469956302e-07, + "loss": 0.2395000457763672, + "num_tokens": 82264.0, + "reward": -0.4905555546283722, + "reward_std": 0.21888889074325563, + "rewards/reward_fn/mean": -0.4905555546283722, + "rewards/reward_fn/std": 0.21888889074325563, + "step": 320, + "step_time": 1.0275981751270593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.2, + "completions/max_terminated_length": 20.2, + "completions/mean_length": 12.15, + "completions/mean_terminated_length": 12.15, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.7988920092582703, + "epoch": 0.33505154639175255, + "frac_reward_zero_std": 0.8, + "grad_norm": 22.0, + "learning_rate": 4.931918844650094e-07, + "loss": 0.04101354777812958, + "num_tokens": 83523.0, + "reward": -0.4850000023841858, + "reward_std": 0.10754843950271606, + "rewards/reward_fn/mean": -0.4850000023841858, + "rewards/reward_fn/std": 0.10754843950271606, + "step": 325, + "step_time": 0.5553480639122427 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 34.6, + "completions/max_terminated_length": 34.6, + "completions/mean_length": 15.25, + "completions/mean_terminated_length": 15.25, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.7929889589548111, + "epoch": 0.3402061855670103, + "frac_reward_zero_std": 0.0, + "grad_norm": 24.75, + "learning_rate": 4.926607386564898e-07, + "loss": 0.14523402452468873, + "num_tokens": 84840.0, + "reward": -0.6350000023841857, + "reward_std": 0.32301849126815796, + "rewards/reward_fn/mean": -0.6350000023841857, + "rewards/reward_fn/std": 0.32301849126815796, + "step": 330, + "step_time": 0.7962054027244448 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.6, + "completions/max_terminated_length": 9.6, + "completions/mean_length": 7.5, + "completions/mean_terminated_length": 7.5, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 0.48006263077259065, + "epoch": 0.34536082474226804, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.921099525595423e-07, + "loss": 0.10299931764602661, + "num_tokens": 85986.0, + "reward": -0.6, + "reward_std": 0.15773502588272095, + "rewards/reward_fn/mean": -0.6, + "rewards/reward_fn/std": 0.15773502588272095, + "step": 335, + "step_time": 0.38521298291161654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.8, + "completions/max_terminated_length": 17.8, + "completions/mean_length": 10.75, + "completions/mean_terminated_length": 10.75, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.9134504497051239, + "epoch": 0.35051546391752575, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 4.915395707532691e-07, + "loss": 0.017949485778808595, + "num_tokens": 87201.0, + "reward": -0.55, + "reward_std": 0.1, + "rewards/reward_fn/mean": -0.55, + "rewards/reward_fn/std": 0.1, + "step": 340, + "step_time": 0.5092797569930554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.2, + "completions/max_terminated_length": 13.2, + "completions/mean_length": 10.25, + "completions/mean_terminated_length": 10.25, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.9801588356494904, + "epoch": 0.3556701030927835, + "frac_reward_zero_std": 0.4, + "grad_norm": 24.125, + "learning_rate": 4.909496394027944e-07, + "loss": 0.12110986709594726, + "num_tokens": 88402.0, + "reward": -0.575, + "reward_std": 0.15, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.15, + "step": 345, + "step_time": 0.44093081960454583 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 91.8, + "completions/max_terminated_length": 91.8, + "completions/mean_length": 32.55, + "completions/mean_terminated_length": 32.55, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 1.510153776407242, + "epoch": 0.36082474226804123, + "frac_reward_zero_std": 0.4, + "grad_norm": 7.4375, + "learning_rate": 4.903402062555289e-07, + "loss": -0.05523960590362549, + "num_tokens": 90061.0, + "reward": -0.625, + "reward_std": 0.16547005176544188, + "rewards/reward_fn/mean": -0.625, + "rewards/reward_fn/std": 0.16547005176544188, + "step": 350, + "step_time": 1.6976578251458705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 25.0, + "completions/max_terminated_length": 25.0, + "completions/mean_length": 11.8, + "completions/mean_terminated_length": 11.8, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 0.9649739861488342, + "epoch": 0.36597938144329895, + "frac_reward_zero_std": 0.2, + "grad_norm": 18.5, + "learning_rate": 4.897113206373042e-07, + "loss": 0.3782911539077759, + "num_tokens": 91317.0, + "reward": -0.425, + "reward_std": 0.32320507764816286, + "rewards/reward_fn/mean": -0.425, + "rewards/reward_fn/std": 0.32320507764816286, + "step": 355, + "step_time": 0.6405022434890271 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.6, + "completions/max_terminated_length": 17.6, + "completions/mean_length": 9.95, + "completions/mean_terminated_length": 9.95, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 0.9136595696210861, + "epoch": 0.3711340206185567, + "frac_reward_zero_std": 0.0, + "grad_norm": 22.125, + "learning_rate": 4.890630334483814e-07, + "loss": 0.13049967288970948, + "num_tokens": 92528.0, + "reward": -0.4025000035762787, + "reward_std": 0.5875245332717896, + "rewards/reward_fn/mean": -0.4025000035762787, + "rewards/reward_fn/std": 0.5875245332717896, + "step": 360, + "step_time": 0.5148123029619456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.0, + "completions/max_terminated_length": 20.0, + "completions/mean_length": 11.0, + "completions/mean_terminated_length": 11.0, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 1.0597147643566132, + "epoch": 0.37628865979381443, + "frac_reward_zero_std": 0.0, + "grad_norm": 16.25, + "learning_rate": 4.883953971593308e-07, + "loss": 0.4041938304901123, + "num_tokens": 93772.0, + "reward": -0.625, + "reward_std": 0.25, + "rewards/reward_fn/mean": -0.625, + "rewards/reward_fn/std": 0.25, + "step": 365, + "step_time": 0.6115478255786002 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 52.8, + "completions/max_terminated_length": 52.8, + "completions/mean_length": 20.05, + "completions/mean_terminated_length": 20.05, + "completions/min_length": 7.4, + "completions/min_terminated_length": 7.4, + "entropy": 0.7085862278938293, + "epoch": 0.38144329896907214, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.6875, + "learning_rate": 4.877084658067855e-07, + "loss": 0.40793662071228026, + "num_tokens": 95209.0, + "reward": -0.6233333319425582, + "reward_std": 0.3110683560371399, + "rewards/reward_fn/mean": -0.6233333319425582, + "rewards/reward_fn/std": 0.31106836795806886, + "step": 370, + "step_time": 1.0920773649588227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.2, + "completions/max_terminated_length": 17.2, + "completions/mean_length": 11.65, + "completions/mean_terminated_length": 11.65, + "completions/min_length": 8.4, + "completions/min_terminated_length": 8.4, + "entropy": 0.7929205089807511, + "epoch": 0.3865979381443299, + "frac_reward_zero_std": 0.4, + "grad_norm": 42.75, + "learning_rate": 4.870022949890676e-07, + "loss": 0.1025731086730957, + "num_tokens": 96430.0, + "reward": -0.3235000044107437, + "reward_std": 0.4177010774612427, + "rewards/reward_fn/mean": -0.3235000044107437, + "rewards/reward_fn/std": 0.41770108938217165, + "step": 375, + "step_time": 0.5476652001962066 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.6, + "completions/max_terminated_length": 14.6, + "completions/mean_length": 9.85, + "completions/mean_terminated_length": 9.85, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.8785172998905182, + "epoch": 0.3917525773195876, + "frac_reward_zero_std": 0.0, + "grad_norm": 13.6875, + "learning_rate": 4.862769418616881e-07, + "loss": 0.01562568247318268, + "num_tokens": 97623.0, + "reward": -0.5175000011920929, + "reward_std": 0.42273502349853515, + "rewards/reward_fn/mean": -0.5175000011920929, + "rewards/reward_fn/std": 0.42273502349853515, + "step": 380, + "step_time": 0.5373218949884176 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 43.0, + "completions/max_terminated_length": 43.0, + "completions/mean_length": 17.65, + "completions/mean_terminated_length": 17.65, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 0.6945732861757279, + "epoch": 0.39690721649484534, + "frac_reward_zero_std": 0.0, + "grad_norm": 25.0, + "learning_rate": 4.855324651327211e-07, + "loss": 0.32841565608978274, + "num_tokens": 98984.0, + "reward": -0.5350000023841858, + "reward_std": 0.5768915295600892, + "rewards/reward_fn/mean": -0.5350000023841858, + "rewards/reward_fn/std": 0.5768915295600892, + "step": 385, + "step_time": 1.0561855778098106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.2, + "completions/max_terminated_length": 14.2, + "completions/mean_length": 7.8, + "completions/mean_terminated_length": 7.8, + "completions/min_length": 4.6, + "completions/min_terminated_length": 4.6, + "entropy": 1.1521853357553482, + "epoch": 0.4020618556701031, + "frac_reward_zero_std": 0.2, + "grad_norm": 42.75, + "learning_rate": 4.847689250580522e-07, + "loss": 0.1270517587661743, + "num_tokens": 100152.0, + "reward": -0.45, + "reward_std": 0.35615527629852295, + "rewards/reward_fn/mean": -0.45, + "rewards/reward_fn/std": 0.35615527629852295, + "step": 390, + "step_time": 0.5240602082572877 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 27.8, + "completions/max_terminated_length": 27.8, + "completions/mean_length": 15.45, + "completions/mean_terminated_length": 15.45, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 0.6790560841560364, + "epoch": 0.4072164948453608, + "frac_reward_zero_std": 0.2, + "grad_norm": 8.125, + "learning_rate": 4.839863834365013e-07, + "loss": 0.03930776119232178, + "num_tokens": 101485.0, + "reward": -0.75, + "reward_std": 0.2154700517654419, + "rewards/reward_fn/mean": -0.75, + "rewards/reward_fn/std": 0.2154700517654419, + "step": 395, + "step_time": 0.769484898634255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.4, + "completions/max_terminated_length": 17.4, + "completions/mean_length": 11.0, + "completions/mean_terminated_length": 11.0, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.970324519276619, + "epoch": 0.41237113402061853, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.831849036048209e-07, + "loss": 0.12541403770446777, + "num_tokens": 102733.0, + "reward": -0.4425000011920929, + "reward_std": 0.2863911747932434, + "rewards/reward_fn/mean": -0.4425000011920929, + "rewards/reward_fn/std": 0.2863911747932434, + "step": 400, + "step_time": 0.7362870733253658 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.6, + "completions/max_terminated_length": 14.6, + "completions/mean_length": 10.0, + "completions/mean_terminated_length": 10.0, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.7727616012096405, + "epoch": 0.4175257731958763, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 4.823645504325699e-07, + "loss": 0.09246260523796082, + "num_tokens": 103969.0, + "reward": -0.525, + "reward_std": 0.32320507764816286, + "rewards/reward_fn/mean": -0.525, + "rewards/reward_fn/std": 0.32320507764816286, + "step": 405, + "step_time": 0.5494077072478831 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 26.2, + "completions/max_terminated_length": 26.2, + "completions/mean_length": 15.35, + "completions/mean_terminated_length": 15.35, + "completions/min_length": 9.2, + "completions/min_terminated_length": 9.2, + "entropy": 0.9095003753900528, + "epoch": 0.422680412371134, + "frac_reward_zero_std": 0.4, + "grad_norm": 11.875, + "learning_rate": 4.815253903168629e-07, + "loss": 0.04959979057312012, + "num_tokens": 105292.0, + "reward": -0.48850000500679014, + "reward_std": 0.338838791847229, + "rewards/reward_fn/mean": -0.48850000500679014, + "rewards/reward_fn/std": 0.338838791847229, + "step": 410, + "step_time": 0.7390698280185461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.0, + "completions/max_terminated_length": 17.0, + "completions/mean_length": 12.45, + "completions/mean_terminated_length": 12.45, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.7868008881807327, + "epoch": 0.42783505154639173, + "frac_reward_zero_std": 0.2, + "grad_norm": 24.625, + "learning_rate": 4.806674911769968e-07, + "loss": 0.1086654782295227, + "num_tokens": 106557.0, + "reward": -0.5350000023841858, + "reward_std": 0.2300000011920929, + "rewards/reward_fn/mean": -0.5350000023841858, + "rewards/reward_fn/std": 0.2300000011920929, + "step": 415, + "step_time": 0.5790117777884006 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.6, + "completions/max_terminated_length": 8.6, + "completions/mean_length": 7.15, + "completions/mean_terminated_length": 7.15, + "completions/min_length": 5.2, + "completions/min_terminated_length": 5.2, + "entropy": 0.7389998614788056, + "epoch": 0.4329896907216495, + "frac_reward_zero_std": 0.6, + "grad_norm": 13.375, + "learning_rate": 4.79790922448953e-07, + "loss": 0.06091185808181763, + "num_tokens": 107692.0, + "reward": -0.4850000023841858, + "reward_std": 0.1300000011920929, + "rewards/reward_fn/mean": -0.4850000023841858, + "rewards/reward_fn/std": 0.1300000011920929, + "step": 420, + "step_time": 0.43482481176033616 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.8, + "completions/max_terminated_length": 14.8, + "completions/mean_length": 10.5, + "completions/mean_terminated_length": 10.5, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.9143629461526871, + "epoch": 0.4381443298969072, + "frac_reward_zero_std": 0.2, + "grad_norm": 17.75, + "learning_rate": 4.788957550797777e-07, + "loss": -0.00275689959526062, + "num_tokens": 108958.0, + "reward": -0.08333334028720855, + "reward_std": 0.412871390581131, + "rewards/reward_fn/mean": -0.08333334028720855, + "rewards/reward_fn/std": 0.412871390581131, + "step": 425, + "step_time": 0.5315653963014484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10.6, + "completions/max_terminated_length": 10.6, + "completions/mean_length": 8.7, + "completions/mean_terminated_length": 8.7, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 0.6365309089422226, + "epoch": 0.44329896907216493, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.779820615218398e-07, + "loss": 0.04290743470191956, + "num_tokens": 110160.0, + "reward": -0.5366666674613952, + "reward_std": 0.2586212992668152, + "rewards/reward_fn/mean": -0.5366666674613952, + "rewards/reward_fn/std": 0.2586212992668152, + "step": 430, + "step_time": 0.5582886083982885 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.2, + "completions/max_terminated_length": 15.2, + "completions/mean_length": 11.95, + "completions/mean_terminated_length": 11.95, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.6503553047776223, + "epoch": 0.4484536082474227, + "frac_reward_zero_std": 0.2, + "grad_norm": 9.5625, + "learning_rate": 4.770499157269663e-07, + "loss": -0.07173663973808289, + "num_tokens": 111435.0, + "reward": -0.3712500035762787, + "reward_std": 0.32023502588272096, + "rewards/reward_fn/mean": -0.3712500035762787, + "rewards/reward_fn/std": 0.32023502588272096, + "step": 435, + "step_time": 0.5316761738620699 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.4, + "completions/max_terminated_length": 14.4, + "completions/mean_length": 10.7, + "completions/mean_terminated_length": 10.7, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 0.6362611055374146, + "epoch": 0.4536082474226804, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.7609939314045755e-07, + "loss": 0.08669210076332093, + "num_tokens": 112669.0, + "reward": -0.5, + "reward_std": 0.2732050776481628, + "rewards/reward_fn/mean": -0.5, + "rewards/reward_fn/std": 0.2732050776481628, + "step": 440, + "step_time": 0.5227897326461971 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.0, + "completions/max_terminated_length": 16.0, + "completions/mean_length": 10.25, + "completions/mean_terminated_length": 10.25, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 0.5979740798473359, + "epoch": 0.4587628865979381, + "frac_reward_zero_std": 0.6, + "grad_norm": 12.75, + "learning_rate": 4.7513057069498026e-07, + "loss": 0.1255246639251709, + "num_tokens": 113866.0, + "reward": -0.39499999582767487, + "reward_std": 0.30333333015441893, + "rewards/reward_fn/mean": -0.39499999582767487, + "rewards/reward_fn/std": 0.30333333015441893, + "step": 445, + "step_time": 0.5488045696169138 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10.6, + "completions/max_terminated_length": 10.6, + "completions/mean_length": 9.0, + "completions/mean_terminated_length": 9.0, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.7348838299512863, + "epoch": 0.4639175257731959, + "frac_reward_zero_std": 0.2, + "grad_norm": 22.75, + "learning_rate": 4.741435268043411e-07, + "loss": 0.10615365505218506, + "num_tokens": 115050.0, + "reward": -0.575, + "reward_std": 0.3470319628715515, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.3470319747924805, + "step": 450, + "step_time": 0.45176689634099604 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 44.2, + "completions/max_terminated_length": 44.2, + "completions/mean_length": 18.3, + "completions/mean_terminated_length": 18.3, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.7693002432584762, + "epoch": 0.4690721649484536, + "frac_reward_zero_std": 0.2, + "grad_norm": 13.6875, + "learning_rate": 4.7313834135714036e-07, + "loss": 0.41129140853881835, + "num_tokens": 116436.0, + "reward": -0.55, + "reward_std": 0.3154700517654419, + "rewards/reward_fn/mean": -0.55, + "rewards/reward_fn/std": 0.3154700517654419, + "step": 455, + "step_time": 1.0780604547820984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.4, + "completions/max_terminated_length": 14.4, + "completions/mean_length": 11.65, + "completions/mean_terminated_length": 11.65, + "completions/min_length": 8.8, + "completions/min_terminated_length": 8.8, + "entropy": 0.7763857960700988, + "epoch": 0.4742268041237113, + "frac_reward_zero_std": 0.4, + "grad_norm": 19.75, + "learning_rate": 4.721150957103053e-07, + "loss": 0.03669398725032806, + "num_tokens": 117713.0, + "reward": -0.5924999952316284, + "reward_std": 0.2727350234985352, + "rewards/reward_fn/mean": -0.5924999952316284, + "rewards/reward_fn/std": 0.2727350234985352, + "step": 460, + "step_time": 0.5201641391962767 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.6, + "completions/max_terminated_length": 13.6, + "completions/mean_length": 9.4, + "completions/mean_terminated_length": 9.4, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.8593772113323211, + "epoch": 0.4793814432989691, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 4.7107387268250586e-07, + "loss": 0.06270080804824829, + "num_tokens": 118905.0, + "reward": -0.575, + "reward_std": 0.10773502588272095, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.10773502588272095, + "step": 465, + "step_time": 0.527864100690931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.0, + "completions/max_terminated_length": 18.0, + "completions/mean_length": 13.5, + "completions/mean_terminated_length": 13.5, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 1.0802289962768554, + "epoch": 0.4845360824742268, + "frac_reward_zero_std": 0.4, + "grad_norm": 31.625, + "learning_rate": 4.700147565474517e-07, + "loss": -0.023255640268325807, + "num_tokens": 120207.0, + "reward": -0.28666666746139524, + "reward_std": 0.2598089694976807, + "rewards/reward_fn/mean": -0.28666666746139524, + "rewards/reward_fn/std": 0.2598089694976807, + "step": 470, + "step_time": 0.5613622773438692 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 56.4, + "completions/max_terminated_length": 56.4, + "completions/mean_length": 23.7, + "completions/mean_terminated_length": 23.7, + "completions/min_length": 9.2, + "completions/min_terminated_length": 9.2, + "entropy": 0.9632985845208168, + "epoch": 0.4896907216494845, + "frac_reward_zero_std": 0.2, + "grad_norm": 28.875, + "learning_rate": 4.689378330270707e-07, + "loss": 0.08240939378738403, + "num_tokens": 121725.0, + "reward": -0.3850000023841858, + "reward_std": 0.3727463483810425, + "rewards/reward_fn/mean": -0.3850000023841858, + "rewards/reward_fn/std": 0.3727463483810425, + "step": 475, + "step_time": 1.2693270226940512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 28.8, + "completions/max_terminated_length": 28.8, + "completions/mean_length": 16.3, + "completions/mean_terminated_length": 16.3, + "completions/min_length": 10.8, + "completions/min_terminated_length": 10.8, + "entropy": 0.5772907376289368, + "epoch": 0.4948453608247423, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.678431892845713e-07, + "loss": -0.026522517204284668, + "num_tokens": 123051.0, + "reward": -0.502999997138977, + "reward_std": 0.27458140850067136, + "rewards/reward_fn/mean": -0.502999997138977, + "rewards/reward_fn/std": 0.27458140850067136, + "step": 480, + "step_time": 0.721957803145051 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 46.2, + "completions/max_terminated_length": 46.2, + "completions/mean_length": 21.55, + "completions/mean_terminated_length": 21.55, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "entropy": 0.7705437213182449, + "epoch": 0.5, + "frac_reward_zero_std": 0.2, + "grad_norm": 8.1875, + "learning_rate": 4.6673091391738785e-07, + "loss": 0.1539371132850647, + "num_tokens": 124502.0, + "reward": -0.575555557012558, + "reward_std": 0.3588018655776978, + "rewards/reward_fn/mean": -0.575555557012558, + "rewards/reward_fn/std": 0.3588018655776978, + "step": 485, + "step_time": 1.0199918281286955 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 50.4, + "completions/max_terminated_length": 50.4, + "completions/mean_length": 20.05, + "completions/mean_terminated_length": 20.05, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.837895593047142, + "epoch": 0.5051546391752577, + "frac_reward_zero_std": 0.0, + "grad_norm": 12.3125, + "learning_rate": 4.656010969500094e-07, + "loss": 0.3263136863708496, + "num_tokens": 125931.0, + "reward": -0.4266666680574417, + "reward_std": 0.46954628825187683, + "rewards/reward_fn/mean": -0.4266666680574417, + "rewards/reward_fn/std": 0.4695463001728058, + "step": 490, + "step_time": 1.08096160069108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.6, + "completions/max_terminated_length": 21.6, + "completions/mean_length": 15.5, + "completions/mean_terminated_length": 15.5, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "entropy": 0.680565445125103, + "epoch": 0.5103092783505154, + "frac_reward_zero_std": 0.4, + "grad_norm": 23.25, + "learning_rate": 4.6445382982669354e-07, + "loss": 0.13636120557785034, + "num_tokens": 127289.0, + "reward": -0.5175000011920929, + "reward_std": 0.23985110521316527, + "rewards/reward_fn/mean": -0.5175000011920929, + "rewards/reward_fn/std": 0.23985110521316527, + "step": 495, + "step_time": 0.5912062424235046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 27.2, + "completions/max_terminated_length": 27.2, + "completions/mean_length": 12.95, + "completions/mean_terminated_length": 12.95, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 1.105911335349083, + "epoch": 0.5154639175257731, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.6328920540406536e-07, + "loss": 0.17237958908081055, + "num_tokens": 128560.0, + "reward": -0.5600000023841858, + "reward_std": 0.21528346538543702, + "rewards/reward_fn/mean": -0.5600000023841858, + "rewards/reward_fn/std": 0.21528346538543702, + "step": 500, + "step_time": 0.6783841417171061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.2, + "completions/max_terminated_length": 16.2, + "completions/mean_length": 12.6, + "completions/mean_terminated_length": 12.6, + "completions/min_length": 9.4, + "completions/min_terminated_length": 9.4, + "entropy": 0.8051907598972321, + "epoch": 0.520618556701031, + "frac_reward_zero_std": 0.0, + "grad_norm": 16.5, + "learning_rate": 4.621073179436015e-07, + "loss": 0.10203053951263427, + "num_tokens": 129836.0, + "reward": -0.35850000083446504, + "reward_std": 0.6104013562202454, + "rewards/reward_fn/mean": -0.35850000083446504, + "rewards/reward_fn/std": 0.6104013681411743, + "step": 505, + "step_time": 0.5657726871781051 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.8, + "completions/max_terminated_length": 20.8, + "completions/mean_length": 11.55, + "completions/mean_terminated_length": 11.55, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 0.7712950989603996, + "epoch": 0.5257731958762887, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.5625, + "learning_rate": 4.609082631040011e-07, + "loss": 0.1616443634033203, + "num_tokens": 131107.0, + "reward": -0.4925000011920929, + "reward_std": 0.44758613109588624, + "rewards/reward_fn/mean": -0.4925000011920929, + "rewards/reward_fn/std": 0.44758613109588624, + "step": 510, + "step_time": 0.6361785892397165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 27.2, + "completions/max_terminated_length": 27.2, + "completions/mean_length": 16.35, + "completions/mean_terminated_length": 16.35, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "entropy": 1.0122096598148347, + "epoch": 0.5309278350515464, + "frac_reward_zero_std": 0.0, + "grad_norm": 20.75, + "learning_rate": 4.5969213793344373e-07, + "loss": 0.028887468576431274, + "num_tokens": 132482.0, + "reward": -0.38416666984558107, + "reward_std": 0.3881505250930786, + "rewards/reward_fn/mean": -0.38416666984558107, + "rewards/reward_fn/std": 0.3881505370140076, + "step": 515, + "step_time": 0.755103122163564 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 31.0, + "completions/max_terminated_length": 31.0, + "completions/mean_length": 17.75, + "completions/mean_terminated_length": 17.75, + "completions/min_length": 10.4, + "completions/min_terminated_length": 10.4, + "entropy": 0.8129868268966675, + "epoch": 0.5360824742268041, + "frac_reward_zero_std": 0.2, + "grad_norm": 23.25, + "learning_rate": 4.5845904086173405e-07, + "loss": 0.20167853832244872, + "num_tokens": 133845.0, + "reward": -0.725, + "reward_std": 0.22320507764816283, + "rewards/reward_fn/mean": -0.725, + "rewards/reward_fn/std": 0.22320507764816283, + "step": 520, + "step_time": 0.9605252320878208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.8, + "completions/max_terminated_length": 17.8, + "completions/mean_length": 12.3, + "completions/mean_terminated_length": 12.3, + "completions/min_length": 9.2, + "completions/min_terminated_length": 9.2, + "entropy": 0.8394870400428772, + "epoch": 0.5412371134020618, + "frac_reward_zero_std": 0.0, + "grad_norm": 12.4375, + "learning_rate": 4.572090716923353e-07, + "loss": 0.1577498197555542, + "num_tokens": 135115.0, + "reward": -0.3600000023841858, + "reward_std": 0.41528346538543703, + "rewards/reward_fn/mean": -0.3600000023841858, + "rewards/reward_fn/std": 0.41528346538543703, + "step": 525, + "step_time": 0.5848557983525098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 31.6, + "completions/max_terminated_length": 31.6, + "completions/mean_length": 15.45, + "completions/mean_terminated_length": 15.45, + "completions/min_length": 7.4, + "completions/min_terminated_length": 7.4, + "entropy": 1.1088833928108215, + "epoch": 0.5463917525773195, + "frac_reward_zero_std": 0.4, + "grad_norm": 26.375, + "learning_rate": 4.5594233159429185e-07, + "loss": 0.06881465315818787, + "num_tokens": 136436.0, + "reward": -0.4225000023841858, + "reward_std": 0.13499999642372132, + "rewards/reward_fn/mean": -0.4225000023841858, + "rewards/reward_fn/std": 0.13499999642372132, + "step": 530, + "step_time": 0.840742127224803 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 40.2, + "completions/max_terminated_length": 40.2, + "completions/mean_length": 18.7, + "completions/mean_terminated_length": 18.7, + "completions/min_length": 8.4, + "completions/min_terminated_length": 8.4, + "entropy": 0.5888572871685028, + "epoch": 0.5515463917525774, + "frac_reward_zero_std": 0.4, + "grad_norm": 16.375, + "learning_rate": 4.5465892309404065e-07, + "loss": 0.19101574420928955, + "num_tokens": 137866.0, + "reward": -0.4560000032186508, + "reward_std": 0.26554843187332156, + "rewards/reward_fn/mean": -0.4560000032186508, + "rewards/reward_fn/std": 0.26554843187332156, + "step": 535, + "step_time": 0.9919451894238591 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.6, + "completions/max_terminated_length": 14.6, + "completions/mean_length": 11.0, + "completions/mean_terminated_length": 11.0, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.7863050103187561, + "epoch": 0.5567010309278351, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 4.5335895006711257e-07, + "loss": 0.058580482006073, + "num_tokens": 139102.0, + "reward": -0.575, + "reward_std": 0.10773502588272095, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.10773502588272095, + "step": 540, + "step_time": 0.5204128349199891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 32.2, + "completions/max_terminated_length": 32.2, + "completions/mean_length": 14.75, + "completions/mean_terminated_length": 14.75, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.8185445249080658, + "epoch": 0.5618556701030928, + "frac_reward_zero_std": 0.4, + "grad_norm": 13.375, + "learning_rate": 4.520425177297259e-07, + "loss": 0.25057742595672605, + "num_tokens": 140433.0, + "reward": -0.3316666603088379, + "reward_std": 0.12333332747220993, + "rewards/reward_fn/mean": -0.3316666603088379, + "rewards/reward_fn/std": 0.12333333045244217, + "step": 545, + "step_time": 0.8382557584904134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.6, + "completions/max_terminated_length": 13.6, + "completions/mean_length": 11.05, + "completions/mean_terminated_length": 11.05, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "entropy": 0.7716027140617371, + "epoch": 0.5670103092783505, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 4.507097326302696e-07, + "loss": 0.040840917825698854, + "num_tokens": 141698.0, + "reward": -0.4600000023841858, + "reward_std": 0.2957271337509155, + "rewards/reward_fn/mean": -0.4600000023841858, + "rewards/reward_fn/std": 0.29572715759277346, + "step": 550, + "step_time": 0.5102354141883552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.8, + "completions/max_terminated_length": 19.8, + "completions/mean_length": 12.75, + "completions/mean_terminated_length": 12.75, + "completions/min_length": 8.4, + "completions/min_terminated_length": 8.4, + "entropy": 0.938120248913765, + "epoch": 0.5721649484536082, + "frac_reward_zero_std": 0.4, + "grad_norm": 29.125, + "learning_rate": 4.4936070264068016e-07, + "loss": 0.204956579208374, + "num_tokens": 142965.0, + "reward": -0.3550000011920929, + "reward_std": 0.14041451513767242, + "rewards/reward_fn/mean": -0.3550000011920929, + "rewards/reward_fn/std": 0.14041451513767242, + "step": 555, + "step_time": 0.6310633135959506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.2, + "completions/max_terminated_length": 11.2, + "completions/mean_length": 9.2, + "completions/mean_terminated_length": 9.2, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.7356794983148575, + "epoch": 0.5773195876288659, + "frac_reward_zero_std": 0.6, + "grad_norm": 16.0, + "learning_rate": 4.4799553694771043e-07, + "loss": 0.024954798817634582, + "num_tokens": 144169.0, + "reward": -0.2474999964237213, + "reward_std": 0.13833333253860475, + "rewards/reward_fn/mean": -0.2474999964237213, + "rewards/reward_fn/std": 0.13833333253860475, + "step": 560, + "step_time": 0.5636249963194132 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.8, + "completions/max_terminated_length": 14.8, + "completions/mean_length": 10.1, + "completions/mean_terminated_length": 10.1, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.6894051700830459, + "epoch": 0.5824742268041238, + "frac_reward_zero_std": 0.0, + "grad_norm": 17.375, + "learning_rate": 4.4661434604409234e-07, + "loss": 0.09334607720375061, + "num_tokens": 145399.0, + "reward": -0.3425000011920929, + "reward_std": 0.4114371001720428, + "rewards/reward_fn/mean": -0.3425000011920929, + "rewards/reward_fn/std": 0.4114371001720428, + "step": 565, + "step_time": 0.5348106060177088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 48.4, + "completions/max_terminated_length": 48.4, + "completions/mean_length": 22.6, + "completions/mean_terminated_length": 22.6, + "completions/min_length": 9.2, + "completions/min_terminated_length": 9.2, + "entropy": 0.6998355552554131, + "epoch": 0.5876288659793815, + "frac_reward_zero_std": 0.0, + "grad_norm": 19.375, + "learning_rate": 4.45217241719594e-07, + "loss": 0.21809990406036378, + "num_tokens": 146891.0, + "reward": -0.5775000035762787, + "reward_std": 0.3775861322879791, + "rewards/reward_fn/mean": -0.5775000035762787, + "rewards/reward_fn/std": 0.3775861322879791, + "step": 570, + "step_time": 1.1578531011939048 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.0, + "completions/max_terminated_length": 13.0, + "completions/mean_length": 9.7, + "completions/mean_terminated_length": 9.7, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.7059618055820465, + "epoch": 0.5927835051546392, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 4.4380433705197193e-07, + "loss": 0.14664100408554076, + "num_tokens": 148093.0, + "reward": -0.65, + "reward_std": 0.2, + "rewards/reward_fn/mean": -0.65, + "rewards/reward_fn/std": 0.2, + "step": 575, + "step_time": 0.49879590775817634 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.0, + "completions/max_terminated_length": 18.0, + "completions/mean_length": 10.2, + "completions/mean_terminated_length": 10.2, + "completions/min_length": 5.2, + "completions/min_terminated_length": 5.2, + "entropy": 0.8498676598072052, + "epoch": 0.5979381443298969, + "frac_reward_zero_std": 0.4, + "grad_norm": 25.75, + "learning_rate": 4.423757463978186e-07, + "loss": 0.1832141637802124, + "num_tokens": 149313.0, + "reward": -0.5200000047683716, + "reward_std": 0.23754844069480896, + "rewards/reward_fn/mean": -0.5200000047683716, + "rewards/reward_fn/std": 0.23754844069480896, + "step": 580, + "step_time": 0.5867984751239419 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 53.8, + "completions/max_terminated_length": 53.8, + "completions/mean_length": 25.25, + "completions/mean_terminated_length": 25.25, + "completions/min_length": 8.4, + "completions/min_terminated_length": 8.4, + "entropy": 0.9533327341079711, + "epoch": 0.6030927835051546, + "frac_reward_zero_std": 0.0, + "grad_norm": 14.3125, + "learning_rate": 4.409315853833067e-07, + "loss": 0.22886526584625244, + "num_tokens": 150850.0, + "reward": -0.5675000011920929, + "reward_std": 0.32273502349853517, + "rewards/reward_fn/mean": -0.5675000011920929, + "rewards/reward_fn/std": 0.32273502349853517, + "step": 585, + "step_time": 1.2400014178827405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.4, + "completions/max_terminated_length": 21.4, + "completions/mean_length": 13.65, + "completions/mean_terminated_length": 13.65, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 1.055218482017517, + "epoch": 0.6082474226804123, + "frac_reward_zero_std": 0.2, + "grad_norm": 20.125, + "learning_rate": 4.394719708948309e-07, + "loss": 0.11721107959747315, + "num_tokens": 152179.0, + "reward": -0.3637500017881393, + "reward_std": 0.5170861959457398, + "rewards/reward_fn/mean": -0.3637500017881393, + "rewards/reward_fn/std": 0.5170861840248108, + "step": 590, + "step_time": 0.6499694635160267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.6, + "completions/max_terminated_length": 15.6, + "completions/mean_length": 10.1, + "completions/mean_terminated_length": 10.1, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.8289062976837158, + "epoch": 0.6134020618556701, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.3799702106954725e-07, + "loss": 0.054711729288101196, + "num_tokens": 153401.0, + "reward": -0.47750000059604647, + "reward_std": 0.24997222423553467, + "rewards/reward_fn/mean": -0.47750000059604647, + "rewards/reward_fn/std": 0.2499722123146057, + "step": 595, + "step_time": 0.5453162741847336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 61.6, + "completions/max_terminated_length": 61.6, + "completions/mean_length": 23.55, + "completions/mean_terminated_length": 23.55, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.9436829313635826, + "epoch": 0.6185567010309279, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 4.365068552858115e-07, + "loss": 0.03441246449947357, + "num_tokens": 154896.0, + "reward": -0.525, + "reward_std": 0.05, + "rewards/reward_fn/mean": -0.525, + "rewards/reward_fn/std": 0.05, + "step": 600, + "step_time": 1.3744830467738212 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.0, + "completions/max_terminated_length": 19.0, + "completions/mean_length": 11.65, + "completions/mean_terminated_length": 11.65, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.8497404426336288, + "epoch": 0.6237113402061856, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 4.350015941535169e-07, + "loss": 0.1358797073364258, + "num_tokens": 156149.0, + "reward": -0.17900000214576722, + "reward_std": 0.48975425958633423, + "rewards/reward_fn/mean": -0.17900000214576722, + "rewards/reward_fn/std": 0.48975429534912107, + "step": 605, + "step_time": 0.7100505581125617 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.0, + "completions/max_terminated_length": 19.0, + "completions/mean_length": 13.7, + "completions/mean_terminated_length": 13.7, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "entropy": 1.0203462660312652, + "epoch": 0.6288659793814433, + "frac_reward_zero_std": 0.2, + "grad_norm": 18.875, + "learning_rate": 4.334813595043325e-07, + "loss": 0.15752297639846802, + "num_tokens": 157443.0, + "reward": -0.4810000002384186, + "reward_std": 0.2585418462753296, + "rewards/reward_fn/mean": -0.4810000002384186, + "rewards/reward_fn/std": 0.2585418462753296, + "step": 610, + "step_time": 0.6184165499173104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.2, + "completions/max_terminated_length": 12.2, + "completions/mean_length": 8.6, + "completions/mean_terminated_length": 8.6, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.6573485657572746, + "epoch": 0.634020618556701, + "frac_reward_zero_std": 0.2, + "grad_norm": 24.625, + "learning_rate": 4.3194627438184233e-07, + "loss": 0.09532037973403931, + "num_tokens": 158619.0, + "reward": -0.20666666626930236, + "reward_std": 0.4414422929286957, + "rewards/reward_fn/mean": -0.20666666626930236, + "rewards/reward_fn/std": 0.44144230484962466, + "step": 615, + "step_time": 0.4910893154330552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 22.8, + "completions/max_terminated_length": 22.8, + "completions/mean_length": 14.35, + "completions/mean_terminated_length": 14.35, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.8949583232402801, + "epoch": 0.6391752577319587, + "frac_reward_zero_std": 0.6, + "grad_norm": 21.625, + "learning_rate": 4.303964630315866e-07, + "loss": -0.06437566876411438, + "num_tokens": 159926.0, + "reward": -0.5116666674613952, + "reward_std": 0.21679993867874145, + "rewards/reward_fn/mean": -0.5116666674613952, + "rewards/reward_fn/std": 0.21679995059967042, + "step": 620, + "step_time": 0.6933138659223914 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.0, + "completions/max_terminated_length": 20.0, + "completions/mean_length": 12.0, + "completions/mean_terminated_length": 12.0, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 1.2098346531391144, + "epoch": 0.6443298969072165, + "frac_reward_zero_std": 0.2, + "grad_norm": 17.75, + "learning_rate": 4.288320508910057e-07, + "loss": 0.07893999814987182, + "num_tokens": 161206.0, + "reward": -0.4925000011920929, + "reward_std": 0.36073286533355714, + "rewards/reward_fn/mean": -0.4925000011920929, + "rewards/reward_fn/std": 0.36073286533355714, + "step": 625, + "step_time": 0.6323956601321697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.8, + "completions/max_terminated_length": 11.8, + "completions/mean_length": 10.2, + "completions/mean_terminated_length": 10.2, + "completions/min_length": 8.8, + "completions/min_terminated_length": 8.8, + "entropy": 0.48069052398204803, + "epoch": 0.6494845360824743, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.272531645792876e-07, + "loss": 0.08115171790122985, + "num_tokens": 162410.0, + "reward": -0.31124999821186067, + "reward_std": 0.4536625385284424, + "rewards/reward_fn/mean": -0.31124999821186067, + "rewards/reward_fn/std": 0.4536625385284424, + "step": 630, + "step_time": 0.48424859074875715 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.0, + "completions/max_terminated_length": 17.0, + "completions/mean_length": 10.35, + "completions/mean_terminated_length": 10.35, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.8250537186861038, + "epoch": 0.654639175257732, + "frac_reward_zero_std": 0.4, + "grad_norm": 14.125, + "learning_rate": 4.256599318871193e-07, + "loss": 0.17227303981781006, + "num_tokens": 163649.0, + "reward": -0.3962500035762787, + "reward_std": 0.3790696501731873, + "rewards/reward_fn/mean": -0.3962500035762787, + "rewards/reward_fn/std": 0.37906967401504515, + "step": 635, + "step_time": 0.5772591234184802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.8, + "completions/max_terminated_length": 20.8, + "completions/mean_length": 14.5, + "completions/mean_terminated_length": 14.5, + "completions/min_length": 9.4, + "completions/min_terminated_length": 9.4, + "entropy": 0.912872189283371, + "epoch": 0.6597938144329897, + "frac_reward_zero_std": 0.0, + "grad_norm": 17.125, + "learning_rate": 4.240524817663445e-07, + "loss": 0.10593240261077881, + "num_tokens": 164959.0, + "reward": -0.5704999923706054, + "reward_std": 0.4167350232601166, + "rewards/reward_fn/mean": -0.5704999923706054, + "rewards/reward_fn/std": 0.4167350232601166, + "step": 640, + "step_time": 0.7264971459284425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.8, + "completions/max_terminated_length": 20.8, + "completions/mean_length": 15.1, + "completions/mean_terminated_length": 15.1, + "completions/min_length": 10.6, + "completions/min_terminated_length": 10.6, + "entropy": 0.776880145072937, + "epoch": 0.6649484536082474, + "frac_reward_zero_std": 0.2, + "grad_norm": 7.90625, + "learning_rate": 4.2243094431952607e-07, + "loss": 0.006071268022060395, + "num_tokens": 166273.0, + "reward": -0.5350000023841858, + "reward_std": 0.2300000011920929, + "rewards/reward_fn/mean": -0.5350000023841858, + "rewards/reward_fn/std": 0.2300000011920929, + "step": 645, + "step_time": 0.6461667789146304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.6, + "completions/max_terminated_length": 19.6, + "completions/mean_length": 12.25, + "completions/mean_terminated_length": 12.25, + "completions/min_length": 8.4, + "completions/min_terminated_length": 8.4, + "entropy": 0.8619225978851318, + "epoch": 0.6701030927835051, + "frac_reward_zero_std": 0.6, + "grad_norm": 11.375, + "learning_rate": 4.2079545078941593e-07, + "loss": 0.007808300852775574, + "num_tokens": 167550.0, + "reward": -0.32999999523162843, + "reward_std": 0.07333333492279052, + "rewards/reward_fn/mean": -0.32999999523162843, + "rewards/reward_fn/std": 0.07333333492279052, + "step": 650, + "step_time": 0.6256520722061396 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.6, + "completions/max_terminated_length": 21.6, + "completions/mean_length": 12.75, + "completions/mean_terminated_length": 12.75, + "completions/min_length": 7.4, + "completions/min_terminated_length": 7.4, + "entropy": 0.7383227556943893, + "epoch": 0.6752577319587629, + "frac_reward_zero_std": 0.4, + "grad_norm": 11.75, + "learning_rate": 4.1914613354833297e-07, + "loss": 0.0942760944366455, + "num_tokens": 168825.0, + "reward": -0.625, + "reward_std": 0.16547005176544188, + "rewards/reward_fn/mean": -0.625, + "rewards/reward_fn/std": 0.16547005176544188, + "step": 655, + "step_time": 0.6979141506366432 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.6, + "completions/max_terminated_length": 11.6, + "completions/mean_length": 8.5, + "completions/mean_terminated_length": 8.5, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.7141625940799713, + "epoch": 0.6804123711340206, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.1748312608744886e-07, + "loss": 0.08471410274505616, + "num_tokens": 170023.0, + "reward": -0.5, + "reward_std": 0.25773502588272096, + "rewards/reward_fn/mean": -0.5, + "rewards/reward_fn/std": 0.25773502588272096, + "step": 660, + "step_time": 0.4793179755099118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 48.2, + "completions/max_terminated_length": 48.2, + "completions/mean_length": 17.95, + "completions/mean_terminated_length": 17.95, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 0.6713347196578979, + "epoch": 0.6855670103092784, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.625, + "learning_rate": 4.158065630059837e-07, + "loss": 0.25584611892700193, + "num_tokens": 171406.0, + "reward": -0.38000000417232516, + "reward_std": 0.418888658285141, + "rewards/reward_fn/mean": -0.38000000417232516, + "rewards/reward_fn/std": 0.418888658285141, + "step": 665, + "step_time": 1.1557064381428064 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.8, + "completions/max_terminated_length": 15.8, + "completions/mean_length": 11.4, + "completions/mean_terminated_length": 11.4, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 0.8641545414924622, + "epoch": 0.6907216494845361, + "frac_reward_zero_std": 0.0, + "grad_norm": 35.0, + "learning_rate": 4.1411658000031217e-07, + "loss": 0.11413931846618652, + "num_tokens": 172646.0, + "reward": -0.5350000023841858, + "reward_std": 0.40320507884025575, + "rewards/reward_fn/mean": -0.5350000023841858, + "rewards/reward_fn/std": 0.40320507884025575, + "step": 670, + "step_time": 0.63343042479828 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.2, + "completions/max_terminated_length": 15.2, + "completions/mean_length": 10.15, + "completions/mean_terminated_length": 10.15, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.8342425405979157, + "epoch": 0.6958762886597938, + "frac_reward_zero_std": 0.2, + "grad_norm": 19.875, + "learning_rate": 4.124133138529803e-07, + "loss": 0.1467257022857666, + "num_tokens": 173877.0, + "reward": -0.38166667222976686, + "reward_std": 0.3997236371040344, + "rewards/reward_fn/mean": -0.38166667222976686, + "rewards/reward_fn/std": 0.3997236490249634, + "step": 675, + "step_time": 0.5436047066934406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.0, + "completions/max_terminated_length": 17.0, + "completions/mean_length": 11.75, + "completions/mean_terminated_length": 11.75, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 1.0363103717565536, + "epoch": 0.7010309278350515, + "frac_reward_zero_std": 0.2, + "grad_norm": 27.125, + "learning_rate": 4.106969024216348e-07, + "loss": 0.08919697999954224, + "num_tokens": 175136.0, + "reward": -0.27999999523162844, + "reward_std": 0.32804334759712217, + "rewards/reward_fn/mean": -0.27999999523162844, + "rewards/reward_fn/std": 0.32804334759712217, + "step": 680, + "step_time": 0.5779976899735629 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 32.4, + "completions/max_terminated_length": 32.4, + "completions/mean_length": 14.95, + "completions/mean_terminated_length": 14.95, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.9448502540588379, + "epoch": 0.7061855670103093, + "frac_reward_zero_std": 0.0, + "grad_norm": 18.375, + "learning_rate": 4.089674846278656e-07, + "loss": 0.04907660484313965, + "num_tokens": 176463.0, + "reward": -0.525, + "reward_std": 0.35, + "rewards/reward_fn/mean": -0.525, + "rewards/reward_fn/std": 0.35, + "step": 685, + "step_time": 0.8561360297724605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.4, + "completions/max_terminated_length": 18.4, + "completions/mean_length": 11.05, + "completions/mean_terminated_length": 11.05, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 1.0519244968891144, + "epoch": 0.711340206185567, + "frac_reward_zero_std": 0.2, + "grad_norm": 29.75, + "learning_rate": 4.072252004459611e-07, + "loss": 0.20497164726257325, + "num_tokens": 177712.0, + "reward": -0.39749999940395353, + "reward_std": 0.34408590793609617, + "rewards/reward_fn/mean": -0.39749999940395353, + "rewards/reward_fn/std": 0.34408590793609617, + "step": 690, + "step_time": 0.5954566929489374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.8, + "completions/max_terminated_length": 18.8, + "completions/mean_length": 13.7, + "completions/mean_terminated_length": 13.7, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 0.8845273166894912, + "epoch": 0.7164948453608248, + "frac_reward_zero_std": 0.2, + "grad_norm": 13.8125, + "learning_rate": 4.0547019089158004e-07, + "loss": 0.05732879638671875, + "num_tokens": 179006.0, + "reward": -0.5483333319425583, + "reward_std": 0.2610683560371399, + "rewards/reward_fn/mean": -0.5483333319425583, + "rewards/reward_fn/std": 0.26106836795806887, + "step": 695, + "step_time": 0.6034244776703417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.8, + "completions/max_terminated_length": 21.8, + "completions/mean_length": 12.25, + "completions/mean_terminated_length": 12.25, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.8579754889011383, + "epoch": 0.7216494845360825, + "frac_reward_zero_std": 0.2, + "grad_norm": 15.3125, + "learning_rate": 4.037025980103372e-07, + "loss": 0.15266115665435792, + "num_tokens": 180275.0, + "reward": -0.5233333319425583, + "reward_std": 0.25333333015441895, + "rewards/reward_fn/mean": -0.5233333319425583, + "rewards/reward_fn/std": 0.2533333420753479, + "step": 700, + "step_time": 0.657803092431277 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.0, + "completions/max_terminated_length": 15.0, + "completions/mean_length": 9.0, + "completions/mean_terminated_length": 9.0, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 0.6864415079355239, + "epoch": 0.7268041237113402, + "frac_reward_zero_std": 0.4, + "grad_norm": 38.0, + "learning_rate": 4.019225648663072e-07, + "loss": 0.16153377294540405, + "num_tokens": 181455.0, + "reward": -0.29500000476837157, + "reward_std": 0.3967333257198334, + "rewards/reward_fn/mean": -0.29500000476837157, + "rewards/reward_fn/std": 0.3967333257198334, + "step": 705, + "step_time": 0.5445711076259613 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.4, + "completions/max_terminated_length": 12.4, + "completions/mean_length": 8.95, + "completions/mean_terminated_length": 8.95, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.8134978413581848, + "epoch": 0.7319587628865979, + "frac_reward_zero_std": 0.0, + "grad_norm": 21.25, + "learning_rate": 4.0013023553044504e-07, + "loss": 0.09724601507186889, + "num_tokens": 182658.0, + "reward": -0.375, + "reward_std": 0.506155276298523, + "rewards/reward_fn/mean": -0.375, + "rewards/reward_fn/std": 0.506155276298523, + "step": 710, + "step_time": 0.5672263609245419 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.6, + "completions/max_terminated_length": 20.6, + "completions/mean_length": 12.95, + "completions/mean_terminated_length": 12.95, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "entropy": 1.1158703505992889, + "epoch": 0.7371134020618557, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 3.983257550689255e-07, + "loss": 0.02380000948905945, + "num_tokens": 183929.0, + "reward": -0.525, + "reward_std": 0.05, + "rewards/reward_fn/mean": -0.525, + "rewards/reward_fn/std": 0.05, + "step": 715, + "step_time": 0.6364484421908856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.8, + "completions/max_terminated_length": 13.8, + "completions/mean_length": 8.25, + "completions/mean_terminated_length": 8.25, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 0.8417123228311538, + "epoch": 0.7422680412371134, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 3.9650926953140173e-07, + "loss": 0.08947134613990784, + "num_tokens": 185134.0, + "reward": -0.5199999988079071, + "reward_std": 0.20784609913825988, + "rewards/reward_fn/mean": -0.5199999988079071, + "rewards/reward_fn/std": 0.20784609913825988, + "step": 720, + "step_time": 0.51900653578341 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.6, + "completions/max_terminated_length": 9.6, + "completions/mean_length": 7.85, + "completions/mean_terminated_length": 7.85, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 0.868690624833107, + "epoch": 0.7474226804123711, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 3.946809259391846e-07, + "loss": 0.018198567628860473, + "num_tokens": 186303.0, + "reward": -0.20000000894069672, + "reward_std": 0.3599011540412903, + "rewards/reward_fn/mean": -0.20000000894069672, + "rewards/reward_fn/std": 0.3599011540412903, + "step": 725, + "step_time": 0.4509535185061395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.4, + "completions/max_terminated_length": 12.4, + "completions/mean_length": 8.8, + "completions/mean_terminated_length": 8.8, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 0.7654918372631073, + "epoch": 0.7525773195876289, + "frac_reward_zero_std": 0.2, + "grad_norm": 23.25, + "learning_rate": 3.9284087227334283e-07, + "loss": 0.19318230152130128, + "num_tokens": 187483.0, + "reward": -0.65, + "reward_std": 0.2154700517654419, + "rewards/reward_fn/mean": -0.65, + "rewards/reward_fn/std": 0.2154700517654419, + "step": 730, + "step_time": 0.48818655135110023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.6, + "completions/max_terminated_length": 13.6, + "completions/mean_length": 8.6, + "completions/mean_terminated_length": 8.6, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 0.9335297107696533, + "epoch": 0.7577319587628866, + "frac_reward_zero_std": 0.4, + "grad_norm": 30.375, + "learning_rate": 3.909892574627266e-07, + "loss": 0.11663612127304077, + "num_tokens": 188679.0, + "reward": -0.37750000357627866, + "reward_std": 0.2149999976158142, + "rewards/reward_fn/mean": -0.37750000357627866, + "rewards/reward_fn/std": 0.2149999976158142, + "step": 735, + "step_time": 0.5085009665228426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.8, + "completions/max_terminated_length": 8.8, + "completions/mean_length": 6.35, + "completions/mean_terminated_length": 6.35, + "completions/min_length": 4.4, + "completions/min_terminated_length": 4.4, + "entropy": 0.6853881716728211, + "epoch": 0.7628865979381443, + "frac_reward_zero_std": 0.2, + "grad_norm": 53.5, + "learning_rate": 3.891262313719127e-07, + "loss": 0.11004903316497802, + "num_tokens": 189822.0, + "reward": 0.03999999761581421, + "reward_std": 0.45320507884025574, + "rewards/reward_fn/mean": 0.03999999761581421, + "rewards/reward_fn/std": 0.45320507884025574, + "step": 740, + "step_time": 0.42834897292777896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 37.6, + "completions/max_terminated_length": 37.6, + "completions/mean_length": 16.5, + "completions/mean_terminated_length": 16.5, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 0.8831792533397674, + "epoch": 0.7680412371134021, + "frac_reward_zero_std": 0.2, + "grad_norm": 20.375, + "learning_rate": 3.872519447890755e-07, + "loss": 0.2623473644256592, + "num_tokens": 191184.0, + "reward": -0.4025000035762787, + "reward_std": 0.2649999976158142, + "rewards/reward_fn/mean": -0.4025000035762787, + "rewards/reward_fn/std": 0.2649999976158142, + "step": 745, + "step_time": 1.0432677429169417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.4, + "completions/max_terminated_length": 20.4, + "completions/mean_length": 13.25, + "completions/mean_terminated_length": 13.25, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 0.7962756127119064, + "epoch": 0.7731958762886598, + "frac_reward_zero_std": 0.0, + "grad_norm": 21.875, + "learning_rate": 3.8536654941378247e-07, + "loss": 0.1507752299308777, + "num_tokens": 192497.0, + "reward": -0.5350000023841858, + "reward_std": 0.34826061725616453, + "rewards/reward_fn/mean": -0.5350000023841858, + "rewards/reward_fn/std": 0.34826061725616453, + "step": 750, + "step_time": 0.6385473405010998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 48.6, + "completions/max_terminated_length": 48.6, + "completions/mean_length": 20.95, + "completions/mean_terminated_length": 20.95, + "completions/min_length": 10.4, + "completions/min_terminated_length": 10.4, + "entropy": 1.0322799324989318, + "epoch": 0.7783505154639175, + "frac_reward_zero_std": 0.0, + "grad_norm": 48.0, + "learning_rate": 3.834701978447159e-07, + "loss": 0.3006690502166748, + "num_tokens": 193956.0, + "reward": -0.41600000858306885, + "reward_std": 0.48129103779792787, + "rewards/reward_fn/mean": -0.41600000858306885, + "rewards/reward_fn/std": 0.48129103779792787, + "step": 755, + "step_time": 1.164021307323128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 46.8, + "completions/max_terminated_length": 46.8, + "completions/mean_length": 21.65, + "completions/mean_terminated_length": 21.65, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.7505376458168029, + "epoch": 0.7835051546391752, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.6875, + "learning_rate": 3.8156304356732203e-07, + "loss": 0.18233208656311034, + "num_tokens": 195397.0, + "reward": -0.6100000023841858, + "reward_std": 0.2800000011920929, + "rewards/reward_fn/mean": -0.6100000023841858, + "rewards/reward_fn/std": 0.2800000011920929, + "step": 760, + "step_time": 1.1370311999693512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 24.6, + "completions/max_terminated_length": 24.6, + "completions/mean_length": 14.5, + "completions/mean_terminated_length": 14.5, + "completions/min_length": 9.6, + "completions/min_terminated_length": 9.6, + "entropy": 0.5627210974693299, + "epoch": 0.788659793814433, + "frac_reward_zero_std": 0.2, + "grad_norm": 9.875, + "learning_rate": 3.796452409413887e-07, + "loss": -0.004269063472747803, + "num_tokens": 196715.0, + "reward": -0.1147499993443489, + "reward_std": 0.5147351384162903, + "rewards/reward_fn/mean": -0.1147499993443489, + "rewards/reward_fn/std": 0.5147351384162903, + "step": 765, + "step_time": 0.716858146712184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.6, + "completions/max_terminated_length": 9.6, + "completions/mean_length": 6.95, + "completions/mean_terminated_length": 6.95, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "entropy": 1.1454240024089812, + "epoch": 0.7938144329896907, + "frac_reward_zero_std": 0.0, + "grad_norm": 22.375, + "learning_rate": 3.7771694518855113e-07, + "loss": 0.15136839151382447, + "num_tokens": 197870.0, + "reward": -0.09916666746139527, + "reward_std": 0.47160890102386477, + "rewards/reward_fn/mean": -0.09916666746139527, + "rewards/reward_fn/std": 0.47160890102386477, + "step": 770, + "step_time": 0.4394523199647665 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.4, + "completions/max_terminated_length": 21.4, + "completions/mean_length": 12.8, + "completions/mean_terminated_length": 12.8, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.9384787321090698, + "epoch": 0.7989690721649485, + "frac_reward_zero_std": 0.2, + "grad_norm": 25.875, + "learning_rate": 3.7577831237972965e-07, + "loss": 0.1645453691482544, + "num_tokens": 199170.0, + "reward": -0.15049999952316284, + "reward_std": 0.3982495129108429, + "rewards/reward_fn/mean": -0.15049999952316284, + "rewards/reward_fn/std": 0.3982495129108429, + "step": 775, + "step_time": 0.6595026399940253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 35.4, + "completions/max_terminated_length": 35.4, + "completions/mean_length": 17.7, + "completions/mean_terminated_length": 17.7, + "completions/min_length": 9.4, + "completions/min_terminated_length": 9.4, + "entropy": 0.908982464671135, + "epoch": 0.8041237113402062, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 3.738294994224969e-07, + "loss": 0.16901147365570068, + "num_tokens": 200548.0, + "reward": -0.4850000023841858, + "reward_std": 0.24826061725616455, + "rewards/reward_fn/mean": -0.4850000023841858, + "rewards/reward_fn/std": 0.24826061725616455, + "step": 780, + "step_time": 0.9208143584430217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.6, + "completions/max_terminated_length": 17.6, + "completions/mean_length": 14.0, + "completions/mean_terminated_length": 14.0, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.608512119948864, + "epoch": 0.8092783505154639, + "frac_reward_zero_std": 0.8, + "grad_norm": 12.375, + "learning_rate": 3.7187066404837885e-07, + "loss": 0.022144103050231935, + "num_tokens": 201868.0, + "reward": -0.3925000011920929, + "reward_std": 0.1863911747932434, + "rewards/reward_fn/mean": -0.3925000011920929, + "rewards/reward_fn/std": 0.1863911747932434, + "step": 785, + "step_time": 0.6640401553362608 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.8, + "completions/max_terminated_length": 13.8, + "completions/mean_length": 9.5, + "completions/mean_terminated_length": 9.5, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.7681404441595078, + "epoch": 0.8144329896907216, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 3.69901964800088e-07, + "loss": 0.12942266464233398, + "num_tokens": 203070.0, + "reward": -0.4525000035762787, + "reward_std": 0.32273502349853517, + "rewards/reward_fn/mean": -0.4525000035762787, + "rewards/reward_fn/std": 0.32273502349853517, + "step": 790, + "step_time": 0.5212488909251988 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.0, + "completions/max_terminated_length": 13.0, + "completions/mean_length": 10.65, + "completions/mean_terminated_length": 10.65, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.8925413608551025, + "epoch": 0.8195876288659794, + "frac_reward_zero_std": 0.2, + "grad_norm": 18.625, + "learning_rate": 3.6792356101869156e-07, + "loss": 0.12305893898010253, + "num_tokens": 204299.0, + "reward": -0.2700000047683716, + "reward_std": 0.45541470050811766, + "rewards/reward_fn/mean": -0.2700000047683716, + "rewards/reward_fn/std": 0.45541470050811766, + "step": 795, + "step_time": 0.49430387681350113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.4, + "completions/max_terminated_length": 11.4, + "completions/mean_length": 9.6, + "completions/mean_terminated_length": 9.6, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.6558867126703263, + "epoch": 0.8247422680412371, + "frac_reward_zero_std": 0.2, + "grad_norm": 17.125, + "learning_rate": 3.659356128307147e-07, + "loss": 0.08286584615707397, + "num_tokens": 205515.0, + "reward": -0.2750000059604645, + "reward_std": 0.2779629558324814, + "rewards/reward_fn/mean": -0.2750000059604645, + "rewards/reward_fn/std": 0.2779629558324814, + "step": 800, + "step_time": 0.4695729834027588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.0, + "completions/max_terminated_length": 13.0, + "completions/mean_length": 10.65, + "completions/mean_terminated_length": 10.65, + "completions/min_length": 8.4, + "completions/min_terminated_length": 8.4, + "entropy": 0.6253852218389511, + "epoch": 0.8298969072164949, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 3.639382811351806e-07, + "loss": 0.06376473903656006, + "num_tokens": 206756.0, + "reward": -0.4025000035762787, + "reward_std": 0.29779056310653684, + "rewards/reward_fn/mean": -0.4025000035762787, + "rewards/reward_fn/std": 0.29779056310653684, + "step": 805, + "step_time": 0.5086913261562586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.4, + "completions/max_terminated_length": 12.4, + "completions/mean_length": 8.75, + "completions/mean_terminated_length": 8.75, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 1.1803167939186097, + "epoch": 0.8350515463917526, + "frac_reward_zero_std": 0.2, + "grad_norm": 21.5, + "learning_rate": 3.619317275905874e-07, + "loss": 0.040542510151863095, + "num_tokens": 207911.0, + "reward": -0.20699999630451202, + "reward_std": 0.3604273974895477, + "rewards/reward_fn/mean": -0.20699999630451202, + "rewards/reward_fn/std": 0.36042741537094114, + "step": 810, + "step_time": 0.4909832508303225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.0, + "completions/max_terminated_length": 16.0, + "completions/mean_length": 12.6, + "completions/mean_terminated_length": 12.6, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "entropy": 0.9079984992742538, + "epoch": 0.8402061855670103, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 3.5991611460182426e-07, + "loss": 0.014759808778762817, + "num_tokens": 209179.0, + "reward": -0.4949999988079071, + "reward_std": 0.20011106729507447, + "rewards/reward_fn/mean": -0.4949999988079071, + "rewards/reward_fn/std": 0.20011106729507447, + "step": 815, + "step_time": 0.5565207180567086 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 28.0, + "completions/max_terminated_length": 28.0, + "completions/mean_length": 14.85, + "completions/mean_terminated_length": 14.85, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.9968389987945556, + "epoch": 0.845360824742268, + "frac_reward_zero_std": 0.4, + "grad_norm": 13.1875, + "learning_rate": 3.578916053070267e-07, + "loss": 0.18342533111572265, + "num_tokens": 210516.0, + "reward": -0.725, + "reward_std": 0.16547005176544188, + "rewards/reward_fn/mean": -0.725, + "rewards/reward_fn/std": 0.16547005176544188, + "step": 820, + "step_time": 0.7959272848442197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 26.8, + "completions/max_terminated_length": 26.8, + "completions/mean_length": 13.5, + "completions/mean_terminated_length": 13.5, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.8428839266300201, + "epoch": 0.8505154639175257, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 3.558583635643726e-07, + "loss": 0.12303779125213624, + "num_tokens": 211806.0, + "reward": -0.3550000011920929, + "reward_std": 0.3120825529098511, + "rewards/reward_fn/mean": -0.3550000011920929, + "rewards/reward_fn/std": 0.3120825529098511, + "step": 825, + "step_time": 0.8458523162640631 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 44.6, + "completions/max_terminated_length": 44.6, + "completions/mean_length": 19.8, + "completions/mean_terminated_length": 19.8, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.9209296464920044, + "epoch": 0.8556701030927835, + "frac_reward_zero_std": 0.2, + "grad_norm": 13.9375, + "learning_rate": 3.5381655393881984e-07, + "loss": 0.16743425130844117, + "num_tokens": 213238.0, + "reward": -0.6175000011920929, + "reward_std": 0.2975861310958862, + "rewards/reward_fn/mean": -0.6175000011920929, + "rewards/reward_fn/std": 0.2975861310958862, + "step": 830, + "step_time": 1.088095944840461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05, + "completions/max_length": 66.0, + "completions/max_terminated_length": 18.4, + "completions/mean_length": 26.3, + "completions/mean_terminated_length": 14.216666793823242, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "entropy": 0.6402861684560776, + "epoch": 0.8608247422680413, + "frac_reward_zero_std": 0.2, + "grad_norm": 15.0625, + "learning_rate": 3.5176634168878723e-07, + "loss": 0.15643105506896973, + "num_tokens": 214812.0, + "reward": -0.56700000166893, + "reward_std": 0.31664379835128786, + "rewards/reward_fn/mean": -0.56700000166893, + "rewards/reward_fn/std": 0.31664379835128786, + "step": 835, + "step_time": 1.4822699318639934 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.4, + "completions/max_terminated_length": 14.4, + "completions/mean_length": 10.25, + "completions/mean_terminated_length": 10.25, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.566778889298439, + "epoch": 0.865979381443299, + "frac_reward_zero_std": 0.0, + "grad_norm": 19.125, + "learning_rate": 3.497078927527787e-07, + "loss": 0.13858178853988648, + "num_tokens": 216065.0, + "reward": -0.5950000047683716, + "reward_std": 0.3880298137664795, + "rewards/reward_fn/mean": -0.5950000047683716, + "rewards/reward_fn/std": 0.3880298137664795, + "step": 840, + "step_time": 0.5335512371733785 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.0, + "completions/max_terminated_length": 12.0, + "completions/mean_length": 8.95, + "completions/mean_terminated_length": 8.95, + "completions/min_length": 4.8, + "completions/min_terminated_length": 4.8, + "entropy": 0.7644638597965241, + "epoch": 0.8711340206185567, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 3.476413737359527e-07, + "loss": 0.08275651931762695, + "num_tokens": 217268.0, + "reward": -0.3950000047683716, + "reward_std": 0.29033899307250977, + "rewards/reward_fn/mean": -0.3950000047683716, + "rewards/reward_fn/std": 0.29033900499343873, + "step": 845, + "step_time": 0.48589533697813747 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.8, + "completions/max_terminated_length": 15.8, + "completions/mean_length": 9.95, + "completions/mean_terminated_length": 9.95, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 1.178800243139267, + "epoch": 0.8762886597938144, + "frac_reward_zero_std": 0.4, + "grad_norm": 19.375, + "learning_rate": 3.455669518966378e-07, + "loss": 0.06842224597930908, + "num_tokens": 218483.0, + "reward": -0.4350000023841858, + "reward_std": 0.3048511028289795, + "rewards/reward_fn/mean": -0.4350000023841858, + "rewards/reward_fn/std": 0.3048511028289795, + "step": 850, + "step_time": 0.5550074013881385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 32.4, + "completions/max_terminated_length": 32.4, + "completions/mean_length": 16.55, + "completions/mean_terminated_length": 16.55, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 1.0851195693016051, + "epoch": 0.8814432989690721, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 3.4348479513279484e-07, + "loss": 0.21348931789398193, + "num_tokens": 219874.0, + "reward": -0.5600000023841858, + "reward_std": 0.25754843950271605, + "rewards/reward_fn/mean": -0.5600000023841858, + "rewards/reward_fn/std": 0.25754843950271605, + "step": 855, + "step_time": 0.8693027082830668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 58.0, + "completions/max_terminated_length": 58.0, + "completions/mean_length": 29.15, + "completions/mean_terminated_length": 29.15, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.9453159362077713, + "epoch": 0.8865979381443299, + "frac_reward_zero_std": 0.4, + "grad_norm": 12.75, + "learning_rate": 3.4139507196842837e-07, + "loss": 0.165838360786438, + "num_tokens": 221461.0, + "reward": -0.5233333319425583, + "reward_std": 0.2110683560371399, + "rewards/reward_fn/mean": -0.5233333319425583, + "rewards/reward_fn/std": 0.21106836795806885, + "step": 860, + "step_time": 1.4117650521919132 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.8, + "completions/max_terminated_length": 16.8, + "completions/mean_length": 10.7, + "completions/mean_terminated_length": 10.7, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 1.284914356470108, + "epoch": 0.8917525773195877, + "frac_reward_zero_std": 0.2, + "grad_norm": 29.875, + "learning_rate": 3.3929795153994624e-07, + "loss": 0.030526399612426758, + "num_tokens": 222691.0, + "reward": -0.23999999165534974, + "reward_std": 0.40157958269119265, + "rewards/reward_fn/mean": -0.23999999165534974, + "rewards/reward_fn/std": 0.4015796065330505, + "step": 865, + "step_time": 0.5726813119836152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.2, + "completions/max_terminated_length": 21.2, + "completions/mean_length": 11.15, + "completions/mean_terminated_length": 11.15, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 1.0081340759992599, + "epoch": 0.8969072164948454, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 3.3719360358247053e-07, + "loss": 0.07258962392807007, + "num_tokens": 223930.0, + "reward": -0.575, + "reward_std": 0.10773502588272095, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.10773502588272095, + "step": 870, + "step_time": 0.6567351558245719 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.0, + "completions/max_terminated_length": 21.0, + "completions/mean_length": 13.0, + "completions/mean_terminated_length": 13.0, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.9066625654697418, + "epoch": 0.9020618556701031, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 3.350821984160994e-07, + "loss": 0.12728797197341918, + "num_tokens": 225222.0, + "reward": -0.425, + "reward_std": 0.2809401035308838, + "rewards/reward_fn/mean": -0.425, + "rewards/reward_fn/std": 0.2809401035308838, + "step": 875, + "step_time": 0.6553768229670822 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.0, + "completions/max_terminated_length": 12.0, + "completions/mean_length": 9.4, + "completions/mean_terminated_length": 9.4, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.8609405964612961, + "epoch": 0.9072164948453608, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.6875, + "learning_rate": 3.329639069321221e-07, + "loss": 0.1672358274459839, + "num_tokens": 226454.0, + "reward": -0.36833333373069765, + "reward_std": 0.5804607629776001, + "rewards/reward_fn/mean": -0.36833333373069765, + "rewards/reward_fn/std": 0.5804607629776001, + "step": 880, + "step_time": 0.48266850765794517 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.6, + "completions/max_terminated_length": 17.6, + "completions/mean_length": 10.6, + "completions/mean_terminated_length": 10.6, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 1.1898846477270126, + "epoch": 0.9123711340206185, + "frac_reward_zero_std": 0.4, + "grad_norm": 25.0, + "learning_rate": 3.308389005791871e-07, + "loss": 0.12706210613250732, + "num_tokens": 227674.0, + "reward": -0.6, + "reward_std": 0.15773502588272095, + "rewards/reward_fn/mean": -0.6, + "rewards/reward_fn/std": 0.15773502588272095, + "step": 885, + "step_time": 0.5870971732772887 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.2, + "completions/max_terminated_length": 11.2, + "completions/mean_length": 8.5, + "completions/mean_terminated_length": 8.5, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 1.0536468535661698, + "epoch": 0.9175257731958762, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 3.287073513494261e-07, + "loss": 0.0786515474319458, + "num_tokens": 228868.0, + "reward": -0.4025000035762787, + "reward_std": 0.22273502349853516, + "rewards/reward_fn/mean": -0.4025000035762787, + "rewards/reward_fn/std": 0.22273502349853516, + "step": 890, + "step_time": 0.46912989038974046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 22.0, + "completions/max_terminated_length": 22.0, + "completions/mean_length": 12.05, + "completions/mean_terminated_length": 12.05, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 0.8401788771152496, + "epoch": 0.9226804123711341, + "frac_reward_zero_std": 0.4, + "grad_norm": 21.125, + "learning_rate": 3.265694317645328e-07, + "loss": 0.06491990685462952, + "num_tokens": 230125.0, + "reward": -0.4433333337306976, + "reward_std": 0.37993608713150023, + "rewards/reward_fn/mean": -0.4433333337306976, + "rewards/reward_fn/std": 0.37993608713150023, + "step": 895, + "step_time": 0.730783850606531 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 44.4, + "completions/max_terminated_length": 44.4, + "completions/mean_length": 22.75, + "completions/mean_terminated_length": 22.75, + "completions/min_length": 12.4, + "completions/min_terminated_length": 12.4, + "entropy": 0.8007380843162537, + "epoch": 0.9278350515463918, + "frac_reward_zero_std": 0.4, + "grad_norm": 11.875, + "learning_rate": 3.244253148618001e-07, + "loss": 0.05340578556060791, + "num_tokens": 231620.0, + "reward": -0.4360606074333191, + "reward_std": 0.30347265005111695, + "rewards/reward_fn/mean": -0.4360606074333191, + "rewards/reward_fn/std": 0.3034726619720459, + "step": 900, + "step_time": 1.0834071793593467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05, + "completions/max_length": 67.4, + "completions/max_terminated_length": 21.6, + "completions/mean_length": 24.3, + "completions/mean_terminated_length": 12.25, + "completions/min_length": 5.4, + "completions/min_terminated_length": 5.4, + "entropy": 0.9434569358825684, + "epoch": 0.9329896907216495, + "frac_reward_zero_std": 0.4, + "grad_norm": 10.3125, + "learning_rate": 3.222751741801146e-07, + "loss": 0.07329256534576416, + "num_tokens": 233102.0, + "reward": -0.5100000023841857, + "reward_std": 0.1800000011920929, + "rewards/reward_fn/mean": -0.5100000023841857, + "rewards/reward_fn/std": 0.1800000011920929, + "step": 905, + "step_time": 1.5177901348099112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.6, + "completions/max_terminated_length": 15.6, + "completions/mean_length": 12.05, + "completions/mean_terminated_length": 12.05, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.8976251482963562, + "epoch": 0.9381443298969072, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 3.201191837459107e-07, + "loss": 0.02294694781303406, + "num_tokens": 234375.0, + "reward": -0.4889999985694885, + "reward_std": 0.1797350287437439, + "rewards/reward_fn/mean": -0.4889999985694885, + "rewards/reward_fn/std": 0.17973504066467286, + "step": 910, + "step_time": 0.5585914126597344 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 24.4, + "completions/max_terminated_length": 24.4, + "completions/mean_length": 14.5, + "completions/mean_terminated_length": 14.5, + "completions/min_length": 7.4, + "completions/min_terminated_length": 7.4, + "entropy": 1.120745688676834, + "epoch": 0.9432989690721649, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 3.179575180590857e-07, + "loss": -0.041665342450141904, + "num_tokens": 235701.0, + "reward": -0.3719090908765793, + "reward_std": 0.26021685600280764, + "rewards/reward_fn/mean": -0.3719090908765793, + "rewards/reward_fn/std": 0.26021685600280764, + "step": 915, + "step_time": 0.7208410411141812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 31.2, + "completions/max_terminated_length": 31.2, + "completions/mean_length": 18.15, + "completions/mean_terminated_length": 18.15, + "completions/min_length": 10.6, + "completions/min_terminated_length": 10.6, + "entropy": 0.7649198234081268, + "epoch": 0.9484536082474226, + "frac_reward_zero_std": 0.4, + "grad_norm": 15.9375, + "learning_rate": 3.1579035207887624e-07, + "loss": 0.09516732692718506, + "num_tokens": 237104.0, + "reward": -0.446000000834465, + "reward_std": 0.33894009590148927, + "rewards/reward_fn/mean": -0.446000000834465, + "rewards/reward_fn/std": 0.33894009590148927, + "step": 920, + "step_time": 0.8414601643569768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 24.2, + "completions/max_terminated_length": 24.2, + "completions/mean_length": 12.7, + "completions/mean_terminated_length": 12.7, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "entropy": 0.7877166599035264, + "epoch": 0.9536082474226805, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 3.136178612096973e-07, + "loss": 0.04098041653633118, + "num_tokens": 238386.0, + "reward": -0.3866666674613953, + "reward_std": 0.21086571216583253, + "rewards/reward_fn/mean": -0.3866666674613953, + "rewards/reward_fn/std": 0.21086571216583253, + "step": 925, + "step_time": 0.712461163289845 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.2, + "completions/max_terminated_length": 17.2, + "completions/mean_length": 12.55, + "completions/mean_terminated_length": 12.55, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.8109033226966857, + "epoch": 0.9587628865979382, + "frac_reward_zero_std": 0.0, + "grad_norm": 15.25, + "learning_rate": 3.114402212869458e-07, + "loss": 0.03635913133621216, + "num_tokens": 239649.0, + "reward": -0.30333333015441893, + "reward_std": 0.5505508422851563, + "rewards/reward_fn/mean": -0.30333333015441893, + "rewards/reward_fn/std": 0.5505508542060852, + "step": 930, + "step_time": 0.585801807884127 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.0, + "completions/max_terminated_length": 12.0, + "completions/mean_length": 9.4, + "completions/mean_terminated_length": 9.4, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.7167331606149674, + "epoch": 0.9639175257731959, + "frac_reward_zero_std": 0.6, + "grad_norm": 21.5, + "learning_rate": 3.092576085627686e-07, + "loss": 0.04640328586101532, + "num_tokens": 240881.0, + "reward": -0.4983333319425583, + "reward_std": 0.1610683560371399, + "rewards/reward_fn/mean": -0.4983333319425583, + "rewards/reward_fn/std": 0.16106836795806884, + "step": 935, + "step_time": 0.4838869537226856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.4, + "completions/max_terminated_length": 14.4, + "completions/mean_length": 10.2, + "completions/mean_terminated_length": 10.2, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 0.8657872408628464, + "epoch": 0.9690721649484536, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 3.070701996917976e-07, + "loss": 0.11601418256759644, + "num_tokens": 242105.0, + "reward": -0.0800000011920929, + "reward_std": 0.5176253199577332, + "rewards/reward_fn/mean": -0.0800000011920929, + "rewards/reward_fn/std": 0.5176253080368042, + "step": 940, + "step_time": 0.5930247722193599 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.2, + "completions/max_terminated_length": 18.2, + "completions/mean_length": 13.4, + "completions/mean_terminated_length": 13.4, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.8240721940994262, + "epoch": 0.9742268041237113, + "frac_reward_zero_std": 0.0, + "grad_norm": 22.875, + "learning_rate": 3.048781717168513e-07, + "loss": -0.08106042742729187, + "num_tokens": 243389.0, + "reward": -0.2540909111499786, + "reward_std": 0.44955320954322814, + "rewards/reward_fn/mean": -0.2540909111499786, + "rewards/reward_fn/std": 0.44955320954322814, + "step": 945, + "step_time": 0.5971614887937904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 34.0, + "completions/max_terminated_length": 34.0, + "completions/mean_length": 15.0, + "completions/mean_terminated_length": 15.0, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 1.3446922928094864, + "epoch": 0.979381443298969, + "frac_reward_zero_std": 0.0, + "grad_norm": 24.125, + "learning_rate": 3.0268170205460566e-07, + "loss": -0.017026874423027038, + "num_tokens": 244697.0, + "reward": -0.3649999976158142, + "reward_std": 0.4780433356761932, + "rewards/reward_fn/mean": -0.3649999976158142, + "rewards/reward_fn/std": 0.4780433475971222, + "step": 950, + "step_time": 0.8890752191655338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.4, + "completions/max_terminated_length": 13.4, + "completions/mean_length": 8.5, + "completions/mean_terminated_length": 8.5, + "completions/min_length": 5.4, + "completions/min_terminated_length": 5.4, + "entropy": 1.2739114731550216, + "epoch": 0.9845360824742269, + "frac_reward_zero_std": 0.2, + "grad_norm": 14.9375, + "learning_rate": 3.004809684812349e-07, + "loss": 0.057057321071624756, + "num_tokens": 245879.0, + "reward": -0.4950000047683716, + "reward_std": 0.26773502826690676, + "rewards/reward_fn/mean": -0.4950000047683716, + "rewards/reward_fn/std": 0.26773502826690676, + "step": 955, + "step_time": 0.5191133976913989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 26.2, + "completions/max_terminated_length": 26.2, + "completions/mean_length": 17.7, + "completions/mean_terminated_length": 17.7, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "entropy": 1.1563934952020645, + "epoch": 0.9896907216494846, + "frac_reward_zero_std": 0.2, + "grad_norm": 20.125, + "learning_rate": 2.98276149118022e-07, + "loss": 0.07603020668029785, + "num_tokens": 247273.0, + "reward": -0.5050000011920929, + "reward_std": 0.34321573972702024, + "rewards/reward_fn/mean": -0.5050000011920929, + "rewards/reward_fn/std": 0.34321573972702024, + "step": 960, + "step_time": 0.764554787799716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.0, + "completions/max_terminated_length": 12.0, + "completions/mean_length": 8.35, + "completions/mean_terminated_length": 8.35, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 1.2172771751880647, + "epoch": 0.9948453608247423, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 2.9606742241694265e-07, + "loss": 0.03891005516052246, + "num_tokens": 248456.0, + "reward": -0.4550000011920929, + "reward_std": 0.2651223957538605, + "rewards/reward_fn/mean": -0.4550000011920929, + "rewards/reward_fn/std": 0.2651223957538605, + "step": 965, + "step_time": 0.4914631086401641 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10.2, + "completions/max_terminated_length": 10.2, + "completions/mean_length": 7.9, + "completions/mean_terminated_length": 7.9, + "completions/min_length": 5.4, + "completions/min_terminated_length": 5.4, + "entropy": 1.2364417910575867, + "epoch": 1.0, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 2.938549671462216e-07, + "loss": 0.016531860828399657, + "num_tokens": 249638.0, + "reward": -0.3433333277702332, + "reward_std": 0.1, + "rewards/reward_fn/mean": -0.3433333277702332, + "rewards/reward_fn/std": 0.1, + "step": 970, + "step_time": 0.4553267262876034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.2, + "completions/max_terminated_length": 20.2, + "completions/mean_length": 14.65, + "completions/mean_terminated_length": 14.65, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "entropy": 0.8596020042896271, + "epoch": 1.0051546391752577, + "frac_reward_zero_std": 0.2, + "grad_norm": 8.6875, + "learning_rate": 2.916389623758636e-07, + "loss": 0.11736297607421875, + "num_tokens": 250955.0, + "reward": -0.22090909481048585, + "reward_std": 0.3392897605895996, + "rewards/reward_fn/mean": -0.22090909481048585, + "rewards/reward_fn/std": 0.3392897605895996, + "step": 975, + "step_time": 0.6459065237082541 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.2, + "completions/max_terminated_length": 11.2, + "completions/mean_length": 8.15, + "completions/mean_terminated_length": 8.15, + "completions/min_length": 5.2, + "completions/min_terminated_length": 5.2, + "entropy": 0.8980956047773361, + "epoch": 1.0103092783505154, + "frac_reward_zero_std": 0.4, + "grad_norm": 39.0, + "learning_rate": 2.8941958746316036e-07, + "loss": 0.032901537418365476, + "num_tokens": 252130.0, + "reward": -0.24749999940395356, + "reward_std": 0.3519443690776825, + "rewards/reward_fn/mean": -0.24749999940395356, + "rewards/reward_fn/std": 0.3519443690776825, + "step": 980, + "step_time": 0.47193123353645205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.8, + "completions/max_terminated_length": 11.8, + "completions/mean_length": 9.4, + "completions/mean_terminated_length": 9.4, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.6604278087615967, + "epoch": 1.0154639175257731, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 2.871970220381733e-07, + "loss": 0.14088039398193358, + "num_tokens": 253358.0, + "reward": -0.4949999988079071, + "reward_std": 0.42496423721313475, + "rewards/reward_fn/mean": -0.4949999988079071, + "rewards/reward_fn/std": 0.4249642491340637, + "step": 985, + "step_time": 0.49146103104576466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.6, + "completions/max_terminated_length": 14.6, + "completions/mean_length": 9.5, + "completions/mean_terminated_length": 9.5, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.7506533205509186, + "epoch": 1.0206185567010309, + "frac_reward_zero_std": 0.0, + "grad_norm": 34.75, + "learning_rate": 2.8497144598919524e-07, + "loss": 0.2226261615753174, + "num_tokens": 254556.0, + "reward": -0.4300000071525574, + "reward_std": 0.4620167911052704, + "rewards/reward_fn/mean": -0.4300000071525574, + "rewards/reward_fn/std": 0.4620167911052704, + "step": 990, + "step_time": 0.5410918489098548 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.4, + "completions/max_terminated_length": 14.4, + "completions/mean_length": 10.2, + "completions/mean_terminated_length": 10.2, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 1.0815811634063721, + "epoch": 1.0257731958762886, + "frac_reward_zero_std": 0.4, + "grad_norm": 11.0625, + "learning_rate": 2.8274303944819045e-07, + "loss": 0.11785383224487304, + "num_tokens": 255784.0, + "reward": -0.19799999594688417, + "reward_std": 0.3982208132743835, + "rewards/reward_fn/mean": -0.19799999594688417, + "rewards/reward_fn/std": 0.3982208132743835, + "step": 995, + "step_time": 0.5314617343246937 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.8, + "completions/max_terminated_length": 21.8, + "completions/mean_length": 13.55, + "completions/mean_terminated_length": 13.55, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "entropy": 0.7796996653079986, + "epoch": 1.0309278350515463, + "frac_reward_zero_std": 0.6, + "grad_norm": 35.5, + "learning_rate": 2.8051198277621544e-07, + "loss": 0.11956088542938233, + "num_tokens": 257067.0, + "reward": -0.4350000023841858, + "reward_std": 0.21673332452774047, + "rewards/reward_fn/mean": -0.4350000023841858, + "rewards/reward_fn/std": 0.21673332452774047, + "step": 1000, + "step_time": 0.6741534044034779 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 27.6, + "completions/max_terminated_length": 27.6, + "completions/mean_length": 13.65, + "completions/mean_terminated_length": 13.65, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 1.1909677296876908, + "epoch": 1.0360824742268042, + "frac_reward_zero_std": 0.2, + "grad_norm": 21.75, + "learning_rate": 2.782784565488211e-07, + "loss": 0.22131562232971191, + "num_tokens": 258368.0, + "reward": -0.26075000017881395, + "reward_std": 0.4857735872268677, + "rewards/reward_fn/mean": -0.26075000017881395, + "rewards/reward_fn/std": 0.48577356338500977, + "step": 1005, + "step_time": 0.8473773039877415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 44.2, + "completions/max_terminated_length": 44.2, + "completions/mean_length": 23.95, + "completions/mean_terminated_length": 23.95, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 0.8882090777158738, + "epoch": 1.041237113402062, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 2.760426415414371e-07, + "loss": 0.13278703689575194, + "num_tokens": 259863.0, + "reward": -0.6425000011920929, + "reward_std": 0.2727350234985352, + "rewards/reward_fn/mean": -0.6425000011920929, + "rewards/reward_fn/std": 0.2727350234985352, + "step": 1010, + "step_time": 1.0934121519327165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 26.8, + "completions/max_terminated_length": 26.8, + "completions/mean_length": 15.75, + "completions/mean_terminated_length": 15.75, + "completions/min_length": 10.2, + "completions/min_terminated_length": 10.2, + "entropy": 0.9788724243640899, + "epoch": 1.0463917525773196, + "frac_reward_zero_std": 0.2, + "grad_norm": 12.8125, + "learning_rate": 2.738047187147406e-07, + "loss": 0.035887351632118224, + "num_tokens": 261214.0, + "reward": -0.4050000011920929, + "reward_std": 0.20241495370864868, + "rewards/reward_fn/mean": -0.4050000011920929, + "rewards/reward_fn/std": 0.20241495370864868, + "step": 1015, + "step_time": 0.7664700558409094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.2, + "completions/max_terminated_length": 21.2, + "completions/mean_length": 13.0, + "completions/mean_terminated_length": 13.0, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.7931116044521331, + "epoch": 1.0515463917525774, + "frac_reward_zero_std": 0.2, + "grad_norm": 15.0, + "learning_rate": 2.715648692000102e-07, + "loss": 0.1386839747428894, + "num_tokens": 262506.0, + "reward": -0.4069999992847443, + "reward_std": 0.4169401049613953, + "rewards/reward_fn/mean": -0.4069999992847443, + "rewards/reward_fn/std": 0.4169400930404663, + "step": 1020, + "step_time": 0.6620019136928021 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 38.0, + "completions/max_terminated_length": 38.0, + "completions/mean_length": 19.25, + "completions/mean_terminated_length": 19.25, + "completions/min_length": 9.8, + "completions/min_terminated_length": 9.8, + "entropy": 1.357260411977768, + "epoch": 1.056701030927835, + "frac_reward_zero_std": 0.4, + "grad_norm": 9.75, + "learning_rate": 2.693232742844649e-07, + "loss": 0.09496567249298096, + "num_tokens": 263963.0, + "reward": -0.5210000038146972, + "reward_std": 0.2332816481590271, + "rewards/reward_fn/mean": -0.5210000038146972, + "rewards/reward_fn/std": 0.23328166007995604, + "step": 1025, + "step_time": 0.9738308310508728 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 22.8, + "completions/max_terminated_length": 22.8, + "completions/mean_length": 13.2, + "completions/mean_terminated_length": 13.2, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "entropy": 0.9426976352930069, + "epoch": 1.0618556701030928, + "frac_reward_zero_std": 0.4, + "grad_norm": 16.0, + "learning_rate": 2.6708011539659165e-07, + "loss": 0.1510265588760376, + "num_tokens": 265263.0, + "reward": -0.575, + "reward_std": 0.15, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.15, + "step": 1030, + "step_time": 0.6886733044870198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.8, + "completions/max_terminated_length": 15.8, + "completions/mean_length": 11.3, + "completions/mean_terminated_length": 11.3, + "completions/min_length": 8.8, + "completions/min_terminated_length": 8.8, + "entropy": 0.8660927087068557, + "epoch": 1.0670103092783505, + "frac_reward_zero_std": 0.2, + "grad_norm": 42.75, + "learning_rate": 2.648355740914613e-07, + "loss": 0.15204911231994628, + "num_tokens": 266541.0, + "reward": -0.5639999985694886, + "reward_std": 0.3041954874992371, + "rewards/reward_fn/mean": -0.5639999985694886, + "rewards/reward_fn/std": 0.3041954874992371, + "step": 1035, + "step_time": 0.6211226489394903 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.4, + "completions/max_terminated_length": 16.4, + "completions/mean_length": 9.85, + "completions/mean_terminated_length": 9.85, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 1.0684709191322326, + "epoch": 1.0721649484536082, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 2.6258983203603336e-07, + "loss": 0.16374725103378296, + "num_tokens": 267758.0, + "reward": -0.28499999940395354, + "reward_std": 0.44684439301490786, + "rewards/reward_fn/mean": -0.28499999940395354, + "rewards/reward_fn/std": 0.44684439301490786, + "step": 1040, + "step_time": 0.5705590627156198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.8, + "completions/max_terminated_length": 17.8, + "completions/mean_length": 12.65, + "completions/mean_terminated_length": 12.65, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.8728094905614853, + "epoch": 1.077319587628866, + "frac_reward_zero_std": 0.0, + "grad_norm": 14.125, + "learning_rate": 2.603430709944529e-07, + "loss": 0.08976652026176453, + "num_tokens": 269051.0, + "reward": -0.2824999988079071, + "reward_std": 0.3431844413280487, + "rewards/reward_fn/mean": -0.2824999988079071, + "rewards/reward_fn/std": 0.3431844413280487, + "step": 1045, + "step_time": 0.600045460741967 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.8, + "completions/max_terminated_length": 13.8, + "completions/mean_length": 10.1, + "completions/mean_terminated_length": 10.1, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 0.7100171774625779, + "epoch": 1.0824742268041236, + "frac_reward_zero_std": 0.2, + "grad_norm": 15.5, + "learning_rate": 2.58095472813339e-07, + "loss": 0.04074999690055847, + "num_tokens": 270285.0, + "reward": -0.4716666668653488, + "reward_std": 0.332260525226593, + "rewards/reward_fn/mean": -0.4716666668653488, + "rewards/reward_fn/std": 0.33226053714752196, + "step": 1050, + "step_time": 0.5243586430326104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 40.2, + "completions/max_terminated_length": 40.2, + "completions/mean_length": 15.9, + "completions/mean_terminated_length": 15.9, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 1.0960396260023118, + "epoch": 1.0876288659793814, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 2.558472194070662e-07, + "loss": 0.06408804059028625, + "num_tokens": 271615.0, + "reward": -0.4333333373069763, + "reward_std": 0.20892719626426698, + "rewards/reward_fn/mean": -0.4333333373069763, + "rewards/reward_fn/std": 0.20892720818519592, + "step": 1055, + "step_time": 1.026824015751481 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 25.6, + "completions/max_terminated_length": 25.6, + "completions/mean_length": 15.3, + "completions/mean_terminated_length": 15.3, + "completions/min_length": 9.2, + "completions/min_terminated_length": 9.2, + "entropy": 1.1162247240543366, + "epoch": 1.0927835051546393, + "frac_reward_zero_std": 0.0, + "grad_norm": 12.375, + "learning_rate": 2.5359849274304163e-07, + "loss": -0.01492423415184021, + "num_tokens": 272933.0, + "reward": -0.7, + "reward_std": 0.2732050776481628, + "rewards/reward_fn/mean": -0.7, + "rewards/reward_fn/std": 0.2732050776481628, + "step": 1060, + "step_time": 0.7419044939801097 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10.2, + "completions/max_terminated_length": 10.2, + "completions/mean_length": 6.65, + "completions/mean_terminated_length": 6.65, + "completions/min_length": 3.8, + "completions/min_terminated_length": 3.8, + "entropy": 1.1226806700229646, + "epoch": 1.097938144329897, + "frac_reward_zero_std": 0.4, + "grad_norm": 31.5, + "learning_rate": 2.513494748269761e-07, + "loss": 0.20585639476776124, + "num_tokens": 274062.0, + "reward": -0.3850000023841858, + "reward_std": 0.3784202814102173, + "rewards/reward_fn/mean": -0.3850000023841858, + "rewards/reward_fn/std": 0.3784202814102173, + "step": 1065, + "step_time": 0.4867300412617624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.8, + "completions/max_terminated_length": 11.8, + "completions/mean_length": 9.05, + "completions/mean_terminated_length": 9.05, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.6160762578248977, + "epoch": 1.1030927835051547, + "frac_reward_zero_std": 0.2, + "grad_norm": 12.625, + "learning_rate": 2.4910034768815383e-07, + "loss": 0.1410534143447876, + "num_tokens": 275279.0, + "reward": -0.5408333361148834, + "reward_std": 0.38424835205078123, + "rewards/reward_fn/mean": -0.5408333361148834, + "rewards/reward_fn/std": 0.3842483639717102, + "step": 1070, + "step_time": 0.48538791984319685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.2, + "completions/max_terminated_length": 13.2, + "completions/mean_length": 8.5, + "completions/mean_terminated_length": 8.5, + "completions/min_length": 5.2, + "completions/min_terminated_length": 5.2, + "entropy": 1.0447974920272827, + "epoch": 1.1082474226804124, + "frac_reward_zero_std": 0.4, + "grad_norm": 22.0, + "learning_rate": 2.468512933646992e-07, + "loss": 0.14037505388259888, + "num_tokens": 276461.0, + "reward": -0.4675000011920929, + "reward_std": 0.36337884664535525, + "rewards/reward_fn/mean": -0.4675000011920929, + "rewards/reward_fn/std": 0.36337885856628416, + "step": 1075, + "step_time": 0.5826695743016899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.6, + "completions/max_terminated_length": 17.6, + "completions/mean_length": 11.85, + "completions/mean_terminated_length": 11.85, + "completions/min_length": 7.4, + "completions/min_terminated_length": 7.4, + "entropy": 0.7518991678953171, + "epoch": 1.1134020618556701, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.5, + "learning_rate": 2.4460249388884307e-07, + "loss": 0.039956438541412356, + "num_tokens": 277722.0, + "reward": -0.4816666662693024, + "reward_std": 0.3914422929286957, + "rewards/reward_fn/mean": -0.4816666662693024, + "rewards/reward_fn/std": 0.3914422929286957, + "step": 1080, + "step_time": 0.5958723069168628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.8, + "completions/max_terminated_length": 16.8, + "completions/mean_length": 9.85, + "completions/mean_terminated_length": 9.85, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 0.8753577500581742, + "epoch": 1.1185567010309279, + "frac_reward_zero_std": 0.8, + "grad_norm": 41.25, + "learning_rate": 2.423541312721896e-07, + "loss": 0.018834894895553587, + "num_tokens": 278939.0, + "reward": -0.34500000476837156, + "reward_std": 0.10847426652908325, + "rewards/reward_fn/mean": -0.34500000476837156, + "rewards/reward_fn/std": 0.10847426652908325, + "step": 1085, + "step_time": 0.5817077334038914 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.4, + "completions/max_terminated_length": 18.4, + "completions/mean_length": 13.3, + "completions/mean_terminated_length": 13.3, + "completions/min_length": 9.4, + "completions/min_terminated_length": 9.4, + "entropy": 0.985750263929367, + "epoch": 1.1237113402061856, + "frac_reward_zero_std": 0.0, + "grad_norm": 15.75, + "learning_rate": 2.401063874909848e-07, + "loss": 0.17092158794403076, + "num_tokens": 280241.0, + "reward": -0.5675000011920929, + "reward_std": 0.32273502349853517, + "rewards/reward_fn/mean": -0.5675000011920929, + "rewards/reward_fn/std": 0.32273502349853517, + "step": 1090, + "step_time": 0.604114449955523 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.0, + "completions/max_terminated_length": 9.0, + "completions/mean_length": 7.1, + "completions/mean_terminated_length": 7.1, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 0.6755490630865097, + "epoch": 1.1288659793814433, + "frac_reward_zero_std": 0.2, + "grad_norm": 20.375, + "learning_rate": 2.37859444471388e-07, + "loss": 0.07062238454818726, + "num_tokens": 281391.0, + "reward": -0.2740000009536743, + "reward_std": 0.4252050817012787, + "rewards/reward_fn/mean": -0.2740000009536743, + "rewards/reward_fn/std": 0.42520509362220765, + "step": 1095, + "step_time": 0.43289085691794754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 38.6, + "completions/max_terminated_length": 38.6, + "completions/mean_length": 18.3, + "completions/mean_terminated_length": 18.3, + "completions/min_length": 8.8, + "completions/min_terminated_length": 8.8, + "entropy": 0.9542642265558243, + "epoch": 1.134020618556701, + "frac_reward_zero_std": 0.0, + "grad_norm": 38.5, + "learning_rate": 2.3561348407474678e-07, + "loss": 0.18230464458465576, + "num_tokens": 282765.0, + "reward": -0.5875, + "reward_std": 0.30516021251678466, + "rewards/reward_fn/mean": -0.5875, + "rewards/reward_fn/std": 0.30516021251678466, + "step": 1100, + "step_time": 0.9872395430691541 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.0, + "completions/max_terminated_length": 20.0, + "completions/mean_length": 13.65, + "completions/mean_terminated_length": 13.65, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 1.1501558601856232, + "epoch": 1.1391752577319587, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 2.333686880828784e-07, + "loss": 0.01851527690887451, + "num_tokens": 284058.0, + "reward": -0.08750000596046448, + "reward_std": 0.23499999642372132, + "rewards/reward_fn/mean": -0.08750000596046448, + "rewards/reward_fn/std": 0.23499999642372132, + "step": 1105, + "step_time": 0.6418257012963295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 60.2, + "completions/max_terminated_length": 60.2, + "completions/mean_length": 22.15, + "completions/mean_terminated_length": 22.15, + "completions/min_length": 8.8, + "completions/min_terminated_length": 8.8, + "entropy": 0.7380276054143906, + "epoch": 1.1443298969072164, + "frac_reward_zero_std": 0.6, + "grad_norm": 24.625, + "learning_rate": 2.3112523818335605e-07, + "loss": 0.04171156883239746, + "num_tokens": 285521.0, + "reward": -0.575, + "reward_std": 0.10773502588272095, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.10773502588272095, + "step": 1110, + "step_time": 1.4538947951979935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.2, + "completions/max_terminated_length": 14.2, + "completions/mean_length": 8.9, + "completions/mean_terminated_length": 8.9, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 0.997655501961708, + "epoch": 1.1494845360824741, + "frac_reward_zero_std": 0.2, + "grad_norm": 25.75, + "learning_rate": 2.2888331595480397e-07, + "loss": -0.0034986168146133425, + "num_tokens": 286707.0, + "reward": -0.08833332657814026, + "reward_std": 0.36946714818477633, + "rewards/reward_fn/mean": -0.08833332657814026, + "rewards/reward_fn/std": 0.36946714818477633, + "step": 1115, + "step_time": 0.5262484772130847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10.0, + "completions/max_terminated_length": 10.0, + "completions/mean_length": 7.6, + "completions/mean_terminated_length": 7.6, + "completions/min_length": 4.6, + "completions/min_terminated_length": 4.6, + "entropy": 1.0448400229215622, + "epoch": 1.1546391752577319, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 2.2664310285220112e-07, + "loss": 0.10587654113769532, + "num_tokens": 287859.0, + "reward": -0.5350000023841858, + "reward_std": 0.165283465385437, + "rewards/reward_fn/mean": -0.5350000023841858, + "rewards/reward_fn/std": 0.165283465385437, + "step": 1120, + "step_time": 0.4480899182148278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 25.6, + "completions/max_terminated_length": 25.6, + "completions/mean_length": 13.3, + "completions/mean_terminated_length": 13.3, + "completions/min_length": 7.4, + "completions/min_terminated_length": 7.4, + "entropy": 1.2160271763801576, + "epoch": 1.1597938144329896, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.2440478019219437e-07, + "loss": 0.0, + "num_tokens": 289173.0, + "reward": -0.5, + "reward_std": 0.0, + "rewards/reward_fn/mean": -0.5, + "rewards/reward_fn/std": 0.0, + "step": 1125, + "step_time": 0.7431498907506466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 22.0, + "completions/max_terminated_length": 22.0, + "completions/mean_length": 10.95, + "completions/mean_terminated_length": 10.95, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 0.8884743601083755, + "epoch": 1.1649484536082475, + "frac_reward_zero_std": 0.2, + "grad_norm": 57.5, + "learning_rate": 2.2216852913842347e-07, + "loss": 0.09710804224014283, + "num_tokens": 290424.0, + "reward": -0.3950000047683716, + "reward_std": 0.3127905666828156, + "rewards/reward_fn/mean": -0.3950000047683716, + "rewards/reward_fn/std": 0.3127905666828156, + "step": 1130, + "step_time": 0.6823917054571211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 56.6, + "completions/max_terminated_length": 56.6, + "completions/mean_length": 21.35, + "completions/mean_terminated_length": 21.35, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.8476454958319664, + "epoch": 1.1701030927835052, + "frac_reward_zero_std": 0.0, + "grad_norm": 48.5, + "learning_rate": 2.1993453068685807e-07, + "loss": 0.21475210189819335, + "num_tokens": 291855.0, + "reward": -0.11125000417232514, + "reward_std": 0.3925000071525574, + "rewards/reward_fn/mean": -0.11125000417232514, + "rewards/reward_fn/std": 0.3925000011920929, + "step": 1135, + "step_time": 1.3351986355148255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.4, + "completions/max_terminated_length": 13.4, + "completions/mean_length": 9.4, + "completions/mean_terminated_length": 9.4, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 1.01234889626503, + "epoch": 1.175257731958763, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 2.1770296565114846e-07, + "loss": -0.0250102162361145, + "num_tokens": 293075.0, + "reward": -0.3625, + "reward_std": 0.2073760449886322, + "rewards/reward_fn/mean": -0.3625, + "rewards/reward_fn/std": 0.2073760449886322, + "step": 1140, + "step_time": 0.5155650717206299 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.0, + "completions/max_terminated_length": 18.0, + "completions/mean_length": 11.3, + "completions/mean_terminated_length": 11.3, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 1.0420640349388122, + "epoch": 1.1804123711340206, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 2.1547401464799096e-07, + "loss": 0.12414703369140626, + "num_tokens": 294325.0, + "reward": -0.575, + "reward_std": 0.10773502588272095, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.10773502588272095, + "step": 1145, + "step_time": 0.602156451717019 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.8, + "completions/max_terminated_length": 15.8, + "completions/mean_length": 12.55, + "completions/mean_terminated_length": 12.55, + "completions/min_length": 9.2, + "completions/min_terminated_length": 9.2, + "entropy": 0.7143237292766571, + "epoch": 1.1855670103092784, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 2.132478580825093e-07, + "loss": 0.041160008311271666, + "num_tokens": 295608.0, + "reward": -0.525, + "reward_std": 0.05, + "rewards/reward_fn/mean": -0.525, + "rewards/reward_fn/std": 0.05, + "step": 1150, + "step_time": 0.5545094406232238 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.8, + "completions/max_terminated_length": 11.8, + "completions/mean_length": 9.8, + "completions/mean_terminated_length": 9.8, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 0.8000463306903839, + "epoch": 1.190721649484536, + "frac_reward_zero_std": 0.6, + "grad_norm": 17.375, + "learning_rate": 2.1102467613365334e-07, + "loss": -0.008892300724983215, + "num_tokens": 296856.0, + "reward": -0.3304166689515114, + "reward_std": 0.2501828670501709, + "rewards/reward_fn/mean": -0.3304166689515114, + "rewards/reward_fn/std": 0.25018285512924193, + "step": 1155, + "step_time": 0.5423662343993783 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.0, + "completions/max_terminated_length": 19.0, + "completions/mean_length": 11.5, + "completions/mean_terminated_length": 11.5, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 1.0487438976764678, + "epoch": 1.1958762886597938, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 2.0880464873961553e-07, + "loss": 0.008957856893539428, + "num_tokens": 298118.0, + "reward": -0.33583333194255827, + "reward_std": 0.39479472637176516, + "rewards/reward_fn/mean": -0.33583333194255827, + "rewards/reward_fn/std": 0.39479475021362304, + "step": 1160, + "step_time": 0.6194642005488277 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.0, + "completions/max_terminated_length": 20.0, + "completions/mean_length": 16.4, + "completions/mean_terminated_length": 16.4, + "completions/min_length": 12.2, + "completions/min_terminated_length": 12.2, + "entropy": 0.7133869394659996, + "epoch": 1.2010309278350515, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 2.065879555832674e-07, + "loss": -0.03838225901126861, + "num_tokens": 299458.0, + "reward": -0.45999999940395353, + "reward_std": 0.25230334997177123, + "rewards/reward_fn/mean": -0.45999999940395353, + "rewards/reward_fn/std": 0.25230334997177123, + "step": 1165, + "step_time": 0.6352453227154911 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 22.2, + "completions/max_terminated_length": 22.2, + "completions/mean_length": 12.05, + "completions/mean_terminated_length": 12.05, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 1.2859386861324311, + "epoch": 1.2061855670103092, + "frac_reward_zero_std": 0.4, + "grad_norm": 15.5, + "learning_rate": 2.0437477607761655e-07, + "loss": 0.16457252502441405, + "num_tokens": 300711.0, + "reward": -0.3799999952316284, + "reward_std": 0.13106836080551149, + "rewards/reward_fn/mean": -0.3799999952316284, + "rewards/reward_fn/std": 0.13106836080551149, + "step": 1170, + "step_time": 0.6807820100337267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 24.4, + "completions/max_terminated_length": 24.4, + "completions/mean_length": 14.95, + "completions/mean_terminated_length": 14.95, + "completions/min_length": 10.4, + "completions/min_terminated_length": 10.4, + "entropy": 1.0816936790943146, + "epoch": 1.211340206185567, + "frac_reward_zero_std": 0.2, + "grad_norm": 12.375, + "learning_rate": 2.021652893512854e-07, + "loss": 0.25116305351257323, + "num_tokens": 302046.0, + "reward": -0.5069999992847443, + "reward_std": 0.28600000143051146, + "rewards/reward_fn/mean": -0.5069999992847443, + "rewards/reward_fn/std": 0.2859999895095825, + "step": 1175, + "step_time": 0.7220191801898181 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.2, + "completions/max_terminated_length": 19.2, + "completions/mean_length": 12.95, + "completions/mean_terminated_length": 12.95, + "completions/min_length": 9.2, + "completions/min_terminated_length": 9.2, + "entropy": 0.9652112066745758, + "epoch": 1.2164948453608249, + "frac_reward_zero_std": 0.2, + "grad_norm": 21.125, + "learning_rate": 1.9995967423401306e-07, + "loss": 0.10934715270996094, + "num_tokens": 303353.0, + "reward": -0.19833333194255828, + "reward_std": 0.3716987192630768, + "rewards/reward_fn/mean": -0.19833333194255828, + "rewards/reward_fn/std": 0.3716987192630768, + "step": 1180, + "step_time": 0.6236526501365006 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.4, + "completions/max_terminated_length": 19.4, + "completions/mean_length": 14.1, + "completions/mean_terminated_length": 14.1, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "entropy": 0.9499202251434327, + "epoch": 1.2216494845360826, + "frac_reward_zero_std": 0.4, + "grad_norm": 14.0, + "learning_rate": 1.977581092421812e-07, + "loss": 0.07946513891220093, + "num_tokens": 304687.0, + "reward": -0.625, + "reward_std": 0.15, + "rewards/reward_fn/mean": -0.625, + "rewards/reward_fn/std": 0.15, + "step": 1185, + "step_time": 0.6262445945292712 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.0, + "completions/max_terminated_length": 16.0, + "completions/mean_length": 9.95, + "completions/mean_terminated_length": 9.95, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 0.8996234953403472, + "epoch": 1.2268041237113403, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 1.9556077256436586e-07, + "loss": 0.10128650665283204, + "num_tokens": 305902.0, + "reward": -0.4949999988079071, + "reward_std": 0.20011107325553895, + "rewards/reward_fn/mean": -0.4949999988079071, + "rewards/reward_fn/std": 0.20011107325553895, + "step": 1190, + "step_time": 0.6217918395996094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05, + "completions/max_length": 76.4, + "completions/max_terminated_length": 26.8, + "completions/mean_length": 26.35, + "completions/mean_terminated_length": 13.88333339691162, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 1.2090439796447754, + "epoch": 1.231958762886598, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 1.933678420469147e-07, + "loss": -0.007263195514678955, + "num_tokens": 307457.0, + "reward": -0.5066666662693023, + "reward_std": 0.3612557053565979, + "rewards/reward_fn/mean": -0.5066666662693023, + "rewards/reward_fn/std": 0.3612557053565979, + "step": 1195, + "step_time": 1.6938601299189031 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 22.6, + "completions/max_terminated_length": 22.6, + "completions/mean_length": 13.8, + "completions/mean_terminated_length": 13.8, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "entropy": 0.7910245209932327, + "epoch": 1.2371134020618557, + "frac_reward_zero_std": 0.2, + "grad_norm": 21.75, + "learning_rate": 1.911794951795531e-07, + "loss": 0.04146863520145416, + "num_tokens": 308777.0, + "reward": -0.4508333384990692, + "reward_std": 0.41441943049430846, + "rewards/reward_fn/mean": -0.4508333384990692, + "rewards/reward_fn/std": 0.41441943049430846, + "step": 1200, + "step_time": 0.685492187552154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 25.0, + "completions/max_terminated_length": 25.0, + "completions/mean_length": 15.25, + "completions/mean_terminated_length": 15.25, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.9586571276187896, + "epoch": 1.2422680412371134, + "frac_reward_zero_std": 0.6, + "grad_norm": 21.625, + "learning_rate": 1.889959090810185e-07, + "loss": 0.11326441764831544, + "num_tokens": 310094.0, + "reward": -0.45833333134651183, + "reward_std": 0.165283465385437, + "rewards/reward_fn/mean": -0.45833333134651183, + "rewards/reward_fn/std": 0.165283465385437, + "step": 1205, + "step_time": 0.7412053862586617 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.2, + "completions/max_terminated_length": 21.2, + "completions/mean_length": 11.55, + "completions/mean_terminated_length": 11.55, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 0.8803381383419037, + "epoch": 1.2474226804123711, + "frac_reward_zero_std": 0.2, + "grad_norm": 13.125, + "learning_rate": 1.8681726048472497e-07, + "loss": 0.23633675575256347, + "num_tokens": 311329.0, + "reward": -0.4, + "reward_std": 0.4, + "rewards/reward_fn/mean": -0.4, + "rewards/reward_fn/std": 0.4, + "step": 1210, + "step_time": 0.6590411723591387 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 53.6, + "completions/max_terminated_length": 53.6, + "completions/mean_length": 24.1, + "completions/mean_terminated_length": 24.1, + "completions/min_length": 11.8, + "completions/min_terminated_length": 11.8, + "entropy": 1.015632688999176, + "epoch": 1.2525773195876289, + "frac_reward_zero_std": 0.4, + "grad_norm": 22.25, + "learning_rate": 1.8464372572445863e-07, + "loss": 0.07433696389198304, + "num_tokens": 312823.0, + "reward": -0.4983333319425583, + "reward_std": 0.20333333015441896, + "rewards/reward_fn/mean": -0.4983333319425583, + "rewards/reward_fn/std": 0.2033333420753479, + "step": 1215, + "step_time": 1.2660644174553455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 22.4, + "completions/max_terminated_length": 22.4, + "completions/mean_length": 11.4, + "completions/mean_terminated_length": 11.4, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 0.7511100977659225, + "epoch": 1.2577319587628866, + "frac_reward_zero_std": 0.0, + "grad_norm": 10.875, + "learning_rate": 1.8247548072010593e-07, + "loss": 0.20432829856872559, + "num_tokens": 314051.0, + "reward": -0.5385000020265579, + "reward_std": 0.4213788390159607, + "rewards/reward_fn/mean": -0.5385000020265579, + "rewards/reward_fn/std": 0.42137885093688965, + "step": 1220, + "step_time": 0.6830242997035384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 28.8, + "completions/max_terminated_length": 28.8, + "completions/mean_length": 15.35, + "completions/mean_terminated_length": 15.35, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.7591416954994201, + "epoch": 1.2628865979381443, + "frac_reward_zero_std": 0.4, + "grad_norm": 7.0625, + "learning_rate": 1.8031270096341533e-07, + "loss": 0.03440877199172974, + "num_tokens": 315406.0, + "reward": -0.575, + "reward_std": 0.15, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.15, + "step": 1225, + "step_time": 0.8744408298283816 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 23.2, + "completions/max_terminated_length": 23.2, + "completions/mean_length": 13.15, + "completions/mean_terminated_length": 13.15, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 0.8470461785793304, + "epoch": 1.268041237113402, + "frac_reward_zero_std": 0.2, + "grad_norm": 20.25, + "learning_rate": 1.7815556150379296e-07, + "loss": 0.12400050163269043, + "num_tokens": 316697.0, + "reward": -0.4033333361148834, + "reward_std": 0.4449474155902863, + "rewards/reward_fn/mean": -0.4033333361148834, + "rewards/reward_fn/std": 0.4449474155902863, + "step": 1230, + "step_time": 0.7029364148154855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.2, + "completions/max_terminated_length": 18.2, + "completions/mean_length": 12.05, + "completions/mean_terminated_length": 12.05, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.7155319631099701, + "epoch": 1.2731958762886597, + "frac_reward_zero_std": 0.4, + "grad_norm": 10.0, + "learning_rate": 1.7600423693413507e-07, + "loss": 0.1241507887840271, + "num_tokens": 317986.0, + "reward": -0.625, + "reward_std": 0.16547005176544188, + "rewards/reward_fn/mean": -0.625, + "rewards/reward_fn/std": 0.16547005176544188, + "step": 1235, + "step_time": 0.6039237166754902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.6, + "completions/max_terminated_length": 16.6, + "completions/mean_length": 11.1, + "completions/mean_terminated_length": 11.1, + "completions/min_length": 5.4, + "completions/min_terminated_length": 5.4, + "entropy": 0.9233552157878876, + "epoch": 1.2783505154639174, + "frac_reward_zero_std": 0.4, + "grad_norm": 13.75, + "learning_rate": 1.738589013766969e-07, + "loss": 0.028127449750900268, + "num_tokens": 319208.0, + "reward": -0.2425000011920929, + "reward_std": 0.3149999976158142, + "rewards/reward_fn/mean": -0.2425000011920929, + "rewards/reward_fn/std": 0.3149999976158142, + "step": 1240, + "step_time": 0.5808714678511023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.2, + "completions/max_terminated_length": 16.2, + "completions/mean_length": 11.5, + "completions/mean_terminated_length": 11.5, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.9115808337926865, + "epoch": 1.2835051546391751, + "frac_reward_zero_std": 0.0, + "grad_norm": 21.0, + "learning_rate": 1.7171972846899941e-07, + "loss": 0.05150402188301086, + "num_tokens": 320466.0, + "reward": -0.3074999988079071, + "reward_std": 0.469794100522995, + "rewards/reward_fn/mean": -0.3074999988079071, + "rewards/reward_fn/std": 0.46979411244392394, + "step": 1245, + "step_time": 0.567639524769038 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.0, + "completions/max_terminated_length": 14.0, + "completions/mean_length": 11.25, + "completions/mean_terminated_length": 11.25, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "entropy": 0.6332197517156601, + "epoch": 1.2886597938144329, + "frac_reward_zero_std": 0.2, + "grad_norm": 11.0625, + "learning_rate": 1.6958689134977587e-07, + "loss": 0.06763789653778077, + "num_tokens": 321707.0, + "reward": -0.5, + "reward_std": 0.3, + "rewards/reward_fn/mean": -0.5, + "rewards/reward_fn/std": 0.3, + "step": 1250, + "step_time": 0.5252592659555375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.4, + "completions/max_terminated_length": 16.4, + "completions/mean_length": 9.55, + "completions/mean_terminated_length": 9.55, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 1.0738073289394379, + "epoch": 1.2938144329896908, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 1.6746056264495846e-07, + "loss": 0.02612420916557312, + "num_tokens": 322922.0, + "reward": -0.425, + "reward_std": 0.30615527629852296, + "rewards/reward_fn/mean": -0.425, + "rewards/reward_fn/std": 0.30615527629852296, + "step": 1255, + "step_time": 0.5707598288543523 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.6, + "completions/max_terminated_length": 14.6, + "completions/mean_length": 9.55, + "completions/mean_terminated_length": 9.55, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "entropy": 0.6144456475973129, + "epoch": 1.2989690721649485, + "frac_reward_zero_std": 0.0, + "grad_norm": 23.5, + "learning_rate": 1.65340914453706e-07, + "loss": 0.20289556980133056, + "num_tokens": 324121.0, + "reward": -0.4025000035762787, + "reward_std": 0.4698511064052582, + "rewards/reward_fn/mean": -0.4025000035762787, + "rewards/reward_fn/std": 0.4698511064052582, + "step": 1260, + "step_time": 0.5400445449166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 29.2, + "completions/max_terminated_length": 29.2, + "completions/mean_length": 15.3, + "completions/mean_terminated_length": 15.3, + "completions/min_length": 8.8, + "completions/min_terminated_length": 8.8, + "entropy": 0.7304955810308457, + "epoch": 1.3041237113402062, + "frac_reward_zero_std": 0.2, + "grad_norm": 13.75, + "learning_rate": 1.632281183344756e-07, + "loss": 0.13050819635391236, + "num_tokens": 325455.0, + "reward": -0.38333333730697633, + "reward_std": 0.4089271903038025, + "rewards/reward_fn/mean": -0.38333333730697633, + "rewards/reward_fn/std": 0.40892720222473145, + "step": 1265, + "step_time": 0.873385751992464 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.6, + "completions/max_terminated_length": 15.6, + "completions/mean_length": 11.95, + "completions/mean_terminated_length": 11.95, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.6056427478790283, + "epoch": 1.309278350515464, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 1.6112234529113606e-07, + "loss": 0.10200026035308837, + "num_tokens": 326730.0, + "reward": -0.55, + "reward_std": 0.33094010353088377, + "rewards/reward_fn/mean": -0.55, + "rewards/reward_fn/std": 0.33094010353088377, + "step": 1270, + "step_time": 0.5555100217461586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.4, + "completions/max_terminated_length": 19.4, + "completions/mean_length": 10.7, + "completions/mean_terminated_length": 10.7, + "completions/min_length": 5.2, + "completions/min_terminated_length": 5.2, + "entropy": 1.0357014298439027, + "epoch": 1.3144329896907216, + "frac_reward_zero_std": 0.0, + "grad_norm": 15.125, + "learning_rate": 1.5902376575912814e-07, + "loss": 0.2978448629379272, + "num_tokens": 327960.0, + "reward": -0.5262499988079071, + "reward_std": 0.4544562935829163, + "rewards/reward_fn/mean": -0.5262499988079071, + "rewards/reward_fn/std": 0.4544562935829163, + "step": 1275, + "step_time": 0.6274704253301024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.0, + "completions/max_terminated_length": 14.0, + "completions/mean_length": 9.85, + "completions/mean_terminated_length": 9.85, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 0.8286823898553848, + "epoch": 1.3195876288659794, + "frac_reward_zero_std": 0.4, + "grad_norm": 13.6875, + "learning_rate": 1.569325495916696e-07, + "loss": 0.02840469479560852, + "num_tokens": 329177.0, + "reward": -0.3458333373069763, + "reward_std": 0.2295683205127716, + "rewards/reward_fn/mean": -0.3458333373069763, + "rewards/reward_fn/std": 0.22956833243370056, + "step": 1280, + "step_time": 0.5265060624107718 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.2, + "completions/max_terminated_length": 11.2, + "completions/mean_length": 7.75, + "completions/mean_terminated_length": 7.75, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 0.8311231344938278, + "epoch": 1.324742268041237, + "frac_reward_zero_std": 0.0, + "grad_norm": 20.625, + "learning_rate": 1.5484886604600795e-07, + "loss": 0.1401970148086548, + "num_tokens": 330348.0, + "reward": -0.36166666746139525, + "reward_std": 0.45088627338409426, + "rewards/reward_fn/mean": -0.36166666746139525, + "rewards/reward_fn/std": 0.45088627338409426, + "step": 1285, + "step_time": 0.4739662584848702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.2, + "completions/max_terminated_length": 16.2, + "completions/mean_length": 10.6, + "completions/mean_terminated_length": 10.6, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.894581812620163, + "epoch": 1.3298969072164948, + "frac_reward_zero_std": 0.0, + "grad_norm": 16.375, + "learning_rate": 1.5277288376972114e-07, + "loss": 0.10380651950836181, + "num_tokens": 331592.0, + "reward": -0.19583333134651185, + "reward_std": 0.530957841873169, + "rewards/reward_fn/mean": -0.19583333134651185, + "rewards/reward_fn/std": 0.530957841873169, + "step": 1290, + "step_time": 0.5679388328455388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 23.2, + "completions/max_terminated_length": 23.2, + "completions/mean_length": 14.7, + "completions/mean_terminated_length": 14.7, + "completions/min_length": 9.4, + "completions/min_terminated_length": 9.4, + "entropy": 0.7432743966579437, + "epoch": 1.3350515463917525, + "frac_reward_zero_std": 0.4, + "grad_norm": 10.875, + "learning_rate": 1.5070477078706755e-07, + "loss": 0.003392466902732849, + "num_tokens": 332938.0, + "reward": -0.3454999983310699, + "reward_std": 0.3838511049747467, + "rewards/reward_fn/mean": -0.3454999983310699, + "rewards/reward_fn/std": 0.3838511049747467, + "step": 1295, + "step_time": 0.6983847383409738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.2, + "completions/max_terminated_length": 15.2, + "completions/mean_length": 10.9, + "completions/mean_terminated_length": 10.9, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.8243459284305572, + "epoch": 1.3402061855670104, + "frac_reward_zero_std": 0.2, + "grad_norm": 23.625, + "learning_rate": 1.4864469448538696e-07, + "loss": 0.10561842918395996, + "num_tokens": 334168.0, + "reward": -0.4600000023841858, + "reward_std": 0.3377350211143494, + "rewards/reward_fn/mean": -0.4600000023841858, + "rewards/reward_fn/std": 0.3377350211143494, + "step": 1300, + "step_time": 0.6084295228123665 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.4, + "completions/max_terminated_length": 12.4, + "completions/mean_length": 10.0, + "completions/mean_terminated_length": 10.0, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.7640111565589904, + "epoch": 1.3453608247422681, + "frac_reward_zero_std": 0.8, + "grad_norm": 33.25, + "learning_rate": 1.465928216015522e-07, + "loss": 0.027267986536026002, + "num_tokens": 335364.0, + "reward": -0.5425000011920929, + "reward_std": 0.11499999761581421, + "rewards/reward_fn/mean": -0.5425000011920929, + "rewards/reward_fn/std": 0.11499999761581421, + "step": 1305, + "step_time": 0.5023958536796271 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 61.0, + "completions/max_terminated_length": 61.0, + "completions/mean_length": 25.35, + "completions/mean_terminated_length": 25.35, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 0.8650216937065125, + "epoch": 1.3505154639175259, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.375, + "learning_rate": 1.445493182084743e-07, + "loss": -0.0330998033285141, + "num_tokens": 336903.0, + "reward": -0.5233333319425583, + "reward_std": 0.25333333015441895, + "rewards/reward_fn/mean": -0.5233333319425583, + "rewards/reward_fn/std": 0.2533333420753479, + "step": 1310, + "step_time": 1.4153380719013513 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9.8, + "completions/max_terminated_length": 9.8, + "completions/mean_length": 6.7, + "completions/mean_terminated_length": 6.7, + "completions/min_length": 4.8, + "completions/min_terminated_length": 4.8, + "entropy": 0.8046780616044998, + "epoch": 1.3556701030927836, + "frac_reward_zero_std": 0.4, + "grad_norm": 18.625, + "learning_rate": 1.425143497016608e-07, + "loss": 0.1706416368484497, + "num_tokens": 338045.0, + "reward": -0.1850000023841858, + "reward_std": 0.2744683504104614, + "rewards/reward_fn/mean": -0.1850000023841858, + "rewards/reward_fn/std": 0.2744683504104614, + "step": 1315, + "step_time": 0.4627469423227012 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.0, + "completions/max_terminated_length": 18.0, + "completions/mean_length": 9.65, + "completions/mean_terminated_length": 9.65, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 1.0882736980915069, + "epoch": 1.3608247422680413, + "frac_reward_zero_std": 0.2, + "grad_norm": 34.0, + "learning_rate": 1.4048808078582942e-07, + "loss": 0.1820279359817505, + "num_tokens": 339246.0, + "reward": -0.3949999988079071, + "reward_std": 0.342376047372818, + "rewards/reward_fn/mean": -0.3949999988079071, + "rewards/reward_fn/std": 0.342376047372818, + "step": 1320, + "step_time": 0.6051038973964751 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 31.2, + "completions/max_terminated_length": 31.2, + "completions/mean_length": 15.0, + "completions/mean_terminated_length": 15.0, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 1.1348010390996932, + "epoch": 1.365979381443299, + "frac_reward_zero_std": 0.2, + "grad_norm": 36.5, + "learning_rate": 1.384706754615767e-07, + "loss": -0.08873192071914673, + "num_tokens": 340586.0, + "reward": -0.15250000059604646, + "reward_std": 0.2449999988079071, + "rewards/reward_fn/mean": -0.15250000059604646, + "rewards/reward_fn/std": 0.2449999988079071, + "step": 1325, + "step_time": 0.853823087643832 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 24.2, + "completions/max_terminated_length": 24.2, + "completions/mean_length": 17.3, + "completions/mean_terminated_length": 17.3, + "completions/min_length": 13.0, + "completions/min_terminated_length": 13.0, + "entropy": 0.8157379269599915, + "epoch": 1.3711340206185567, + "frac_reward_zero_std": 0.2, + "grad_norm": 5.6875, + "learning_rate": 1.3646229701210488e-07, + "loss": 0.0920086920261383, + "num_tokens": 341964.0, + "reward": -0.5612500071525574, + "reward_std": 0.3543502926826477, + "rewards/reward_fn/mean": -0.5612500071525574, + "rewards/reward_fn/std": 0.3543502926826477, + "step": 1330, + "step_time": 0.7886355902068317 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.0, + "completions/max_terminated_length": 13.0, + "completions/mean_length": 8.4, + "completions/mean_terminated_length": 8.4, + "completions/min_length": 4.6, + "completions/min_terminated_length": 4.6, + "entropy": 1.2810046076774597, + "epoch": 1.3762886597938144, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 1.3446310799000575e-07, + "loss": 0.09169626832008362, + "num_tokens": 343160.0, + "reward": -0.4050000011920929, + "reward_std": 0.2223760485649109, + "rewards/reward_fn/mean": -0.4050000011920929, + "rewards/reward_fn/std": 0.2223760485649109, + "step": 1335, + "step_time": 0.5072437161579728 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 26.6, + "completions/max_terminated_length": 26.6, + "completions/mean_length": 15.05, + "completions/mean_terminated_length": 15.05, + "completions/min_length": 9.4, + "completions/min_terminated_length": 9.4, + "entropy": 0.6845301300287246, + "epoch": 1.3814432989690721, + "frac_reward_zero_std": 0.4, + "grad_norm": 20.375, + "learning_rate": 1.3247327020410452e-07, + "loss": 0.016708952188491822, + "num_tokens": 344497.0, + "reward": -0.2340000033378601, + "reward_std": 0.37685110569000246, + "rewards/reward_fn/mean": -0.2340000033378601, + "rewards/reward_fn/std": 0.3768511176109314, + "step": 1340, + "step_time": 0.7704925104044378 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.2, + "completions/max_terminated_length": 16.2, + "completions/mean_length": 10.4, + "completions/mean_terminated_length": 10.4, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.8923976421356201, + "epoch": 1.3865979381443299, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 1.3049294470636302e-07, + "loss": 0.12361987829208373, + "num_tokens": 345705.0, + "reward": -0.2, + "reward_std": 0.3, + "rewards/reward_fn/mean": -0.2, + "rewards/reward_fn/std": 0.3, + "step": 1345, + "step_time": 0.6526330811902881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 24.0, + "completions/max_terminated_length": 24.0, + "completions/mean_length": 12.75, + "completions/mean_terminated_length": 12.75, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 1.0238151282072068, + "epoch": 1.3917525773195876, + "frac_reward_zero_std": 0.4, + "grad_norm": 29.125, + "learning_rate": 1.285222917788449e-07, + "loss": 0.10306930541992188, + "num_tokens": 347004.0, + "reward": -0.25499999821186065, + "reward_std": 0.3032050788402557, + "rewards/reward_fn/mean": -0.25499999821186065, + "rewards/reward_fn/std": 0.3032050788402557, + "step": 1350, + "step_time": 0.7229462908580899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.4, + "completions/max_terminated_length": 16.4, + "completions/mean_length": 9.4, + "completions/mean_terminated_length": 9.4, + "completions/min_length": 4.8, + "completions/min_terminated_length": 4.8, + "entropy": 1.2665182173252105, + "epoch": 1.3969072164948453, + "frac_reward_zero_std": 0.2, + "grad_norm": 34.25, + "learning_rate": 1.2656147092074276e-07, + "loss": 0.27374863624572754, + "num_tokens": 348196.0, + "reward": -0.625, + "reward_std": 0.20773502588272094, + "rewards/reward_fn/mean": -0.625, + "rewards/reward_fn/std": 0.20773502588272094, + "step": 1355, + "step_time": 0.5801575570367277 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.2, + "completions/max_terminated_length": 16.2, + "completions/mean_length": 10.6, + "completions/mean_terminated_length": 10.6, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 1.2069101631641388, + "epoch": 1.402061855670103, + "frac_reward_zero_std": 0.0, + "grad_norm": 14.875, + "learning_rate": 1.2461064083546872e-07, + "loss": 0.16335666179656982, + "num_tokens": 349416.0, + "reward": -0.01749999225139618, + "reward_std": 0.32012745440006257, + "rewards/reward_fn/mean": -0.01749999225139618, + "rewards/reward_fn/std": 0.32012745440006257, + "step": 1360, + "step_time": 0.5706661109812557 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.2, + "completions/max_terminated_length": 12.2, + "completions/mean_length": 9.0, + "completions/mean_terminated_length": 9.0, + "completions/min_length": 5.4, + "completions/min_terminated_length": 5.4, + "entropy": 0.8017214298248291, + "epoch": 1.4072164948453607, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 1.2266995941780933e-07, + "loss": 0.14401029348373412, + "num_tokens": 350640.0, + "reward": -0.30999999642372134, + "reward_std": 0.2600000023841858, + "rewards/reward_fn/mean": -0.30999999642372134, + "rewards/reward_fn/std": 0.2600000023841858, + "step": 1365, + "step_time": 0.5673609586432576 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.2, + "completions/max_terminated_length": 30.2, + "completions/mean_length": 17.1, + "completions/mean_terminated_length": 17.1, + "completions/min_length": 9.4, + "completions/min_terminated_length": 9.4, + "entropy": 0.6343318462371826, + "epoch": 1.4123711340206184, + "frac_reward_zero_std": 0.4, + "grad_norm": 19.5, + "learning_rate": 1.2073958374114608e-07, + "loss": 0.15244705677032472, + "num_tokens": 352022.0, + "reward": -0.5025000035762787, + "reward_std": 0.26985110640525817, + "rewards/reward_fn/mean": -0.5025000035762787, + "rewards/reward_fn/std": 0.26985110640525817, + "step": 1370, + "step_time": 0.8377051496878266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05, + "completions/max_length": 76.4, + "completions/max_terminated_length": 27.2, + "completions/mean_length": 27.35, + "completions/mean_terminated_length": 14.866666698455811, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 1.4102221041917802, + "epoch": 1.4175257731958764, + "frac_reward_zero_std": 0.4, + "grad_norm": 16.875, + "learning_rate": 1.1881967004474255e-07, + "loss": 0.1017134428024292, + "num_tokens": 353585.0, + "reward": -0.30613636672496797, + "reward_std": 0.3646326959133148, + "rewards/reward_fn/mean": -0.30613636672496797, + "rewards/reward_fn/std": 0.3646326959133148, + "step": 1375, + "step_time": 1.7935701003298163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 26.2, + "completions/max_terminated_length": 26.2, + "completions/mean_length": 16.55, + "completions/mean_terminated_length": 16.55, + "completions/min_length": 11.0, + "completions/min_terminated_length": 11.0, + "entropy": 1.0183711111545564, + "epoch": 1.422680412371134, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 1.1691037372109832e-07, + "loss": -0.013654737174510956, + "num_tokens": 354944.0, + "reward": -0.35833333134651185, + "reward_std": 0.30754843950271604, + "rewards/reward_fn/mean": -0.35833333134651185, + "rewards/reward_fn/std": 0.30754843950271604, + "step": 1380, + "step_time": 0.7602020460180938 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.2, + "completions/max_terminated_length": 16.2, + "completions/mean_length": 10.75, + "completions/mean_terminated_length": 10.75, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 0.7927883446216584, + "epoch": 1.4278350515463918, + "frac_reward_zero_std": 0.4, + "grad_norm": 18.75, + "learning_rate": 1.1501184930337235e-07, + "loss": 0.03684360682964325, + "num_tokens": 356155.0, + "reward": -0.36833333075046537, + "reward_std": 0.36272807121276857, + "rewards/reward_fn/mean": -0.36833333075046537, + "rewards/reward_fn/std": 0.36272807121276857, + "step": 1385, + "step_time": 0.5737183832563459 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.2, + "completions/max_terminated_length": 18.2, + "completions/mean_length": 10.85, + "completions/mean_terminated_length": 10.85, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 0.8488032281398773, + "epoch": 1.4329896907216495, + "frac_reward_zero_std": 0.4, + "grad_norm": 20.0, + "learning_rate": 1.1312425045287524e-07, + "loss": 0.10943784713745117, + "num_tokens": 357380.0, + "reward": -0.5100000023841857, + "reward_std": 0.1800000011920929, + "rewards/reward_fn/mean": -0.5100000023841857, + "rewards/reward_fn/std": 0.1800000011920929, + "step": 1390, + "step_time": 0.670763380266726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10.2, + "completions/max_terminated_length": 10.2, + "completions/mean_length": 7.7, + "completions/mean_terminated_length": 7.7, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.6782841980457306, + "epoch": 1.4381443298969072, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 1.1124772994663256e-07, + "loss": 0.10716173648834229, + "num_tokens": 358538.0, + "reward": -0.21916666626930237, + "reward_std": 0.24967803657054902, + "rewards/reward_fn/mean": -0.21916666626930237, + "rewards/reward_fn/std": 0.24967803657054902, + "step": 1395, + "step_time": 0.46183384573087094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 56.8, + "completions/max_terminated_length": 56.8, + "completions/mean_length": 23.45, + "completions/mean_terminated_length": 23.45, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 1.0040556222200394, + "epoch": 1.443298969072165, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 1.0938243966501929e-07, + "loss": 0.22731239795684816, + "num_tokens": 360027.0, + "reward": -0.4699999988079071, + "reward_std": 0.17793227434158326, + "rewards/reward_fn/mean": -0.4699999988079071, + "rewards/reward_fn/std": 0.17793227434158326, + "step": 1400, + "step_time": 1.3617384696379304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.4, + "completions/max_terminated_length": 14.4, + "completions/mean_length": 11.55, + "completions/mean_terminated_length": 11.55, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.7580155253410339, + "epoch": 1.4484536082474226, + "frac_reward_zero_std": 0.0, + "grad_norm": 13.0, + "learning_rate": 1.075285305794671e-07, + "loss": 0.11078314781188965, + "num_tokens": 361294.0, + "reward": -0.3039999961853027, + "reward_std": 0.4995263934135437, + "rewards/reward_fn/mean": -0.3039999961853027, + "rewards/reward_fn/std": 0.4995263934135437, + "step": 1405, + "step_time": 0.6288512685336173 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05, + "completions/max_length": 61.2, + "completions/max_terminated_length": 45.4, + "completions/mean_length": 30.8, + "completions/mean_terminated_length": 21.483334350585938, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.8140331745147705, + "epoch": 1.4536082474226804, + "frac_reward_zero_std": 0.2, + "grad_norm": 3.734375, + "learning_rate": 1.056861527402452e-07, + "loss": 0.0343685507774353, + "num_tokens": 362922.0, + "reward": -0.625, + "reward_std": 0.20773502588272094, + "rewards/reward_fn/mean": -0.625, + "rewards/reward_fn/std": 0.20773502588272094, + "step": 1410, + "step_time": 1.5017110424116253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.6, + "completions/max_terminated_length": 15.6, + "completions/mean_length": 9.0, + "completions/mean_terminated_length": 9.0, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 1.2074449896812438, + "epoch": 1.458762886597938, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 1.0385545526431566e-07, + "loss": 0.154154634475708, + "num_tokens": 364126.0, + "reward": -0.3650000005960464, + "reward_std": 0.2099999964237213, + "rewards/reward_fn/mean": -0.3650000005960464, + "rewards/reward_fn/std": 0.21000000834465027, + "step": 1415, + "step_time": 0.5890727814286947 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.6, + "completions/max_terminated_length": 17.6, + "completions/mean_length": 12.1, + "completions/mean_terminated_length": 12.1, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "entropy": 0.8329922676086425, + "epoch": 1.463917525773196, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 1.0203658632326423e-07, + "loss": 0.11881840229034424, + "num_tokens": 365384.0, + "reward": -0.5350000023841858, + "reward_std": 0.18773502707481385, + "rewards/reward_fn/mean": -0.5350000023841858, + "rewards/reward_fn/std": 0.18773502707481385, + "step": 1420, + "step_time": 0.7082118081860245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.4, + "completions/max_terminated_length": 19.4, + "completions/mean_length": 11.05, + "completions/mean_terminated_length": 11.05, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 0.9463332563638687, + "epoch": 1.4690721649484537, + "frac_reward_zero_std": 0.4, + "grad_norm": 15.8125, + "learning_rate": 1.0022969313130773e-07, + "loss": -0.11116218566894531, + "num_tokens": 366617.0, + "reward": -0.4077777773141861, + "reward_std": 0.35764952898025515, + "rewards/reward_fn/mean": -0.4077777773141861, + "rewards/reward_fn/std": 0.35764952898025515, + "step": 1425, + "step_time": 0.6454660271294415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.8, + "completions/max_terminated_length": 18.8, + "completions/mean_length": 12.45, + "completions/mean_terminated_length": 12.45, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "entropy": 0.8313162535429001, + "epoch": 1.4742268041237114, + "frac_reward_zero_std": 0.2, + "grad_norm": 16.875, + "learning_rate": 9.843492193337923e-08, + "loss": 0.10078572034835816, + "num_tokens": 367866.0, + "reward": -0.19883333146572113, + "reward_std": 0.5281381130218505, + "rewards/reward_fn/mean": -0.19883333146572113, + "rewards/reward_fn/std": 0.5281381130218505, + "step": 1430, + "step_time": 0.6323665215633809 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.4, + "completions/max_terminated_length": 7.4, + "completions/mean_length": 6.0, + "completions/mean_terminated_length": 6.0, + "completions/min_length": 4.6, + "completions/min_terminated_length": 4.6, + "entropy": 0.7071571081876755, + "epoch": 1.4793814432989691, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 9.665241799329096e-08, + "loss": 0.05014275312423706, + "num_tokens": 368998.0, + "reward": 0.27749999761581423, + "reward_std": 0.4774251878261566, + "rewards/reward_fn/mean": 0.27749999761581423, + "rewards/reward_fn/std": 0.4774251878261566, + "step": 1435, + "step_time": 0.46755328895524145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 25.6, + "completions/max_terminated_length": 25.6, + "completions/mean_length": 12.1, + "completions/mean_terminated_length": 12.1, + "completions/min_length": 4.6, + "completions/min_terminated_length": 4.6, + "entropy": 0.8592796742916107, + "epoch": 1.4845360824742269, + "frac_reward_zero_std": 0.2, + "grad_norm": 15.625, + "learning_rate": 9.488232558197731e-08, + "loss": 0.217482328414917, + "num_tokens": 370272.0, + "reward": -0.3266666650772095, + "reward_std": 0.3772513210773468, + "rewards/reward_fn/mean": -0.3266666650772095, + "rewards/reward_fn/std": 0.37725133299827573, + "step": 1440, + "step_time": 0.7577955449931324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.6, + "completions/max_terminated_length": 16.6, + "completions/mean_length": 12.85, + "completions/mean_terminated_length": 12.85, + "completions/min_length": 8.8, + "completions/min_terminated_length": 8.8, + "entropy": 0.6260365337133408, + "epoch": 1.4896907216494846, + "frac_reward_zero_std": 0.2, + "grad_norm": 21.875, + "learning_rate": 9.312478796581791e-08, + "loss": 0.1230657935142517, + "num_tokens": 371541.0, + "reward": -0.46194444596767426, + "reward_std": 0.3338461399078369, + "rewards/reward_fn/mean": -0.46194444596767426, + "rewards/reward_fn/std": 0.33384612798690794, + "step": 1445, + "step_time": 0.5881177311763167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.4, + "completions/max_terminated_length": 8.4, + "completions/mean_length": 6.35, + "completions/mean_terminated_length": 6.35, + "completions/min_length": 4.8, + "completions/min_terminated_length": 4.8, + "entropy": 1.007636147737503, + "epoch": 1.4948453608247423, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 9.137994739504199e-08, + "loss": 0.014282146096229553, + "num_tokens": 372688.0, + "reward": -0.31999999582767485, + "reward_std": 0.0800000011920929, + "rewards/reward_fn/mean": -0.31999999582767485, + "rewards/reward_fn/std": 0.0800000011920929, + "step": 1450, + "step_time": 0.4345912431366742 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.4, + "completions/max_terminated_length": 8.4, + "completions/mean_length": 6.8, + "completions/mean_terminated_length": 6.8, + "completions/min_length": 5.2, + "completions/min_terminated_length": 5.2, + "entropy": 0.5342540591955185, + "epoch": 1.5, + "frac_reward_zero_std": 0.4, + "grad_norm": 37.5, + "learning_rate": 8.964794509221507e-08, + "loss": 0.10609520673751831, + "num_tokens": 373848.0, + "reward": -0.0625, + "reward_std": 0.39801986813545226, + "rewards/reward_fn/mean": -0.0625, + "rewards/reward_fn/std": 0.3980198800563812, + "step": 1455, + "step_time": 0.5045116319321096 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10.0, + "completions/max_terminated_length": 10.0, + "completions/mean_length": 7.8, + "completions/mean_terminated_length": 7.8, + "completions/min_length": 5.4, + "completions/min_terminated_length": 5.4, + "entropy": 0.7498216450214386, + "epoch": 1.5051546391752577, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 8.79289212408088e-08, + "loss": 0.04797559082508087, + "num_tokens": 375004.0, + "reward": -0.28333333432674407, + "reward_std": 0.28076876401901246, + "rewards/reward_fn/mean": -0.28333333432674407, + "rewards/reward_fn/std": 0.28076878786087034, + "step": 1460, + "step_time": 0.4600403653457761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.4, + "completions/max_terminated_length": 17.4, + "completions/mean_length": 12.9, + "completions/mean_terminated_length": 12.9, + "completions/min_length": 8.8, + "completions/min_terminated_length": 8.8, + "entropy": 0.8871105581521987, + "epoch": 1.5103092783505154, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 8.622301497385506e-08, + "loss": -0.021851116418838502, + "num_tokens": 376282.0, + "reward": 0.06249999701976776, + "reward_std": 0.29870229959487915, + "rewards/reward_fn/mean": 0.06249999701976776, + "rewards/reward_fn/std": 0.2987023115158081, + "step": 1465, + "step_time": 0.6089170153252781 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.0, + "completions/max_terminated_length": 16.0, + "completions/mean_length": 11.45, + "completions/mean_terminated_length": 11.45, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.88620465695858, + "epoch": 1.5154639175257731, + "frac_reward_zero_std": 0.2, + "grad_norm": 18.125, + "learning_rate": 8.453036436268457e-08, + "loss": 0.03725805878639221, + "num_tokens": 377543.0, + "reward": -0.3990000009536743, + "reward_std": 0.3792510271072388, + "rewards/reward_fn/mean": -0.3990000009536743, + "rewards/reward_fn/std": 0.3792510271072388, + "step": 1470, + "step_time": 0.5841910772025585 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 49.6, + "completions/max_terminated_length": 49.6, + "completions/mean_length": 20.3, + "completions/mean_terminated_length": 20.3, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.589074644446373, + "epoch": 1.5206185567010309, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 8.285110640575199e-08, + "loss": -0.00042560696601867676, + "num_tokens": 378973.0, + "reward": -0.5600000023841858, + "reward_std": 0.21528346538543702, + "rewards/reward_fn/mean": -0.5600000023841858, + "rewards/reward_fn/std": 0.21528346538543702, + "step": 1475, + "step_time": 1.345842866692692 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 43.6, + "completions/max_terminated_length": 43.6, + "completions/mean_length": 20.05, + "completions/mean_terminated_length": 20.05, + "completions/min_length": 8.4, + "completions/min_terminated_length": 8.4, + "entropy": 1.0215669929981233, + "epoch": 1.5257731958762886, + "frac_reward_zero_std": 0.4, + "grad_norm": 10.6875, + "learning_rate": 8.118537701754766e-08, + "loss": 0.04796600043773651, + "num_tokens": 380426.0, + "reward": -0.6, + "reward_std": 0.15773502588272095, + "rewards/reward_fn/mean": -0.6, + "rewards/reward_fn/std": 0.15773502588272095, + "step": 1480, + "step_time": 1.1404463457874954 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 27.6, + "completions/max_terminated_length": 27.6, + "completions/mean_length": 13.8, + "completions/mean_terminated_length": 13.8, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.6877721130847931, + "epoch": 1.5309278350515463, + "frac_reward_zero_std": 0.6, + "grad_norm": 20.375, + "learning_rate": 7.953331101759705e-08, + "loss": 0.06992399096488952, + "num_tokens": 381722.0, + "reward": -0.6, + "reward_std": 0.1154700517654419, + "rewards/reward_fn/mean": -0.6, + "rewards/reward_fn/std": 0.1154700517654419, + "step": 1485, + "step_time": 0.8708475268445909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 22.0, + "completions/max_terminated_length": 22.0, + "completions/mean_length": 12.35, + "completions/mean_terminated_length": 12.35, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 0.8309216499328613, + "epoch": 1.536082474226804, + "frac_reward_zero_std": 0.6, + "grad_norm": 11.4375, + "learning_rate": 7.789504211954865e-08, + "loss": -0.06701267361640931, + "num_tokens": 382973.0, + "reward": -0.575, + "reward_std": 0.10773502588272095, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.10773502588272095, + "step": 1490, + "step_time": 0.7045149521902203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.4, + "completions/max_terminated_length": 14.4, + "completions/mean_length": 10.25, + "completions/mean_terminated_length": 10.25, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.8640795558691025, + "epoch": 1.5412371134020617, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 7.627070292035201e-08, + "loss": -0.02734326720237732, + "num_tokens": 384206.0, + "reward": -0.39000000059604645, + "reward_std": 0.2600000023841858, + "rewards/reward_fn/mean": -0.39000000059604645, + "rewards/reward_fn/std": 0.2600000023841858, + "step": 1495, + "step_time": 0.5708974854089319 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.0, + "completions/max_terminated_length": 18.0, + "completions/mean_length": 11.4, + "completions/mean_terminated_length": 11.4, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.948620742559433, + "epoch": 1.5463917525773194, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 7.46604248895252e-08, + "loss": -0.024460136890411377, + "num_tokens": 385470.0, + "reward": -0.4800000011920929, + "reward_std": 0.27285742163658144, + "rewards/reward_fn/mean": -0.4800000011920929, + "rewards/reward_fn/std": 0.27285742163658144, + "step": 1500, + "step_time": 0.7104676578193903 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.4, + "completions/max_terminated_length": 11.4, + "completions/mean_length": 8.3, + "completions/mean_terminated_length": 8.3, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.8726276814937591, + "epoch": 1.5515463917525774, + "frac_reward_zero_std": 0.8, + "grad_norm": 25.0, + "learning_rate": 7.306433835851422e-08, + "loss": 0.10536327362060546, + "num_tokens": 386652.0, + "reward": -0.525, + "reward_std": 0.05, + "rewards/reward_fn/mean": -0.525, + "rewards/reward_fn/std": 0.05, + "step": 1505, + "step_time": 0.4956268293783069 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.8, + "completions/max_terminated_length": 11.8, + "completions/mean_length": 7.35, + "completions/mean_terminated_length": 7.35, + "completions/min_length": 5.2, + "completions/min_terminated_length": 5.2, + "entropy": 1.0207952618598939, + "epoch": 1.556701030927835, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 7.148257251014453e-08, + "loss": 0.024337270855903627, + "num_tokens": 387815.0, + "reward": -0.4983333319425583, + "reward_std": 0.20333333015441896, + "rewards/reward_fn/mean": -0.4983333319425583, + "rewards/reward_fn/std": 0.2033333420753479, + "step": 1510, + "step_time": 0.5121777442283928 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.0, + "completions/max_terminated_length": 14.0, + "completions/mean_length": 7.8, + "completions/mean_terminated_length": 7.8, + "completions/min_length": 4.4, + "completions/min_terminated_length": 4.4, + "entropy": 1.017931866645813, + "epoch": 1.5618556701030928, + "frac_reward_zero_std": 0.2, + "grad_norm": 32.5, + "learning_rate": 6.991525536816497e-08, + "loss": 0.13234100341796876, + "num_tokens": 388979.0, + "reward": -0.17000000178813934, + "reward_std": 0.33085031509399415, + "rewards/reward_fn/mean": -0.17000000178813934, + "rewards/reward_fn/std": 0.33085031509399415, + "step": 1515, + "step_time": 0.5632654077373445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 7.6, + "completions/max_terminated_length": 7.6, + "completions/mean_length": 6.35, + "completions/mean_terminated_length": 6.35, + "completions/min_length": 5.2, + "completions/min_terminated_length": 5.2, + "entropy": 0.9235482051968574, + "epoch": 1.5670103092783505, + "frac_reward_zero_std": 0.2, + "grad_norm": 29.125, + "learning_rate": 6.836251378688631e-08, + "loss": 0.10203149318695068, + "num_tokens": 390118.0, + "reward": -0.3066666662693024, + "reward_std": 0.3993190586566925, + "rewards/reward_fn/mean": -0.3066666662693024, + "rewards/reward_fn/std": 0.3993190586566925, + "step": 1520, + "step_time": 0.49033016450703143 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05, + "completions/max_length": 65.2, + "completions/max_terminated_length": 18.6, + "completions/mean_length": 25.65, + "completions/mean_terminated_length": 13.7, + "completions/min_length": 9.2, + "completions/min_terminated_length": 9.2, + "entropy": 0.6301730372011661, + "epoch": 1.5721649484536082, + "frac_reward_zero_std": 0.4, + "grad_norm": 15.5, + "learning_rate": 6.682447344091363e-08, + "loss": 0.14748450517654418, + "num_tokens": 391663.0, + "reward": -0.43458333015441897, + "reward_std": 0.29935399293899534, + "rewards/reward_fn/mean": -0.43458333015441897, + "rewards/reward_fn/std": 0.2993540048599243, + "step": 1525, + "step_time": 1.5304752157069743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 24.8, + "completions/max_terminated_length": 24.8, + "completions/mean_length": 15.95, + "completions/mean_terminated_length": 15.95, + "completions/min_length": 9.6, + "completions/min_terminated_length": 9.6, + "entropy": 1.306685358285904, + "epoch": 1.577319587628866, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 6.530125881497473e-08, + "loss": 0.06663668155670166, + "num_tokens": 393002.0, + "reward": -0.3899999976158142, + "reward_std": 0.2558645486831665, + "rewards/reward_fn/mean": -0.3899999976158142, + "rewards/reward_fn/std": 0.2558645486831665, + "step": 1530, + "step_time": 0.8066560757346452 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.4, + "completions/max_terminated_length": 13.4, + "completions/mean_length": 10.75, + "completions/mean_terminated_length": 10.75, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.8304047286510468, + "epoch": 1.5824742268041239, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 6.37929931938447e-08, + "loss": 0.008444239944219589, + "num_tokens": 394249.0, + "reward": -0.32750000059604645, + "reward_std": 0.24915027618408203, + "rewards/reward_fn/mean": -0.32750000059604645, + "rewards/reward_fn/std": 0.24915027618408203, + "step": 1535, + "step_time": 0.5235663454048336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.6, + "completions/max_terminated_length": 17.6, + "completions/mean_length": 10.5, + "completions/mean_terminated_length": 10.5, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 0.7640501141548157, + "epoch": 1.5876288659793816, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 6.229979865236768e-08, + "loss": -0.030684226751327516, + "num_tokens": 395455.0, + "reward": -0.3512500047683716, + "reward_std": 0.19023503065109254, + "rewards/reward_fn/mean": -0.3512500047683716, + "rewards/reward_fn/std": 0.19023503065109254, + "step": 1540, + "step_time": 0.6071365992538631 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.2, + "completions/max_terminated_length": 20.2, + "completions/mean_length": 12.7, + "completions/mean_terminated_length": 12.7, + "completions/min_length": 8.4, + "completions/min_terminated_length": 8.4, + "entropy": 0.9274045705795289, + "epoch": 1.5927835051546393, + "frac_reward_zero_std": 0.2, + "grad_norm": 38.25, + "learning_rate": 6.082179604557616e-08, + "loss": 0.047584915161132814, + "num_tokens": 396741.0, + "reward": -0.11250001192092896, + "reward_std": 0.21273502707481384, + "rewards/reward_fn/mean": -0.11250001192092896, + "rewards/reward_fn/std": 0.21273502707481384, + "step": 1545, + "step_time": 0.6507150128483772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.6, + "completions/max_terminated_length": 20.6, + "completions/mean_length": 13.25, + "completions/mean_terminated_length": 13.25, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 0.9783258199691772, + "epoch": 1.597938144329897, + "frac_reward_zero_std": 0.2, + "grad_norm": 12.5625, + "learning_rate": 5.9359104998909617e-08, + "loss": 0.1127540111541748, + "num_tokens": 398022.0, + "reward": -0.27166666388511657, + "reward_std": 0.5395560979843139, + "rewards/reward_fn/mean": -0.27166666388511657, + "rewards/reward_fn/std": 0.5395561099052429, + "step": 1550, + "step_time": 0.7179786643013358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.0, + "completions/max_terminated_length": 14.0, + "completions/mean_length": 11.65, + "completions/mean_terminated_length": 11.65, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.7478360235691071, + "epoch": 1.6030927835051547, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.8125, + "learning_rate": 5.791184389853212e-08, + "loss": 0.02044408619403839, + "num_tokens": 399287.0, + "reward": -0.23333333432674408, + "reward_std": 0.5817121744155884, + "rewards/reward_fn/mean": -0.23333333432674408, + "rewards/reward_fn/std": 0.5817121982574462, + "step": 1555, + "step_time": 0.5401966189965606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 24.6, + "completions/max_terminated_length": 24.6, + "completions/mean_length": 11.9, + "completions/mean_terminated_length": 11.9, + "completions/min_length": 5.4, + "completions/min_terminated_length": 5.4, + "entropy": 1.0005066066980361, + "epoch": 1.6082474226804124, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 5.648012988175074e-08, + "loss": 0.07772735953330993, + "num_tokens": 400533.0, + "reward": -0.325, + "reward_std": 0.3809401035308838, + "rewards/reward_fn/mean": -0.325, + "rewards/reward_fn/std": 0.3809401035308838, + "step": 1560, + "step_time": 0.7347628603689372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 45.2, + "completions/max_terminated_length": 45.2, + "completions/mean_length": 20.0, + "completions/mean_terminated_length": 20.0, + "completions/min_length": 9.4, + "completions/min_terminated_length": 9.4, + "entropy": 0.878107276558876, + "epoch": 1.6134020618556701, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 5.5064078827534554e-08, + "loss": -0.0046801909804344176, + "num_tokens": 401977.0, + "reward": -0.527999997138977, + "reward_std": 0.20173502564430237, + "rewards/reward_fn/mean": -0.527999997138977, + "rewards/reward_fn/std": 0.20173502564430237, + "step": 1565, + "step_time": 1.186389725934714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.0, + "completions/max_terminated_length": 14.0, + "completions/mean_length": 9.4, + "completions/mean_terminated_length": 9.4, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 0.9705159187316894, + "epoch": 1.6185567010309279, + "frac_reward_zero_std": 0.4, + "grad_norm": 19.125, + "learning_rate": 5.36638053471358e-08, + "loss": -0.07211000919342041, + "num_tokens": 403185.0, + "reward": -0.5350000023841858, + "reward_std": 0.18773502707481385, + "rewards/reward_fn/mean": -0.5350000023841858, + "rewards/reward_fn/std": 0.18773502707481385, + "step": 1570, + "step_time": 0.5305851818062365 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 23.2, + "completions/max_terminated_length": 23.2, + "completions/mean_length": 15.4, + "completions/mean_terminated_length": 15.4, + "completions/min_length": 9.4, + "completions/min_terminated_length": 9.4, + "entropy": 0.7032105147838592, + "epoch": 1.6237113402061856, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 5.227942277481362e-08, + "loss": 0.010414773225784301, + "num_tokens": 404513.0, + "reward": -0.19199999868869783, + "reward_std": 0.2105449080467224, + "rewards/reward_fn/mean": -0.19199999868869783, + "rewards/reward_fn/std": 0.2105449080467224, + "step": 1575, + "step_time": 0.6994988144375384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.6, + "completions/max_terminated_length": 15.6, + "completions/mean_length": 9.15, + "completions/mean_terminated_length": 9.15, + "completions/min_length": 4.4, + "completions/min_terminated_length": 4.4, + "entropy": 1.0182935357093812, + "epoch": 1.6288659793814433, + "frac_reward_zero_std": 0.2, + "grad_norm": 19.25, + "learning_rate": 5.091104315866101e-08, + "loss": 0.27594072818756105, + "num_tokens": 405704.0, + "reward": -0.4875000029802322, + "reward_std": 0.36525567770004275, + "rewards/reward_fn/mean": -0.4875000029802322, + "rewards/reward_fn/std": 0.36525567770004275, + "step": 1580, + "step_time": 0.6210156822577119 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 25.2, + "completions/max_terminated_length": 25.2, + "completions/mean_length": 12.6, + "completions/mean_terminated_length": 12.6, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.6944295287132263, + "epoch": 1.634020618556701, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.955877725153604e-08, + "loss": 0.25443143844604493, + "num_tokens": 406960.0, + "reward": -0.4175000011920929, + "reward_std": 0.3398511052131653, + "rewards/reward_fn/mean": -0.4175000011920929, + "rewards/reward_fn/std": 0.3398511052131653, + "step": 1585, + "step_time": 0.7473113781772553 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 50.0, + "completions/max_terminated_length": 50.0, + "completions/mean_length": 21.4, + "completions/mean_terminated_length": 21.4, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 1.065959817171097, + "epoch": 1.6391752577319587, + "frac_reward_zero_std": 0.0, + "grad_norm": 22.375, + "learning_rate": 4.8222734502097655e-08, + "loss": 0.39557130336761476, + "num_tokens": 408396.0, + "reward": -0.5983333349227905, + "reward_std": 0.336662220954895, + "rewards/reward_fn/mean": -0.5983333349227905, + "rewards/reward_fn/std": 0.336662220954895, + "step": 1590, + "step_time": 1.2763810374774038 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18.4, + "completions/max_terminated_length": 18.4, + "completions/mean_length": 9.7, + "completions/mean_terminated_length": 9.7, + "completions/min_length": 4.8, + "completions/min_terminated_length": 4.8, + "entropy": 0.8178534567356109, + "epoch": 1.6443298969072164, + "frac_reward_zero_std": 0.2, + "grad_norm": 24.75, + "learning_rate": 4.690302304594754e-08, + "loss": 0.16044366359710693, + "num_tokens": 409602.0, + "reward": -0.23000001013278962, + "reward_std": 0.2588886618614197, + "rewards/reward_fn/mean": -0.23000001013278962, + "rewards/reward_fn/std": 0.2588886618614197, + "step": 1595, + "step_time": 0.628109234571457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.6, + "completions/max_terminated_length": 12.6, + "completions/mean_length": 9.7, + "completions/mean_terminated_length": 9.7, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 1.0267748922109603, + "epoch": 1.6494845360824741, + "frac_reward_zero_std": 0.6, + "grad_norm": 20.5, + "learning_rate": 4.559974969687755e-08, + "loss": 0.053169417381286624, + "num_tokens": 410820.0, + "reward": -0.5100000023841857, + "reward_std": 0.13773502707481383, + "rewards/reward_fn/mean": -0.5100000023841857, + "rewards/reward_fn/std": 0.13773502707481383, + "step": 1600, + "step_time": 0.5113442252390087 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.6, + "completions/max_terminated_length": 11.6, + "completions/mean_length": 7.3, + "completions/mean_terminated_length": 7.3, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "entropy": 0.8996017038822174, + "epoch": 1.6546391752577319, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 4.43130199382247e-08, + "loss": 0.21870932579040528, + "num_tokens": 411974.0, + "reward": -0.28500000238418577, + "reward_std": 0.4861552774906158, + "rewards/reward_fn/mean": -0.28500000238418577, + "rewards/reward_fn/std": 0.4861552774906158, + "step": 1605, + "step_time": 0.49553243080154064 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10.4, + "completions/max_terminated_length": 10.4, + "completions/mean_length": 7.25, + "completions/mean_terminated_length": 7.25, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "entropy": 0.769911852478981, + "epoch": 1.6597938144329896, + "frac_reward_zero_std": 0.4, + "grad_norm": 36.5, + "learning_rate": 4.304293791433358e-08, + "loss": 0.06664889454841613, + "num_tokens": 413139.0, + "reward": -0.325, + "reward_std": 0.25, + "rewards/reward_fn/mean": -0.325, + "rewards/reward_fn/std": 0.25, + "step": 1610, + "step_time": 0.5342640136368573 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 25.4, + "completions/max_terminated_length": 25.4, + "completions/mean_length": 15.65, + "completions/mean_terminated_length": 15.65, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "entropy": 1.128392007946968, + "epoch": 1.6649484536082473, + "frac_reward_zero_std": 0.4, + "grad_norm": 14.25, + "learning_rate": 4.1789606422127226e-08, + "loss": 0.07596928477287293, + "num_tokens": 414492.0, + "reward": -0.3168181836605072, + "reward_std": 0.353405237197876, + "rewards/reward_fn/mean": -0.3168181836605072, + "rewards/reward_fn/std": 0.353405225276947, + "step": 1615, + "step_time": 0.745933323726058 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 25.0, + "completions/max_terminated_length": 25.0, + "completions/mean_length": 17.4, + "completions/mean_terminated_length": 17.4, + "completions/min_length": 9.2, + "completions/min_terminated_length": 9.2, + "entropy": 0.685271966457367, + "epoch": 1.670103092783505, + "frac_reward_zero_std": 0.4, + "grad_norm": 7.78125, + "learning_rate": 4.0553126902787005e-08, + "loss": 0.08662630915641785, + "num_tokens": 415836.0, + "reward": -0.38583333492279054, + "reward_std": 0.3215514063835144, + "rewards/reward_fn/mean": -0.38583333492279054, + "rewards/reward_fn/std": 0.3215514302253723, + "step": 1620, + "step_time": 0.7388565543107688 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.4, + "completions/max_terminated_length": 30.4, + "completions/mean_length": 14.75, + "completions/mean_terminated_length": 14.75, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 1.1614010393619538, + "epoch": 1.675257731958763, + "frac_reward_zero_std": 0.0, + "grad_norm": 27.375, + "learning_rate": 3.933359943354228e-08, + "loss": 0.12167187929153442, + "num_tokens": 417167.0, + "reward": -0.2925000011920929, + "reward_std": 0.45460320115089414, + "rewards/reward_fn/mean": -0.2925000011920929, + "rewards/reward_fn/std": 0.45460320115089414, + "step": 1625, + "step_time": 0.903398260846734 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.6, + "completions/max_terminated_length": 16.6, + "completions/mean_length": 10.45, + "completions/mean_terminated_length": 10.45, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 1.0969272643327712, + "epoch": 1.6804123711340206, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 3.813112271957039e-08, + "loss": 0.14156380891799927, + "num_tokens": 418412.0, + "reward": -0.575, + "reward_std": 0.15, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.15, + "step": 1630, + "step_time": 0.5806521772406995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.8, + "completions/max_terminated_length": 12.8, + "completions/mean_length": 9.35, + "completions/mean_terminated_length": 9.35, + "completions/min_length": 5.4, + "completions/min_terminated_length": 5.4, + "entropy": 0.7530741691589355, + "epoch": 1.6855670103092784, + "frac_reward_zero_std": 0.4, + "grad_norm": 53.75, + "learning_rate": 3.6945794086007705e-08, + "loss": 0.0776121437549591, + "num_tokens": 419619.0, + "reward": -0.1850000023841858, + "reward_std": 0.36603364944458006, + "rewards/reward_fn/mean": -0.1850000023841858, + "rewards/reward_fn/std": 0.36603364944458006, + "step": 1635, + "step_time": 0.5080023482441902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 27.6, + "completions/max_terminated_length": 27.6, + "completions/mean_length": 17.8, + "completions/mean_terminated_length": 17.8, + "completions/min_length": 10.6, + "completions/min_terminated_length": 10.6, + "entropy": 0.9120415449142456, + "epoch": 1.690721649484536, + "frac_reward_zero_std": 0.4, + "grad_norm": 12.6875, + "learning_rate": 3.577770947007253e-08, + "loss": 0.12926599979400635, + "num_tokens": 421003.0, + "reward": -0.3950000047683716, + "reward_std": 0.29712470173835753, + "rewards/reward_fn/mean": -0.3950000047683716, + "rewards/reward_fn/std": 0.29712470173835753, + "step": 1640, + "step_time": 0.8596902235411108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10.6, + "completions/max_terminated_length": 10.6, + "completions/mean_length": 8.75, + "completions/mean_terminated_length": 8.75, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.6245357424020768, + "epoch": 1.6958762886597938, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 3.462696341329996e-08, + "loss": 0.09731593132019042, + "num_tokens": 422210.0, + "reward": -0.21000000238418579, + "reward_std": 0.43075350522994993, + "rewards/reward_fn/mean": -0.21000000238418579, + "rewards/reward_fn/std": 0.4307535171508789, + "step": 1645, + "step_time": 0.4654439380392432 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.8, + "completions/max_terminated_length": 21.8, + "completions/mean_length": 12.3, + "completions/mean_terminated_length": 12.3, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "entropy": 1.0625550508499146, + "epoch": 1.7010309278350515, + "frac_reward_zero_std": 0.4, + "grad_norm": 24.25, + "learning_rate": 3.349364905389032e-08, + "loss": 0.025496214628219604, + "num_tokens": 423480.0, + "reward": -0.4909090906381607, + "reward_std": 0.218181836605072, + "rewards/reward_fn/mean": -0.4909090906381607, + "rewards/reward_fn/std": 0.2181818127632141, + "step": 1650, + "step_time": 0.6790977258235216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.2, + "completions/max_terminated_length": 12.2, + "completions/mean_length": 9.45, + "completions/mean_terminated_length": 9.45, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.7438314110040665, + "epoch": 1.7061855670103094, + "frac_reward_zero_std": 0.4, + "grad_norm": 20.75, + "learning_rate": 3.237785811917049e-08, + "loss": 0.05355839133262634, + "num_tokens": 424693.0, + "reward": -0.5550000011920929, + "reward_std": 0.3228574156761169, + "rewards/reward_fn/mean": -0.5550000011920929, + "rewards/reward_fn/std": 0.3228574156761169, + "step": 1655, + "step_time": 0.5530511302873492 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.6, + "completions/max_terminated_length": 15.6, + "completions/mean_length": 10.15, + "completions/mean_terminated_length": 10.15, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 0.8672337532043457, + "epoch": 1.7113402061855671, + "frac_reward_zero_std": 0.6, + "grad_norm": 17.0, + "learning_rate": 3.127968091816985e-08, + "loss": 0.12111912965774536, + "num_tokens": 425920.0, + "reward": -0.575, + "reward_std": 0.10773502588272095, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.10773502588272095, + "step": 1660, + "step_time": 0.5605017002671957 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.8, + "completions/max_terminated_length": 14.8, + "completions/mean_length": 10.65, + "completions/mean_terminated_length": 10.65, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.7234238475561142, + "epoch": 1.7164948453608249, + "frac_reward_zero_std": 0.8, + "grad_norm": 18.125, + "learning_rate": 3.0199206334310945e-08, + "loss": -0.043664944171905515, + "num_tokens": 427157.0, + "reward": -0.10833333134651184, + "reward_std": 0.10754843950271606, + "rewards/reward_fn/mean": -0.10833333134651184, + "rewards/reward_fn/std": 0.10754843950271606, + "step": 1665, + "step_time": 0.5437676094472408 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 38.6, + "completions/max_terminated_length": 38.6, + "completions/mean_length": 18.1, + "completions/mean_terminated_length": 18.1, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "entropy": 1.1868691265583038, + "epoch": 1.7216494845360826, + "frac_reward_zero_std": 0.2, + "grad_norm": 15.5, + "learning_rate": 2.913652181821552e-08, + "loss": 0.12590534687042237, + "num_tokens": 428543.0, + "reward": -0.3076666623353958, + "reward_std": 0.4518565535545349, + "rewards/reward_fn/mean": -0.3076666623353958, + "rewards/reward_fn/std": 0.4518565535545349, + "step": 1670, + "step_time": 1.0609127783216536 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.0, + "completions/max_terminated_length": 20.0, + "completions/mean_length": 11.25, + "completions/mean_terminated_length": 11.25, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 1.1474024474620819, + "epoch": 1.7268041237113403, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 2.8091713380626493e-08, + "loss": -0.016077935695648193, + "num_tokens": 429788.0, + "reward": -0.6, + "reward_std": 0.15773502588272095, + "rewards/reward_fn/mean": -0.6, + "rewards/reward_fn/std": 0.15773502588272095, + "step": 1675, + "step_time": 0.6428652757778763 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.8, + "completions/max_terminated_length": 13.8, + "completions/mean_length": 11.05, + "completions/mean_terminated_length": 11.05, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "entropy": 0.8905054450035095, + "epoch": 1.731958762886598, + "frac_reward_zero_std": 0.0, + "grad_norm": 24.25, + "learning_rate": 2.7064865585446433e-08, + "loss": 0.0036585628986358643, + "num_tokens": 431053.0, + "reward": -0.28500000536441805, + "reward_std": 0.4567333281040192, + "rewards/reward_fn/mean": -0.28500000536441805, + "rewards/reward_fn/std": 0.4567333281040192, + "step": 1680, + "step_time": 0.5253810392692685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 40.0, + "completions/max_terminated_length": 40.0, + "completions/mean_length": 16.8, + "completions/mean_terminated_length": 16.8, + "completions/min_length": 7.6, + "completions/min_terminated_length": 7.6, + "entropy": 0.8407724499702454, + "epoch": 1.7371134020618557, + "frac_reward_zero_std": 0.6, + "grad_norm": 6.78125, + "learning_rate": 2.6056061542893222e-08, + "loss": -0.14318621158599854, + "num_tokens": 432389.0, + "reward": -0.4699999988079071, + "reward_std": 0.15011107325553893, + "rewards/reward_fn/mean": -0.4699999988079071, + "rewards/reward_fn/std": 0.15011107325553893, + "step": 1685, + "step_time": 1.0814969715662301 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 25.8, + "completions/max_terminated_length": 25.8, + "completions/mean_length": 16.65, + "completions/mean_terminated_length": 16.65, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "entropy": 0.9036176294088364, + "epoch": 1.7422680412371134, + "frac_reward_zero_std": 0.2, + "grad_norm": 17.875, + "learning_rate": 2.5065382902773435e-08, + "loss": 0.17168867588043213, + "num_tokens": 433758.0, + "reward": -0.5699999988079071, + "reward_std": 0.2934023141860962, + "rewards/reward_fn/mean": -0.5699999988079071, + "rewards/reward_fn/std": 0.29340232610702516, + "step": 1690, + "step_time": 0.7546438516117633 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.8, + "completions/max_terminated_length": 17.8, + "completions/mean_length": 13.45, + "completions/mean_terminated_length": 13.45, + "completions/min_length": 9.6, + "completions/min_terminated_length": 9.6, + "entropy": 0.9092921882867813, + "epoch": 1.7474226804123711, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 2.409290984787371e-08, + "loss": -0.08626242876052856, + "num_tokens": 435043.0, + "reward": -0.65, + "reward_std": 0.15773502588272095, + "rewards/reward_fn/mean": -0.65, + "rewards/reward_fn/std": 0.15773502588272095, + "step": 1695, + "step_time": 0.6577458389103412 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 21.4, + "completions/max_terminated_length": 21.4, + "completions/mean_length": 14.8, + "completions/mean_terminated_length": 14.8, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.6884515404701232, + "epoch": 1.7525773195876289, + "frac_reward_zero_std": 0.6, + "grad_norm": 13.9375, + "learning_rate": 2.313872108747092e-08, + "loss": -0.023072260618209838, + "num_tokens": 436375.0, + "reward": -0.45, + "reward_std": 0.2, + "rewards/reward_fn/mean": -0.45, + "rewards/reward_fn/std": 0.2, + "step": 1700, + "step_time": 0.6705449435859918 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.8, + "completions/max_terminated_length": 19.8, + "completions/mean_length": 9.65, + "completions/mean_terminated_length": 9.65, + "completions/min_length": 4.2, + "completions/min_terminated_length": 4.2, + "entropy": 1.0143103897571564, + "epoch": 1.7577319587628866, + "frac_reward_zero_std": 0.4, + "grad_norm": 9.125, + "learning_rate": 2.220289385096194e-08, + "loss": 0.18412907123565675, + "num_tokens": 437580.0, + "reward": -0.35999999940395355, + "reward_std": 0.3762573838233948, + "rewards/reward_fn/mean": -0.35999999940395355, + "rewards/reward_fn/std": 0.3762573838233948, + "step": 1705, + "step_time": 0.7051920691505075 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.6, + "completions/max_terminated_length": 14.6, + "completions/mean_length": 10.6, + "completions/mean_terminated_length": 10.6, + "completions/min_length": 7.4, + "completions/min_terminated_length": 7.4, + "entropy": 0.6810103952884674, + "epoch": 1.7628865979381443, + "frac_reward_zero_std": 0.4, + "grad_norm": 17.0, + "learning_rate": 2.1285503881612627e-08, + "loss": -0.005234807729721069, + "num_tokens": 438788.0, + "reward": 0.1824999988079071, + "reward_std": 0.2471906840801239, + "rewards/reward_fn/mean": 0.1824999988079071, + "rewards/reward_fn/std": 0.2471906840801239, + "step": 1710, + "step_time": 0.5405286100693047 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.8, + "completions/max_terminated_length": 15.8, + "completions/mean_length": 11.0, + "completions/mean_terminated_length": 11.0, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.78079913854599, + "epoch": 1.768041237113402, + "frac_reward_zero_std": 0.4, + "grad_norm": 15.375, + "learning_rate": 2.0386625430427434e-08, + "loss": 0.11818069219589233, + "num_tokens": 440024.0, + "reward": -0.5425000011920929, + "reward_std": 0.24758613109588623, + "rewards/reward_fn/mean": -0.5425000011920929, + "rewards/reward_fn/std": 0.24758613109588623, + "step": 1715, + "step_time": 0.6267786368727684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.4, + "completions/max_terminated_length": 14.4, + "completions/mean_length": 10.3, + "completions/mean_terminated_length": 10.3, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.9158282101154327, + "epoch": 1.7731958762886597, + "frac_reward_zero_std": 0.2, + "grad_norm": 16.875, + "learning_rate": 1.9506331250139896e-08, + "loss": 0.15086617469787597, + "num_tokens": 441246.0, + "reward": -0.4775000035762787, + "reward_std": 0.37412620186805723, + "rewards/reward_fn/mean": -0.4775000035762787, + "rewards/reward_fn/std": 0.37412620186805723, + "step": 1720, + "step_time": 0.5440597709268331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.4, + "completions/max_terminated_length": 12.4, + "completions/mean_length": 8.6, + "completions/mean_terminated_length": 8.6, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 0.7902150690555573, + "epoch": 1.7783505154639174, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 1.8644692589323967e-08, + "loss": 0.05799698829650879, + "num_tokens": 442418.0, + "reward": -0.44666666388511655, + "reward_std": 0.20370726585388182, + "rewards/reward_fn/mean": -0.44666666388511655, + "rewards/reward_fn/std": 0.2037072777748108, + "step": 1725, + "step_time": 0.5111340196803212 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.4, + "completions/max_terminated_length": 16.4, + "completions/mean_length": 10.65, + "completions/mean_terminated_length": 10.65, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.6754415512084961, + "epoch": 1.7835051546391751, + "frac_reward_zero_std": 0.4, + "grad_norm": 21.625, + "learning_rate": 1.7801779186627648e-08, + "loss": 0.1614132523536682, + "num_tokens": 443643.0, + "reward": -0.45499999821186066, + "reward_std": 0.22999999523162842, + "rewards/reward_fn/mean": -0.45499999821186066, + "rewards/reward_fn/std": 0.23000000715255736, + "step": 1730, + "step_time": 0.6408032779581845 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.6, + "completions/max_terminated_length": 30.6, + "completions/mean_length": 17.55, + "completions/mean_terminated_length": 17.55, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.8354218006134033, + "epoch": 1.7886597938144329, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 1.697765926512823e-08, + "loss": 0.02780742049217224, + "num_tokens": 445034.0, + "reward": -0.5012500047683716, + "reward_std": 0.1390668272972107, + "rewards/reward_fn/mean": -0.5012500047683716, + "rewards/reward_fn/std": 0.1390668272972107, + "step": 1735, + "step_time": 0.8523523797281086 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.6, + "completions/max_terminated_length": 11.6, + "completions/mean_length": 8.1, + "completions/mean_terminated_length": 8.1, + "completions/min_length": 5.6, + "completions/min_terminated_length": 5.6, + "entropy": 0.8720086574554443, + "epoch": 1.7938144329896906, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 1.617239952681082e-08, + "loss": 0.12526261806488037, + "num_tokens": 446220.0, + "reward": -0.3550000011920929, + "reward_std": 0.32516661286354065, + "rewards/reward_fn/mean": -0.3550000011920929, + "rewards/reward_fn/std": 0.32516661286354065, + "step": 1740, + "step_time": 0.5580897599458694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.6, + "completions/max_terminated_length": 15.6, + "completions/mean_length": 12.1, + "completions/mean_terminated_length": 12.1, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 0.9813948631286621, + "epoch": 1.7989690721649485, + "frac_reward_zero_std": 0.4, + "grad_norm": 16.5, + "learning_rate": 1.538606514716939e-08, + "loss": 0.0448380708694458, + "num_tokens": 447486.0, + "reward": -0.4350000023841858, + "reward_std": 0.3048511028289795, + "rewards/reward_fn/mean": -0.4350000023841858, + "rewards/reward_fn/std": 0.3048511028289795, + "step": 1745, + "step_time": 0.5642483637668192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 25.6, + "completions/max_terminated_length": 25.6, + "completions/mean_length": 16.5, + "completions/mean_terminated_length": 16.5, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "entropy": 0.971468162536621, + "epoch": 1.8041237113402062, + "frac_reward_zero_std": 0.2, + "grad_norm": 12.1875, + "learning_rate": 1.4618719769931843e-08, + "loss": 0.2018949031829834, + "num_tokens": 448864.0, + "reward": -0.5600000023841858, + "reward_std": 0.25754843950271605, + "rewards/reward_fn/mean": -0.5600000023841858, + "rewards/reward_fn/std": 0.25754843950271605, + "step": 1750, + "step_time": 0.8179386577568948 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 37.4, + "completions/max_terminated_length": 37.4, + "completions/mean_length": 18.5, + "completions/mean_terminated_length": 18.5, + "completions/min_length": 8.8, + "completions/min_terminated_length": 8.8, + "entropy": 0.6841971307992936, + "epoch": 1.809278350515464, + "frac_reward_zero_std": 0.4, + "grad_norm": 16.25, + "learning_rate": 1.3870425501908672e-08, + "loss": 0.1817900061607361, + "num_tokens": 450234.0, + "reward": -0.4199999988079071, + "reward_std": 0.21237728595733643, + "rewards/reward_fn/mean": -0.4199999988079071, + "rewards/reward_fn/std": 0.21237727403640747, + "step": 1755, + "step_time": 1.0719246904365718 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20.4, + "completions/max_terminated_length": 20.4, + "completions/mean_length": 12.0, + "completions/mean_terminated_length": 12.0, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.6847515821456909, + "epoch": 1.8144329896907216, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 1.314124290796656e-08, + "loss": 0.15299254655838013, + "num_tokens": 451514.0, + "reward": -0.4262499988079071, + "reward_std": 0.32143821716308596, + "rewards/reward_fn/mean": -0.4262499988079071, + "rewards/reward_fn/std": 0.32143821716308596, + "step": 1760, + "step_time": 0.6563241709955037 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 23.4, + "completions/max_terminated_length": 23.4, + "completions/mean_length": 12.65, + "completions/mean_terminated_length": 12.65, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.9309171110391616, + "epoch": 1.8195876288659794, + "frac_reward_zero_std": 0.0, + "grad_norm": 14.875, + "learning_rate": 1.2431231006126003e-08, + "loss": 0.23779988288879395, + "num_tokens": 452787.0, + "reward": -0.2858333379030228, + "reward_std": 0.6213802993297577, + "rewards/reward_fn/mean": -0.2858333379030228, + "rewards/reward_fn/std": 0.6213802993297577, + "step": 1765, + "step_time": 0.788817260786891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 22.8, + "completions/max_terminated_length": 22.8, + "completions/mean_length": 14.3, + "completions/mean_terminated_length": 14.3, + "completions/min_length": 9.8, + "completions/min_terminated_length": 9.8, + "entropy": 0.655038520693779, + "epoch": 1.824742268041237, + "frac_reward_zero_std": 0.2, + "grad_norm": 8.8125, + "learning_rate": 1.174044726278478e-08, + "loss": 0.15090932846069335, + "num_tokens": 454117.0, + "reward": -0.1236666738986969, + "reward_std": 0.5083053946495056, + "rewards/reward_fn/mean": -0.1236666738986969, + "rewards/reward_fn/std": 0.5083053946495056, + "step": 1770, + "step_time": 0.7027385210618377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.6, + "completions/max_terminated_length": 30.6, + "completions/mean_length": 16.4, + "completions/mean_terminated_length": 16.4, + "completions/min_length": 7.8, + "completions/min_terminated_length": 7.8, + "entropy": 0.8989251792430878, + "epoch": 1.829896907216495, + "frac_reward_zero_std": 0.2, + "grad_norm": 17.75, + "learning_rate": 1.1068947588066812e-08, + "loss": 0.07146713733673096, + "num_tokens": 455481.0, + "reward": -0.4300000011920929, + "reward_std": 0.3527463436126709, + "rewards/reward_fn/mean": -0.4300000011920929, + "rewards/reward_fn/std": 0.3527463436126709, + "step": 1775, + "step_time": 0.9245791801251471 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 22.6, + "completions/max_terminated_length": 22.6, + "completions/mean_length": 13.4, + "completions/mean_terminated_length": 13.4, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "entropy": 1.0093101501464843, + "epoch": 1.8350515463917527, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.25, + "learning_rate": 1.0416786331296861e-08, + "loss": 0.11369644403457642, + "num_tokens": 456769.0, + "reward": -0.33700000643730166, + "reward_std": 0.5056420981884002, + "rewards/reward_fn/mean": -0.33700000643730166, + "rewards/reward_fn/std": 0.5056420981884002, + "step": 1780, + "step_time": 0.7672192833386362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.0, + "completions/max_terminated_length": 14.0, + "completions/mean_length": 10.9, + "completions/mean_terminated_length": 10.9, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "entropy": 0.7461501806974411, + "epoch": 1.8402061855670104, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 9.784016276601609e-09, + "loss": 0.04938405454158783, + "num_tokens": 457979.0, + "reward": -0.3700000047683716, + "reward_std": 0.15847426652908325, + "rewards/reward_fn/mean": -0.3700000047683716, + "rewards/reward_fn/std": 0.15847426652908325, + "step": 1785, + "step_time": 0.5356096717528999 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 22.6, + "completions/max_terminated_length": 22.6, + "completions/mean_length": 14.65, + "completions/mean_terminated_length": 14.65, + "completions/min_length": 8.8, + "completions/min_terminated_length": 8.8, + "entropy": 0.9771642625331879, + "epoch": 1.8453608247422681, + "frac_reward_zero_std": 0.2, + "grad_norm": 8.875, + "learning_rate": 9.170688638637653e-09, + "loss": 0.12961161136627197, + "num_tokens": 459316.0, + "reward": -0.4425000011920929, + "reward_std": 0.3898511052131653, + "rewards/reward_fn/mean": -0.4425000011920929, + "rewards/reward_fn/std": 0.3898511052131653, + "step": 1790, + "step_time": 0.7050401211716235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.8, + "completions/max_terminated_length": 19.8, + "completions/mean_length": 13.75, + "completions/mean_terminated_length": 13.75, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 1.1843935072422027, + "epoch": 1.8505154639175259, + "frac_reward_zero_std": 0.4, + "grad_norm": 14.5, + "learning_rate": 8.576853058446076e-09, + "loss": 0.09116497039794921, + "num_tokens": 460615.0, + "reward": -0.49600000083446505, + "reward_std": 0.20799999237060546, + "rewards/reward_fn/mean": -0.49600000083446505, + "rewards/reward_fn/std": 0.20799999237060546, + "step": 1795, + "step_time": 0.7088314037770033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.2, + "completions/max_terminated_length": 11.2, + "completions/mean_length": 8.35, + "completions/mean_terminated_length": 8.35, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 1.0617978870868683, + "epoch": 1.8556701030927836, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 8.002557599434801e-09, + "loss": -0.02811015844345093, + "num_tokens": 461794.0, + "reward": -0.1716666638851166, + "reward_std": 0.1277350217103958, + "rewards/reward_fn/mean": -0.1716666638851166, + "rewards/reward_fn/std": 0.12773502469062806, + "step": 1800, + "step_time": 0.48176195118576287 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.8, + "completions/max_terminated_length": 15.8, + "completions/mean_length": 10.1, + "completions/mean_terminated_length": 10.1, + "completions/min_length": 6.6, + "completions/min_terminated_length": 6.6, + "entropy": 0.7022380590438843, + "epoch": 1.8608247422680413, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 7.447848743488555e-09, + "loss": 0.13740893602371215, + "num_tokens": 463028.0, + "reward": -0.3600000023841858, + "reward_std": 0.23859126567840577, + "rewards/reward_fn/mean": -0.3600000023841858, + "rewards/reward_fn/std": 0.2385912775993347, + "step": 1805, + "step_time": 0.6413388851098716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.6, + "completions/max_terminated_length": 15.6, + "completions/mean_length": 12.85, + "completions/mean_terminated_length": 12.85, + "completions/min_length": 9.8, + "completions/min_terminated_length": 9.8, + "entropy": 0.5699267834424973, + "epoch": 1.865979381443299, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 6.912771387206473e-09, + "loss": 0.11363432407379151, + "num_tokens": 464317.0, + "reward": -0.5100000023841857, + "reward_std": 0.1800000011920929, + "rewards/reward_fn/mean": -0.5100000023841857, + "rewards/reward_fn/std": 0.1800000011920929, + "step": 1810, + "step_time": 0.5692544280551374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 48.4, + "completions/max_terminated_length": 48.4, + "completions/mean_length": 19.45, + "completions/mean_terminated_length": 19.45, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 0.8490649104118347, + "epoch": 1.8711340206185567, + "frac_reward_zero_std": 0.0, + "grad_norm": 17.25, + "learning_rate": 6.397368838268496e-09, + "loss": 0.26308455467224123, + "num_tokens": 465726.0, + "reward": -0.4316666662693024, + "reward_std": 0.3493190586566925, + "rewards/reward_fn/mean": -0.4316666662693024, + "rewards/reward_fn/std": 0.3493190586566925, + "step": 1815, + "step_time": 1.2811257994733751 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10.4, + "completions/max_terminated_length": 10.4, + "completions/mean_length": 8.2, + "completions/mean_terminated_length": 8.2, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 0.7323252975940704, + "epoch": 1.8762886597938144, + "frac_reward_zero_std": 0.6, + "grad_norm": 29.25, + "learning_rate": 5.901682811930181e-09, + "loss": 0.09926102161407471, + "num_tokens": 466898.0, + "reward": -0.55, + "reward_std": 0.1, + "rewards/reward_fn/mean": -0.55, + "rewards/reward_fn/std": 0.1, + "step": 1820, + "step_time": 0.5264862560667097 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 42.0, + "completions/max_terminated_length": 42.0, + "completions/mean_length": 19.55, + "completions/mean_terminated_length": 19.55, + "completions/min_length": 8.4, + "completions/min_terminated_length": 8.4, + "entropy": 0.744831657409668, + "epoch": 1.8814432989690721, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 5.425753427646257e-09, + "loss": 0.16098196506500245, + "num_tokens": 468313.0, + "reward": -0.38875000178813934, + "reward_std": 0.3985625863075256, + "rewards/reward_fn/mean": -0.38875000178813934, + "rewards/reward_fn/std": 0.3985625863075256, + "step": 1825, + "step_time": 1.0875293998979032 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.6, + "completions/max_terminated_length": 16.6, + "completions/mean_length": 11.25, + "completions/mean_terminated_length": 11.25, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 0.9310593396425247, + "epoch": 1.8865979381443299, + "frac_reward_zero_std": 0.2, + "grad_norm": 27.25, + "learning_rate": 4.9696192058236164e-09, + "loss": 0.09600334167480469, + "num_tokens": 469550.0, + "reward": -0.5350000023841858, + "reward_std": 0.33220337629318236, + "rewards/reward_fn/mean": -0.5350000023841858, + "rewards/reward_fn/std": 0.33220337629318236, + "step": 1830, + "step_time": 0.6508773274719715 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12.8, + "completions/max_terminated_length": 12.8, + "completions/mean_length": 8.7, + "completions/mean_terminated_length": 8.7, + "completions/min_length": 5.8, + "completions/min_terminated_length": 5.8, + "entropy": 0.6754906833171844, + "epoch": 1.8917525773195876, + "frac_reward_zero_std": 0.2, + "grad_norm": 23.125, + "learning_rate": 4.53331706470339e-09, + "loss": 0.14865694046020508, + "num_tokens": 470724.0, + "reward": -0.4199999988079071, + "reward_std": 0.3779322624206543, + "rewards/reward_fn/mean": -0.4199999988079071, + "rewards/reward_fn/std": 0.3779322743415833, + "step": 1835, + "step_time": 0.518319695442915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16.0, + "completions/max_terminated_length": 16.0, + "completions/mean_length": 9.2, + "completions/mean_terminated_length": 9.2, + "completions/min_length": 5.4, + "completions/min_terminated_length": 5.4, + "entropy": 1.0082854956388474, + "epoch": 1.8969072164948453, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 4.116882317373116e-09, + "loss": -0.042311012744903564, + "num_tokens": 471920.0, + "reward": -0.3195833295583725, + "reward_std": 0.4602280676364899, + "rewards/reward_fn/mean": -0.3195833295583725, + "rewards/reward_fn/std": 0.46022807359695433, + "step": 1840, + "step_time": 0.6366292778402567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.2, + "completions/max_terminated_length": 17.2, + "completions/mean_length": 10.65, + "completions/mean_terminated_length": 10.65, + "completions/min_length": 7.4, + "completions/min_terminated_length": 7.4, + "entropy": 0.9684360563755036, + "epoch": 1.902061855670103, + "frac_reward_zero_std": 0.4, + "grad_norm": 12.6875, + "learning_rate": 3.720348668908385e-09, + "loss": 0.10864226818084717, + "num_tokens": 473149.0, + "reward": -0.34166666567325593, + "reward_std": 0.25666666626930235, + "rewards/reward_fn/mean": -0.34166666567325593, + "rewards/reward_fn/std": 0.25666666626930235, + "step": 1845, + "step_time": 0.6000545247457921 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 8.6, + "completions/max_terminated_length": 8.6, + "completions/mean_length": 7.75, + "completions/mean_terminated_length": 7.75, + "completions/min_length": 6.2, + "completions/min_terminated_length": 6.2, + "entropy": 0.7907846450805665, + "epoch": 1.9072164948453607, + "frac_reward_zero_std": 0.6, + "grad_norm": 11.875, + "learning_rate": 3.343748213645026e-09, + "loss": -0.018713557720184328, + "num_tokens": 474300.0, + "reward": -0.3925000011920929, + "reward_std": 0.20564382076263427, + "rewards/reward_fn/mean": -0.3925000011920929, + "rewards/reward_fn/std": 0.20564383268356323, + "step": 1850, + "step_time": 0.48386304890736936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15.8, + "completions/max_terminated_length": 15.8, + "completions/mean_length": 10.35, + "completions/mean_terminated_length": 10.35, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 1.116754847764969, + "epoch": 1.9123711340206184, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 2.9871114325812907e-09, + "loss": 0.028201365470886232, + "num_tokens": 475499.0, + "reward": -0.425, + "reward_std": 0.15, + "rewards/reward_fn/mean": -0.425, + "rewards/reward_fn/std": 0.15, + "step": 1855, + "step_time": 0.586841206252575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.6, + "completions/max_terminated_length": 14.6, + "completions/mean_length": 11.35, + "completions/mean_terminated_length": 11.35, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 0.9712529420852661, + "epoch": 1.9175257731958761, + "frac_reward_zero_std": 0.4, + "grad_norm": 13.1875, + "learning_rate": 2.6504671909109988e-09, + "loss": 0.055383771657943726, + "num_tokens": 476762.0, + "reward": -0.6, + "reward_std": 0.15773502588272095, + "rewards/reward_fn/mean": -0.6, + "rewards/reward_fn/std": 0.15773502588272095, + "step": 1860, + "step_time": 0.5633186590857804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 36.6, + "completions/max_terminated_length": 36.6, + "completions/mean_length": 15.85, + "completions/mean_terminated_length": 15.85, + "completions/min_length": 6.4, + "completions/min_terminated_length": 6.4, + "entropy": 1.062751868367195, + "epoch": 1.922680412371134, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 2.333842735687097e-09, + "loss": 0.10891166925430298, + "num_tokens": 478095.0, + "reward": -0.525, + "reward_std": 0.05, + "rewards/reward_fn/mean": -0.525, + "rewards/reward_fn/std": 0.05, + "step": 1865, + "step_time": 1.0668349410407245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 23.8, + "completions/max_terminated_length": 23.8, + "completions/mean_length": 16.4, + "completions/mean_terminated_length": 16.4, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "entropy": 1.1400189995765686, + "epoch": 1.9278350515463918, + "frac_reward_zero_std": 0.4, + "grad_norm": 10.5, + "learning_rate": 2.0372636936165368e-09, + "loss": 0.017370551824569702, + "num_tokens": 479451.0, + "reward": -0.4213888913393021, + "reward_std": 0.3149572372436523, + "rewards/reward_fn/mean": -0.4213888913393021, + "rewards/reward_fn/std": 0.3149572491645813, + "step": 1870, + "step_time": 0.8135622454807162 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.2, + "completions/max_terminated_length": 17.2, + "completions/mean_length": 12.35, + "completions/mean_terminated_length": 12.35, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 1.0274198472499847, + "epoch": 1.9329896907216495, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 1.7607540689859035e-09, + "loss": 0.022668755054473876, + "num_tokens": 480746.0, + "reward": -0.5350000023841858, + "reward_std": 0.18773502707481385, + "rewards/reward_fn/mean": -0.5350000023841858, + "rewards/reward_fn/std": 0.18773502707481385, + "step": 1875, + "step_time": 0.6297862586565316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13.2, + "completions/max_terminated_length": 13.2, + "completions/mean_length": 9.95, + "completions/mean_terminated_length": 9.95, + "completions/min_length": 7.4, + "completions/min_terminated_length": 7.4, + "entropy": 0.7384592443704605, + "epoch": 1.9381443298969072, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 1.5043362417187787e-09, + "loss": 0.11721787452697754, + "num_tokens": 481953.0, + "reward": -0.2975000083446503, + "reward_std": 0.3525484263896942, + "rewards/reward_fn/mean": -0.2975000083446503, + "rewards/reward_fn/std": 0.35254843831062316, + "step": 1880, + "step_time": 0.6035839405842125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19.2, + "completions/max_terminated_length": 19.2, + "completions/mean_length": 12.75, + "completions/mean_terminated_length": 12.75, + "completions/min_length": 6.8, + "completions/min_terminated_length": 6.8, + "entropy": 0.9961306691169739, + "epoch": 1.943298969072165, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 1.268030965564243e-09, + "loss": 0.061715888977050784, + "num_tokens": 483212.0, + "reward": -0.4899999976158142, + "reward_std": 0.1, + "rewards/reward_fn/mean": -0.4899999976158142, + "rewards/reward_fn/std": 0.1, + "step": 1885, + "step_time": 0.6409509646706283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11.8, + "completions/max_terminated_length": 11.8, + "completions/mean_length": 9.45, + "completions/mean_terminated_length": 9.45, + "completions/min_length": 7.2, + "completions/min_terminated_length": 7.2, + "entropy": 0.6782282710075378, + "epoch": 1.9484536082474226, + "frac_reward_zero_std": 0.2, + "grad_norm": 11.6875, + "learning_rate": 1.0518573664172192e-09, + "loss": 0.09830584526062011, + "num_tokens": 484409.0, + "reward": -0.4433333337306976, + "reward_std": 0.3213032424449921, + "rewards/reward_fn/mean": -0.4433333337306976, + "rewards/reward_fn/std": 0.3213032424449921, + "step": 1890, + "step_time": 0.5699493732303381 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14.6, + "completions/max_terminated_length": 14.6, + "completions/mean_length": 11.9, + "completions/mean_terminated_length": 11.9, + "completions/min_length": 8.6, + "completions/min_terminated_length": 8.6, + "entropy": 0.7531616747379303, + "epoch": 1.9536082474226806, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 8.558329407703513e-10, + "loss": 0.040635806322097776, + "num_tokens": 485675.0, + "reward": -0.30833333134651186, + "reward_std": 0.36525357961654664, + "rewards/reward_fn/mean": -0.30833333134651186, + "rewards/reward_fn/std": 0.3652535915374756, + "step": 1895, + "step_time": 0.5324823048897087 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17.6, + "completions/max_terminated_length": 17.6, + "completions/mean_length": 12.4, + "completions/mean_terminated_length": 12.4, + "completions/min_length": 8.2, + "completions/min_terminated_length": 8.2, + "entropy": 0.8260966598987579, + "epoch": 1.9587628865979383, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.0, + "learning_rate": 6.799735542979967e-10, + "loss": 0.06493161916732788, + "num_tokens": 486963.0, + "reward": -0.575, + "reward_std": 0.30773502588272095, + "rewards/reward_fn/mean": -0.575, + "rewards/reward_fn/std": 0.30773502588272095, + "step": 1900, + "step_time": 0.6176068928092718 + } + ], + "logging_steps": 5, + "max_steps": 1940, + "num_input_tokens_seen": 486963, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/last-checkpoint/training_args.bin b/last-checkpoint/training_args.bin new file mode 100644 index 0000000..ae7c476 --- /dev/null +++ b/last-checkpoint/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50850549de1715f3e4dd20d7aca1656c274ffb64c89eb9126d58c44d67a8b149 +size 7057 diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..02c1585 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d57536ba92255d37237213e5d744fb3d486140a30ece47b42aeba17ee149bad2 +size 988097824 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..e4fb8c3 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 32768, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..ae7c476 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50850549de1715f3e4dd20d7aca1656c274ffb64c89eb9126d58c44d67a8b149 +size 7057