From a53def3b08248af132fb32429a2909082571aa7b Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 16 Jun 2026 08:06:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: pvs333/supergames-grpo Source: Original Platform --- .gitattributes | 46 + README.md | 67 + chat_template.jinja | 54 + checkpoint-100/chat_template.jinja | 54 + checkpoint-100/config.json | 61 + checkpoint-100/generation_config.json | 13 + checkpoint-100/model.safetensors | 3 + checkpoint-100/optimizer.pt | 3 + checkpoint-100/rng_state.pth | 3 + checkpoint-100/scheduler.pt | 3 + checkpoint-100/tokenizer.json | 3 + checkpoint-100/tokenizer_config.json | 30 + checkpoint-100/trainer_state.json | 2934 +++++++++++++ checkpoint-100/training_args.bin | 3 + checkpoint-120/chat_template.jinja | 54 + checkpoint-120/config.json | 61 + checkpoint-120/generation_config.json | 13 + checkpoint-120/model.safetensors | 3 + checkpoint-120/optimizer.pt | 3 + checkpoint-120/rng_state.pth | 3 + checkpoint-120/scheduler.pt | 3 + checkpoint-120/tokenizer.json | 3 + checkpoint-120/tokenizer_config.json | 30 + checkpoint-120/trainer_state.json | 3514 +++++++++++++++ checkpoint-120/training_args.bin | 3 + checkpoint-140/chat_template.jinja | 54 + checkpoint-140/config.json | 61 + checkpoint-140/generation_config.json | 13 + checkpoint-140/model.safetensors | 3 + checkpoint-140/optimizer.pt | 3 + checkpoint-140/rng_state.pth | 3 + checkpoint-140/scheduler.pt | 3 + checkpoint-140/tokenizer.json | 3 + checkpoint-140/tokenizer_config.json | 30 + checkpoint-140/trainer_state.json | 4094 +++++++++++++++++ checkpoint-140/training_args.bin | 3 + checkpoint-160/chat_template.jinja | 54 + checkpoint-160/config.json | 61 + checkpoint-160/generation_config.json | 13 + checkpoint-160/model.safetensors | 3 + checkpoint-160/optimizer.pt | 3 + checkpoint-160/rng_state.pth | 3 + checkpoint-160/scheduler.pt | 3 + checkpoint-160/tokenizer.json | 3 + checkpoint-160/tokenizer_config.json | 30 + checkpoint-160/trainer_state.json | 4674 ++++++++++++++++++++ checkpoint-160/training_args.bin | 3 + checkpoint-180/chat_template.jinja | 54 + checkpoint-180/config.json | 61 + checkpoint-180/generation_config.json | 13 + checkpoint-180/model.safetensors | 3 + checkpoint-180/optimizer.pt | 3 + checkpoint-180/rng_state.pth | 3 + checkpoint-180/scheduler.pt | 3 + checkpoint-180/tokenizer.json | 3 + checkpoint-180/tokenizer_config.json | 30 + checkpoint-180/trainer_state.json | 5254 ++++++++++++++++++++++ checkpoint-180/training_args.bin | 3 + checkpoint-20/chat_template.jinja | 54 + checkpoint-20/config.json | 61 + checkpoint-20/generation_config.json | 13 + checkpoint-20/model.safetensors | 3 + checkpoint-20/optimizer.pt | 3 + checkpoint-20/rng_state.pth | 3 + checkpoint-20/scheduler.pt | 3 + checkpoint-20/tokenizer.json | 3 + checkpoint-20/tokenizer_config.json | 30 + checkpoint-20/trainer_state.json | 614 +++ checkpoint-20/training_args.bin | 3 + checkpoint-200/chat_template.jinja | 54 + checkpoint-200/config.json | 61 + checkpoint-200/generation_config.json | 13 + checkpoint-200/model.safetensors | 3 + checkpoint-200/optimizer.pt | 3 + checkpoint-200/rng_state.pth | 3 + checkpoint-200/scheduler.pt | 3 + checkpoint-200/tokenizer.json | 3 + checkpoint-200/tokenizer_config.json | 30 + checkpoint-200/trainer_state.json | 5834 +++++++++++++++++++++++++ checkpoint-200/training_args.bin | 3 + checkpoint-40/chat_template.jinja | 54 + checkpoint-40/config.json | 61 + checkpoint-40/generation_config.json | 13 + checkpoint-40/model.safetensors | 3 + checkpoint-40/optimizer.pt | 3 + checkpoint-40/rng_state.pth | 3 + checkpoint-40/scheduler.pt | 3 + checkpoint-40/tokenizer.json | 3 + checkpoint-40/tokenizer_config.json | 30 + checkpoint-40/trainer_state.json | 1194 +++++ checkpoint-40/training_args.bin | 3 + checkpoint-60/chat_template.jinja | 54 + checkpoint-60/config.json | 61 + checkpoint-60/generation_config.json | 13 + checkpoint-60/model.safetensors | 3 + checkpoint-60/optimizer.pt | 3 + checkpoint-60/rng_state.pth | 3 + checkpoint-60/scheduler.pt | 3 + checkpoint-60/tokenizer.json | 3 + checkpoint-60/tokenizer_config.json | 30 + checkpoint-60/trainer_state.json | 1774 ++++++++ checkpoint-60/training_args.bin | 3 + checkpoint-80/chat_template.jinja | 54 + checkpoint-80/config.json | 61 + checkpoint-80/generation_config.json | 13 + checkpoint-80/model.safetensors | 3 + checkpoint-80/optimizer.pt | 3 + checkpoint-80/rng_state.pth | 3 + checkpoint-80/scheduler.pt | 3 + checkpoint-80/tokenizer.json | 3 + checkpoint-80/tokenizer_config.json | 30 + checkpoint-80/trainer_state.json | 2354 ++++++++++ checkpoint-80/training_args.bin | 3 + config.json | 61 + generation_config.json | 13 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 30 + training_args.bin | 3 + untrained_vs_trained.png | Bin 0 -> 82257 bytes 120 files changed, 34280 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 checkpoint-100/chat_template.jinja create mode 100644 checkpoint-100/config.json create mode 100644 checkpoint-100/generation_config.json create mode 100644 checkpoint-100/model.safetensors create mode 100644 checkpoint-100/optimizer.pt create mode 100644 checkpoint-100/rng_state.pth create mode 100644 checkpoint-100/scheduler.pt create mode 100644 checkpoint-100/tokenizer.json create mode 100644 checkpoint-100/tokenizer_config.json create mode 100644 checkpoint-100/trainer_state.json create mode 100644 checkpoint-100/training_args.bin create mode 100644 checkpoint-120/chat_template.jinja create mode 100644 checkpoint-120/config.json create mode 100644 checkpoint-120/generation_config.json create mode 100644 checkpoint-120/model.safetensors create mode 100644 checkpoint-120/optimizer.pt create mode 100644 checkpoint-120/rng_state.pth create mode 100644 checkpoint-120/scheduler.pt create mode 100644 checkpoint-120/tokenizer.json create mode 100644 checkpoint-120/tokenizer_config.json create mode 100644 checkpoint-120/trainer_state.json create mode 100644 checkpoint-120/training_args.bin create mode 100644 checkpoint-140/chat_template.jinja create mode 100644 checkpoint-140/config.json create mode 100644 checkpoint-140/generation_config.json create mode 100644 checkpoint-140/model.safetensors create mode 100644 checkpoint-140/optimizer.pt create mode 100644 checkpoint-140/rng_state.pth create mode 100644 checkpoint-140/scheduler.pt create mode 100644 checkpoint-140/tokenizer.json create mode 100644 checkpoint-140/tokenizer_config.json create mode 100644 checkpoint-140/trainer_state.json create mode 100644 checkpoint-140/training_args.bin create mode 100644 checkpoint-160/chat_template.jinja create mode 100644 checkpoint-160/config.json create mode 100644 checkpoint-160/generation_config.json create mode 100644 checkpoint-160/model.safetensors create mode 100644 checkpoint-160/optimizer.pt create mode 100644 checkpoint-160/rng_state.pth create mode 100644 checkpoint-160/scheduler.pt create mode 100644 checkpoint-160/tokenizer.json create mode 100644 checkpoint-160/tokenizer_config.json create mode 100644 checkpoint-160/trainer_state.json create mode 100644 checkpoint-160/training_args.bin create mode 100644 checkpoint-180/chat_template.jinja create mode 100644 checkpoint-180/config.json create mode 100644 checkpoint-180/generation_config.json create mode 100644 checkpoint-180/model.safetensors create mode 100644 checkpoint-180/optimizer.pt create mode 100644 checkpoint-180/rng_state.pth create mode 100644 checkpoint-180/scheduler.pt create mode 100644 checkpoint-180/tokenizer.json create mode 100644 checkpoint-180/tokenizer_config.json create mode 100644 checkpoint-180/trainer_state.json create mode 100644 checkpoint-180/training_args.bin create mode 100644 checkpoint-20/chat_template.jinja create mode 100644 checkpoint-20/config.json create mode 100644 checkpoint-20/generation_config.json create mode 100644 checkpoint-20/model.safetensors create mode 100644 checkpoint-20/optimizer.pt create mode 100644 checkpoint-20/rng_state.pth create mode 100644 checkpoint-20/scheduler.pt create mode 100644 checkpoint-20/tokenizer.json create mode 100644 checkpoint-20/tokenizer_config.json create mode 100644 checkpoint-20/trainer_state.json create mode 100644 checkpoint-20/training_args.bin create mode 100644 checkpoint-200/chat_template.jinja create mode 100644 checkpoint-200/config.json create mode 100644 checkpoint-200/generation_config.json create mode 100644 checkpoint-200/model.safetensors create mode 100644 checkpoint-200/optimizer.pt create mode 100644 checkpoint-200/rng_state.pth create mode 100644 checkpoint-200/scheduler.pt create mode 100644 checkpoint-200/tokenizer.json create mode 100644 checkpoint-200/tokenizer_config.json create mode 100644 checkpoint-200/trainer_state.json create mode 100644 checkpoint-200/training_args.bin create mode 100644 checkpoint-40/chat_template.jinja create mode 100644 checkpoint-40/config.json create mode 100644 checkpoint-40/generation_config.json create mode 100644 checkpoint-40/model.safetensors create mode 100644 checkpoint-40/optimizer.pt create mode 100644 checkpoint-40/rng_state.pth create mode 100644 checkpoint-40/scheduler.pt create mode 100644 checkpoint-40/tokenizer.json create mode 100644 checkpoint-40/tokenizer_config.json create mode 100644 checkpoint-40/trainer_state.json create mode 100644 checkpoint-40/training_args.bin create mode 100644 checkpoint-60/chat_template.jinja create mode 100644 checkpoint-60/config.json create mode 100644 checkpoint-60/generation_config.json create mode 100644 checkpoint-60/model.safetensors create mode 100644 checkpoint-60/optimizer.pt create mode 100644 checkpoint-60/rng_state.pth create mode 100644 checkpoint-60/scheduler.pt create mode 100644 checkpoint-60/tokenizer.json create mode 100644 checkpoint-60/tokenizer_config.json create mode 100644 checkpoint-60/trainer_state.json create mode 100644 checkpoint-60/training_args.bin create mode 100644 checkpoint-80/chat_template.jinja create mode 100644 checkpoint-80/config.json create mode 100644 checkpoint-80/generation_config.json create mode 100644 checkpoint-80/model.safetensors create mode 100644 checkpoint-80/optimizer.pt create mode 100644 checkpoint-80/rng_state.pth create mode 100644 checkpoint-80/scheduler.pt create mode 100644 checkpoint-80/tokenizer.json create mode 100644 checkpoint-80/tokenizer_config.json create mode 100644 checkpoint-80/trainer_state.json create mode 100644 checkpoint-80/training_args.bin create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 training_args.bin create mode 100644 untrained_vs_trained.png diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a017861 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,46 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-100/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-120/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-140/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-160/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-180/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-20/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-200/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-40/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-60/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-80/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..670dfc2 --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +--- +base_model: Qwen/Qwen2.5-1.5B-Instruct +library_name: transformers +model_name: supergames-grpo200-1.5B +tags: +- generated_from_trainer +- grpo +- trl +licence: license +--- + +# Model Card for supergames-grpo200-1.5B + +This model is a fine-tuned version of [Qwen/Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + + + + + +This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300). + +### Framework versions + +- TRL: 1.2.0 +- Transformers: 5.6.2 +- Pytorch: 2.11.0 +- Datasets: 4.8.4 +- Tokenizers: 0.22.2 + +## Citations + +Cite GRPO as: + +```bibtex +@article{shao2024deepseekmath, + title = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}}, + author = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo}, + year = 2024, + eprint = {arXiv:2402.03300}, +} +``` + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-100/chat_template.jinja b/checkpoint-100/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/checkpoint-100/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-100/config.json b/checkpoint-100/config.json new file mode 100644 index 0000000..f57b09b --- /dev/null +++ b/checkpoint-100/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-100/generation_config.json b/checkpoint-100/generation_config.json new file mode 100644 index 0000000..1b2bba9 --- /dev/null +++ b/checkpoint-100/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/checkpoint-100/model.safetensors b/checkpoint-100/model.safetensors new file mode 100644 index 0000000..bb9332f --- /dev/null +++ b/checkpoint-100/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:016ca4a7af59ae5340813c9f0b26ea5ebd58afe4cb3ed06f59d7fa1fe9828095 +size 6174895536 diff --git a/checkpoint-100/optimizer.pt b/checkpoint-100/optimizer.pt new file mode 100644 index 0000000..37c939b --- /dev/null +++ b/checkpoint-100/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8fa58ba705beb0a7bbc366e9d67c072abecfdd9f655213dfa382d9e628925a6 +size 12350013801 diff --git a/checkpoint-100/rng_state.pth b/checkpoint-100/rng_state.pth new file mode 100644 index 0000000..406bb59 --- /dev/null +++ b/checkpoint-100/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9869cc2d5f67dae6ae945d3ceb0eb17366022213b0956cc572cab18268befe71 +size 14645 diff --git a/checkpoint-100/scheduler.pt b/checkpoint-100/scheduler.pt new file mode 100644 index 0000000..8a88dc3 --- /dev/null +++ b/checkpoint-100/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6151631075f7a98a9131e237ab6732efc7b20c8af25b6f984599c180e47967c +size 1465 diff --git a/checkpoint-100/tokenizer.json b/checkpoint-100/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-100/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-100/tokenizer_config.json b/checkpoint-100/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-100/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-100/trainer_state.json b/checkpoint-100/trainer_state.json new file mode 100644 index 0000000..83c67d3 --- /dev/null +++ b/checkpoint-100/trainer_state.json @@ -0,0 +1,2934 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5, + "eval_steps": 500, + "global_step": 100, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 191.0, + "completions/mean_terminated_length": 191.0, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "entropy": 0.5566893219947815, + "epoch": 0.005, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.370361328125, + "learning_rate": 1e-06, + "loss": -0.24658073484897614, + "num_tokens": 10832.0, + "reward": -3.3000030517578125, + "reward_std": 85.62333679199219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.4000015258789062, + "rewards/supergames_reward/std": 85.62333679199219, + "step": 1, + "step_time": 22.779711072013015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/max_terminated_length": 216.0, + "completions/mean_length": 131.5, + "completions/mean_terminated_length": 131.5, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.6395841240882874, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.09774398803711, + "learning_rate": 9.95e-07, + "loss": 0.20567649602890015, + "num_tokens": 16404.0, + "reward": -12.422499656677246, + "reward_std": 7.134707450866699, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.522500038146973, + "rewards/supergames_reward/std": 7.134707927703857, + "step": 2, + "step_time": 9.960156448010821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/max_terminated_length": 287.0, + "completions/mean_length": 156.625, + "completions/mean_terminated_length": 156.625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "entropy": 0.562222421169281, + "epoch": 0.015, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9053616523742676, + "learning_rate": 9.9e-07, + "loss": 0.09602774679660797, + "num_tokens": 26953.0, + "reward": -74.94999694824219, + "reward_std": 70.73094177246094, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -75.0, + "rewards/supergames_reward/std": 70.71067810058594, + "step": 3, + "step_time": 13.686320498993155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 154.125, + "completions/mean_terminated_length": 154.125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7323317527770996, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.933310031890869, + "learning_rate": 9.849999999999999e-07, + "loss": 0.28249427676200867, + "num_tokens": 36514.0, + "reward": -40.7599983215332, + "reward_std": 81.73140716552734, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -40.80999755859375, + "rewards/supergames_reward/std": 81.68995666503906, + "step": 4, + "step_time": 12.805880262021674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 205.25, + "completions/mean_terminated_length": 205.25, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.6167430281639099, + "epoch": 0.025, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.344135046005249, + "learning_rate": 9.8e-07, + "loss": 0.1433994024991989, + "num_tokens": 47476.0, + "reward": -97.6612548828125, + "reward_std": 6.3321428298950195, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.76124572753906, + "rewards/supergames_reward/std": 6.3321428298950195, + "step": 5, + "step_time": 18.789364666008623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/max_terminated_length": 276.0, + "completions/mean_length": 167.25, + "completions/mean_terminated_length": 167.25, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.7583790421485901, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.172371864318848, + "learning_rate": 9.75e-07, + "loss": -0.12127404659986496, + "num_tokens": 55246.0, + "reward": -58.687503814697266, + "reward_std": 58.64059829711914, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -58.76250457763672, + "rewards/supergames_reward/std": 58.620460510253906, + "step": 6, + "step_time": 12.948570273991209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 133.875, + "completions/mean_terminated_length": 133.875, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.7447654604911804, + "epoch": 0.035, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.428595066070557, + "learning_rate": 9.7e-07, + "loss": 0.10135584324598312, + "num_tokens": 60837.0, + "reward": -32.45000076293945, + "reward_std": 41.754486083984375, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -32.5, + "rewards/supergames_reward/std": 41.66190719604492, + "step": 7, + "step_time": 12.058315072004916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 414.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 209.125, + "completions/mean_terminated_length": 209.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.3860666751861572, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.806255578994751, + "learning_rate": 9.649999999999999e-07, + "loss": -0.33238139748573303, + "num_tokens": 70838.0, + "reward": -80.50375366210938, + "reward_std": 38.82380294799805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -80.60375213623047, + "rewards/supergames_reward/std": 38.82379913330078, + "step": 8, + "step_time": 19.323370319994865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 247.125, + "completions/mean_terminated_length": 158.83334350585938, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.5816237330436707, + "epoch": 0.045, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3209943771362305, + "learning_rate": 9.6e-07, + "loss": 0.36752766370773315, + "num_tokens": 82135.0, + "reward": -27.688751220703125, + "reward_std": 82.8358154296875, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -27.738750457763672, + "rewards/supergames_reward/std": 82.78590393066406, + "step": 9, + "step_time": 21.948575104994234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/max_terminated_length": 240.0, + "completions/mean_length": 153.0, + "completions/mean_terminated_length": 153.0, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6301521062850952, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.125359058380127, + "learning_rate": 9.55e-07, + "loss": 0.06129350885748863, + "num_tokens": 89807.0, + "reward": -78.39624786376953, + "reward_std": 40.17042922973633, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -78.44625091552734, + "rewards/supergames_reward/std": 40.1396484375, + "step": 10, + "step_time": 9.098202993016457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/max_terminated_length": 133.0, + "completions/mean_length": 112.0, + "completions/mean_terminated_length": 112.0, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.5521990656852722, + "epoch": 0.055, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.616060256958008, + "learning_rate": 9.499999999999999e-07, + "loss": -0.006313305348157883, + "num_tokens": 95231.0, + "reward": -12.161249160766602, + "reward_std": 6.39578104019165, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.261249542236328, + "rewards/supergames_reward/std": 6.39578104019165, + "step": 11, + "step_time": 5.136311663984088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 214.875, + "completions/mean_terminated_length": 214.875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.5505905151367188, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.289583206176758, + "learning_rate": 9.45e-07, + "loss": -0.29802003502845764, + "num_tokens": 106238.0, + "reward": -91.66999816894531, + "reward_std": 23.2779541015625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -91.77000427246094, + "rewards/supergames_reward/std": 23.277956008911133, + "step": 12, + "step_time": 14.945365622988902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 167.375, + "completions/mean_terminated_length": 167.375, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "entropy": 0.540415346622467, + "epoch": 0.065, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.858291149139404, + "learning_rate": 9.399999999999999e-07, + "loss": 0.24004128575325012, + "num_tokens": 116937.0, + "reward": -20.38249969482422, + "reward_std": 87.55204010009766, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.457500457763672, + "rewards/supergames_reward/std": 87.52605438232422, + "step": 13, + "step_time": 10.482285185018554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "entropy": 0.7237679958343506, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.083980560302734, + "learning_rate": 9.35e-07, + "loss": -0.19025824964046478, + "num_tokens": 124640.0, + "reward": -51.397499084472656, + "reward_std": 54.486454010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -51.49749755859375, + "rewards/supergames_reward/std": 54.486454010009766, + "step": 14, + "step_time": 9.597231683001155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 141.375, + "completions/mean_terminated_length": 141.375, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.754334568977356, + "epoch": 0.075, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.720428943634033, + "learning_rate": 9.3e-07, + "loss": -0.16830675303936005, + "num_tokens": 132219.0, + "reward": -56.226253509521484, + "reward_std": 60.115657806396484, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -56.32625198364258, + "rewards/supergames_reward/std": 60.11566162109375, + "step": 15, + "step_time": 8.998362872982398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.0, + "completions/max_terminated_length": 209.0, + "completions/mean_length": 136.375, + "completions/mean_terminated_length": 136.375, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.6280568838119507, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.9681806564331055, + "learning_rate": 9.25e-07, + "loss": -0.01910916529595852, + "num_tokens": 139726.0, + "reward": -68.24000549316406, + "reward_std": 52.238914489746094, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.33999633789062, + "rewards/supergames_reward/std": 52.238914489746094, + "step": 16, + "step_time": 8.034480294008972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 211.875, + "completions/mean_terminated_length": 211.875, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.899176836013794, + "epoch": 0.085, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.691746711730957, + "learning_rate": 9.2e-07, + "loss": 0.25513291358947754, + "num_tokens": 147869.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 17, + "step_time": 13.655792869016295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 387.0, + "completions/max_terminated_length": 387.0, + "completions/mean_length": 238.0, + "completions/mean_terminated_length": 238.0, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.5596873760223389, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6711347103118896, + "learning_rate": 9.15e-07, + "loss": -0.2628335654735565, + "num_tokens": 159101.0, + "reward": -24.900001525878906, + "reward_std": 103.50983428955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.0, + "rewards/supergames_reward/std": 103.50983428955078, + "step": 18, + "step_time": 14.784329420013819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 277.0, + "completions/mean_length": 219.75, + "completions/mean_terminated_length": 178.00001525878906, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.9245517253875732, + "epoch": 0.095, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55502986907959, + "learning_rate": 9.1e-07, + "loss": 0.13060179352760315, + "num_tokens": 167299.0, + "reward": -86.89125061035156, + "reward_std": 36.95603561401367, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -86.94125366210938, + "rewards/supergames_reward/std": 36.93572235107422, + "step": 19, + "step_time": 18.832684700988466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/max_terminated_length": 267.0, + "completions/mean_length": 177.25, + "completions/mean_terminated_length": 177.25, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.7331254482269287, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.538497447967529, + "learning_rate": 9.05e-07, + "loss": -0.16418980062007904, + "num_tokens": 175141.0, + "reward": -68.52874755859375, + "reward_std": 58.48719787597656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.62875366210938, + "rewards/supergames_reward/std": 58.48720169067383, + "step": 20, + "step_time": 10.090975169994636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 190.625, + "completions/mean_terminated_length": 190.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.6659425497055054, + "epoch": 0.105, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.485373497009277, + "learning_rate": 9e-07, + "loss": -0.10106432437896729, + "num_tokens": 183098.0, + "reward": -71.48875427246094, + "reward_std": 55.33565139770508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -71.5887451171875, + "rewards/supergames_reward/std": 55.33565139770508, + "step": 21, + "step_time": 11.255001295008697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 179.0, + "completions/mean_terminated_length": 179.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.49365949630737305, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.829419136047363, + "learning_rate": 8.95e-07, + "loss": 0.10587231814861298, + "num_tokens": 192826.0, + "reward": -75.62000274658203, + "reward_std": 45.01959991455078, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -75.69499969482422, + "rewards/supergames_reward/std": 45.00411605834961, + "step": 22, + "step_time": 10.611246599000879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/max_terminated_length": 315.0, + "completions/mean_length": 212.625, + "completions/mean_terminated_length": 212.625, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.43159469962120056, + "epoch": 0.115, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.541125774383545, + "learning_rate": 8.9e-07, + "loss": -0.1681259125471115, + "num_tokens": 202815.0, + "reward": 23.67624855041504, + "reward_std": 67.74340057373047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 23.576250076293945, + "rewards/supergames_reward/std": 67.743408203125, + "step": 23, + "step_time": 11.731771531980485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/max_terminated_length": 168.0, + "completions/mean_length": 101.625, + "completions/mean_terminated_length": 101.625, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.3972298502922058, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.8285417556762695, + "learning_rate": 8.85e-07, + "loss": -0.011585958302021027, + "num_tokens": 208188.0, + "reward": 58.849998474121094, + "reward_std": 56.9304084777832, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 58.75, + "rewards/supergames_reward/std": 56.9304084777832, + "step": 24, + "step_time": 6.187504950998118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 303.0, + "completions/max_terminated_length": 303.0, + "completions/mean_length": 165.375, + "completions/mean_terminated_length": 165.375, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "entropy": 0.6339899301528931, + "epoch": 0.125, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.823799133300781, + "learning_rate": 8.799999999999999e-07, + "loss": 0.05159185826778412, + "num_tokens": 215935.0, + "reward": -23.521251678466797, + "reward_std": 74.23067474365234, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -23.621248245239258, + "rewards/supergames_reward/std": 74.23067474365234, + "step": 25, + "step_time": 11.059416300005978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 349.0, + "completions/max_terminated_length": 349.0, + "completions/mean_length": 204.0, + "completions/mean_terminated_length": 204.0, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.6462895274162292, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.969292640686035, + "learning_rate": 8.75e-07, + "loss": 0.0115677984431386, + "num_tokens": 223999.0, + "reward": -74.8175048828125, + "reward_std": 40.94221115112305, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -74.89250183105469, + "rewards/supergames_reward/std": 40.924625396728516, + "step": 26, + "step_time": 12.666237785975682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 166.5, + "completions/mean_terminated_length": 166.5, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4756850600242615, + "epoch": 0.135, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915069103240967, + "learning_rate": 8.699999999999999e-07, + "loss": -0.13321346044540405, + "num_tokens": 233643.0, + "reward": -52.125, + "reward_std": 19.405282974243164, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -52.20000076293945, + "rewards/supergames_reward/std": 19.33464813232422, + "step": 27, + "step_time": 11.172539752995363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 137.125, + "completions/mean_terminated_length": 137.125, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.5834778547286987, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.146666526794434, + "learning_rate": 8.65e-07, + "loss": 0.060454584658145905, + "num_tokens": 241204.0, + "reward": -68.99250793457031, + "reward_std": 51.010581970214844, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -69.09249877929688, + "rewards/supergames_reward/std": 51.010581970214844, + "step": 28, + "step_time": 7.025046669004951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 182.375, + "completions/mean_terminated_length": 182.375, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.6368551254272461, + "epoch": 0.145, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.15535306930542, + "learning_rate": 8.599999999999999e-07, + "loss": -0.3323673903942108, + "num_tokens": 249087.0, + "reward": -44.17000198364258, + "reward_std": 80.5373306274414, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -44.27000045776367, + "rewards/supergames_reward/std": 80.5373306274414, + "step": 29, + "step_time": 12.079259724996518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 127.75, + "completions/mean_terminated_length": 127.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4491196870803833, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.896186828613281, + "learning_rate": 8.55e-07, + "loss": -0.12887656688690186, + "num_tokens": 259445.0, + "reward": 57.446250915527344, + "reward_std": 38.78837585449219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 57.34625244140625, + "rewards/supergames_reward/std": 38.78837966918945, + "step": 30, + "step_time": 7.483972909016302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/max_terminated_length": 166.0, + "completions/mean_length": 112.875, + "completions/mean_terminated_length": 112.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4858454465866089, + "epoch": 0.155, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.742906093597412, + "learning_rate": 8.499999999999999e-07, + "loss": 0.06930096447467804, + "num_tokens": 264924.0, + "reward": -21.174999237060547, + "reward_std": 31.89051628112793, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -21.25, + "rewards/supergames_reward/std": 31.819807052612305, + "step": 31, + "step_time": 6.1866529019898735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/max_terminated_length": 148.0, + "completions/mean_length": 118.125, + "completions/mean_terminated_length": 118.125, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.45428749918937683, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.663414478302002, + "learning_rate": 8.45e-07, + "loss": 0.05735419690608978, + "num_tokens": 270381.0, + "reward": 0.4687504768371582, + "reward_std": 38.66371154785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 0.36875057220458984, + "rewards/supergames_reward/std": 38.6637077331543, + "step": 32, + "step_time": 5.577042757999152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 166.875, + "completions/mean_terminated_length": 166.875, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.6107293367385864, + "epoch": 0.165, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.75006628036499, + "learning_rate": 8.399999999999999e-07, + "loss": -0.14760127663612366, + "num_tokens": 278156.0, + "reward": -87.84750366210938, + "reward_std": 26.84708023071289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -87.94749450683594, + "rewards/supergames_reward/std": 26.847076416015625, + "step": 33, + "step_time": 12.552876825997373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 360.0, + "completions/mean_length": 231.0, + "completions/mean_terminated_length": 190.85714721679688, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.6089653372764587, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.781267166137695, + "learning_rate": 8.349999999999999e-07, + "loss": 0.4294504225254059, + "num_tokens": 289364.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 34, + "step_time": 18.961819477990502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 134.125, + "completions/mean_terminated_length": 134.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46158918738365173, + "epoch": 0.175, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.016353607177734, + "learning_rate": 8.299999999999999e-07, + "loss": 0.09251818805932999, + "num_tokens": 298733.0, + "reward": -35.558753967285156, + "reward_std": 15.881204605102539, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -35.65875244140625, + "rewards/supergames_reward/std": 15.881203651428223, + "step": 35, + "step_time": 8.38491243700264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/max_terminated_length": 207.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.5104788541793823, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.592893123626709, + "learning_rate": 8.249999999999999e-07, + "loss": -0.0410832017660141, + "num_tokens": 306420.0, + "reward": 5.4212493896484375, + "reward_std": 93.50404357910156, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 5.346250534057617, + "rewards/supergames_reward/std": 93.4718246459961, + "step": 36, + "step_time": 7.811868985998444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/max_terminated_length": 241.0, + "completions/mean_length": 178.0, + "completions/mean_terminated_length": 178.0, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.7527878880500793, + "epoch": 0.185, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8812546730041504, + "learning_rate": 8.199999999999999e-07, + "loss": 0.12539231777191162, + "num_tokens": 314252.0, + "reward": -89.58500671386719, + "reward_std": 29.256120681762695, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -89.66000366210938, + "rewards/supergames_reward/std": 29.24593734741211, + "step": 37, + "step_time": 8.959742914012168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 133.375, + "completions/mean_terminated_length": 133.375, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.3347901403903961, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.205414295196533, + "learning_rate": 8.149999999999999e-07, + "loss": -0.06991486996412277, + "num_tokens": 319839.0, + "reward": -14.498749732971191, + "reward_std": 13.007229804992676, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -14.598750114440918, + "rewards/supergames_reward/std": 13.007229804992676, + "step": 38, + "step_time": 7.377183554985095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/max_terminated_length": 181.0, + "completions/mean_length": 114.375, + "completions/mean_terminated_length": 114.375, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.4445965886116028, + "epoch": 0.195, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.561298847198486, + "learning_rate": 8.1e-07, + "loss": -0.06235164776444435, + "num_tokens": 325290.0, + "reward": -7.425000190734863, + "reward_std": 53.701602935791016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -7.5, + "rewards/supergames_reward/std": 53.652320861816406, + "step": 39, + "step_time": 6.753862089011818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 187.625, + "completions/mean_terminated_length": 187.625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.3950234353542328, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5988712310791016, + "learning_rate": 8.05e-07, + "loss": -0.1381440907716751, + "num_tokens": 335119.0, + "reward": -33.68375015258789, + "reward_std": 42.069435119628906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -33.783748626708984, + "rewards/supergames_reward/std": 42.069435119628906, + "step": 40, + "step_time": 13.718958162004128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 371.0, + "completions/mean_length": 251.125, + "completions/mean_terminated_length": 213.85714721679688, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.507067084312439, + "epoch": 0.205, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3898658752441406, + "learning_rate": 8e-07, + "loss": 0.0078964838758111, + "num_tokens": 346464.0, + "reward": -86.48625183105469, + "reward_std": 38.020694732666016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -86.56124877929688, + "rewards/supergames_reward/std": 38.010528564453125, + "step": 41, + "step_time": 18.90960379401804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 152.0, + "completions/max_terminated_length": 152.0, + "completions/mean_length": 116.375, + "completions/mean_terminated_length": 116.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3801707923412323, + "epoch": 0.21, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.95e-07, + "loss": 0.0, + "num_tokens": 351955.0, + "reward": -9.899999618530273, + "reward_std": 0.0, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -10.0, + "rewards/supergames_reward/std": 0.0, + "step": 42, + "step_time": 5.711630532023264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/max_terminated_length": 308.0, + "completions/mean_length": 247.375, + "completions/mean_terminated_length": 247.375, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.3185043931007385, + "epoch": 0.215, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.168762445449829, + "learning_rate": 7.9e-07, + "loss": 0.1970764398574829, + "num_tokens": 363278.0, + "reward": -53.89249801635742, + "reward_std": 63.02619934082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -53.99250030517578, + "rewards/supergames_reward/std": 63.02619934082031, + "step": 43, + "step_time": 11.661934906995157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/max_terminated_length": 330.0, + "completions/mean_length": 212.5, + "completions/mean_terminated_length": 212.5, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4446268379688263, + "epoch": 0.22, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.305844306945801, + "learning_rate": 7.85e-07, + "loss": -0.06735197454690933, + "num_tokens": 374306.0, + "reward": 18.059999465942383, + "reward_std": 98.9417953491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 17.959999084472656, + "rewards/supergames_reward/std": 98.9417953491211, + "step": 44, + "step_time": 12.492453911982011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.0, + "completions/max_terminated_length": 316.0, + "completions/mean_length": 183.0, + "completions/mean_terminated_length": 183.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.5269614458084106, + "epoch": 0.225, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8006370067596436, + "learning_rate": 7.799999999999999e-07, + "loss": -0.08870815485715866, + "num_tokens": 382210.0, + "reward": -54.928749084472656, + "reward_std": 72.5127944946289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -55.02874755859375, + "rewards/supergames_reward/std": 72.5127944946289, + "step": 45, + "step_time": 11.7636467939883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.0, + "completions/max_terminated_length": 244.0, + "completions/mean_length": 136.625, + "completions/mean_terminated_length": 136.625, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.34820204973220825, + "epoch": 0.23, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.911736011505127, + "learning_rate": 7.75e-07, + "loss": 0.04397330805659294, + "num_tokens": 391583.0, + "reward": -19.10375213623047, + "reward_std": 23.967702865600586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -19.203750610351562, + "rewards/supergames_reward/std": 23.967702865600586, + "step": 46, + "step_time": 10.119426151999505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 160.375, + "completions/mean_terminated_length": 160.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.611914336681366, + "epoch": 0.235, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.571913242340088, + "learning_rate": 7.699999999999999e-07, + "loss": -0.044293053448200226, + "num_tokens": 399290.0, + "reward": -25.186250686645508, + "reward_std": 71.46305084228516, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.286251068115234, + "rewards/supergames_reward/std": 71.46305084228516, + "step": 47, + "step_time": 8.563868903991533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/max_terminated_length": 167.0, + "completions/mean_length": 111.125, + "completions/mean_terminated_length": 111.125, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.46585777401924133, + "epoch": 0.24, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.654109954833984, + "learning_rate": 7.65e-07, + "loss": -0.035151124000549316, + "num_tokens": 404707.0, + "reward": -0.408750057220459, + "reward_std": 31.939748764038086, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -0.5087499618530273, + "rewards/supergames_reward/std": 31.93975257873535, + "step": 48, + "step_time": 6.438482580007985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/max_terminated_length": 306.0, + "completions/mean_length": 216.625, + "completions/mean_terminated_length": 216.625, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.33054471015930176, + "epoch": 0.245, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.700303792953491, + "learning_rate": 7.599999999999999e-07, + "loss": -0.04393656551837921, + "num_tokens": 414736.0, + "reward": -97.26000213623047, + "reward_std": 1.6294406652450562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.36000061035156, + "rewards/supergames_reward/std": 1.629441738128662, + "step": 49, + "step_time": 11.636040106008295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 378.0, + "completions/max_terminated_length": 378.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.48711177706718445, + "epoch": 0.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9650564193725586, + "learning_rate": 7.55e-07, + "loss": -0.3529520332813263, + "num_tokens": 424795.0, + "reward": 47.408748626708984, + "reward_std": 72.72083282470703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 47.30875015258789, + "rewards/supergames_reward/std": 72.72083282470703, + "step": 50, + "step_time": 14.258096842997475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 267.625, + "completions/mean_terminated_length": 232.71429443359375, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4482860863208771, + "epoch": 0.255, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2998197078704834, + "learning_rate": 7.5e-07, + "loss": 0.2493654191493988, + "num_tokens": 436288.0, + "reward": 50.61125183105469, + "reward_std": 75.3349838256836, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.53624725341797, + "rewards/supergames_reward/std": 75.27783966064453, + "step": 51, + "step_time": 19.20827590499539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 176.0, + "completions/max_terminated_length": 176.0, + "completions/mean_length": 144.5, + "completions/mean_terminated_length": 144.5, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "entropy": 0.5378735661506653, + "epoch": 0.26, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.114461898803711, + "learning_rate": 7.45e-07, + "loss": 0.0036597400903701782, + "num_tokens": 441980.0, + "reward": -15.20625114440918, + "reward_std": 45.35526657104492, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.28125, + "rewards/supergames_reward/std": 45.301815032958984, + "step": 52, + "step_time": 6.660627231001854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/max_terminated_length": 136.0, + "completions/mean_length": 119.0, + "completions/mean_terminated_length": 119.0, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "entropy": 0.41796669363975525, + "epoch": 0.265, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.059505462646484, + "learning_rate": 7.4e-07, + "loss": -0.014973883517086506, + "num_tokens": 447468.0, + "reward": 15.170000076293945, + "reward_std": 46.658329010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 15.070000648498535, + "rewards/supergames_reward/std": 46.658329010009766, + "step": 53, + "step_time": 5.271571868011961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/max_terminated_length": 259.0, + "completions/mean_length": 191.125, + "completions/mean_terminated_length": 191.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.45421987771987915, + "epoch": 0.27, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4454867839813232, + "learning_rate": 7.35e-07, + "loss": -0.1128973588347435, + "num_tokens": 457301.0, + "reward": 38.89875030517578, + "reward_std": 62.1772575378418, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 38.79874801635742, + "rewards/supergames_reward/std": 62.1772575378418, + "step": 54, + "step_time": 10.524528659996577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/max_terminated_length": 319.0, + "completions/mean_length": 233.375, + "completions/mean_terminated_length": 233.375, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "entropy": 0.4449213445186615, + "epoch": 0.275, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2065398693084717, + "learning_rate": 7.3e-07, + "loss": 0.0016277075046673417, + "num_tokens": 467520.0, + "reward": 27.372499465942383, + "reward_std": 81.23126983642578, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.27250099182129, + "rewards/supergames_reward/std": 81.23126983642578, + "step": 55, + "step_time": 12.592280682991259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 357.0, + "completions/mean_length": 261.0, + "completions/mean_terminated_length": 225.1428680419922, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.506920337677002, + "epoch": 0.28, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.058388710021973, + "learning_rate": 7.249999999999999e-07, + "loss": 0.3289242386817932, + "num_tokens": 476056.0, + "reward": 47.05249786376953, + "reward_std": 80.97753143310547, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 46.977500915527344, + "rewards/supergames_reward/std": 80.9256362915039, + "step": 56, + "step_time": 18.845177220006008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.0, + "completions/max_terminated_length": 250.0, + "completions/mean_length": 192.75, + "completions/mean_terminated_length": 192.75, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "entropy": 0.5197336077690125, + "epoch": 0.285, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.230815410614014, + "learning_rate": 7.2e-07, + "loss": -0.15186546742916107, + "num_tokens": 484014.0, + "reward": -64.5050048828125, + "reward_std": 50.35145568847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -64.60499572753906, + "rewards/supergames_reward/std": 50.3514518737793, + "step": 57, + "step_time": 9.576232638006331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 249.0, + "completions/mean_terminated_length": 249.0, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "entropy": 0.42913514375686646, + "epoch": 0.29, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.8544528484344482, + "learning_rate": 7.149999999999999e-07, + "loss": -0.056405920535326004, + "num_tokens": 494390.0, + "reward": 40.123748779296875, + "reward_std": 84.164794921875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 40.02375030517578, + "rewards/supergames_reward/std": 84.164794921875, + "step": 58, + "step_time": 12.385353341000155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 138.0, + "completions/max_terminated_length": 138.0, + "completions/mean_length": 107.25, + "completions/mean_terminated_length": 107.25, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.34386488795280457, + "epoch": 0.295, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.6915154457092285, + "learning_rate": 7.1e-07, + "loss": 0.0865321159362793, + "num_tokens": 499816.0, + "reward": -8.244999885559082, + "reward_std": 4.681046962738037, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.345000267028809, + "rewards/supergames_reward/std": 4.681046962738037, + "step": 59, + "step_time": 5.42325339600211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/max_terminated_length": 187.0, + "completions/mean_length": 127.25, + "completions/mean_terminated_length": 127.25, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.5430045127868652, + "epoch": 0.3, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.635311126708984, + "learning_rate": 7.049999999999999e-07, + "loss": -0.05412375554442406, + "num_tokens": 505410.0, + "reward": 20.005001068115234, + "reward_std": 55.156005859375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 19.904998779296875, + "rewards/supergames_reward/std": 55.156005859375, + "step": 60, + "step_time": 7.068010902003152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/max_terminated_length": 286.0, + "completions/mean_length": 192.125, + "completions/mean_terminated_length": 192.125, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4213869869709015, + "epoch": 0.305, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4664618968963623, + "learning_rate": 7e-07, + "loss": -0.0495159812271595, + "num_tokens": 516283.0, + "reward": -47.212501525878906, + "reward_std": 91.2253646850586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -47.3125, + "rewards/supergames_reward/std": 91.2253646850586, + "step": 61, + "step_time": 11.176304235996213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 171.0, + "completions/mean_terminated_length": 171.0, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.4626811146736145, + "epoch": 0.31, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.949252128601074, + "learning_rate": 6.949999999999999e-07, + "loss": 0.192047581076622, + "num_tokens": 522251.0, + "reward": -15.092500686645508, + "reward_std": 49.82962417602539, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.167499542236328, + "rewards/supergames_reward/std": 49.78091049194336, + "step": 62, + "step_time": 12.075224861997413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 429.0, + "completions/mean_length": 283.5, + "completions/mean_terminated_length": 250.85714721679688, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4814216196537018, + "epoch": 0.315, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7918753623962402, + "learning_rate": 6.9e-07, + "loss": 0.15706156194210052, + "num_tokens": 533815.0, + "reward": -20.86625099182129, + "reward_std": 85.99250030517578, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.94124984741211, + "rewards/supergames_reward/std": 85.96620178222656, + "step": 63, + "step_time": 19.448832260008203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.0, + "completions/max_terminated_length": 189.0, + "completions/mean_length": 131.25, + "completions/mean_terminated_length": 131.25, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4929414987564087, + "epoch": 0.32, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.774101734161377, + "learning_rate": 6.85e-07, + "loss": -0.060691747814416885, + "num_tokens": 539465.0, + "reward": -21.0674991607666, + "reward_std": 6.892730236053467, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -21.167499542236328, + "rewards/supergames_reward/std": 6.892730236053467, + "step": 64, + "step_time": 7.141569407976931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 454.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 281.375, + "completions/mean_terminated_length": 281.375, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "entropy": 0.45866572856903076, + "epoch": 0.325, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.092717409133911, + "learning_rate": 6.800000000000001e-07, + "loss": 0.11694959551095963, + "num_tokens": 551012.0, + "reward": 49.938751220703125, + "reward_std": 79.1865005493164, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 49.8387451171875, + "rewards/supergames_reward/std": 79.1865005493164, + "step": 65, + "step_time": 17.06170882002334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/max_terminated_length": 203.0, + "completions/mean_length": 128.625, + "completions/mean_terminated_length": 128.625, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.5005782246589661, + "epoch": 0.33, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.77616024017334, + "learning_rate": 6.75e-07, + "loss": 0.11242837458848953, + "num_tokens": 556577.0, + "reward": 48.875, + "reward_std": 49.13539123535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.775001525878906, + "rewards/supergames_reward/std": 49.13539123535156, + "step": 66, + "step_time": 7.695410129002994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 436.0, + "completions/max_terminated_length": 436.0, + "completions/mean_length": 281.875, + "completions/mean_terminated_length": 281.875, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "entropy": 0.507097065448761, + "epoch": 0.335, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.911184787750244, + "learning_rate": 6.7e-07, + "loss": -0.08151137083768845, + "num_tokens": 567120.0, + "reward": -12.886249542236328, + "reward_std": 84.04269409179688, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.986251831054688, + "rewards/supergames_reward/std": 84.04269409179688, + "step": 67, + "step_time": 16.409127942984924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 318.375, + "completions/mean_terminated_length": 290.71429443359375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.486402690410614, + "epoch": 0.34, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9461607933044434, + "learning_rate": 6.65e-07, + "loss": 0.2442178875207901, + "num_tokens": 578995.0, + "reward": 2.8387489318847656, + "reward_std": 98.47579956054688, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 2.7637500762939453, + "rewards/supergames_reward/std": 98.44595336914062, + "step": 68, + "step_time": 19.32364999302081 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 291.0, + "completions/max_terminated_length": 291.0, + "completions/mean_length": 219.625, + "completions/mean_terminated_length": 219.625, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.37856727838516235, + "epoch": 0.345, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0204732418060303, + "learning_rate": 6.6e-07, + "loss": -0.08764916658401489, + "num_tokens": 589072.0, + "reward": -54.18375015258789, + "reward_std": 71.45012664794922, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -54.283748626708984, + "rewards/supergames_reward/std": 71.45013427734375, + "step": 69, + "step_time": 11.126611230982235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 305.0, + "completions/mean_length": 254.75, + "completions/mean_terminated_length": 218.00001525878906, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46423614025115967, + "epoch": 0.35, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0140154361724854, + "learning_rate": 6.55e-07, + "loss": 0.26797160506248474, + "num_tokens": 600438.0, + "reward": -32.04875183105469, + "reward_std": 69.86957550048828, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -32.12375259399414, + "rewards/supergames_reward/std": 69.84178161621094, + "step": 70, + "step_time": 19.12617294798838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 372.0, + "completions/max_terminated_length": 372.0, + "completions/mean_length": 194.125, + "completions/mean_terminated_length": 194.125, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.3427293300628662, + "epoch": 0.355, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.781383991241455, + "learning_rate": 6.5e-07, + "loss": 0.354524701833725, + "num_tokens": 610327.0, + "reward": -6.913749694824219, + "reward_std": 87.56071472167969, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -6.988750457763672, + "rewards/supergames_reward/std": 87.53033447265625, + "step": 71, + "step_time": 14.041668199002743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/max_terminated_length": 197.0, + "completions/mean_length": 124.75, + "completions/mean_terminated_length": 124.75, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.4455118477344513, + "epoch": 0.36, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.726032733917236, + "learning_rate": 6.45e-07, + "loss": 0.18206289410591125, + "num_tokens": 615853.0, + "reward": 41.95124816894531, + "reward_std": 60.640811920166016, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 41.85124969482422, + "rewards/supergames_reward/std": 60.640811920166016, + "step": 72, + "step_time": 7.416493425989756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.0, + "completions/max_terminated_length": 323.0, + "completions/mean_length": 147.375, + "completions/mean_terminated_length": 147.375, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.480733186006546, + "epoch": 0.365, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.853621482849121, + "learning_rate": 6.4e-07, + "loss": 0.04558124020695686, + "num_tokens": 621584.0, + "reward": 14.884998321533203, + "reward_std": 46.25857162475586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 14.78499984741211, + "rewards/supergames_reward/std": 46.25857162475586, + "step": 73, + "step_time": 11.95379540900467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 283.0, + "completions/max_terminated_length": 283.0, + "completions/mean_length": 188.125, + "completions/mean_terminated_length": 188.125, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.5526013374328613, + "epoch": 0.37, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.729883909225464, + "learning_rate": 6.35e-07, + "loss": 0.06268303096294403, + "num_tokens": 629545.0, + "reward": -68.05750274658203, + "reward_std": 59.44808578491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.15750122070312, + "rewards/supergames_reward/std": 59.44808578491211, + "step": 74, + "step_time": 10.64124580900534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 141.0, + "completions/max_terminated_length": 141.0, + "completions/mean_length": 117.375, + "completions/mean_terminated_length": 117.375, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.3744319677352905, + "epoch": 0.375, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.775154113769531, + "learning_rate": 6.3e-07, + "loss": -0.023284845054149628, + "num_tokens": 635052.0, + "reward": 3.9374990463256836, + "reward_std": 61.85771179199219, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 3.862499237060547, + "rewards/supergames_reward/std": 61.80967712402344, + "step": 75, + "step_time": 5.432648951013107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 157.0, + "completions/mean_terminated_length": 157.0, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4822140336036682, + "epoch": 0.38, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.715704917907715, + "learning_rate": 6.249999999999999e-07, + "loss": -0.04918142035603523, + "num_tokens": 644644.0, + "reward": 48.64125061035156, + "reward_std": 78.0205078125, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 48.56624984741211, + "rewards/supergames_reward/std": 77.96605682373047, + "step": 76, + "step_time": 10.348935816989979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/max_terminated_length": 380.0, + "completions/mean_length": 210.875, + "completions/mean_terminated_length": 210.875, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "entropy": 0.45722317695617676, + "epoch": 0.385, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6057567596435547, + "learning_rate": 6.2e-07, + "loss": 0.058352649211883545, + "num_tokens": 652771.0, + "reward": -8.283750534057617, + "reward_std": 77.68436431884766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.383749008178711, + "rewards/supergames_reward/std": 77.68437194824219, + "step": 77, + "step_time": 14.142948755004909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 188.25, + "completions/mean_terminated_length": 188.25, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.33380764722824097, + "epoch": 0.39, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2239863872528076, + "learning_rate": 6.149999999999999e-07, + "loss": -0.04592633992433548, + "num_tokens": 662653.0, + "reward": 11.153749465942383, + "reward_std": 89.88329315185547, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 11.053749084472656, + "rewards/supergames_reward/std": 89.88329315185547, + "step": 78, + "step_time": 12.277474621019792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/max_terminated_length": 143.0, + "completions/mean_length": 108.875, + "completions/mean_terminated_length": 108.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.3518812656402588, + "epoch": 0.395, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.967776298522949, + "learning_rate": 6.1e-07, + "loss": -0.10800496488809586, + "num_tokens": 668076.0, + "reward": 70.33250427246094, + "reward_std": 49.9145393371582, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 70.23249816894531, + "rewards/supergames_reward/std": 49.91453552246094, + "step": 79, + "step_time": 5.52714040101273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 152.125, + "completions/mean_terminated_length": 152.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4699896574020386, + "epoch": 0.4, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.561161994934082, + "learning_rate": 6.049999999999999e-07, + "loss": 0.09505834430456161, + "num_tokens": 675701.0, + "reward": -20.25625228881836, + "reward_std": 53.78542709350586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -20.35624885559082, + "rewards/supergames_reward/std": 53.78542709350586, + "step": 80, + "step_time": 7.155081019998761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 288.875, + "completions/mean_terminated_length": 257.0, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.4086494445800781, + "epoch": 0.405, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.3719611167907715, + "learning_rate": 6e-07, + "loss": 0.04727627709507942, + "num_tokens": 687332.0, + "reward": -37.20750045776367, + "reward_std": 91.7676010131836, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -37.25749969482422, + "rewards/supergames_reward/std": 91.72847747802734, + "step": 81, + "step_time": 19.231399144016905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 376.0, + "completions/max_terminated_length": 376.0, + "completions/mean_length": 227.75, + "completions/mean_terminated_length": 227.75, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.43233707547187805, + "epoch": 0.41, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3391714096069336, + "learning_rate": 5.949999999999999e-07, + "loss": -0.25524550676345825, + "num_tokens": 698474.0, + "reward": -3.7287511825561523, + "reward_std": 91.25379943847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.828749656677246, + "rewards/supergames_reward/std": 91.25379943847656, + "step": 82, + "step_time": 14.329176368017215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 178.125, + "completions/mean_terminated_length": 178.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.4293636977672577, + "epoch": 0.415, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.7271742820739746, + "learning_rate": 5.9e-07, + "loss": 0.09273672848939896, + "num_tokens": 708243.0, + "reward": 12.802498817443848, + "reward_std": 86.23526000976562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 12.702500343322754, + "rewards/supergames_reward/std": 86.23526000976562, + "step": 83, + "step_time": 11.419686011999147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 425.0, + "completions/max_terminated_length": 425.0, + "completions/mean_length": 255.375, + "completions/mean_terminated_length": 255.375, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "entropy": 0.38068246841430664, + "epoch": 0.42, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.6244184970855713, + "learning_rate": 5.849999999999999e-07, + "loss": 0.02006213366985321, + "num_tokens": 718606.0, + "reward": 37.803749084472656, + "reward_std": 57.58624267578125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 37.70375061035156, + "rewards/supergames_reward/std": 57.58624267578125, + "step": 84, + "step_time": 15.975198492989875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 116.0, + "completions/max_terminated_length": 116.0, + "completions/mean_length": 108.125, + "completions/mean_terminated_length": 108.125, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.339572012424469, + "epoch": 0.425, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.754759311676025, + "learning_rate": 5.8e-07, + "loss": -0.011015941388905048, + "num_tokens": 723983.0, + "reward": -3.8212504386901855, + "reward_std": 61.624786376953125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.921250820159912, + "rewards/supergames_reward/std": 61.624786376953125, + "step": 85, + "step_time": 4.542777584982105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 447.0, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 291.375, + "completions/mean_terminated_length": 291.375, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "entropy": 0.5846173763275146, + "epoch": 0.43, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55724835395813, + "learning_rate": 5.749999999999999e-07, + "loss": -0.04981120675802231, + "num_tokens": 735642.0, + "reward": 50.07499694824219, + "reward_std": 92.6283187866211, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.0, + "rewards/supergames_reward/std": 92.5820083618164, + "step": 86, + "step_time": 16.777178087009815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/max_terminated_length": 232.0, + "completions/mean_length": 192.0, + "completions/mean_terminated_length": 192.0, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "entropy": 0.5139474272727966, + "epoch": 0.435, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8860411643981934, + "learning_rate": 5.699999999999999e-07, + "loss": -0.02194221317768097, + "num_tokens": 743602.0, + "reward": -52.522499084472656, + "reward_std": 73.8686294555664, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -52.62249755859375, + "rewards/supergames_reward/std": 73.86863708496094, + "step": 87, + "step_time": 8.772893911984283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/max_terminated_length": 266.0, + "completions/mean_length": 146.875, + "completions/mean_terminated_length": 146.875, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.3428497910499573, + "epoch": 0.44, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.509483337402344, + "learning_rate": 5.649999999999999e-07, + "loss": 0.19857533276081085, + "num_tokens": 753065.0, + "reward": 63.63874816894531, + "reward_std": 67.54963684082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 63.53874969482422, + "rewards/supergames_reward/std": 67.54963684082031, + "step": 88, + "step_time": 10.230529835011112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 446.0, + "completions/max_terminated_length": 446.0, + "completions/mean_length": 270.625, + "completions/mean_terminated_length": 270.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.4417419731616974, + "epoch": 0.445, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.973951816558838, + "learning_rate": 5.6e-07, + "loss": -0.00031440958264283836, + "num_tokens": 764534.0, + "reward": -9.736251831054688, + "reward_std": 92.25879669189453, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -9.836250305175781, + "rewards/supergames_reward/std": 92.25880432128906, + "step": 89, + "step_time": 17.40752330099349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 398.0, + "completions/max_terminated_length": 398.0, + "completions/mean_length": 227.375, + "completions/mean_terminated_length": 227.375, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.4732024073600769, + "epoch": 0.45, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2560815811157227, + "learning_rate": 5.55e-07, + "loss": 0.03820464387536049, + "num_tokens": 772809.0, + "reward": 5.982499599456787, + "reward_std": 58.833797454833984, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 5.8824992179870605, + "rewards/supergames_reward/std": 58.83380126953125, + "step": 90, + "step_time": 15.007269965979503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 207.5, + "completions/mean_terminated_length": 207.5, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.42038559913635254, + "epoch": 0.455, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3649117946624756, + "learning_rate": 5.5e-07, + "loss": 0.1510048508644104, + "num_tokens": 782805.0, + "reward": -5.61500358581543, + "reward_std": 82.67152404785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -5.714998245239258, + "rewards/supergames_reward/std": 82.67152404785156, + "step": 91, + "step_time": 10.386137172987219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 416.0, + "completions/max_terminated_length": 416.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.3971734642982483, + "epoch": 0.46, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.433772563934326, + "learning_rate": 5.45e-07, + "loss": -0.3027213513851166, + "num_tokens": 793840.0, + "reward": -29.06500244140625, + "reward_std": 91.68325805664062, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -29.16499900817871, + "rewards/supergames_reward/std": 91.68325805664062, + "step": 92, + "step_time": 15.790611553995404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 206.625, + "completions/mean_terminated_length": 206.625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.5906907320022583, + "epoch": 0.465, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.293088912963867, + "learning_rate": 5.4e-07, + "loss": 0.09760032594203949, + "num_tokens": 803821.0, + "reward": -72.73999786376953, + "reward_std": 50.72484588623047, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -72.81500244140625, + "rewards/supergames_reward/std": 50.7094841003418, + "step": 93, + "step_time": 13.043119941983605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/max_terminated_length": 143.0, + "completions/mean_length": 115.75, + "completions/mean_terminated_length": 115.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4233115315437317, + "epoch": 0.47, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.58886194229126, + "learning_rate": 5.35e-07, + "loss": -0.04040650278329849, + "num_tokens": 809299.0, + "reward": -3.3050003051757812, + "reward_std": 24.443714141845703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.405000686645508, + "rewards/supergames_reward/std": 24.443714141845703, + "step": 94, + "step_time": 5.512874762003776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 122.0, + "completions/max_terminated_length": 122.0, + "completions/mean_length": 103.125, + "completions/mean_terminated_length": 103.125, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.4055764377117157, + "epoch": 0.475, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.981549263000488, + "learning_rate": 5.3e-07, + "loss": -0.07588938623666763, + "num_tokens": 814716.0, + "reward": 60.06999969482422, + "reward_std": 42.708473205566406, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 59.970001220703125, + "rewards/supergames_reward/std": 42.708473205566406, + "step": 95, + "step_time": 4.766389057011111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 475.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.408913254737854, + "epoch": 0.48, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.79402756690979, + "learning_rate": 5.25e-07, + "loss": -0.3087541460990906, + "num_tokens": 824791.0, + "reward": -15.042500495910645, + "reward_std": 84.37246704101562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -15.142499923706055, + "rewards/supergames_reward/std": 84.37246704101562, + "step": 96, + "step_time": 17.779843941010768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/max_terminated_length": 154.0, + "completions/mean_length": 121.625, + "completions/mean_terminated_length": 121.625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.529015839099884, + "epoch": 0.485, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.142816066741943, + "learning_rate": 5.2e-07, + "loss": 0.07216037809848785, + "num_tokens": 830300.0, + "reward": -32.743751525878906, + "reward_std": 54.61775588989258, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -32.81875228881836, + "rewards/supergames_reward/std": 54.58256149291992, + "step": 97, + "step_time": 5.913989890017547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 126.0, + "completions/max_terminated_length": 126.0, + "completions/mean_length": 95.625, + "completions/mean_terminated_length": 95.625, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.33743274211883545, + "epoch": 0.49, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.197273254394531, + "learning_rate": 5.149999999999999e-07, + "loss": 0.07500407844781876, + "num_tokens": 835633.0, + "reward": 11.00999927520752, + "reward_std": 62.73848342895508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 10.910000801086426, + "rewards/supergames_reward/std": 62.73848342895508, + "step": 98, + "step_time": 4.9155233050114475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 108.5, + "completions/mean_terminated_length": 108.5, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.3106803297996521, + "epoch": 0.495, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.864002227783203, + "learning_rate": 5.1e-07, + "loss": -0.1206185445189476, + "num_tokens": 841045.0, + "reward": 14.238749504089355, + "reward_std": 51.77016830444336, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 14.138750076293945, + "rewards/supergames_reward/std": 51.77016830444336, + "step": 99, + "step_time": 5.7507751359953545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 97.125, + "completions/mean_terminated_length": 97.125, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.35160186886787415, + "epoch": 0.5, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915731906890869, + "learning_rate": 5.049999999999999e-07, + "loss": -0.07951541244983673, + "num_tokens": 846398.0, + "reward": 67.78125, + "reward_std": 38.15436553955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 67.68124389648438, + "rewards/supergames_reward/std": 38.15436553955078, + "step": 100, + "step_time": 5.994720440998208 + } + ], + "logging_steps": 1, + "max_steps": 200, + "num_input_tokens_seen": 846398, + "num_train_epochs": 1, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-100/training_args.bin b/checkpoint-100/training_args.bin new file mode 100644 index 0000000..730dec0 --- /dev/null +++ b/checkpoint-100/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691fe5be2d7d1d56160dae63866d136877a72efb30e15ff7b3249192f998a788 +size 7185 diff --git a/checkpoint-120/chat_template.jinja b/checkpoint-120/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/checkpoint-120/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-120/config.json b/checkpoint-120/config.json new file mode 100644 index 0000000..f57b09b --- /dev/null +++ b/checkpoint-120/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-120/generation_config.json b/checkpoint-120/generation_config.json new file mode 100644 index 0000000..1b2bba9 --- /dev/null +++ b/checkpoint-120/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/checkpoint-120/model.safetensors b/checkpoint-120/model.safetensors new file mode 100644 index 0000000..6a9d42b --- /dev/null +++ b/checkpoint-120/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c4da3a283becce8e8206f10d185aee8fe0be0f1db7ddd6e86f8bb6c27c8c355 +size 6174895536 diff --git a/checkpoint-120/optimizer.pt b/checkpoint-120/optimizer.pt new file mode 100644 index 0000000..370be41 --- /dev/null +++ b/checkpoint-120/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d819fb438d02d0584096eedc109a92dcee131a733136d155f10f0608cd23c99 +size 12350013801 diff --git a/checkpoint-120/rng_state.pth b/checkpoint-120/rng_state.pth new file mode 100644 index 0000000..fab1d60 --- /dev/null +++ b/checkpoint-120/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23f4512500fe387abc57142c0ca62eb2d6ae3b2f6fa7cfce743b4111042fa89b +size 14645 diff --git a/checkpoint-120/scheduler.pt b/checkpoint-120/scheduler.pt new file mode 100644 index 0000000..7ffc8c1 --- /dev/null +++ b/checkpoint-120/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:440de387e2beaf1948cdb5c020c66cf05b31275efda10896ac069f091dc596e7 +size 1465 diff --git a/checkpoint-120/tokenizer.json b/checkpoint-120/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-120/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-120/tokenizer_config.json b/checkpoint-120/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-120/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-120/trainer_state.json b/checkpoint-120/trainer_state.json new file mode 100644 index 0000000..3960e1f --- /dev/null +++ b/checkpoint-120/trainer_state.json @@ -0,0 +1,3514 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6, + "eval_steps": 500, + "global_step": 120, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 191.0, + "completions/mean_terminated_length": 191.0, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "entropy": 0.5566893219947815, + "epoch": 0.005, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.370361328125, + "learning_rate": 1e-06, + "loss": -0.24658073484897614, + "num_tokens": 10832.0, + "reward": -3.3000030517578125, + "reward_std": 85.62333679199219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.4000015258789062, + "rewards/supergames_reward/std": 85.62333679199219, + "step": 1, + "step_time": 22.779711072013015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/max_terminated_length": 216.0, + "completions/mean_length": 131.5, + "completions/mean_terminated_length": 131.5, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.6395841240882874, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.09774398803711, + "learning_rate": 9.95e-07, + "loss": 0.20567649602890015, + "num_tokens": 16404.0, + "reward": -12.422499656677246, + "reward_std": 7.134707450866699, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.522500038146973, + "rewards/supergames_reward/std": 7.134707927703857, + "step": 2, + "step_time": 9.960156448010821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/max_terminated_length": 287.0, + "completions/mean_length": 156.625, + "completions/mean_terminated_length": 156.625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "entropy": 0.562222421169281, + "epoch": 0.015, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9053616523742676, + "learning_rate": 9.9e-07, + "loss": 0.09602774679660797, + "num_tokens": 26953.0, + "reward": -74.94999694824219, + "reward_std": 70.73094177246094, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -75.0, + "rewards/supergames_reward/std": 70.71067810058594, + "step": 3, + "step_time": 13.686320498993155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 154.125, + "completions/mean_terminated_length": 154.125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7323317527770996, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.933310031890869, + "learning_rate": 9.849999999999999e-07, + "loss": 0.28249427676200867, + "num_tokens": 36514.0, + "reward": -40.7599983215332, + "reward_std": 81.73140716552734, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -40.80999755859375, + "rewards/supergames_reward/std": 81.68995666503906, + "step": 4, + "step_time": 12.805880262021674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 205.25, + "completions/mean_terminated_length": 205.25, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.6167430281639099, + "epoch": 0.025, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.344135046005249, + "learning_rate": 9.8e-07, + "loss": 0.1433994024991989, + "num_tokens": 47476.0, + "reward": -97.6612548828125, + "reward_std": 6.3321428298950195, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.76124572753906, + "rewards/supergames_reward/std": 6.3321428298950195, + "step": 5, + "step_time": 18.789364666008623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/max_terminated_length": 276.0, + "completions/mean_length": 167.25, + "completions/mean_terminated_length": 167.25, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.7583790421485901, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.172371864318848, + "learning_rate": 9.75e-07, + "loss": -0.12127404659986496, + "num_tokens": 55246.0, + "reward": -58.687503814697266, + "reward_std": 58.64059829711914, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -58.76250457763672, + "rewards/supergames_reward/std": 58.620460510253906, + "step": 6, + "step_time": 12.948570273991209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 133.875, + "completions/mean_terminated_length": 133.875, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.7447654604911804, + "epoch": 0.035, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.428595066070557, + "learning_rate": 9.7e-07, + "loss": 0.10135584324598312, + "num_tokens": 60837.0, + "reward": -32.45000076293945, + "reward_std": 41.754486083984375, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -32.5, + "rewards/supergames_reward/std": 41.66190719604492, + "step": 7, + "step_time": 12.058315072004916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 414.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 209.125, + "completions/mean_terminated_length": 209.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.3860666751861572, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.806255578994751, + "learning_rate": 9.649999999999999e-07, + "loss": -0.33238139748573303, + "num_tokens": 70838.0, + "reward": -80.50375366210938, + "reward_std": 38.82380294799805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -80.60375213623047, + "rewards/supergames_reward/std": 38.82379913330078, + "step": 8, + "step_time": 19.323370319994865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 247.125, + "completions/mean_terminated_length": 158.83334350585938, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.5816237330436707, + "epoch": 0.045, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3209943771362305, + "learning_rate": 9.6e-07, + "loss": 0.36752766370773315, + "num_tokens": 82135.0, + "reward": -27.688751220703125, + "reward_std": 82.8358154296875, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -27.738750457763672, + "rewards/supergames_reward/std": 82.78590393066406, + "step": 9, + "step_time": 21.948575104994234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/max_terminated_length": 240.0, + "completions/mean_length": 153.0, + "completions/mean_terminated_length": 153.0, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6301521062850952, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.125359058380127, + "learning_rate": 9.55e-07, + "loss": 0.06129350885748863, + "num_tokens": 89807.0, + "reward": -78.39624786376953, + "reward_std": 40.17042922973633, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -78.44625091552734, + "rewards/supergames_reward/std": 40.1396484375, + "step": 10, + "step_time": 9.098202993016457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/max_terminated_length": 133.0, + "completions/mean_length": 112.0, + "completions/mean_terminated_length": 112.0, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.5521990656852722, + "epoch": 0.055, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.616060256958008, + "learning_rate": 9.499999999999999e-07, + "loss": -0.006313305348157883, + "num_tokens": 95231.0, + "reward": -12.161249160766602, + "reward_std": 6.39578104019165, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.261249542236328, + "rewards/supergames_reward/std": 6.39578104019165, + "step": 11, + "step_time": 5.136311663984088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 214.875, + "completions/mean_terminated_length": 214.875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.5505905151367188, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.289583206176758, + "learning_rate": 9.45e-07, + "loss": -0.29802003502845764, + "num_tokens": 106238.0, + "reward": -91.66999816894531, + "reward_std": 23.2779541015625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -91.77000427246094, + "rewards/supergames_reward/std": 23.277956008911133, + "step": 12, + "step_time": 14.945365622988902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 167.375, + "completions/mean_terminated_length": 167.375, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "entropy": 0.540415346622467, + "epoch": 0.065, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.858291149139404, + "learning_rate": 9.399999999999999e-07, + "loss": 0.24004128575325012, + "num_tokens": 116937.0, + "reward": -20.38249969482422, + "reward_std": 87.55204010009766, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.457500457763672, + "rewards/supergames_reward/std": 87.52605438232422, + "step": 13, + "step_time": 10.482285185018554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "entropy": 0.7237679958343506, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.083980560302734, + "learning_rate": 9.35e-07, + "loss": -0.19025824964046478, + "num_tokens": 124640.0, + "reward": -51.397499084472656, + "reward_std": 54.486454010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -51.49749755859375, + "rewards/supergames_reward/std": 54.486454010009766, + "step": 14, + "step_time": 9.597231683001155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 141.375, + "completions/mean_terminated_length": 141.375, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.754334568977356, + "epoch": 0.075, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.720428943634033, + "learning_rate": 9.3e-07, + "loss": -0.16830675303936005, + "num_tokens": 132219.0, + "reward": -56.226253509521484, + "reward_std": 60.115657806396484, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -56.32625198364258, + "rewards/supergames_reward/std": 60.11566162109375, + "step": 15, + "step_time": 8.998362872982398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.0, + "completions/max_terminated_length": 209.0, + "completions/mean_length": 136.375, + "completions/mean_terminated_length": 136.375, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.6280568838119507, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.9681806564331055, + "learning_rate": 9.25e-07, + "loss": -0.01910916529595852, + "num_tokens": 139726.0, + "reward": -68.24000549316406, + "reward_std": 52.238914489746094, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.33999633789062, + "rewards/supergames_reward/std": 52.238914489746094, + "step": 16, + "step_time": 8.034480294008972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 211.875, + "completions/mean_terminated_length": 211.875, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.899176836013794, + "epoch": 0.085, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.691746711730957, + "learning_rate": 9.2e-07, + "loss": 0.25513291358947754, + "num_tokens": 147869.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 17, + "step_time": 13.655792869016295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 387.0, + "completions/max_terminated_length": 387.0, + "completions/mean_length": 238.0, + "completions/mean_terminated_length": 238.0, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.5596873760223389, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6711347103118896, + "learning_rate": 9.15e-07, + "loss": -0.2628335654735565, + "num_tokens": 159101.0, + "reward": -24.900001525878906, + "reward_std": 103.50983428955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.0, + "rewards/supergames_reward/std": 103.50983428955078, + "step": 18, + "step_time": 14.784329420013819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 277.0, + "completions/mean_length": 219.75, + "completions/mean_terminated_length": 178.00001525878906, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.9245517253875732, + "epoch": 0.095, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55502986907959, + "learning_rate": 9.1e-07, + "loss": 0.13060179352760315, + "num_tokens": 167299.0, + "reward": -86.89125061035156, + "reward_std": 36.95603561401367, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -86.94125366210938, + "rewards/supergames_reward/std": 36.93572235107422, + "step": 19, + "step_time": 18.832684700988466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/max_terminated_length": 267.0, + "completions/mean_length": 177.25, + "completions/mean_terminated_length": 177.25, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.7331254482269287, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.538497447967529, + "learning_rate": 9.05e-07, + "loss": -0.16418980062007904, + "num_tokens": 175141.0, + "reward": -68.52874755859375, + "reward_std": 58.48719787597656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.62875366210938, + "rewards/supergames_reward/std": 58.48720169067383, + "step": 20, + "step_time": 10.090975169994636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 190.625, + "completions/mean_terminated_length": 190.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.6659425497055054, + "epoch": 0.105, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.485373497009277, + "learning_rate": 9e-07, + "loss": -0.10106432437896729, + "num_tokens": 183098.0, + "reward": -71.48875427246094, + "reward_std": 55.33565139770508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -71.5887451171875, + "rewards/supergames_reward/std": 55.33565139770508, + "step": 21, + "step_time": 11.255001295008697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 179.0, + "completions/mean_terminated_length": 179.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.49365949630737305, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.829419136047363, + "learning_rate": 8.95e-07, + "loss": 0.10587231814861298, + "num_tokens": 192826.0, + "reward": -75.62000274658203, + "reward_std": 45.01959991455078, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -75.69499969482422, + "rewards/supergames_reward/std": 45.00411605834961, + "step": 22, + "step_time": 10.611246599000879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/max_terminated_length": 315.0, + "completions/mean_length": 212.625, + "completions/mean_terminated_length": 212.625, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.43159469962120056, + "epoch": 0.115, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.541125774383545, + "learning_rate": 8.9e-07, + "loss": -0.1681259125471115, + "num_tokens": 202815.0, + "reward": 23.67624855041504, + "reward_std": 67.74340057373047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 23.576250076293945, + "rewards/supergames_reward/std": 67.743408203125, + "step": 23, + "step_time": 11.731771531980485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/max_terminated_length": 168.0, + "completions/mean_length": 101.625, + "completions/mean_terminated_length": 101.625, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.3972298502922058, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.8285417556762695, + "learning_rate": 8.85e-07, + "loss": -0.011585958302021027, + "num_tokens": 208188.0, + "reward": 58.849998474121094, + "reward_std": 56.9304084777832, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 58.75, + "rewards/supergames_reward/std": 56.9304084777832, + "step": 24, + "step_time": 6.187504950998118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 303.0, + "completions/max_terminated_length": 303.0, + "completions/mean_length": 165.375, + "completions/mean_terminated_length": 165.375, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "entropy": 0.6339899301528931, + "epoch": 0.125, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.823799133300781, + "learning_rate": 8.799999999999999e-07, + "loss": 0.05159185826778412, + "num_tokens": 215935.0, + "reward": -23.521251678466797, + "reward_std": 74.23067474365234, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -23.621248245239258, + "rewards/supergames_reward/std": 74.23067474365234, + "step": 25, + "step_time": 11.059416300005978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 349.0, + "completions/max_terminated_length": 349.0, + "completions/mean_length": 204.0, + "completions/mean_terminated_length": 204.0, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.6462895274162292, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.969292640686035, + "learning_rate": 8.75e-07, + "loss": 0.0115677984431386, + "num_tokens": 223999.0, + "reward": -74.8175048828125, + "reward_std": 40.94221115112305, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -74.89250183105469, + "rewards/supergames_reward/std": 40.924625396728516, + "step": 26, + "step_time": 12.666237785975682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 166.5, + "completions/mean_terminated_length": 166.5, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4756850600242615, + "epoch": 0.135, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915069103240967, + "learning_rate": 8.699999999999999e-07, + "loss": -0.13321346044540405, + "num_tokens": 233643.0, + "reward": -52.125, + "reward_std": 19.405282974243164, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -52.20000076293945, + "rewards/supergames_reward/std": 19.33464813232422, + "step": 27, + "step_time": 11.172539752995363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 137.125, + "completions/mean_terminated_length": 137.125, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.5834778547286987, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.146666526794434, + "learning_rate": 8.65e-07, + "loss": 0.060454584658145905, + "num_tokens": 241204.0, + "reward": -68.99250793457031, + "reward_std": 51.010581970214844, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -69.09249877929688, + "rewards/supergames_reward/std": 51.010581970214844, + "step": 28, + "step_time": 7.025046669004951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 182.375, + "completions/mean_terminated_length": 182.375, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.6368551254272461, + "epoch": 0.145, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.15535306930542, + "learning_rate": 8.599999999999999e-07, + "loss": -0.3323673903942108, + "num_tokens": 249087.0, + "reward": -44.17000198364258, + "reward_std": 80.5373306274414, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -44.27000045776367, + "rewards/supergames_reward/std": 80.5373306274414, + "step": 29, + "step_time": 12.079259724996518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 127.75, + "completions/mean_terminated_length": 127.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4491196870803833, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.896186828613281, + "learning_rate": 8.55e-07, + "loss": -0.12887656688690186, + "num_tokens": 259445.0, + "reward": 57.446250915527344, + "reward_std": 38.78837585449219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 57.34625244140625, + "rewards/supergames_reward/std": 38.78837966918945, + "step": 30, + "step_time": 7.483972909016302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/max_terminated_length": 166.0, + "completions/mean_length": 112.875, + "completions/mean_terminated_length": 112.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4858454465866089, + "epoch": 0.155, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.742906093597412, + "learning_rate": 8.499999999999999e-07, + "loss": 0.06930096447467804, + "num_tokens": 264924.0, + "reward": -21.174999237060547, + "reward_std": 31.89051628112793, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -21.25, + "rewards/supergames_reward/std": 31.819807052612305, + "step": 31, + "step_time": 6.1866529019898735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/max_terminated_length": 148.0, + "completions/mean_length": 118.125, + "completions/mean_terminated_length": 118.125, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.45428749918937683, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.663414478302002, + "learning_rate": 8.45e-07, + "loss": 0.05735419690608978, + "num_tokens": 270381.0, + "reward": 0.4687504768371582, + "reward_std": 38.66371154785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 0.36875057220458984, + "rewards/supergames_reward/std": 38.6637077331543, + "step": 32, + "step_time": 5.577042757999152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 166.875, + "completions/mean_terminated_length": 166.875, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.6107293367385864, + "epoch": 0.165, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.75006628036499, + "learning_rate": 8.399999999999999e-07, + "loss": -0.14760127663612366, + "num_tokens": 278156.0, + "reward": -87.84750366210938, + "reward_std": 26.84708023071289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -87.94749450683594, + "rewards/supergames_reward/std": 26.847076416015625, + "step": 33, + "step_time": 12.552876825997373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 360.0, + "completions/mean_length": 231.0, + "completions/mean_terminated_length": 190.85714721679688, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.6089653372764587, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.781267166137695, + "learning_rate": 8.349999999999999e-07, + "loss": 0.4294504225254059, + "num_tokens": 289364.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 34, + "step_time": 18.961819477990502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 134.125, + "completions/mean_terminated_length": 134.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46158918738365173, + "epoch": 0.175, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.016353607177734, + "learning_rate": 8.299999999999999e-07, + "loss": 0.09251818805932999, + "num_tokens": 298733.0, + "reward": -35.558753967285156, + "reward_std": 15.881204605102539, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -35.65875244140625, + "rewards/supergames_reward/std": 15.881203651428223, + "step": 35, + "step_time": 8.38491243700264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/max_terminated_length": 207.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.5104788541793823, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.592893123626709, + "learning_rate": 8.249999999999999e-07, + "loss": -0.0410832017660141, + "num_tokens": 306420.0, + "reward": 5.4212493896484375, + "reward_std": 93.50404357910156, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 5.346250534057617, + "rewards/supergames_reward/std": 93.4718246459961, + "step": 36, + "step_time": 7.811868985998444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/max_terminated_length": 241.0, + "completions/mean_length": 178.0, + "completions/mean_terminated_length": 178.0, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.7527878880500793, + "epoch": 0.185, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8812546730041504, + "learning_rate": 8.199999999999999e-07, + "loss": 0.12539231777191162, + "num_tokens": 314252.0, + "reward": -89.58500671386719, + "reward_std": 29.256120681762695, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -89.66000366210938, + "rewards/supergames_reward/std": 29.24593734741211, + "step": 37, + "step_time": 8.959742914012168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 133.375, + "completions/mean_terminated_length": 133.375, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.3347901403903961, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.205414295196533, + "learning_rate": 8.149999999999999e-07, + "loss": -0.06991486996412277, + "num_tokens": 319839.0, + "reward": -14.498749732971191, + "reward_std": 13.007229804992676, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -14.598750114440918, + "rewards/supergames_reward/std": 13.007229804992676, + "step": 38, + "step_time": 7.377183554985095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/max_terminated_length": 181.0, + "completions/mean_length": 114.375, + "completions/mean_terminated_length": 114.375, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.4445965886116028, + "epoch": 0.195, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.561298847198486, + "learning_rate": 8.1e-07, + "loss": -0.06235164776444435, + "num_tokens": 325290.0, + "reward": -7.425000190734863, + "reward_std": 53.701602935791016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -7.5, + "rewards/supergames_reward/std": 53.652320861816406, + "step": 39, + "step_time": 6.753862089011818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 187.625, + "completions/mean_terminated_length": 187.625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.3950234353542328, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5988712310791016, + "learning_rate": 8.05e-07, + "loss": -0.1381440907716751, + "num_tokens": 335119.0, + "reward": -33.68375015258789, + "reward_std": 42.069435119628906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -33.783748626708984, + "rewards/supergames_reward/std": 42.069435119628906, + "step": 40, + "step_time": 13.718958162004128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 371.0, + "completions/mean_length": 251.125, + "completions/mean_terminated_length": 213.85714721679688, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.507067084312439, + "epoch": 0.205, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3898658752441406, + "learning_rate": 8e-07, + "loss": 0.0078964838758111, + "num_tokens": 346464.0, + "reward": -86.48625183105469, + "reward_std": 38.020694732666016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -86.56124877929688, + "rewards/supergames_reward/std": 38.010528564453125, + "step": 41, + "step_time": 18.90960379401804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 152.0, + "completions/max_terminated_length": 152.0, + "completions/mean_length": 116.375, + "completions/mean_terminated_length": 116.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3801707923412323, + "epoch": 0.21, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.95e-07, + "loss": 0.0, + "num_tokens": 351955.0, + "reward": -9.899999618530273, + "reward_std": 0.0, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -10.0, + "rewards/supergames_reward/std": 0.0, + "step": 42, + "step_time": 5.711630532023264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/max_terminated_length": 308.0, + "completions/mean_length": 247.375, + "completions/mean_terminated_length": 247.375, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.3185043931007385, + "epoch": 0.215, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.168762445449829, + "learning_rate": 7.9e-07, + "loss": 0.1970764398574829, + "num_tokens": 363278.0, + "reward": -53.89249801635742, + "reward_std": 63.02619934082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -53.99250030517578, + "rewards/supergames_reward/std": 63.02619934082031, + "step": 43, + "step_time": 11.661934906995157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/max_terminated_length": 330.0, + "completions/mean_length": 212.5, + "completions/mean_terminated_length": 212.5, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4446268379688263, + "epoch": 0.22, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.305844306945801, + "learning_rate": 7.85e-07, + "loss": -0.06735197454690933, + "num_tokens": 374306.0, + "reward": 18.059999465942383, + "reward_std": 98.9417953491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 17.959999084472656, + "rewards/supergames_reward/std": 98.9417953491211, + "step": 44, + "step_time": 12.492453911982011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.0, + "completions/max_terminated_length": 316.0, + "completions/mean_length": 183.0, + "completions/mean_terminated_length": 183.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.5269614458084106, + "epoch": 0.225, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8006370067596436, + "learning_rate": 7.799999999999999e-07, + "loss": -0.08870815485715866, + "num_tokens": 382210.0, + "reward": -54.928749084472656, + "reward_std": 72.5127944946289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -55.02874755859375, + "rewards/supergames_reward/std": 72.5127944946289, + "step": 45, + "step_time": 11.7636467939883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.0, + "completions/max_terminated_length": 244.0, + "completions/mean_length": 136.625, + "completions/mean_terminated_length": 136.625, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.34820204973220825, + "epoch": 0.23, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.911736011505127, + "learning_rate": 7.75e-07, + "loss": 0.04397330805659294, + "num_tokens": 391583.0, + "reward": -19.10375213623047, + "reward_std": 23.967702865600586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -19.203750610351562, + "rewards/supergames_reward/std": 23.967702865600586, + "step": 46, + "step_time": 10.119426151999505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 160.375, + "completions/mean_terminated_length": 160.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.611914336681366, + "epoch": 0.235, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.571913242340088, + "learning_rate": 7.699999999999999e-07, + "loss": -0.044293053448200226, + "num_tokens": 399290.0, + "reward": -25.186250686645508, + "reward_std": 71.46305084228516, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.286251068115234, + "rewards/supergames_reward/std": 71.46305084228516, + "step": 47, + "step_time": 8.563868903991533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/max_terminated_length": 167.0, + "completions/mean_length": 111.125, + "completions/mean_terminated_length": 111.125, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.46585777401924133, + "epoch": 0.24, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.654109954833984, + "learning_rate": 7.65e-07, + "loss": -0.035151124000549316, + "num_tokens": 404707.0, + "reward": -0.408750057220459, + "reward_std": 31.939748764038086, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -0.5087499618530273, + "rewards/supergames_reward/std": 31.93975257873535, + "step": 48, + "step_time": 6.438482580007985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/max_terminated_length": 306.0, + "completions/mean_length": 216.625, + "completions/mean_terminated_length": 216.625, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.33054471015930176, + "epoch": 0.245, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.700303792953491, + "learning_rate": 7.599999999999999e-07, + "loss": -0.04393656551837921, + "num_tokens": 414736.0, + "reward": -97.26000213623047, + "reward_std": 1.6294406652450562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.36000061035156, + "rewards/supergames_reward/std": 1.629441738128662, + "step": 49, + "step_time": 11.636040106008295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 378.0, + "completions/max_terminated_length": 378.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.48711177706718445, + "epoch": 0.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9650564193725586, + "learning_rate": 7.55e-07, + "loss": -0.3529520332813263, + "num_tokens": 424795.0, + "reward": 47.408748626708984, + "reward_std": 72.72083282470703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 47.30875015258789, + "rewards/supergames_reward/std": 72.72083282470703, + "step": 50, + "step_time": 14.258096842997475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 267.625, + "completions/mean_terminated_length": 232.71429443359375, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4482860863208771, + "epoch": 0.255, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2998197078704834, + "learning_rate": 7.5e-07, + "loss": 0.2493654191493988, + "num_tokens": 436288.0, + "reward": 50.61125183105469, + "reward_std": 75.3349838256836, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.53624725341797, + "rewards/supergames_reward/std": 75.27783966064453, + "step": 51, + "step_time": 19.20827590499539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 176.0, + "completions/max_terminated_length": 176.0, + "completions/mean_length": 144.5, + "completions/mean_terminated_length": 144.5, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "entropy": 0.5378735661506653, + "epoch": 0.26, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.114461898803711, + "learning_rate": 7.45e-07, + "loss": 0.0036597400903701782, + "num_tokens": 441980.0, + "reward": -15.20625114440918, + "reward_std": 45.35526657104492, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.28125, + "rewards/supergames_reward/std": 45.301815032958984, + "step": 52, + "step_time": 6.660627231001854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/max_terminated_length": 136.0, + "completions/mean_length": 119.0, + "completions/mean_terminated_length": 119.0, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "entropy": 0.41796669363975525, + "epoch": 0.265, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.059505462646484, + "learning_rate": 7.4e-07, + "loss": -0.014973883517086506, + "num_tokens": 447468.0, + "reward": 15.170000076293945, + "reward_std": 46.658329010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 15.070000648498535, + "rewards/supergames_reward/std": 46.658329010009766, + "step": 53, + "step_time": 5.271571868011961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/max_terminated_length": 259.0, + "completions/mean_length": 191.125, + "completions/mean_terminated_length": 191.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.45421987771987915, + "epoch": 0.27, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4454867839813232, + "learning_rate": 7.35e-07, + "loss": -0.1128973588347435, + "num_tokens": 457301.0, + "reward": 38.89875030517578, + "reward_std": 62.1772575378418, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 38.79874801635742, + "rewards/supergames_reward/std": 62.1772575378418, + "step": 54, + "step_time": 10.524528659996577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/max_terminated_length": 319.0, + "completions/mean_length": 233.375, + "completions/mean_terminated_length": 233.375, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "entropy": 0.4449213445186615, + "epoch": 0.275, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2065398693084717, + "learning_rate": 7.3e-07, + "loss": 0.0016277075046673417, + "num_tokens": 467520.0, + "reward": 27.372499465942383, + "reward_std": 81.23126983642578, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.27250099182129, + "rewards/supergames_reward/std": 81.23126983642578, + "step": 55, + "step_time": 12.592280682991259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 357.0, + "completions/mean_length": 261.0, + "completions/mean_terminated_length": 225.1428680419922, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.506920337677002, + "epoch": 0.28, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.058388710021973, + "learning_rate": 7.249999999999999e-07, + "loss": 0.3289242386817932, + "num_tokens": 476056.0, + "reward": 47.05249786376953, + "reward_std": 80.97753143310547, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 46.977500915527344, + "rewards/supergames_reward/std": 80.9256362915039, + "step": 56, + "step_time": 18.845177220006008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.0, + "completions/max_terminated_length": 250.0, + "completions/mean_length": 192.75, + "completions/mean_terminated_length": 192.75, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "entropy": 0.5197336077690125, + "epoch": 0.285, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.230815410614014, + "learning_rate": 7.2e-07, + "loss": -0.15186546742916107, + "num_tokens": 484014.0, + "reward": -64.5050048828125, + "reward_std": 50.35145568847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -64.60499572753906, + "rewards/supergames_reward/std": 50.3514518737793, + "step": 57, + "step_time": 9.576232638006331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 249.0, + "completions/mean_terminated_length": 249.0, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "entropy": 0.42913514375686646, + "epoch": 0.29, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.8544528484344482, + "learning_rate": 7.149999999999999e-07, + "loss": -0.056405920535326004, + "num_tokens": 494390.0, + "reward": 40.123748779296875, + "reward_std": 84.164794921875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 40.02375030517578, + "rewards/supergames_reward/std": 84.164794921875, + "step": 58, + "step_time": 12.385353341000155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 138.0, + "completions/max_terminated_length": 138.0, + "completions/mean_length": 107.25, + "completions/mean_terminated_length": 107.25, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.34386488795280457, + "epoch": 0.295, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.6915154457092285, + "learning_rate": 7.1e-07, + "loss": 0.0865321159362793, + "num_tokens": 499816.0, + "reward": -8.244999885559082, + "reward_std": 4.681046962738037, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.345000267028809, + "rewards/supergames_reward/std": 4.681046962738037, + "step": 59, + "step_time": 5.42325339600211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/max_terminated_length": 187.0, + "completions/mean_length": 127.25, + "completions/mean_terminated_length": 127.25, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.5430045127868652, + "epoch": 0.3, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.635311126708984, + "learning_rate": 7.049999999999999e-07, + "loss": -0.05412375554442406, + "num_tokens": 505410.0, + "reward": 20.005001068115234, + "reward_std": 55.156005859375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 19.904998779296875, + "rewards/supergames_reward/std": 55.156005859375, + "step": 60, + "step_time": 7.068010902003152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/max_terminated_length": 286.0, + "completions/mean_length": 192.125, + "completions/mean_terminated_length": 192.125, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4213869869709015, + "epoch": 0.305, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4664618968963623, + "learning_rate": 7e-07, + "loss": -0.0495159812271595, + "num_tokens": 516283.0, + "reward": -47.212501525878906, + "reward_std": 91.2253646850586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -47.3125, + "rewards/supergames_reward/std": 91.2253646850586, + "step": 61, + "step_time": 11.176304235996213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 171.0, + "completions/mean_terminated_length": 171.0, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.4626811146736145, + "epoch": 0.31, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.949252128601074, + "learning_rate": 6.949999999999999e-07, + "loss": 0.192047581076622, + "num_tokens": 522251.0, + "reward": -15.092500686645508, + "reward_std": 49.82962417602539, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.167499542236328, + "rewards/supergames_reward/std": 49.78091049194336, + "step": 62, + "step_time": 12.075224861997413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 429.0, + "completions/mean_length": 283.5, + "completions/mean_terminated_length": 250.85714721679688, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4814216196537018, + "epoch": 0.315, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7918753623962402, + "learning_rate": 6.9e-07, + "loss": 0.15706156194210052, + "num_tokens": 533815.0, + "reward": -20.86625099182129, + "reward_std": 85.99250030517578, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.94124984741211, + "rewards/supergames_reward/std": 85.96620178222656, + "step": 63, + "step_time": 19.448832260008203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.0, + "completions/max_terminated_length": 189.0, + "completions/mean_length": 131.25, + "completions/mean_terminated_length": 131.25, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4929414987564087, + "epoch": 0.32, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.774101734161377, + "learning_rate": 6.85e-07, + "loss": -0.060691747814416885, + "num_tokens": 539465.0, + "reward": -21.0674991607666, + "reward_std": 6.892730236053467, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -21.167499542236328, + "rewards/supergames_reward/std": 6.892730236053467, + "step": 64, + "step_time": 7.141569407976931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 454.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 281.375, + "completions/mean_terminated_length": 281.375, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "entropy": 0.45866572856903076, + "epoch": 0.325, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.092717409133911, + "learning_rate": 6.800000000000001e-07, + "loss": 0.11694959551095963, + "num_tokens": 551012.0, + "reward": 49.938751220703125, + "reward_std": 79.1865005493164, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 49.8387451171875, + "rewards/supergames_reward/std": 79.1865005493164, + "step": 65, + "step_time": 17.06170882002334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/max_terminated_length": 203.0, + "completions/mean_length": 128.625, + "completions/mean_terminated_length": 128.625, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.5005782246589661, + "epoch": 0.33, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.77616024017334, + "learning_rate": 6.75e-07, + "loss": 0.11242837458848953, + "num_tokens": 556577.0, + "reward": 48.875, + "reward_std": 49.13539123535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.775001525878906, + "rewards/supergames_reward/std": 49.13539123535156, + "step": 66, + "step_time": 7.695410129002994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 436.0, + "completions/max_terminated_length": 436.0, + "completions/mean_length": 281.875, + "completions/mean_terminated_length": 281.875, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "entropy": 0.507097065448761, + "epoch": 0.335, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.911184787750244, + "learning_rate": 6.7e-07, + "loss": -0.08151137083768845, + "num_tokens": 567120.0, + "reward": -12.886249542236328, + "reward_std": 84.04269409179688, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.986251831054688, + "rewards/supergames_reward/std": 84.04269409179688, + "step": 67, + "step_time": 16.409127942984924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 318.375, + "completions/mean_terminated_length": 290.71429443359375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.486402690410614, + "epoch": 0.34, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9461607933044434, + "learning_rate": 6.65e-07, + "loss": 0.2442178875207901, + "num_tokens": 578995.0, + "reward": 2.8387489318847656, + "reward_std": 98.47579956054688, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 2.7637500762939453, + "rewards/supergames_reward/std": 98.44595336914062, + "step": 68, + "step_time": 19.32364999302081 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 291.0, + "completions/max_terminated_length": 291.0, + "completions/mean_length": 219.625, + "completions/mean_terminated_length": 219.625, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.37856727838516235, + "epoch": 0.345, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0204732418060303, + "learning_rate": 6.6e-07, + "loss": -0.08764916658401489, + "num_tokens": 589072.0, + "reward": -54.18375015258789, + "reward_std": 71.45012664794922, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -54.283748626708984, + "rewards/supergames_reward/std": 71.45013427734375, + "step": 69, + "step_time": 11.126611230982235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 305.0, + "completions/mean_length": 254.75, + "completions/mean_terminated_length": 218.00001525878906, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46423614025115967, + "epoch": 0.35, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0140154361724854, + "learning_rate": 6.55e-07, + "loss": 0.26797160506248474, + "num_tokens": 600438.0, + "reward": -32.04875183105469, + "reward_std": 69.86957550048828, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -32.12375259399414, + "rewards/supergames_reward/std": 69.84178161621094, + "step": 70, + "step_time": 19.12617294798838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 372.0, + "completions/max_terminated_length": 372.0, + "completions/mean_length": 194.125, + "completions/mean_terminated_length": 194.125, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.3427293300628662, + "epoch": 0.355, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.781383991241455, + "learning_rate": 6.5e-07, + "loss": 0.354524701833725, + "num_tokens": 610327.0, + "reward": -6.913749694824219, + "reward_std": 87.56071472167969, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -6.988750457763672, + "rewards/supergames_reward/std": 87.53033447265625, + "step": 71, + "step_time": 14.041668199002743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/max_terminated_length": 197.0, + "completions/mean_length": 124.75, + "completions/mean_terminated_length": 124.75, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.4455118477344513, + "epoch": 0.36, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.726032733917236, + "learning_rate": 6.45e-07, + "loss": 0.18206289410591125, + "num_tokens": 615853.0, + "reward": 41.95124816894531, + "reward_std": 60.640811920166016, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 41.85124969482422, + "rewards/supergames_reward/std": 60.640811920166016, + "step": 72, + "step_time": 7.416493425989756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.0, + "completions/max_terminated_length": 323.0, + "completions/mean_length": 147.375, + "completions/mean_terminated_length": 147.375, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.480733186006546, + "epoch": 0.365, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.853621482849121, + "learning_rate": 6.4e-07, + "loss": 0.04558124020695686, + "num_tokens": 621584.0, + "reward": 14.884998321533203, + "reward_std": 46.25857162475586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 14.78499984741211, + "rewards/supergames_reward/std": 46.25857162475586, + "step": 73, + "step_time": 11.95379540900467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 283.0, + "completions/max_terminated_length": 283.0, + "completions/mean_length": 188.125, + "completions/mean_terminated_length": 188.125, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.5526013374328613, + "epoch": 0.37, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.729883909225464, + "learning_rate": 6.35e-07, + "loss": 0.06268303096294403, + "num_tokens": 629545.0, + "reward": -68.05750274658203, + "reward_std": 59.44808578491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.15750122070312, + "rewards/supergames_reward/std": 59.44808578491211, + "step": 74, + "step_time": 10.64124580900534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 141.0, + "completions/max_terminated_length": 141.0, + "completions/mean_length": 117.375, + "completions/mean_terminated_length": 117.375, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.3744319677352905, + "epoch": 0.375, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.775154113769531, + "learning_rate": 6.3e-07, + "loss": -0.023284845054149628, + "num_tokens": 635052.0, + "reward": 3.9374990463256836, + "reward_std": 61.85771179199219, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 3.862499237060547, + "rewards/supergames_reward/std": 61.80967712402344, + "step": 75, + "step_time": 5.432648951013107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 157.0, + "completions/mean_terminated_length": 157.0, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4822140336036682, + "epoch": 0.38, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.715704917907715, + "learning_rate": 6.249999999999999e-07, + "loss": -0.04918142035603523, + "num_tokens": 644644.0, + "reward": 48.64125061035156, + "reward_std": 78.0205078125, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 48.56624984741211, + "rewards/supergames_reward/std": 77.96605682373047, + "step": 76, + "step_time": 10.348935816989979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/max_terminated_length": 380.0, + "completions/mean_length": 210.875, + "completions/mean_terminated_length": 210.875, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "entropy": 0.45722317695617676, + "epoch": 0.385, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6057567596435547, + "learning_rate": 6.2e-07, + "loss": 0.058352649211883545, + "num_tokens": 652771.0, + "reward": -8.283750534057617, + "reward_std": 77.68436431884766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.383749008178711, + "rewards/supergames_reward/std": 77.68437194824219, + "step": 77, + "step_time": 14.142948755004909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 188.25, + "completions/mean_terminated_length": 188.25, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.33380764722824097, + "epoch": 0.39, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2239863872528076, + "learning_rate": 6.149999999999999e-07, + "loss": -0.04592633992433548, + "num_tokens": 662653.0, + "reward": 11.153749465942383, + "reward_std": 89.88329315185547, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 11.053749084472656, + "rewards/supergames_reward/std": 89.88329315185547, + "step": 78, + "step_time": 12.277474621019792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/max_terminated_length": 143.0, + "completions/mean_length": 108.875, + "completions/mean_terminated_length": 108.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.3518812656402588, + "epoch": 0.395, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.967776298522949, + "learning_rate": 6.1e-07, + "loss": -0.10800496488809586, + "num_tokens": 668076.0, + "reward": 70.33250427246094, + "reward_std": 49.9145393371582, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 70.23249816894531, + "rewards/supergames_reward/std": 49.91453552246094, + "step": 79, + "step_time": 5.52714040101273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 152.125, + "completions/mean_terminated_length": 152.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4699896574020386, + "epoch": 0.4, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.561161994934082, + "learning_rate": 6.049999999999999e-07, + "loss": 0.09505834430456161, + "num_tokens": 675701.0, + "reward": -20.25625228881836, + "reward_std": 53.78542709350586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -20.35624885559082, + "rewards/supergames_reward/std": 53.78542709350586, + "step": 80, + "step_time": 7.155081019998761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 288.875, + "completions/mean_terminated_length": 257.0, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.4086494445800781, + "epoch": 0.405, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.3719611167907715, + "learning_rate": 6e-07, + "loss": 0.04727627709507942, + "num_tokens": 687332.0, + "reward": -37.20750045776367, + "reward_std": 91.7676010131836, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -37.25749969482422, + "rewards/supergames_reward/std": 91.72847747802734, + "step": 81, + "step_time": 19.231399144016905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 376.0, + "completions/max_terminated_length": 376.0, + "completions/mean_length": 227.75, + "completions/mean_terminated_length": 227.75, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.43233707547187805, + "epoch": 0.41, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3391714096069336, + "learning_rate": 5.949999999999999e-07, + "loss": -0.25524550676345825, + "num_tokens": 698474.0, + "reward": -3.7287511825561523, + "reward_std": 91.25379943847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.828749656677246, + "rewards/supergames_reward/std": 91.25379943847656, + "step": 82, + "step_time": 14.329176368017215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 178.125, + "completions/mean_terminated_length": 178.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.4293636977672577, + "epoch": 0.415, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.7271742820739746, + "learning_rate": 5.9e-07, + "loss": 0.09273672848939896, + "num_tokens": 708243.0, + "reward": 12.802498817443848, + "reward_std": 86.23526000976562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 12.702500343322754, + "rewards/supergames_reward/std": 86.23526000976562, + "step": 83, + "step_time": 11.419686011999147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 425.0, + "completions/max_terminated_length": 425.0, + "completions/mean_length": 255.375, + "completions/mean_terminated_length": 255.375, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "entropy": 0.38068246841430664, + "epoch": 0.42, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.6244184970855713, + "learning_rate": 5.849999999999999e-07, + "loss": 0.02006213366985321, + "num_tokens": 718606.0, + "reward": 37.803749084472656, + "reward_std": 57.58624267578125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 37.70375061035156, + "rewards/supergames_reward/std": 57.58624267578125, + "step": 84, + "step_time": 15.975198492989875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 116.0, + "completions/max_terminated_length": 116.0, + "completions/mean_length": 108.125, + "completions/mean_terminated_length": 108.125, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.339572012424469, + "epoch": 0.425, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.754759311676025, + "learning_rate": 5.8e-07, + "loss": -0.011015941388905048, + "num_tokens": 723983.0, + "reward": -3.8212504386901855, + "reward_std": 61.624786376953125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.921250820159912, + "rewards/supergames_reward/std": 61.624786376953125, + "step": 85, + "step_time": 4.542777584982105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 447.0, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 291.375, + "completions/mean_terminated_length": 291.375, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "entropy": 0.5846173763275146, + "epoch": 0.43, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55724835395813, + "learning_rate": 5.749999999999999e-07, + "loss": -0.04981120675802231, + "num_tokens": 735642.0, + "reward": 50.07499694824219, + "reward_std": 92.6283187866211, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.0, + "rewards/supergames_reward/std": 92.5820083618164, + "step": 86, + "step_time": 16.777178087009815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/max_terminated_length": 232.0, + "completions/mean_length": 192.0, + "completions/mean_terminated_length": 192.0, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "entropy": 0.5139474272727966, + "epoch": 0.435, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8860411643981934, + "learning_rate": 5.699999999999999e-07, + "loss": -0.02194221317768097, + "num_tokens": 743602.0, + "reward": -52.522499084472656, + "reward_std": 73.8686294555664, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -52.62249755859375, + "rewards/supergames_reward/std": 73.86863708496094, + "step": 87, + "step_time": 8.772893911984283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/max_terminated_length": 266.0, + "completions/mean_length": 146.875, + "completions/mean_terminated_length": 146.875, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.3428497910499573, + "epoch": 0.44, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.509483337402344, + "learning_rate": 5.649999999999999e-07, + "loss": 0.19857533276081085, + "num_tokens": 753065.0, + "reward": 63.63874816894531, + "reward_std": 67.54963684082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 63.53874969482422, + "rewards/supergames_reward/std": 67.54963684082031, + "step": 88, + "step_time": 10.230529835011112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 446.0, + "completions/max_terminated_length": 446.0, + "completions/mean_length": 270.625, + "completions/mean_terminated_length": 270.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.4417419731616974, + "epoch": 0.445, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.973951816558838, + "learning_rate": 5.6e-07, + "loss": -0.00031440958264283836, + "num_tokens": 764534.0, + "reward": -9.736251831054688, + "reward_std": 92.25879669189453, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -9.836250305175781, + "rewards/supergames_reward/std": 92.25880432128906, + "step": 89, + "step_time": 17.40752330099349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 398.0, + "completions/max_terminated_length": 398.0, + "completions/mean_length": 227.375, + "completions/mean_terminated_length": 227.375, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.4732024073600769, + "epoch": 0.45, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2560815811157227, + "learning_rate": 5.55e-07, + "loss": 0.03820464387536049, + "num_tokens": 772809.0, + "reward": 5.982499599456787, + "reward_std": 58.833797454833984, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 5.8824992179870605, + "rewards/supergames_reward/std": 58.83380126953125, + "step": 90, + "step_time": 15.007269965979503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 207.5, + "completions/mean_terminated_length": 207.5, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.42038559913635254, + "epoch": 0.455, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3649117946624756, + "learning_rate": 5.5e-07, + "loss": 0.1510048508644104, + "num_tokens": 782805.0, + "reward": -5.61500358581543, + "reward_std": 82.67152404785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -5.714998245239258, + "rewards/supergames_reward/std": 82.67152404785156, + "step": 91, + "step_time": 10.386137172987219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 416.0, + "completions/max_terminated_length": 416.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.3971734642982483, + "epoch": 0.46, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.433772563934326, + "learning_rate": 5.45e-07, + "loss": -0.3027213513851166, + "num_tokens": 793840.0, + "reward": -29.06500244140625, + "reward_std": 91.68325805664062, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -29.16499900817871, + "rewards/supergames_reward/std": 91.68325805664062, + "step": 92, + "step_time": 15.790611553995404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 206.625, + "completions/mean_terminated_length": 206.625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.5906907320022583, + "epoch": 0.465, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.293088912963867, + "learning_rate": 5.4e-07, + "loss": 0.09760032594203949, + "num_tokens": 803821.0, + "reward": -72.73999786376953, + "reward_std": 50.72484588623047, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -72.81500244140625, + "rewards/supergames_reward/std": 50.7094841003418, + "step": 93, + "step_time": 13.043119941983605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/max_terminated_length": 143.0, + "completions/mean_length": 115.75, + "completions/mean_terminated_length": 115.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4233115315437317, + "epoch": 0.47, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.58886194229126, + "learning_rate": 5.35e-07, + "loss": -0.04040650278329849, + "num_tokens": 809299.0, + "reward": -3.3050003051757812, + "reward_std": 24.443714141845703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.405000686645508, + "rewards/supergames_reward/std": 24.443714141845703, + "step": 94, + "step_time": 5.512874762003776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 122.0, + "completions/max_terminated_length": 122.0, + "completions/mean_length": 103.125, + "completions/mean_terminated_length": 103.125, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.4055764377117157, + "epoch": 0.475, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.981549263000488, + "learning_rate": 5.3e-07, + "loss": -0.07588938623666763, + "num_tokens": 814716.0, + "reward": 60.06999969482422, + "reward_std": 42.708473205566406, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 59.970001220703125, + "rewards/supergames_reward/std": 42.708473205566406, + "step": 95, + "step_time": 4.766389057011111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 475.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.408913254737854, + "epoch": 0.48, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.79402756690979, + "learning_rate": 5.25e-07, + "loss": -0.3087541460990906, + "num_tokens": 824791.0, + "reward": -15.042500495910645, + "reward_std": 84.37246704101562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -15.142499923706055, + "rewards/supergames_reward/std": 84.37246704101562, + "step": 96, + "step_time": 17.779843941010768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/max_terminated_length": 154.0, + "completions/mean_length": 121.625, + "completions/mean_terminated_length": 121.625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.529015839099884, + "epoch": 0.485, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.142816066741943, + "learning_rate": 5.2e-07, + "loss": 0.07216037809848785, + "num_tokens": 830300.0, + "reward": -32.743751525878906, + "reward_std": 54.61775588989258, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -32.81875228881836, + "rewards/supergames_reward/std": 54.58256149291992, + "step": 97, + "step_time": 5.913989890017547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 126.0, + "completions/max_terminated_length": 126.0, + "completions/mean_length": 95.625, + "completions/mean_terminated_length": 95.625, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.33743274211883545, + "epoch": 0.49, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.197273254394531, + "learning_rate": 5.149999999999999e-07, + "loss": 0.07500407844781876, + "num_tokens": 835633.0, + "reward": 11.00999927520752, + "reward_std": 62.73848342895508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 10.910000801086426, + "rewards/supergames_reward/std": 62.73848342895508, + "step": 98, + "step_time": 4.9155233050114475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 108.5, + "completions/mean_terminated_length": 108.5, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.3106803297996521, + "epoch": 0.495, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.864002227783203, + "learning_rate": 5.1e-07, + "loss": -0.1206185445189476, + "num_tokens": 841045.0, + "reward": 14.238749504089355, + "reward_std": 51.77016830444336, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 14.138750076293945, + "rewards/supergames_reward/std": 51.77016830444336, + "step": 99, + "step_time": 5.7507751359953545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 97.125, + "completions/mean_terminated_length": 97.125, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.35160186886787415, + "epoch": 0.5, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915731906890869, + "learning_rate": 5.049999999999999e-07, + "loss": -0.07951541244983673, + "num_tokens": 846398.0, + "reward": 67.78125, + "reward_std": 38.15436553955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 67.68124389648438, + "rewards/supergames_reward/std": 38.15436553955078, + "step": 100, + "step_time": 5.994720440998208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 320.0, + "completions/max_terminated_length": 320.0, + "completions/mean_length": 208.75, + "completions/mean_terminated_length": 208.75, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.452095091342926, + "epoch": 0.505, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.065176963806152, + "learning_rate": 5e-07, + "loss": -0.1397111564874649, + "num_tokens": 857420.0, + "reward": -5.801251411437988, + "reward_std": 73.56117248535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -5.90125036239624, + "rewards/supergames_reward/std": 73.5611801147461, + "step": 101, + "step_time": 12.28178753197426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/max_terminated_length": 230.0, + "completions/mean_length": 171.375, + "completions/mean_terminated_length": 171.375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4138447642326355, + "epoch": 0.51, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.998823404312134, + "learning_rate": 4.95e-07, + "loss": -0.08941879868507385, + "num_tokens": 867103.0, + "reward": 92.24749755859375, + "reward_std": 14.540005683898926, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 92.14749908447266, + "rewards/supergames_reward/std": 14.540006637573242, + "step": 102, + "step_time": 8.992682139010867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 311.0, + "completions/max_terminated_length": 311.0, + "completions/mean_length": 163.0, + "completions/mean_terminated_length": 163.0, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.4182547330856323, + "epoch": 0.515, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.419595241546631, + "learning_rate": 4.9e-07, + "loss": -0.19304415583610535, + "num_tokens": 876679.0, + "reward": 79.13249969482422, + "reward_std": 59.3050422668457, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 79.03250122070312, + "rewards/supergames_reward/std": 59.30504608154297, + "step": 103, + "step_time": 11.836970244999975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 253.0, + "completions/max_terminated_length": 253.0, + "completions/mean_length": 155.625, + "completions/mean_terminated_length": 155.625, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.49889081716537476, + "epoch": 0.52, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.463719367980957, + "learning_rate": 4.85e-07, + "loss": -0.04786720871925354, + "num_tokens": 884348.0, + "reward": 48.616249084472656, + "reward_std": 73.30741119384766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.51625061035156, + "rewards/supergames_reward/std": 73.30741119384766, + "step": 104, + "step_time": 9.579507841990562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/max_terminated_length": 380.0, + "completions/mean_length": 269.5, + "completions/mean_terminated_length": 269.5, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "entropy": 0.5286482572555542, + "epoch": 0.525, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.993295431137085, + "learning_rate": 4.8e-07, + "loss": -0.08071611076593399, + "num_tokens": 894856.0, + "reward": 34.970001220703125, + "reward_std": 85.324462890625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 34.869998931884766, + "rewards/supergames_reward/std": 85.324462890625, + "step": 105, + "step_time": 14.295730627985904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 203.25, + "completions/mean_terminated_length": 203.25, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.6049264073371887, + "epoch": 0.53, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.787306785583496, + "learning_rate": 4.7499999999999995e-07, + "loss": 0.19928883016109467, + "num_tokens": 902922.0, + "reward": 32.44499969482422, + "reward_std": 70.4663314819336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 32.369998931884766, + "rewards/supergames_reward/std": 70.41261291503906, + "step": 106, + "step_time": 9.84894504098338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 194.0, + "completions/mean_terminated_length": 194.0, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "entropy": 0.5115755200386047, + "epoch": 0.535, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.277207374572754, + "learning_rate": 4.6999999999999995e-07, + "loss": 0.09118300676345825, + "num_tokens": 910906.0, + "reward": 40.432498931884766, + "reward_std": 48.54548263549805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 40.33250045776367, + "rewards/supergames_reward/std": 48.54548263549805, + "step": 107, + "step_time": 10.466603949986165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 332.0, + "completions/max_terminated_length": 332.0, + "completions/mean_length": 195.875, + "completions/mean_terminated_length": 195.875, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4192962944507599, + "epoch": 0.54, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.308061122894287, + "learning_rate": 4.65e-07, + "loss": 0.28064602613449097, + "num_tokens": 920745.0, + "reward": -21.982500076293945, + "reward_std": 66.31195068359375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -22.08249855041504, + "rewards/supergames_reward/std": 66.31195068359375, + "step": 108, + "step_time": 12.595848233992001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 195.125, + "completions/mean_terminated_length": 195.125, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.37523961067199707, + "epoch": 0.545, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.802706718444824, + "learning_rate": 4.6e-07, + "loss": -0.06614465266466141, + "num_tokens": 930618.0, + "reward": 71.75, + "reward_std": 51.14524459838867, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 71.6500015258789, + "rewards/supergames_reward/std": 51.1452522277832, + "step": 109, + "step_time": 10.33884758799104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 167.75, + "completions/mean_terminated_length": 167.75, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.4339042007923126, + "epoch": 0.55, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.254003047943115, + "learning_rate": 4.55e-07, + "loss": 0.34405598044395447, + "num_tokens": 941256.0, + "reward": 99.21875, + "reward_std": 2.492553234100342, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 99.1187515258789, + "rewards/supergames_reward/std": 2.4925525188446045, + "step": 110, + "step_time": 12.657525141985388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 285.0, + "completions/max_terminated_length": 285.0, + "completions/mean_length": 211.0, + "completions/mean_terminated_length": 211.0, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "entropy": 0.46809861063957214, + "epoch": 0.555, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.811166286468506, + "learning_rate": 4.5e-07, + "loss": 0.045113153755664825, + "num_tokens": 951296.0, + "reward": 55.27750015258789, + "reward_std": 47.90687561035156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 55.1775016784668, + "rewards/supergames_reward/std": 47.9068717956543, + "step": 111, + "step_time": 10.91118389699841 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 145.0, + "completions/max_terminated_length": 145.0, + "completions/mean_length": 108.375, + "completions/mean_terminated_length": 108.375, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.34181955456733704, + "epoch": 0.56, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.537296772003174, + "learning_rate": 4.45e-07, + "loss": -0.052413541823625565, + "num_tokens": 956675.0, + "reward": 47.56624984741211, + "reward_std": 49.185157775878906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 47.466251373291016, + "rewards/supergames_reward/std": 49.18516159057617, + "step": 112, + "step_time": 5.552288047998445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/max_terminated_length": 245.0, + "completions/mean_length": 189.125, + "completions/mean_terminated_length": 189.125, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "entropy": 0.5034605860710144, + "epoch": 0.565, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.349876642227173, + "learning_rate": 4.3999999999999997e-07, + "loss": -0.08186715841293335, + "num_tokens": 964636.0, + "reward": 81.32374572753906, + "reward_std": 36.85074996948242, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 81.2237548828125, + "rewards/supergames_reward/std": 36.85074996948242, + "step": 113, + "step_time": 9.267611294984818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/max_terminated_length": 178.0, + "completions/mean_length": 111.25, + "completions/mean_terminated_length": 111.25, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "entropy": 0.3229002058506012, + "epoch": 0.57, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.245575428009033, + "learning_rate": 4.3499999999999996e-07, + "loss": 0.06544198095798492, + "num_tokens": 970070.0, + "reward": 63.1775016784668, + "reward_std": 52.30624008178711, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 63.07749938964844, + "rewards/supergames_reward/std": 52.30624008178711, + "step": 114, + "step_time": 6.731139309995342 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 224.0, + "completions/max_terminated_length": 224.0, + "completions/mean_length": 170.0, + "completions/mean_terminated_length": 170.0, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.3274807929992676, + "epoch": 0.575, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5544140338897705, + "learning_rate": 4.2999999999999996e-07, + "loss": -0.06953569501638412, + "num_tokens": 979702.0, + "reward": 25.62125015258789, + "reward_std": 64.52003479003906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 25.521244049072266, + "rewards/supergames_reward/std": 64.52003479003906, + "step": 115, + "step_time": 8.702618776995223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 427.0, + "completions/max_terminated_length": 427.0, + "completions/mean_length": 209.5, + "completions/mean_terminated_length": 209.5, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.4970880150794983, + "epoch": 0.58, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4932165145874023, + "learning_rate": 4.2499999999999995e-07, + "loss": -0.2281273603439331, + "num_tokens": 990690.0, + "reward": 50.54750061035156, + "reward_std": 70.5126953125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 50.4474983215332, + "rewards/supergames_reward/std": 70.51270294189453, + "step": 116, + "step_time": 16.204386608995264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.0, + "completions/max_terminated_length": 228.0, + "completions/mean_length": 132.375, + "completions/mean_terminated_length": 132.375, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4380227029323578, + "epoch": 0.585, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.089909076690674, + "learning_rate": 4.1999999999999995e-07, + "loss": 0.12121254950761795, + "num_tokens": 996333.0, + "reward": -15.333749771118164, + "reward_std": 47.200164794921875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -15.433748245239258, + "rewards/supergames_reward/std": 47.200164794921875, + "step": 117, + "step_time": 8.546233230998041 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.0, + "completions/max_terminated_length": 323.0, + "completions/mean_length": 137.625, + "completions/mean_terminated_length": 137.625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4177803695201874, + "epoch": 0.59, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8712472915649414, + "learning_rate": 4.1499999999999994e-07, + "loss": 0.14192955195903778, + "num_tokens": 1002010.0, + "reward": -30.700000762939453, + "reward_std": 17.224035263061523, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -30.799999237060547, + "rewards/supergames_reward/std": 17.224035263061523, + "step": 118, + "step_time": 11.878434990998358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 453.0, + "completions/max_terminated_length": 453.0, + "completions/mean_length": 238.125, + "completions/mean_terminated_length": 238.125, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "entropy": 0.5640217065811157, + "epoch": 0.595, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.064737319946289, + "learning_rate": 4.0999999999999994e-07, + "loss": -0.023621462285518646, + "num_tokens": 1012259.0, + "reward": 43.683746337890625, + "reward_std": 70.4210205078125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 43.5837516784668, + "rewards/supergames_reward/std": 70.4210205078125, + "step": 119, + "step_time": 16.829514264973113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 193.75, + "completions/mean_terminated_length": 193.75, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "entropy": 0.5795091390609741, + "epoch": 0.6, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.423007965087891, + "learning_rate": 4.05e-07, + "loss": -0.0666111558675766, + "num_tokens": 1020241.0, + "reward": -85.86000061035156, + "reward_std": 27.4442081451416, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -85.90999603271484, + "rewards/supergames_reward/std": 27.41469955444336, + "step": 120, + "step_time": 8.95496519000153 + } + ], + "logging_steps": 1, + "max_steps": 200, + "num_input_tokens_seen": 1020241, + "num_train_epochs": 1, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-120/training_args.bin b/checkpoint-120/training_args.bin new file mode 100644 index 0000000..730dec0 --- /dev/null +++ b/checkpoint-120/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691fe5be2d7d1d56160dae63866d136877a72efb30e15ff7b3249192f998a788 +size 7185 diff --git a/checkpoint-140/chat_template.jinja b/checkpoint-140/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/checkpoint-140/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-140/config.json b/checkpoint-140/config.json new file mode 100644 index 0000000..f57b09b --- /dev/null +++ b/checkpoint-140/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-140/generation_config.json b/checkpoint-140/generation_config.json new file mode 100644 index 0000000..1b2bba9 --- /dev/null +++ b/checkpoint-140/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/checkpoint-140/model.safetensors b/checkpoint-140/model.safetensors new file mode 100644 index 0000000..387af85 --- /dev/null +++ b/checkpoint-140/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fdd60d1fa7be0cb10b8b346de5fc6cae3622c2276f19c10936a3d951a7a51af +size 6174895536 diff --git a/checkpoint-140/optimizer.pt b/checkpoint-140/optimizer.pt new file mode 100644 index 0000000..0bf27c4 --- /dev/null +++ b/checkpoint-140/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8249d1c911756c4ae68efa3d86cd5d15c283e6a678dc1914f57d936d46e7e7b7 +size 12350013801 diff --git a/checkpoint-140/rng_state.pth b/checkpoint-140/rng_state.pth new file mode 100644 index 0000000..1cbff49 --- /dev/null +++ b/checkpoint-140/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f30aa6f3347e5a84497bbadb0dd2e02b31bd49b38300a56ce489f81df37084c6 +size 14645 diff --git a/checkpoint-140/scheduler.pt b/checkpoint-140/scheduler.pt new file mode 100644 index 0000000..4bcce52 --- /dev/null +++ b/checkpoint-140/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06e07d7192e2daaedeea2396d67721d207d9a0175b75327949bf0087d7d181eb +size 1465 diff --git a/checkpoint-140/tokenizer.json b/checkpoint-140/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-140/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-140/tokenizer_config.json b/checkpoint-140/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-140/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-140/trainer_state.json b/checkpoint-140/trainer_state.json new file mode 100644 index 0000000..cfef8e8 --- /dev/null +++ b/checkpoint-140/trainer_state.json @@ -0,0 +1,4094 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.7, + "eval_steps": 500, + "global_step": 140, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 191.0, + "completions/mean_terminated_length": 191.0, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "entropy": 0.5566893219947815, + "epoch": 0.005, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.370361328125, + "learning_rate": 1e-06, + "loss": -0.24658073484897614, + "num_tokens": 10832.0, + "reward": -3.3000030517578125, + "reward_std": 85.62333679199219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.4000015258789062, + "rewards/supergames_reward/std": 85.62333679199219, + "step": 1, + "step_time": 22.779711072013015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/max_terminated_length": 216.0, + "completions/mean_length": 131.5, + "completions/mean_terminated_length": 131.5, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.6395841240882874, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.09774398803711, + "learning_rate": 9.95e-07, + "loss": 0.20567649602890015, + "num_tokens": 16404.0, + "reward": -12.422499656677246, + "reward_std": 7.134707450866699, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.522500038146973, + "rewards/supergames_reward/std": 7.134707927703857, + "step": 2, + "step_time": 9.960156448010821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/max_terminated_length": 287.0, + "completions/mean_length": 156.625, + "completions/mean_terminated_length": 156.625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "entropy": 0.562222421169281, + "epoch": 0.015, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9053616523742676, + "learning_rate": 9.9e-07, + "loss": 0.09602774679660797, + "num_tokens": 26953.0, + "reward": -74.94999694824219, + "reward_std": 70.73094177246094, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -75.0, + "rewards/supergames_reward/std": 70.71067810058594, + "step": 3, + "step_time": 13.686320498993155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 154.125, + "completions/mean_terminated_length": 154.125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7323317527770996, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.933310031890869, + "learning_rate": 9.849999999999999e-07, + "loss": 0.28249427676200867, + "num_tokens": 36514.0, + "reward": -40.7599983215332, + "reward_std": 81.73140716552734, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -40.80999755859375, + "rewards/supergames_reward/std": 81.68995666503906, + "step": 4, + "step_time": 12.805880262021674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 205.25, + "completions/mean_terminated_length": 205.25, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.6167430281639099, + "epoch": 0.025, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.344135046005249, + "learning_rate": 9.8e-07, + "loss": 0.1433994024991989, + "num_tokens": 47476.0, + "reward": -97.6612548828125, + "reward_std": 6.3321428298950195, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.76124572753906, + "rewards/supergames_reward/std": 6.3321428298950195, + "step": 5, + "step_time": 18.789364666008623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/max_terminated_length": 276.0, + "completions/mean_length": 167.25, + "completions/mean_terminated_length": 167.25, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.7583790421485901, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.172371864318848, + "learning_rate": 9.75e-07, + "loss": -0.12127404659986496, + "num_tokens": 55246.0, + "reward": -58.687503814697266, + "reward_std": 58.64059829711914, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -58.76250457763672, + "rewards/supergames_reward/std": 58.620460510253906, + "step": 6, + "step_time": 12.948570273991209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 133.875, + "completions/mean_terminated_length": 133.875, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.7447654604911804, + "epoch": 0.035, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.428595066070557, + "learning_rate": 9.7e-07, + "loss": 0.10135584324598312, + "num_tokens": 60837.0, + "reward": -32.45000076293945, + "reward_std": 41.754486083984375, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -32.5, + "rewards/supergames_reward/std": 41.66190719604492, + "step": 7, + "step_time": 12.058315072004916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 414.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 209.125, + "completions/mean_terminated_length": 209.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.3860666751861572, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.806255578994751, + "learning_rate": 9.649999999999999e-07, + "loss": -0.33238139748573303, + "num_tokens": 70838.0, + "reward": -80.50375366210938, + "reward_std": 38.82380294799805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -80.60375213623047, + "rewards/supergames_reward/std": 38.82379913330078, + "step": 8, + "step_time": 19.323370319994865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 247.125, + "completions/mean_terminated_length": 158.83334350585938, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.5816237330436707, + "epoch": 0.045, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3209943771362305, + "learning_rate": 9.6e-07, + "loss": 0.36752766370773315, + "num_tokens": 82135.0, + "reward": -27.688751220703125, + "reward_std": 82.8358154296875, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -27.738750457763672, + "rewards/supergames_reward/std": 82.78590393066406, + "step": 9, + "step_time": 21.948575104994234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/max_terminated_length": 240.0, + "completions/mean_length": 153.0, + "completions/mean_terminated_length": 153.0, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6301521062850952, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.125359058380127, + "learning_rate": 9.55e-07, + "loss": 0.06129350885748863, + "num_tokens": 89807.0, + "reward": -78.39624786376953, + "reward_std": 40.17042922973633, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -78.44625091552734, + "rewards/supergames_reward/std": 40.1396484375, + "step": 10, + "step_time": 9.098202993016457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/max_terminated_length": 133.0, + "completions/mean_length": 112.0, + "completions/mean_terminated_length": 112.0, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.5521990656852722, + "epoch": 0.055, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.616060256958008, + "learning_rate": 9.499999999999999e-07, + "loss": -0.006313305348157883, + "num_tokens": 95231.0, + "reward": -12.161249160766602, + "reward_std": 6.39578104019165, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.261249542236328, + "rewards/supergames_reward/std": 6.39578104019165, + "step": 11, + "step_time": 5.136311663984088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 214.875, + "completions/mean_terminated_length": 214.875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.5505905151367188, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.289583206176758, + "learning_rate": 9.45e-07, + "loss": -0.29802003502845764, + "num_tokens": 106238.0, + "reward": -91.66999816894531, + "reward_std": 23.2779541015625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -91.77000427246094, + "rewards/supergames_reward/std": 23.277956008911133, + "step": 12, + "step_time": 14.945365622988902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 167.375, + "completions/mean_terminated_length": 167.375, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "entropy": 0.540415346622467, + "epoch": 0.065, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.858291149139404, + "learning_rate": 9.399999999999999e-07, + "loss": 0.24004128575325012, + "num_tokens": 116937.0, + "reward": -20.38249969482422, + "reward_std": 87.55204010009766, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.457500457763672, + "rewards/supergames_reward/std": 87.52605438232422, + "step": 13, + "step_time": 10.482285185018554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "entropy": 0.7237679958343506, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.083980560302734, + "learning_rate": 9.35e-07, + "loss": -0.19025824964046478, + "num_tokens": 124640.0, + "reward": -51.397499084472656, + "reward_std": 54.486454010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -51.49749755859375, + "rewards/supergames_reward/std": 54.486454010009766, + "step": 14, + "step_time": 9.597231683001155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 141.375, + "completions/mean_terminated_length": 141.375, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.754334568977356, + "epoch": 0.075, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.720428943634033, + "learning_rate": 9.3e-07, + "loss": -0.16830675303936005, + "num_tokens": 132219.0, + "reward": -56.226253509521484, + "reward_std": 60.115657806396484, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -56.32625198364258, + "rewards/supergames_reward/std": 60.11566162109375, + "step": 15, + "step_time": 8.998362872982398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.0, + "completions/max_terminated_length": 209.0, + "completions/mean_length": 136.375, + "completions/mean_terminated_length": 136.375, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.6280568838119507, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.9681806564331055, + "learning_rate": 9.25e-07, + "loss": -0.01910916529595852, + "num_tokens": 139726.0, + "reward": -68.24000549316406, + "reward_std": 52.238914489746094, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.33999633789062, + "rewards/supergames_reward/std": 52.238914489746094, + "step": 16, + "step_time": 8.034480294008972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 211.875, + "completions/mean_terminated_length": 211.875, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.899176836013794, + "epoch": 0.085, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.691746711730957, + "learning_rate": 9.2e-07, + "loss": 0.25513291358947754, + "num_tokens": 147869.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 17, + "step_time": 13.655792869016295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 387.0, + "completions/max_terminated_length": 387.0, + "completions/mean_length": 238.0, + "completions/mean_terminated_length": 238.0, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.5596873760223389, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6711347103118896, + "learning_rate": 9.15e-07, + "loss": -0.2628335654735565, + "num_tokens": 159101.0, + "reward": -24.900001525878906, + "reward_std": 103.50983428955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.0, + "rewards/supergames_reward/std": 103.50983428955078, + "step": 18, + "step_time": 14.784329420013819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 277.0, + "completions/mean_length": 219.75, + "completions/mean_terminated_length": 178.00001525878906, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.9245517253875732, + "epoch": 0.095, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55502986907959, + "learning_rate": 9.1e-07, + "loss": 0.13060179352760315, + "num_tokens": 167299.0, + "reward": -86.89125061035156, + "reward_std": 36.95603561401367, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -86.94125366210938, + "rewards/supergames_reward/std": 36.93572235107422, + "step": 19, + "step_time": 18.832684700988466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/max_terminated_length": 267.0, + "completions/mean_length": 177.25, + "completions/mean_terminated_length": 177.25, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.7331254482269287, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.538497447967529, + "learning_rate": 9.05e-07, + "loss": -0.16418980062007904, + "num_tokens": 175141.0, + "reward": -68.52874755859375, + "reward_std": 58.48719787597656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.62875366210938, + "rewards/supergames_reward/std": 58.48720169067383, + "step": 20, + "step_time": 10.090975169994636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 190.625, + "completions/mean_terminated_length": 190.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.6659425497055054, + "epoch": 0.105, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.485373497009277, + "learning_rate": 9e-07, + "loss": -0.10106432437896729, + "num_tokens": 183098.0, + "reward": -71.48875427246094, + "reward_std": 55.33565139770508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -71.5887451171875, + "rewards/supergames_reward/std": 55.33565139770508, + "step": 21, + "step_time": 11.255001295008697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 179.0, + "completions/mean_terminated_length": 179.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.49365949630737305, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.829419136047363, + "learning_rate": 8.95e-07, + "loss": 0.10587231814861298, + "num_tokens": 192826.0, + "reward": -75.62000274658203, + "reward_std": 45.01959991455078, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -75.69499969482422, + "rewards/supergames_reward/std": 45.00411605834961, + "step": 22, + "step_time": 10.611246599000879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/max_terminated_length": 315.0, + "completions/mean_length": 212.625, + "completions/mean_terminated_length": 212.625, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.43159469962120056, + "epoch": 0.115, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.541125774383545, + "learning_rate": 8.9e-07, + "loss": -0.1681259125471115, + "num_tokens": 202815.0, + "reward": 23.67624855041504, + "reward_std": 67.74340057373047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 23.576250076293945, + "rewards/supergames_reward/std": 67.743408203125, + "step": 23, + "step_time": 11.731771531980485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/max_terminated_length": 168.0, + "completions/mean_length": 101.625, + "completions/mean_terminated_length": 101.625, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.3972298502922058, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.8285417556762695, + "learning_rate": 8.85e-07, + "loss": -0.011585958302021027, + "num_tokens": 208188.0, + "reward": 58.849998474121094, + "reward_std": 56.9304084777832, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 58.75, + "rewards/supergames_reward/std": 56.9304084777832, + "step": 24, + "step_time": 6.187504950998118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 303.0, + "completions/max_terminated_length": 303.0, + "completions/mean_length": 165.375, + "completions/mean_terminated_length": 165.375, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "entropy": 0.6339899301528931, + "epoch": 0.125, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.823799133300781, + "learning_rate": 8.799999999999999e-07, + "loss": 0.05159185826778412, + "num_tokens": 215935.0, + "reward": -23.521251678466797, + "reward_std": 74.23067474365234, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -23.621248245239258, + "rewards/supergames_reward/std": 74.23067474365234, + "step": 25, + "step_time": 11.059416300005978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 349.0, + "completions/max_terminated_length": 349.0, + "completions/mean_length": 204.0, + "completions/mean_terminated_length": 204.0, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.6462895274162292, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.969292640686035, + "learning_rate": 8.75e-07, + "loss": 0.0115677984431386, + "num_tokens": 223999.0, + "reward": -74.8175048828125, + "reward_std": 40.94221115112305, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -74.89250183105469, + "rewards/supergames_reward/std": 40.924625396728516, + "step": 26, + "step_time": 12.666237785975682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 166.5, + "completions/mean_terminated_length": 166.5, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4756850600242615, + "epoch": 0.135, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915069103240967, + "learning_rate": 8.699999999999999e-07, + "loss": -0.13321346044540405, + "num_tokens": 233643.0, + "reward": -52.125, + "reward_std": 19.405282974243164, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -52.20000076293945, + "rewards/supergames_reward/std": 19.33464813232422, + "step": 27, + "step_time": 11.172539752995363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 137.125, + "completions/mean_terminated_length": 137.125, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.5834778547286987, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.146666526794434, + "learning_rate": 8.65e-07, + "loss": 0.060454584658145905, + "num_tokens": 241204.0, + "reward": -68.99250793457031, + "reward_std": 51.010581970214844, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -69.09249877929688, + "rewards/supergames_reward/std": 51.010581970214844, + "step": 28, + "step_time": 7.025046669004951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 182.375, + "completions/mean_terminated_length": 182.375, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.6368551254272461, + "epoch": 0.145, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.15535306930542, + "learning_rate": 8.599999999999999e-07, + "loss": -0.3323673903942108, + "num_tokens": 249087.0, + "reward": -44.17000198364258, + "reward_std": 80.5373306274414, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -44.27000045776367, + "rewards/supergames_reward/std": 80.5373306274414, + "step": 29, + "step_time": 12.079259724996518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 127.75, + "completions/mean_terminated_length": 127.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4491196870803833, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.896186828613281, + "learning_rate": 8.55e-07, + "loss": -0.12887656688690186, + "num_tokens": 259445.0, + "reward": 57.446250915527344, + "reward_std": 38.78837585449219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 57.34625244140625, + "rewards/supergames_reward/std": 38.78837966918945, + "step": 30, + "step_time": 7.483972909016302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/max_terminated_length": 166.0, + "completions/mean_length": 112.875, + "completions/mean_terminated_length": 112.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4858454465866089, + "epoch": 0.155, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.742906093597412, + "learning_rate": 8.499999999999999e-07, + "loss": 0.06930096447467804, + "num_tokens": 264924.0, + "reward": -21.174999237060547, + "reward_std": 31.89051628112793, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -21.25, + "rewards/supergames_reward/std": 31.819807052612305, + "step": 31, + "step_time": 6.1866529019898735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/max_terminated_length": 148.0, + "completions/mean_length": 118.125, + "completions/mean_terminated_length": 118.125, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.45428749918937683, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.663414478302002, + "learning_rate": 8.45e-07, + "loss": 0.05735419690608978, + "num_tokens": 270381.0, + "reward": 0.4687504768371582, + "reward_std": 38.66371154785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 0.36875057220458984, + "rewards/supergames_reward/std": 38.6637077331543, + "step": 32, + "step_time": 5.577042757999152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 166.875, + "completions/mean_terminated_length": 166.875, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.6107293367385864, + "epoch": 0.165, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.75006628036499, + "learning_rate": 8.399999999999999e-07, + "loss": -0.14760127663612366, + "num_tokens": 278156.0, + "reward": -87.84750366210938, + "reward_std": 26.84708023071289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -87.94749450683594, + "rewards/supergames_reward/std": 26.847076416015625, + "step": 33, + "step_time": 12.552876825997373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 360.0, + "completions/mean_length": 231.0, + "completions/mean_terminated_length": 190.85714721679688, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.6089653372764587, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.781267166137695, + "learning_rate": 8.349999999999999e-07, + "loss": 0.4294504225254059, + "num_tokens": 289364.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 34, + "step_time": 18.961819477990502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 134.125, + "completions/mean_terminated_length": 134.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46158918738365173, + "epoch": 0.175, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.016353607177734, + "learning_rate": 8.299999999999999e-07, + "loss": 0.09251818805932999, + "num_tokens": 298733.0, + "reward": -35.558753967285156, + "reward_std": 15.881204605102539, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -35.65875244140625, + "rewards/supergames_reward/std": 15.881203651428223, + "step": 35, + "step_time": 8.38491243700264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/max_terminated_length": 207.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.5104788541793823, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.592893123626709, + "learning_rate": 8.249999999999999e-07, + "loss": -0.0410832017660141, + "num_tokens": 306420.0, + "reward": 5.4212493896484375, + "reward_std": 93.50404357910156, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 5.346250534057617, + "rewards/supergames_reward/std": 93.4718246459961, + "step": 36, + "step_time": 7.811868985998444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/max_terminated_length": 241.0, + "completions/mean_length": 178.0, + "completions/mean_terminated_length": 178.0, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.7527878880500793, + "epoch": 0.185, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8812546730041504, + "learning_rate": 8.199999999999999e-07, + "loss": 0.12539231777191162, + "num_tokens": 314252.0, + "reward": -89.58500671386719, + "reward_std": 29.256120681762695, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -89.66000366210938, + "rewards/supergames_reward/std": 29.24593734741211, + "step": 37, + "step_time": 8.959742914012168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 133.375, + "completions/mean_terminated_length": 133.375, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.3347901403903961, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.205414295196533, + "learning_rate": 8.149999999999999e-07, + "loss": -0.06991486996412277, + "num_tokens": 319839.0, + "reward": -14.498749732971191, + "reward_std": 13.007229804992676, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -14.598750114440918, + "rewards/supergames_reward/std": 13.007229804992676, + "step": 38, + "step_time": 7.377183554985095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/max_terminated_length": 181.0, + "completions/mean_length": 114.375, + "completions/mean_terminated_length": 114.375, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.4445965886116028, + "epoch": 0.195, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.561298847198486, + "learning_rate": 8.1e-07, + "loss": -0.06235164776444435, + "num_tokens": 325290.0, + "reward": -7.425000190734863, + "reward_std": 53.701602935791016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -7.5, + "rewards/supergames_reward/std": 53.652320861816406, + "step": 39, + "step_time": 6.753862089011818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 187.625, + "completions/mean_terminated_length": 187.625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.3950234353542328, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5988712310791016, + "learning_rate": 8.05e-07, + "loss": -0.1381440907716751, + "num_tokens": 335119.0, + "reward": -33.68375015258789, + "reward_std": 42.069435119628906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -33.783748626708984, + "rewards/supergames_reward/std": 42.069435119628906, + "step": 40, + "step_time": 13.718958162004128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 371.0, + "completions/mean_length": 251.125, + "completions/mean_terminated_length": 213.85714721679688, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.507067084312439, + "epoch": 0.205, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3898658752441406, + "learning_rate": 8e-07, + "loss": 0.0078964838758111, + "num_tokens": 346464.0, + "reward": -86.48625183105469, + "reward_std": 38.020694732666016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -86.56124877929688, + "rewards/supergames_reward/std": 38.010528564453125, + "step": 41, + "step_time": 18.90960379401804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 152.0, + "completions/max_terminated_length": 152.0, + "completions/mean_length": 116.375, + "completions/mean_terminated_length": 116.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3801707923412323, + "epoch": 0.21, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.95e-07, + "loss": 0.0, + "num_tokens": 351955.0, + "reward": -9.899999618530273, + "reward_std": 0.0, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -10.0, + "rewards/supergames_reward/std": 0.0, + "step": 42, + "step_time": 5.711630532023264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/max_terminated_length": 308.0, + "completions/mean_length": 247.375, + "completions/mean_terminated_length": 247.375, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.3185043931007385, + "epoch": 0.215, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.168762445449829, + "learning_rate": 7.9e-07, + "loss": 0.1970764398574829, + "num_tokens": 363278.0, + "reward": -53.89249801635742, + "reward_std": 63.02619934082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -53.99250030517578, + "rewards/supergames_reward/std": 63.02619934082031, + "step": 43, + "step_time": 11.661934906995157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/max_terminated_length": 330.0, + "completions/mean_length": 212.5, + "completions/mean_terminated_length": 212.5, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4446268379688263, + "epoch": 0.22, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.305844306945801, + "learning_rate": 7.85e-07, + "loss": -0.06735197454690933, + "num_tokens": 374306.0, + "reward": 18.059999465942383, + "reward_std": 98.9417953491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 17.959999084472656, + "rewards/supergames_reward/std": 98.9417953491211, + "step": 44, + "step_time": 12.492453911982011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.0, + "completions/max_terminated_length": 316.0, + "completions/mean_length": 183.0, + "completions/mean_terminated_length": 183.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.5269614458084106, + "epoch": 0.225, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8006370067596436, + "learning_rate": 7.799999999999999e-07, + "loss": -0.08870815485715866, + "num_tokens": 382210.0, + "reward": -54.928749084472656, + "reward_std": 72.5127944946289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -55.02874755859375, + "rewards/supergames_reward/std": 72.5127944946289, + "step": 45, + "step_time": 11.7636467939883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.0, + "completions/max_terminated_length": 244.0, + "completions/mean_length": 136.625, + "completions/mean_terminated_length": 136.625, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.34820204973220825, + "epoch": 0.23, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.911736011505127, + "learning_rate": 7.75e-07, + "loss": 0.04397330805659294, + "num_tokens": 391583.0, + "reward": -19.10375213623047, + "reward_std": 23.967702865600586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -19.203750610351562, + "rewards/supergames_reward/std": 23.967702865600586, + "step": 46, + "step_time": 10.119426151999505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 160.375, + "completions/mean_terminated_length": 160.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.611914336681366, + "epoch": 0.235, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.571913242340088, + "learning_rate": 7.699999999999999e-07, + "loss": -0.044293053448200226, + "num_tokens": 399290.0, + "reward": -25.186250686645508, + "reward_std": 71.46305084228516, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.286251068115234, + "rewards/supergames_reward/std": 71.46305084228516, + "step": 47, + "step_time": 8.563868903991533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/max_terminated_length": 167.0, + "completions/mean_length": 111.125, + "completions/mean_terminated_length": 111.125, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.46585777401924133, + "epoch": 0.24, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.654109954833984, + "learning_rate": 7.65e-07, + "loss": -0.035151124000549316, + "num_tokens": 404707.0, + "reward": -0.408750057220459, + "reward_std": 31.939748764038086, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -0.5087499618530273, + "rewards/supergames_reward/std": 31.93975257873535, + "step": 48, + "step_time": 6.438482580007985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/max_terminated_length": 306.0, + "completions/mean_length": 216.625, + "completions/mean_terminated_length": 216.625, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.33054471015930176, + "epoch": 0.245, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.700303792953491, + "learning_rate": 7.599999999999999e-07, + "loss": -0.04393656551837921, + "num_tokens": 414736.0, + "reward": -97.26000213623047, + "reward_std": 1.6294406652450562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.36000061035156, + "rewards/supergames_reward/std": 1.629441738128662, + "step": 49, + "step_time": 11.636040106008295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 378.0, + "completions/max_terminated_length": 378.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.48711177706718445, + "epoch": 0.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9650564193725586, + "learning_rate": 7.55e-07, + "loss": -0.3529520332813263, + "num_tokens": 424795.0, + "reward": 47.408748626708984, + "reward_std": 72.72083282470703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 47.30875015258789, + "rewards/supergames_reward/std": 72.72083282470703, + "step": 50, + "step_time": 14.258096842997475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 267.625, + "completions/mean_terminated_length": 232.71429443359375, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4482860863208771, + "epoch": 0.255, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2998197078704834, + "learning_rate": 7.5e-07, + "loss": 0.2493654191493988, + "num_tokens": 436288.0, + "reward": 50.61125183105469, + "reward_std": 75.3349838256836, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.53624725341797, + "rewards/supergames_reward/std": 75.27783966064453, + "step": 51, + "step_time": 19.20827590499539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 176.0, + "completions/max_terminated_length": 176.0, + "completions/mean_length": 144.5, + "completions/mean_terminated_length": 144.5, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "entropy": 0.5378735661506653, + "epoch": 0.26, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.114461898803711, + "learning_rate": 7.45e-07, + "loss": 0.0036597400903701782, + "num_tokens": 441980.0, + "reward": -15.20625114440918, + "reward_std": 45.35526657104492, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.28125, + "rewards/supergames_reward/std": 45.301815032958984, + "step": 52, + "step_time": 6.660627231001854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/max_terminated_length": 136.0, + "completions/mean_length": 119.0, + "completions/mean_terminated_length": 119.0, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "entropy": 0.41796669363975525, + "epoch": 0.265, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.059505462646484, + "learning_rate": 7.4e-07, + "loss": -0.014973883517086506, + "num_tokens": 447468.0, + "reward": 15.170000076293945, + "reward_std": 46.658329010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 15.070000648498535, + "rewards/supergames_reward/std": 46.658329010009766, + "step": 53, + "step_time": 5.271571868011961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/max_terminated_length": 259.0, + "completions/mean_length": 191.125, + "completions/mean_terminated_length": 191.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.45421987771987915, + "epoch": 0.27, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4454867839813232, + "learning_rate": 7.35e-07, + "loss": -0.1128973588347435, + "num_tokens": 457301.0, + "reward": 38.89875030517578, + "reward_std": 62.1772575378418, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 38.79874801635742, + "rewards/supergames_reward/std": 62.1772575378418, + "step": 54, + "step_time": 10.524528659996577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/max_terminated_length": 319.0, + "completions/mean_length": 233.375, + "completions/mean_terminated_length": 233.375, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "entropy": 0.4449213445186615, + "epoch": 0.275, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2065398693084717, + "learning_rate": 7.3e-07, + "loss": 0.0016277075046673417, + "num_tokens": 467520.0, + "reward": 27.372499465942383, + "reward_std": 81.23126983642578, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.27250099182129, + "rewards/supergames_reward/std": 81.23126983642578, + "step": 55, + "step_time": 12.592280682991259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 357.0, + "completions/mean_length": 261.0, + "completions/mean_terminated_length": 225.1428680419922, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.506920337677002, + "epoch": 0.28, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.058388710021973, + "learning_rate": 7.249999999999999e-07, + "loss": 0.3289242386817932, + "num_tokens": 476056.0, + "reward": 47.05249786376953, + "reward_std": 80.97753143310547, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 46.977500915527344, + "rewards/supergames_reward/std": 80.9256362915039, + "step": 56, + "step_time": 18.845177220006008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.0, + "completions/max_terminated_length": 250.0, + "completions/mean_length": 192.75, + "completions/mean_terminated_length": 192.75, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "entropy": 0.5197336077690125, + "epoch": 0.285, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.230815410614014, + "learning_rate": 7.2e-07, + "loss": -0.15186546742916107, + "num_tokens": 484014.0, + "reward": -64.5050048828125, + "reward_std": 50.35145568847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -64.60499572753906, + "rewards/supergames_reward/std": 50.3514518737793, + "step": 57, + "step_time": 9.576232638006331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 249.0, + "completions/mean_terminated_length": 249.0, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "entropy": 0.42913514375686646, + "epoch": 0.29, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.8544528484344482, + "learning_rate": 7.149999999999999e-07, + "loss": -0.056405920535326004, + "num_tokens": 494390.0, + "reward": 40.123748779296875, + "reward_std": 84.164794921875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 40.02375030517578, + "rewards/supergames_reward/std": 84.164794921875, + "step": 58, + "step_time": 12.385353341000155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 138.0, + "completions/max_terminated_length": 138.0, + "completions/mean_length": 107.25, + "completions/mean_terminated_length": 107.25, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.34386488795280457, + "epoch": 0.295, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.6915154457092285, + "learning_rate": 7.1e-07, + "loss": 0.0865321159362793, + "num_tokens": 499816.0, + "reward": -8.244999885559082, + "reward_std": 4.681046962738037, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.345000267028809, + "rewards/supergames_reward/std": 4.681046962738037, + "step": 59, + "step_time": 5.42325339600211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/max_terminated_length": 187.0, + "completions/mean_length": 127.25, + "completions/mean_terminated_length": 127.25, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.5430045127868652, + "epoch": 0.3, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.635311126708984, + "learning_rate": 7.049999999999999e-07, + "loss": -0.05412375554442406, + "num_tokens": 505410.0, + "reward": 20.005001068115234, + "reward_std": 55.156005859375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 19.904998779296875, + "rewards/supergames_reward/std": 55.156005859375, + "step": 60, + "step_time": 7.068010902003152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/max_terminated_length": 286.0, + "completions/mean_length": 192.125, + "completions/mean_terminated_length": 192.125, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4213869869709015, + "epoch": 0.305, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4664618968963623, + "learning_rate": 7e-07, + "loss": -0.0495159812271595, + "num_tokens": 516283.0, + "reward": -47.212501525878906, + "reward_std": 91.2253646850586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -47.3125, + "rewards/supergames_reward/std": 91.2253646850586, + "step": 61, + "step_time": 11.176304235996213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 171.0, + "completions/mean_terminated_length": 171.0, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.4626811146736145, + "epoch": 0.31, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.949252128601074, + "learning_rate": 6.949999999999999e-07, + "loss": 0.192047581076622, + "num_tokens": 522251.0, + "reward": -15.092500686645508, + "reward_std": 49.82962417602539, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.167499542236328, + "rewards/supergames_reward/std": 49.78091049194336, + "step": 62, + "step_time": 12.075224861997413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 429.0, + "completions/mean_length": 283.5, + "completions/mean_terminated_length": 250.85714721679688, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4814216196537018, + "epoch": 0.315, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7918753623962402, + "learning_rate": 6.9e-07, + "loss": 0.15706156194210052, + "num_tokens": 533815.0, + "reward": -20.86625099182129, + "reward_std": 85.99250030517578, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.94124984741211, + "rewards/supergames_reward/std": 85.96620178222656, + "step": 63, + "step_time": 19.448832260008203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.0, + "completions/max_terminated_length": 189.0, + "completions/mean_length": 131.25, + "completions/mean_terminated_length": 131.25, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4929414987564087, + "epoch": 0.32, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.774101734161377, + "learning_rate": 6.85e-07, + "loss": -0.060691747814416885, + "num_tokens": 539465.0, + "reward": -21.0674991607666, + "reward_std": 6.892730236053467, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -21.167499542236328, + "rewards/supergames_reward/std": 6.892730236053467, + "step": 64, + "step_time": 7.141569407976931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 454.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 281.375, + "completions/mean_terminated_length": 281.375, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "entropy": 0.45866572856903076, + "epoch": 0.325, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.092717409133911, + "learning_rate": 6.800000000000001e-07, + "loss": 0.11694959551095963, + "num_tokens": 551012.0, + "reward": 49.938751220703125, + "reward_std": 79.1865005493164, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 49.8387451171875, + "rewards/supergames_reward/std": 79.1865005493164, + "step": 65, + "step_time": 17.06170882002334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/max_terminated_length": 203.0, + "completions/mean_length": 128.625, + "completions/mean_terminated_length": 128.625, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.5005782246589661, + "epoch": 0.33, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.77616024017334, + "learning_rate": 6.75e-07, + "loss": 0.11242837458848953, + "num_tokens": 556577.0, + "reward": 48.875, + "reward_std": 49.13539123535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.775001525878906, + "rewards/supergames_reward/std": 49.13539123535156, + "step": 66, + "step_time": 7.695410129002994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 436.0, + "completions/max_terminated_length": 436.0, + "completions/mean_length": 281.875, + "completions/mean_terminated_length": 281.875, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "entropy": 0.507097065448761, + "epoch": 0.335, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.911184787750244, + "learning_rate": 6.7e-07, + "loss": -0.08151137083768845, + "num_tokens": 567120.0, + "reward": -12.886249542236328, + "reward_std": 84.04269409179688, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.986251831054688, + "rewards/supergames_reward/std": 84.04269409179688, + "step": 67, + "step_time": 16.409127942984924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 318.375, + "completions/mean_terminated_length": 290.71429443359375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.486402690410614, + "epoch": 0.34, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9461607933044434, + "learning_rate": 6.65e-07, + "loss": 0.2442178875207901, + "num_tokens": 578995.0, + "reward": 2.8387489318847656, + "reward_std": 98.47579956054688, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 2.7637500762939453, + "rewards/supergames_reward/std": 98.44595336914062, + "step": 68, + "step_time": 19.32364999302081 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 291.0, + "completions/max_terminated_length": 291.0, + "completions/mean_length": 219.625, + "completions/mean_terminated_length": 219.625, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.37856727838516235, + "epoch": 0.345, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0204732418060303, + "learning_rate": 6.6e-07, + "loss": -0.08764916658401489, + "num_tokens": 589072.0, + "reward": -54.18375015258789, + "reward_std": 71.45012664794922, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -54.283748626708984, + "rewards/supergames_reward/std": 71.45013427734375, + "step": 69, + "step_time": 11.126611230982235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 305.0, + "completions/mean_length": 254.75, + "completions/mean_terminated_length": 218.00001525878906, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46423614025115967, + "epoch": 0.35, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0140154361724854, + "learning_rate": 6.55e-07, + "loss": 0.26797160506248474, + "num_tokens": 600438.0, + "reward": -32.04875183105469, + "reward_std": 69.86957550048828, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -32.12375259399414, + "rewards/supergames_reward/std": 69.84178161621094, + "step": 70, + "step_time": 19.12617294798838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 372.0, + "completions/max_terminated_length": 372.0, + "completions/mean_length": 194.125, + "completions/mean_terminated_length": 194.125, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.3427293300628662, + "epoch": 0.355, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.781383991241455, + "learning_rate": 6.5e-07, + "loss": 0.354524701833725, + "num_tokens": 610327.0, + "reward": -6.913749694824219, + "reward_std": 87.56071472167969, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -6.988750457763672, + "rewards/supergames_reward/std": 87.53033447265625, + "step": 71, + "step_time": 14.041668199002743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/max_terminated_length": 197.0, + "completions/mean_length": 124.75, + "completions/mean_terminated_length": 124.75, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.4455118477344513, + "epoch": 0.36, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.726032733917236, + "learning_rate": 6.45e-07, + "loss": 0.18206289410591125, + "num_tokens": 615853.0, + "reward": 41.95124816894531, + "reward_std": 60.640811920166016, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 41.85124969482422, + "rewards/supergames_reward/std": 60.640811920166016, + "step": 72, + "step_time": 7.416493425989756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.0, + "completions/max_terminated_length": 323.0, + "completions/mean_length": 147.375, + "completions/mean_terminated_length": 147.375, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.480733186006546, + "epoch": 0.365, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.853621482849121, + "learning_rate": 6.4e-07, + "loss": 0.04558124020695686, + "num_tokens": 621584.0, + "reward": 14.884998321533203, + "reward_std": 46.25857162475586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 14.78499984741211, + "rewards/supergames_reward/std": 46.25857162475586, + "step": 73, + "step_time": 11.95379540900467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 283.0, + "completions/max_terminated_length": 283.0, + "completions/mean_length": 188.125, + "completions/mean_terminated_length": 188.125, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.5526013374328613, + "epoch": 0.37, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.729883909225464, + "learning_rate": 6.35e-07, + "loss": 0.06268303096294403, + "num_tokens": 629545.0, + "reward": -68.05750274658203, + "reward_std": 59.44808578491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.15750122070312, + "rewards/supergames_reward/std": 59.44808578491211, + "step": 74, + "step_time": 10.64124580900534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 141.0, + "completions/max_terminated_length": 141.0, + "completions/mean_length": 117.375, + "completions/mean_terminated_length": 117.375, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.3744319677352905, + "epoch": 0.375, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.775154113769531, + "learning_rate": 6.3e-07, + "loss": -0.023284845054149628, + "num_tokens": 635052.0, + "reward": 3.9374990463256836, + "reward_std": 61.85771179199219, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 3.862499237060547, + "rewards/supergames_reward/std": 61.80967712402344, + "step": 75, + "step_time": 5.432648951013107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 157.0, + "completions/mean_terminated_length": 157.0, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4822140336036682, + "epoch": 0.38, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.715704917907715, + "learning_rate": 6.249999999999999e-07, + "loss": -0.04918142035603523, + "num_tokens": 644644.0, + "reward": 48.64125061035156, + "reward_std": 78.0205078125, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 48.56624984741211, + "rewards/supergames_reward/std": 77.96605682373047, + "step": 76, + "step_time": 10.348935816989979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/max_terminated_length": 380.0, + "completions/mean_length": 210.875, + "completions/mean_terminated_length": 210.875, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "entropy": 0.45722317695617676, + "epoch": 0.385, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6057567596435547, + "learning_rate": 6.2e-07, + "loss": 0.058352649211883545, + "num_tokens": 652771.0, + "reward": -8.283750534057617, + "reward_std": 77.68436431884766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.383749008178711, + "rewards/supergames_reward/std": 77.68437194824219, + "step": 77, + "step_time": 14.142948755004909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 188.25, + "completions/mean_terminated_length": 188.25, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.33380764722824097, + "epoch": 0.39, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2239863872528076, + "learning_rate": 6.149999999999999e-07, + "loss": -0.04592633992433548, + "num_tokens": 662653.0, + "reward": 11.153749465942383, + "reward_std": 89.88329315185547, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 11.053749084472656, + "rewards/supergames_reward/std": 89.88329315185547, + "step": 78, + "step_time": 12.277474621019792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/max_terminated_length": 143.0, + "completions/mean_length": 108.875, + "completions/mean_terminated_length": 108.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.3518812656402588, + "epoch": 0.395, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.967776298522949, + "learning_rate": 6.1e-07, + "loss": -0.10800496488809586, + "num_tokens": 668076.0, + "reward": 70.33250427246094, + "reward_std": 49.9145393371582, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 70.23249816894531, + "rewards/supergames_reward/std": 49.91453552246094, + "step": 79, + "step_time": 5.52714040101273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 152.125, + "completions/mean_terminated_length": 152.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4699896574020386, + "epoch": 0.4, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.561161994934082, + "learning_rate": 6.049999999999999e-07, + "loss": 0.09505834430456161, + "num_tokens": 675701.0, + "reward": -20.25625228881836, + "reward_std": 53.78542709350586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -20.35624885559082, + "rewards/supergames_reward/std": 53.78542709350586, + "step": 80, + "step_time": 7.155081019998761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 288.875, + "completions/mean_terminated_length": 257.0, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.4086494445800781, + "epoch": 0.405, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.3719611167907715, + "learning_rate": 6e-07, + "loss": 0.04727627709507942, + "num_tokens": 687332.0, + "reward": -37.20750045776367, + "reward_std": 91.7676010131836, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -37.25749969482422, + "rewards/supergames_reward/std": 91.72847747802734, + "step": 81, + "step_time": 19.231399144016905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 376.0, + "completions/max_terminated_length": 376.0, + "completions/mean_length": 227.75, + "completions/mean_terminated_length": 227.75, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.43233707547187805, + "epoch": 0.41, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3391714096069336, + "learning_rate": 5.949999999999999e-07, + "loss": -0.25524550676345825, + "num_tokens": 698474.0, + "reward": -3.7287511825561523, + "reward_std": 91.25379943847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.828749656677246, + "rewards/supergames_reward/std": 91.25379943847656, + "step": 82, + "step_time": 14.329176368017215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 178.125, + "completions/mean_terminated_length": 178.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.4293636977672577, + "epoch": 0.415, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.7271742820739746, + "learning_rate": 5.9e-07, + "loss": 0.09273672848939896, + "num_tokens": 708243.0, + "reward": 12.802498817443848, + "reward_std": 86.23526000976562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 12.702500343322754, + "rewards/supergames_reward/std": 86.23526000976562, + "step": 83, + "step_time": 11.419686011999147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 425.0, + "completions/max_terminated_length": 425.0, + "completions/mean_length": 255.375, + "completions/mean_terminated_length": 255.375, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "entropy": 0.38068246841430664, + "epoch": 0.42, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.6244184970855713, + "learning_rate": 5.849999999999999e-07, + "loss": 0.02006213366985321, + "num_tokens": 718606.0, + "reward": 37.803749084472656, + "reward_std": 57.58624267578125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 37.70375061035156, + "rewards/supergames_reward/std": 57.58624267578125, + "step": 84, + "step_time": 15.975198492989875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 116.0, + "completions/max_terminated_length": 116.0, + "completions/mean_length": 108.125, + "completions/mean_terminated_length": 108.125, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.339572012424469, + "epoch": 0.425, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.754759311676025, + "learning_rate": 5.8e-07, + "loss": -0.011015941388905048, + "num_tokens": 723983.0, + "reward": -3.8212504386901855, + "reward_std": 61.624786376953125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.921250820159912, + "rewards/supergames_reward/std": 61.624786376953125, + "step": 85, + "step_time": 4.542777584982105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 447.0, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 291.375, + "completions/mean_terminated_length": 291.375, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "entropy": 0.5846173763275146, + "epoch": 0.43, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55724835395813, + "learning_rate": 5.749999999999999e-07, + "loss": -0.04981120675802231, + "num_tokens": 735642.0, + "reward": 50.07499694824219, + "reward_std": 92.6283187866211, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.0, + "rewards/supergames_reward/std": 92.5820083618164, + "step": 86, + "step_time": 16.777178087009815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/max_terminated_length": 232.0, + "completions/mean_length": 192.0, + "completions/mean_terminated_length": 192.0, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "entropy": 0.5139474272727966, + "epoch": 0.435, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8860411643981934, + "learning_rate": 5.699999999999999e-07, + "loss": -0.02194221317768097, + "num_tokens": 743602.0, + "reward": -52.522499084472656, + "reward_std": 73.8686294555664, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -52.62249755859375, + "rewards/supergames_reward/std": 73.86863708496094, + "step": 87, + "step_time": 8.772893911984283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/max_terminated_length": 266.0, + "completions/mean_length": 146.875, + "completions/mean_terminated_length": 146.875, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.3428497910499573, + "epoch": 0.44, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.509483337402344, + "learning_rate": 5.649999999999999e-07, + "loss": 0.19857533276081085, + "num_tokens": 753065.0, + "reward": 63.63874816894531, + "reward_std": 67.54963684082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 63.53874969482422, + "rewards/supergames_reward/std": 67.54963684082031, + "step": 88, + "step_time": 10.230529835011112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 446.0, + "completions/max_terminated_length": 446.0, + "completions/mean_length": 270.625, + "completions/mean_terminated_length": 270.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.4417419731616974, + "epoch": 0.445, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.973951816558838, + "learning_rate": 5.6e-07, + "loss": -0.00031440958264283836, + "num_tokens": 764534.0, + "reward": -9.736251831054688, + "reward_std": 92.25879669189453, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -9.836250305175781, + "rewards/supergames_reward/std": 92.25880432128906, + "step": 89, + "step_time": 17.40752330099349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 398.0, + "completions/max_terminated_length": 398.0, + "completions/mean_length": 227.375, + "completions/mean_terminated_length": 227.375, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.4732024073600769, + "epoch": 0.45, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2560815811157227, + "learning_rate": 5.55e-07, + "loss": 0.03820464387536049, + "num_tokens": 772809.0, + "reward": 5.982499599456787, + "reward_std": 58.833797454833984, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 5.8824992179870605, + "rewards/supergames_reward/std": 58.83380126953125, + "step": 90, + "step_time": 15.007269965979503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 207.5, + "completions/mean_terminated_length": 207.5, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.42038559913635254, + "epoch": 0.455, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3649117946624756, + "learning_rate": 5.5e-07, + "loss": 0.1510048508644104, + "num_tokens": 782805.0, + "reward": -5.61500358581543, + "reward_std": 82.67152404785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -5.714998245239258, + "rewards/supergames_reward/std": 82.67152404785156, + "step": 91, + "step_time": 10.386137172987219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 416.0, + "completions/max_terminated_length": 416.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.3971734642982483, + "epoch": 0.46, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.433772563934326, + "learning_rate": 5.45e-07, + "loss": -0.3027213513851166, + "num_tokens": 793840.0, + "reward": -29.06500244140625, + "reward_std": 91.68325805664062, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -29.16499900817871, + "rewards/supergames_reward/std": 91.68325805664062, + "step": 92, + "step_time": 15.790611553995404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 206.625, + "completions/mean_terminated_length": 206.625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.5906907320022583, + "epoch": 0.465, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.293088912963867, + "learning_rate": 5.4e-07, + "loss": 0.09760032594203949, + "num_tokens": 803821.0, + "reward": -72.73999786376953, + "reward_std": 50.72484588623047, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -72.81500244140625, + "rewards/supergames_reward/std": 50.7094841003418, + "step": 93, + "step_time": 13.043119941983605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/max_terminated_length": 143.0, + "completions/mean_length": 115.75, + "completions/mean_terminated_length": 115.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4233115315437317, + "epoch": 0.47, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.58886194229126, + "learning_rate": 5.35e-07, + "loss": -0.04040650278329849, + "num_tokens": 809299.0, + "reward": -3.3050003051757812, + "reward_std": 24.443714141845703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.405000686645508, + "rewards/supergames_reward/std": 24.443714141845703, + "step": 94, + "step_time": 5.512874762003776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 122.0, + "completions/max_terminated_length": 122.0, + "completions/mean_length": 103.125, + "completions/mean_terminated_length": 103.125, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.4055764377117157, + "epoch": 0.475, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.981549263000488, + "learning_rate": 5.3e-07, + "loss": -0.07588938623666763, + "num_tokens": 814716.0, + "reward": 60.06999969482422, + "reward_std": 42.708473205566406, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 59.970001220703125, + "rewards/supergames_reward/std": 42.708473205566406, + "step": 95, + "step_time": 4.766389057011111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 475.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.408913254737854, + "epoch": 0.48, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.79402756690979, + "learning_rate": 5.25e-07, + "loss": -0.3087541460990906, + "num_tokens": 824791.0, + "reward": -15.042500495910645, + "reward_std": 84.37246704101562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -15.142499923706055, + "rewards/supergames_reward/std": 84.37246704101562, + "step": 96, + "step_time": 17.779843941010768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/max_terminated_length": 154.0, + "completions/mean_length": 121.625, + "completions/mean_terminated_length": 121.625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.529015839099884, + "epoch": 0.485, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.142816066741943, + "learning_rate": 5.2e-07, + "loss": 0.07216037809848785, + "num_tokens": 830300.0, + "reward": -32.743751525878906, + "reward_std": 54.61775588989258, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -32.81875228881836, + "rewards/supergames_reward/std": 54.58256149291992, + "step": 97, + "step_time": 5.913989890017547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 126.0, + "completions/max_terminated_length": 126.0, + "completions/mean_length": 95.625, + "completions/mean_terminated_length": 95.625, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.33743274211883545, + "epoch": 0.49, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.197273254394531, + "learning_rate": 5.149999999999999e-07, + "loss": 0.07500407844781876, + "num_tokens": 835633.0, + "reward": 11.00999927520752, + "reward_std": 62.73848342895508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 10.910000801086426, + "rewards/supergames_reward/std": 62.73848342895508, + "step": 98, + "step_time": 4.9155233050114475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 108.5, + "completions/mean_terminated_length": 108.5, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.3106803297996521, + "epoch": 0.495, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.864002227783203, + "learning_rate": 5.1e-07, + "loss": -0.1206185445189476, + "num_tokens": 841045.0, + "reward": 14.238749504089355, + "reward_std": 51.77016830444336, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 14.138750076293945, + "rewards/supergames_reward/std": 51.77016830444336, + "step": 99, + "step_time": 5.7507751359953545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 97.125, + "completions/mean_terminated_length": 97.125, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.35160186886787415, + "epoch": 0.5, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915731906890869, + "learning_rate": 5.049999999999999e-07, + "loss": -0.07951541244983673, + "num_tokens": 846398.0, + "reward": 67.78125, + "reward_std": 38.15436553955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 67.68124389648438, + "rewards/supergames_reward/std": 38.15436553955078, + "step": 100, + "step_time": 5.994720440998208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 320.0, + "completions/max_terminated_length": 320.0, + "completions/mean_length": 208.75, + "completions/mean_terminated_length": 208.75, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.452095091342926, + "epoch": 0.505, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.065176963806152, + "learning_rate": 5e-07, + "loss": -0.1397111564874649, + "num_tokens": 857420.0, + "reward": -5.801251411437988, + "reward_std": 73.56117248535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -5.90125036239624, + "rewards/supergames_reward/std": 73.5611801147461, + "step": 101, + "step_time": 12.28178753197426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/max_terminated_length": 230.0, + "completions/mean_length": 171.375, + "completions/mean_terminated_length": 171.375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4138447642326355, + "epoch": 0.51, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.998823404312134, + "learning_rate": 4.95e-07, + "loss": -0.08941879868507385, + "num_tokens": 867103.0, + "reward": 92.24749755859375, + "reward_std": 14.540005683898926, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 92.14749908447266, + "rewards/supergames_reward/std": 14.540006637573242, + "step": 102, + "step_time": 8.992682139010867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 311.0, + "completions/max_terminated_length": 311.0, + "completions/mean_length": 163.0, + "completions/mean_terminated_length": 163.0, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.4182547330856323, + "epoch": 0.515, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.419595241546631, + "learning_rate": 4.9e-07, + "loss": -0.19304415583610535, + "num_tokens": 876679.0, + "reward": 79.13249969482422, + "reward_std": 59.3050422668457, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 79.03250122070312, + "rewards/supergames_reward/std": 59.30504608154297, + "step": 103, + "step_time": 11.836970244999975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 253.0, + "completions/max_terminated_length": 253.0, + "completions/mean_length": 155.625, + "completions/mean_terminated_length": 155.625, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.49889081716537476, + "epoch": 0.52, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.463719367980957, + "learning_rate": 4.85e-07, + "loss": -0.04786720871925354, + "num_tokens": 884348.0, + "reward": 48.616249084472656, + "reward_std": 73.30741119384766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.51625061035156, + "rewards/supergames_reward/std": 73.30741119384766, + "step": 104, + "step_time": 9.579507841990562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/max_terminated_length": 380.0, + "completions/mean_length": 269.5, + "completions/mean_terminated_length": 269.5, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "entropy": 0.5286482572555542, + "epoch": 0.525, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.993295431137085, + "learning_rate": 4.8e-07, + "loss": -0.08071611076593399, + "num_tokens": 894856.0, + "reward": 34.970001220703125, + "reward_std": 85.324462890625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 34.869998931884766, + "rewards/supergames_reward/std": 85.324462890625, + "step": 105, + "step_time": 14.295730627985904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 203.25, + "completions/mean_terminated_length": 203.25, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.6049264073371887, + "epoch": 0.53, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.787306785583496, + "learning_rate": 4.7499999999999995e-07, + "loss": 0.19928883016109467, + "num_tokens": 902922.0, + "reward": 32.44499969482422, + "reward_std": 70.4663314819336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 32.369998931884766, + "rewards/supergames_reward/std": 70.41261291503906, + "step": 106, + "step_time": 9.84894504098338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 194.0, + "completions/mean_terminated_length": 194.0, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "entropy": 0.5115755200386047, + "epoch": 0.535, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.277207374572754, + "learning_rate": 4.6999999999999995e-07, + "loss": 0.09118300676345825, + "num_tokens": 910906.0, + "reward": 40.432498931884766, + "reward_std": 48.54548263549805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 40.33250045776367, + "rewards/supergames_reward/std": 48.54548263549805, + "step": 107, + "step_time": 10.466603949986165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 332.0, + "completions/max_terminated_length": 332.0, + "completions/mean_length": 195.875, + "completions/mean_terminated_length": 195.875, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4192962944507599, + "epoch": 0.54, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.308061122894287, + "learning_rate": 4.65e-07, + "loss": 0.28064602613449097, + "num_tokens": 920745.0, + "reward": -21.982500076293945, + "reward_std": 66.31195068359375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -22.08249855041504, + "rewards/supergames_reward/std": 66.31195068359375, + "step": 108, + "step_time": 12.595848233992001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 195.125, + "completions/mean_terminated_length": 195.125, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.37523961067199707, + "epoch": 0.545, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.802706718444824, + "learning_rate": 4.6e-07, + "loss": -0.06614465266466141, + "num_tokens": 930618.0, + "reward": 71.75, + "reward_std": 51.14524459838867, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 71.6500015258789, + "rewards/supergames_reward/std": 51.1452522277832, + "step": 109, + "step_time": 10.33884758799104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 167.75, + "completions/mean_terminated_length": 167.75, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.4339042007923126, + "epoch": 0.55, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.254003047943115, + "learning_rate": 4.55e-07, + "loss": 0.34405598044395447, + "num_tokens": 941256.0, + "reward": 99.21875, + "reward_std": 2.492553234100342, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 99.1187515258789, + "rewards/supergames_reward/std": 2.4925525188446045, + "step": 110, + "step_time": 12.657525141985388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 285.0, + "completions/max_terminated_length": 285.0, + "completions/mean_length": 211.0, + "completions/mean_terminated_length": 211.0, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "entropy": 0.46809861063957214, + "epoch": 0.555, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.811166286468506, + "learning_rate": 4.5e-07, + "loss": 0.045113153755664825, + "num_tokens": 951296.0, + "reward": 55.27750015258789, + "reward_std": 47.90687561035156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 55.1775016784668, + "rewards/supergames_reward/std": 47.9068717956543, + "step": 111, + "step_time": 10.91118389699841 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 145.0, + "completions/max_terminated_length": 145.0, + "completions/mean_length": 108.375, + "completions/mean_terminated_length": 108.375, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.34181955456733704, + "epoch": 0.56, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.537296772003174, + "learning_rate": 4.45e-07, + "loss": -0.052413541823625565, + "num_tokens": 956675.0, + "reward": 47.56624984741211, + "reward_std": 49.185157775878906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 47.466251373291016, + "rewards/supergames_reward/std": 49.18516159057617, + "step": 112, + "step_time": 5.552288047998445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/max_terminated_length": 245.0, + "completions/mean_length": 189.125, + "completions/mean_terminated_length": 189.125, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "entropy": 0.5034605860710144, + "epoch": 0.565, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.349876642227173, + "learning_rate": 4.3999999999999997e-07, + "loss": -0.08186715841293335, + "num_tokens": 964636.0, + "reward": 81.32374572753906, + "reward_std": 36.85074996948242, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 81.2237548828125, + "rewards/supergames_reward/std": 36.85074996948242, + "step": 113, + "step_time": 9.267611294984818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/max_terminated_length": 178.0, + "completions/mean_length": 111.25, + "completions/mean_terminated_length": 111.25, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "entropy": 0.3229002058506012, + "epoch": 0.57, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.245575428009033, + "learning_rate": 4.3499999999999996e-07, + "loss": 0.06544198095798492, + "num_tokens": 970070.0, + "reward": 63.1775016784668, + "reward_std": 52.30624008178711, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 63.07749938964844, + "rewards/supergames_reward/std": 52.30624008178711, + "step": 114, + "step_time": 6.731139309995342 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 224.0, + "completions/max_terminated_length": 224.0, + "completions/mean_length": 170.0, + "completions/mean_terminated_length": 170.0, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.3274807929992676, + "epoch": 0.575, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5544140338897705, + "learning_rate": 4.2999999999999996e-07, + "loss": -0.06953569501638412, + "num_tokens": 979702.0, + "reward": 25.62125015258789, + "reward_std": 64.52003479003906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 25.521244049072266, + "rewards/supergames_reward/std": 64.52003479003906, + "step": 115, + "step_time": 8.702618776995223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 427.0, + "completions/max_terminated_length": 427.0, + "completions/mean_length": 209.5, + "completions/mean_terminated_length": 209.5, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.4970880150794983, + "epoch": 0.58, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4932165145874023, + "learning_rate": 4.2499999999999995e-07, + "loss": -0.2281273603439331, + "num_tokens": 990690.0, + "reward": 50.54750061035156, + "reward_std": 70.5126953125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 50.4474983215332, + "rewards/supergames_reward/std": 70.51270294189453, + "step": 116, + "step_time": 16.204386608995264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.0, + "completions/max_terminated_length": 228.0, + "completions/mean_length": 132.375, + "completions/mean_terminated_length": 132.375, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4380227029323578, + "epoch": 0.585, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.089909076690674, + "learning_rate": 4.1999999999999995e-07, + "loss": 0.12121254950761795, + "num_tokens": 996333.0, + "reward": -15.333749771118164, + "reward_std": 47.200164794921875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -15.433748245239258, + "rewards/supergames_reward/std": 47.200164794921875, + "step": 117, + "step_time": 8.546233230998041 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.0, + "completions/max_terminated_length": 323.0, + "completions/mean_length": 137.625, + "completions/mean_terminated_length": 137.625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4177803695201874, + "epoch": 0.59, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8712472915649414, + "learning_rate": 4.1499999999999994e-07, + "loss": 0.14192955195903778, + "num_tokens": 1002010.0, + "reward": -30.700000762939453, + "reward_std": 17.224035263061523, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -30.799999237060547, + "rewards/supergames_reward/std": 17.224035263061523, + "step": 118, + "step_time": 11.878434990998358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 453.0, + "completions/max_terminated_length": 453.0, + "completions/mean_length": 238.125, + "completions/mean_terminated_length": 238.125, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "entropy": 0.5640217065811157, + "epoch": 0.595, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.064737319946289, + "learning_rate": 4.0999999999999994e-07, + "loss": -0.023621462285518646, + "num_tokens": 1012259.0, + "reward": 43.683746337890625, + "reward_std": 70.4210205078125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 43.5837516784668, + "rewards/supergames_reward/std": 70.4210205078125, + "step": 119, + "step_time": 16.829514264973113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 193.75, + "completions/mean_terminated_length": 193.75, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "entropy": 0.5795091390609741, + "epoch": 0.6, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.423007965087891, + "learning_rate": 4.05e-07, + "loss": -0.0666111558675766, + "num_tokens": 1020241.0, + "reward": -85.86000061035156, + "reward_std": 27.4442081451416, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -85.90999603271484, + "rewards/supergames_reward/std": 27.41469955444336, + "step": 120, + "step_time": 8.95496519000153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 231.0, + "completions/mean_terminated_length": 231.0, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "entropy": 0.3767699897289276, + "epoch": 0.605, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7434473037719727, + "learning_rate": 4e-07, + "loss": -0.1165132150053978, + "num_tokens": 1031465.0, + "reward": 61.45624923706055, + "reward_std": 72.21080780029297, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 61.35625076293945, + "rewards/supergames_reward/std": 72.2108154296875, + "step": 121, + "step_time": 12.588820598000893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 297.0, + "completions/max_terminated_length": 297.0, + "completions/mean_length": 223.375, + "completions/mean_terminated_length": 223.375, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "entropy": 0.4537277817726135, + "epoch": 0.61, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4382436275482178, + "learning_rate": 3.95e-07, + "loss": -0.1038316935300827, + "num_tokens": 1042548.0, + "reward": 5.658749580383301, + "reward_std": 83.77420043945312, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 5.558750152587891, + "rewards/supergames_reward/std": 83.77420043945312, + "step": 122, + "step_time": 11.39525846898323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 456.0, + "completions/max_terminated_length": 456.0, + "completions/mean_length": 314.25, + "completions/mean_terminated_length": 314.25, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4113953411579132, + "epoch": 0.615, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.449777841567993, + "learning_rate": 3.8999999999999997e-07, + "loss": -0.1303664743900299, + "num_tokens": 1054422.0, + "reward": 35.98249816894531, + "reward_std": 88.6131362915039, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 35.88249969482422, + "rewards/supergames_reward/std": 88.6131362915039, + "step": 123, + "step_time": 17.001118954998674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 242.0, + "completions/max_terminated_length": 242.0, + "completions/mean_length": 172.5, + "completions/mean_terminated_length": 172.5, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.4670729339122772, + "epoch": 0.62, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.951946973800659, + "learning_rate": 3.8499999999999997e-07, + "loss": 0.05888795852661133, + "num_tokens": 1062210.0, + "reward": 43.368751525878906, + "reward_std": 40.30733871459961, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 43.26874542236328, + "rewards/supergames_reward/std": 40.307342529296875, + "step": 124, + "step_time": 9.142582255997695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/max_terminated_length": 148.0, + "completions/mean_length": 106.625, + "completions/mean_terminated_length": 106.625, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "entropy": 0.30219921469688416, + "epoch": 0.625, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.391322135925293, + "learning_rate": 3.7999999999999996e-07, + "loss": 0.14367049932479858, + "num_tokens": 1067599.0, + "reward": 42.63374710083008, + "reward_std": 58.20897674560547, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 42.533748626708984, + "rewards/supergames_reward/std": 58.208984375, + "step": 125, + "step_time": 5.636063549987739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.0, + "completions/max_terminated_length": 243.0, + "completions/mean_length": 179.25, + "completions/mean_terminated_length": 179.25, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "entropy": 0.4834887683391571, + "epoch": 0.63, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9391286373138428, + "learning_rate": 3.75e-07, + "loss": -0.06614185124635696, + "num_tokens": 1075457.0, + "reward": 8.302498817443848, + "reward_std": 75.10364532470703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 8.202500343322754, + "rewards/supergames_reward/std": 75.10364532470703, + "step": 126, + "step_time": 9.130600645992672 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 196.0, + "completions/max_terminated_length": 196.0, + "completions/mean_length": 157.0, + "completions/mean_terminated_length": 157.0, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.4801807701587677, + "epoch": 0.635, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.425615310668945, + "learning_rate": 3.7e-07, + "loss": -0.08830522745847702, + "num_tokens": 1083129.0, + "reward": -10.563751220703125, + "reward_std": 64.00443267822266, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -10.663749694824219, + "rewards/supergames_reward/std": 64.00444030761719, + "step": 127, + "step_time": 7.4972667430120055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 412.0, + "completions/max_terminated_length": 412.0, + "completions/mean_length": 239.25, + "completions/mean_terminated_length": 239.25, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.4667063355445862, + "epoch": 0.64, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.199113130569458, + "learning_rate": 3.65e-07, + "loss": -0.043856702744960785, + "num_tokens": 1094355.0, + "reward": -55.89250183105469, + "reward_std": 53.05980682373047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -55.99250030517578, + "rewards/supergames_reward/std": 53.05980682373047, + "step": 128, + "step_time": 15.429580625001108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 244.75, + "completions/mean_terminated_length": 244.75, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "entropy": 0.57573401927948, + "epoch": 0.645, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6800079345703125, + "learning_rate": 3.6e-07, + "loss": 0.09521033614873886, + "num_tokens": 1105633.0, + "reward": -23.35250473022461, + "reward_std": 102.31401062011719, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -23.452499389648438, + "rewards/supergames_reward/std": 102.31402587890625, + "step": 129, + "step_time": 13.920327747007832 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/max_terminated_length": 258.0, + "completions/mean_length": 188.25, + "completions/mean_terminated_length": 188.25, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "entropy": 0.5341438055038452, + "epoch": 0.65, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.97955060005188, + "learning_rate": 3.55e-07, + "loss": -0.09512650966644287, + "num_tokens": 1113571.0, + "reward": 29.181249618530273, + "reward_std": 80.10585021972656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 29.081249237060547, + "rewards/supergames_reward/std": 80.10585021972656, + "step": 130, + "step_time": 9.69286120700417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 257.0, + "completions/max_terminated_length": 257.0, + "completions/mean_length": 180.75, + "completions/mean_terminated_length": 180.75, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "entropy": 0.46591487526893616, + "epoch": 0.655, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3518130779266357, + "learning_rate": 3.5e-07, + "loss": 0.04714229330420494, + "num_tokens": 1121457.0, + "reward": -8.287501335144043, + "reward_std": 76.35875701904297, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.387499809265137, + "rewards/supergames_reward/std": 76.35875701904297, + "step": 131, + "step_time": 9.636637121002423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 147.75, + "completions/mean_terminated_length": 147.75, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4407735764980316, + "epoch": 0.66, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.498741626739502, + "learning_rate": 3.45e-07, + "loss": -0.03996007889509201, + "num_tokens": 1129095.0, + "reward": 0.1999988555908203, + "reward_std": 84.63554382324219, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 0.125, + "rewards/supergames_reward/std": 84.6017074584961, + "step": 132, + "step_time": 7.510833573003765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 358.0, + "completions/max_terminated_length": 358.0, + "completions/mean_length": 213.875, + "completions/mean_terminated_length": 213.875, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.5186149477958679, + "epoch": 0.665, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3987107276916504, + "learning_rate": 3.4000000000000003e-07, + "loss": 0.1863594949245453, + "num_tokens": 1139110.0, + "reward": -21.328752517700195, + "reward_std": 88.9102783203125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -21.428749084472656, + "rewards/supergames_reward/std": 88.9102783203125, + "step": 133, + "step_time": 13.393839450000087 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/max_terminated_length": 230.0, + "completions/mean_length": 169.875, + "completions/mean_terminated_length": 169.875, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.49445784091949463, + "epoch": 0.67, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.479673862457275, + "learning_rate": 3.35e-07, + "loss": 0.03618276119232178, + "num_tokens": 1146885.0, + "reward": -14.042501449584961, + "reward_std": 83.22671508789062, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -14.142499923706055, + "rewards/supergames_reward/std": 83.22671508789062, + "step": 134, + "step_time": 8.703399487014394 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/max_terminated_length": 168.0, + "completions/mean_length": 124.75, + "completions/mean_terminated_length": 124.75, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "entropy": 0.2997814118862152, + "epoch": 0.675, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.440976619720459, + "learning_rate": 3.3e-07, + "loss": -0.029299044981598854, + "num_tokens": 1152427.0, + "reward": 45.274993896484375, + "reward_std": 44.401519775390625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 45.17499923706055, + "rewards/supergames_reward/std": 44.40152359008789, + "step": 135, + "step_time": 6.334954546997324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 161.0, + "completions/max_terminated_length": 161.0, + "completions/mean_length": 123.625, + "completions/mean_terminated_length": 123.625, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3462243974208832, + "epoch": 0.68, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.1878814697265625, + "learning_rate": 3.25e-07, + "loss": -0.10772529244422913, + "num_tokens": 1157952.0, + "reward": 44.186248779296875, + "reward_std": 51.78197479248047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 44.08625030517578, + "rewards/supergames_reward/std": 51.7819709777832, + "step": 136, + "step_time": 6.067531232984038 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 155.0, + "completions/max_terminated_length": 155.0, + "completions/mean_length": 106.75, + "completions/mean_terminated_length": 106.75, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.3662136197090149, + "epoch": 0.685, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.508971214294434, + "learning_rate": 3.2e-07, + "loss": -0.11172202974557877, + "num_tokens": 1163334.0, + "reward": 53.0574951171875, + "reward_std": 58.36653518676758, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 52.957496643066406, + "rewards/supergames_reward/std": 58.36653518676758, + "step": 137, + "step_time": 5.8906258280039765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 175.0, + "completions/max_terminated_length": 175.0, + "completions/mean_length": 125.375, + "completions/mean_terminated_length": 125.375, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.3352622091770172, + "epoch": 0.69, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.1937079429626465, + "learning_rate": 3.15e-07, + "loss": 0.08307601511478424, + "num_tokens": 1168929.0, + "reward": 11.796250343322754, + "reward_std": 53.48870849609375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 11.696250915527344, + "rewards/supergames_reward/std": 53.48870849609375, + "step": 138, + "step_time": 6.595962718012743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 193.0, + "completions/mean_terminated_length": 193.0, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.5486535429954529, + "epoch": 0.695, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.861871242523193, + "learning_rate": 3.1e-07, + "loss": 0.0321456640958786, + "num_tokens": 1176937.0, + "reward": 50.463748931884766, + "reward_std": 70.37553405761719, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.38875198364258, + "rewards/supergames_reward/std": 70.31440734863281, + "step": 139, + "step_time": 9.544256388006033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 189.5, + "completions/mean_terminated_length": 189.5, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.42230603098869324, + "epoch": 0.7, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.830677032470703, + "learning_rate": 3.05e-07, + "loss": -0.07222295552492142, + "num_tokens": 1186789.0, + "reward": 33.44999694824219, + "reward_std": 67.1298599243164, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 33.349998474121094, + "rewards/supergames_reward/std": 67.1298599243164, + "step": 140, + "step_time": 10.002322309010196 + } + ], + "logging_steps": 1, + "max_steps": 200, + "num_input_tokens_seen": 1186789, + "num_train_epochs": 1, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-140/training_args.bin b/checkpoint-140/training_args.bin new file mode 100644 index 0000000..730dec0 --- /dev/null +++ b/checkpoint-140/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691fe5be2d7d1d56160dae63866d136877a72efb30e15ff7b3249192f998a788 +size 7185 diff --git a/checkpoint-160/chat_template.jinja b/checkpoint-160/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/checkpoint-160/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-160/config.json b/checkpoint-160/config.json new file mode 100644 index 0000000..f57b09b --- /dev/null +++ b/checkpoint-160/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-160/generation_config.json b/checkpoint-160/generation_config.json new file mode 100644 index 0000000..1b2bba9 --- /dev/null +++ b/checkpoint-160/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/checkpoint-160/model.safetensors b/checkpoint-160/model.safetensors new file mode 100644 index 0000000..1f2a9b9 --- /dev/null +++ b/checkpoint-160/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e0d5c0c17e78ede34e38c84f732b9754796cabe771cda20dfba617e134780ff +size 6174895536 diff --git a/checkpoint-160/optimizer.pt b/checkpoint-160/optimizer.pt new file mode 100644 index 0000000..b8f2764 --- /dev/null +++ b/checkpoint-160/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50807e72f23133d36d3657a582732486baf0740b84182caf65002921eddd9a47 +size 12350013801 diff --git a/checkpoint-160/rng_state.pth b/checkpoint-160/rng_state.pth new file mode 100644 index 0000000..9d5e465 --- /dev/null +++ b/checkpoint-160/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:573f5fe5c1703fc9f10a995c424d9deec8188be21ffead3f210839eff346240e +size 14645 diff --git a/checkpoint-160/scheduler.pt b/checkpoint-160/scheduler.pt new file mode 100644 index 0000000..66de7f6 --- /dev/null +++ b/checkpoint-160/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7254de8fa1e1e54523ae878ad08e005e688587df236e1a0c0daa9706e54ac55 +size 1465 diff --git a/checkpoint-160/tokenizer.json b/checkpoint-160/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-160/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-160/tokenizer_config.json b/checkpoint-160/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-160/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-160/trainer_state.json b/checkpoint-160/trainer_state.json new file mode 100644 index 0000000..acc872a --- /dev/null +++ b/checkpoint-160/trainer_state.json @@ -0,0 +1,4674 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8, + "eval_steps": 500, + "global_step": 160, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 191.0, + "completions/mean_terminated_length": 191.0, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "entropy": 0.5566893219947815, + "epoch": 0.005, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.370361328125, + "learning_rate": 1e-06, + "loss": -0.24658073484897614, + "num_tokens": 10832.0, + "reward": -3.3000030517578125, + "reward_std": 85.62333679199219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.4000015258789062, + "rewards/supergames_reward/std": 85.62333679199219, + "step": 1, + "step_time": 22.779711072013015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/max_terminated_length": 216.0, + "completions/mean_length": 131.5, + "completions/mean_terminated_length": 131.5, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.6395841240882874, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.09774398803711, + "learning_rate": 9.95e-07, + "loss": 0.20567649602890015, + "num_tokens": 16404.0, + "reward": -12.422499656677246, + "reward_std": 7.134707450866699, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.522500038146973, + "rewards/supergames_reward/std": 7.134707927703857, + "step": 2, + "step_time": 9.960156448010821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/max_terminated_length": 287.0, + "completions/mean_length": 156.625, + "completions/mean_terminated_length": 156.625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "entropy": 0.562222421169281, + "epoch": 0.015, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9053616523742676, + "learning_rate": 9.9e-07, + "loss": 0.09602774679660797, + "num_tokens": 26953.0, + "reward": -74.94999694824219, + "reward_std": 70.73094177246094, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -75.0, + "rewards/supergames_reward/std": 70.71067810058594, + "step": 3, + "step_time": 13.686320498993155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 154.125, + "completions/mean_terminated_length": 154.125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7323317527770996, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.933310031890869, + "learning_rate": 9.849999999999999e-07, + "loss": 0.28249427676200867, + "num_tokens": 36514.0, + "reward": -40.7599983215332, + "reward_std": 81.73140716552734, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -40.80999755859375, + "rewards/supergames_reward/std": 81.68995666503906, + "step": 4, + "step_time": 12.805880262021674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 205.25, + "completions/mean_terminated_length": 205.25, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.6167430281639099, + "epoch": 0.025, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.344135046005249, + "learning_rate": 9.8e-07, + "loss": 0.1433994024991989, + "num_tokens": 47476.0, + "reward": -97.6612548828125, + "reward_std": 6.3321428298950195, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.76124572753906, + "rewards/supergames_reward/std": 6.3321428298950195, + "step": 5, + "step_time": 18.789364666008623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/max_terminated_length": 276.0, + "completions/mean_length": 167.25, + "completions/mean_terminated_length": 167.25, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.7583790421485901, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.172371864318848, + "learning_rate": 9.75e-07, + "loss": -0.12127404659986496, + "num_tokens": 55246.0, + "reward": -58.687503814697266, + "reward_std": 58.64059829711914, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -58.76250457763672, + "rewards/supergames_reward/std": 58.620460510253906, + "step": 6, + "step_time": 12.948570273991209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 133.875, + "completions/mean_terminated_length": 133.875, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.7447654604911804, + "epoch": 0.035, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.428595066070557, + "learning_rate": 9.7e-07, + "loss": 0.10135584324598312, + "num_tokens": 60837.0, + "reward": -32.45000076293945, + "reward_std": 41.754486083984375, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -32.5, + "rewards/supergames_reward/std": 41.66190719604492, + "step": 7, + "step_time": 12.058315072004916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 414.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 209.125, + "completions/mean_terminated_length": 209.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.3860666751861572, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.806255578994751, + "learning_rate": 9.649999999999999e-07, + "loss": -0.33238139748573303, + "num_tokens": 70838.0, + "reward": -80.50375366210938, + "reward_std": 38.82380294799805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -80.60375213623047, + "rewards/supergames_reward/std": 38.82379913330078, + "step": 8, + "step_time": 19.323370319994865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 247.125, + "completions/mean_terminated_length": 158.83334350585938, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.5816237330436707, + "epoch": 0.045, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3209943771362305, + "learning_rate": 9.6e-07, + "loss": 0.36752766370773315, + "num_tokens": 82135.0, + "reward": -27.688751220703125, + "reward_std": 82.8358154296875, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -27.738750457763672, + "rewards/supergames_reward/std": 82.78590393066406, + "step": 9, + "step_time": 21.948575104994234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/max_terminated_length": 240.0, + "completions/mean_length": 153.0, + "completions/mean_terminated_length": 153.0, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6301521062850952, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.125359058380127, + "learning_rate": 9.55e-07, + "loss": 0.06129350885748863, + "num_tokens": 89807.0, + "reward": -78.39624786376953, + "reward_std": 40.17042922973633, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -78.44625091552734, + "rewards/supergames_reward/std": 40.1396484375, + "step": 10, + "step_time": 9.098202993016457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/max_terminated_length": 133.0, + "completions/mean_length": 112.0, + "completions/mean_terminated_length": 112.0, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.5521990656852722, + "epoch": 0.055, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.616060256958008, + "learning_rate": 9.499999999999999e-07, + "loss": -0.006313305348157883, + "num_tokens": 95231.0, + "reward": -12.161249160766602, + "reward_std": 6.39578104019165, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.261249542236328, + "rewards/supergames_reward/std": 6.39578104019165, + "step": 11, + "step_time": 5.136311663984088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 214.875, + "completions/mean_terminated_length": 214.875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.5505905151367188, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.289583206176758, + "learning_rate": 9.45e-07, + "loss": -0.29802003502845764, + "num_tokens": 106238.0, + "reward": -91.66999816894531, + "reward_std": 23.2779541015625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -91.77000427246094, + "rewards/supergames_reward/std": 23.277956008911133, + "step": 12, + "step_time": 14.945365622988902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 167.375, + "completions/mean_terminated_length": 167.375, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "entropy": 0.540415346622467, + "epoch": 0.065, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.858291149139404, + "learning_rate": 9.399999999999999e-07, + "loss": 0.24004128575325012, + "num_tokens": 116937.0, + "reward": -20.38249969482422, + "reward_std": 87.55204010009766, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.457500457763672, + "rewards/supergames_reward/std": 87.52605438232422, + "step": 13, + "step_time": 10.482285185018554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "entropy": 0.7237679958343506, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.083980560302734, + "learning_rate": 9.35e-07, + "loss": -0.19025824964046478, + "num_tokens": 124640.0, + "reward": -51.397499084472656, + "reward_std": 54.486454010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -51.49749755859375, + "rewards/supergames_reward/std": 54.486454010009766, + "step": 14, + "step_time": 9.597231683001155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 141.375, + "completions/mean_terminated_length": 141.375, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.754334568977356, + "epoch": 0.075, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.720428943634033, + "learning_rate": 9.3e-07, + "loss": -0.16830675303936005, + "num_tokens": 132219.0, + "reward": -56.226253509521484, + "reward_std": 60.115657806396484, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -56.32625198364258, + "rewards/supergames_reward/std": 60.11566162109375, + "step": 15, + "step_time": 8.998362872982398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.0, + "completions/max_terminated_length": 209.0, + "completions/mean_length": 136.375, + "completions/mean_terminated_length": 136.375, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.6280568838119507, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.9681806564331055, + "learning_rate": 9.25e-07, + "loss": -0.01910916529595852, + "num_tokens": 139726.0, + "reward": -68.24000549316406, + "reward_std": 52.238914489746094, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.33999633789062, + "rewards/supergames_reward/std": 52.238914489746094, + "step": 16, + "step_time": 8.034480294008972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 211.875, + "completions/mean_terminated_length": 211.875, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.899176836013794, + "epoch": 0.085, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.691746711730957, + "learning_rate": 9.2e-07, + "loss": 0.25513291358947754, + "num_tokens": 147869.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 17, + "step_time": 13.655792869016295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 387.0, + "completions/max_terminated_length": 387.0, + "completions/mean_length": 238.0, + "completions/mean_terminated_length": 238.0, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.5596873760223389, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6711347103118896, + "learning_rate": 9.15e-07, + "loss": -0.2628335654735565, + "num_tokens": 159101.0, + "reward": -24.900001525878906, + "reward_std": 103.50983428955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.0, + "rewards/supergames_reward/std": 103.50983428955078, + "step": 18, + "step_time": 14.784329420013819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 277.0, + "completions/mean_length": 219.75, + "completions/mean_terminated_length": 178.00001525878906, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.9245517253875732, + "epoch": 0.095, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55502986907959, + "learning_rate": 9.1e-07, + "loss": 0.13060179352760315, + "num_tokens": 167299.0, + "reward": -86.89125061035156, + "reward_std": 36.95603561401367, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -86.94125366210938, + "rewards/supergames_reward/std": 36.93572235107422, + "step": 19, + "step_time": 18.832684700988466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/max_terminated_length": 267.0, + "completions/mean_length": 177.25, + "completions/mean_terminated_length": 177.25, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.7331254482269287, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.538497447967529, + "learning_rate": 9.05e-07, + "loss": -0.16418980062007904, + "num_tokens": 175141.0, + "reward": -68.52874755859375, + "reward_std": 58.48719787597656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.62875366210938, + "rewards/supergames_reward/std": 58.48720169067383, + "step": 20, + "step_time": 10.090975169994636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 190.625, + "completions/mean_terminated_length": 190.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.6659425497055054, + "epoch": 0.105, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.485373497009277, + "learning_rate": 9e-07, + "loss": -0.10106432437896729, + "num_tokens": 183098.0, + "reward": -71.48875427246094, + "reward_std": 55.33565139770508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -71.5887451171875, + "rewards/supergames_reward/std": 55.33565139770508, + "step": 21, + "step_time": 11.255001295008697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 179.0, + "completions/mean_terminated_length": 179.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.49365949630737305, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.829419136047363, + "learning_rate": 8.95e-07, + "loss": 0.10587231814861298, + "num_tokens": 192826.0, + "reward": -75.62000274658203, + "reward_std": 45.01959991455078, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -75.69499969482422, + "rewards/supergames_reward/std": 45.00411605834961, + "step": 22, + "step_time": 10.611246599000879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/max_terminated_length": 315.0, + "completions/mean_length": 212.625, + "completions/mean_terminated_length": 212.625, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.43159469962120056, + "epoch": 0.115, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.541125774383545, + "learning_rate": 8.9e-07, + "loss": -0.1681259125471115, + "num_tokens": 202815.0, + "reward": 23.67624855041504, + "reward_std": 67.74340057373047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 23.576250076293945, + "rewards/supergames_reward/std": 67.743408203125, + "step": 23, + "step_time": 11.731771531980485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/max_terminated_length": 168.0, + "completions/mean_length": 101.625, + "completions/mean_terminated_length": 101.625, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.3972298502922058, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.8285417556762695, + "learning_rate": 8.85e-07, + "loss": -0.011585958302021027, + "num_tokens": 208188.0, + "reward": 58.849998474121094, + "reward_std": 56.9304084777832, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 58.75, + "rewards/supergames_reward/std": 56.9304084777832, + "step": 24, + "step_time": 6.187504950998118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 303.0, + "completions/max_terminated_length": 303.0, + "completions/mean_length": 165.375, + "completions/mean_terminated_length": 165.375, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "entropy": 0.6339899301528931, + "epoch": 0.125, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.823799133300781, + "learning_rate": 8.799999999999999e-07, + "loss": 0.05159185826778412, + "num_tokens": 215935.0, + "reward": -23.521251678466797, + "reward_std": 74.23067474365234, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -23.621248245239258, + "rewards/supergames_reward/std": 74.23067474365234, + "step": 25, + "step_time": 11.059416300005978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 349.0, + "completions/max_terminated_length": 349.0, + "completions/mean_length": 204.0, + "completions/mean_terminated_length": 204.0, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.6462895274162292, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.969292640686035, + "learning_rate": 8.75e-07, + "loss": 0.0115677984431386, + "num_tokens": 223999.0, + "reward": -74.8175048828125, + "reward_std": 40.94221115112305, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -74.89250183105469, + "rewards/supergames_reward/std": 40.924625396728516, + "step": 26, + "step_time": 12.666237785975682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 166.5, + "completions/mean_terminated_length": 166.5, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4756850600242615, + "epoch": 0.135, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915069103240967, + "learning_rate": 8.699999999999999e-07, + "loss": -0.13321346044540405, + "num_tokens": 233643.0, + "reward": -52.125, + "reward_std": 19.405282974243164, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -52.20000076293945, + "rewards/supergames_reward/std": 19.33464813232422, + "step": 27, + "step_time": 11.172539752995363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 137.125, + "completions/mean_terminated_length": 137.125, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.5834778547286987, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.146666526794434, + "learning_rate": 8.65e-07, + "loss": 0.060454584658145905, + "num_tokens": 241204.0, + "reward": -68.99250793457031, + "reward_std": 51.010581970214844, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -69.09249877929688, + "rewards/supergames_reward/std": 51.010581970214844, + "step": 28, + "step_time": 7.025046669004951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 182.375, + "completions/mean_terminated_length": 182.375, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.6368551254272461, + "epoch": 0.145, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.15535306930542, + "learning_rate": 8.599999999999999e-07, + "loss": -0.3323673903942108, + "num_tokens": 249087.0, + "reward": -44.17000198364258, + "reward_std": 80.5373306274414, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -44.27000045776367, + "rewards/supergames_reward/std": 80.5373306274414, + "step": 29, + "step_time": 12.079259724996518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 127.75, + "completions/mean_terminated_length": 127.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4491196870803833, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.896186828613281, + "learning_rate": 8.55e-07, + "loss": -0.12887656688690186, + "num_tokens": 259445.0, + "reward": 57.446250915527344, + "reward_std": 38.78837585449219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 57.34625244140625, + "rewards/supergames_reward/std": 38.78837966918945, + "step": 30, + "step_time": 7.483972909016302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/max_terminated_length": 166.0, + "completions/mean_length": 112.875, + "completions/mean_terminated_length": 112.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4858454465866089, + "epoch": 0.155, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.742906093597412, + "learning_rate": 8.499999999999999e-07, + "loss": 0.06930096447467804, + "num_tokens": 264924.0, + "reward": -21.174999237060547, + "reward_std": 31.89051628112793, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -21.25, + "rewards/supergames_reward/std": 31.819807052612305, + "step": 31, + "step_time": 6.1866529019898735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/max_terminated_length": 148.0, + "completions/mean_length": 118.125, + "completions/mean_terminated_length": 118.125, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.45428749918937683, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.663414478302002, + "learning_rate": 8.45e-07, + "loss": 0.05735419690608978, + "num_tokens": 270381.0, + "reward": 0.4687504768371582, + "reward_std": 38.66371154785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 0.36875057220458984, + "rewards/supergames_reward/std": 38.6637077331543, + "step": 32, + "step_time": 5.577042757999152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 166.875, + "completions/mean_terminated_length": 166.875, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.6107293367385864, + "epoch": 0.165, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.75006628036499, + "learning_rate": 8.399999999999999e-07, + "loss": -0.14760127663612366, + "num_tokens": 278156.0, + "reward": -87.84750366210938, + "reward_std": 26.84708023071289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -87.94749450683594, + "rewards/supergames_reward/std": 26.847076416015625, + "step": 33, + "step_time": 12.552876825997373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 360.0, + "completions/mean_length": 231.0, + "completions/mean_terminated_length": 190.85714721679688, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.6089653372764587, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.781267166137695, + "learning_rate": 8.349999999999999e-07, + "loss": 0.4294504225254059, + "num_tokens": 289364.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 34, + "step_time": 18.961819477990502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 134.125, + "completions/mean_terminated_length": 134.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46158918738365173, + "epoch": 0.175, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.016353607177734, + "learning_rate": 8.299999999999999e-07, + "loss": 0.09251818805932999, + "num_tokens": 298733.0, + "reward": -35.558753967285156, + "reward_std": 15.881204605102539, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -35.65875244140625, + "rewards/supergames_reward/std": 15.881203651428223, + "step": 35, + "step_time": 8.38491243700264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/max_terminated_length": 207.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.5104788541793823, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.592893123626709, + "learning_rate": 8.249999999999999e-07, + "loss": -0.0410832017660141, + "num_tokens": 306420.0, + "reward": 5.4212493896484375, + "reward_std": 93.50404357910156, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 5.346250534057617, + "rewards/supergames_reward/std": 93.4718246459961, + "step": 36, + "step_time": 7.811868985998444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/max_terminated_length": 241.0, + "completions/mean_length": 178.0, + "completions/mean_terminated_length": 178.0, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.7527878880500793, + "epoch": 0.185, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8812546730041504, + "learning_rate": 8.199999999999999e-07, + "loss": 0.12539231777191162, + "num_tokens": 314252.0, + "reward": -89.58500671386719, + "reward_std": 29.256120681762695, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -89.66000366210938, + "rewards/supergames_reward/std": 29.24593734741211, + "step": 37, + "step_time": 8.959742914012168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 133.375, + "completions/mean_terminated_length": 133.375, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.3347901403903961, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.205414295196533, + "learning_rate": 8.149999999999999e-07, + "loss": -0.06991486996412277, + "num_tokens": 319839.0, + "reward": -14.498749732971191, + "reward_std": 13.007229804992676, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -14.598750114440918, + "rewards/supergames_reward/std": 13.007229804992676, + "step": 38, + "step_time": 7.377183554985095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/max_terminated_length": 181.0, + "completions/mean_length": 114.375, + "completions/mean_terminated_length": 114.375, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.4445965886116028, + "epoch": 0.195, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.561298847198486, + "learning_rate": 8.1e-07, + "loss": -0.06235164776444435, + "num_tokens": 325290.0, + "reward": -7.425000190734863, + "reward_std": 53.701602935791016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -7.5, + "rewards/supergames_reward/std": 53.652320861816406, + "step": 39, + "step_time": 6.753862089011818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 187.625, + "completions/mean_terminated_length": 187.625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.3950234353542328, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5988712310791016, + "learning_rate": 8.05e-07, + "loss": -0.1381440907716751, + "num_tokens": 335119.0, + "reward": -33.68375015258789, + "reward_std": 42.069435119628906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -33.783748626708984, + "rewards/supergames_reward/std": 42.069435119628906, + "step": 40, + "step_time": 13.718958162004128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 371.0, + "completions/mean_length": 251.125, + "completions/mean_terminated_length": 213.85714721679688, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.507067084312439, + "epoch": 0.205, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3898658752441406, + "learning_rate": 8e-07, + "loss": 0.0078964838758111, + "num_tokens": 346464.0, + "reward": -86.48625183105469, + "reward_std": 38.020694732666016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -86.56124877929688, + "rewards/supergames_reward/std": 38.010528564453125, + "step": 41, + "step_time": 18.90960379401804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 152.0, + "completions/max_terminated_length": 152.0, + "completions/mean_length": 116.375, + "completions/mean_terminated_length": 116.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3801707923412323, + "epoch": 0.21, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.95e-07, + "loss": 0.0, + "num_tokens": 351955.0, + "reward": -9.899999618530273, + "reward_std": 0.0, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -10.0, + "rewards/supergames_reward/std": 0.0, + "step": 42, + "step_time": 5.711630532023264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/max_terminated_length": 308.0, + "completions/mean_length": 247.375, + "completions/mean_terminated_length": 247.375, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.3185043931007385, + "epoch": 0.215, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.168762445449829, + "learning_rate": 7.9e-07, + "loss": 0.1970764398574829, + "num_tokens": 363278.0, + "reward": -53.89249801635742, + "reward_std": 63.02619934082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -53.99250030517578, + "rewards/supergames_reward/std": 63.02619934082031, + "step": 43, + "step_time": 11.661934906995157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/max_terminated_length": 330.0, + "completions/mean_length": 212.5, + "completions/mean_terminated_length": 212.5, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4446268379688263, + "epoch": 0.22, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.305844306945801, + "learning_rate": 7.85e-07, + "loss": -0.06735197454690933, + "num_tokens": 374306.0, + "reward": 18.059999465942383, + "reward_std": 98.9417953491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 17.959999084472656, + "rewards/supergames_reward/std": 98.9417953491211, + "step": 44, + "step_time": 12.492453911982011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.0, + "completions/max_terminated_length": 316.0, + "completions/mean_length": 183.0, + "completions/mean_terminated_length": 183.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.5269614458084106, + "epoch": 0.225, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8006370067596436, + "learning_rate": 7.799999999999999e-07, + "loss": -0.08870815485715866, + "num_tokens": 382210.0, + "reward": -54.928749084472656, + "reward_std": 72.5127944946289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -55.02874755859375, + "rewards/supergames_reward/std": 72.5127944946289, + "step": 45, + "step_time": 11.7636467939883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.0, + "completions/max_terminated_length": 244.0, + "completions/mean_length": 136.625, + "completions/mean_terminated_length": 136.625, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.34820204973220825, + "epoch": 0.23, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.911736011505127, + "learning_rate": 7.75e-07, + "loss": 0.04397330805659294, + "num_tokens": 391583.0, + "reward": -19.10375213623047, + "reward_std": 23.967702865600586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -19.203750610351562, + "rewards/supergames_reward/std": 23.967702865600586, + "step": 46, + "step_time": 10.119426151999505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 160.375, + "completions/mean_terminated_length": 160.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.611914336681366, + "epoch": 0.235, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.571913242340088, + "learning_rate": 7.699999999999999e-07, + "loss": -0.044293053448200226, + "num_tokens": 399290.0, + "reward": -25.186250686645508, + "reward_std": 71.46305084228516, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.286251068115234, + "rewards/supergames_reward/std": 71.46305084228516, + "step": 47, + "step_time": 8.563868903991533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/max_terminated_length": 167.0, + "completions/mean_length": 111.125, + "completions/mean_terminated_length": 111.125, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.46585777401924133, + "epoch": 0.24, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.654109954833984, + "learning_rate": 7.65e-07, + "loss": -0.035151124000549316, + "num_tokens": 404707.0, + "reward": -0.408750057220459, + "reward_std": 31.939748764038086, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -0.5087499618530273, + "rewards/supergames_reward/std": 31.93975257873535, + "step": 48, + "step_time": 6.438482580007985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/max_terminated_length": 306.0, + "completions/mean_length": 216.625, + "completions/mean_terminated_length": 216.625, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.33054471015930176, + "epoch": 0.245, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.700303792953491, + "learning_rate": 7.599999999999999e-07, + "loss": -0.04393656551837921, + "num_tokens": 414736.0, + "reward": -97.26000213623047, + "reward_std": 1.6294406652450562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.36000061035156, + "rewards/supergames_reward/std": 1.629441738128662, + "step": 49, + "step_time": 11.636040106008295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 378.0, + "completions/max_terminated_length": 378.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.48711177706718445, + "epoch": 0.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9650564193725586, + "learning_rate": 7.55e-07, + "loss": -0.3529520332813263, + "num_tokens": 424795.0, + "reward": 47.408748626708984, + "reward_std": 72.72083282470703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 47.30875015258789, + "rewards/supergames_reward/std": 72.72083282470703, + "step": 50, + "step_time": 14.258096842997475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 267.625, + "completions/mean_terminated_length": 232.71429443359375, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4482860863208771, + "epoch": 0.255, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2998197078704834, + "learning_rate": 7.5e-07, + "loss": 0.2493654191493988, + "num_tokens": 436288.0, + "reward": 50.61125183105469, + "reward_std": 75.3349838256836, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.53624725341797, + "rewards/supergames_reward/std": 75.27783966064453, + "step": 51, + "step_time": 19.20827590499539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 176.0, + "completions/max_terminated_length": 176.0, + "completions/mean_length": 144.5, + "completions/mean_terminated_length": 144.5, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "entropy": 0.5378735661506653, + "epoch": 0.26, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.114461898803711, + "learning_rate": 7.45e-07, + "loss": 0.0036597400903701782, + "num_tokens": 441980.0, + "reward": -15.20625114440918, + "reward_std": 45.35526657104492, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.28125, + "rewards/supergames_reward/std": 45.301815032958984, + "step": 52, + "step_time": 6.660627231001854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/max_terminated_length": 136.0, + "completions/mean_length": 119.0, + "completions/mean_terminated_length": 119.0, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "entropy": 0.41796669363975525, + "epoch": 0.265, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.059505462646484, + "learning_rate": 7.4e-07, + "loss": -0.014973883517086506, + "num_tokens": 447468.0, + "reward": 15.170000076293945, + "reward_std": 46.658329010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 15.070000648498535, + "rewards/supergames_reward/std": 46.658329010009766, + "step": 53, + "step_time": 5.271571868011961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/max_terminated_length": 259.0, + "completions/mean_length": 191.125, + "completions/mean_terminated_length": 191.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.45421987771987915, + "epoch": 0.27, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4454867839813232, + "learning_rate": 7.35e-07, + "loss": -0.1128973588347435, + "num_tokens": 457301.0, + "reward": 38.89875030517578, + "reward_std": 62.1772575378418, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 38.79874801635742, + "rewards/supergames_reward/std": 62.1772575378418, + "step": 54, + "step_time": 10.524528659996577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/max_terminated_length": 319.0, + "completions/mean_length": 233.375, + "completions/mean_terminated_length": 233.375, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "entropy": 0.4449213445186615, + "epoch": 0.275, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2065398693084717, + "learning_rate": 7.3e-07, + "loss": 0.0016277075046673417, + "num_tokens": 467520.0, + "reward": 27.372499465942383, + "reward_std": 81.23126983642578, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.27250099182129, + "rewards/supergames_reward/std": 81.23126983642578, + "step": 55, + "step_time": 12.592280682991259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 357.0, + "completions/mean_length": 261.0, + "completions/mean_terminated_length": 225.1428680419922, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.506920337677002, + "epoch": 0.28, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.058388710021973, + "learning_rate": 7.249999999999999e-07, + "loss": 0.3289242386817932, + "num_tokens": 476056.0, + "reward": 47.05249786376953, + "reward_std": 80.97753143310547, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 46.977500915527344, + "rewards/supergames_reward/std": 80.9256362915039, + "step": 56, + "step_time": 18.845177220006008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.0, + "completions/max_terminated_length": 250.0, + "completions/mean_length": 192.75, + "completions/mean_terminated_length": 192.75, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "entropy": 0.5197336077690125, + "epoch": 0.285, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.230815410614014, + "learning_rate": 7.2e-07, + "loss": -0.15186546742916107, + "num_tokens": 484014.0, + "reward": -64.5050048828125, + "reward_std": 50.35145568847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -64.60499572753906, + "rewards/supergames_reward/std": 50.3514518737793, + "step": 57, + "step_time": 9.576232638006331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 249.0, + "completions/mean_terminated_length": 249.0, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "entropy": 0.42913514375686646, + "epoch": 0.29, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.8544528484344482, + "learning_rate": 7.149999999999999e-07, + "loss": -0.056405920535326004, + "num_tokens": 494390.0, + "reward": 40.123748779296875, + "reward_std": 84.164794921875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 40.02375030517578, + "rewards/supergames_reward/std": 84.164794921875, + "step": 58, + "step_time": 12.385353341000155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 138.0, + "completions/max_terminated_length": 138.0, + "completions/mean_length": 107.25, + "completions/mean_terminated_length": 107.25, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.34386488795280457, + "epoch": 0.295, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.6915154457092285, + "learning_rate": 7.1e-07, + "loss": 0.0865321159362793, + "num_tokens": 499816.0, + "reward": -8.244999885559082, + "reward_std": 4.681046962738037, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.345000267028809, + "rewards/supergames_reward/std": 4.681046962738037, + "step": 59, + "step_time": 5.42325339600211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/max_terminated_length": 187.0, + "completions/mean_length": 127.25, + "completions/mean_terminated_length": 127.25, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.5430045127868652, + "epoch": 0.3, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.635311126708984, + "learning_rate": 7.049999999999999e-07, + "loss": -0.05412375554442406, + "num_tokens": 505410.0, + "reward": 20.005001068115234, + "reward_std": 55.156005859375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 19.904998779296875, + "rewards/supergames_reward/std": 55.156005859375, + "step": 60, + "step_time": 7.068010902003152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/max_terminated_length": 286.0, + "completions/mean_length": 192.125, + "completions/mean_terminated_length": 192.125, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4213869869709015, + "epoch": 0.305, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4664618968963623, + "learning_rate": 7e-07, + "loss": -0.0495159812271595, + "num_tokens": 516283.0, + "reward": -47.212501525878906, + "reward_std": 91.2253646850586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -47.3125, + "rewards/supergames_reward/std": 91.2253646850586, + "step": 61, + "step_time": 11.176304235996213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 171.0, + "completions/mean_terminated_length": 171.0, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.4626811146736145, + "epoch": 0.31, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.949252128601074, + "learning_rate": 6.949999999999999e-07, + "loss": 0.192047581076622, + "num_tokens": 522251.0, + "reward": -15.092500686645508, + "reward_std": 49.82962417602539, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.167499542236328, + "rewards/supergames_reward/std": 49.78091049194336, + "step": 62, + "step_time": 12.075224861997413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 429.0, + "completions/mean_length": 283.5, + "completions/mean_terminated_length": 250.85714721679688, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4814216196537018, + "epoch": 0.315, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7918753623962402, + "learning_rate": 6.9e-07, + "loss": 0.15706156194210052, + "num_tokens": 533815.0, + "reward": -20.86625099182129, + "reward_std": 85.99250030517578, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.94124984741211, + "rewards/supergames_reward/std": 85.96620178222656, + "step": 63, + "step_time": 19.448832260008203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.0, + "completions/max_terminated_length": 189.0, + "completions/mean_length": 131.25, + "completions/mean_terminated_length": 131.25, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4929414987564087, + "epoch": 0.32, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.774101734161377, + "learning_rate": 6.85e-07, + "loss": -0.060691747814416885, + "num_tokens": 539465.0, + "reward": -21.0674991607666, + "reward_std": 6.892730236053467, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -21.167499542236328, + "rewards/supergames_reward/std": 6.892730236053467, + "step": 64, + "step_time": 7.141569407976931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 454.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 281.375, + "completions/mean_terminated_length": 281.375, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "entropy": 0.45866572856903076, + "epoch": 0.325, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.092717409133911, + "learning_rate": 6.800000000000001e-07, + "loss": 0.11694959551095963, + "num_tokens": 551012.0, + "reward": 49.938751220703125, + "reward_std": 79.1865005493164, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 49.8387451171875, + "rewards/supergames_reward/std": 79.1865005493164, + "step": 65, + "step_time": 17.06170882002334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/max_terminated_length": 203.0, + "completions/mean_length": 128.625, + "completions/mean_terminated_length": 128.625, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.5005782246589661, + "epoch": 0.33, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.77616024017334, + "learning_rate": 6.75e-07, + "loss": 0.11242837458848953, + "num_tokens": 556577.0, + "reward": 48.875, + "reward_std": 49.13539123535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.775001525878906, + "rewards/supergames_reward/std": 49.13539123535156, + "step": 66, + "step_time": 7.695410129002994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 436.0, + "completions/max_terminated_length": 436.0, + "completions/mean_length": 281.875, + "completions/mean_terminated_length": 281.875, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "entropy": 0.507097065448761, + "epoch": 0.335, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.911184787750244, + "learning_rate": 6.7e-07, + "loss": -0.08151137083768845, + "num_tokens": 567120.0, + "reward": -12.886249542236328, + "reward_std": 84.04269409179688, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.986251831054688, + "rewards/supergames_reward/std": 84.04269409179688, + "step": 67, + "step_time": 16.409127942984924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 318.375, + "completions/mean_terminated_length": 290.71429443359375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.486402690410614, + "epoch": 0.34, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9461607933044434, + "learning_rate": 6.65e-07, + "loss": 0.2442178875207901, + "num_tokens": 578995.0, + "reward": 2.8387489318847656, + "reward_std": 98.47579956054688, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 2.7637500762939453, + "rewards/supergames_reward/std": 98.44595336914062, + "step": 68, + "step_time": 19.32364999302081 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 291.0, + "completions/max_terminated_length": 291.0, + "completions/mean_length": 219.625, + "completions/mean_terminated_length": 219.625, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.37856727838516235, + "epoch": 0.345, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0204732418060303, + "learning_rate": 6.6e-07, + "loss": -0.08764916658401489, + "num_tokens": 589072.0, + "reward": -54.18375015258789, + "reward_std": 71.45012664794922, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -54.283748626708984, + "rewards/supergames_reward/std": 71.45013427734375, + "step": 69, + "step_time": 11.126611230982235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 305.0, + "completions/mean_length": 254.75, + "completions/mean_terminated_length": 218.00001525878906, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46423614025115967, + "epoch": 0.35, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0140154361724854, + "learning_rate": 6.55e-07, + "loss": 0.26797160506248474, + "num_tokens": 600438.0, + "reward": -32.04875183105469, + "reward_std": 69.86957550048828, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -32.12375259399414, + "rewards/supergames_reward/std": 69.84178161621094, + "step": 70, + "step_time": 19.12617294798838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 372.0, + "completions/max_terminated_length": 372.0, + "completions/mean_length": 194.125, + "completions/mean_terminated_length": 194.125, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.3427293300628662, + "epoch": 0.355, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.781383991241455, + "learning_rate": 6.5e-07, + "loss": 0.354524701833725, + "num_tokens": 610327.0, + "reward": -6.913749694824219, + "reward_std": 87.56071472167969, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -6.988750457763672, + "rewards/supergames_reward/std": 87.53033447265625, + "step": 71, + "step_time": 14.041668199002743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/max_terminated_length": 197.0, + "completions/mean_length": 124.75, + "completions/mean_terminated_length": 124.75, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.4455118477344513, + "epoch": 0.36, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.726032733917236, + "learning_rate": 6.45e-07, + "loss": 0.18206289410591125, + "num_tokens": 615853.0, + "reward": 41.95124816894531, + "reward_std": 60.640811920166016, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 41.85124969482422, + "rewards/supergames_reward/std": 60.640811920166016, + "step": 72, + "step_time": 7.416493425989756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.0, + "completions/max_terminated_length": 323.0, + "completions/mean_length": 147.375, + "completions/mean_terminated_length": 147.375, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.480733186006546, + "epoch": 0.365, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.853621482849121, + "learning_rate": 6.4e-07, + "loss": 0.04558124020695686, + "num_tokens": 621584.0, + "reward": 14.884998321533203, + "reward_std": 46.25857162475586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 14.78499984741211, + "rewards/supergames_reward/std": 46.25857162475586, + "step": 73, + "step_time": 11.95379540900467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 283.0, + "completions/max_terminated_length": 283.0, + "completions/mean_length": 188.125, + "completions/mean_terminated_length": 188.125, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.5526013374328613, + "epoch": 0.37, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.729883909225464, + "learning_rate": 6.35e-07, + "loss": 0.06268303096294403, + "num_tokens": 629545.0, + "reward": -68.05750274658203, + "reward_std": 59.44808578491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.15750122070312, + "rewards/supergames_reward/std": 59.44808578491211, + "step": 74, + "step_time": 10.64124580900534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 141.0, + "completions/max_terminated_length": 141.0, + "completions/mean_length": 117.375, + "completions/mean_terminated_length": 117.375, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.3744319677352905, + "epoch": 0.375, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.775154113769531, + "learning_rate": 6.3e-07, + "loss": -0.023284845054149628, + "num_tokens": 635052.0, + "reward": 3.9374990463256836, + "reward_std": 61.85771179199219, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 3.862499237060547, + "rewards/supergames_reward/std": 61.80967712402344, + "step": 75, + "step_time": 5.432648951013107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 157.0, + "completions/mean_terminated_length": 157.0, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4822140336036682, + "epoch": 0.38, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.715704917907715, + "learning_rate": 6.249999999999999e-07, + "loss": -0.04918142035603523, + "num_tokens": 644644.0, + "reward": 48.64125061035156, + "reward_std": 78.0205078125, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 48.56624984741211, + "rewards/supergames_reward/std": 77.96605682373047, + "step": 76, + "step_time": 10.348935816989979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/max_terminated_length": 380.0, + "completions/mean_length": 210.875, + "completions/mean_terminated_length": 210.875, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "entropy": 0.45722317695617676, + "epoch": 0.385, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6057567596435547, + "learning_rate": 6.2e-07, + "loss": 0.058352649211883545, + "num_tokens": 652771.0, + "reward": -8.283750534057617, + "reward_std": 77.68436431884766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.383749008178711, + "rewards/supergames_reward/std": 77.68437194824219, + "step": 77, + "step_time": 14.142948755004909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 188.25, + "completions/mean_terminated_length": 188.25, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.33380764722824097, + "epoch": 0.39, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2239863872528076, + "learning_rate": 6.149999999999999e-07, + "loss": -0.04592633992433548, + "num_tokens": 662653.0, + "reward": 11.153749465942383, + "reward_std": 89.88329315185547, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 11.053749084472656, + "rewards/supergames_reward/std": 89.88329315185547, + "step": 78, + "step_time": 12.277474621019792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/max_terminated_length": 143.0, + "completions/mean_length": 108.875, + "completions/mean_terminated_length": 108.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.3518812656402588, + "epoch": 0.395, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.967776298522949, + "learning_rate": 6.1e-07, + "loss": -0.10800496488809586, + "num_tokens": 668076.0, + "reward": 70.33250427246094, + "reward_std": 49.9145393371582, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 70.23249816894531, + "rewards/supergames_reward/std": 49.91453552246094, + "step": 79, + "step_time": 5.52714040101273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 152.125, + "completions/mean_terminated_length": 152.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4699896574020386, + "epoch": 0.4, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.561161994934082, + "learning_rate": 6.049999999999999e-07, + "loss": 0.09505834430456161, + "num_tokens": 675701.0, + "reward": -20.25625228881836, + "reward_std": 53.78542709350586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -20.35624885559082, + "rewards/supergames_reward/std": 53.78542709350586, + "step": 80, + "step_time": 7.155081019998761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 288.875, + "completions/mean_terminated_length": 257.0, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.4086494445800781, + "epoch": 0.405, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.3719611167907715, + "learning_rate": 6e-07, + "loss": 0.04727627709507942, + "num_tokens": 687332.0, + "reward": -37.20750045776367, + "reward_std": 91.7676010131836, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -37.25749969482422, + "rewards/supergames_reward/std": 91.72847747802734, + "step": 81, + "step_time": 19.231399144016905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 376.0, + "completions/max_terminated_length": 376.0, + "completions/mean_length": 227.75, + "completions/mean_terminated_length": 227.75, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.43233707547187805, + "epoch": 0.41, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3391714096069336, + "learning_rate": 5.949999999999999e-07, + "loss": -0.25524550676345825, + "num_tokens": 698474.0, + "reward": -3.7287511825561523, + "reward_std": 91.25379943847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.828749656677246, + "rewards/supergames_reward/std": 91.25379943847656, + "step": 82, + "step_time": 14.329176368017215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 178.125, + "completions/mean_terminated_length": 178.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.4293636977672577, + "epoch": 0.415, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.7271742820739746, + "learning_rate": 5.9e-07, + "loss": 0.09273672848939896, + "num_tokens": 708243.0, + "reward": 12.802498817443848, + "reward_std": 86.23526000976562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 12.702500343322754, + "rewards/supergames_reward/std": 86.23526000976562, + "step": 83, + "step_time": 11.419686011999147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 425.0, + "completions/max_terminated_length": 425.0, + "completions/mean_length": 255.375, + "completions/mean_terminated_length": 255.375, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "entropy": 0.38068246841430664, + "epoch": 0.42, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.6244184970855713, + "learning_rate": 5.849999999999999e-07, + "loss": 0.02006213366985321, + "num_tokens": 718606.0, + "reward": 37.803749084472656, + "reward_std": 57.58624267578125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 37.70375061035156, + "rewards/supergames_reward/std": 57.58624267578125, + "step": 84, + "step_time": 15.975198492989875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 116.0, + "completions/max_terminated_length": 116.0, + "completions/mean_length": 108.125, + "completions/mean_terminated_length": 108.125, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.339572012424469, + "epoch": 0.425, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.754759311676025, + "learning_rate": 5.8e-07, + "loss": -0.011015941388905048, + "num_tokens": 723983.0, + "reward": -3.8212504386901855, + "reward_std": 61.624786376953125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.921250820159912, + "rewards/supergames_reward/std": 61.624786376953125, + "step": 85, + "step_time": 4.542777584982105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 447.0, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 291.375, + "completions/mean_terminated_length": 291.375, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "entropy": 0.5846173763275146, + "epoch": 0.43, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55724835395813, + "learning_rate": 5.749999999999999e-07, + "loss": -0.04981120675802231, + "num_tokens": 735642.0, + "reward": 50.07499694824219, + "reward_std": 92.6283187866211, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.0, + "rewards/supergames_reward/std": 92.5820083618164, + "step": 86, + "step_time": 16.777178087009815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/max_terminated_length": 232.0, + "completions/mean_length": 192.0, + "completions/mean_terminated_length": 192.0, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "entropy": 0.5139474272727966, + "epoch": 0.435, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8860411643981934, + "learning_rate": 5.699999999999999e-07, + "loss": -0.02194221317768097, + "num_tokens": 743602.0, + "reward": -52.522499084472656, + "reward_std": 73.8686294555664, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -52.62249755859375, + "rewards/supergames_reward/std": 73.86863708496094, + "step": 87, + "step_time": 8.772893911984283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/max_terminated_length": 266.0, + "completions/mean_length": 146.875, + "completions/mean_terminated_length": 146.875, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.3428497910499573, + "epoch": 0.44, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.509483337402344, + "learning_rate": 5.649999999999999e-07, + "loss": 0.19857533276081085, + "num_tokens": 753065.0, + "reward": 63.63874816894531, + "reward_std": 67.54963684082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 63.53874969482422, + "rewards/supergames_reward/std": 67.54963684082031, + "step": 88, + "step_time": 10.230529835011112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 446.0, + "completions/max_terminated_length": 446.0, + "completions/mean_length": 270.625, + "completions/mean_terminated_length": 270.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.4417419731616974, + "epoch": 0.445, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.973951816558838, + "learning_rate": 5.6e-07, + "loss": -0.00031440958264283836, + "num_tokens": 764534.0, + "reward": -9.736251831054688, + "reward_std": 92.25879669189453, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -9.836250305175781, + "rewards/supergames_reward/std": 92.25880432128906, + "step": 89, + "step_time": 17.40752330099349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 398.0, + "completions/max_terminated_length": 398.0, + "completions/mean_length": 227.375, + "completions/mean_terminated_length": 227.375, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.4732024073600769, + "epoch": 0.45, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2560815811157227, + "learning_rate": 5.55e-07, + "loss": 0.03820464387536049, + "num_tokens": 772809.0, + "reward": 5.982499599456787, + "reward_std": 58.833797454833984, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 5.8824992179870605, + "rewards/supergames_reward/std": 58.83380126953125, + "step": 90, + "step_time": 15.007269965979503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 207.5, + "completions/mean_terminated_length": 207.5, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.42038559913635254, + "epoch": 0.455, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3649117946624756, + "learning_rate": 5.5e-07, + "loss": 0.1510048508644104, + "num_tokens": 782805.0, + "reward": -5.61500358581543, + "reward_std": 82.67152404785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -5.714998245239258, + "rewards/supergames_reward/std": 82.67152404785156, + "step": 91, + "step_time": 10.386137172987219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 416.0, + "completions/max_terminated_length": 416.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.3971734642982483, + "epoch": 0.46, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.433772563934326, + "learning_rate": 5.45e-07, + "loss": -0.3027213513851166, + "num_tokens": 793840.0, + "reward": -29.06500244140625, + "reward_std": 91.68325805664062, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -29.16499900817871, + "rewards/supergames_reward/std": 91.68325805664062, + "step": 92, + "step_time": 15.790611553995404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 206.625, + "completions/mean_terminated_length": 206.625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.5906907320022583, + "epoch": 0.465, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.293088912963867, + "learning_rate": 5.4e-07, + "loss": 0.09760032594203949, + "num_tokens": 803821.0, + "reward": -72.73999786376953, + "reward_std": 50.72484588623047, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -72.81500244140625, + "rewards/supergames_reward/std": 50.7094841003418, + "step": 93, + "step_time": 13.043119941983605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/max_terminated_length": 143.0, + "completions/mean_length": 115.75, + "completions/mean_terminated_length": 115.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4233115315437317, + "epoch": 0.47, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.58886194229126, + "learning_rate": 5.35e-07, + "loss": -0.04040650278329849, + "num_tokens": 809299.0, + "reward": -3.3050003051757812, + "reward_std": 24.443714141845703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.405000686645508, + "rewards/supergames_reward/std": 24.443714141845703, + "step": 94, + "step_time": 5.512874762003776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 122.0, + "completions/max_terminated_length": 122.0, + "completions/mean_length": 103.125, + "completions/mean_terminated_length": 103.125, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.4055764377117157, + "epoch": 0.475, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.981549263000488, + "learning_rate": 5.3e-07, + "loss": -0.07588938623666763, + "num_tokens": 814716.0, + "reward": 60.06999969482422, + "reward_std": 42.708473205566406, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 59.970001220703125, + "rewards/supergames_reward/std": 42.708473205566406, + "step": 95, + "step_time": 4.766389057011111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 475.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.408913254737854, + "epoch": 0.48, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.79402756690979, + "learning_rate": 5.25e-07, + "loss": -0.3087541460990906, + "num_tokens": 824791.0, + "reward": -15.042500495910645, + "reward_std": 84.37246704101562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -15.142499923706055, + "rewards/supergames_reward/std": 84.37246704101562, + "step": 96, + "step_time": 17.779843941010768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/max_terminated_length": 154.0, + "completions/mean_length": 121.625, + "completions/mean_terminated_length": 121.625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.529015839099884, + "epoch": 0.485, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.142816066741943, + "learning_rate": 5.2e-07, + "loss": 0.07216037809848785, + "num_tokens": 830300.0, + "reward": -32.743751525878906, + "reward_std": 54.61775588989258, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -32.81875228881836, + "rewards/supergames_reward/std": 54.58256149291992, + "step": 97, + "step_time": 5.913989890017547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 126.0, + "completions/max_terminated_length": 126.0, + "completions/mean_length": 95.625, + "completions/mean_terminated_length": 95.625, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.33743274211883545, + "epoch": 0.49, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.197273254394531, + "learning_rate": 5.149999999999999e-07, + "loss": 0.07500407844781876, + "num_tokens": 835633.0, + "reward": 11.00999927520752, + "reward_std": 62.73848342895508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 10.910000801086426, + "rewards/supergames_reward/std": 62.73848342895508, + "step": 98, + "step_time": 4.9155233050114475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 108.5, + "completions/mean_terminated_length": 108.5, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.3106803297996521, + "epoch": 0.495, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.864002227783203, + "learning_rate": 5.1e-07, + "loss": -0.1206185445189476, + "num_tokens": 841045.0, + "reward": 14.238749504089355, + "reward_std": 51.77016830444336, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 14.138750076293945, + "rewards/supergames_reward/std": 51.77016830444336, + "step": 99, + "step_time": 5.7507751359953545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 97.125, + "completions/mean_terminated_length": 97.125, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.35160186886787415, + "epoch": 0.5, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915731906890869, + "learning_rate": 5.049999999999999e-07, + "loss": -0.07951541244983673, + "num_tokens": 846398.0, + "reward": 67.78125, + "reward_std": 38.15436553955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 67.68124389648438, + "rewards/supergames_reward/std": 38.15436553955078, + "step": 100, + "step_time": 5.994720440998208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 320.0, + "completions/max_terminated_length": 320.0, + "completions/mean_length": 208.75, + "completions/mean_terminated_length": 208.75, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.452095091342926, + "epoch": 0.505, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.065176963806152, + "learning_rate": 5e-07, + "loss": -0.1397111564874649, + "num_tokens": 857420.0, + "reward": -5.801251411437988, + "reward_std": 73.56117248535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -5.90125036239624, + "rewards/supergames_reward/std": 73.5611801147461, + "step": 101, + "step_time": 12.28178753197426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/max_terminated_length": 230.0, + "completions/mean_length": 171.375, + "completions/mean_terminated_length": 171.375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4138447642326355, + "epoch": 0.51, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.998823404312134, + "learning_rate": 4.95e-07, + "loss": -0.08941879868507385, + "num_tokens": 867103.0, + "reward": 92.24749755859375, + "reward_std": 14.540005683898926, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 92.14749908447266, + "rewards/supergames_reward/std": 14.540006637573242, + "step": 102, + "step_time": 8.992682139010867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 311.0, + "completions/max_terminated_length": 311.0, + "completions/mean_length": 163.0, + "completions/mean_terminated_length": 163.0, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.4182547330856323, + "epoch": 0.515, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.419595241546631, + "learning_rate": 4.9e-07, + "loss": -0.19304415583610535, + "num_tokens": 876679.0, + "reward": 79.13249969482422, + "reward_std": 59.3050422668457, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 79.03250122070312, + "rewards/supergames_reward/std": 59.30504608154297, + "step": 103, + "step_time": 11.836970244999975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 253.0, + "completions/max_terminated_length": 253.0, + "completions/mean_length": 155.625, + "completions/mean_terminated_length": 155.625, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.49889081716537476, + "epoch": 0.52, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.463719367980957, + "learning_rate": 4.85e-07, + "loss": -0.04786720871925354, + "num_tokens": 884348.0, + "reward": 48.616249084472656, + "reward_std": 73.30741119384766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.51625061035156, + "rewards/supergames_reward/std": 73.30741119384766, + "step": 104, + "step_time": 9.579507841990562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/max_terminated_length": 380.0, + "completions/mean_length": 269.5, + "completions/mean_terminated_length": 269.5, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "entropy": 0.5286482572555542, + "epoch": 0.525, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.993295431137085, + "learning_rate": 4.8e-07, + "loss": -0.08071611076593399, + "num_tokens": 894856.0, + "reward": 34.970001220703125, + "reward_std": 85.324462890625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 34.869998931884766, + "rewards/supergames_reward/std": 85.324462890625, + "step": 105, + "step_time": 14.295730627985904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 203.25, + "completions/mean_terminated_length": 203.25, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.6049264073371887, + "epoch": 0.53, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.787306785583496, + "learning_rate": 4.7499999999999995e-07, + "loss": 0.19928883016109467, + "num_tokens": 902922.0, + "reward": 32.44499969482422, + "reward_std": 70.4663314819336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 32.369998931884766, + "rewards/supergames_reward/std": 70.41261291503906, + "step": 106, + "step_time": 9.84894504098338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 194.0, + "completions/mean_terminated_length": 194.0, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "entropy": 0.5115755200386047, + "epoch": 0.535, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.277207374572754, + "learning_rate": 4.6999999999999995e-07, + "loss": 0.09118300676345825, + "num_tokens": 910906.0, + "reward": 40.432498931884766, + "reward_std": 48.54548263549805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 40.33250045776367, + "rewards/supergames_reward/std": 48.54548263549805, + "step": 107, + "step_time": 10.466603949986165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 332.0, + "completions/max_terminated_length": 332.0, + "completions/mean_length": 195.875, + "completions/mean_terminated_length": 195.875, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4192962944507599, + "epoch": 0.54, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.308061122894287, + "learning_rate": 4.65e-07, + "loss": 0.28064602613449097, + "num_tokens": 920745.0, + "reward": -21.982500076293945, + "reward_std": 66.31195068359375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -22.08249855041504, + "rewards/supergames_reward/std": 66.31195068359375, + "step": 108, + "step_time": 12.595848233992001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 195.125, + "completions/mean_terminated_length": 195.125, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.37523961067199707, + "epoch": 0.545, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.802706718444824, + "learning_rate": 4.6e-07, + "loss": -0.06614465266466141, + "num_tokens": 930618.0, + "reward": 71.75, + "reward_std": 51.14524459838867, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 71.6500015258789, + "rewards/supergames_reward/std": 51.1452522277832, + "step": 109, + "step_time": 10.33884758799104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 167.75, + "completions/mean_terminated_length": 167.75, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.4339042007923126, + "epoch": 0.55, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.254003047943115, + "learning_rate": 4.55e-07, + "loss": 0.34405598044395447, + "num_tokens": 941256.0, + "reward": 99.21875, + "reward_std": 2.492553234100342, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 99.1187515258789, + "rewards/supergames_reward/std": 2.4925525188446045, + "step": 110, + "step_time": 12.657525141985388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 285.0, + "completions/max_terminated_length": 285.0, + "completions/mean_length": 211.0, + "completions/mean_terminated_length": 211.0, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "entropy": 0.46809861063957214, + "epoch": 0.555, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.811166286468506, + "learning_rate": 4.5e-07, + "loss": 0.045113153755664825, + "num_tokens": 951296.0, + "reward": 55.27750015258789, + "reward_std": 47.90687561035156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 55.1775016784668, + "rewards/supergames_reward/std": 47.9068717956543, + "step": 111, + "step_time": 10.91118389699841 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 145.0, + "completions/max_terminated_length": 145.0, + "completions/mean_length": 108.375, + "completions/mean_terminated_length": 108.375, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.34181955456733704, + "epoch": 0.56, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.537296772003174, + "learning_rate": 4.45e-07, + "loss": -0.052413541823625565, + "num_tokens": 956675.0, + "reward": 47.56624984741211, + "reward_std": 49.185157775878906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 47.466251373291016, + "rewards/supergames_reward/std": 49.18516159057617, + "step": 112, + "step_time": 5.552288047998445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/max_terminated_length": 245.0, + "completions/mean_length": 189.125, + "completions/mean_terminated_length": 189.125, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "entropy": 0.5034605860710144, + "epoch": 0.565, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.349876642227173, + "learning_rate": 4.3999999999999997e-07, + "loss": -0.08186715841293335, + "num_tokens": 964636.0, + "reward": 81.32374572753906, + "reward_std": 36.85074996948242, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 81.2237548828125, + "rewards/supergames_reward/std": 36.85074996948242, + "step": 113, + "step_time": 9.267611294984818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/max_terminated_length": 178.0, + "completions/mean_length": 111.25, + "completions/mean_terminated_length": 111.25, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "entropy": 0.3229002058506012, + "epoch": 0.57, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.245575428009033, + "learning_rate": 4.3499999999999996e-07, + "loss": 0.06544198095798492, + "num_tokens": 970070.0, + "reward": 63.1775016784668, + "reward_std": 52.30624008178711, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 63.07749938964844, + "rewards/supergames_reward/std": 52.30624008178711, + "step": 114, + "step_time": 6.731139309995342 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 224.0, + "completions/max_terminated_length": 224.0, + "completions/mean_length": 170.0, + "completions/mean_terminated_length": 170.0, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.3274807929992676, + "epoch": 0.575, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5544140338897705, + "learning_rate": 4.2999999999999996e-07, + "loss": -0.06953569501638412, + "num_tokens": 979702.0, + "reward": 25.62125015258789, + "reward_std": 64.52003479003906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 25.521244049072266, + "rewards/supergames_reward/std": 64.52003479003906, + "step": 115, + "step_time": 8.702618776995223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 427.0, + "completions/max_terminated_length": 427.0, + "completions/mean_length": 209.5, + "completions/mean_terminated_length": 209.5, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.4970880150794983, + "epoch": 0.58, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4932165145874023, + "learning_rate": 4.2499999999999995e-07, + "loss": -0.2281273603439331, + "num_tokens": 990690.0, + "reward": 50.54750061035156, + "reward_std": 70.5126953125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 50.4474983215332, + "rewards/supergames_reward/std": 70.51270294189453, + "step": 116, + "step_time": 16.204386608995264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.0, + "completions/max_terminated_length": 228.0, + "completions/mean_length": 132.375, + "completions/mean_terminated_length": 132.375, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4380227029323578, + "epoch": 0.585, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.089909076690674, + "learning_rate": 4.1999999999999995e-07, + "loss": 0.12121254950761795, + "num_tokens": 996333.0, + "reward": -15.333749771118164, + "reward_std": 47.200164794921875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -15.433748245239258, + "rewards/supergames_reward/std": 47.200164794921875, + "step": 117, + "step_time": 8.546233230998041 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.0, + "completions/max_terminated_length": 323.0, + "completions/mean_length": 137.625, + "completions/mean_terminated_length": 137.625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4177803695201874, + "epoch": 0.59, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8712472915649414, + "learning_rate": 4.1499999999999994e-07, + "loss": 0.14192955195903778, + "num_tokens": 1002010.0, + "reward": -30.700000762939453, + "reward_std": 17.224035263061523, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -30.799999237060547, + "rewards/supergames_reward/std": 17.224035263061523, + "step": 118, + "step_time": 11.878434990998358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 453.0, + "completions/max_terminated_length": 453.0, + "completions/mean_length": 238.125, + "completions/mean_terminated_length": 238.125, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "entropy": 0.5640217065811157, + "epoch": 0.595, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.064737319946289, + "learning_rate": 4.0999999999999994e-07, + "loss": -0.023621462285518646, + "num_tokens": 1012259.0, + "reward": 43.683746337890625, + "reward_std": 70.4210205078125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 43.5837516784668, + "rewards/supergames_reward/std": 70.4210205078125, + "step": 119, + "step_time": 16.829514264973113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 193.75, + "completions/mean_terminated_length": 193.75, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "entropy": 0.5795091390609741, + "epoch": 0.6, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.423007965087891, + "learning_rate": 4.05e-07, + "loss": -0.0666111558675766, + "num_tokens": 1020241.0, + "reward": -85.86000061035156, + "reward_std": 27.4442081451416, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -85.90999603271484, + "rewards/supergames_reward/std": 27.41469955444336, + "step": 120, + "step_time": 8.95496519000153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 231.0, + "completions/mean_terminated_length": 231.0, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "entropy": 0.3767699897289276, + "epoch": 0.605, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7434473037719727, + "learning_rate": 4e-07, + "loss": -0.1165132150053978, + "num_tokens": 1031465.0, + "reward": 61.45624923706055, + "reward_std": 72.21080780029297, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 61.35625076293945, + "rewards/supergames_reward/std": 72.2108154296875, + "step": 121, + "step_time": 12.588820598000893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 297.0, + "completions/max_terminated_length": 297.0, + "completions/mean_length": 223.375, + "completions/mean_terminated_length": 223.375, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "entropy": 0.4537277817726135, + "epoch": 0.61, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4382436275482178, + "learning_rate": 3.95e-07, + "loss": -0.1038316935300827, + "num_tokens": 1042548.0, + "reward": 5.658749580383301, + "reward_std": 83.77420043945312, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 5.558750152587891, + "rewards/supergames_reward/std": 83.77420043945312, + "step": 122, + "step_time": 11.39525846898323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 456.0, + "completions/max_terminated_length": 456.0, + "completions/mean_length": 314.25, + "completions/mean_terminated_length": 314.25, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4113953411579132, + "epoch": 0.615, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.449777841567993, + "learning_rate": 3.8999999999999997e-07, + "loss": -0.1303664743900299, + "num_tokens": 1054422.0, + "reward": 35.98249816894531, + "reward_std": 88.6131362915039, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 35.88249969482422, + "rewards/supergames_reward/std": 88.6131362915039, + "step": 123, + "step_time": 17.001118954998674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 242.0, + "completions/max_terminated_length": 242.0, + "completions/mean_length": 172.5, + "completions/mean_terminated_length": 172.5, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.4670729339122772, + "epoch": 0.62, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.951946973800659, + "learning_rate": 3.8499999999999997e-07, + "loss": 0.05888795852661133, + "num_tokens": 1062210.0, + "reward": 43.368751525878906, + "reward_std": 40.30733871459961, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 43.26874542236328, + "rewards/supergames_reward/std": 40.307342529296875, + "step": 124, + "step_time": 9.142582255997695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/max_terminated_length": 148.0, + "completions/mean_length": 106.625, + "completions/mean_terminated_length": 106.625, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "entropy": 0.30219921469688416, + "epoch": 0.625, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.391322135925293, + "learning_rate": 3.7999999999999996e-07, + "loss": 0.14367049932479858, + "num_tokens": 1067599.0, + "reward": 42.63374710083008, + "reward_std": 58.20897674560547, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 42.533748626708984, + "rewards/supergames_reward/std": 58.208984375, + "step": 125, + "step_time": 5.636063549987739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.0, + "completions/max_terminated_length": 243.0, + "completions/mean_length": 179.25, + "completions/mean_terminated_length": 179.25, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "entropy": 0.4834887683391571, + "epoch": 0.63, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9391286373138428, + "learning_rate": 3.75e-07, + "loss": -0.06614185124635696, + "num_tokens": 1075457.0, + "reward": 8.302498817443848, + "reward_std": 75.10364532470703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 8.202500343322754, + "rewards/supergames_reward/std": 75.10364532470703, + "step": 126, + "step_time": 9.130600645992672 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 196.0, + "completions/max_terminated_length": 196.0, + "completions/mean_length": 157.0, + "completions/mean_terminated_length": 157.0, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.4801807701587677, + "epoch": 0.635, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.425615310668945, + "learning_rate": 3.7e-07, + "loss": -0.08830522745847702, + "num_tokens": 1083129.0, + "reward": -10.563751220703125, + "reward_std": 64.00443267822266, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -10.663749694824219, + "rewards/supergames_reward/std": 64.00444030761719, + "step": 127, + "step_time": 7.4972667430120055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 412.0, + "completions/max_terminated_length": 412.0, + "completions/mean_length": 239.25, + "completions/mean_terminated_length": 239.25, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.4667063355445862, + "epoch": 0.64, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.199113130569458, + "learning_rate": 3.65e-07, + "loss": -0.043856702744960785, + "num_tokens": 1094355.0, + "reward": -55.89250183105469, + "reward_std": 53.05980682373047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -55.99250030517578, + "rewards/supergames_reward/std": 53.05980682373047, + "step": 128, + "step_time": 15.429580625001108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 244.75, + "completions/mean_terminated_length": 244.75, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "entropy": 0.57573401927948, + "epoch": 0.645, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6800079345703125, + "learning_rate": 3.6e-07, + "loss": 0.09521033614873886, + "num_tokens": 1105633.0, + "reward": -23.35250473022461, + "reward_std": 102.31401062011719, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -23.452499389648438, + "rewards/supergames_reward/std": 102.31402587890625, + "step": 129, + "step_time": 13.920327747007832 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/max_terminated_length": 258.0, + "completions/mean_length": 188.25, + "completions/mean_terminated_length": 188.25, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "entropy": 0.5341438055038452, + "epoch": 0.65, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.97955060005188, + "learning_rate": 3.55e-07, + "loss": -0.09512650966644287, + "num_tokens": 1113571.0, + "reward": 29.181249618530273, + "reward_std": 80.10585021972656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 29.081249237060547, + "rewards/supergames_reward/std": 80.10585021972656, + "step": 130, + "step_time": 9.69286120700417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 257.0, + "completions/max_terminated_length": 257.0, + "completions/mean_length": 180.75, + "completions/mean_terminated_length": 180.75, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "entropy": 0.46591487526893616, + "epoch": 0.655, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3518130779266357, + "learning_rate": 3.5e-07, + "loss": 0.04714229330420494, + "num_tokens": 1121457.0, + "reward": -8.287501335144043, + "reward_std": 76.35875701904297, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.387499809265137, + "rewards/supergames_reward/std": 76.35875701904297, + "step": 131, + "step_time": 9.636637121002423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 147.75, + "completions/mean_terminated_length": 147.75, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4407735764980316, + "epoch": 0.66, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.498741626739502, + "learning_rate": 3.45e-07, + "loss": -0.03996007889509201, + "num_tokens": 1129095.0, + "reward": 0.1999988555908203, + "reward_std": 84.63554382324219, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 0.125, + "rewards/supergames_reward/std": 84.6017074584961, + "step": 132, + "step_time": 7.510833573003765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 358.0, + "completions/max_terminated_length": 358.0, + "completions/mean_length": 213.875, + "completions/mean_terminated_length": 213.875, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.5186149477958679, + "epoch": 0.665, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3987107276916504, + "learning_rate": 3.4000000000000003e-07, + "loss": 0.1863594949245453, + "num_tokens": 1139110.0, + "reward": -21.328752517700195, + "reward_std": 88.9102783203125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -21.428749084472656, + "rewards/supergames_reward/std": 88.9102783203125, + "step": 133, + "step_time": 13.393839450000087 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/max_terminated_length": 230.0, + "completions/mean_length": 169.875, + "completions/mean_terminated_length": 169.875, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.49445784091949463, + "epoch": 0.67, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.479673862457275, + "learning_rate": 3.35e-07, + "loss": 0.03618276119232178, + "num_tokens": 1146885.0, + "reward": -14.042501449584961, + "reward_std": 83.22671508789062, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -14.142499923706055, + "rewards/supergames_reward/std": 83.22671508789062, + "step": 134, + "step_time": 8.703399487014394 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/max_terminated_length": 168.0, + "completions/mean_length": 124.75, + "completions/mean_terminated_length": 124.75, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "entropy": 0.2997814118862152, + "epoch": 0.675, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.440976619720459, + "learning_rate": 3.3e-07, + "loss": -0.029299044981598854, + "num_tokens": 1152427.0, + "reward": 45.274993896484375, + "reward_std": 44.401519775390625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 45.17499923706055, + "rewards/supergames_reward/std": 44.40152359008789, + "step": 135, + "step_time": 6.334954546997324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 161.0, + "completions/max_terminated_length": 161.0, + "completions/mean_length": 123.625, + "completions/mean_terminated_length": 123.625, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3462243974208832, + "epoch": 0.68, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.1878814697265625, + "learning_rate": 3.25e-07, + "loss": -0.10772529244422913, + "num_tokens": 1157952.0, + "reward": 44.186248779296875, + "reward_std": 51.78197479248047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 44.08625030517578, + "rewards/supergames_reward/std": 51.7819709777832, + "step": 136, + "step_time": 6.067531232984038 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 155.0, + "completions/max_terminated_length": 155.0, + "completions/mean_length": 106.75, + "completions/mean_terminated_length": 106.75, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.3662136197090149, + "epoch": 0.685, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.508971214294434, + "learning_rate": 3.2e-07, + "loss": -0.11172202974557877, + "num_tokens": 1163334.0, + "reward": 53.0574951171875, + "reward_std": 58.36653518676758, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 52.957496643066406, + "rewards/supergames_reward/std": 58.36653518676758, + "step": 137, + "step_time": 5.8906258280039765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 175.0, + "completions/max_terminated_length": 175.0, + "completions/mean_length": 125.375, + "completions/mean_terminated_length": 125.375, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.3352622091770172, + "epoch": 0.69, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.1937079429626465, + "learning_rate": 3.15e-07, + "loss": 0.08307601511478424, + "num_tokens": 1168929.0, + "reward": 11.796250343322754, + "reward_std": 53.48870849609375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 11.696250915527344, + "rewards/supergames_reward/std": 53.48870849609375, + "step": 138, + "step_time": 6.595962718012743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 193.0, + "completions/mean_terminated_length": 193.0, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.5486535429954529, + "epoch": 0.695, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.861871242523193, + "learning_rate": 3.1e-07, + "loss": 0.0321456640958786, + "num_tokens": 1176937.0, + "reward": 50.463748931884766, + "reward_std": 70.37553405761719, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.38875198364258, + "rewards/supergames_reward/std": 70.31440734863281, + "step": 139, + "step_time": 9.544256388006033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 189.5, + "completions/mean_terminated_length": 189.5, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.42230603098869324, + "epoch": 0.7, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.830677032470703, + "learning_rate": 3.05e-07, + "loss": -0.07222295552492142, + "num_tokens": 1186789.0, + "reward": 33.44999694824219, + "reward_std": 67.1298599243164, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 33.349998474121094, + "rewards/supergames_reward/std": 67.1298599243164, + "step": 140, + "step_time": 10.002322309010196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 282.0, + "completions/max_terminated_length": 282.0, + "completions/mean_length": 197.375, + "completions/mean_terminated_length": 197.375, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.33588480949401855, + "epoch": 0.705, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.028353452682495, + "learning_rate": 3e-07, + "loss": -0.07543312013149261, + "num_tokens": 1196672.0, + "reward": 17.10249900817871, + "reward_std": 73.94812774658203, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 17.002500534057617, + "rewards/supergames_reward/std": 73.94813537597656, + "step": 141, + "step_time": 10.684560916008195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/max_terminated_length": 235.0, + "completions/mean_length": 209.25, + "completions/mean_terminated_length": 209.25, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "entropy": 0.3622831702232361, + "epoch": 0.71, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.047222137451172, + "learning_rate": 2.95e-07, + "loss": -0.04569581151008606, + "num_tokens": 1206658.0, + "reward": 49.57624816894531, + "reward_std": 78.21060180664062, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 49.476253509521484, + "rewards/supergames_reward/std": 78.21060943603516, + "step": 142, + "step_time": 8.97679864402744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 161.0, + "completions/max_terminated_length": 161.0, + "completions/mean_length": 116.25, + "completions/mean_terminated_length": 116.25, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.43277010321617126, + "epoch": 0.715, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.345203876495361, + "learning_rate": 2.9e-07, + "loss": -0.16899466514587402, + "num_tokens": 1212140.0, + "reward": 27.42875099182129, + "reward_std": 48.59138870239258, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.328750610351562, + "rewards/supergames_reward/std": 48.591392517089844, + "step": 143, + "step_time": 6.1606669370085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 205.0, + "completions/max_terminated_length": 205.0, + "completions/mean_length": 173.75, + "completions/mean_terminated_length": 173.75, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "entropy": 0.43806371092796326, + "epoch": 0.72, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.858840227127075, + "learning_rate": 2.8499999999999997e-07, + "loss": -0.06502802670001984, + "num_tokens": 1219954.0, + "reward": 38.69249725341797, + "reward_std": 68.78203582763672, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 38.592498779296875, + "rewards/supergames_reward/std": 68.78202819824219, + "step": 144, + "step_time": 7.88639413099736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 169.0, + "completions/max_terminated_length": 169.0, + "completions/mean_length": 110.625, + "completions/mean_terminated_length": 110.625, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.3383479118347168, + "epoch": 0.725, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.225733757019043, + "learning_rate": 2.8e-07, + "loss": -0.042058318853378296, + "num_tokens": 1225399.0, + "reward": 64.85874938964844, + "reward_std": 25.170387268066406, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 64.75875091552734, + "rewards/supergames_reward/std": 25.170391082763672, + "step": 145, + "step_time": 6.432729878026294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 147.0, + "completions/max_terminated_length": 147.0, + "completions/mean_length": 129.5, + "completions/mean_terminated_length": 129.5, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3088987171649933, + "epoch": 0.73, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.75e-07, + "loss": 0.0, + "num_tokens": 1231043.0, + "reward": 100.0999984741211, + "reward_std": 0.0, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 100.0, + "rewards/supergames_reward/std": 0.0, + "step": 146, + "step_time": 5.634907905012369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 160.0, + "completions/max_terminated_length": 160.0, + "completions/mean_length": 120.5, + "completions/mean_terminated_length": 120.5, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.37292322516441345, + "epoch": 0.735, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.174577713012695, + "learning_rate": 2.7e-07, + "loss": 0.11778222769498825, + "num_tokens": 1236607.0, + "reward": 34.82374572753906, + "reward_std": 58.513397216796875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 34.72374725341797, + "rewards/supergames_reward/std": 58.513397216796875, + "step": 147, + "step_time": 6.078690663998714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 172.0, + "completions/max_terminated_length": 172.0, + "completions/mean_length": 125.625, + "completions/mean_terminated_length": 125.625, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.308654248714447, + "epoch": 0.74, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8483362197875977, + "learning_rate": 2.65e-07, + "loss": -0.09181805700063705, + "num_tokens": 1242140.0, + "reward": 85.06375122070312, + "reward_std": 42.52893829345703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 84.9637451171875, + "rewards/supergames_reward/std": 42.52893829345703, + "step": 148, + "step_time": 6.501119122985983 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 275.0, + "completions/max_terminated_length": 275.0, + "completions/mean_length": 211.125, + "completions/mean_terminated_length": 211.125, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "entropy": 0.5405827164649963, + "epoch": 0.745, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.774259567260742, + "learning_rate": 2.6e-07, + "loss": -0.03056861087679863, + "num_tokens": 1250205.0, + "reward": 84.63125610351562, + "reward_std": 34.3358154296875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 84.53125, + "rewards/supergames_reward/std": 34.3358154296875, + "step": 149, + "step_time": 10.352729418984381 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 257.0, + "completions/max_terminated_length": 257.0, + "completions/mean_length": 218.5, + "completions/mean_terminated_length": 218.5, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "entropy": 0.3852894604206085, + "epoch": 0.75, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6103010177612305, + "learning_rate": 2.55e-07, + "loss": 0.034789010882377625, + "num_tokens": 1260249.0, + "reward": 81.81375122070312, + "reward_std": 51.72132873535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 81.7137451171875, + "rewards/supergames_reward/std": 51.72132873535156, + "step": 150, + "step_time": 9.937249842012534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 351.0, + "completions/max_terminated_length": 351.0, + "completions/mean_length": 232.625, + "completions/mean_terminated_length": 232.625, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.4323120415210724, + "epoch": 0.755, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.640308380126953, + "learning_rate": 2.5e-07, + "loss": -0.11181996762752533, + "num_tokens": 1271430.0, + "reward": 56.33625030517578, + "reward_std": 78.97452545166016, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 56.23625183105469, + "rewards/supergames_reward/std": 78.97452545166016, + "step": 151, + "step_time": 13.323547195002902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 359.0, + "completions/max_terminated_length": 359.0, + "completions/mean_length": 247.5, + "completions/mean_terminated_length": 247.5, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "entropy": 0.4038960039615631, + "epoch": 0.76, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.5704824924468994, + "learning_rate": 2.45e-07, + "loss": 0.060525521636009216, + "num_tokens": 1282722.0, + "reward": -68.53875732421875, + "reward_std": 50.590599060058594, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.63874816894531, + "rewards/supergames_reward/std": 50.590599060058594, + "step": 152, + "step_time": 13.593127576023107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 383.0, + "completions/max_terminated_length": 383.0, + "completions/mean_length": 246.375, + "completions/mean_terminated_length": 246.375, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "entropy": 0.4110933542251587, + "epoch": 0.765, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.233025312423706, + "learning_rate": 2.4e-07, + "loss": 0.06547188758850098, + "num_tokens": 1294061.0, + "reward": 95.91499328613281, + "reward_std": 11.836968421936035, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 95.81500244140625, + "rewards/supergames_reward/std": 11.836968421936035, + "step": 153, + "step_time": 14.499402174988063 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 338.0, + "completions/max_terminated_length": 338.0, + "completions/mean_length": 233.125, + "completions/mean_terminated_length": 233.125, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.34559860825538635, + "epoch": 0.77, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.6887102127075195, + "learning_rate": 2.3499999999999997e-07, + "loss": 0.15905122458934784, + "num_tokens": 1304262.0, + "reward": 88.92375183105469, + "reward_std": 31.611207962036133, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 88.82374572753906, + "rewards/supergames_reward/std": 31.6112117767334, + "step": 154, + "step_time": 12.704337224975461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 208.0, + "completions/max_terminated_length": 208.0, + "completions/mean_length": 178.5, + "completions/mean_terminated_length": 178.5, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "entropy": 0.4646885097026825, + "epoch": 0.775, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.161937236785889, + "learning_rate": 2.3e-07, + "loss": -0.04354584217071533, + "num_tokens": 1312098.0, + "reward": 27.80875015258789, + "reward_std": 60.81097412109375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.708751678466797, + "rewards/supergames_reward/std": 60.81097412109375, + "step": 155, + "step_time": 7.911955468007363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 391.0, + "completions/max_terminated_length": 391.0, + "completions/mean_length": 235.375, + "completions/mean_terminated_length": 235.375, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "entropy": 0.40985700488090515, + "epoch": 0.78, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.12153959274292, + "learning_rate": 2.25e-07, + "loss": -0.024320494383573532, + "num_tokens": 1322301.0, + "reward": 64.14624786376953, + "reward_std": 49.90530014038086, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 64.04624938964844, + "rewards/supergames_reward/std": 49.90530014038086, + "step": 156, + "step_time": 14.631677505996777 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/max_terminated_length": 330.0, + "completions/mean_length": 220.625, + "completions/mean_terminated_length": 220.625, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3543764054775238, + "epoch": 0.785, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0111091136932373, + "learning_rate": 2.1999999999999998e-07, + "loss": -0.13996481895446777, + "num_tokens": 1333386.0, + "reward": 3.6812496185302734, + "reward_std": 103.5096664428711, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 3.606250762939453, + "rewards/supergames_reward/std": 103.48104858398438, + "step": 157, + "step_time": 12.613216657977318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.0, + "completions/max_terminated_length": 225.0, + "completions/mean_length": 193.125, + "completions/mean_terminated_length": 193.125, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "entropy": 0.5682947635650635, + "epoch": 0.79, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.67927885055542, + "learning_rate": 2.1499999999999998e-07, + "loss": 0.0535324327647686, + "num_tokens": 1341355.0, + "reward": -19.78999900817871, + "reward_std": 69.72611236572266, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -19.889997482299805, + "rewards/supergames_reward/std": 69.72611236572266, + "step": 158, + "step_time": 8.585773958999198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.0, + "completions/max_terminated_length": 316.0, + "completions/mean_length": 237.625, + "completions/mean_terminated_length": 237.625, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "entropy": 0.3637359142303467, + "epoch": 0.795, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.033409357070923, + "learning_rate": 2.0999999999999997e-07, + "loss": 0.018884683027863503, + "num_tokens": 1352632.0, + "reward": 52.063751220703125, + "reward_std": 65.35551452636719, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 51.96375274658203, + "rewards/supergames_reward/std": 65.35551452636719, + "step": 159, + "step_time": 12.306456914986484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.0, + "completions/max_terminated_length": 225.0, + "completions/mean_length": 184.0, + "completions/mean_terminated_length": 184.0, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.47005388140678406, + "epoch": 0.8, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.585808515548706, + "learning_rate": 2.0499999999999997e-07, + "loss": -0.021751077845692635, + "num_tokens": 1360528.0, + "reward": 33.73124694824219, + "reward_std": 78.67134857177734, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 33.631248474121094, + "rewards/supergames_reward/std": 78.67134857177734, + "step": 160, + "step_time": 8.517202848976012 + } + ], + "logging_steps": 1, + "max_steps": 200, + "num_input_tokens_seen": 1360528, + "num_train_epochs": 1, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-160/training_args.bin b/checkpoint-160/training_args.bin new file mode 100644 index 0000000..730dec0 --- /dev/null +++ b/checkpoint-160/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691fe5be2d7d1d56160dae63866d136877a72efb30e15ff7b3249192f998a788 +size 7185 diff --git a/checkpoint-180/chat_template.jinja b/checkpoint-180/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/checkpoint-180/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-180/config.json b/checkpoint-180/config.json new file mode 100644 index 0000000..f57b09b --- /dev/null +++ b/checkpoint-180/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-180/generation_config.json b/checkpoint-180/generation_config.json new file mode 100644 index 0000000..1b2bba9 --- /dev/null +++ b/checkpoint-180/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/checkpoint-180/model.safetensors b/checkpoint-180/model.safetensors new file mode 100644 index 0000000..19d6536 --- /dev/null +++ b/checkpoint-180/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa72ec17dea3b938582c9d0c66180015e24ba492541ada4b1ff62122b46a81f6 +size 6174895536 diff --git a/checkpoint-180/optimizer.pt b/checkpoint-180/optimizer.pt new file mode 100644 index 0000000..60dda89 --- /dev/null +++ b/checkpoint-180/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32366444f69b40b1c07efaa61259dd4d894f310967b0b29be3249b78176ab9cb +size 12350013801 diff --git a/checkpoint-180/rng_state.pth b/checkpoint-180/rng_state.pth new file mode 100644 index 0000000..cdda4a8 --- /dev/null +++ b/checkpoint-180/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09b3b8e2168fb1e067c78a7650e6b875052ff8cd483bbf8e2cf27a7d0b6ed6dc +size 14645 diff --git a/checkpoint-180/scheduler.pt b/checkpoint-180/scheduler.pt new file mode 100644 index 0000000..d8c6f76 --- /dev/null +++ b/checkpoint-180/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3906df9ece476453bfcc305277c81bbdd9072fad7ee821bcdaff0ae528f6fa6b +size 1465 diff --git a/checkpoint-180/tokenizer.json b/checkpoint-180/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-180/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-180/tokenizer_config.json b/checkpoint-180/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-180/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-180/trainer_state.json b/checkpoint-180/trainer_state.json new file mode 100644 index 0000000..b5fb2c5 --- /dev/null +++ b/checkpoint-180/trainer_state.json @@ -0,0 +1,5254 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9, + "eval_steps": 500, + "global_step": 180, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 191.0, + "completions/mean_terminated_length": 191.0, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "entropy": 0.5566893219947815, + "epoch": 0.005, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.370361328125, + "learning_rate": 1e-06, + "loss": -0.24658073484897614, + "num_tokens": 10832.0, + "reward": -3.3000030517578125, + "reward_std": 85.62333679199219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.4000015258789062, + "rewards/supergames_reward/std": 85.62333679199219, + "step": 1, + "step_time": 22.779711072013015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/max_terminated_length": 216.0, + "completions/mean_length": 131.5, + "completions/mean_terminated_length": 131.5, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.6395841240882874, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.09774398803711, + "learning_rate": 9.95e-07, + "loss": 0.20567649602890015, + "num_tokens": 16404.0, + "reward": -12.422499656677246, + "reward_std": 7.134707450866699, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.522500038146973, + "rewards/supergames_reward/std": 7.134707927703857, + "step": 2, + "step_time": 9.960156448010821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/max_terminated_length": 287.0, + "completions/mean_length": 156.625, + "completions/mean_terminated_length": 156.625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "entropy": 0.562222421169281, + "epoch": 0.015, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9053616523742676, + "learning_rate": 9.9e-07, + "loss": 0.09602774679660797, + "num_tokens": 26953.0, + "reward": -74.94999694824219, + "reward_std": 70.73094177246094, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -75.0, + "rewards/supergames_reward/std": 70.71067810058594, + "step": 3, + "step_time": 13.686320498993155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 154.125, + "completions/mean_terminated_length": 154.125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7323317527770996, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.933310031890869, + "learning_rate": 9.849999999999999e-07, + "loss": 0.28249427676200867, + "num_tokens": 36514.0, + "reward": -40.7599983215332, + "reward_std": 81.73140716552734, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -40.80999755859375, + "rewards/supergames_reward/std": 81.68995666503906, + "step": 4, + "step_time": 12.805880262021674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 205.25, + "completions/mean_terminated_length": 205.25, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.6167430281639099, + "epoch": 0.025, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.344135046005249, + "learning_rate": 9.8e-07, + "loss": 0.1433994024991989, + "num_tokens": 47476.0, + "reward": -97.6612548828125, + "reward_std": 6.3321428298950195, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.76124572753906, + "rewards/supergames_reward/std": 6.3321428298950195, + "step": 5, + "step_time": 18.789364666008623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/max_terminated_length": 276.0, + "completions/mean_length": 167.25, + "completions/mean_terminated_length": 167.25, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.7583790421485901, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.172371864318848, + "learning_rate": 9.75e-07, + "loss": -0.12127404659986496, + "num_tokens": 55246.0, + "reward": -58.687503814697266, + "reward_std": 58.64059829711914, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -58.76250457763672, + "rewards/supergames_reward/std": 58.620460510253906, + "step": 6, + "step_time": 12.948570273991209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 133.875, + "completions/mean_terminated_length": 133.875, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.7447654604911804, + "epoch": 0.035, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.428595066070557, + "learning_rate": 9.7e-07, + "loss": 0.10135584324598312, + "num_tokens": 60837.0, + "reward": -32.45000076293945, + "reward_std": 41.754486083984375, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -32.5, + "rewards/supergames_reward/std": 41.66190719604492, + "step": 7, + "step_time": 12.058315072004916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 414.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 209.125, + "completions/mean_terminated_length": 209.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.3860666751861572, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.806255578994751, + "learning_rate": 9.649999999999999e-07, + "loss": -0.33238139748573303, + "num_tokens": 70838.0, + "reward": -80.50375366210938, + "reward_std": 38.82380294799805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -80.60375213623047, + "rewards/supergames_reward/std": 38.82379913330078, + "step": 8, + "step_time": 19.323370319994865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 247.125, + "completions/mean_terminated_length": 158.83334350585938, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.5816237330436707, + "epoch": 0.045, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3209943771362305, + "learning_rate": 9.6e-07, + "loss": 0.36752766370773315, + "num_tokens": 82135.0, + "reward": -27.688751220703125, + "reward_std": 82.8358154296875, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -27.738750457763672, + "rewards/supergames_reward/std": 82.78590393066406, + "step": 9, + "step_time": 21.948575104994234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/max_terminated_length": 240.0, + "completions/mean_length": 153.0, + "completions/mean_terminated_length": 153.0, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6301521062850952, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.125359058380127, + "learning_rate": 9.55e-07, + "loss": 0.06129350885748863, + "num_tokens": 89807.0, + "reward": -78.39624786376953, + "reward_std": 40.17042922973633, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -78.44625091552734, + "rewards/supergames_reward/std": 40.1396484375, + "step": 10, + "step_time": 9.098202993016457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/max_terminated_length": 133.0, + "completions/mean_length": 112.0, + "completions/mean_terminated_length": 112.0, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.5521990656852722, + "epoch": 0.055, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.616060256958008, + "learning_rate": 9.499999999999999e-07, + "loss": -0.006313305348157883, + "num_tokens": 95231.0, + "reward": -12.161249160766602, + "reward_std": 6.39578104019165, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.261249542236328, + "rewards/supergames_reward/std": 6.39578104019165, + "step": 11, + "step_time": 5.136311663984088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 214.875, + "completions/mean_terminated_length": 214.875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.5505905151367188, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.289583206176758, + "learning_rate": 9.45e-07, + "loss": -0.29802003502845764, + "num_tokens": 106238.0, + "reward": -91.66999816894531, + "reward_std": 23.2779541015625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -91.77000427246094, + "rewards/supergames_reward/std": 23.277956008911133, + "step": 12, + "step_time": 14.945365622988902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 167.375, + "completions/mean_terminated_length": 167.375, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "entropy": 0.540415346622467, + "epoch": 0.065, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.858291149139404, + "learning_rate": 9.399999999999999e-07, + "loss": 0.24004128575325012, + "num_tokens": 116937.0, + "reward": -20.38249969482422, + "reward_std": 87.55204010009766, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.457500457763672, + "rewards/supergames_reward/std": 87.52605438232422, + "step": 13, + "step_time": 10.482285185018554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "entropy": 0.7237679958343506, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.083980560302734, + "learning_rate": 9.35e-07, + "loss": -0.19025824964046478, + "num_tokens": 124640.0, + "reward": -51.397499084472656, + "reward_std": 54.486454010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -51.49749755859375, + "rewards/supergames_reward/std": 54.486454010009766, + "step": 14, + "step_time": 9.597231683001155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 141.375, + "completions/mean_terminated_length": 141.375, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.754334568977356, + "epoch": 0.075, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.720428943634033, + "learning_rate": 9.3e-07, + "loss": -0.16830675303936005, + "num_tokens": 132219.0, + "reward": -56.226253509521484, + "reward_std": 60.115657806396484, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -56.32625198364258, + "rewards/supergames_reward/std": 60.11566162109375, + "step": 15, + "step_time": 8.998362872982398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.0, + "completions/max_terminated_length": 209.0, + "completions/mean_length": 136.375, + "completions/mean_terminated_length": 136.375, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.6280568838119507, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.9681806564331055, + "learning_rate": 9.25e-07, + "loss": -0.01910916529595852, + "num_tokens": 139726.0, + "reward": -68.24000549316406, + "reward_std": 52.238914489746094, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.33999633789062, + "rewards/supergames_reward/std": 52.238914489746094, + "step": 16, + "step_time": 8.034480294008972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 211.875, + "completions/mean_terminated_length": 211.875, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.899176836013794, + "epoch": 0.085, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.691746711730957, + "learning_rate": 9.2e-07, + "loss": 0.25513291358947754, + "num_tokens": 147869.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 17, + "step_time": 13.655792869016295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 387.0, + "completions/max_terminated_length": 387.0, + "completions/mean_length": 238.0, + "completions/mean_terminated_length": 238.0, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.5596873760223389, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6711347103118896, + "learning_rate": 9.15e-07, + "loss": -0.2628335654735565, + "num_tokens": 159101.0, + "reward": -24.900001525878906, + "reward_std": 103.50983428955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.0, + "rewards/supergames_reward/std": 103.50983428955078, + "step": 18, + "step_time": 14.784329420013819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 277.0, + "completions/mean_length": 219.75, + "completions/mean_terminated_length": 178.00001525878906, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.9245517253875732, + "epoch": 0.095, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55502986907959, + "learning_rate": 9.1e-07, + "loss": 0.13060179352760315, + "num_tokens": 167299.0, + "reward": -86.89125061035156, + "reward_std": 36.95603561401367, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -86.94125366210938, + "rewards/supergames_reward/std": 36.93572235107422, + "step": 19, + "step_time": 18.832684700988466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/max_terminated_length": 267.0, + "completions/mean_length": 177.25, + "completions/mean_terminated_length": 177.25, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.7331254482269287, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.538497447967529, + "learning_rate": 9.05e-07, + "loss": -0.16418980062007904, + "num_tokens": 175141.0, + "reward": -68.52874755859375, + "reward_std": 58.48719787597656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.62875366210938, + "rewards/supergames_reward/std": 58.48720169067383, + "step": 20, + "step_time": 10.090975169994636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 190.625, + "completions/mean_terminated_length": 190.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.6659425497055054, + "epoch": 0.105, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.485373497009277, + "learning_rate": 9e-07, + "loss": -0.10106432437896729, + "num_tokens": 183098.0, + "reward": -71.48875427246094, + "reward_std": 55.33565139770508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -71.5887451171875, + "rewards/supergames_reward/std": 55.33565139770508, + "step": 21, + "step_time": 11.255001295008697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 179.0, + "completions/mean_terminated_length": 179.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.49365949630737305, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.829419136047363, + "learning_rate": 8.95e-07, + "loss": 0.10587231814861298, + "num_tokens": 192826.0, + "reward": -75.62000274658203, + "reward_std": 45.01959991455078, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -75.69499969482422, + "rewards/supergames_reward/std": 45.00411605834961, + "step": 22, + "step_time": 10.611246599000879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/max_terminated_length": 315.0, + "completions/mean_length": 212.625, + "completions/mean_terminated_length": 212.625, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.43159469962120056, + "epoch": 0.115, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.541125774383545, + "learning_rate": 8.9e-07, + "loss": -0.1681259125471115, + "num_tokens": 202815.0, + "reward": 23.67624855041504, + "reward_std": 67.74340057373047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 23.576250076293945, + "rewards/supergames_reward/std": 67.743408203125, + "step": 23, + "step_time": 11.731771531980485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/max_terminated_length": 168.0, + "completions/mean_length": 101.625, + "completions/mean_terminated_length": 101.625, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.3972298502922058, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.8285417556762695, + "learning_rate": 8.85e-07, + "loss": -0.011585958302021027, + "num_tokens": 208188.0, + "reward": 58.849998474121094, + "reward_std": 56.9304084777832, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 58.75, + "rewards/supergames_reward/std": 56.9304084777832, + "step": 24, + "step_time": 6.187504950998118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 303.0, + "completions/max_terminated_length": 303.0, + "completions/mean_length": 165.375, + "completions/mean_terminated_length": 165.375, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "entropy": 0.6339899301528931, + "epoch": 0.125, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.823799133300781, + "learning_rate": 8.799999999999999e-07, + "loss": 0.05159185826778412, + "num_tokens": 215935.0, + "reward": -23.521251678466797, + "reward_std": 74.23067474365234, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -23.621248245239258, + "rewards/supergames_reward/std": 74.23067474365234, + "step": 25, + "step_time": 11.059416300005978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 349.0, + "completions/max_terminated_length": 349.0, + "completions/mean_length": 204.0, + "completions/mean_terminated_length": 204.0, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.6462895274162292, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.969292640686035, + "learning_rate": 8.75e-07, + "loss": 0.0115677984431386, + "num_tokens": 223999.0, + "reward": -74.8175048828125, + "reward_std": 40.94221115112305, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -74.89250183105469, + "rewards/supergames_reward/std": 40.924625396728516, + "step": 26, + "step_time": 12.666237785975682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 166.5, + "completions/mean_terminated_length": 166.5, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4756850600242615, + "epoch": 0.135, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915069103240967, + "learning_rate": 8.699999999999999e-07, + "loss": -0.13321346044540405, + "num_tokens": 233643.0, + "reward": -52.125, + "reward_std": 19.405282974243164, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -52.20000076293945, + "rewards/supergames_reward/std": 19.33464813232422, + "step": 27, + "step_time": 11.172539752995363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 137.125, + "completions/mean_terminated_length": 137.125, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.5834778547286987, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.146666526794434, + "learning_rate": 8.65e-07, + "loss": 0.060454584658145905, + "num_tokens": 241204.0, + "reward": -68.99250793457031, + "reward_std": 51.010581970214844, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -69.09249877929688, + "rewards/supergames_reward/std": 51.010581970214844, + "step": 28, + "step_time": 7.025046669004951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 182.375, + "completions/mean_terminated_length": 182.375, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.6368551254272461, + "epoch": 0.145, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.15535306930542, + "learning_rate": 8.599999999999999e-07, + "loss": -0.3323673903942108, + "num_tokens": 249087.0, + "reward": -44.17000198364258, + "reward_std": 80.5373306274414, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -44.27000045776367, + "rewards/supergames_reward/std": 80.5373306274414, + "step": 29, + "step_time": 12.079259724996518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 127.75, + "completions/mean_terminated_length": 127.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4491196870803833, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.896186828613281, + "learning_rate": 8.55e-07, + "loss": -0.12887656688690186, + "num_tokens": 259445.0, + "reward": 57.446250915527344, + "reward_std": 38.78837585449219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 57.34625244140625, + "rewards/supergames_reward/std": 38.78837966918945, + "step": 30, + "step_time": 7.483972909016302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/max_terminated_length": 166.0, + "completions/mean_length": 112.875, + "completions/mean_terminated_length": 112.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4858454465866089, + "epoch": 0.155, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.742906093597412, + "learning_rate": 8.499999999999999e-07, + "loss": 0.06930096447467804, + "num_tokens": 264924.0, + "reward": -21.174999237060547, + "reward_std": 31.89051628112793, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -21.25, + "rewards/supergames_reward/std": 31.819807052612305, + "step": 31, + "step_time": 6.1866529019898735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/max_terminated_length": 148.0, + "completions/mean_length": 118.125, + "completions/mean_terminated_length": 118.125, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.45428749918937683, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.663414478302002, + "learning_rate": 8.45e-07, + "loss": 0.05735419690608978, + "num_tokens": 270381.0, + "reward": 0.4687504768371582, + "reward_std": 38.66371154785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 0.36875057220458984, + "rewards/supergames_reward/std": 38.6637077331543, + "step": 32, + "step_time": 5.577042757999152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 166.875, + "completions/mean_terminated_length": 166.875, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.6107293367385864, + "epoch": 0.165, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.75006628036499, + "learning_rate": 8.399999999999999e-07, + "loss": -0.14760127663612366, + "num_tokens": 278156.0, + "reward": -87.84750366210938, + "reward_std": 26.84708023071289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -87.94749450683594, + "rewards/supergames_reward/std": 26.847076416015625, + "step": 33, + "step_time": 12.552876825997373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 360.0, + "completions/mean_length": 231.0, + "completions/mean_terminated_length": 190.85714721679688, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.6089653372764587, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.781267166137695, + "learning_rate": 8.349999999999999e-07, + "loss": 0.4294504225254059, + "num_tokens": 289364.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 34, + "step_time": 18.961819477990502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 134.125, + "completions/mean_terminated_length": 134.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46158918738365173, + "epoch": 0.175, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.016353607177734, + "learning_rate": 8.299999999999999e-07, + "loss": 0.09251818805932999, + "num_tokens": 298733.0, + "reward": -35.558753967285156, + "reward_std": 15.881204605102539, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -35.65875244140625, + "rewards/supergames_reward/std": 15.881203651428223, + "step": 35, + "step_time": 8.38491243700264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/max_terminated_length": 207.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.5104788541793823, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.592893123626709, + "learning_rate": 8.249999999999999e-07, + "loss": -0.0410832017660141, + "num_tokens": 306420.0, + "reward": 5.4212493896484375, + "reward_std": 93.50404357910156, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 5.346250534057617, + "rewards/supergames_reward/std": 93.4718246459961, + "step": 36, + "step_time": 7.811868985998444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/max_terminated_length": 241.0, + "completions/mean_length": 178.0, + "completions/mean_terminated_length": 178.0, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.7527878880500793, + "epoch": 0.185, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8812546730041504, + "learning_rate": 8.199999999999999e-07, + "loss": 0.12539231777191162, + "num_tokens": 314252.0, + "reward": -89.58500671386719, + "reward_std": 29.256120681762695, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -89.66000366210938, + "rewards/supergames_reward/std": 29.24593734741211, + "step": 37, + "step_time": 8.959742914012168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 133.375, + "completions/mean_terminated_length": 133.375, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.3347901403903961, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.205414295196533, + "learning_rate": 8.149999999999999e-07, + "loss": -0.06991486996412277, + "num_tokens": 319839.0, + "reward": -14.498749732971191, + "reward_std": 13.007229804992676, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -14.598750114440918, + "rewards/supergames_reward/std": 13.007229804992676, + "step": 38, + "step_time": 7.377183554985095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/max_terminated_length": 181.0, + "completions/mean_length": 114.375, + "completions/mean_terminated_length": 114.375, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.4445965886116028, + "epoch": 0.195, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.561298847198486, + "learning_rate": 8.1e-07, + "loss": -0.06235164776444435, + "num_tokens": 325290.0, + "reward": -7.425000190734863, + "reward_std": 53.701602935791016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -7.5, + "rewards/supergames_reward/std": 53.652320861816406, + "step": 39, + "step_time": 6.753862089011818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 187.625, + "completions/mean_terminated_length": 187.625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.3950234353542328, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5988712310791016, + "learning_rate": 8.05e-07, + "loss": -0.1381440907716751, + "num_tokens": 335119.0, + "reward": -33.68375015258789, + "reward_std": 42.069435119628906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -33.783748626708984, + "rewards/supergames_reward/std": 42.069435119628906, + "step": 40, + "step_time": 13.718958162004128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 371.0, + "completions/mean_length": 251.125, + "completions/mean_terminated_length": 213.85714721679688, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.507067084312439, + "epoch": 0.205, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3898658752441406, + "learning_rate": 8e-07, + "loss": 0.0078964838758111, + "num_tokens": 346464.0, + "reward": -86.48625183105469, + "reward_std": 38.020694732666016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -86.56124877929688, + "rewards/supergames_reward/std": 38.010528564453125, + "step": 41, + "step_time": 18.90960379401804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 152.0, + "completions/max_terminated_length": 152.0, + "completions/mean_length": 116.375, + "completions/mean_terminated_length": 116.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3801707923412323, + "epoch": 0.21, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.95e-07, + "loss": 0.0, + "num_tokens": 351955.0, + "reward": -9.899999618530273, + "reward_std": 0.0, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -10.0, + "rewards/supergames_reward/std": 0.0, + "step": 42, + "step_time": 5.711630532023264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/max_terminated_length": 308.0, + "completions/mean_length": 247.375, + "completions/mean_terminated_length": 247.375, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.3185043931007385, + "epoch": 0.215, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.168762445449829, + "learning_rate": 7.9e-07, + "loss": 0.1970764398574829, + "num_tokens": 363278.0, + "reward": -53.89249801635742, + "reward_std": 63.02619934082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -53.99250030517578, + "rewards/supergames_reward/std": 63.02619934082031, + "step": 43, + "step_time": 11.661934906995157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/max_terminated_length": 330.0, + "completions/mean_length": 212.5, + "completions/mean_terminated_length": 212.5, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4446268379688263, + "epoch": 0.22, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.305844306945801, + "learning_rate": 7.85e-07, + "loss": -0.06735197454690933, + "num_tokens": 374306.0, + "reward": 18.059999465942383, + "reward_std": 98.9417953491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 17.959999084472656, + "rewards/supergames_reward/std": 98.9417953491211, + "step": 44, + "step_time": 12.492453911982011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.0, + "completions/max_terminated_length": 316.0, + "completions/mean_length": 183.0, + "completions/mean_terminated_length": 183.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.5269614458084106, + "epoch": 0.225, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8006370067596436, + "learning_rate": 7.799999999999999e-07, + "loss": -0.08870815485715866, + "num_tokens": 382210.0, + "reward": -54.928749084472656, + "reward_std": 72.5127944946289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -55.02874755859375, + "rewards/supergames_reward/std": 72.5127944946289, + "step": 45, + "step_time": 11.7636467939883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.0, + "completions/max_terminated_length": 244.0, + "completions/mean_length": 136.625, + "completions/mean_terminated_length": 136.625, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.34820204973220825, + "epoch": 0.23, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.911736011505127, + "learning_rate": 7.75e-07, + "loss": 0.04397330805659294, + "num_tokens": 391583.0, + "reward": -19.10375213623047, + "reward_std": 23.967702865600586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -19.203750610351562, + "rewards/supergames_reward/std": 23.967702865600586, + "step": 46, + "step_time": 10.119426151999505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 160.375, + "completions/mean_terminated_length": 160.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.611914336681366, + "epoch": 0.235, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.571913242340088, + "learning_rate": 7.699999999999999e-07, + "loss": -0.044293053448200226, + "num_tokens": 399290.0, + "reward": -25.186250686645508, + "reward_std": 71.46305084228516, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.286251068115234, + "rewards/supergames_reward/std": 71.46305084228516, + "step": 47, + "step_time": 8.563868903991533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/max_terminated_length": 167.0, + "completions/mean_length": 111.125, + "completions/mean_terminated_length": 111.125, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.46585777401924133, + "epoch": 0.24, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.654109954833984, + "learning_rate": 7.65e-07, + "loss": -0.035151124000549316, + "num_tokens": 404707.0, + "reward": -0.408750057220459, + "reward_std": 31.939748764038086, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -0.5087499618530273, + "rewards/supergames_reward/std": 31.93975257873535, + "step": 48, + "step_time": 6.438482580007985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/max_terminated_length": 306.0, + "completions/mean_length": 216.625, + "completions/mean_terminated_length": 216.625, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.33054471015930176, + "epoch": 0.245, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.700303792953491, + "learning_rate": 7.599999999999999e-07, + "loss": -0.04393656551837921, + "num_tokens": 414736.0, + "reward": -97.26000213623047, + "reward_std": 1.6294406652450562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.36000061035156, + "rewards/supergames_reward/std": 1.629441738128662, + "step": 49, + "step_time": 11.636040106008295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 378.0, + "completions/max_terminated_length": 378.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.48711177706718445, + "epoch": 0.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9650564193725586, + "learning_rate": 7.55e-07, + "loss": -0.3529520332813263, + "num_tokens": 424795.0, + "reward": 47.408748626708984, + "reward_std": 72.72083282470703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 47.30875015258789, + "rewards/supergames_reward/std": 72.72083282470703, + "step": 50, + "step_time": 14.258096842997475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 267.625, + "completions/mean_terminated_length": 232.71429443359375, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4482860863208771, + "epoch": 0.255, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2998197078704834, + "learning_rate": 7.5e-07, + "loss": 0.2493654191493988, + "num_tokens": 436288.0, + "reward": 50.61125183105469, + "reward_std": 75.3349838256836, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.53624725341797, + "rewards/supergames_reward/std": 75.27783966064453, + "step": 51, + "step_time": 19.20827590499539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 176.0, + "completions/max_terminated_length": 176.0, + "completions/mean_length": 144.5, + "completions/mean_terminated_length": 144.5, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "entropy": 0.5378735661506653, + "epoch": 0.26, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.114461898803711, + "learning_rate": 7.45e-07, + "loss": 0.0036597400903701782, + "num_tokens": 441980.0, + "reward": -15.20625114440918, + "reward_std": 45.35526657104492, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.28125, + "rewards/supergames_reward/std": 45.301815032958984, + "step": 52, + "step_time": 6.660627231001854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/max_terminated_length": 136.0, + "completions/mean_length": 119.0, + "completions/mean_terminated_length": 119.0, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "entropy": 0.41796669363975525, + "epoch": 0.265, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.059505462646484, + "learning_rate": 7.4e-07, + "loss": -0.014973883517086506, + "num_tokens": 447468.0, + "reward": 15.170000076293945, + "reward_std": 46.658329010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 15.070000648498535, + "rewards/supergames_reward/std": 46.658329010009766, + "step": 53, + "step_time": 5.271571868011961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/max_terminated_length": 259.0, + "completions/mean_length": 191.125, + "completions/mean_terminated_length": 191.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.45421987771987915, + "epoch": 0.27, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4454867839813232, + "learning_rate": 7.35e-07, + "loss": -0.1128973588347435, + "num_tokens": 457301.0, + "reward": 38.89875030517578, + "reward_std": 62.1772575378418, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 38.79874801635742, + "rewards/supergames_reward/std": 62.1772575378418, + "step": 54, + "step_time": 10.524528659996577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/max_terminated_length": 319.0, + "completions/mean_length": 233.375, + "completions/mean_terminated_length": 233.375, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "entropy": 0.4449213445186615, + "epoch": 0.275, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2065398693084717, + "learning_rate": 7.3e-07, + "loss": 0.0016277075046673417, + "num_tokens": 467520.0, + "reward": 27.372499465942383, + "reward_std": 81.23126983642578, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.27250099182129, + "rewards/supergames_reward/std": 81.23126983642578, + "step": 55, + "step_time": 12.592280682991259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 357.0, + "completions/mean_length": 261.0, + "completions/mean_terminated_length": 225.1428680419922, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.506920337677002, + "epoch": 0.28, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.058388710021973, + "learning_rate": 7.249999999999999e-07, + "loss": 0.3289242386817932, + "num_tokens": 476056.0, + "reward": 47.05249786376953, + "reward_std": 80.97753143310547, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 46.977500915527344, + "rewards/supergames_reward/std": 80.9256362915039, + "step": 56, + "step_time": 18.845177220006008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.0, + "completions/max_terminated_length": 250.0, + "completions/mean_length": 192.75, + "completions/mean_terminated_length": 192.75, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "entropy": 0.5197336077690125, + "epoch": 0.285, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.230815410614014, + "learning_rate": 7.2e-07, + "loss": -0.15186546742916107, + "num_tokens": 484014.0, + "reward": -64.5050048828125, + "reward_std": 50.35145568847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -64.60499572753906, + "rewards/supergames_reward/std": 50.3514518737793, + "step": 57, + "step_time": 9.576232638006331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 249.0, + "completions/mean_terminated_length": 249.0, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "entropy": 0.42913514375686646, + "epoch": 0.29, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.8544528484344482, + "learning_rate": 7.149999999999999e-07, + "loss": -0.056405920535326004, + "num_tokens": 494390.0, + "reward": 40.123748779296875, + "reward_std": 84.164794921875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 40.02375030517578, + "rewards/supergames_reward/std": 84.164794921875, + "step": 58, + "step_time": 12.385353341000155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 138.0, + "completions/max_terminated_length": 138.0, + "completions/mean_length": 107.25, + "completions/mean_terminated_length": 107.25, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.34386488795280457, + "epoch": 0.295, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.6915154457092285, + "learning_rate": 7.1e-07, + "loss": 0.0865321159362793, + "num_tokens": 499816.0, + "reward": -8.244999885559082, + "reward_std": 4.681046962738037, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.345000267028809, + "rewards/supergames_reward/std": 4.681046962738037, + "step": 59, + "step_time": 5.42325339600211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/max_terminated_length": 187.0, + "completions/mean_length": 127.25, + "completions/mean_terminated_length": 127.25, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.5430045127868652, + "epoch": 0.3, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.635311126708984, + "learning_rate": 7.049999999999999e-07, + "loss": -0.05412375554442406, + "num_tokens": 505410.0, + "reward": 20.005001068115234, + "reward_std": 55.156005859375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 19.904998779296875, + "rewards/supergames_reward/std": 55.156005859375, + "step": 60, + "step_time": 7.068010902003152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/max_terminated_length": 286.0, + "completions/mean_length": 192.125, + "completions/mean_terminated_length": 192.125, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4213869869709015, + "epoch": 0.305, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4664618968963623, + "learning_rate": 7e-07, + "loss": -0.0495159812271595, + "num_tokens": 516283.0, + "reward": -47.212501525878906, + "reward_std": 91.2253646850586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -47.3125, + "rewards/supergames_reward/std": 91.2253646850586, + "step": 61, + "step_time": 11.176304235996213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 171.0, + "completions/mean_terminated_length": 171.0, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.4626811146736145, + "epoch": 0.31, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.949252128601074, + "learning_rate": 6.949999999999999e-07, + "loss": 0.192047581076622, + "num_tokens": 522251.0, + "reward": -15.092500686645508, + "reward_std": 49.82962417602539, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.167499542236328, + "rewards/supergames_reward/std": 49.78091049194336, + "step": 62, + "step_time": 12.075224861997413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 429.0, + "completions/mean_length": 283.5, + "completions/mean_terminated_length": 250.85714721679688, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4814216196537018, + "epoch": 0.315, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7918753623962402, + "learning_rate": 6.9e-07, + "loss": 0.15706156194210052, + "num_tokens": 533815.0, + "reward": -20.86625099182129, + "reward_std": 85.99250030517578, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.94124984741211, + "rewards/supergames_reward/std": 85.96620178222656, + "step": 63, + "step_time": 19.448832260008203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.0, + "completions/max_terminated_length": 189.0, + "completions/mean_length": 131.25, + "completions/mean_terminated_length": 131.25, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4929414987564087, + "epoch": 0.32, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.774101734161377, + "learning_rate": 6.85e-07, + "loss": -0.060691747814416885, + "num_tokens": 539465.0, + "reward": -21.0674991607666, + "reward_std": 6.892730236053467, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -21.167499542236328, + "rewards/supergames_reward/std": 6.892730236053467, + "step": 64, + "step_time": 7.141569407976931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 454.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 281.375, + "completions/mean_terminated_length": 281.375, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "entropy": 0.45866572856903076, + "epoch": 0.325, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.092717409133911, + "learning_rate": 6.800000000000001e-07, + "loss": 0.11694959551095963, + "num_tokens": 551012.0, + "reward": 49.938751220703125, + "reward_std": 79.1865005493164, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 49.8387451171875, + "rewards/supergames_reward/std": 79.1865005493164, + "step": 65, + "step_time": 17.06170882002334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/max_terminated_length": 203.0, + "completions/mean_length": 128.625, + "completions/mean_terminated_length": 128.625, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.5005782246589661, + "epoch": 0.33, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.77616024017334, + "learning_rate": 6.75e-07, + "loss": 0.11242837458848953, + "num_tokens": 556577.0, + "reward": 48.875, + "reward_std": 49.13539123535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.775001525878906, + "rewards/supergames_reward/std": 49.13539123535156, + "step": 66, + "step_time": 7.695410129002994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 436.0, + "completions/max_terminated_length": 436.0, + "completions/mean_length": 281.875, + "completions/mean_terminated_length": 281.875, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "entropy": 0.507097065448761, + "epoch": 0.335, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.911184787750244, + "learning_rate": 6.7e-07, + "loss": -0.08151137083768845, + "num_tokens": 567120.0, + "reward": -12.886249542236328, + "reward_std": 84.04269409179688, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.986251831054688, + "rewards/supergames_reward/std": 84.04269409179688, + "step": 67, + "step_time": 16.409127942984924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 318.375, + "completions/mean_terminated_length": 290.71429443359375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.486402690410614, + "epoch": 0.34, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9461607933044434, + "learning_rate": 6.65e-07, + "loss": 0.2442178875207901, + "num_tokens": 578995.0, + "reward": 2.8387489318847656, + "reward_std": 98.47579956054688, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 2.7637500762939453, + "rewards/supergames_reward/std": 98.44595336914062, + "step": 68, + "step_time": 19.32364999302081 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 291.0, + "completions/max_terminated_length": 291.0, + "completions/mean_length": 219.625, + "completions/mean_terminated_length": 219.625, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.37856727838516235, + "epoch": 0.345, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0204732418060303, + "learning_rate": 6.6e-07, + "loss": -0.08764916658401489, + "num_tokens": 589072.0, + "reward": -54.18375015258789, + "reward_std": 71.45012664794922, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -54.283748626708984, + "rewards/supergames_reward/std": 71.45013427734375, + "step": 69, + "step_time": 11.126611230982235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 305.0, + "completions/mean_length": 254.75, + "completions/mean_terminated_length": 218.00001525878906, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46423614025115967, + "epoch": 0.35, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0140154361724854, + "learning_rate": 6.55e-07, + "loss": 0.26797160506248474, + "num_tokens": 600438.0, + "reward": -32.04875183105469, + "reward_std": 69.86957550048828, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -32.12375259399414, + "rewards/supergames_reward/std": 69.84178161621094, + "step": 70, + "step_time": 19.12617294798838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 372.0, + "completions/max_terminated_length": 372.0, + "completions/mean_length": 194.125, + "completions/mean_terminated_length": 194.125, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.3427293300628662, + "epoch": 0.355, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.781383991241455, + "learning_rate": 6.5e-07, + "loss": 0.354524701833725, + "num_tokens": 610327.0, + "reward": -6.913749694824219, + "reward_std": 87.56071472167969, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -6.988750457763672, + "rewards/supergames_reward/std": 87.53033447265625, + "step": 71, + "step_time": 14.041668199002743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/max_terminated_length": 197.0, + "completions/mean_length": 124.75, + "completions/mean_terminated_length": 124.75, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.4455118477344513, + "epoch": 0.36, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.726032733917236, + "learning_rate": 6.45e-07, + "loss": 0.18206289410591125, + "num_tokens": 615853.0, + "reward": 41.95124816894531, + "reward_std": 60.640811920166016, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 41.85124969482422, + "rewards/supergames_reward/std": 60.640811920166016, + "step": 72, + "step_time": 7.416493425989756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.0, + "completions/max_terminated_length": 323.0, + "completions/mean_length": 147.375, + "completions/mean_terminated_length": 147.375, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.480733186006546, + "epoch": 0.365, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.853621482849121, + "learning_rate": 6.4e-07, + "loss": 0.04558124020695686, + "num_tokens": 621584.0, + "reward": 14.884998321533203, + "reward_std": 46.25857162475586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 14.78499984741211, + "rewards/supergames_reward/std": 46.25857162475586, + "step": 73, + "step_time": 11.95379540900467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 283.0, + "completions/max_terminated_length": 283.0, + "completions/mean_length": 188.125, + "completions/mean_terminated_length": 188.125, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.5526013374328613, + "epoch": 0.37, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.729883909225464, + "learning_rate": 6.35e-07, + "loss": 0.06268303096294403, + "num_tokens": 629545.0, + "reward": -68.05750274658203, + "reward_std": 59.44808578491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.15750122070312, + "rewards/supergames_reward/std": 59.44808578491211, + "step": 74, + "step_time": 10.64124580900534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 141.0, + "completions/max_terminated_length": 141.0, + "completions/mean_length": 117.375, + "completions/mean_terminated_length": 117.375, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.3744319677352905, + "epoch": 0.375, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.775154113769531, + "learning_rate": 6.3e-07, + "loss": -0.023284845054149628, + "num_tokens": 635052.0, + "reward": 3.9374990463256836, + "reward_std": 61.85771179199219, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 3.862499237060547, + "rewards/supergames_reward/std": 61.80967712402344, + "step": 75, + "step_time": 5.432648951013107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 157.0, + "completions/mean_terminated_length": 157.0, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4822140336036682, + "epoch": 0.38, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.715704917907715, + "learning_rate": 6.249999999999999e-07, + "loss": -0.04918142035603523, + "num_tokens": 644644.0, + "reward": 48.64125061035156, + "reward_std": 78.0205078125, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 48.56624984741211, + "rewards/supergames_reward/std": 77.96605682373047, + "step": 76, + "step_time": 10.348935816989979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/max_terminated_length": 380.0, + "completions/mean_length": 210.875, + "completions/mean_terminated_length": 210.875, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "entropy": 0.45722317695617676, + "epoch": 0.385, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6057567596435547, + "learning_rate": 6.2e-07, + "loss": 0.058352649211883545, + "num_tokens": 652771.0, + "reward": -8.283750534057617, + "reward_std": 77.68436431884766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.383749008178711, + "rewards/supergames_reward/std": 77.68437194824219, + "step": 77, + "step_time": 14.142948755004909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 188.25, + "completions/mean_terminated_length": 188.25, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.33380764722824097, + "epoch": 0.39, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2239863872528076, + "learning_rate": 6.149999999999999e-07, + "loss": -0.04592633992433548, + "num_tokens": 662653.0, + "reward": 11.153749465942383, + "reward_std": 89.88329315185547, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 11.053749084472656, + "rewards/supergames_reward/std": 89.88329315185547, + "step": 78, + "step_time": 12.277474621019792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/max_terminated_length": 143.0, + "completions/mean_length": 108.875, + "completions/mean_terminated_length": 108.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.3518812656402588, + "epoch": 0.395, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.967776298522949, + "learning_rate": 6.1e-07, + "loss": -0.10800496488809586, + "num_tokens": 668076.0, + "reward": 70.33250427246094, + "reward_std": 49.9145393371582, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 70.23249816894531, + "rewards/supergames_reward/std": 49.91453552246094, + "step": 79, + "step_time": 5.52714040101273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 152.125, + "completions/mean_terminated_length": 152.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4699896574020386, + "epoch": 0.4, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.561161994934082, + "learning_rate": 6.049999999999999e-07, + "loss": 0.09505834430456161, + "num_tokens": 675701.0, + "reward": -20.25625228881836, + "reward_std": 53.78542709350586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -20.35624885559082, + "rewards/supergames_reward/std": 53.78542709350586, + "step": 80, + "step_time": 7.155081019998761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 288.875, + "completions/mean_terminated_length": 257.0, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.4086494445800781, + "epoch": 0.405, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.3719611167907715, + "learning_rate": 6e-07, + "loss": 0.04727627709507942, + "num_tokens": 687332.0, + "reward": -37.20750045776367, + "reward_std": 91.7676010131836, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -37.25749969482422, + "rewards/supergames_reward/std": 91.72847747802734, + "step": 81, + "step_time": 19.231399144016905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 376.0, + "completions/max_terminated_length": 376.0, + "completions/mean_length": 227.75, + "completions/mean_terminated_length": 227.75, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.43233707547187805, + "epoch": 0.41, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3391714096069336, + "learning_rate": 5.949999999999999e-07, + "loss": -0.25524550676345825, + "num_tokens": 698474.0, + "reward": -3.7287511825561523, + "reward_std": 91.25379943847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.828749656677246, + "rewards/supergames_reward/std": 91.25379943847656, + "step": 82, + "step_time": 14.329176368017215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 178.125, + "completions/mean_terminated_length": 178.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.4293636977672577, + "epoch": 0.415, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.7271742820739746, + "learning_rate": 5.9e-07, + "loss": 0.09273672848939896, + "num_tokens": 708243.0, + "reward": 12.802498817443848, + "reward_std": 86.23526000976562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 12.702500343322754, + "rewards/supergames_reward/std": 86.23526000976562, + "step": 83, + "step_time": 11.419686011999147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 425.0, + "completions/max_terminated_length": 425.0, + "completions/mean_length": 255.375, + "completions/mean_terminated_length": 255.375, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "entropy": 0.38068246841430664, + "epoch": 0.42, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.6244184970855713, + "learning_rate": 5.849999999999999e-07, + "loss": 0.02006213366985321, + "num_tokens": 718606.0, + "reward": 37.803749084472656, + "reward_std": 57.58624267578125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 37.70375061035156, + "rewards/supergames_reward/std": 57.58624267578125, + "step": 84, + "step_time": 15.975198492989875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 116.0, + "completions/max_terminated_length": 116.0, + "completions/mean_length": 108.125, + "completions/mean_terminated_length": 108.125, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.339572012424469, + "epoch": 0.425, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.754759311676025, + "learning_rate": 5.8e-07, + "loss": -0.011015941388905048, + "num_tokens": 723983.0, + "reward": -3.8212504386901855, + "reward_std": 61.624786376953125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.921250820159912, + "rewards/supergames_reward/std": 61.624786376953125, + "step": 85, + "step_time": 4.542777584982105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 447.0, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 291.375, + "completions/mean_terminated_length": 291.375, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "entropy": 0.5846173763275146, + "epoch": 0.43, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55724835395813, + "learning_rate": 5.749999999999999e-07, + "loss": -0.04981120675802231, + "num_tokens": 735642.0, + "reward": 50.07499694824219, + "reward_std": 92.6283187866211, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.0, + "rewards/supergames_reward/std": 92.5820083618164, + "step": 86, + "step_time": 16.777178087009815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/max_terminated_length": 232.0, + "completions/mean_length": 192.0, + "completions/mean_terminated_length": 192.0, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "entropy": 0.5139474272727966, + "epoch": 0.435, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8860411643981934, + "learning_rate": 5.699999999999999e-07, + "loss": -0.02194221317768097, + "num_tokens": 743602.0, + "reward": -52.522499084472656, + "reward_std": 73.8686294555664, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -52.62249755859375, + "rewards/supergames_reward/std": 73.86863708496094, + "step": 87, + "step_time": 8.772893911984283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/max_terminated_length": 266.0, + "completions/mean_length": 146.875, + "completions/mean_terminated_length": 146.875, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.3428497910499573, + "epoch": 0.44, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.509483337402344, + "learning_rate": 5.649999999999999e-07, + "loss": 0.19857533276081085, + "num_tokens": 753065.0, + "reward": 63.63874816894531, + "reward_std": 67.54963684082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 63.53874969482422, + "rewards/supergames_reward/std": 67.54963684082031, + "step": 88, + "step_time": 10.230529835011112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 446.0, + "completions/max_terminated_length": 446.0, + "completions/mean_length": 270.625, + "completions/mean_terminated_length": 270.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.4417419731616974, + "epoch": 0.445, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.973951816558838, + "learning_rate": 5.6e-07, + "loss": -0.00031440958264283836, + "num_tokens": 764534.0, + "reward": -9.736251831054688, + "reward_std": 92.25879669189453, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -9.836250305175781, + "rewards/supergames_reward/std": 92.25880432128906, + "step": 89, + "step_time": 17.40752330099349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 398.0, + "completions/max_terminated_length": 398.0, + "completions/mean_length": 227.375, + "completions/mean_terminated_length": 227.375, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.4732024073600769, + "epoch": 0.45, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2560815811157227, + "learning_rate": 5.55e-07, + "loss": 0.03820464387536049, + "num_tokens": 772809.0, + "reward": 5.982499599456787, + "reward_std": 58.833797454833984, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 5.8824992179870605, + "rewards/supergames_reward/std": 58.83380126953125, + "step": 90, + "step_time": 15.007269965979503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 207.5, + "completions/mean_terminated_length": 207.5, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.42038559913635254, + "epoch": 0.455, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3649117946624756, + "learning_rate": 5.5e-07, + "loss": 0.1510048508644104, + "num_tokens": 782805.0, + "reward": -5.61500358581543, + "reward_std": 82.67152404785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -5.714998245239258, + "rewards/supergames_reward/std": 82.67152404785156, + "step": 91, + "step_time": 10.386137172987219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 416.0, + "completions/max_terminated_length": 416.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.3971734642982483, + "epoch": 0.46, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.433772563934326, + "learning_rate": 5.45e-07, + "loss": -0.3027213513851166, + "num_tokens": 793840.0, + "reward": -29.06500244140625, + "reward_std": 91.68325805664062, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -29.16499900817871, + "rewards/supergames_reward/std": 91.68325805664062, + "step": 92, + "step_time": 15.790611553995404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 206.625, + "completions/mean_terminated_length": 206.625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.5906907320022583, + "epoch": 0.465, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.293088912963867, + "learning_rate": 5.4e-07, + "loss": 0.09760032594203949, + "num_tokens": 803821.0, + "reward": -72.73999786376953, + "reward_std": 50.72484588623047, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -72.81500244140625, + "rewards/supergames_reward/std": 50.7094841003418, + "step": 93, + "step_time": 13.043119941983605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/max_terminated_length": 143.0, + "completions/mean_length": 115.75, + "completions/mean_terminated_length": 115.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4233115315437317, + "epoch": 0.47, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.58886194229126, + "learning_rate": 5.35e-07, + "loss": -0.04040650278329849, + "num_tokens": 809299.0, + "reward": -3.3050003051757812, + "reward_std": 24.443714141845703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.405000686645508, + "rewards/supergames_reward/std": 24.443714141845703, + "step": 94, + "step_time": 5.512874762003776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 122.0, + "completions/max_terminated_length": 122.0, + "completions/mean_length": 103.125, + "completions/mean_terminated_length": 103.125, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.4055764377117157, + "epoch": 0.475, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.981549263000488, + "learning_rate": 5.3e-07, + "loss": -0.07588938623666763, + "num_tokens": 814716.0, + "reward": 60.06999969482422, + "reward_std": 42.708473205566406, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 59.970001220703125, + "rewards/supergames_reward/std": 42.708473205566406, + "step": 95, + "step_time": 4.766389057011111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 475.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.408913254737854, + "epoch": 0.48, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.79402756690979, + "learning_rate": 5.25e-07, + "loss": -0.3087541460990906, + "num_tokens": 824791.0, + "reward": -15.042500495910645, + "reward_std": 84.37246704101562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -15.142499923706055, + "rewards/supergames_reward/std": 84.37246704101562, + "step": 96, + "step_time": 17.779843941010768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/max_terminated_length": 154.0, + "completions/mean_length": 121.625, + "completions/mean_terminated_length": 121.625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.529015839099884, + "epoch": 0.485, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.142816066741943, + "learning_rate": 5.2e-07, + "loss": 0.07216037809848785, + "num_tokens": 830300.0, + "reward": -32.743751525878906, + "reward_std": 54.61775588989258, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -32.81875228881836, + "rewards/supergames_reward/std": 54.58256149291992, + "step": 97, + "step_time": 5.913989890017547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 126.0, + "completions/max_terminated_length": 126.0, + "completions/mean_length": 95.625, + "completions/mean_terminated_length": 95.625, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.33743274211883545, + "epoch": 0.49, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.197273254394531, + "learning_rate": 5.149999999999999e-07, + "loss": 0.07500407844781876, + "num_tokens": 835633.0, + "reward": 11.00999927520752, + "reward_std": 62.73848342895508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 10.910000801086426, + "rewards/supergames_reward/std": 62.73848342895508, + "step": 98, + "step_time": 4.9155233050114475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 108.5, + "completions/mean_terminated_length": 108.5, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.3106803297996521, + "epoch": 0.495, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.864002227783203, + "learning_rate": 5.1e-07, + "loss": -0.1206185445189476, + "num_tokens": 841045.0, + "reward": 14.238749504089355, + "reward_std": 51.77016830444336, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 14.138750076293945, + "rewards/supergames_reward/std": 51.77016830444336, + "step": 99, + "step_time": 5.7507751359953545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 97.125, + "completions/mean_terminated_length": 97.125, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.35160186886787415, + "epoch": 0.5, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915731906890869, + "learning_rate": 5.049999999999999e-07, + "loss": -0.07951541244983673, + "num_tokens": 846398.0, + "reward": 67.78125, + "reward_std": 38.15436553955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 67.68124389648438, + "rewards/supergames_reward/std": 38.15436553955078, + "step": 100, + "step_time": 5.994720440998208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 320.0, + "completions/max_terminated_length": 320.0, + "completions/mean_length": 208.75, + "completions/mean_terminated_length": 208.75, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.452095091342926, + "epoch": 0.505, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.065176963806152, + "learning_rate": 5e-07, + "loss": -0.1397111564874649, + "num_tokens": 857420.0, + "reward": -5.801251411437988, + "reward_std": 73.56117248535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -5.90125036239624, + "rewards/supergames_reward/std": 73.5611801147461, + "step": 101, + "step_time": 12.28178753197426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/max_terminated_length": 230.0, + "completions/mean_length": 171.375, + "completions/mean_terminated_length": 171.375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4138447642326355, + "epoch": 0.51, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.998823404312134, + "learning_rate": 4.95e-07, + "loss": -0.08941879868507385, + "num_tokens": 867103.0, + "reward": 92.24749755859375, + "reward_std": 14.540005683898926, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 92.14749908447266, + "rewards/supergames_reward/std": 14.540006637573242, + "step": 102, + "step_time": 8.992682139010867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 311.0, + "completions/max_terminated_length": 311.0, + "completions/mean_length": 163.0, + "completions/mean_terminated_length": 163.0, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.4182547330856323, + "epoch": 0.515, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.419595241546631, + "learning_rate": 4.9e-07, + "loss": -0.19304415583610535, + "num_tokens": 876679.0, + "reward": 79.13249969482422, + "reward_std": 59.3050422668457, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 79.03250122070312, + "rewards/supergames_reward/std": 59.30504608154297, + "step": 103, + "step_time": 11.836970244999975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 253.0, + "completions/max_terminated_length": 253.0, + "completions/mean_length": 155.625, + "completions/mean_terminated_length": 155.625, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.49889081716537476, + "epoch": 0.52, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.463719367980957, + "learning_rate": 4.85e-07, + "loss": -0.04786720871925354, + "num_tokens": 884348.0, + "reward": 48.616249084472656, + "reward_std": 73.30741119384766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.51625061035156, + "rewards/supergames_reward/std": 73.30741119384766, + "step": 104, + "step_time": 9.579507841990562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/max_terminated_length": 380.0, + "completions/mean_length": 269.5, + "completions/mean_terminated_length": 269.5, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "entropy": 0.5286482572555542, + "epoch": 0.525, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.993295431137085, + "learning_rate": 4.8e-07, + "loss": -0.08071611076593399, + "num_tokens": 894856.0, + "reward": 34.970001220703125, + "reward_std": 85.324462890625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 34.869998931884766, + "rewards/supergames_reward/std": 85.324462890625, + "step": 105, + "step_time": 14.295730627985904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 203.25, + "completions/mean_terminated_length": 203.25, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.6049264073371887, + "epoch": 0.53, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.787306785583496, + "learning_rate": 4.7499999999999995e-07, + "loss": 0.19928883016109467, + "num_tokens": 902922.0, + "reward": 32.44499969482422, + "reward_std": 70.4663314819336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 32.369998931884766, + "rewards/supergames_reward/std": 70.41261291503906, + "step": 106, + "step_time": 9.84894504098338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 194.0, + "completions/mean_terminated_length": 194.0, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "entropy": 0.5115755200386047, + "epoch": 0.535, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.277207374572754, + "learning_rate": 4.6999999999999995e-07, + "loss": 0.09118300676345825, + "num_tokens": 910906.0, + "reward": 40.432498931884766, + "reward_std": 48.54548263549805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 40.33250045776367, + "rewards/supergames_reward/std": 48.54548263549805, + "step": 107, + "step_time": 10.466603949986165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 332.0, + "completions/max_terminated_length": 332.0, + "completions/mean_length": 195.875, + "completions/mean_terminated_length": 195.875, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4192962944507599, + "epoch": 0.54, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.308061122894287, + "learning_rate": 4.65e-07, + "loss": 0.28064602613449097, + "num_tokens": 920745.0, + "reward": -21.982500076293945, + "reward_std": 66.31195068359375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -22.08249855041504, + "rewards/supergames_reward/std": 66.31195068359375, + "step": 108, + "step_time": 12.595848233992001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 195.125, + "completions/mean_terminated_length": 195.125, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.37523961067199707, + "epoch": 0.545, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.802706718444824, + "learning_rate": 4.6e-07, + "loss": -0.06614465266466141, + "num_tokens": 930618.0, + "reward": 71.75, + "reward_std": 51.14524459838867, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 71.6500015258789, + "rewards/supergames_reward/std": 51.1452522277832, + "step": 109, + "step_time": 10.33884758799104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 167.75, + "completions/mean_terminated_length": 167.75, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.4339042007923126, + "epoch": 0.55, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.254003047943115, + "learning_rate": 4.55e-07, + "loss": 0.34405598044395447, + "num_tokens": 941256.0, + "reward": 99.21875, + "reward_std": 2.492553234100342, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 99.1187515258789, + "rewards/supergames_reward/std": 2.4925525188446045, + "step": 110, + "step_time": 12.657525141985388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 285.0, + "completions/max_terminated_length": 285.0, + "completions/mean_length": 211.0, + "completions/mean_terminated_length": 211.0, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "entropy": 0.46809861063957214, + "epoch": 0.555, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.811166286468506, + "learning_rate": 4.5e-07, + "loss": 0.045113153755664825, + "num_tokens": 951296.0, + "reward": 55.27750015258789, + "reward_std": 47.90687561035156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 55.1775016784668, + "rewards/supergames_reward/std": 47.9068717956543, + "step": 111, + "step_time": 10.91118389699841 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 145.0, + "completions/max_terminated_length": 145.0, + "completions/mean_length": 108.375, + "completions/mean_terminated_length": 108.375, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.34181955456733704, + "epoch": 0.56, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.537296772003174, + "learning_rate": 4.45e-07, + "loss": -0.052413541823625565, + "num_tokens": 956675.0, + "reward": 47.56624984741211, + "reward_std": 49.185157775878906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 47.466251373291016, + "rewards/supergames_reward/std": 49.18516159057617, + "step": 112, + "step_time": 5.552288047998445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/max_terminated_length": 245.0, + "completions/mean_length": 189.125, + "completions/mean_terminated_length": 189.125, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "entropy": 0.5034605860710144, + "epoch": 0.565, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.349876642227173, + "learning_rate": 4.3999999999999997e-07, + "loss": -0.08186715841293335, + "num_tokens": 964636.0, + "reward": 81.32374572753906, + "reward_std": 36.85074996948242, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 81.2237548828125, + "rewards/supergames_reward/std": 36.85074996948242, + "step": 113, + "step_time": 9.267611294984818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/max_terminated_length": 178.0, + "completions/mean_length": 111.25, + "completions/mean_terminated_length": 111.25, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "entropy": 0.3229002058506012, + "epoch": 0.57, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.245575428009033, + "learning_rate": 4.3499999999999996e-07, + "loss": 0.06544198095798492, + "num_tokens": 970070.0, + "reward": 63.1775016784668, + "reward_std": 52.30624008178711, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 63.07749938964844, + "rewards/supergames_reward/std": 52.30624008178711, + "step": 114, + "step_time": 6.731139309995342 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 224.0, + "completions/max_terminated_length": 224.0, + "completions/mean_length": 170.0, + "completions/mean_terminated_length": 170.0, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.3274807929992676, + "epoch": 0.575, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5544140338897705, + "learning_rate": 4.2999999999999996e-07, + "loss": -0.06953569501638412, + "num_tokens": 979702.0, + "reward": 25.62125015258789, + "reward_std": 64.52003479003906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 25.521244049072266, + "rewards/supergames_reward/std": 64.52003479003906, + "step": 115, + "step_time": 8.702618776995223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 427.0, + "completions/max_terminated_length": 427.0, + "completions/mean_length": 209.5, + "completions/mean_terminated_length": 209.5, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.4970880150794983, + "epoch": 0.58, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4932165145874023, + "learning_rate": 4.2499999999999995e-07, + "loss": -0.2281273603439331, + "num_tokens": 990690.0, + "reward": 50.54750061035156, + "reward_std": 70.5126953125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 50.4474983215332, + "rewards/supergames_reward/std": 70.51270294189453, + "step": 116, + "step_time": 16.204386608995264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.0, + "completions/max_terminated_length": 228.0, + "completions/mean_length": 132.375, + "completions/mean_terminated_length": 132.375, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4380227029323578, + "epoch": 0.585, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.089909076690674, + "learning_rate": 4.1999999999999995e-07, + "loss": 0.12121254950761795, + "num_tokens": 996333.0, + "reward": -15.333749771118164, + "reward_std": 47.200164794921875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -15.433748245239258, + "rewards/supergames_reward/std": 47.200164794921875, + "step": 117, + "step_time": 8.546233230998041 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.0, + "completions/max_terminated_length": 323.0, + "completions/mean_length": 137.625, + "completions/mean_terminated_length": 137.625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4177803695201874, + "epoch": 0.59, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8712472915649414, + "learning_rate": 4.1499999999999994e-07, + "loss": 0.14192955195903778, + "num_tokens": 1002010.0, + "reward": -30.700000762939453, + "reward_std": 17.224035263061523, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -30.799999237060547, + "rewards/supergames_reward/std": 17.224035263061523, + "step": 118, + "step_time": 11.878434990998358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 453.0, + "completions/max_terminated_length": 453.0, + "completions/mean_length": 238.125, + "completions/mean_terminated_length": 238.125, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "entropy": 0.5640217065811157, + "epoch": 0.595, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.064737319946289, + "learning_rate": 4.0999999999999994e-07, + "loss": -0.023621462285518646, + "num_tokens": 1012259.0, + "reward": 43.683746337890625, + "reward_std": 70.4210205078125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 43.5837516784668, + "rewards/supergames_reward/std": 70.4210205078125, + "step": 119, + "step_time": 16.829514264973113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 193.75, + "completions/mean_terminated_length": 193.75, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "entropy": 0.5795091390609741, + "epoch": 0.6, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.423007965087891, + "learning_rate": 4.05e-07, + "loss": -0.0666111558675766, + "num_tokens": 1020241.0, + "reward": -85.86000061035156, + "reward_std": 27.4442081451416, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -85.90999603271484, + "rewards/supergames_reward/std": 27.41469955444336, + "step": 120, + "step_time": 8.95496519000153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 231.0, + "completions/mean_terminated_length": 231.0, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "entropy": 0.3767699897289276, + "epoch": 0.605, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7434473037719727, + "learning_rate": 4e-07, + "loss": -0.1165132150053978, + "num_tokens": 1031465.0, + "reward": 61.45624923706055, + "reward_std": 72.21080780029297, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 61.35625076293945, + "rewards/supergames_reward/std": 72.2108154296875, + "step": 121, + "step_time": 12.588820598000893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 297.0, + "completions/max_terminated_length": 297.0, + "completions/mean_length": 223.375, + "completions/mean_terminated_length": 223.375, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "entropy": 0.4537277817726135, + "epoch": 0.61, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4382436275482178, + "learning_rate": 3.95e-07, + "loss": -0.1038316935300827, + "num_tokens": 1042548.0, + "reward": 5.658749580383301, + "reward_std": 83.77420043945312, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 5.558750152587891, + "rewards/supergames_reward/std": 83.77420043945312, + "step": 122, + "step_time": 11.39525846898323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 456.0, + "completions/max_terminated_length": 456.0, + "completions/mean_length": 314.25, + "completions/mean_terminated_length": 314.25, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4113953411579132, + "epoch": 0.615, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.449777841567993, + "learning_rate": 3.8999999999999997e-07, + "loss": -0.1303664743900299, + "num_tokens": 1054422.0, + "reward": 35.98249816894531, + "reward_std": 88.6131362915039, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 35.88249969482422, + "rewards/supergames_reward/std": 88.6131362915039, + "step": 123, + "step_time": 17.001118954998674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 242.0, + "completions/max_terminated_length": 242.0, + "completions/mean_length": 172.5, + "completions/mean_terminated_length": 172.5, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.4670729339122772, + "epoch": 0.62, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.951946973800659, + "learning_rate": 3.8499999999999997e-07, + "loss": 0.05888795852661133, + "num_tokens": 1062210.0, + "reward": 43.368751525878906, + "reward_std": 40.30733871459961, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 43.26874542236328, + "rewards/supergames_reward/std": 40.307342529296875, + "step": 124, + "step_time": 9.142582255997695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/max_terminated_length": 148.0, + "completions/mean_length": 106.625, + "completions/mean_terminated_length": 106.625, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "entropy": 0.30219921469688416, + "epoch": 0.625, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.391322135925293, + "learning_rate": 3.7999999999999996e-07, + "loss": 0.14367049932479858, + "num_tokens": 1067599.0, + "reward": 42.63374710083008, + "reward_std": 58.20897674560547, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 42.533748626708984, + "rewards/supergames_reward/std": 58.208984375, + "step": 125, + "step_time": 5.636063549987739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.0, + "completions/max_terminated_length": 243.0, + "completions/mean_length": 179.25, + "completions/mean_terminated_length": 179.25, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "entropy": 0.4834887683391571, + "epoch": 0.63, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9391286373138428, + "learning_rate": 3.75e-07, + "loss": -0.06614185124635696, + "num_tokens": 1075457.0, + "reward": 8.302498817443848, + "reward_std": 75.10364532470703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 8.202500343322754, + "rewards/supergames_reward/std": 75.10364532470703, + "step": 126, + "step_time": 9.130600645992672 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 196.0, + "completions/max_terminated_length": 196.0, + "completions/mean_length": 157.0, + "completions/mean_terminated_length": 157.0, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.4801807701587677, + "epoch": 0.635, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.425615310668945, + "learning_rate": 3.7e-07, + "loss": -0.08830522745847702, + "num_tokens": 1083129.0, + "reward": -10.563751220703125, + "reward_std": 64.00443267822266, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -10.663749694824219, + "rewards/supergames_reward/std": 64.00444030761719, + "step": 127, + "step_time": 7.4972667430120055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 412.0, + "completions/max_terminated_length": 412.0, + "completions/mean_length": 239.25, + "completions/mean_terminated_length": 239.25, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.4667063355445862, + "epoch": 0.64, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.199113130569458, + "learning_rate": 3.65e-07, + "loss": -0.043856702744960785, + "num_tokens": 1094355.0, + "reward": -55.89250183105469, + "reward_std": 53.05980682373047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -55.99250030517578, + "rewards/supergames_reward/std": 53.05980682373047, + "step": 128, + "step_time": 15.429580625001108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 244.75, + "completions/mean_terminated_length": 244.75, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "entropy": 0.57573401927948, + "epoch": 0.645, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6800079345703125, + "learning_rate": 3.6e-07, + "loss": 0.09521033614873886, + "num_tokens": 1105633.0, + "reward": -23.35250473022461, + "reward_std": 102.31401062011719, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -23.452499389648438, + "rewards/supergames_reward/std": 102.31402587890625, + "step": 129, + "step_time": 13.920327747007832 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/max_terminated_length": 258.0, + "completions/mean_length": 188.25, + "completions/mean_terminated_length": 188.25, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "entropy": 0.5341438055038452, + "epoch": 0.65, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.97955060005188, + "learning_rate": 3.55e-07, + "loss": -0.09512650966644287, + "num_tokens": 1113571.0, + "reward": 29.181249618530273, + "reward_std": 80.10585021972656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 29.081249237060547, + "rewards/supergames_reward/std": 80.10585021972656, + "step": 130, + "step_time": 9.69286120700417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 257.0, + "completions/max_terminated_length": 257.0, + "completions/mean_length": 180.75, + "completions/mean_terminated_length": 180.75, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "entropy": 0.46591487526893616, + "epoch": 0.655, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3518130779266357, + "learning_rate": 3.5e-07, + "loss": 0.04714229330420494, + "num_tokens": 1121457.0, + "reward": -8.287501335144043, + "reward_std": 76.35875701904297, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.387499809265137, + "rewards/supergames_reward/std": 76.35875701904297, + "step": 131, + "step_time": 9.636637121002423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 147.75, + "completions/mean_terminated_length": 147.75, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4407735764980316, + "epoch": 0.66, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.498741626739502, + "learning_rate": 3.45e-07, + "loss": -0.03996007889509201, + "num_tokens": 1129095.0, + "reward": 0.1999988555908203, + "reward_std": 84.63554382324219, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 0.125, + "rewards/supergames_reward/std": 84.6017074584961, + "step": 132, + "step_time": 7.510833573003765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 358.0, + "completions/max_terminated_length": 358.0, + "completions/mean_length": 213.875, + "completions/mean_terminated_length": 213.875, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.5186149477958679, + "epoch": 0.665, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3987107276916504, + "learning_rate": 3.4000000000000003e-07, + "loss": 0.1863594949245453, + "num_tokens": 1139110.0, + "reward": -21.328752517700195, + "reward_std": 88.9102783203125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -21.428749084472656, + "rewards/supergames_reward/std": 88.9102783203125, + "step": 133, + "step_time": 13.393839450000087 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/max_terminated_length": 230.0, + "completions/mean_length": 169.875, + "completions/mean_terminated_length": 169.875, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.49445784091949463, + "epoch": 0.67, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.479673862457275, + "learning_rate": 3.35e-07, + "loss": 0.03618276119232178, + "num_tokens": 1146885.0, + "reward": -14.042501449584961, + "reward_std": 83.22671508789062, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -14.142499923706055, + "rewards/supergames_reward/std": 83.22671508789062, + "step": 134, + "step_time": 8.703399487014394 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/max_terminated_length": 168.0, + "completions/mean_length": 124.75, + "completions/mean_terminated_length": 124.75, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "entropy": 0.2997814118862152, + "epoch": 0.675, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.440976619720459, + "learning_rate": 3.3e-07, + "loss": -0.029299044981598854, + "num_tokens": 1152427.0, + "reward": 45.274993896484375, + "reward_std": 44.401519775390625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 45.17499923706055, + "rewards/supergames_reward/std": 44.40152359008789, + "step": 135, + "step_time": 6.334954546997324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 161.0, + "completions/max_terminated_length": 161.0, + "completions/mean_length": 123.625, + "completions/mean_terminated_length": 123.625, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3462243974208832, + "epoch": 0.68, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.1878814697265625, + "learning_rate": 3.25e-07, + "loss": -0.10772529244422913, + "num_tokens": 1157952.0, + "reward": 44.186248779296875, + "reward_std": 51.78197479248047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 44.08625030517578, + "rewards/supergames_reward/std": 51.7819709777832, + "step": 136, + "step_time": 6.067531232984038 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 155.0, + "completions/max_terminated_length": 155.0, + "completions/mean_length": 106.75, + "completions/mean_terminated_length": 106.75, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.3662136197090149, + "epoch": 0.685, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.508971214294434, + "learning_rate": 3.2e-07, + "loss": -0.11172202974557877, + "num_tokens": 1163334.0, + "reward": 53.0574951171875, + "reward_std": 58.36653518676758, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 52.957496643066406, + "rewards/supergames_reward/std": 58.36653518676758, + "step": 137, + "step_time": 5.8906258280039765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 175.0, + "completions/max_terminated_length": 175.0, + "completions/mean_length": 125.375, + "completions/mean_terminated_length": 125.375, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.3352622091770172, + "epoch": 0.69, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.1937079429626465, + "learning_rate": 3.15e-07, + "loss": 0.08307601511478424, + "num_tokens": 1168929.0, + "reward": 11.796250343322754, + "reward_std": 53.48870849609375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 11.696250915527344, + "rewards/supergames_reward/std": 53.48870849609375, + "step": 138, + "step_time": 6.595962718012743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 193.0, + "completions/mean_terminated_length": 193.0, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.5486535429954529, + "epoch": 0.695, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.861871242523193, + "learning_rate": 3.1e-07, + "loss": 0.0321456640958786, + "num_tokens": 1176937.0, + "reward": 50.463748931884766, + "reward_std": 70.37553405761719, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.38875198364258, + "rewards/supergames_reward/std": 70.31440734863281, + "step": 139, + "step_time": 9.544256388006033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 189.5, + "completions/mean_terminated_length": 189.5, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.42230603098869324, + "epoch": 0.7, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.830677032470703, + "learning_rate": 3.05e-07, + "loss": -0.07222295552492142, + "num_tokens": 1186789.0, + "reward": 33.44999694824219, + "reward_std": 67.1298599243164, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 33.349998474121094, + "rewards/supergames_reward/std": 67.1298599243164, + "step": 140, + "step_time": 10.002322309010196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 282.0, + "completions/max_terminated_length": 282.0, + "completions/mean_length": 197.375, + "completions/mean_terminated_length": 197.375, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.33588480949401855, + "epoch": 0.705, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.028353452682495, + "learning_rate": 3e-07, + "loss": -0.07543312013149261, + "num_tokens": 1196672.0, + "reward": 17.10249900817871, + "reward_std": 73.94812774658203, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 17.002500534057617, + "rewards/supergames_reward/std": 73.94813537597656, + "step": 141, + "step_time": 10.684560916008195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/max_terminated_length": 235.0, + "completions/mean_length": 209.25, + "completions/mean_terminated_length": 209.25, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "entropy": 0.3622831702232361, + "epoch": 0.71, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.047222137451172, + "learning_rate": 2.95e-07, + "loss": -0.04569581151008606, + "num_tokens": 1206658.0, + "reward": 49.57624816894531, + "reward_std": 78.21060180664062, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 49.476253509521484, + "rewards/supergames_reward/std": 78.21060943603516, + "step": 142, + "step_time": 8.97679864402744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 161.0, + "completions/max_terminated_length": 161.0, + "completions/mean_length": 116.25, + "completions/mean_terminated_length": 116.25, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.43277010321617126, + "epoch": 0.715, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.345203876495361, + "learning_rate": 2.9e-07, + "loss": -0.16899466514587402, + "num_tokens": 1212140.0, + "reward": 27.42875099182129, + "reward_std": 48.59138870239258, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.328750610351562, + "rewards/supergames_reward/std": 48.591392517089844, + "step": 143, + "step_time": 6.1606669370085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 205.0, + "completions/max_terminated_length": 205.0, + "completions/mean_length": 173.75, + "completions/mean_terminated_length": 173.75, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "entropy": 0.43806371092796326, + "epoch": 0.72, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.858840227127075, + "learning_rate": 2.8499999999999997e-07, + "loss": -0.06502802670001984, + "num_tokens": 1219954.0, + "reward": 38.69249725341797, + "reward_std": 68.78203582763672, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 38.592498779296875, + "rewards/supergames_reward/std": 68.78202819824219, + "step": 144, + "step_time": 7.88639413099736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 169.0, + "completions/max_terminated_length": 169.0, + "completions/mean_length": 110.625, + "completions/mean_terminated_length": 110.625, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.3383479118347168, + "epoch": 0.725, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.225733757019043, + "learning_rate": 2.8e-07, + "loss": -0.042058318853378296, + "num_tokens": 1225399.0, + "reward": 64.85874938964844, + "reward_std": 25.170387268066406, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 64.75875091552734, + "rewards/supergames_reward/std": 25.170391082763672, + "step": 145, + "step_time": 6.432729878026294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 147.0, + "completions/max_terminated_length": 147.0, + "completions/mean_length": 129.5, + "completions/mean_terminated_length": 129.5, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3088987171649933, + "epoch": 0.73, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.75e-07, + "loss": 0.0, + "num_tokens": 1231043.0, + "reward": 100.0999984741211, + "reward_std": 0.0, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 100.0, + "rewards/supergames_reward/std": 0.0, + "step": 146, + "step_time": 5.634907905012369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 160.0, + "completions/max_terminated_length": 160.0, + "completions/mean_length": 120.5, + "completions/mean_terminated_length": 120.5, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.37292322516441345, + "epoch": 0.735, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.174577713012695, + "learning_rate": 2.7e-07, + "loss": 0.11778222769498825, + "num_tokens": 1236607.0, + "reward": 34.82374572753906, + "reward_std": 58.513397216796875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 34.72374725341797, + "rewards/supergames_reward/std": 58.513397216796875, + "step": 147, + "step_time": 6.078690663998714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 172.0, + "completions/max_terminated_length": 172.0, + "completions/mean_length": 125.625, + "completions/mean_terminated_length": 125.625, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.308654248714447, + "epoch": 0.74, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8483362197875977, + "learning_rate": 2.65e-07, + "loss": -0.09181805700063705, + "num_tokens": 1242140.0, + "reward": 85.06375122070312, + "reward_std": 42.52893829345703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 84.9637451171875, + "rewards/supergames_reward/std": 42.52893829345703, + "step": 148, + "step_time": 6.501119122985983 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 275.0, + "completions/max_terminated_length": 275.0, + "completions/mean_length": 211.125, + "completions/mean_terminated_length": 211.125, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "entropy": 0.5405827164649963, + "epoch": 0.745, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.774259567260742, + "learning_rate": 2.6e-07, + "loss": -0.03056861087679863, + "num_tokens": 1250205.0, + "reward": 84.63125610351562, + "reward_std": 34.3358154296875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 84.53125, + "rewards/supergames_reward/std": 34.3358154296875, + "step": 149, + "step_time": 10.352729418984381 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 257.0, + "completions/max_terminated_length": 257.0, + "completions/mean_length": 218.5, + "completions/mean_terminated_length": 218.5, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "entropy": 0.3852894604206085, + "epoch": 0.75, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6103010177612305, + "learning_rate": 2.55e-07, + "loss": 0.034789010882377625, + "num_tokens": 1260249.0, + "reward": 81.81375122070312, + "reward_std": 51.72132873535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 81.7137451171875, + "rewards/supergames_reward/std": 51.72132873535156, + "step": 150, + "step_time": 9.937249842012534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 351.0, + "completions/max_terminated_length": 351.0, + "completions/mean_length": 232.625, + "completions/mean_terminated_length": 232.625, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.4323120415210724, + "epoch": 0.755, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.640308380126953, + "learning_rate": 2.5e-07, + "loss": -0.11181996762752533, + "num_tokens": 1271430.0, + "reward": 56.33625030517578, + "reward_std": 78.97452545166016, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 56.23625183105469, + "rewards/supergames_reward/std": 78.97452545166016, + "step": 151, + "step_time": 13.323547195002902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 359.0, + "completions/max_terminated_length": 359.0, + "completions/mean_length": 247.5, + "completions/mean_terminated_length": 247.5, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "entropy": 0.4038960039615631, + "epoch": 0.76, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.5704824924468994, + "learning_rate": 2.45e-07, + "loss": 0.060525521636009216, + "num_tokens": 1282722.0, + "reward": -68.53875732421875, + "reward_std": 50.590599060058594, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.63874816894531, + "rewards/supergames_reward/std": 50.590599060058594, + "step": 152, + "step_time": 13.593127576023107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 383.0, + "completions/max_terminated_length": 383.0, + "completions/mean_length": 246.375, + "completions/mean_terminated_length": 246.375, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "entropy": 0.4110933542251587, + "epoch": 0.765, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.233025312423706, + "learning_rate": 2.4e-07, + "loss": 0.06547188758850098, + "num_tokens": 1294061.0, + "reward": 95.91499328613281, + "reward_std": 11.836968421936035, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 95.81500244140625, + "rewards/supergames_reward/std": 11.836968421936035, + "step": 153, + "step_time": 14.499402174988063 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 338.0, + "completions/max_terminated_length": 338.0, + "completions/mean_length": 233.125, + "completions/mean_terminated_length": 233.125, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.34559860825538635, + "epoch": 0.77, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.6887102127075195, + "learning_rate": 2.3499999999999997e-07, + "loss": 0.15905122458934784, + "num_tokens": 1304262.0, + "reward": 88.92375183105469, + "reward_std": 31.611207962036133, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 88.82374572753906, + "rewards/supergames_reward/std": 31.6112117767334, + "step": 154, + "step_time": 12.704337224975461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 208.0, + "completions/max_terminated_length": 208.0, + "completions/mean_length": 178.5, + "completions/mean_terminated_length": 178.5, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "entropy": 0.4646885097026825, + "epoch": 0.775, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.161937236785889, + "learning_rate": 2.3e-07, + "loss": -0.04354584217071533, + "num_tokens": 1312098.0, + "reward": 27.80875015258789, + "reward_std": 60.81097412109375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.708751678466797, + "rewards/supergames_reward/std": 60.81097412109375, + "step": 155, + "step_time": 7.911955468007363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 391.0, + "completions/max_terminated_length": 391.0, + "completions/mean_length": 235.375, + "completions/mean_terminated_length": 235.375, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "entropy": 0.40985700488090515, + "epoch": 0.78, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.12153959274292, + "learning_rate": 2.25e-07, + "loss": -0.024320494383573532, + "num_tokens": 1322301.0, + "reward": 64.14624786376953, + "reward_std": 49.90530014038086, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 64.04624938964844, + "rewards/supergames_reward/std": 49.90530014038086, + "step": 156, + "step_time": 14.631677505996777 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/max_terminated_length": 330.0, + "completions/mean_length": 220.625, + "completions/mean_terminated_length": 220.625, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3543764054775238, + "epoch": 0.785, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0111091136932373, + "learning_rate": 2.1999999999999998e-07, + "loss": -0.13996481895446777, + "num_tokens": 1333386.0, + "reward": 3.6812496185302734, + "reward_std": 103.5096664428711, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 3.606250762939453, + "rewards/supergames_reward/std": 103.48104858398438, + "step": 157, + "step_time": 12.613216657977318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.0, + "completions/max_terminated_length": 225.0, + "completions/mean_length": 193.125, + "completions/mean_terminated_length": 193.125, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "entropy": 0.5682947635650635, + "epoch": 0.79, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.67927885055542, + "learning_rate": 2.1499999999999998e-07, + "loss": 0.0535324327647686, + "num_tokens": 1341355.0, + "reward": -19.78999900817871, + "reward_std": 69.72611236572266, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -19.889997482299805, + "rewards/supergames_reward/std": 69.72611236572266, + "step": 158, + "step_time": 8.585773958999198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.0, + "completions/max_terminated_length": 316.0, + "completions/mean_length": 237.625, + "completions/mean_terminated_length": 237.625, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "entropy": 0.3637359142303467, + "epoch": 0.795, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.033409357070923, + "learning_rate": 2.0999999999999997e-07, + "loss": 0.018884683027863503, + "num_tokens": 1352632.0, + "reward": 52.063751220703125, + "reward_std": 65.35551452636719, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 51.96375274658203, + "rewards/supergames_reward/std": 65.35551452636719, + "step": 159, + "step_time": 12.306456914986484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.0, + "completions/max_terminated_length": 225.0, + "completions/mean_length": 184.0, + "completions/mean_terminated_length": 184.0, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.47005388140678406, + "epoch": 0.8, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.585808515548706, + "learning_rate": 2.0499999999999997e-07, + "loss": -0.021751077845692635, + "num_tokens": 1360528.0, + "reward": 33.73124694824219, + "reward_std": 78.67134857177734, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 33.631248474121094, + "rewards/supergames_reward/std": 78.67134857177734, + "step": 160, + "step_time": 8.517202848976012 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 456.0, + "completions/max_terminated_length": 456.0, + "completions/mean_length": 248.75, + "completions/mean_terminated_length": 248.75, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.39807969331741333, + "epoch": 0.805, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.8931398391723633, + "learning_rate": 2e-07, + "loss": -0.05018138885498047, + "num_tokens": 1371854.0, + "reward": 4.002498626708984, + "reward_std": 103.03063201904297, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 3.9274978637695312, + "rewards/supergames_reward/std": 103.00178527832031, + "step": 161, + "step_time": 17.225145012984285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 400.0, + "completions/max_terminated_length": 400.0, + "completions/mean_length": 231.125, + "completions/mean_terminated_length": 231.125, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.49391740560531616, + "epoch": 0.81, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5471365451812744, + "learning_rate": 1.9499999999999999e-07, + "loss": -0.18011468648910522, + "num_tokens": 1383015.0, + "reward": 67.2449951171875, + "reward_std": 69.01223754882812, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 67.14500427246094, + "rewards/supergames_reward/std": 69.01224517822266, + "step": 162, + "step_time": 15.188760302000446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 262.0, + "completions/max_terminated_length": 262.0, + "completions/mean_length": 221.875, + "completions/mean_terminated_length": 221.875, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.388366162776947, + "epoch": 0.815, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.662609815597534, + "learning_rate": 1.8999999999999998e-07, + "loss": 1.8455075405654497e-05, + "num_tokens": 1393150.0, + "reward": 59.23624801635742, + "reward_std": 76.00599670410156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 59.13624954223633, + "rewards/supergames_reward/std": 76.00599670410156, + "step": 163, + "step_time": 10.180389262997778 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 323.625, + "completions/mean_terminated_length": 323.625, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.45717528462409973, + "epoch": 0.82, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.580068826675415, + "learning_rate": 1.85e-07, + "loss": 0.024594159796833992, + "num_tokens": 1405067.0, + "reward": 6.318748474121094, + "reward_std": 88.21497344970703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 6.21875, + "rewards/supergames_reward/std": 88.21497344970703, + "step": 164, + "step_time": 15.074436638999032 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 163.0, + "completions/max_terminated_length": 163.0, + "completions/mean_length": 133.875, + "completions/mean_terminated_length": 133.875, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.3449660539627075, + "epoch": 0.825, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.045867443084717, + "learning_rate": 1.8e-07, + "loss": -0.02517889253795147, + "num_tokens": 1410730.0, + "reward": 84.35499572753906, + "reward_std": 38.135921478271484, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 84.2550048828125, + "rewards/supergames_reward/std": 38.13592529296875, + "step": 165, + "step_time": 6.275273430015659 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 144.0, + "completions/max_terminated_length": 144.0, + "completions/mean_length": 105.875, + "completions/mean_terminated_length": 105.875, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "entropy": 0.286411851644516, + "epoch": 0.83, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.855896472930908, + "learning_rate": 1.75e-07, + "loss": 0.014497723430395126, + "num_tokens": 1416081.0, + "reward": 48.40374755859375, + "reward_std": 47.61183547973633, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.303749084472656, + "rewards/supergames_reward/std": 47.611839294433594, + "step": 166, + "step_time": 5.5562305669882335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 226.125, + "completions/mean_terminated_length": 226.125, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "entropy": 0.39676305651664734, + "epoch": 0.835, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.633286237716675, + "learning_rate": 1.7000000000000001e-07, + "loss": -0.008210637606680393, + "num_tokens": 1426194.0, + "reward": 67.51000213623047, + "reward_std": 58.9198112487793, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 67.41000366210938, + "rewards/supergames_reward/std": 58.9198112487793, + "step": 167, + "step_time": 10.435288048989605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 210.0, + "completions/max_terminated_length": 210.0, + "completions/mean_length": 168.5, + "completions/mean_terminated_length": 168.5, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.4233987629413605, + "epoch": 0.84, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.654274940490723, + "learning_rate": 1.65e-07, + "loss": -0.0319184847176075, + "num_tokens": 1433990.0, + "reward": 50.342498779296875, + "reward_std": 67.66273498535156, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.26750183105469, + "rewards/supergames_reward/std": 67.5992202758789, + "step": 168, + "step_time": 8.111304188991198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 397.0, + "completions/max_terminated_length": 397.0, + "completions/mean_length": 217.75, + "completions/mean_terminated_length": 217.75, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.41511332988739014, + "epoch": 0.845, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.156782150268555, + "learning_rate": 1.6e-07, + "loss": 0.30166593194007874, + "num_tokens": 1444068.0, + "reward": 48.64500045776367, + "reward_std": 51.50307846069336, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.54499816894531, + "rewards/supergames_reward/std": 51.50307846069336, + "step": 169, + "step_time": 14.915170635009417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/max_terminated_length": 245.0, + "completions/mean_length": 218.625, + "completions/mean_terminated_length": 218.625, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "entropy": 0.3981763422489166, + "epoch": 0.85, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.818580389022827, + "learning_rate": 1.55e-07, + "loss": -0.08915036171674728, + "num_tokens": 1454153.0, + "reward": 82.69249725341797, + "reward_std": 24.65846824645996, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 82.59249877929688, + "rewards/supergames_reward/std": 24.65846824645996, + "step": 170, + "step_time": 9.446723325003404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 386.0, + "completions/max_terminated_length": 386.0, + "completions/mean_length": 280.375, + "completions/mean_terminated_length": 280.375, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.39121532440185547, + "epoch": 0.855, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.067988634109497, + "learning_rate": 1.5e-07, + "loss": -0.28749391436576843, + "num_tokens": 1465692.0, + "reward": 66.02375030517578, + "reward_std": 70.814697265625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 65.92375183105469, + "rewards/supergames_reward/std": 70.814697265625, + "step": 171, + "step_time": 14.603269515006104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 188.0, + "completions/max_terminated_length": 188.0, + "completions/mean_length": 131.125, + "completions/mean_terminated_length": 131.125, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.435172438621521, + "epoch": 0.86, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.832774639129639, + "learning_rate": 1.45e-07, + "loss": -0.1204412579536438, + "num_tokens": 1471253.0, + "reward": 39.80875015258789, + "reward_std": 51.58554458618164, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 39.7087516784668, + "rewards/supergames_reward/std": 51.585548400878906, + "step": 172, + "step_time": 7.057335219986271 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 391.0, + "completions/max_terminated_length": 391.0, + "completions/mean_length": 271.375, + "completions/mean_terminated_length": 271.375, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "entropy": 0.4081321060657501, + "epoch": 0.865, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2880308628082275, + "learning_rate": 1.4e-07, + "loss": 0.03748488798737526, + "num_tokens": 1482744.0, + "reward": 53.62249755859375, + "reward_std": 75.1306381225586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 53.522499084472656, + "rewards/supergames_reward/std": 75.1306381225586, + "step": 173, + "step_time": 14.96632127999328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 304.0, + "completions/max_terminated_length": 304.0, + "completions/mean_length": 203.875, + "completions/mean_terminated_length": 203.875, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "entropy": 0.4152996242046356, + "epoch": 0.87, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.1565890312194824, + "learning_rate": 1.35e-07, + "loss": -0.047037553042173386, + "num_tokens": 1492679.0, + "reward": 37.48374938964844, + "reward_std": 72.71965789794922, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 37.383750915527344, + "rewards/supergames_reward/std": 72.71966552734375, + "step": 174, + "step_time": 11.566132662002929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/max_terminated_length": 187.0, + "completions/mean_length": 133.375, + "completions/mean_terminated_length": 133.375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.3746980130672455, + "epoch": 0.875, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.724977970123291, + "learning_rate": 1.3e-07, + "loss": 0.05197479575872421, + "num_tokens": 1498330.0, + "reward": 37.86625289916992, + "reward_std": 52.43158721923828, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 37.76625061035156, + "rewards/supergames_reward/std": 52.43158721923828, + "step": 175, + "step_time": 7.080777855007909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 448.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 277.25, + "completions/mean_terminated_length": 277.25, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "entropy": 0.397225022315979, + "epoch": 0.88, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.438502788543701, + "learning_rate": 1.25e-07, + "loss": 0.1022070124745369, + "num_tokens": 1509860.0, + "reward": -41.323753356933594, + "reward_std": 79.704345703125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -41.42374801635742, + "rewards/supergames_reward/std": 79.704345703125, + "step": 176, + "step_time": 16.852883081999607 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/max_terminated_length": 187.0, + "completions/mean_length": 145.875, + "completions/mean_terminated_length": 145.875, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.34882763028144836, + "epoch": 0.885, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.683617115020752, + "learning_rate": 1.2e-07, + "loss": 0.0609549917280674, + "num_tokens": 1515587.0, + "reward": 84.27999877929688, + "reward_std": 38.06671905517578, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 84.17999267578125, + "rewards/supergames_reward/std": 38.06672286987305, + "step": 177, + "step_time": 7.09286071601673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 217.0, + "completions/max_terminated_length": 217.0, + "completions/mean_length": 181.875, + "completions/mean_terminated_length": 181.875, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "entropy": 0.423494815826416, + "epoch": 0.89, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3993117809295654, + "learning_rate": 1.15e-07, + "loss": 0.04823008552193642, + "num_tokens": 1523490.0, + "reward": 22.46124839782715, + "reward_std": 89.67146301269531, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 22.361249923706055, + "rewards/supergames_reward/std": 89.67146301269531, + "step": 178, + "step_time": 8.281584940006724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/max_terminated_length": 315.0, + "completions/mean_length": 273.25, + "completions/mean_terminated_length": 273.25, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.3831624984741211, + "epoch": 0.895, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7383997440338135, + "learning_rate": 1.0999999999999999e-07, + "loss": -0.010000350885093212, + "num_tokens": 1535012.0, + "reward": 2.237499237060547, + "reward_std": 93.32353210449219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 2.137500762939453, + "rewards/supergames_reward/std": 93.32353973388672, + "step": 179, + "step_time": 12.098909514024854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 395.0, + "completions/max_terminated_length": 395.0, + "completions/mean_length": 197.375, + "completions/mean_terminated_length": 197.375, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.5201653242111206, + "epoch": 0.9, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4596073627471924, + "learning_rate": 1.0499999999999999e-07, + "loss": -0.11710439622402191, + "num_tokens": 1543023.0, + "reward": 88.1875, + "reward_std": 33.69363784790039, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 88.0875015258789, + "rewards/supergames_reward/std": 33.693641662597656, + "step": 180, + "step_time": 14.662479719001567 + } + ], + "logging_steps": 1, + "max_steps": 200, + "num_input_tokens_seen": 1543023, + "num_train_epochs": 1, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-180/training_args.bin b/checkpoint-180/training_args.bin new file mode 100644 index 0000000..730dec0 --- /dev/null +++ b/checkpoint-180/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691fe5be2d7d1d56160dae63866d136877a72efb30e15ff7b3249192f998a788 +size 7185 diff --git a/checkpoint-20/chat_template.jinja b/checkpoint-20/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/checkpoint-20/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-20/config.json b/checkpoint-20/config.json new file mode 100644 index 0000000..f57b09b --- /dev/null +++ b/checkpoint-20/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-20/generation_config.json b/checkpoint-20/generation_config.json new file mode 100644 index 0000000..1b2bba9 --- /dev/null +++ b/checkpoint-20/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/checkpoint-20/model.safetensors b/checkpoint-20/model.safetensors new file mode 100644 index 0000000..db32b2f --- /dev/null +++ b/checkpoint-20/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a39aec9f313a668fc041aff46341277464455362e1e1a310b26d9e9d14adf24d +size 6174895536 diff --git a/checkpoint-20/optimizer.pt b/checkpoint-20/optimizer.pt new file mode 100644 index 0000000..1dcbec2 --- /dev/null +++ b/checkpoint-20/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7700c459a5327b2ab9aee3a5bb3d23e73c8bb01230b33ba79014e18313adfef8 +size 12350013801 diff --git a/checkpoint-20/rng_state.pth b/checkpoint-20/rng_state.pth new file mode 100644 index 0000000..f9f8073 --- /dev/null +++ b/checkpoint-20/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93bf4adc8891698c782639b2c71621fcef18a7f4aa56009307e85e227fac0b8f +size 14645 diff --git a/checkpoint-20/scheduler.pt b/checkpoint-20/scheduler.pt new file mode 100644 index 0000000..1d7ac3e --- /dev/null +++ b/checkpoint-20/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6532ac530958ee6e596569b01cb0102818e20b71baecdf6e49fd861fbe7fab75 +size 1465 diff --git a/checkpoint-20/tokenizer.json b/checkpoint-20/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-20/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-20/tokenizer_config.json b/checkpoint-20/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-20/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-20/trainer_state.json b/checkpoint-20/trainer_state.json new file mode 100644 index 0000000..a4963a7 --- /dev/null +++ b/checkpoint-20/trainer_state.json @@ -0,0 +1,614 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.1, + "eval_steps": 500, + "global_step": 20, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 191.0, + "completions/mean_terminated_length": 191.0, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "entropy": 0.5566893219947815, + "epoch": 0.005, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.370361328125, + "learning_rate": 1e-06, + "loss": -0.24658073484897614, + "num_tokens": 10832.0, + "reward": -3.3000030517578125, + "reward_std": 85.62333679199219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.4000015258789062, + "rewards/supergames_reward/std": 85.62333679199219, + "step": 1, + "step_time": 22.779711072013015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/max_terminated_length": 216.0, + "completions/mean_length": 131.5, + "completions/mean_terminated_length": 131.5, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.6395841240882874, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.09774398803711, + "learning_rate": 9.95e-07, + "loss": 0.20567649602890015, + "num_tokens": 16404.0, + "reward": -12.422499656677246, + "reward_std": 7.134707450866699, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.522500038146973, + "rewards/supergames_reward/std": 7.134707927703857, + "step": 2, + "step_time": 9.960156448010821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/max_terminated_length": 287.0, + "completions/mean_length": 156.625, + "completions/mean_terminated_length": 156.625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "entropy": 0.562222421169281, + "epoch": 0.015, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9053616523742676, + "learning_rate": 9.9e-07, + "loss": 0.09602774679660797, + "num_tokens": 26953.0, + "reward": -74.94999694824219, + "reward_std": 70.73094177246094, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -75.0, + "rewards/supergames_reward/std": 70.71067810058594, + "step": 3, + "step_time": 13.686320498993155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 154.125, + "completions/mean_terminated_length": 154.125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7323317527770996, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.933310031890869, + "learning_rate": 9.849999999999999e-07, + "loss": 0.28249427676200867, + "num_tokens": 36514.0, + "reward": -40.7599983215332, + "reward_std": 81.73140716552734, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -40.80999755859375, + "rewards/supergames_reward/std": 81.68995666503906, + "step": 4, + "step_time": 12.805880262021674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 205.25, + "completions/mean_terminated_length": 205.25, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.6167430281639099, + "epoch": 0.025, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.344135046005249, + "learning_rate": 9.8e-07, + "loss": 0.1433994024991989, + "num_tokens": 47476.0, + "reward": -97.6612548828125, + "reward_std": 6.3321428298950195, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.76124572753906, + "rewards/supergames_reward/std": 6.3321428298950195, + "step": 5, + "step_time": 18.789364666008623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/max_terminated_length": 276.0, + "completions/mean_length": 167.25, + "completions/mean_terminated_length": 167.25, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.7583790421485901, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.172371864318848, + "learning_rate": 9.75e-07, + "loss": -0.12127404659986496, + "num_tokens": 55246.0, + "reward": -58.687503814697266, + "reward_std": 58.64059829711914, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -58.76250457763672, + "rewards/supergames_reward/std": 58.620460510253906, + "step": 6, + "step_time": 12.948570273991209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 133.875, + "completions/mean_terminated_length": 133.875, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.7447654604911804, + "epoch": 0.035, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.428595066070557, + "learning_rate": 9.7e-07, + "loss": 0.10135584324598312, + "num_tokens": 60837.0, + "reward": -32.45000076293945, + "reward_std": 41.754486083984375, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -32.5, + "rewards/supergames_reward/std": 41.66190719604492, + "step": 7, + "step_time": 12.058315072004916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 414.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 209.125, + "completions/mean_terminated_length": 209.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.3860666751861572, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.806255578994751, + "learning_rate": 9.649999999999999e-07, + "loss": -0.33238139748573303, + "num_tokens": 70838.0, + "reward": -80.50375366210938, + "reward_std": 38.82380294799805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -80.60375213623047, + "rewards/supergames_reward/std": 38.82379913330078, + "step": 8, + "step_time": 19.323370319994865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 247.125, + "completions/mean_terminated_length": 158.83334350585938, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.5816237330436707, + "epoch": 0.045, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3209943771362305, + "learning_rate": 9.6e-07, + "loss": 0.36752766370773315, + "num_tokens": 82135.0, + "reward": -27.688751220703125, + "reward_std": 82.8358154296875, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -27.738750457763672, + "rewards/supergames_reward/std": 82.78590393066406, + "step": 9, + "step_time": 21.948575104994234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/max_terminated_length": 240.0, + "completions/mean_length": 153.0, + "completions/mean_terminated_length": 153.0, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6301521062850952, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.125359058380127, + "learning_rate": 9.55e-07, + "loss": 0.06129350885748863, + "num_tokens": 89807.0, + "reward": -78.39624786376953, + "reward_std": 40.17042922973633, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -78.44625091552734, + "rewards/supergames_reward/std": 40.1396484375, + "step": 10, + "step_time": 9.098202993016457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/max_terminated_length": 133.0, + "completions/mean_length": 112.0, + "completions/mean_terminated_length": 112.0, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.5521990656852722, + "epoch": 0.055, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.616060256958008, + "learning_rate": 9.499999999999999e-07, + "loss": -0.006313305348157883, + "num_tokens": 95231.0, + "reward": -12.161249160766602, + "reward_std": 6.39578104019165, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.261249542236328, + "rewards/supergames_reward/std": 6.39578104019165, + "step": 11, + "step_time": 5.136311663984088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 214.875, + "completions/mean_terminated_length": 214.875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.5505905151367188, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.289583206176758, + "learning_rate": 9.45e-07, + "loss": -0.29802003502845764, + "num_tokens": 106238.0, + "reward": -91.66999816894531, + "reward_std": 23.2779541015625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -91.77000427246094, + "rewards/supergames_reward/std": 23.277956008911133, + "step": 12, + "step_time": 14.945365622988902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 167.375, + "completions/mean_terminated_length": 167.375, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "entropy": 0.540415346622467, + "epoch": 0.065, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.858291149139404, + "learning_rate": 9.399999999999999e-07, + "loss": 0.24004128575325012, + "num_tokens": 116937.0, + "reward": -20.38249969482422, + "reward_std": 87.55204010009766, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.457500457763672, + "rewards/supergames_reward/std": 87.52605438232422, + "step": 13, + "step_time": 10.482285185018554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "entropy": 0.7237679958343506, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.083980560302734, + "learning_rate": 9.35e-07, + "loss": -0.19025824964046478, + "num_tokens": 124640.0, + "reward": -51.397499084472656, + "reward_std": 54.486454010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -51.49749755859375, + "rewards/supergames_reward/std": 54.486454010009766, + "step": 14, + "step_time": 9.597231683001155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 141.375, + "completions/mean_terminated_length": 141.375, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.754334568977356, + "epoch": 0.075, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.720428943634033, + "learning_rate": 9.3e-07, + "loss": -0.16830675303936005, + "num_tokens": 132219.0, + "reward": -56.226253509521484, + "reward_std": 60.115657806396484, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -56.32625198364258, + "rewards/supergames_reward/std": 60.11566162109375, + "step": 15, + "step_time": 8.998362872982398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.0, + "completions/max_terminated_length": 209.0, + "completions/mean_length": 136.375, + "completions/mean_terminated_length": 136.375, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.6280568838119507, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.9681806564331055, + "learning_rate": 9.25e-07, + "loss": -0.01910916529595852, + "num_tokens": 139726.0, + "reward": -68.24000549316406, + "reward_std": 52.238914489746094, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.33999633789062, + "rewards/supergames_reward/std": 52.238914489746094, + "step": 16, + "step_time": 8.034480294008972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 211.875, + "completions/mean_terminated_length": 211.875, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.899176836013794, + "epoch": 0.085, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.691746711730957, + "learning_rate": 9.2e-07, + "loss": 0.25513291358947754, + "num_tokens": 147869.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 17, + "step_time": 13.655792869016295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 387.0, + "completions/max_terminated_length": 387.0, + "completions/mean_length": 238.0, + "completions/mean_terminated_length": 238.0, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.5596873760223389, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6711347103118896, + "learning_rate": 9.15e-07, + "loss": -0.2628335654735565, + "num_tokens": 159101.0, + "reward": -24.900001525878906, + "reward_std": 103.50983428955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.0, + "rewards/supergames_reward/std": 103.50983428955078, + "step": 18, + "step_time": 14.784329420013819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 277.0, + "completions/mean_length": 219.75, + "completions/mean_terminated_length": 178.00001525878906, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.9245517253875732, + "epoch": 0.095, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55502986907959, + "learning_rate": 9.1e-07, + "loss": 0.13060179352760315, + "num_tokens": 167299.0, + "reward": -86.89125061035156, + "reward_std": 36.95603561401367, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -86.94125366210938, + "rewards/supergames_reward/std": 36.93572235107422, + "step": 19, + "step_time": 18.832684700988466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/max_terminated_length": 267.0, + "completions/mean_length": 177.25, + "completions/mean_terminated_length": 177.25, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.7331254482269287, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.538497447967529, + "learning_rate": 9.05e-07, + "loss": -0.16418980062007904, + "num_tokens": 175141.0, + "reward": -68.52874755859375, + "reward_std": 58.48719787597656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.62875366210938, + "rewards/supergames_reward/std": 58.48720169067383, + "step": 20, + "step_time": 10.090975169994636 + } + ], + "logging_steps": 1, + "max_steps": 200, + "num_input_tokens_seen": 175141, + "num_train_epochs": 1, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-20/training_args.bin b/checkpoint-20/training_args.bin new file mode 100644 index 0000000..730dec0 --- /dev/null +++ b/checkpoint-20/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691fe5be2d7d1d56160dae63866d136877a72efb30e15ff7b3249192f998a788 +size 7185 diff --git a/checkpoint-200/chat_template.jinja b/checkpoint-200/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/checkpoint-200/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-200/config.json b/checkpoint-200/config.json new file mode 100644 index 0000000..f57b09b --- /dev/null +++ b/checkpoint-200/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-200/generation_config.json b/checkpoint-200/generation_config.json new file mode 100644 index 0000000..1b2bba9 --- /dev/null +++ b/checkpoint-200/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/checkpoint-200/model.safetensors b/checkpoint-200/model.safetensors new file mode 100644 index 0000000..73c5aaa --- /dev/null +++ b/checkpoint-200/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d768f5bced7c176c364bd8caa06f355a13ba8e84bfa6afc5000b0943d729a1b0 +size 6174895536 diff --git a/checkpoint-200/optimizer.pt b/checkpoint-200/optimizer.pt new file mode 100644 index 0000000..a9aa0ca --- /dev/null +++ b/checkpoint-200/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ecab9b19baa7f145e68200d06e20d1eabf0b812395b056a997f9ad356d423e9 +size 12350013801 diff --git a/checkpoint-200/rng_state.pth b/checkpoint-200/rng_state.pth new file mode 100644 index 0000000..bc226e1 --- /dev/null +++ b/checkpoint-200/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb4f2fcdc9d655fb6008b359f9ac492f3657acb42eb854239f64bcbd2e7ef3c8 +size 14645 diff --git a/checkpoint-200/scheduler.pt b/checkpoint-200/scheduler.pt new file mode 100644 index 0000000..f468c57 --- /dev/null +++ b/checkpoint-200/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62627619e905cc6929ad64a59b955d88cc2414468a59bffeba792e0b2b5af7d8 +size 1465 diff --git a/checkpoint-200/tokenizer.json b/checkpoint-200/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-200/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-200/tokenizer_config.json b/checkpoint-200/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-200/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-200/trainer_state.json b/checkpoint-200/trainer_state.json new file mode 100644 index 0000000..ef8c921 --- /dev/null +++ b/checkpoint-200/trainer_state.json @@ -0,0 +1,5834 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 191.0, + "completions/mean_terminated_length": 191.0, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "entropy": 0.5566893219947815, + "epoch": 0.005, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.370361328125, + "learning_rate": 1e-06, + "loss": -0.24658073484897614, + "num_tokens": 10832.0, + "reward": -3.3000030517578125, + "reward_std": 85.62333679199219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.4000015258789062, + "rewards/supergames_reward/std": 85.62333679199219, + "step": 1, + "step_time": 22.779711072013015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/max_terminated_length": 216.0, + "completions/mean_length": 131.5, + "completions/mean_terminated_length": 131.5, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.6395841240882874, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.09774398803711, + "learning_rate": 9.95e-07, + "loss": 0.20567649602890015, + "num_tokens": 16404.0, + "reward": -12.422499656677246, + "reward_std": 7.134707450866699, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.522500038146973, + "rewards/supergames_reward/std": 7.134707927703857, + "step": 2, + "step_time": 9.960156448010821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/max_terminated_length": 287.0, + "completions/mean_length": 156.625, + "completions/mean_terminated_length": 156.625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "entropy": 0.562222421169281, + "epoch": 0.015, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9053616523742676, + "learning_rate": 9.9e-07, + "loss": 0.09602774679660797, + "num_tokens": 26953.0, + "reward": -74.94999694824219, + "reward_std": 70.73094177246094, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -75.0, + "rewards/supergames_reward/std": 70.71067810058594, + "step": 3, + "step_time": 13.686320498993155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 154.125, + "completions/mean_terminated_length": 154.125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7323317527770996, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.933310031890869, + "learning_rate": 9.849999999999999e-07, + "loss": 0.28249427676200867, + "num_tokens": 36514.0, + "reward": -40.7599983215332, + "reward_std": 81.73140716552734, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -40.80999755859375, + "rewards/supergames_reward/std": 81.68995666503906, + "step": 4, + "step_time": 12.805880262021674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 205.25, + "completions/mean_terminated_length": 205.25, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.6167430281639099, + "epoch": 0.025, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.344135046005249, + "learning_rate": 9.8e-07, + "loss": 0.1433994024991989, + "num_tokens": 47476.0, + "reward": -97.6612548828125, + "reward_std": 6.3321428298950195, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.76124572753906, + "rewards/supergames_reward/std": 6.3321428298950195, + "step": 5, + "step_time": 18.789364666008623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/max_terminated_length": 276.0, + "completions/mean_length": 167.25, + "completions/mean_terminated_length": 167.25, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.7583790421485901, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.172371864318848, + "learning_rate": 9.75e-07, + "loss": -0.12127404659986496, + "num_tokens": 55246.0, + "reward": -58.687503814697266, + "reward_std": 58.64059829711914, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -58.76250457763672, + "rewards/supergames_reward/std": 58.620460510253906, + "step": 6, + "step_time": 12.948570273991209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 133.875, + "completions/mean_terminated_length": 133.875, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.7447654604911804, + "epoch": 0.035, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.428595066070557, + "learning_rate": 9.7e-07, + "loss": 0.10135584324598312, + "num_tokens": 60837.0, + "reward": -32.45000076293945, + "reward_std": 41.754486083984375, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -32.5, + "rewards/supergames_reward/std": 41.66190719604492, + "step": 7, + "step_time": 12.058315072004916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 414.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 209.125, + "completions/mean_terminated_length": 209.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.3860666751861572, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.806255578994751, + "learning_rate": 9.649999999999999e-07, + "loss": -0.33238139748573303, + "num_tokens": 70838.0, + "reward": -80.50375366210938, + "reward_std": 38.82380294799805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -80.60375213623047, + "rewards/supergames_reward/std": 38.82379913330078, + "step": 8, + "step_time": 19.323370319994865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 247.125, + "completions/mean_terminated_length": 158.83334350585938, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.5816237330436707, + "epoch": 0.045, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3209943771362305, + "learning_rate": 9.6e-07, + "loss": 0.36752766370773315, + "num_tokens": 82135.0, + "reward": -27.688751220703125, + "reward_std": 82.8358154296875, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -27.738750457763672, + "rewards/supergames_reward/std": 82.78590393066406, + "step": 9, + "step_time": 21.948575104994234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/max_terminated_length": 240.0, + "completions/mean_length": 153.0, + "completions/mean_terminated_length": 153.0, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6301521062850952, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.125359058380127, + "learning_rate": 9.55e-07, + "loss": 0.06129350885748863, + "num_tokens": 89807.0, + "reward": -78.39624786376953, + "reward_std": 40.17042922973633, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -78.44625091552734, + "rewards/supergames_reward/std": 40.1396484375, + "step": 10, + "step_time": 9.098202993016457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/max_terminated_length": 133.0, + "completions/mean_length": 112.0, + "completions/mean_terminated_length": 112.0, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.5521990656852722, + "epoch": 0.055, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.616060256958008, + "learning_rate": 9.499999999999999e-07, + "loss": -0.006313305348157883, + "num_tokens": 95231.0, + "reward": -12.161249160766602, + "reward_std": 6.39578104019165, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.261249542236328, + "rewards/supergames_reward/std": 6.39578104019165, + "step": 11, + "step_time": 5.136311663984088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 214.875, + "completions/mean_terminated_length": 214.875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.5505905151367188, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.289583206176758, + "learning_rate": 9.45e-07, + "loss": -0.29802003502845764, + "num_tokens": 106238.0, + "reward": -91.66999816894531, + "reward_std": 23.2779541015625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -91.77000427246094, + "rewards/supergames_reward/std": 23.277956008911133, + "step": 12, + "step_time": 14.945365622988902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 167.375, + "completions/mean_terminated_length": 167.375, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "entropy": 0.540415346622467, + "epoch": 0.065, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.858291149139404, + "learning_rate": 9.399999999999999e-07, + "loss": 0.24004128575325012, + "num_tokens": 116937.0, + "reward": -20.38249969482422, + "reward_std": 87.55204010009766, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.457500457763672, + "rewards/supergames_reward/std": 87.52605438232422, + "step": 13, + "step_time": 10.482285185018554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "entropy": 0.7237679958343506, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.083980560302734, + "learning_rate": 9.35e-07, + "loss": -0.19025824964046478, + "num_tokens": 124640.0, + "reward": -51.397499084472656, + "reward_std": 54.486454010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -51.49749755859375, + "rewards/supergames_reward/std": 54.486454010009766, + "step": 14, + "step_time": 9.597231683001155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 141.375, + "completions/mean_terminated_length": 141.375, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.754334568977356, + "epoch": 0.075, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.720428943634033, + "learning_rate": 9.3e-07, + "loss": -0.16830675303936005, + "num_tokens": 132219.0, + "reward": -56.226253509521484, + "reward_std": 60.115657806396484, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -56.32625198364258, + "rewards/supergames_reward/std": 60.11566162109375, + "step": 15, + "step_time": 8.998362872982398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.0, + "completions/max_terminated_length": 209.0, + "completions/mean_length": 136.375, + "completions/mean_terminated_length": 136.375, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.6280568838119507, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.9681806564331055, + "learning_rate": 9.25e-07, + "loss": -0.01910916529595852, + "num_tokens": 139726.0, + "reward": -68.24000549316406, + "reward_std": 52.238914489746094, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.33999633789062, + "rewards/supergames_reward/std": 52.238914489746094, + "step": 16, + "step_time": 8.034480294008972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 211.875, + "completions/mean_terminated_length": 211.875, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.899176836013794, + "epoch": 0.085, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.691746711730957, + "learning_rate": 9.2e-07, + "loss": 0.25513291358947754, + "num_tokens": 147869.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 17, + "step_time": 13.655792869016295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 387.0, + "completions/max_terminated_length": 387.0, + "completions/mean_length": 238.0, + "completions/mean_terminated_length": 238.0, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.5596873760223389, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6711347103118896, + "learning_rate": 9.15e-07, + "loss": -0.2628335654735565, + "num_tokens": 159101.0, + "reward": -24.900001525878906, + "reward_std": 103.50983428955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.0, + "rewards/supergames_reward/std": 103.50983428955078, + "step": 18, + "step_time": 14.784329420013819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 277.0, + "completions/mean_length": 219.75, + "completions/mean_terminated_length": 178.00001525878906, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.9245517253875732, + "epoch": 0.095, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55502986907959, + "learning_rate": 9.1e-07, + "loss": 0.13060179352760315, + "num_tokens": 167299.0, + "reward": -86.89125061035156, + "reward_std": 36.95603561401367, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -86.94125366210938, + "rewards/supergames_reward/std": 36.93572235107422, + "step": 19, + "step_time": 18.832684700988466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/max_terminated_length": 267.0, + "completions/mean_length": 177.25, + "completions/mean_terminated_length": 177.25, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.7331254482269287, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.538497447967529, + "learning_rate": 9.05e-07, + "loss": -0.16418980062007904, + "num_tokens": 175141.0, + "reward": -68.52874755859375, + "reward_std": 58.48719787597656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.62875366210938, + "rewards/supergames_reward/std": 58.48720169067383, + "step": 20, + "step_time": 10.090975169994636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 190.625, + "completions/mean_terminated_length": 190.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.6659425497055054, + "epoch": 0.105, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.485373497009277, + "learning_rate": 9e-07, + "loss": -0.10106432437896729, + "num_tokens": 183098.0, + "reward": -71.48875427246094, + "reward_std": 55.33565139770508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -71.5887451171875, + "rewards/supergames_reward/std": 55.33565139770508, + "step": 21, + "step_time": 11.255001295008697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 179.0, + "completions/mean_terminated_length": 179.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.49365949630737305, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.829419136047363, + "learning_rate": 8.95e-07, + "loss": 0.10587231814861298, + "num_tokens": 192826.0, + "reward": -75.62000274658203, + "reward_std": 45.01959991455078, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -75.69499969482422, + "rewards/supergames_reward/std": 45.00411605834961, + "step": 22, + "step_time": 10.611246599000879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/max_terminated_length": 315.0, + "completions/mean_length": 212.625, + "completions/mean_terminated_length": 212.625, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.43159469962120056, + "epoch": 0.115, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.541125774383545, + "learning_rate": 8.9e-07, + "loss": -0.1681259125471115, + "num_tokens": 202815.0, + "reward": 23.67624855041504, + "reward_std": 67.74340057373047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 23.576250076293945, + "rewards/supergames_reward/std": 67.743408203125, + "step": 23, + "step_time": 11.731771531980485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/max_terminated_length": 168.0, + "completions/mean_length": 101.625, + "completions/mean_terminated_length": 101.625, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.3972298502922058, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.8285417556762695, + "learning_rate": 8.85e-07, + "loss": -0.011585958302021027, + "num_tokens": 208188.0, + "reward": 58.849998474121094, + "reward_std": 56.9304084777832, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 58.75, + "rewards/supergames_reward/std": 56.9304084777832, + "step": 24, + "step_time": 6.187504950998118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 303.0, + "completions/max_terminated_length": 303.0, + "completions/mean_length": 165.375, + "completions/mean_terminated_length": 165.375, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "entropy": 0.6339899301528931, + "epoch": 0.125, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.823799133300781, + "learning_rate": 8.799999999999999e-07, + "loss": 0.05159185826778412, + "num_tokens": 215935.0, + "reward": -23.521251678466797, + "reward_std": 74.23067474365234, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -23.621248245239258, + "rewards/supergames_reward/std": 74.23067474365234, + "step": 25, + "step_time": 11.059416300005978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 349.0, + "completions/max_terminated_length": 349.0, + "completions/mean_length": 204.0, + "completions/mean_terminated_length": 204.0, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.6462895274162292, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.969292640686035, + "learning_rate": 8.75e-07, + "loss": 0.0115677984431386, + "num_tokens": 223999.0, + "reward": -74.8175048828125, + "reward_std": 40.94221115112305, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -74.89250183105469, + "rewards/supergames_reward/std": 40.924625396728516, + "step": 26, + "step_time": 12.666237785975682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 166.5, + "completions/mean_terminated_length": 166.5, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4756850600242615, + "epoch": 0.135, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915069103240967, + "learning_rate": 8.699999999999999e-07, + "loss": -0.13321346044540405, + "num_tokens": 233643.0, + "reward": -52.125, + "reward_std": 19.405282974243164, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -52.20000076293945, + "rewards/supergames_reward/std": 19.33464813232422, + "step": 27, + "step_time": 11.172539752995363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 137.125, + "completions/mean_terminated_length": 137.125, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.5834778547286987, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.146666526794434, + "learning_rate": 8.65e-07, + "loss": 0.060454584658145905, + "num_tokens": 241204.0, + "reward": -68.99250793457031, + "reward_std": 51.010581970214844, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -69.09249877929688, + "rewards/supergames_reward/std": 51.010581970214844, + "step": 28, + "step_time": 7.025046669004951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 182.375, + "completions/mean_terminated_length": 182.375, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.6368551254272461, + "epoch": 0.145, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.15535306930542, + "learning_rate": 8.599999999999999e-07, + "loss": -0.3323673903942108, + "num_tokens": 249087.0, + "reward": -44.17000198364258, + "reward_std": 80.5373306274414, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -44.27000045776367, + "rewards/supergames_reward/std": 80.5373306274414, + "step": 29, + "step_time": 12.079259724996518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 127.75, + "completions/mean_terminated_length": 127.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4491196870803833, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.896186828613281, + "learning_rate": 8.55e-07, + "loss": -0.12887656688690186, + "num_tokens": 259445.0, + "reward": 57.446250915527344, + "reward_std": 38.78837585449219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 57.34625244140625, + "rewards/supergames_reward/std": 38.78837966918945, + "step": 30, + "step_time": 7.483972909016302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/max_terminated_length": 166.0, + "completions/mean_length": 112.875, + "completions/mean_terminated_length": 112.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4858454465866089, + "epoch": 0.155, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.742906093597412, + "learning_rate": 8.499999999999999e-07, + "loss": 0.06930096447467804, + "num_tokens": 264924.0, + "reward": -21.174999237060547, + "reward_std": 31.89051628112793, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -21.25, + "rewards/supergames_reward/std": 31.819807052612305, + "step": 31, + "step_time": 6.1866529019898735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/max_terminated_length": 148.0, + "completions/mean_length": 118.125, + "completions/mean_terminated_length": 118.125, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.45428749918937683, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.663414478302002, + "learning_rate": 8.45e-07, + "loss": 0.05735419690608978, + "num_tokens": 270381.0, + "reward": 0.4687504768371582, + "reward_std": 38.66371154785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 0.36875057220458984, + "rewards/supergames_reward/std": 38.6637077331543, + "step": 32, + "step_time": 5.577042757999152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 166.875, + "completions/mean_terminated_length": 166.875, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.6107293367385864, + "epoch": 0.165, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.75006628036499, + "learning_rate": 8.399999999999999e-07, + "loss": -0.14760127663612366, + "num_tokens": 278156.0, + "reward": -87.84750366210938, + "reward_std": 26.84708023071289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -87.94749450683594, + "rewards/supergames_reward/std": 26.847076416015625, + "step": 33, + "step_time": 12.552876825997373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 360.0, + "completions/mean_length": 231.0, + "completions/mean_terminated_length": 190.85714721679688, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.6089653372764587, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.781267166137695, + "learning_rate": 8.349999999999999e-07, + "loss": 0.4294504225254059, + "num_tokens": 289364.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 34, + "step_time": 18.961819477990502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 134.125, + "completions/mean_terminated_length": 134.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46158918738365173, + "epoch": 0.175, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.016353607177734, + "learning_rate": 8.299999999999999e-07, + "loss": 0.09251818805932999, + "num_tokens": 298733.0, + "reward": -35.558753967285156, + "reward_std": 15.881204605102539, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -35.65875244140625, + "rewards/supergames_reward/std": 15.881203651428223, + "step": 35, + "step_time": 8.38491243700264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/max_terminated_length": 207.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.5104788541793823, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.592893123626709, + "learning_rate": 8.249999999999999e-07, + "loss": -0.0410832017660141, + "num_tokens": 306420.0, + "reward": 5.4212493896484375, + "reward_std": 93.50404357910156, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 5.346250534057617, + "rewards/supergames_reward/std": 93.4718246459961, + "step": 36, + "step_time": 7.811868985998444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/max_terminated_length": 241.0, + "completions/mean_length": 178.0, + "completions/mean_terminated_length": 178.0, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.7527878880500793, + "epoch": 0.185, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8812546730041504, + "learning_rate": 8.199999999999999e-07, + "loss": 0.12539231777191162, + "num_tokens": 314252.0, + "reward": -89.58500671386719, + "reward_std": 29.256120681762695, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -89.66000366210938, + "rewards/supergames_reward/std": 29.24593734741211, + "step": 37, + "step_time": 8.959742914012168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 133.375, + "completions/mean_terminated_length": 133.375, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.3347901403903961, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.205414295196533, + "learning_rate": 8.149999999999999e-07, + "loss": -0.06991486996412277, + "num_tokens": 319839.0, + "reward": -14.498749732971191, + "reward_std": 13.007229804992676, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -14.598750114440918, + "rewards/supergames_reward/std": 13.007229804992676, + "step": 38, + "step_time": 7.377183554985095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/max_terminated_length": 181.0, + "completions/mean_length": 114.375, + "completions/mean_terminated_length": 114.375, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.4445965886116028, + "epoch": 0.195, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.561298847198486, + "learning_rate": 8.1e-07, + "loss": -0.06235164776444435, + "num_tokens": 325290.0, + "reward": -7.425000190734863, + "reward_std": 53.701602935791016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -7.5, + "rewards/supergames_reward/std": 53.652320861816406, + "step": 39, + "step_time": 6.753862089011818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 187.625, + "completions/mean_terminated_length": 187.625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.3950234353542328, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5988712310791016, + "learning_rate": 8.05e-07, + "loss": -0.1381440907716751, + "num_tokens": 335119.0, + "reward": -33.68375015258789, + "reward_std": 42.069435119628906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -33.783748626708984, + "rewards/supergames_reward/std": 42.069435119628906, + "step": 40, + "step_time": 13.718958162004128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 371.0, + "completions/mean_length": 251.125, + "completions/mean_terminated_length": 213.85714721679688, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.507067084312439, + "epoch": 0.205, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3898658752441406, + "learning_rate": 8e-07, + "loss": 0.0078964838758111, + "num_tokens": 346464.0, + "reward": -86.48625183105469, + "reward_std": 38.020694732666016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -86.56124877929688, + "rewards/supergames_reward/std": 38.010528564453125, + "step": 41, + "step_time": 18.90960379401804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 152.0, + "completions/max_terminated_length": 152.0, + "completions/mean_length": 116.375, + "completions/mean_terminated_length": 116.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3801707923412323, + "epoch": 0.21, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.95e-07, + "loss": 0.0, + "num_tokens": 351955.0, + "reward": -9.899999618530273, + "reward_std": 0.0, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -10.0, + "rewards/supergames_reward/std": 0.0, + "step": 42, + "step_time": 5.711630532023264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/max_terminated_length": 308.0, + "completions/mean_length": 247.375, + "completions/mean_terminated_length": 247.375, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.3185043931007385, + "epoch": 0.215, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.168762445449829, + "learning_rate": 7.9e-07, + "loss": 0.1970764398574829, + "num_tokens": 363278.0, + "reward": -53.89249801635742, + "reward_std": 63.02619934082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -53.99250030517578, + "rewards/supergames_reward/std": 63.02619934082031, + "step": 43, + "step_time": 11.661934906995157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/max_terminated_length": 330.0, + "completions/mean_length": 212.5, + "completions/mean_terminated_length": 212.5, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4446268379688263, + "epoch": 0.22, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.305844306945801, + "learning_rate": 7.85e-07, + "loss": -0.06735197454690933, + "num_tokens": 374306.0, + "reward": 18.059999465942383, + "reward_std": 98.9417953491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 17.959999084472656, + "rewards/supergames_reward/std": 98.9417953491211, + "step": 44, + "step_time": 12.492453911982011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.0, + "completions/max_terminated_length": 316.0, + "completions/mean_length": 183.0, + "completions/mean_terminated_length": 183.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.5269614458084106, + "epoch": 0.225, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8006370067596436, + "learning_rate": 7.799999999999999e-07, + "loss": -0.08870815485715866, + "num_tokens": 382210.0, + "reward": -54.928749084472656, + "reward_std": 72.5127944946289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -55.02874755859375, + "rewards/supergames_reward/std": 72.5127944946289, + "step": 45, + "step_time": 11.7636467939883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.0, + "completions/max_terminated_length": 244.0, + "completions/mean_length": 136.625, + "completions/mean_terminated_length": 136.625, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.34820204973220825, + "epoch": 0.23, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.911736011505127, + "learning_rate": 7.75e-07, + "loss": 0.04397330805659294, + "num_tokens": 391583.0, + "reward": -19.10375213623047, + "reward_std": 23.967702865600586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -19.203750610351562, + "rewards/supergames_reward/std": 23.967702865600586, + "step": 46, + "step_time": 10.119426151999505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 160.375, + "completions/mean_terminated_length": 160.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.611914336681366, + "epoch": 0.235, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.571913242340088, + "learning_rate": 7.699999999999999e-07, + "loss": -0.044293053448200226, + "num_tokens": 399290.0, + "reward": -25.186250686645508, + "reward_std": 71.46305084228516, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.286251068115234, + "rewards/supergames_reward/std": 71.46305084228516, + "step": 47, + "step_time": 8.563868903991533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/max_terminated_length": 167.0, + "completions/mean_length": 111.125, + "completions/mean_terminated_length": 111.125, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.46585777401924133, + "epoch": 0.24, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.654109954833984, + "learning_rate": 7.65e-07, + "loss": -0.035151124000549316, + "num_tokens": 404707.0, + "reward": -0.408750057220459, + "reward_std": 31.939748764038086, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -0.5087499618530273, + "rewards/supergames_reward/std": 31.93975257873535, + "step": 48, + "step_time": 6.438482580007985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/max_terminated_length": 306.0, + "completions/mean_length": 216.625, + "completions/mean_terminated_length": 216.625, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.33054471015930176, + "epoch": 0.245, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.700303792953491, + "learning_rate": 7.599999999999999e-07, + "loss": -0.04393656551837921, + "num_tokens": 414736.0, + "reward": -97.26000213623047, + "reward_std": 1.6294406652450562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.36000061035156, + "rewards/supergames_reward/std": 1.629441738128662, + "step": 49, + "step_time": 11.636040106008295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 378.0, + "completions/max_terminated_length": 378.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.48711177706718445, + "epoch": 0.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9650564193725586, + "learning_rate": 7.55e-07, + "loss": -0.3529520332813263, + "num_tokens": 424795.0, + "reward": 47.408748626708984, + "reward_std": 72.72083282470703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 47.30875015258789, + "rewards/supergames_reward/std": 72.72083282470703, + "step": 50, + "step_time": 14.258096842997475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 267.625, + "completions/mean_terminated_length": 232.71429443359375, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4482860863208771, + "epoch": 0.255, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2998197078704834, + "learning_rate": 7.5e-07, + "loss": 0.2493654191493988, + "num_tokens": 436288.0, + "reward": 50.61125183105469, + "reward_std": 75.3349838256836, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.53624725341797, + "rewards/supergames_reward/std": 75.27783966064453, + "step": 51, + "step_time": 19.20827590499539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 176.0, + "completions/max_terminated_length": 176.0, + "completions/mean_length": 144.5, + "completions/mean_terminated_length": 144.5, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "entropy": 0.5378735661506653, + "epoch": 0.26, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.114461898803711, + "learning_rate": 7.45e-07, + "loss": 0.0036597400903701782, + "num_tokens": 441980.0, + "reward": -15.20625114440918, + "reward_std": 45.35526657104492, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.28125, + "rewards/supergames_reward/std": 45.301815032958984, + "step": 52, + "step_time": 6.660627231001854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/max_terminated_length": 136.0, + "completions/mean_length": 119.0, + "completions/mean_terminated_length": 119.0, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "entropy": 0.41796669363975525, + "epoch": 0.265, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.059505462646484, + "learning_rate": 7.4e-07, + "loss": -0.014973883517086506, + "num_tokens": 447468.0, + "reward": 15.170000076293945, + "reward_std": 46.658329010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 15.070000648498535, + "rewards/supergames_reward/std": 46.658329010009766, + "step": 53, + "step_time": 5.271571868011961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/max_terminated_length": 259.0, + "completions/mean_length": 191.125, + "completions/mean_terminated_length": 191.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.45421987771987915, + "epoch": 0.27, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4454867839813232, + "learning_rate": 7.35e-07, + "loss": -0.1128973588347435, + "num_tokens": 457301.0, + "reward": 38.89875030517578, + "reward_std": 62.1772575378418, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 38.79874801635742, + "rewards/supergames_reward/std": 62.1772575378418, + "step": 54, + "step_time": 10.524528659996577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/max_terminated_length": 319.0, + "completions/mean_length": 233.375, + "completions/mean_terminated_length": 233.375, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "entropy": 0.4449213445186615, + "epoch": 0.275, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2065398693084717, + "learning_rate": 7.3e-07, + "loss": 0.0016277075046673417, + "num_tokens": 467520.0, + "reward": 27.372499465942383, + "reward_std": 81.23126983642578, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.27250099182129, + "rewards/supergames_reward/std": 81.23126983642578, + "step": 55, + "step_time": 12.592280682991259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 357.0, + "completions/mean_length": 261.0, + "completions/mean_terminated_length": 225.1428680419922, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.506920337677002, + "epoch": 0.28, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.058388710021973, + "learning_rate": 7.249999999999999e-07, + "loss": 0.3289242386817932, + "num_tokens": 476056.0, + "reward": 47.05249786376953, + "reward_std": 80.97753143310547, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 46.977500915527344, + "rewards/supergames_reward/std": 80.9256362915039, + "step": 56, + "step_time": 18.845177220006008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.0, + "completions/max_terminated_length": 250.0, + "completions/mean_length": 192.75, + "completions/mean_terminated_length": 192.75, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "entropy": 0.5197336077690125, + "epoch": 0.285, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.230815410614014, + "learning_rate": 7.2e-07, + "loss": -0.15186546742916107, + "num_tokens": 484014.0, + "reward": -64.5050048828125, + "reward_std": 50.35145568847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -64.60499572753906, + "rewards/supergames_reward/std": 50.3514518737793, + "step": 57, + "step_time": 9.576232638006331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 249.0, + "completions/mean_terminated_length": 249.0, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "entropy": 0.42913514375686646, + "epoch": 0.29, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.8544528484344482, + "learning_rate": 7.149999999999999e-07, + "loss": -0.056405920535326004, + "num_tokens": 494390.0, + "reward": 40.123748779296875, + "reward_std": 84.164794921875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 40.02375030517578, + "rewards/supergames_reward/std": 84.164794921875, + "step": 58, + "step_time": 12.385353341000155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 138.0, + "completions/max_terminated_length": 138.0, + "completions/mean_length": 107.25, + "completions/mean_terminated_length": 107.25, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.34386488795280457, + "epoch": 0.295, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.6915154457092285, + "learning_rate": 7.1e-07, + "loss": 0.0865321159362793, + "num_tokens": 499816.0, + "reward": -8.244999885559082, + "reward_std": 4.681046962738037, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.345000267028809, + "rewards/supergames_reward/std": 4.681046962738037, + "step": 59, + "step_time": 5.42325339600211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/max_terminated_length": 187.0, + "completions/mean_length": 127.25, + "completions/mean_terminated_length": 127.25, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.5430045127868652, + "epoch": 0.3, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.635311126708984, + "learning_rate": 7.049999999999999e-07, + "loss": -0.05412375554442406, + "num_tokens": 505410.0, + "reward": 20.005001068115234, + "reward_std": 55.156005859375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 19.904998779296875, + "rewards/supergames_reward/std": 55.156005859375, + "step": 60, + "step_time": 7.068010902003152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/max_terminated_length": 286.0, + "completions/mean_length": 192.125, + "completions/mean_terminated_length": 192.125, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4213869869709015, + "epoch": 0.305, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4664618968963623, + "learning_rate": 7e-07, + "loss": -0.0495159812271595, + "num_tokens": 516283.0, + "reward": -47.212501525878906, + "reward_std": 91.2253646850586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -47.3125, + "rewards/supergames_reward/std": 91.2253646850586, + "step": 61, + "step_time": 11.176304235996213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 171.0, + "completions/mean_terminated_length": 171.0, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.4626811146736145, + "epoch": 0.31, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.949252128601074, + "learning_rate": 6.949999999999999e-07, + "loss": 0.192047581076622, + "num_tokens": 522251.0, + "reward": -15.092500686645508, + "reward_std": 49.82962417602539, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.167499542236328, + "rewards/supergames_reward/std": 49.78091049194336, + "step": 62, + "step_time": 12.075224861997413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 429.0, + "completions/mean_length": 283.5, + "completions/mean_terminated_length": 250.85714721679688, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4814216196537018, + "epoch": 0.315, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7918753623962402, + "learning_rate": 6.9e-07, + "loss": 0.15706156194210052, + "num_tokens": 533815.0, + "reward": -20.86625099182129, + "reward_std": 85.99250030517578, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.94124984741211, + "rewards/supergames_reward/std": 85.96620178222656, + "step": 63, + "step_time": 19.448832260008203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.0, + "completions/max_terminated_length": 189.0, + "completions/mean_length": 131.25, + "completions/mean_terminated_length": 131.25, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4929414987564087, + "epoch": 0.32, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.774101734161377, + "learning_rate": 6.85e-07, + "loss": -0.060691747814416885, + "num_tokens": 539465.0, + "reward": -21.0674991607666, + "reward_std": 6.892730236053467, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -21.167499542236328, + "rewards/supergames_reward/std": 6.892730236053467, + "step": 64, + "step_time": 7.141569407976931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 454.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 281.375, + "completions/mean_terminated_length": 281.375, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "entropy": 0.45866572856903076, + "epoch": 0.325, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.092717409133911, + "learning_rate": 6.800000000000001e-07, + "loss": 0.11694959551095963, + "num_tokens": 551012.0, + "reward": 49.938751220703125, + "reward_std": 79.1865005493164, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 49.8387451171875, + "rewards/supergames_reward/std": 79.1865005493164, + "step": 65, + "step_time": 17.06170882002334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/max_terminated_length": 203.0, + "completions/mean_length": 128.625, + "completions/mean_terminated_length": 128.625, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.5005782246589661, + "epoch": 0.33, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.77616024017334, + "learning_rate": 6.75e-07, + "loss": 0.11242837458848953, + "num_tokens": 556577.0, + "reward": 48.875, + "reward_std": 49.13539123535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.775001525878906, + "rewards/supergames_reward/std": 49.13539123535156, + "step": 66, + "step_time": 7.695410129002994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 436.0, + "completions/max_terminated_length": 436.0, + "completions/mean_length": 281.875, + "completions/mean_terminated_length": 281.875, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "entropy": 0.507097065448761, + "epoch": 0.335, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.911184787750244, + "learning_rate": 6.7e-07, + "loss": -0.08151137083768845, + "num_tokens": 567120.0, + "reward": -12.886249542236328, + "reward_std": 84.04269409179688, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.986251831054688, + "rewards/supergames_reward/std": 84.04269409179688, + "step": 67, + "step_time": 16.409127942984924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 318.375, + "completions/mean_terminated_length": 290.71429443359375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.486402690410614, + "epoch": 0.34, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9461607933044434, + "learning_rate": 6.65e-07, + "loss": 0.2442178875207901, + "num_tokens": 578995.0, + "reward": 2.8387489318847656, + "reward_std": 98.47579956054688, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 2.7637500762939453, + "rewards/supergames_reward/std": 98.44595336914062, + "step": 68, + "step_time": 19.32364999302081 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 291.0, + "completions/max_terminated_length": 291.0, + "completions/mean_length": 219.625, + "completions/mean_terminated_length": 219.625, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.37856727838516235, + "epoch": 0.345, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0204732418060303, + "learning_rate": 6.6e-07, + "loss": -0.08764916658401489, + "num_tokens": 589072.0, + "reward": -54.18375015258789, + "reward_std": 71.45012664794922, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -54.283748626708984, + "rewards/supergames_reward/std": 71.45013427734375, + "step": 69, + "step_time": 11.126611230982235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 305.0, + "completions/mean_length": 254.75, + "completions/mean_terminated_length": 218.00001525878906, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46423614025115967, + "epoch": 0.35, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0140154361724854, + "learning_rate": 6.55e-07, + "loss": 0.26797160506248474, + "num_tokens": 600438.0, + "reward": -32.04875183105469, + "reward_std": 69.86957550048828, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -32.12375259399414, + "rewards/supergames_reward/std": 69.84178161621094, + "step": 70, + "step_time": 19.12617294798838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 372.0, + "completions/max_terminated_length": 372.0, + "completions/mean_length": 194.125, + "completions/mean_terminated_length": 194.125, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.3427293300628662, + "epoch": 0.355, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.781383991241455, + "learning_rate": 6.5e-07, + "loss": 0.354524701833725, + "num_tokens": 610327.0, + "reward": -6.913749694824219, + "reward_std": 87.56071472167969, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -6.988750457763672, + "rewards/supergames_reward/std": 87.53033447265625, + "step": 71, + "step_time": 14.041668199002743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/max_terminated_length": 197.0, + "completions/mean_length": 124.75, + "completions/mean_terminated_length": 124.75, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.4455118477344513, + "epoch": 0.36, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.726032733917236, + "learning_rate": 6.45e-07, + "loss": 0.18206289410591125, + "num_tokens": 615853.0, + "reward": 41.95124816894531, + "reward_std": 60.640811920166016, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 41.85124969482422, + "rewards/supergames_reward/std": 60.640811920166016, + "step": 72, + "step_time": 7.416493425989756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.0, + "completions/max_terminated_length": 323.0, + "completions/mean_length": 147.375, + "completions/mean_terminated_length": 147.375, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.480733186006546, + "epoch": 0.365, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.853621482849121, + "learning_rate": 6.4e-07, + "loss": 0.04558124020695686, + "num_tokens": 621584.0, + "reward": 14.884998321533203, + "reward_std": 46.25857162475586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 14.78499984741211, + "rewards/supergames_reward/std": 46.25857162475586, + "step": 73, + "step_time": 11.95379540900467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 283.0, + "completions/max_terminated_length": 283.0, + "completions/mean_length": 188.125, + "completions/mean_terminated_length": 188.125, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.5526013374328613, + "epoch": 0.37, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.729883909225464, + "learning_rate": 6.35e-07, + "loss": 0.06268303096294403, + "num_tokens": 629545.0, + "reward": -68.05750274658203, + "reward_std": 59.44808578491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.15750122070312, + "rewards/supergames_reward/std": 59.44808578491211, + "step": 74, + "step_time": 10.64124580900534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 141.0, + "completions/max_terminated_length": 141.0, + "completions/mean_length": 117.375, + "completions/mean_terminated_length": 117.375, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.3744319677352905, + "epoch": 0.375, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.775154113769531, + "learning_rate": 6.3e-07, + "loss": -0.023284845054149628, + "num_tokens": 635052.0, + "reward": 3.9374990463256836, + "reward_std": 61.85771179199219, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 3.862499237060547, + "rewards/supergames_reward/std": 61.80967712402344, + "step": 75, + "step_time": 5.432648951013107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 157.0, + "completions/mean_terminated_length": 157.0, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4822140336036682, + "epoch": 0.38, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.715704917907715, + "learning_rate": 6.249999999999999e-07, + "loss": -0.04918142035603523, + "num_tokens": 644644.0, + "reward": 48.64125061035156, + "reward_std": 78.0205078125, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 48.56624984741211, + "rewards/supergames_reward/std": 77.96605682373047, + "step": 76, + "step_time": 10.348935816989979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/max_terminated_length": 380.0, + "completions/mean_length": 210.875, + "completions/mean_terminated_length": 210.875, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "entropy": 0.45722317695617676, + "epoch": 0.385, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6057567596435547, + "learning_rate": 6.2e-07, + "loss": 0.058352649211883545, + "num_tokens": 652771.0, + "reward": -8.283750534057617, + "reward_std": 77.68436431884766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.383749008178711, + "rewards/supergames_reward/std": 77.68437194824219, + "step": 77, + "step_time": 14.142948755004909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 188.25, + "completions/mean_terminated_length": 188.25, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.33380764722824097, + "epoch": 0.39, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2239863872528076, + "learning_rate": 6.149999999999999e-07, + "loss": -0.04592633992433548, + "num_tokens": 662653.0, + "reward": 11.153749465942383, + "reward_std": 89.88329315185547, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 11.053749084472656, + "rewards/supergames_reward/std": 89.88329315185547, + "step": 78, + "step_time": 12.277474621019792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/max_terminated_length": 143.0, + "completions/mean_length": 108.875, + "completions/mean_terminated_length": 108.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.3518812656402588, + "epoch": 0.395, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.967776298522949, + "learning_rate": 6.1e-07, + "loss": -0.10800496488809586, + "num_tokens": 668076.0, + "reward": 70.33250427246094, + "reward_std": 49.9145393371582, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 70.23249816894531, + "rewards/supergames_reward/std": 49.91453552246094, + "step": 79, + "step_time": 5.52714040101273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 152.125, + "completions/mean_terminated_length": 152.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4699896574020386, + "epoch": 0.4, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.561161994934082, + "learning_rate": 6.049999999999999e-07, + "loss": 0.09505834430456161, + "num_tokens": 675701.0, + "reward": -20.25625228881836, + "reward_std": 53.78542709350586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -20.35624885559082, + "rewards/supergames_reward/std": 53.78542709350586, + "step": 80, + "step_time": 7.155081019998761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 288.875, + "completions/mean_terminated_length": 257.0, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.4086494445800781, + "epoch": 0.405, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.3719611167907715, + "learning_rate": 6e-07, + "loss": 0.04727627709507942, + "num_tokens": 687332.0, + "reward": -37.20750045776367, + "reward_std": 91.7676010131836, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -37.25749969482422, + "rewards/supergames_reward/std": 91.72847747802734, + "step": 81, + "step_time": 19.231399144016905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 376.0, + "completions/max_terminated_length": 376.0, + "completions/mean_length": 227.75, + "completions/mean_terminated_length": 227.75, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.43233707547187805, + "epoch": 0.41, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3391714096069336, + "learning_rate": 5.949999999999999e-07, + "loss": -0.25524550676345825, + "num_tokens": 698474.0, + "reward": -3.7287511825561523, + "reward_std": 91.25379943847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.828749656677246, + "rewards/supergames_reward/std": 91.25379943847656, + "step": 82, + "step_time": 14.329176368017215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 178.125, + "completions/mean_terminated_length": 178.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.4293636977672577, + "epoch": 0.415, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.7271742820739746, + "learning_rate": 5.9e-07, + "loss": 0.09273672848939896, + "num_tokens": 708243.0, + "reward": 12.802498817443848, + "reward_std": 86.23526000976562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 12.702500343322754, + "rewards/supergames_reward/std": 86.23526000976562, + "step": 83, + "step_time": 11.419686011999147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 425.0, + "completions/max_terminated_length": 425.0, + "completions/mean_length": 255.375, + "completions/mean_terminated_length": 255.375, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "entropy": 0.38068246841430664, + "epoch": 0.42, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.6244184970855713, + "learning_rate": 5.849999999999999e-07, + "loss": 0.02006213366985321, + "num_tokens": 718606.0, + "reward": 37.803749084472656, + "reward_std": 57.58624267578125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 37.70375061035156, + "rewards/supergames_reward/std": 57.58624267578125, + "step": 84, + "step_time": 15.975198492989875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 116.0, + "completions/max_terminated_length": 116.0, + "completions/mean_length": 108.125, + "completions/mean_terminated_length": 108.125, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.339572012424469, + "epoch": 0.425, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.754759311676025, + "learning_rate": 5.8e-07, + "loss": -0.011015941388905048, + "num_tokens": 723983.0, + "reward": -3.8212504386901855, + "reward_std": 61.624786376953125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.921250820159912, + "rewards/supergames_reward/std": 61.624786376953125, + "step": 85, + "step_time": 4.542777584982105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 447.0, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 291.375, + "completions/mean_terminated_length": 291.375, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "entropy": 0.5846173763275146, + "epoch": 0.43, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55724835395813, + "learning_rate": 5.749999999999999e-07, + "loss": -0.04981120675802231, + "num_tokens": 735642.0, + "reward": 50.07499694824219, + "reward_std": 92.6283187866211, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.0, + "rewards/supergames_reward/std": 92.5820083618164, + "step": 86, + "step_time": 16.777178087009815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/max_terminated_length": 232.0, + "completions/mean_length": 192.0, + "completions/mean_terminated_length": 192.0, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "entropy": 0.5139474272727966, + "epoch": 0.435, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8860411643981934, + "learning_rate": 5.699999999999999e-07, + "loss": -0.02194221317768097, + "num_tokens": 743602.0, + "reward": -52.522499084472656, + "reward_std": 73.8686294555664, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -52.62249755859375, + "rewards/supergames_reward/std": 73.86863708496094, + "step": 87, + "step_time": 8.772893911984283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/max_terminated_length": 266.0, + "completions/mean_length": 146.875, + "completions/mean_terminated_length": 146.875, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.3428497910499573, + "epoch": 0.44, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.509483337402344, + "learning_rate": 5.649999999999999e-07, + "loss": 0.19857533276081085, + "num_tokens": 753065.0, + "reward": 63.63874816894531, + "reward_std": 67.54963684082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 63.53874969482422, + "rewards/supergames_reward/std": 67.54963684082031, + "step": 88, + "step_time": 10.230529835011112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 446.0, + "completions/max_terminated_length": 446.0, + "completions/mean_length": 270.625, + "completions/mean_terminated_length": 270.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.4417419731616974, + "epoch": 0.445, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.973951816558838, + "learning_rate": 5.6e-07, + "loss": -0.00031440958264283836, + "num_tokens": 764534.0, + "reward": -9.736251831054688, + "reward_std": 92.25879669189453, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -9.836250305175781, + "rewards/supergames_reward/std": 92.25880432128906, + "step": 89, + "step_time": 17.40752330099349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 398.0, + "completions/max_terminated_length": 398.0, + "completions/mean_length": 227.375, + "completions/mean_terminated_length": 227.375, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.4732024073600769, + "epoch": 0.45, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2560815811157227, + "learning_rate": 5.55e-07, + "loss": 0.03820464387536049, + "num_tokens": 772809.0, + "reward": 5.982499599456787, + "reward_std": 58.833797454833984, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 5.8824992179870605, + "rewards/supergames_reward/std": 58.83380126953125, + "step": 90, + "step_time": 15.007269965979503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 207.5, + "completions/mean_terminated_length": 207.5, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.42038559913635254, + "epoch": 0.455, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3649117946624756, + "learning_rate": 5.5e-07, + "loss": 0.1510048508644104, + "num_tokens": 782805.0, + "reward": -5.61500358581543, + "reward_std": 82.67152404785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -5.714998245239258, + "rewards/supergames_reward/std": 82.67152404785156, + "step": 91, + "step_time": 10.386137172987219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 416.0, + "completions/max_terminated_length": 416.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.3971734642982483, + "epoch": 0.46, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.433772563934326, + "learning_rate": 5.45e-07, + "loss": -0.3027213513851166, + "num_tokens": 793840.0, + "reward": -29.06500244140625, + "reward_std": 91.68325805664062, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -29.16499900817871, + "rewards/supergames_reward/std": 91.68325805664062, + "step": 92, + "step_time": 15.790611553995404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 206.625, + "completions/mean_terminated_length": 206.625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.5906907320022583, + "epoch": 0.465, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.293088912963867, + "learning_rate": 5.4e-07, + "loss": 0.09760032594203949, + "num_tokens": 803821.0, + "reward": -72.73999786376953, + "reward_std": 50.72484588623047, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -72.81500244140625, + "rewards/supergames_reward/std": 50.7094841003418, + "step": 93, + "step_time": 13.043119941983605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/max_terminated_length": 143.0, + "completions/mean_length": 115.75, + "completions/mean_terminated_length": 115.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4233115315437317, + "epoch": 0.47, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.58886194229126, + "learning_rate": 5.35e-07, + "loss": -0.04040650278329849, + "num_tokens": 809299.0, + "reward": -3.3050003051757812, + "reward_std": 24.443714141845703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.405000686645508, + "rewards/supergames_reward/std": 24.443714141845703, + "step": 94, + "step_time": 5.512874762003776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 122.0, + "completions/max_terminated_length": 122.0, + "completions/mean_length": 103.125, + "completions/mean_terminated_length": 103.125, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.4055764377117157, + "epoch": 0.475, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.981549263000488, + "learning_rate": 5.3e-07, + "loss": -0.07588938623666763, + "num_tokens": 814716.0, + "reward": 60.06999969482422, + "reward_std": 42.708473205566406, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 59.970001220703125, + "rewards/supergames_reward/std": 42.708473205566406, + "step": 95, + "step_time": 4.766389057011111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 475.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.408913254737854, + "epoch": 0.48, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.79402756690979, + "learning_rate": 5.25e-07, + "loss": -0.3087541460990906, + "num_tokens": 824791.0, + "reward": -15.042500495910645, + "reward_std": 84.37246704101562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -15.142499923706055, + "rewards/supergames_reward/std": 84.37246704101562, + "step": 96, + "step_time": 17.779843941010768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/max_terminated_length": 154.0, + "completions/mean_length": 121.625, + "completions/mean_terminated_length": 121.625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.529015839099884, + "epoch": 0.485, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.142816066741943, + "learning_rate": 5.2e-07, + "loss": 0.07216037809848785, + "num_tokens": 830300.0, + "reward": -32.743751525878906, + "reward_std": 54.61775588989258, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -32.81875228881836, + "rewards/supergames_reward/std": 54.58256149291992, + "step": 97, + "step_time": 5.913989890017547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 126.0, + "completions/max_terminated_length": 126.0, + "completions/mean_length": 95.625, + "completions/mean_terminated_length": 95.625, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.33743274211883545, + "epoch": 0.49, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.197273254394531, + "learning_rate": 5.149999999999999e-07, + "loss": 0.07500407844781876, + "num_tokens": 835633.0, + "reward": 11.00999927520752, + "reward_std": 62.73848342895508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 10.910000801086426, + "rewards/supergames_reward/std": 62.73848342895508, + "step": 98, + "step_time": 4.9155233050114475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 108.5, + "completions/mean_terminated_length": 108.5, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.3106803297996521, + "epoch": 0.495, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.864002227783203, + "learning_rate": 5.1e-07, + "loss": -0.1206185445189476, + "num_tokens": 841045.0, + "reward": 14.238749504089355, + "reward_std": 51.77016830444336, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 14.138750076293945, + "rewards/supergames_reward/std": 51.77016830444336, + "step": 99, + "step_time": 5.7507751359953545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 97.125, + "completions/mean_terminated_length": 97.125, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.35160186886787415, + "epoch": 0.5, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915731906890869, + "learning_rate": 5.049999999999999e-07, + "loss": -0.07951541244983673, + "num_tokens": 846398.0, + "reward": 67.78125, + "reward_std": 38.15436553955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 67.68124389648438, + "rewards/supergames_reward/std": 38.15436553955078, + "step": 100, + "step_time": 5.994720440998208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 320.0, + "completions/max_terminated_length": 320.0, + "completions/mean_length": 208.75, + "completions/mean_terminated_length": 208.75, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.452095091342926, + "epoch": 0.505, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.065176963806152, + "learning_rate": 5e-07, + "loss": -0.1397111564874649, + "num_tokens": 857420.0, + "reward": -5.801251411437988, + "reward_std": 73.56117248535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -5.90125036239624, + "rewards/supergames_reward/std": 73.5611801147461, + "step": 101, + "step_time": 12.28178753197426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/max_terminated_length": 230.0, + "completions/mean_length": 171.375, + "completions/mean_terminated_length": 171.375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4138447642326355, + "epoch": 0.51, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.998823404312134, + "learning_rate": 4.95e-07, + "loss": -0.08941879868507385, + "num_tokens": 867103.0, + "reward": 92.24749755859375, + "reward_std": 14.540005683898926, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 92.14749908447266, + "rewards/supergames_reward/std": 14.540006637573242, + "step": 102, + "step_time": 8.992682139010867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 311.0, + "completions/max_terminated_length": 311.0, + "completions/mean_length": 163.0, + "completions/mean_terminated_length": 163.0, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.4182547330856323, + "epoch": 0.515, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.419595241546631, + "learning_rate": 4.9e-07, + "loss": -0.19304415583610535, + "num_tokens": 876679.0, + "reward": 79.13249969482422, + "reward_std": 59.3050422668457, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 79.03250122070312, + "rewards/supergames_reward/std": 59.30504608154297, + "step": 103, + "step_time": 11.836970244999975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 253.0, + "completions/max_terminated_length": 253.0, + "completions/mean_length": 155.625, + "completions/mean_terminated_length": 155.625, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.49889081716537476, + "epoch": 0.52, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.463719367980957, + "learning_rate": 4.85e-07, + "loss": -0.04786720871925354, + "num_tokens": 884348.0, + "reward": 48.616249084472656, + "reward_std": 73.30741119384766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.51625061035156, + "rewards/supergames_reward/std": 73.30741119384766, + "step": 104, + "step_time": 9.579507841990562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/max_terminated_length": 380.0, + "completions/mean_length": 269.5, + "completions/mean_terminated_length": 269.5, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "entropy": 0.5286482572555542, + "epoch": 0.525, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.993295431137085, + "learning_rate": 4.8e-07, + "loss": -0.08071611076593399, + "num_tokens": 894856.0, + "reward": 34.970001220703125, + "reward_std": 85.324462890625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 34.869998931884766, + "rewards/supergames_reward/std": 85.324462890625, + "step": 105, + "step_time": 14.295730627985904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 203.25, + "completions/mean_terminated_length": 203.25, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.6049264073371887, + "epoch": 0.53, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.787306785583496, + "learning_rate": 4.7499999999999995e-07, + "loss": 0.19928883016109467, + "num_tokens": 902922.0, + "reward": 32.44499969482422, + "reward_std": 70.4663314819336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 32.369998931884766, + "rewards/supergames_reward/std": 70.41261291503906, + "step": 106, + "step_time": 9.84894504098338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 194.0, + "completions/mean_terminated_length": 194.0, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "entropy": 0.5115755200386047, + "epoch": 0.535, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.277207374572754, + "learning_rate": 4.6999999999999995e-07, + "loss": 0.09118300676345825, + "num_tokens": 910906.0, + "reward": 40.432498931884766, + "reward_std": 48.54548263549805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 40.33250045776367, + "rewards/supergames_reward/std": 48.54548263549805, + "step": 107, + "step_time": 10.466603949986165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 332.0, + "completions/max_terminated_length": 332.0, + "completions/mean_length": 195.875, + "completions/mean_terminated_length": 195.875, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4192962944507599, + "epoch": 0.54, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.308061122894287, + "learning_rate": 4.65e-07, + "loss": 0.28064602613449097, + "num_tokens": 920745.0, + "reward": -21.982500076293945, + "reward_std": 66.31195068359375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -22.08249855041504, + "rewards/supergames_reward/std": 66.31195068359375, + "step": 108, + "step_time": 12.595848233992001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 195.125, + "completions/mean_terminated_length": 195.125, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.37523961067199707, + "epoch": 0.545, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.802706718444824, + "learning_rate": 4.6e-07, + "loss": -0.06614465266466141, + "num_tokens": 930618.0, + "reward": 71.75, + "reward_std": 51.14524459838867, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 71.6500015258789, + "rewards/supergames_reward/std": 51.1452522277832, + "step": 109, + "step_time": 10.33884758799104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 167.75, + "completions/mean_terminated_length": 167.75, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.4339042007923126, + "epoch": 0.55, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.254003047943115, + "learning_rate": 4.55e-07, + "loss": 0.34405598044395447, + "num_tokens": 941256.0, + "reward": 99.21875, + "reward_std": 2.492553234100342, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 99.1187515258789, + "rewards/supergames_reward/std": 2.4925525188446045, + "step": 110, + "step_time": 12.657525141985388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 285.0, + "completions/max_terminated_length": 285.0, + "completions/mean_length": 211.0, + "completions/mean_terminated_length": 211.0, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "entropy": 0.46809861063957214, + "epoch": 0.555, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.811166286468506, + "learning_rate": 4.5e-07, + "loss": 0.045113153755664825, + "num_tokens": 951296.0, + "reward": 55.27750015258789, + "reward_std": 47.90687561035156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 55.1775016784668, + "rewards/supergames_reward/std": 47.9068717956543, + "step": 111, + "step_time": 10.91118389699841 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 145.0, + "completions/max_terminated_length": 145.0, + "completions/mean_length": 108.375, + "completions/mean_terminated_length": 108.375, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.34181955456733704, + "epoch": 0.56, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.537296772003174, + "learning_rate": 4.45e-07, + "loss": -0.052413541823625565, + "num_tokens": 956675.0, + "reward": 47.56624984741211, + "reward_std": 49.185157775878906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 47.466251373291016, + "rewards/supergames_reward/std": 49.18516159057617, + "step": 112, + "step_time": 5.552288047998445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/max_terminated_length": 245.0, + "completions/mean_length": 189.125, + "completions/mean_terminated_length": 189.125, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "entropy": 0.5034605860710144, + "epoch": 0.565, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.349876642227173, + "learning_rate": 4.3999999999999997e-07, + "loss": -0.08186715841293335, + "num_tokens": 964636.0, + "reward": 81.32374572753906, + "reward_std": 36.85074996948242, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 81.2237548828125, + "rewards/supergames_reward/std": 36.85074996948242, + "step": 113, + "step_time": 9.267611294984818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/max_terminated_length": 178.0, + "completions/mean_length": 111.25, + "completions/mean_terminated_length": 111.25, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "entropy": 0.3229002058506012, + "epoch": 0.57, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.245575428009033, + "learning_rate": 4.3499999999999996e-07, + "loss": 0.06544198095798492, + "num_tokens": 970070.0, + "reward": 63.1775016784668, + "reward_std": 52.30624008178711, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 63.07749938964844, + "rewards/supergames_reward/std": 52.30624008178711, + "step": 114, + "step_time": 6.731139309995342 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 224.0, + "completions/max_terminated_length": 224.0, + "completions/mean_length": 170.0, + "completions/mean_terminated_length": 170.0, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.3274807929992676, + "epoch": 0.575, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5544140338897705, + "learning_rate": 4.2999999999999996e-07, + "loss": -0.06953569501638412, + "num_tokens": 979702.0, + "reward": 25.62125015258789, + "reward_std": 64.52003479003906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 25.521244049072266, + "rewards/supergames_reward/std": 64.52003479003906, + "step": 115, + "step_time": 8.702618776995223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 427.0, + "completions/max_terminated_length": 427.0, + "completions/mean_length": 209.5, + "completions/mean_terminated_length": 209.5, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.4970880150794983, + "epoch": 0.58, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4932165145874023, + "learning_rate": 4.2499999999999995e-07, + "loss": -0.2281273603439331, + "num_tokens": 990690.0, + "reward": 50.54750061035156, + "reward_std": 70.5126953125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 50.4474983215332, + "rewards/supergames_reward/std": 70.51270294189453, + "step": 116, + "step_time": 16.204386608995264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.0, + "completions/max_terminated_length": 228.0, + "completions/mean_length": 132.375, + "completions/mean_terminated_length": 132.375, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4380227029323578, + "epoch": 0.585, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.089909076690674, + "learning_rate": 4.1999999999999995e-07, + "loss": 0.12121254950761795, + "num_tokens": 996333.0, + "reward": -15.333749771118164, + "reward_std": 47.200164794921875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -15.433748245239258, + "rewards/supergames_reward/std": 47.200164794921875, + "step": 117, + "step_time": 8.546233230998041 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.0, + "completions/max_terminated_length": 323.0, + "completions/mean_length": 137.625, + "completions/mean_terminated_length": 137.625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4177803695201874, + "epoch": 0.59, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8712472915649414, + "learning_rate": 4.1499999999999994e-07, + "loss": 0.14192955195903778, + "num_tokens": 1002010.0, + "reward": -30.700000762939453, + "reward_std": 17.224035263061523, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -30.799999237060547, + "rewards/supergames_reward/std": 17.224035263061523, + "step": 118, + "step_time": 11.878434990998358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 453.0, + "completions/max_terminated_length": 453.0, + "completions/mean_length": 238.125, + "completions/mean_terminated_length": 238.125, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "entropy": 0.5640217065811157, + "epoch": 0.595, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.064737319946289, + "learning_rate": 4.0999999999999994e-07, + "loss": -0.023621462285518646, + "num_tokens": 1012259.0, + "reward": 43.683746337890625, + "reward_std": 70.4210205078125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 43.5837516784668, + "rewards/supergames_reward/std": 70.4210205078125, + "step": 119, + "step_time": 16.829514264973113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 193.75, + "completions/mean_terminated_length": 193.75, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "entropy": 0.5795091390609741, + "epoch": 0.6, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.423007965087891, + "learning_rate": 4.05e-07, + "loss": -0.0666111558675766, + "num_tokens": 1020241.0, + "reward": -85.86000061035156, + "reward_std": 27.4442081451416, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -85.90999603271484, + "rewards/supergames_reward/std": 27.41469955444336, + "step": 120, + "step_time": 8.95496519000153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 231.0, + "completions/mean_terminated_length": 231.0, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "entropy": 0.3767699897289276, + "epoch": 0.605, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7434473037719727, + "learning_rate": 4e-07, + "loss": -0.1165132150053978, + "num_tokens": 1031465.0, + "reward": 61.45624923706055, + "reward_std": 72.21080780029297, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 61.35625076293945, + "rewards/supergames_reward/std": 72.2108154296875, + "step": 121, + "step_time": 12.588820598000893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 297.0, + "completions/max_terminated_length": 297.0, + "completions/mean_length": 223.375, + "completions/mean_terminated_length": 223.375, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "entropy": 0.4537277817726135, + "epoch": 0.61, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4382436275482178, + "learning_rate": 3.95e-07, + "loss": -0.1038316935300827, + "num_tokens": 1042548.0, + "reward": 5.658749580383301, + "reward_std": 83.77420043945312, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 5.558750152587891, + "rewards/supergames_reward/std": 83.77420043945312, + "step": 122, + "step_time": 11.39525846898323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 456.0, + "completions/max_terminated_length": 456.0, + "completions/mean_length": 314.25, + "completions/mean_terminated_length": 314.25, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4113953411579132, + "epoch": 0.615, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.449777841567993, + "learning_rate": 3.8999999999999997e-07, + "loss": -0.1303664743900299, + "num_tokens": 1054422.0, + "reward": 35.98249816894531, + "reward_std": 88.6131362915039, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 35.88249969482422, + "rewards/supergames_reward/std": 88.6131362915039, + "step": 123, + "step_time": 17.001118954998674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 242.0, + "completions/max_terminated_length": 242.0, + "completions/mean_length": 172.5, + "completions/mean_terminated_length": 172.5, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.4670729339122772, + "epoch": 0.62, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.951946973800659, + "learning_rate": 3.8499999999999997e-07, + "loss": 0.05888795852661133, + "num_tokens": 1062210.0, + "reward": 43.368751525878906, + "reward_std": 40.30733871459961, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 43.26874542236328, + "rewards/supergames_reward/std": 40.307342529296875, + "step": 124, + "step_time": 9.142582255997695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/max_terminated_length": 148.0, + "completions/mean_length": 106.625, + "completions/mean_terminated_length": 106.625, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "entropy": 0.30219921469688416, + "epoch": 0.625, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.391322135925293, + "learning_rate": 3.7999999999999996e-07, + "loss": 0.14367049932479858, + "num_tokens": 1067599.0, + "reward": 42.63374710083008, + "reward_std": 58.20897674560547, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 42.533748626708984, + "rewards/supergames_reward/std": 58.208984375, + "step": 125, + "step_time": 5.636063549987739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.0, + "completions/max_terminated_length": 243.0, + "completions/mean_length": 179.25, + "completions/mean_terminated_length": 179.25, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "entropy": 0.4834887683391571, + "epoch": 0.63, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9391286373138428, + "learning_rate": 3.75e-07, + "loss": -0.06614185124635696, + "num_tokens": 1075457.0, + "reward": 8.302498817443848, + "reward_std": 75.10364532470703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 8.202500343322754, + "rewards/supergames_reward/std": 75.10364532470703, + "step": 126, + "step_time": 9.130600645992672 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 196.0, + "completions/max_terminated_length": 196.0, + "completions/mean_length": 157.0, + "completions/mean_terminated_length": 157.0, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.4801807701587677, + "epoch": 0.635, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.425615310668945, + "learning_rate": 3.7e-07, + "loss": -0.08830522745847702, + "num_tokens": 1083129.0, + "reward": -10.563751220703125, + "reward_std": 64.00443267822266, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -10.663749694824219, + "rewards/supergames_reward/std": 64.00444030761719, + "step": 127, + "step_time": 7.4972667430120055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 412.0, + "completions/max_terminated_length": 412.0, + "completions/mean_length": 239.25, + "completions/mean_terminated_length": 239.25, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.4667063355445862, + "epoch": 0.64, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.199113130569458, + "learning_rate": 3.65e-07, + "loss": -0.043856702744960785, + "num_tokens": 1094355.0, + "reward": -55.89250183105469, + "reward_std": 53.05980682373047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -55.99250030517578, + "rewards/supergames_reward/std": 53.05980682373047, + "step": 128, + "step_time": 15.429580625001108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 244.75, + "completions/mean_terminated_length": 244.75, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "entropy": 0.57573401927948, + "epoch": 0.645, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6800079345703125, + "learning_rate": 3.6e-07, + "loss": 0.09521033614873886, + "num_tokens": 1105633.0, + "reward": -23.35250473022461, + "reward_std": 102.31401062011719, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -23.452499389648438, + "rewards/supergames_reward/std": 102.31402587890625, + "step": 129, + "step_time": 13.920327747007832 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/max_terminated_length": 258.0, + "completions/mean_length": 188.25, + "completions/mean_terminated_length": 188.25, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "entropy": 0.5341438055038452, + "epoch": 0.65, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.97955060005188, + "learning_rate": 3.55e-07, + "loss": -0.09512650966644287, + "num_tokens": 1113571.0, + "reward": 29.181249618530273, + "reward_std": 80.10585021972656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 29.081249237060547, + "rewards/supergames_reward/std": 80.10585021972656, + "step": 130, + "step_time": 9.69286120700417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 257.0, + "completions/max_terminated_length": 257.0, + "completions/mean_length": 180.75, + "completions/mean_terminated_length": 180.75, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "entropy": 0.46591487526893616, + "epoch": 0.655, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3518130779266357, + "learning_rate": 3.5e-07, + "loss": 0.04714229330420494, + "num_tokens": 1121457.0, + "reward": -8.287501335144043, + "reward_std": 76.35875701904297, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.387499809265137, + "rewards/supergames_reward/std": 76.35875701904297, + "step": 131, + "step_time": 9.636637121002423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 147.75, + "completions/mean_terminated_length": 147.75, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4407735764980316, + "epoch": 0.66, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.498741626739502, + "learning_rate": 3.45e-07, + "loss": -0.03996007889509201, + "num_tokens": 1129095.0, + "reward": 0.1999988555908203, + "reward_std": 84.63554382324219, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 0.125, + "rewards/supergames_reward/std": 84.6017074584961, + "step": 132, + "step_time": 7.510833573003765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 358.0, + "completions/max_terminated_length": 358.0, + "completions/mean_length": 213.875, + "completions/mean_terminated_length": 213.875, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.5186149477958679, + "epoch": 0.665, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3987107276916504, + "learning_rate": 3.4000000000000003e-07, + "loss": 0.1863594949245453, + "num_tokens": 1139110.0, + "reward": -21.328752517700195, + "reward_std": 88.9102783203125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -21.428749084472656, + "rewards/supergames_reward/std": 88.9102783203125, + "step": 133, + "step_time": 13.393839450000087 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/max_terminated_length": 230.0, + "completions/mean_length": 169.875, + "completions/mean_terminated_length": 169.875, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.49445784091949463, + "epoch": 0.67, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.479673862457275, + "learning_rate": 3.35e-07, + "loss": 0.03618276119232178, + "num_tokens": 1146885.0, + "reward": -14.042501449584961, + "reward_std": 83.22671508789062, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -14.142499923706055, + "rewards/supergames_reward/std": 83.22671508789062, + "step": 134, + "step_time": 8.703399487014394 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/max_terminated_length": 168.0, + "completions/mean_length": 124.75, + "completions/mean_terminated_length": 124.75, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "entropy": 0.2997814118862152, + "epoch": 0.675, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.440976619720459, + "learning_rate": 3.3e-07, + "loss": -0.029299044981598854, + "num_tokens": 1152427.0, + "reward": 45.274993896484375, + "reward_std": 44.401519775390625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 45.17499923706055, + "rewards/supergames_reward/std": 44.40152359008789, + "step": 135, + "step_time": 6.334954546997324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 161.0, + "completions/max_terminated_length": 161.0, + "completions/mean_length": 123.625, + "completions/mean_terminated_length": 123.625, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3462243974208832, + "epoch": 0.68, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.1878814697265625, + "learning_rate": 3.25e-07, + "loss": -0.10772529244422913, + "num_tokens": 1157952.0, + "reward": 44.186248779296875, + "reward_std": 51.78197479248047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 44.08625030517578, + "rewards/supergames_reward/std": 51.7819709777832, + "step": 136, + "step_time": 6.067531232984038 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 155.0, + "completions/max_terminated_length": 155.0, + "completions/mean_length": 106.75, + "completions/mean_terminated_length": 106.75, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.3662136197090149, + "epoch": 0.685, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.508971214294434, + "learning_rate": 3.2e-07, + "loss": -0.11172202974557877, + "num_tokens": 1163334.0, + "reward": 53.0574951171875, + "reward_std": 58.36653518676758, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 52.957496643066406, + "rewards/supergames_reward/std": 58.36653518676758, + "step": 137, + "step_time": 5.8906258280039765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 175.0, + "completions/max_terminated_length": 175.0, + "completions/mean_length": 125.375, + "completions/mean_terminated_length": 125.375, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.3352622091770172, + "epoch": 0.69, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.1937079429626465, + "learning_rate": 3.15e-07, + "loss": 0.08307601511478424, + "num_tokens": 1168929.0, + "reward": 11.796250343322754, + "reward_std": 53.48870849609375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 11.696250915527344, + "rewards/supergames_reward/std": 53.48870849609375, + "step": 138, + "step_time": 6.595962718012743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 193.0, + "completions/mean_terminated_length": 193.0, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.5486535429954529, + "epoch": 0.695, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.861871242523193, + "learning_rate": 3.1e-07, + "loss": 0.0321456640958786, + "num_tokens": 1176937.0, + "reward": 50.463748931884766, + "reward_std": 70.37553405761719, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.38875198364258, + "rewards/supergames_reward/std": 70.31440734863281, + "step": 139, + "step_time": 9.544256388006033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 189.5, + "completions/mean_terminated_length": 189.5, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.42230603098869324, + "epoch": 0.7, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.830677032470703, + "learning_rate": 3.05e-07, + "loss": -0.07222295552492142, + "num_tokens": 1186789.0, + "reward": 33.44999694824219, + "reward_std": 67.1298599243164, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 33.349998474121094, + "rewards/supergames_reward/std": 67.1298599243164, + "step": 140, + "step_time": 10.002322309010196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 282.0, + "completions/max_terminated_length": 282.0, + "completions/mean_length": 197.375, + "completions/mean_terminated_length": 197.375, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.33588480949401855, + "epoch": 0.705, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.028353452682495, + "learning_rate": 3e-07, + "loss": -0.07543312013149261, + "num_tokens": 1196672.0, + "reward": 17.10249900817871, + "reward_std": 73.94812774658203, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 17.002500534057617, + "rewards/supergames_reward/std": 73.94813537597656, + "step": 141, + "step_time": 10.684560916008195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/max_terminated_length": 235.0, + "completions/mean_length": 209.25, + "completions/mean_terminated_length": 209.25, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "entropy": 0.3622831702232361, + "epoch": 0.71, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.047222137451172, + "learning_rate": 2.95e-07, + "loss": -0.04569581151008606, + "num_tokens": 1206658.0, + "reward": 49.57624816894531, + "reward_std": 78.21060180664062, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 49.476253509521484, + "rewards/supergames_reward/std": 78.21060943603516, + "step": 142, + "step_time": 8.97679864402744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 161.0, + "completions/max_terminated_length": 161.0, + "completions/mean_length": 116.25, + "completions/mean_terminated_length": 116.25, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.43277010321617126, + "epoch": 0.715, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.345203876495361, + "learning_rate": 2.9e-07, + "loss": -0.16899466514587402, + "num_tokens": 1212140.0, + "reward": 27.42875099182129, + "reward_std": 48.59138870239258, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.328750610351562, + "rewards/supergames_reward/std": 48.591392517089844, + "step": 143, + "step_time": 6.1606669370085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 205.0, + "completions/max_terminated_length": 205.0, + "completions/mean_length": 173.75, + "completions/mean_terminated_length": 173.75, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "entropy": 0.43806371092796326, + "epoch": 0.72, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.858840227127075, + "learning_rate": 2.8499999999999997e-07, + "loss": -0.06502802670001984, + "num_tokens": 1219954.0, + "reward": 38.69249725341797, + "reward_std": 68.78203582763672, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 38.592498779296875, + "rewards/supergames_reward/std": 68.78202819824219, + "step": 144, + "step_time": 7.88639413099736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 169.0, + "completions/max_terminated_length": 169.0, + "completions/mean_length": 110.625, + "completions/mean_terminated_length": 110.625, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.3383479118347168, + "epoch": 0.725, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.225733757019043, + "learning_rate": 2.8e-07, + "loss": -0.042058318853378296, + "num_tokens": 1225399.0, + "reward": 64.85874938964844, + "reward_std": 25.170387268066406, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 64.75875091552734, + "rewards/supergames_reward/std": 25.170391082763672, + "step": 145, + "step_time": 6.432729878026294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 147.0, + "completions/max_terminated_length": 147.0, + "completions/mean_length": 129.5, + "completions/mean_terminated_length": 129.5, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3088987171649933, + "epoch": 0.73, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.75e-07, + "loss": 0.0, + "num_tokens": 1231043.0, + "reward": 100.0999984741211, + "reward_std": 0.0, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 100.0, + "rewards/supergames_reward/std": 0.0, + "step": 146, + "step_time": 5.634907905012369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 160.0, + "completions/max_terminated_length": 160.0, + "completions/mean_length": 120.5, + "completions/mean_terminated_length": 120.5, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.37292322516441345, + "epoch": 0.735, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.174577713012695, + "learning_rate": 2.7e-07, + "loss": 0.11778222769498825, + "num_tokens": 1236607.0, + "reward": 34.82374572753906, + "reward_std": 58.513397216796875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 34.72374725341797, + "rewards/supergames_reward/std": 58.513397216796875, + "step": 147, + "step_time": 6.078690663998714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 172.0, + "completions/max_terminated_length": 172.0, + "completions/mean_length": 125.625, + "completions/mean_terminated_length": 125.625, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.308654248714447, + "epoch": 0.74, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8483362197875977, + "learning_rate": 2.65e-07, + "loss": -0.09181805700063705, + "num_tokens": 1242140.0, + "reward": 85.06375122070312, + "reward_std": 42.52893829345703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 84.9637451171875, + "rewards/supergames_reward/std": 42.52893829345703, + "step": 148, + "step_time": 6.501119122985983 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 275.0, + "completions/max_terminated_length": 275.0, + "completions/mean_length": 211.125, + "completions/mean_terminated_length": 211.125, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "entropy": 0.5405827164649963, + "epoch": 0.745, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.774259567260742, + "learning_rate": 2.6e-07, + "loss": -0.03056861087679863, + "num_tokens": 1250205.0, + "reward": 84.63125610351562, + "reward_std": 34.3358154296875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 84.53125, + "rewards/supergames_reward/std": 34.3358154296875, + "step": 149, + "step_time": 10.352729418984381 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 257.0, + "completions/max_terminated_length": 257.0, + "completions/mean_length": 218.5, + "completions/mean_terminated_length": 218.5, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "entropy": 0.3852894604206085, + "epoch": 0.75, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6103010177612305, + "learning_rate": 2.55e-07, + "loss": 0.034789010882377625, + "num_tokens": 1260249.0, + "reward": 81.81375122070312, + "reward_std": 51.72132873535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 81.7137451171875, + "rewards/supergames_reward/std": 51.72132873535156, + "step": 150, + "step_time": 9.937249842012534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 351.0, + "completions/max_terminated_length": 351.0, + "completions/mean_length": 232.625, + "completions/mean_terminated_length": 232.625, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.4323120415210724, + "epoch": 0.755, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.640308380126953, + "learning_rate": 2.5e-07, + "loss": -0.11181996762752533, + "num_tokens": 1271430.0, + "reward": 56.33625030517578, + "reward_std": 78.97452545166016, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 56.23625183105469, + "rewards/supergames_reward/std": 78.97452545166016, + "step": 151, + "step_time": 13.323547195002902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 359.0, + "completions/max_terminated_length": 359.0, + "completions/mean_length": 247.5, + "completions/mean_terminated_length": 247.5, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "entropy": 0.4038960039615631, + "epoch": 0.76, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.5704824924468994, + "learning_rate": 2.45e-07, + "loss": 0.060525521636009216, + "num_tokens": 1282722.0, + "reward": -68.53875732421875, + "reward_std": 50.590599060058594, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.63874816894531, + "rewards/supergames_reward/std": 50.590599060058594, + "step": 152, + "step_time": 13.593127576023107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 383.0, + "completions/max_terminated_length": 383.0, + "completions/mean_length": 246.375, + "completions/mean_terminated_length": 246.375, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "entropy": 0.4110933542251587, + "epoch": 0.765, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.233025312423706, + "learning_rate": 2.4e-07, + "loss": 0.06547188758850098, + "num_tokens": 1294061.0, + "reward": 95.91499328613281, + "reward_std": 11.836968421936035, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 95.81500244140625, + "rewards/supergames_reward/std": 11.836968421936035, + "step": 153, + "step_time": 14.499402174988063 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 338.0, + "completions/max_terminated_length": 338.0, + "completions/mean_length": 233.125, + "completions/mean_terminated_length": 233.125, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.34559860825538635, + "epoch": 0.77, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.6887102127075195, + "learning_rate": 2.3499999999999997e-07, + "loss": 0.15905122458934784, + "num_tokens": 1304262.0, + "reward": 88.92375183105469, + "reward_std": 31.611207962036133, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 88.82374572753906, + "rewards/supergames_reward/std": 31.6112117767334, + "step": 154, + "step_time": 12.704337224975461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 208.0, + "completions/max_terminated_length": 208.0, + "completions/mean_length": 178.5, + "completions/mean_terminated_length": 178.5, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "entropy": 0.4646885097026825, + "epoch": 0.775, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.161937236785889, + "learning_rate": 2.3e-07, + "loss": -0.04354584217071533, + "num_tokens": 1312098.0, + "reward": 27.80875015258789, + "reward_std": 60.81097412109375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.708751678466797, + "rewards/supergames_reward/std": 60.81097412109375, + "step": 155, + "step_time": 7.911955468007363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 391.0, + "completions/max_terminated_length": 391.0, + "completions/mean_length": 235.375, + "completions/mean_terminated_length": 235.375, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "entropy": 0.40985700488090515, + "epoch": 0.78, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.12153959274292, + "learning_rate": 2.25e-07, + "loss": -0.024320494383573532, + "num_tokens": 1322301.0, + "reward": 64.14624786376953, + "reward_std": 49.90530014038086, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 64.04624938964844, + "rewards/supergames_reward/std": 49.90530014038086, + "step": 156, + "step_time": 14.631677505996777 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/max_terminated_length": 330.0, + "completions/mean_length": 220.625, + "completions/mean_terminated_length": 220.625, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3543764054775238, + "epoch": 0.785, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0111091136932373, + "learning_rate": 2.1999999999999998e-07, + "loss": -0.13996481895446777, + "num_tokens": 1333386.0, + "reward": 3.6812496185302734, + "reward_std": 103.5096664428711, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 3.606250762939453, + "rewards/supergames_reward/std": 103.48104858398438, + "step": 157, + "step_time": 12.613216657977318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.0, + "completions/max_terminated_length": 225.0, + "completions/mean_length": 193.125, + "completions/mean_terminated_length": 193.125, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "entropy": 0.5682947635650635, + "epoch": 0.79, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.67927885055542, + "learning_rate": 2.1499999999999998e-07, + "loss": 0.0535324327647686, + "num_tokens": 1341355.0, + "reward": -19.78999900817871, + "reward_std": 69.72611236572266, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -19.889997482299805, + "rewards/supergames_reward/std": 69.72611236572266, + "step": 158, + "step_time": 8.585773958999198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.0, + "completions/max_terminated_length": 316.0, + "completions/mean_length": 237.625, + "completions/mean_terminated_length": 237.625, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "entropy": 0.3637359142303467, + "epoch": 0.795, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.033409357070923, + "learning_rate": 2.0999999999999997e-07, + "loss": 0.018884683027863503, + "num_tokens": 1352632.0, + "reward": 52.063751220703125, + "reward_std": 65.35551452636719, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 51.96375274658203, + "rewards/supergames_reward/std": 65.35551452636719, + "step": 159, + "step_time": 12.306456914986484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.0, + "completions/max_terminated_length": 225.0, + "completions/mean_length": 184.0, + "completions/mean_terminated_length": 184.0, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.47005388140678406, + "epoch": 0.8, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.585808515548706, + "learning_rate": 2.0499999999999997e-07, + "loss": -0.021751077845692635, + "num_tokens": 1360528.0, + "reward": 33.73124694824219, + "reward_std": 78.67134857177734, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 33.631248474121094, + "rewards/supergames_reward/std": 78.67134857177734, + "step": 160, + "step_time": 8.517202848976012 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 456.0, + "completions/max_terminated_length": 456.0, + "completions/mean_length": 248.75, + "completions/mean_terminated_length": 248.75, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.39807969331741333, + "epoch": 0.805, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.8931398391723633, + "learning_rate": 2e-07, + "loss": -0.05018138885498047, + "num_tokens": 1371854.0, + "reward": 4.002498626708984, + "reward_std": 103.03063201904297, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 3.9274978637695312, + "rewards/supergames_reward/std": 103.00178527832031, + "step": 161, + "step_time": 17.225145012984285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 400.0, + "completions/max_terminated_length": 400.0, + "completions/mean_length": 231.125, + "completions/mean_terminated_length": 231.125, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.49391740560531616, + "epoch": 0.81, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5471365451812744, + "learning_rate": 1.9499999999999999e-07, + "loss": -0.18011468648910522, + "num_tokens": 1383015.0, + "reward": 67.2449951171875, + "reward_std": 69.01223754882812, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 67.14500427246094, + "rewards/supergames_reward/std": 69.01224517822266, + "step": 162, + "step_time": 15.188760302000446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 262.0, + "completions/max_terminated_length": 262.0, + "completions/mean_length": 221.875, + "completions/mean_terminated_length": 221.875, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.388366162776947, + "epoch": 0.815, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.662609815597534, + "learning_rate": 1.8999999999999998e-07, + "loss": 1.8455075405654497e-05, + "num_tokens": 1393150.0, + "reward": 59.23624801635742, + "reward_std": 76.00599670410156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 59.13624954223633, + "rewards/supergames_reward/std": 76.00599670410156, + "step": 163, + "step_time": 10.180389262997778 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 323.625, + "completions/mean_terminated_length": 323.625, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.45717528462409973, + "epoch": 0.82, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.580068826675415, + "learning_rate": 1.85e-07, + "loss": 0.024594159796833992, + "num_tokens": 1405067.0, + "reward": 6.318748474121094, + "reward_std": 88.21497344970703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 6.21875, + "rewards/supergames_reward/std": 88.21497344970703, + "step": 164, + "step_time": 15.074436638999032 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 163.0, + "completions/max_terminated_length": 163.0, + "completions/mean_length": 133.875, + "completions/mean_terminated_length": 133.875, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.3449660539627075, + "epoch": 0.825, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.045867443084717, + "learning_rate": 1.8e-07, + "loss": -0.02517889253795147, + "num_tokens": 1410730.0, + "reward": 84.35499572753906, + "reward_std": 38.135921478271484, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 84.2550048828125, + "rewards/supergames_reward/std": 38.13592529296875, + "step": 165, + "step_time": 6.275273430015659 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 144.0, + "completions/max_terminated_length": 144.0, + "completions/mean_length": 105.875, + "completions/mean_terminated_length": 105.875, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "entropy": 0.286411851644516, + "epoch": 0.83, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.855896472930908, + "learning_rate": 1.75e-07, + "loss": 0.014497723430395126, + "num_tokens": 1416081.0, + "reward": 48.40374755859375, + "reward_std": 47.61183547973633, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.303749084472656, + "rewards/supergames_reward/std": 47.611839294433594, + "step": 166, + "step_time": 5.5562305669882335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 226.125, + "completions/mean_terminated_length": 226.125, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "entropy": 0.39676305651664734, + "epoch": 0.835, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.633286237716675, + "learning_rate": 1.7000000000000001e-07, + "loss": -0.008210637606680393, + "num_tokens": 1426194.0, + "reward": 67.51000213623047, + "reward_std": 58.9198112487793, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 67.41000366210938, + "rewards/supergames_reward/std": 58.9198112487793, + "step": 167, + "step_time": 10.435288048989605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 210.0, + "completions/max_terminated_length": 210.0, + "completions/mean_length": 168.5, + "completions/mean_terminated_length": 168.5, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.4233987629413605, + "epoch": 0.84, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.654274940490723, + "learning_rate": 1.65e-07, + "loss": -0.0319184847176075, + "num_tokens": 1433990.0, + "reward": 50.342498779296875, + "reward_std": 67.66273498535156, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.26750183105469, + "rewards/supergames_reward/std": 67.5992202758789, + "step": 168, + "step_time": 8.111304188991198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 397.0, + "completions/max_terminated_length": 397.0, + "completions/mean_length": 217.75, + "completions/mean_terminated_length": 217.75, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.41511332988739014, + "epoch": 0.845, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.156782150268555, + "learning_rate": 1.6e-07, + "loss": 0.30166593194007874, + "num_tokens": 1444068.0, + "reward": 48.64500045776367, + "reward_std": 51.50307846069336, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.54499816894531, + "rewards/supergames_reward/std": 51.50307846069336, + "step": 169, + "step_time": 14.915170635009417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/max_terminated_length": 245.0, + "completions/mean_length": 218.625, + "completions/mean_terminated_length": 218.625, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "entropy": 0.3981763422489166, + "epoch": 0.85, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.818580389022827, + "learning_rate": 1.55e-07, + "loss": -0.08915036171674728, + "num_tokens": 1454153.0, + "reward": 82.69249725341797, + "reward_std": 24.65846824645996, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 82.59249877929688, + "rewards/supergames_reward/std": 24.65846824645996, + "step": 170, + "step_time": 9.446723325003404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 386.0, + "completions/max_terminated_length": 386.0, + "completions/mean_length": 280.375, + "completions/mean_terminated_length": 280.375, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.39121532440185547, + "epoch": 0.855, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.067988634109497, + "learning_rate": 1.5e-07, + "loss": -0.28749391436576843, + "num_tokens": 1465692.0, + "reward": 66.02375030517578, + "reward_std": 70.814697265625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 65.92375183105469, + "rewards/supergames_reward/std": 70.814697265625, + "step": 171, + "step_time": 14.603269515006104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 188.0, + "completions/max_terminated_length": 188.0, + "completions/mean_length": 131.125, + "completions/mean_terminated_length": 131.125, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.435172438621521, + "epoch": 0.86, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.832774639129639, + "learning_rate": 1.45e-07, + "loss": -0.1204412579536438, + "num_tokens": 1471253.0, + "reward": 39.80875015258789, + "reward_std": 51.58554458618164, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 39.7087516784668, + "rewards/supergames_reward/std": 51.585548400878906, + "step": 172, + "step_time": 7.057335219986271 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 391.0, + "completions/max_terminated_length": 391.0, + "completions/mean_length": 271.375, + "completions/mean_terminated_length": 271.375, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "entropy": 0.4081321060657501, + "epoch": 0.865, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2880308628082275, + "learning_rate": 1.4e-07, + "loss": 0.03748488798737526, + "num_tokens": 1482744.0, + "reward": 53.62249755859375, + "reward_std": 75.1306381225586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 53.522499084472656, + "rewards/supergames_reward/std": 75.1306381225586, + "step": 173, + "step_time": 14.96632127999328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 304.0, + "completions/max_terminated_length": 304.0, + "completions/mean_length": 203.875, + "completions/mean_terminated_length": 203.875, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "entropy": 0.4152996242046356, + "epoch": 0.87, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.1565890312194824, + "learning_rate": 1.35e-07, + "loss": -0.047037553042173386, + "num_tokens": 1492679.0, + "reward": 37.48374938964844, + "reward_std": 72.71965789794922, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 37.383750915527344, + "rewards/supergames_reward/std": 72.71966552734375, + "step": 174, + "step_time": 11.566132662002929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/max_terminated_length": 187.0, + "completions/mean_length": 133.375, + "completions/mean_terminated_length": 133.375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.3746980130672455, + "epoch": 0.875, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.724977970123291, + "learning_rate": 1.3e-07, + "loss": 0.05197479575872421, + "num_tokens": 1498330.0, + "reward": 37.86625289916992, + "reward_std": 52.43158721923828, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 37.76625061035156, + "rewards/supergames_reward/std": 52.43158721923828, + "step": 175, + "step_time": 7.080777855007909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 448.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 277.25, + "completions/mean_terminated_length": 277.25, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "entropy": 0.397225022315979, + "epoch": 0.88, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.438502788543701, + "learning_rate": 1.25e-07, + "loss": 0.1022070124745369, + "num_tokens": 1509860.0, + "reward": -41.323753356933594, + "reward_std": 79.704345703125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -41.42374801635742, + "rewards/supergames_reward/std": 79.704345703125, + "step": 176, + "step_time": 16.852883081999607 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/max_terminated_length": 187.0, + "completions/mean_length": 145.875, + "completions/mean_terminated_length": 145.875, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.34882763028144836, + "epoch": 0.885, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.683617115020752, + "learning_rate": 1.2e-07, + "loss": 0.0609549917280674, + "num_tokens": 1515587.0, + "reward": 84.27999877929688, + "reward_std": 38.06671905517578, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 84.17999267578125, + "rewards/supergames_reward/std": 38.06672286987305, + "step": 177, + "step_time": 7.09286071601673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 217.0, + "completions/max_terminated_length": 217.0, + "completions/mean_length": 181.875, + "completions/mean_terminated_length": 181.875, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "entropy": 0.423494815826416, + "epoch": 0.89, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3993117809295654, + "learning_rate": 1.15e-07, + "loss": 0.04823008552193642, + "num_tokens": 1523490.0, + "reward": 22.46124839782715, + "reward_std": 89.67146301269531, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 22.361249923706055, + "rewards/supergames_reward/std": 89.67146301269531, + "step": 178, + "step_time": 8.281584940006724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/max_terminated_length": 315.0, + "completions/mean_length": 273.25, + "completions/mean_terminated_length": 273.25, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.3831624984741211, + "epoch": 0.895, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7383997440338135, + "learning_rate": 1.0999999999999999e-07, + "loss": -0.010000350885093212, + "num_tokens": 1535012.0, + "reward": 2.237499237060547, + "reward_std": 93.32353210449219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 2.137500762939453, + "rewards/supergames_reward/std": 93.32353973388672, + "step": 179, + "step_time": 12.098909514024854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 395.0, + "completions/max_terminated_length": 395.0, + "completions/mean_length": 197.375, + "completions/mean_terminated_length": 197.375, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.5201653242111206, + "epoch": 0.9, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4596073627471924, + "learning_rate": 1.0499999999999999e-07, + "loss": -0.11710439622402191, + "num_tokens": 1543023.0, + "reward": 88.1875, + "reward_std": 33.69363784790039, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 88.0875015258789, + "rewards/supergames_reward/std": 33.693641662597656, + "step": 180, + "step_time": 14.662479719001567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 351.0, + "completions/max_terminated_length": 351.0, + "completions/mean_length": 218.75, + "completions/mean_terminated_length": 218.75, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "entropy": 0.4402031898498535, + "epoch": 0.905, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2664496898651123, + "learning_rate": 1e-07, + "loss": -0.038904689252376556, + "num_tokens": 1554085.0, + "reward": 71.10874938964844, + "reward_std": 69.99549102783203, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 71.00875091552734, + "rewards/supergames_reward/std": 69.99549865722656, + "step": 181, + "step_time": 13.245673589000944 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 328.0, + "completions/max_terminated_length": 328.0, + "completions/mean_length": 263.375, + "completions/mean_terminated_length": 263.375, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "entropy": 0.37136757373809814, + "epoch": 0.91, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.946763277053833, + "learning_rate": 9.499999999999999e-08, + "loss": 0.010829905979335308, + "num_tokens": 1565512.0, + "reward": 52.63500213623047, + "reward_std": 73.81818389892578, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 52.534996032714844, + "rewards/supergames_reward/std": 73.81818389892578, + "step": 182, + "step_time": 12.547108427999774 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 333.0, + "completions/max_terminated_length": 333.0, + "completions/mean_length": 263.0, + "completions/mean_terminated_length": 263.0, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "entropy": 0.3846859633922577, + "epoch": 0.915, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7999610900878906, + "learning_rate": 9e-08, + "loss": 0.004038939252495766, + "num_tokens": 1576960.0, + "reward": 71.89500427246094, + "reward_std": 45.18279266357422, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 71.79499816894531, + "rewards/supergames_reward/std": 45.182796478271484, + "step": 183, + "step_time": 12.604182851006044 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 216.375, + "completions/mean_terminated_length": 216.375, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "entropy": 0.36285799741744995, + "epoch": 0.92, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.070614814758301, + "learning_rate": 8.500000000000001e-08, + "loss": 0.08384927362203598, + "num_tokens": 1587019.0, + "reward": -29.672500610351562, + "reward_std": 94.68096923828125, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -29.772499084472656, + "rewards/supergames_reward/std": 94.68096923828125, + "step": 184, + "step_time": 10.202588220010512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 314.0, + "completions/max_terminated_length": 314.0, + "completions/mean_length": 239.375, + "completions/mean_terminated_length": 239.375, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "entropy": 0.43123888969421387, + "epoch": 0.925, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0129008293151855, + "learning_rate": 8e-08, + "loss": 0.03122909925878048, + "num_tokens": 1597230.0, + "reward": 10.326248168945312, + "reward_std": 66.74771881103516, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 10.226249694824219, + "rewards/supergames_reward/std": 66.74771881103516, + "step": 185, + "step_time": 11.8433545169828 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/max_terminated_length": 266.0, + "completions/mean_length": 219.75, + "completions/mean_terminated_length": 219.75, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "entropy": 0.39456790685653687, + "epoch": 0.93, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.786937952041626, + "learning_rate": 7.5e-08, + "loss": 0.01745336316525936, + "num_tokens": 1607300.0, + "reward": 61.45499801635742, + "reward_std": 59.6201171875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 61.35499954223633, + "rewards/supergames_reward/std": 59.620121002197266, + "step": 186, + "step_time": 10.153366808983264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 221.0, + "completions/max_terminated_length": 221.0, + "completions/mean_length": 158.5, + "completions/mean_terminated_length": 158.5, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.399124413728714, + "epoch": 0.935, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.657733917236328, + "learning_rate": 7e-08, + "loss": -0.05617351084947586, + "num_tokens": 1615000.0, + "reward": -12.976250648498535, + "reward_std": 83.94307708740234, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -13.051249504089355, + "rewards/supergames_reward/std": 83.9134521484375, + "step": 187, + "step_time": 8.401661271986086 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 131.25, + "completions/mean_terminated_length": 131.25, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.38221123814582825, + "epoch": 0.94, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.81663703918457, + "learning_rate": 6.5e-08, + "loss": 0.1486133486032486, + "num_tokens": 1620586.0, + "reward": 32.564998626708984, + "reward_std": 68.18844604492188, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 32.46500015258789, + "rewards/supergames_reward/std": 68.18844604492188, + "step": 188, + "step_time": 7.095077903999481 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 369.0, + "completions/max_terminated_length": 369.0, + "completions/mean_length": 273.75, + "completions/mean_terminated_length": 273.75, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "entropy": 0.4164618253707886, + "epoch": 0.945, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0384137630462646, + "learning_rate": 6e-08, + "loss": -0.013533106073737144, + "num_tokens": 1632056.0, + "reward": 28.412498474121094, + "reward_std": 99.22362518310547, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 28.3125, + "rewards/supergames_reward/std": 99.22362518310547, + "step": 189, + "step_time": 13.92420879600104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 146.0, + "completions/max_terminated_length": 146.0, + "completions/mean_length": 116.25, + "completions/mean_terminated_length": 116.25, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.2739017605781555, + "epoch": 0.95, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.924455165863037, + "learning_rate": 5.4999999999999996e-08, + "loss": -0.10890161991119385, + "num_tokens": 1637538.0, + "reward": 57.61750030517578, + "reward_std": 71.02250671386719, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 57.51750183105469, + "rewards/supergames_reward/std": 71.02251434326172, + "step": 190, + "step_time": 5.542224484990584 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 351.0, + "completions/max_terminated_length": 351.0, + "completions/mean_length": 265.375, + "completions/mean_terminated_length": 265.375, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "entropy": 0.38974297046661377, + "epoch": 0.955, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5310401916503906, + "learning_rate": 5e-08, + "loss": 0.10060633718967438, + "num_tokens": 1649029.0, + "reward": 68.22624206542969, + "reward_std": 69.13253021240234, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 68.12625122070312, + "rewards/supergames_reward/std": 69.13253021240234, + "step": 191, + "step_time": 13.248786654992728 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 309.0, + "completions/max_terminated_length": 309.0, + "completions/mean_length": 228.25, + "completions/mean_terminated_length": 228.25, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "entropy": 0.4095747470855713, + "epoch": 0.96, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.988938331604004, + "learning_rate": 4.5e-08, + "loss": 0.012724814936518669, + "num_tokens": 1659151.0, + "reward": 22.46999740600586, + "reward_std": 71.24070739746094, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 22.369998931884766, + "rewards/supergames_reward/std": 71.2406997680664, + "step": 192, + "step_time": 11.694355098996311 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 332.0, + "completions/max_terminated_length": 332.0, + "completions/mean_length": 183.375, + "completions/mean_terminated_length": 183.375, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "entropy": 0.3741522431373596, + "epoch": 0.965, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.120519161224365, + "learning_rate": 4e-08, + "loss": 0.18378837406635284, + "num_tokens": 1667042.0, + "reward": 18.967496871948242, + "reward_std": 91.04219818115234, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 18.86750030517578, + "rewards/supergames_reward/std": 91.04219818115234, + "step": 193, + "step_time": 12.186350238014711 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 287.0, + "completions/mean_length": 275.125, + "completions/mean_terminated_length": 241.2857208251953, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "entropy": 0.31828516721725464, + "epoch": 0.97, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.3239247798919678, + "learning_rate": 3.5e-08, + "loss": 0.2439030557870865, + "num_tokens": 1677579.0, + "reward": 22.55124855041504, + "reward_std": 74.4148941040039, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 22.47624969482422, + "rewards/supergames_reward/std": 74.3678207397461, + "step": 194, + "step_time": 18.785672573983902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 183.0, + "completions/max_terminated_length": 183.0, + "completions/mean_length": 162.625, + "completions/mean_terminated_length": 162.625, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "entropy": 0.37672579288482666, + "epoch": 0.975, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.524571657180786, + "learning_rate": 3e-08, + "loss": -0.002952136332169175, + "num_tokens": 1685304.0, + "reward": 17.362499237060547, + "reward_std": 67.76500701904297, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 17.262500762939453, + "rewards/supergames_reward/std": 67.76500701904297, + "step": 195, + "step_time": 7.0316317139950115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 391.0, + "completions/max_terminated_length": 391.0, + "completions/mean_length": 290.0, + "completions/mean_terminated_length": 290.0, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "entropy": 0.3798663914203644, + "epoch": 0.98, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.5e-08, + "loss": 0.0, + "num_tokens": 1696944.0, + "reward": 100.0999984741211, + "reward_std": 0.0, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 100.0, + "rewards/supergames_reward/std": 0.0, + "step": 196, + "step_time": 14.659201105998363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 307.0, + "completions/max_terminated_length": 307.0, + "completions/mean_length": 230.375, + "completions/mean_terminated_length": 230.375, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "entropy": 0.3792188763618469, + "epoch": 0.985, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9155354499816895, + "learning_rate": 2e-08, + "loss": 0.08655659109354019, + "num_tokens": 1707123.0, + "reward": 80.99250030517578, + "reward_std": 47.69900131225586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 80.89250183105469, + "rewards/supergames_reward/std": 47.69900131225586, + "step": 197, + "step_time": 11.552438066981267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 420.0, + "completions/max_terminated_length": 420.0, + "completions/mean_length": 268.0, + "completions/mean_terminated_length": 268.0, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "entropy": 0.5007117986679077, + "epoch": 0.99, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9498138427734375, + "learning_rate": 1.5e-08, + "loss": 0.0345655158162117, + "num_tokens": 1718579.0, + "reward": 51.65749740600586, + "reward_std": 83.32228088378906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 51.557498931884766, + "rewards/supergames_reward/std": 83.3222885131836, + "step": 198, + "step_time": 15.654064546979498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 157.0, + "completions/max_terminated_length": 157.0, + "completions/mean_length": 120.375, + "completions/mean_terminated_length": 120.375, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "entropy": 0.3440852761268616, + "epoch": 0.995, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.353044509887695, + "learning_rate": 1e-08, + "loss": -0.08407377451658249, + "num_tokens": 1724094.0, + "reward": 77.20124816894531, + "reward_std": 9.252492904663086, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 77.10124969482422, + "rewards/supergames_reward/std": 9.25249195098877, + "step": 199, + "step_time": 5.929525154992007 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 278.0, + "completions/max_terminated_length": 278.0, + "completions/mean_length": 182.5, + "completions/mean_terminated_length": 182.5, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "entropy": 0.46504852175712585, + "epoch": 1.0, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5138256549835205, + "learning_rate": 5e-09, + "loss": -0.025602273643016815, + "num_tokens": 1731978.0, + "reward": 24.10624885559082, + "reward_std": 56.38955307006836, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 24.006248474121094, + "rewards/supergames_reward/std": 56.38955307006836, + "step": 200, + "step_time": 10.38404856598936 + } + ], + "logging_steps": 1, + "max_steps": 200, + "num_input_tokens_seen": 1731978, + "num_train_epochs": 1, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-200/training_args.bin b/checkpoint-200/training_args.bin new file mode 100644 index 0000000..730dec0 --- /dev/null +++ b/checkpoint-200/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691fe5be2d7d1d56160dae63866d136877a72efb30e15ff7b3249192f998a788 +size 7185 diff --git a/checkpoint-40/chat_template.jinja b/checkpoint-40/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/checkpoint-40/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-40/config.json b/checkpoint-40/config.json new file mode 100644 index 0000000..f57b09b --- /dev/null +++ b/checkpoint-40/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-40/generation_config.json b/checkpoint-40/generation_config.json new file mode 100644 index 0000000..1b2bba9 --- /dev/null +++ b/checkpoint-40/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/checkpoint-40/model.safetensors b/checkpoint-40/model.safetensors new file mode 100644 index 0000000..e8583b3 --- /dev/null +++ b/checkpoint-40/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff9808a347f7c1f6723676d9b64c93e8fb373f523e804c982e79f9232ad6dff9 +size 6174895536 diff --git a/checkpoint-40/optimizer.pt b/checkpoint-40/optimizer.pt new file mode 100644 index 0000000..d87ec02 --- /dev/null +++ b/checkpoint-40/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3d9bf1d973ca75b6888d27fd8f016d457eb9428e6ae62d0e481aca478fb2502 +size 12350013801 diff --git a/checkpoint-40/rng_state.pth b/checkpoint-40/rng_state.pth new file mode 100644 index 0000000..b1f5e0b --- /dev/null +++ b/checkpoint-40/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cd4224f54462732698d27ac3a85f81b4b84f7f8f6f508de186eac62c4e74852 +size 14645 diff --git a/checkpoint-40/scheduler.pt b/checkpoint-40/scheduler.pt new file mode 100644 index 0000000..a3822b0 --- /dev/null +++ b/checkpoint-40/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d12172e6d50575e5620355d22f9f62e7d87de349d6a965fe2f8a67e390b9f8cf +size 1465 diff --git a/checkpoint-40/tokenizer.json b/checkpoint-40/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-40/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-40/tokenizer_config.json b/checkpoint-40/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-40/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-40/trainer_state.json b/checkpoint-40/trainer_state.json new file mode 100644 index 0000000..b1f562a --- /dev/null +++ b/checkpoint-40/trainer_state.json @@ -0,0 +1,1194 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.2, + "eval_steps": 500, + "global_step": 40, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 191.0, + "completions/mean_terminated_length": 191.0, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "entropy": 0.5566893219947815, + "epoch": 0.005, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.370361328125, + "learning_rate": 1e-06, + "loss": -0.24658073484897614, + "num_tokens": 10832.0, + "reward": -3.3000030517578125, + "reward_std": 85.62333679199219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.4000015258789062, + "rewards/supergames_reward/std": 85.62333679199219, + "step": 1, + "step_time": 22.779711072013015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/max_terminated_length": 216.0, + "completions/mean_length": 131.5, + "completions/mean_terminated_length": 131.5, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.6395841240882874, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.09774398803711, + "learning_rate": 9.95e-07, + "loss": 0.20567649602890015, + "num_tokens": 16404.0, + "reward": -12.422499656677246, + "reward_std": 7.134707450866699, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.522500038146973, + "rewards/supergames_reward/std": 7.134707927703857, + "step": 2, + "step_time": 9.960156448010821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/max_terminated_length": 287.0, + "completions/mean_length": 156.625, + "completions/mean_terminated_length": 156.625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "entropy": 0.562222421169281, + "epoch": 0.015, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9053616523742676, + "learning_rate": 9.9e-07, + "loss": 0.09602774679660797, + "num_tokens": 26953.0, + "reward": -74.94999694824219, + "reward_std": 70.73094177246094, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -75.0, + "rewards/supergames_reward/std": 70.71067810058594, + "step": 3, + "step_time": 13.686320498993155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 154.125, + "completions/mean_terminated_length": 154.125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7323317527770996, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.933310031890869, + "learning_rate": 9.849999999999999e-07, + "loss": 0.28249427676200867, + "num_tokens": 36514.0, + "reward": -40.7599983215332, + "reward_std": 81.73140716552734, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -40.80999755859375, + "rewards/supergames_reward/std": 81.68995666503906, + "step": 4, + "step_time": 12.805880262021674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 205.25, + "completions/mean_terminated_length": 205.25, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.6167430281639099, + "epoch": 0.025, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.344135046005249, + "learning_rate": 9.8e-07, + "loss": 0.1433994024991989, + "num_tokens": 47476.0, + "reward": -97.6612548828125, + "reward_std": 6.3321428298950195, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.76124572753906, + "rewards/supergames_reward/std": 6.3321428298950195, + "step": 5, + "step_time": 18.789364666008623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/max_terminated_length": 276.0, + "completions/mean_length": 167.25, + "completions/mean_terminated_length": 167.25, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.7583790421485901, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.172371864318848, + "learning_rate": 9.75e-07, + "loss": -0.12127404659986496, + "num_tokens": 55246.0, + "reward": -58.687503814697266, + "reward_std": 58.64059829711914, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -58.76250457763672, + "rewards/supergames_reward/std": 58.620460510253906, + "step": 6, + "step_time": 12.948570273991209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 133.875, + "completions/mean_terminated_length": 133.875, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.7447654604911804, + "epoch": 0.035, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.428595066070557, + "learning_rate": 9.7e-07, + "loss": 0.10135584324598312, + "num_tokens": 60837.0, + "reward": -32.45000076293945, + "reward_std": 41.754486083984375, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -32.5, + "rewards/supergames_reward/std": 41.66190719604492, + "step": 7, + "step_time": 12.058315072004916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 414.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 209.125, + "completions/mean_terminated_length": 209.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.3860666751861572, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.806255578994751, + "learning_rate": 9.649999999999999e-07, + "loss": -0.33238139748573303, + "num_tokens": 70838.0, + "reward": -80.50375366210938, + "reward_std": 38.82380294799805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -80.60375213623047, + "rewards/supergames_reward/std": 38.82379913330078, + "step": 8, + "step_time": 19.323370319994865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 247.125, + "completions/mean_terminated_length": 158.83334350585938, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.5816237330436707, + "epoch": 0.045, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3209943771362305, + "learning_rate": 9.6e-07, + "loss": 0.36752766370773315, + "num_tokens": 82135.0, + "reward": -27.688751220703125, + "reward_std": 82.8358154296875, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -27.738750457763672, + "rewards/supergames_reward/std": 82.78590393066406, + "step": 9, + "step_time": 21.948575104994234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/max_terminated_length": 240.0, + "completions/mean_length": 153.0, + "completions/mean_terminated_length": 153.0, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6301521062850952, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.125359058380127, + "learning_rate": 9.55e-07, + "loss": 0.06129350885748863, + "num_tokens": 89807.0, + "reward": -78.39624786376953, + "reward_std": 40.17042922973633, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -78.44625091552734, + "rewards/supergames_reward/std": 40.1396484375, + "step": 10, + "step_time": 9.098202993016457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/max_terminated_length": 133.0, + "completions/mean_length": 112.0, + "completions/mean_terminated_length": 112.0, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.5521990656852722, + "epoch": 0.055, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.616060256958008, + "learning_rate": 9.499999999999999e-07, + "loss": -0.006313305348157883, + "num_tokens": 95231.0, + "reward": -12.161249160766602, + "reward_std": 6.39578104019165, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.261249542236328, + "rewards/supergames_reward/std": 6.39578104019165, + "step": 11, + "step_time": 5.136311663984088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 214.875, + "completions/mean_terminated_length": 214.875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.5505905151367188, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.289583206176758, + "learning_rate": 9.45e-07, + "loss": -0.29802003502845764, + "num_tokens": 106238.0, + "reward": -91.66999816894531, + "reward_std": 23.2779541015625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -91.77000427246094, + "rewards/supergames_reward/std": 23.277956008911133, + "step": 12, + "step_time": 14.945365622988902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 167.375, + "completions/mean_terminated_length": 167.375, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "entropy": 0.540415346622467, + "epoch": 0.065, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.858291149139404, + "learning_rate": 9.399999999999999e-07, + "loss": 0.24004128575325012, + "num_tokens": 116937.0, + "reward": -20.38249969482422, + "reward_std": 87.55204010009766, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.457500457763672, + "rewards/supergames_reward/std": 87.52605438232422, + "step": 13, + "step_time": 10.482285185018554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "entropy": 0.7237679958343506, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.083980560302734, + "learning_rate": 9.35e-07, + "loss": -0.19025824964046478, + "num_tokens": 124640.0, + "reward": -51.397499084472656, + "reward_std": 54.486454010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -51.49749755859375, + "rewards/supergames_reward/std": 54.486454010009766, + "step": 14, + "step_time": 9.597231683001155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 141.375, + "completions/mean_terminated_length": 141.375, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.754334568977356, + "epoch": 0.075, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.720428943634033, + "learning_rate": 9.3e-07, + "loss": -0.16830675303936005, + "num_tokens": 132219.0, + "reward": -56.226253509521484, + "reward_std": 60.115657806396484, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -56.32625198364258, + "rewards/supergames_reward/std": 60.11566162109375, + "step": 15, + "step_time": 8.998362872982398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.0, + "completions/max_terminated_length": 209.0, + "completions/mean_length": 136.375, + "completions/mean_terminated_length": 136.375, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.6280568838119507, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.9681806564331055, + "learning_rate": 9.25e-07, + "loss": -0.01910916529595852, + "num_tokens": 139726.0, + "reward": -68.24000549316406, + "reward_std": 52.238914489746094, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.33999633789062, + "rewards/supergames_reward/std": 52.238914489746094, + "step": 16, + "step_time": 8.034480294008972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 211.875, + "completions/mean_terminated_length": 211.875, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.899176836013794, + "epoch": 0.085, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.691746711730957, + "learning_rate": 9.2e-07, + "loss": 0.25513291358947754, + "num_tokens": 147869.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 17, + "step_time": 13.655792869016295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 387.0, + "completions/max_terminated_length": 387.0, + "completions/mean_length": 238.0, + "completions/mean_terminated_length": 238.0, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.5596873760223389, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6711347103118896, + "learning_rate": 9.15e-07, + "loss": -0.2628335654735565, + "num_tokens": 159101.0, + "reward": -24.900001525878906, + "reward_std": 103.50983428955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.0, + "rewards/supergames_reward/std": 103.50983428955078, + "step": 18, + "step_time": 14.784329420013819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 277.0, + "completions/mean_length": 219.75, + "completions/mean_terminated_length": 178.00001525878906, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.9245517253875732, + "epoch": 0.095, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55502986907959, + "learning_rate": 9.1e-07, + "loss": 0.13060179352760315, + "num_tokens": 167299.0, + "reward": -86.89125061035156, + "reward_std": 36.95603561401367, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -86.94125366210938, + "rewards/supergames_reward/std": 36.93572235107422, + "step": 19, + "step_time": 18.832684700988466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/max_terminated_length": 267.0, + "completions/mean_length": 177.25, + "completions/mean_terminated_length": 177.25, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.7331254482269287, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.538497447967529, + "learning_rate": 9.05e-07, + "loss": -0.16418980062007904, + "num_tokens": 175141.0, + "reward": -68.52874755859375, + "reward_std": 58.48719787597656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.62875366210938, + "rewards/supergames_reward/std": 58.48720169067383, + "step": 20, + "step_time": 10.090975169994636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 190.625, + "completions/mean_terminated_length": 190.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.6659425497055054, + "epoch": 0.105, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.485373497009277, + "learning_rate": 9e-07, + "loss": -0.10106432437896729, + "num_tokens": 183098.0, + "reward": -71.48875427246094, + "reward_std": 55.33565139770508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -71.5887451171875, + "rewards/supergames_reward/std": 55.33565139770508, + "step": 21, + "step_time": 11.255001295008697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 179.0, + "completions/mean_terminated_length": 179.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.49365949630737305, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.829419136047363, + "learning_rate": 8.95e-07, + "loss": 0.10587231814861298, + "num_tokens": 192826.0, + "reward": -75.62000274658203, + "reward_std": 45.01959991455078, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -75.69499969482422, + "rewards/supergames_reward/std": 45.00411605834961, + "step": 22, + "step_time": 10.611246599000879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/max_terminated_length": 315.0, + "completions/mean_length": 212.625, + "completions/mean_terminated_length": 212.625, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.43159469962120056, + "epoch": 0.115, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.541125774383545, + "learning_rate": 8.9e-07, + "loss": -0.1681259125471115, + "num_tokens": 202815.0, + "reward": 23.67624855041504, + "reward_std": 67.74340057373047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 23.576250076293945, + "rewards/supergames_reward/std": 67.743408203125, + "step": 23, + "step_time": 11.731771531980485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/max_terminated_length": 168.0, + "completions/mean_length": 101.625, + "completions/mean_terminated_length": 101.625, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.3972298502922058, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.8285417556762695, + "learning_rate": 8.85e-07, + "loss": -0.011585958302021027, + "num_tokens": 208188.0, + "reward": 58.849998474121094, + "reward_std": 56.9304084777832, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 58.75, + "rewards/supergames_reward/std": 56.9304084777832, + "step": 24, + "step_time": 6.187504950998118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 303.0, + "completions/max_terminated_length": 303.0, + "completions/mean_length": 165.375, + "completions/mean_terminated_length": 165.375, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "entropy": 0.6339899301528931, + "epoch": 0.125, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.823799133300781, + "learning_rate": 8.799999999999999e-07, + "loss": 0.05159185826778412, + "num_tokens": 215935.0, + "reward": -23.521251678466797, + "reward_std": 74.23067474365234, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -23.621248245239258, + "rewards/supergames_reward/std": 74.23067474365234, + "step": 25, + "step_time": 11.059416300005978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 349.0, + "completions/max_terminated_length": 349.0, + "completions/mean_length": 204.0, + "completions/mean_terminated_length": 204.0, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.6462895274162292, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.969292640686035, + "learning_rate": 8.75e-07, + "loss": 0.0115677984431386, + "num_tokens": 223999.0, + "reward": -74.8175048828125, + "reward_std": 40.94221115112305, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -74.89250183105469, + "rewards/supergames_reward/std": 40.924625396728516, + "step": 26, + "step_time": 12.666237785975682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 166.5, + "completions/mean_terminated_length": 166.5, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4756850600242615, + "epoch": 0.135, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915069103240967, + "learning_rate": 8.699999999999999e-07, + "loss": -0.13321346044540405, + "num_tokens": 233643.0, + "reward": -52.125, + "reward_std": 19.405282974243164, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -52.20000076293945, + "rewards/supergames_reward/std": 19.33464813232422, + "step": 27, + "step_time": 11.172539752995363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 137.125, + "completions/mean_terminated_length": 137.125, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.5834778547286987, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.146666526794434, + "learning_rate": 8.65e-07, + "loss": 0.060454584658145905, + "num_tokens": 241204.0, + "reward": -68.99250793457031, + "reward_std": 51.010581970214844, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -69.09249877929688, + "rewards/supergames_reward/std": 51.010581970214844, + "step": 28, + "step_time": 7.025046669004951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 182.375, + "completions/mean_terminated_length": 182.375, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.6368551254272461, + "epoch": 0.145, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.15535306930542, + "learning_rate": 8.599999999999999e-07, + "loss": -0.3323673903942108, + "num_tokens": 249087.0, + "reward": -44.17000198364258, + "reward_std": 80.5373306274414, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -44.27000045776367, + "rewards/supergames_reward/std": 80.5373306274414, + "step": 29, + "step_time": 12.079259724996518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 127.75, + "completions/mean_terminated_length": 127.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4491196870803833, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.896186828613281, + "learning_rate": 8.55e-07, + "loss": -0.12887656688690186, + "num_tokens": 259445.0, + "reward": 57.446250915527344, + "reward_std": 38.78837585449219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 57.34625244140625, + "rewards/supergames_reward/std": 38.78837966918945, + "step": 30, + "step_time": 7.483972909016302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/max_terminated_length": 166.0, + "completions/mean_length": 112.875, + "completions/mean_terminated_length": 112.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4858454465866089, + "epoch": 0.155, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.742906093597412, + "learning_rate": 8.499999999999999e-07, + "loss": 0.06930096447467804, + "num_tokens": 264924.0, + "reward": -21.174999237060547, + "reward_std": 31.89051628112793, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -21.25, + "rewards/supergames_reward/std": 31.819807052612305, + "step": 31, + "step_time": 6.1866529019898735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/max_terminated_length": 148.0, + "completions/mean_length": 118.125, + "completions/mean_terminated_length": 118.125, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.45428749918937683, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.663414478302002, + "learning_rate": 8.45e-07, + "loss": 0.05735419690608978, + "num_tokens": 270381.0, + "reward": 0.4687504768371582, + "reward_std": 38.66371154785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 0.36875057220458984, + "rewards/supergames_reward/std": 38.6637077331543, + "step": 32, + "step_time": 5.577042757999152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 166.875, + "completions/mean_terminated_length": 166.875, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.6107293367385864, + "epoch": 0.165, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.75006628036499, + "learning_rate": 8.399999999999999e-07, + "loss": -0.14760127663612366, + "num_tokens": 278156.0, + "reward": -87.84750366210938, + "reward_std": 26.84708023071289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -87.94749450683594, + "rewards/supergames_reward/std": 26.847076416015625, + "step": 33, + "step_time": 12.552876825997373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 360.0, + "completions/mean_length": 231.0, + "completions/mean_terminated_length": 190.85714721679688, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.6089653372764587, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.781267166137695, + "learning_rate": 8.349999999999999e-07, + "loss": 0.4294504225254059, + "num_tokens": 289364.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 34, + "step_time": 18.961819477990502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 134.125, + "completions/mean_terminated_length": 134.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46158918738365173, + "epoch": 0.175, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.016353607177734, + "learning_rate": 8.299999999999999e-07, + "loss": 0.09251818805932999, + "num_tokens": 298733.0, + "reward": -35.558753967285156, + "reward_std": 15.881204605102539, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -35.65875244140625, + "rewards/supergames_reward/std": 15.881203651428223, + "step": 35, + "step_time": 8.38491243700264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/max_terminated_length": 207.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.5104788541793823, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.592893123626709, + "learning_rate": 8.249999999999999e-07, + "loss": -0.0410832017660141, + "num_tokens": 306420.0, + "reward": 5.4212493896484375, + "reward_std": 93.50404357910156, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 5.346250534057617, + "rewards/supergames_reward/std": 93.4718246459961, + "step": 36, + "step_time": 7.811868985998444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/max_terminated_length": 241.0, + "completions/mean_length": 178.0, + "completions/mean_terminated_length": 178.0, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.7527878880500793, + "epoch": 0.185, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8812546730041504, + "learning_rate": 8.199999999999999e-07, + "loss": 0.12539231777191162, + "num_tokens": 314252.0, + "reward": -89.58500671386719, + "reward_std": 29.256120681762695, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -89.66000366210938, + "rewards/supergames_reward/std": 29.24593734741211, + "step": 37, + "step_time": 8.959742914012168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 133.375, + "completions/mean_terminated_length": 133.375, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.3347901403903961, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.205414295196533, + "learning_rate": 8.149999999999999e-07, + "loss": -0.06991486996412277, + "num_tokens": 319839.0, + "reward": -14.498749732971191, + "reward_std": 13.007229804992676, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -14.598750114440918, + "rewards/supergames_reward/std": 13.007229804992676, + "step": 38, + "step_time": 7.377183554985095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/max_terminated_length": 181.0, + "completions/mean_length": 114.375, + "completions/mean_terminated_length": 114.375, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.4445965886116028, + "epoch": 0.195, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.561298847198486, + "learning_rate": 8.1e-07, + "loss": -0.06235164776444435, + "num_tokens": 325290.0, + "reward": -7.425000190734863, + "reward_std": 53.701602935791016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -7.5, + "rewards/supergames_reward/std": 53.652320861816406, + "step": 39, + "step_time": 6.753862089011818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 187.625, + "completions/mean_terminated_length": 187.625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.3950234353542328, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5988712310791016, + "learning_rate": 8.05e-07, + "loss": -0.1381440907716751, + "num_tokens": 335119.0, + "reward": -33.68375015258789, + "reward_std": 42.069435119628906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -33.783748626708984, + "rewards/supergames_reward/std": 42.069435119628906, + "step": 40, + "step_time": 13.718958162004128 + } + ], + "logging_steps": 1, + "max_steps": 200, + "num_input_tokens_seen": 335119, + "num_train_epochs": 1, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-40/training_args.bin b/checkpoint-40/training_args.bin new file mode 100644 index 0000000..730dec0 --- /dev/null +++ b/checkpoint-40/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691fe5be2d7d1d56160dae63866d136877a72efb30e15ff7b3249192f998a788 +size 7185 diff --git a/checkpoint-60/chat_template.jinja b/checkpoint-60/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/checkpoint-60/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-60/config.json b/checkpoint-60/config.json new file mode 100644 index 0000000..f57b09b --- /dev/null +++ b/checkpoint-60/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-60/generation_config.json b/checkpoint-60/generation_config.json new file mode 100644 index 0000000..1b2bba9 --- /dev/null +++ b/checkpoint-60/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/checkpoint-60/model.safetensors b/checkpoint-60/model.safetensors new file mode 100644 index 0000000..19e80b5 --- /dev/null +++ b/checkpoint-60/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e62af89c25fad92b5ed3fcff2d61135e77afdbbac6bf1de213b138bc5c886202 +size 6174895536 diff --git a/checkpoint-60/optimizer.pt b/checkpoint-60/optimizer.pt new file mode 100644 index 0000000..b432bf9 --- /dev/null +++ b/checkpoint-60/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff7d04ae88053c662cef57e6d5cc192af79abe8d30a164edb4b312e98b6908dd +size 12350013801 diff --git a/checkpoint-60/rng_state.pth b/checkpoint-60/rng_state.pth new file mode 100644 index 0000000..8b8159f --- /dev/null +++ b/checkpoint-60/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fae16bd6d032942a248f17a0f77c550f505e9c87781ba9a4b79a3b2d897cd14d +size 14645 diff --git a/checkpoint-60/scheduler.pt b/checkpoint-60/scheduler.pt new file mode 100644 index 0000000..ff57925 --- /dev/null +++ b/checkpoint-60/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de04036787404fb7ee6853d8c3f6a8fa92951d7f795a7b73b185729920015ad6 +size 1465 diff --git a/checkpoint-60/tokenizer.json b/checkpoint-60/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-60/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-60/tokenizer_config.json b/checkpoint-60/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-60/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-60/trainer_state.json b/checkpoint-60/trainer_state.json new file mode 100644 index 0000000..c15b75a --- /dev/null +++ b/checkpoint-60/trainer_state.json @@ -0,0 +1,1774 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.3, + "eval_steps": 500, + "global_step": 60, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 191.0, + "completions/mean_terminated_length": 191.0, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "entropy": 0.5566893219947815, + "epoch": 0.005, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.370361328125, + "learning_rate": 1e-06, + "loss": -0.24658073484897614, + "num_tokens": 10832.0, + "reward": -3.3000030517578125, + "reward_std": 85.62333679199219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.4000015258789062, + "rewards/supergames_reward/std": 85.62333679199219, + "step": 1, + "step_time": 22.779711072013015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/max_terminated_length": 216.0, + "completions/mean_length": 131.5, + "completions/mean_terminated_length": 131.5, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.6395841240882874, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.09774398803711, + "learning_rate": 9.95e-07, + "loss": 0.20567649602890015, + "num_tokens": 16404.0, + "reward": -12.422499656677246, + "reward_std": 7.134707450866699, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.522500038146973, + "rewards/supergames_reward/std": 7.134707927703857, + "step": 2, + "step_time": 9.960156448010821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/max_terminated_length": 287.0, + "completions/mean_length": 156.625, + "completions/mean_terminated_length": 156.625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "entropy": 0.562222421169281, + "epoch": 0.015, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9053616523742676, + "learning_rate": 9.9e-07, + "loss": 0.09602774679660797, + "num_tokens": 26953.0, + "reward": -74.94999694824219, + "reward_std": 70.73094177246094, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -75.0, + "rewards/supergames_reward/std": 70.71067810058594, + "step": 3, + "step_time": 13.686320498993155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 154.125, + "completions/mean_terminated_length": 154.125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7323317527770996, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.933310031890869, + "learning_rate": 9.849999999999999e-07, + "loss": 0.28249427676200867, + "num_tokens": 36514.0, + "reward": -40.7599983215332, + "reward_std": 81.73140716552734, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -40.80999755859375, + "rewards/supergames_reward/std": 81.68995666503906, + "step": 4, + "step_time": 12.805880262021674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 205.25, + "completions/mean_terminated_length": 205.25, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.6167430281639099, + "epoch": 0.025, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.344135046005249, + "learning_rate": 9.8e-07, + "loss": 0.1433994024991989, + "num_tokens": 47476.0, + "reward": -97.6612548828125, + "reward_std": 6.3321428298950195, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.76124572753906, + "rewards/supergames_reward/std": 6.3321428298950195, + "step": 5, + "step_time": 18.789364666008623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/max_terminated_length": 276.0, + "completions/mean_length": 167.25, + "completions/mean_terminated_length": 167.25, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.7583790421485901, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.172371864318848, + "learning_rate": 9.75e-07, + "loss": -0.12127404659986496, + "num_tokens": 55246.0, + "reward": -58.687503814697266, + "reward_std": 58.64059829711914, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -58.76250457763672, + "rewards/supergames_reward/std": 58.620460510253906, + "step": 6, + "step_time": 12.948570273991209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 133.875, + "completions/mean_terminated_length": 133.875, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.7447654604911804, + "epoch": 0.035, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.428595066070557, + "learning_rate": 9.7e-07, + "loss": 0.10135584324598312, + "num_tokens": 60837.0, + "reward": -32.45000076293945, + "reward_std": 41.754486083984375, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -32.5, + "rewards/supergames_reward/std": 41.66190719604492, + "step": 7, + "step_time": 12.058315072004916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 414.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 209.125, + "completions/mean_terminated_length": 209.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.3860666751861572, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.806255578994751, + "learning_rate": 9.649999999999999e-07, + "loss": -0.33238139748573303, + "num_tokens": 70838.0, + "reward": -80.50375366210938, + "reward_std": 38.82380294799805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -80.60375213623047, + "rewards/supergames_reward/std": 38.82379913330078, + "step": 8, + "step_time": 19.323370319994865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 247.125, + "completions/mean_terminated_length": 158.83334350585938, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.5816237330436707, + "epoch": 0.045, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3209943771362305, + "learning_rate": 9.6e-07, + "loss": 0.36752766370773315, + "num_tokens": 82135.0, + "reward": -27.688751220703125, + "reward_std": 82.8358154296875, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -27.738750457763672, + "rewards/supergames_reward/std": 82.78590393066406, + "step": 9, + "step_time": 21.948575104994234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/max_terminated_length": 240.0, + "completions/mean_length": 153.0, + "completions/mean_terminated_length": 153.0, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6301521062850952, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.125359058380127, + "learning_rate": 9.55e-07, + "loss": 0.06129350885748863, + "num_tokens": 89807.0, + "reward": -78.39624786376953, + "reward_std": 40.17042922973633, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -78.44625091552734, + "rewards/supergames_reward/std": 40.1396484375, + "step": 10, + "step_time": 9.098202993016457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/max_terminated_length": 133.0, + "completions/mean_length": 112.0, + "completions/mean_terminated_length": 112.0, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.5521990656852722, + "epoch": 0.055, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.616060256958008, + "learning_rate": 9.499999999999999e-07, + "loss": -0.006313305348157883, + "num_tokens": 95231.0, + "reward": -12.161249160766602, + "reward_std": 6.39578104019165, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.261249542236328, + "rewards/supergames_reward/std": 6.39578104019165, + "step": 11, + "step_time": 5.136311663984088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 214.875, + "completions/mean_terminated_length": 214.875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.5505905151367188, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.289583206176758, + "learning_rate": 9.45e-07, + "loss": -0.29802003502845764, + "num_tokens": 106238.0, + "reward": -91.66999816894531, + "reward_std": 23.2779541015625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -91.77000427246094, + "rewards/supergames_reward/std": 23.277956008911133, + "step": 12, + "step_time": 14.945365622988902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 167.375, + "completions/mean_terminated_length": 167.375, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "entropy": 0.540415346622467, + "epoch": 0.065, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.858291149139404, + "learning_rate": 9.399999999999999e-07, + "loss": 0.24004128575325012, + "num_tokens": 116937.0, + "reward": -20.38249969482422, + "reward_std": 87.55204010009766, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.457500457763672, + "rewards/supergames_reward/std": 87.52605438232422, + "step": 13, + "step_time": 10.482285185018554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "entropy": 0.7237679958343506, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.083980560302734, + "learning_rate": 9.35e-07, + "loss": -0.19025824964046478, + "num_tokens": 124640.0, + "reward": -51.397499084472656, + "reward_std": 54.486454010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -51.49749755859375, + "rewards/supergames_reward/std": 54.486454010009766, + "step": 14, + "step_time": 9.597231683001155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 141.375, + "completions/mean_terminated_length": 141.375, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.754334568977356, + "epoch": 0.075, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.720428943634033, + "learning_rate": 9.3e-07, + "loss": -0.16830675303936005, + "num_tokens": 132219.0, + "reward": -56.226253509521484, + "reward_std": 60.115657806396484, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -56.32625198364258, + "rewards/supergames_reward/std": 60.11566162109375, + "step": 15, + "step_time": 8.998362872982398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.0, + "completions/max_terminated_length": 209.0, + "completions/mean_length": 136.375, + "completions/mean_terminated_length": 136.375, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.6280568838119507, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.9681806564331055, + "learning_rate": 9.25e-07, + "loss": -0.01910916529595852, + "num_tokens": 139726.0, + "reward": -68.24000549316406, + "reward_std": 52.238914489746094, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.33999633789062, + "rewards/supergames_reward/std": 52.238914489746094, + "step": 16, + "step_time": 8.034480294008972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 211.875, + "completions/mean_terminated_length": 211.875, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.899176836013794, + "epoch": 0.085, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.691746711730957, + "learning_rate": 9.2e-07, + "loss": 0.25513291358947754, + "num_tokens": 147869.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 17, + "step_time": 13.655792869016295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 387.0, + "completions/max_terminated_length": 387.0, + "completions/mean_length": 238.0, + "completions/mean_terminated_length": 238.0, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.5596873760223389, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6711347103118896, + "learning_rate": 9.15e-07, + "loss": -0.2628335654735565, + "num_tokens": 159101.0, + "reward": -24.900001525878906, + "reward_std": 103.50983428955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.0, + "rewards/supergames_reward/std": 103.50983428955078, + "step": 18, + "step_time": 14.784329420013819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 277.0, + "completions/mean_length": 219.75, + "completions/mean_terminated_length": 178.00001525878906, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.9245517253875732, + "epoch": 0.095, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55502986907959, + "learning_rate": 9.1e-07, + "loss": 0.13060179352760315, + "num_tokens": 167299.0, + "reward": -86.89125061035156, + "reward_std": 36.95603561401367, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -86.94125366210938, + "rewards/supergames_reward/std": 36.93572235107422, + "step": 19, + "step_time": 18.832684700988466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/max_terminated_length": 267.0, + "completions/mean_length": 177.25, + "completions/mean_terminated_length": 177.25, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.7331254482269287, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.538497447967529, + "learning_rate": 9.05e-07, + "loss": -0.16418980062007904, + "num_tokens": 175141.0, + "reward": -68.52874755859375, + "reward_std": 58.48719787597656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.62875366210938, + "rewards/supergames_reward/std": 58.48720169067383, + "step": 20, + "step_time": 10.090975169994636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 190.625, + "completions/mean_terminated_length": 190.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.6659425497055054, + "epoch": 0.105, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.485373497009277, + "learning_rate": 9e-07, + "loss": -0.10106432437896729, + "num_tokens": 183098.0, + "reward": -71.48875427246094, + "reward_std": 55.33565139770508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -71.5887451171875, + "rewards/supergames_reward/std": 55.33565139770508, + "step": 21, + "step_time": 11.255001295008697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 179.0, + "completions/mean_terminated_length": 179.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.49365949630737305, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.829419136047363, + "learning_rate": 8.95e-07, + "loss": 0.10587231814861298, + "num_tokens": 192826.0, + "reward": -75.62000274658203, + "reward_std": 45.01959991455078, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -75.69499969482422, + "rewards/supergames_reward/std": 45.00411605834961, + "step": 22, + "step_time": 10.611246599000879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/max_terminated_length": 315.0, + "completions/mean_length": 212.625, + "completions/mean_terminated_length": 212.625, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.43159469962120056, + "epoch": 0.115, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.541125774383545, + "learning_rate": 8.9e-07, + "loss": -0.1681259125471115, + "num_tokens": 202815.0, + "reward": 23.67624855041504, + "reward_std": 67.74340057373047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 23.576250076293945, + "rewards/supergames_reward/std": 67.743408203125, + "step": 23, + "step_time": 11.731771531980485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/max_terminated_length": 168.0, + "completions/mean_length": 101.625, + "completions/mean_terminated_length": 101.625, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.3972298502922058, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.8285417556762695, + "learning_rate": 8.85e-07, + "loss": -0.011585958302021027, + "num_tokens": 208188.0, + "reward": 58.849998474121094, + "reward_std": 56.9304084777832, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 58.75, + "rewards/supergames_reward/std": 56.9304084777832, + "step": 24, + "step_time": 6.187504950998118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 303.0, + "completions/max_terminated_length": 303.0, + "completions/mean_length": 165.375, + "completions/mean_terminated_length": 165.375, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "entropy": 0.6339899301528931, + "epoch": 0.125, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.823799133300781, + "learning_rate": 8.799999999999999e-07, + "loss": 0.05159185826778412, + "num_tokens": 215935.0, + "reward": -23.521251678466797, + "reward_std": 74.23067474365234, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -23.621248245239258, + "rewards/supergames_reward/std": 74.23067474365234, + "step": 25, + "step_time": 11.059416300005978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 349.0, + "completions/max_terminated_length": 349.0, + "completions/mean_length": 204.0, + "completions/mean_terminated_length": 204.0, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.6462895274162292, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.969292640686035, + "learning_rate": 8.75e-07, + "loss": 0.0115677984431386, + "num_tokens": 223999.0, + "reward": -74.8175048828125, + "reward_std": 40.94221115112305, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -74.89250183105469, + "rewards/supergames_reward/std": 40.924625396728516, + "step": 26, + "step_time": 12.666237785975682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 166.5, + "completions/mean_terminated_length": 166.5, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4756850600242615, + "epoch": 0.135, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915069103240967, + "learning_rate": 8.699999999999999e-07, + "loss": -0.13321346044540405, + "num_tokens": 233643.0, + "reward": -52.125, + "reward_std": 19.405282974243164, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -52.20000076293945, + "rewards/supergames_reward/std": 19.33464813232422, + "step": 27, + "step_time": 11.172539752995363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 137.125, + "completions/mean_terminated_length": 137.125, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.5834778547286987, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.146666526794434, + "learning_rate": 8.65e-07, + "loss": 0.060454584658145905, + "num_tokens": 241204.0, + "reward": -68.99250793457031, + "reward_std": 51.010581970214844, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -69.09249877929688, + "rewards/supergames_reward/std": 51.010581970214844, + "step": 28, + "step_time": 7.025046669004951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 182.375, + "completions/mean_terminated_length": 182.375, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.6368551254272461, + "epoch": 0.145, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.15535306930542, + "learning_rate": 8.599999999999999e-07, + "loss": -0.3323673903942108, + "num_tokens": 249087.0, + "reward": -44.17000198364258, + "reward_std": 80.5373306274414, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -44.27000045776367, + "rewards/supergames_reward/std": 80.5373306274414, + "step": 29, + "step_time": 12.079259724996518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 127.75, + "completions/mean_terminated_length": 127.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4491196870803833, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.896186828613281, + "learning_rate": 8.55e-07, + "loss": -0.12887656688690186, + "num_tokens": 259445.0, + "reward": 57.446250915527344, + "reward_std": 38.78837585449219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 57.34625244140625, + "rewards/supergames_reward/std": 38.78837966918945, + "step": 30, + "step_time": 7.483972909016302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/max_terminated_length": 166.0, + "completions/mean_length": 112.875, + "completions/mean_terminated_length": 112.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4858454465866089, + "epoch": 0.155, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.742906093597412, + "learning_rate": 8.499999999999999e-07, + "loss": 0.06930096447467804, + "num_tokens": 264924.0, + "reward": -21.174999237060547, + "reward_std": 31.89051628112793, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -21.25, + "rewards/supergames_reward/std": 31.819807052612305, + "step": 31, + "step_time": 6.1866529019898735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/max_terminated_length": 148.0, + "completions/mean_length": 118.125, + "completions/mean_terminated_length": 118.125, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.45428749918937683, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.663414478302002, + "learning_rate": 8.45e-07, + "loss": 0.05735419690608978, + "num_tokens": 270381.0, + "reward": 0.4687504768371582, + "reward_std": 38.66371154785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 0.36875057220458984, + "rewards/supergames_reward/std": 38.6637077331543, + "step": 32, + "step_time": 5.577042757999152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 166.875, + "completions/mean_terminated_length": 166.875, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.6107293367385864, + "epoch": 0.165, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.75006628036499, + "learning_rate": 8.399999999999999e-07, + "loss": -0.14760127663612366, + "num_tokens": 278156.0, + "reward": -87.84750366210938, + "reward_std": 26.84708023071289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -87.94749450683594, + "rewards/supergames_reward/std": 26.847076416015625, + "step": 33, + "step_time": 12.552876825997373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 360.0, + "completions/mean_length": 231.0, + "completions/mean_terminated_length": 190.85714721679688, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.6089653372764587, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.781267166137695, + "learning_rate": 8.349999999999999e-07, + "loss": 0.4294504225254059, + "num_tokens": 289364.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 34, + "step_time": 18.961819477990502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 134.125, + "completions/mean_terminated_length": 134.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46158918738365173, + "epoch": 0.175, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.016353607177734, + "learning_rate": 8.299999999999999e-07, + "loss": 0.09251818805932999, + "num_tokens": 298733.0, + "reward": -35.558753967285156, + "reward_std": 15.881204605102539, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -35.65875244140625, + "rewards/supergames_reward/std": 15.881203651428223, + "step": 35, + "step_time": 8.38491243700264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/max_terminated_length": 207.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.5104788541793823, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.592893123626709, + "learning_rate": 8.249999999999999e-07, + "loss": -0.0410832017660141, + "num_tokens": 306420.0, + "reward": 5.4212493896484375, + "reward_std": 93.50404357910156, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 5.346250534057617, + "rewards/supergames_reward/std": 93.4718246459961, + "step": 36, + "step_time": 7.811868985998444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/max_terminated_length": 241.0, + "completions/mean_length": 178.0, + "completions/mean_terminated_length": 178.0, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.7527878880500793, + "epoch": 0.185, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8812546730041504, + "learning_rate": 8.199999999999999e-07, + "loss": 0.12539231777191162, + "num_tokens": 314252.0, + "reward": -89.58500671386719, + "reward_std": 29.256120681762695, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -89.66000366210938, + "rewards/supergames_reward/std": 29.24593734741211, + "step": 37, + "step_time": 8.959742914012168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 133.375, + "completions/mean_terminated_length": 133.375, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.3347901403903961, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.205414295196533, + "learning_rate": 8.149999999999999e-07, + "loss": -0.06991486996412277, + "num_tokens": 319839.0, + "reward": -14.498749732971191, + "reward_std": 13.007229804992676, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -14.598750114440918, + "rewards/supergames_reward/std": 13.007229804992676, + "step": 38, + "step_time": 7.377183554985095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/max_terminated_length": 181.0, + "completions/mean_length": 114.375, + "completions/mean_terminated_length": 114.375, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.4445965886116028, + "epoch": 0.195, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.561298847198486, + "learning_rate": 8.1e-07, + "loss": -0.06235164776444435, + "num_tokens": 325290.0, + "reward": -7.425000190734863, + "reward_std": 53.701602935791016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -7.5, + "rewards/supergames_reward/std": 53.652320861816406, + "step": 39, + "step_time": 6.753862089011818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 187.625, + "completions/mean_terminated_length": 187.625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.3950234353542328, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5988712310791016, + "learning_rate": 8.05e-07, + "loss": -0.1381440907716751, + "num_tokens": 335119.0, + "reward": -33.68375015258789, + "reward_std": 42.069435119628906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -33.783748626708984, + "rewards/supergames_reward/std": 42.069435119628906, + "step": 40, + "step_time": 13.718958162004128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 371.0, + "completions/mean_length": 251.125, + "completions/mean_terminated_length": 213.85714721679688, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.507067084312439, + "epoch": 0.205, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3898658752441406, + "learning_rate": 8e-07, + "loss": 0.0078964838758111, + "num_tokens": 346464.0, + "reward": -86.48625183105469, + "reward_std": 38.020694732666016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -86.56124877929688, + "rewards/supergames_reward/std": 38.010528564453125, + "step": 41, + "step_time": 18.90960379401804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 152.0, + "completions/max_terminated_length": 152.0, + "completions/mean_length": 116.375, + "completions/mean_terminated_length": 116.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3801707923412323, + "epoch": 0.21, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.95e-07, + "loss": 0.0, + "num_tokens": 351955.0, + "reward": -9.899999618530273, + "reward_std": 0.0, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -10.0, + "rewards/supergames_reward/std": 0.0, + "step": 42, + "step_time": 5.711630532023264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/max_terminated_length": 308.0, + "completions/mean_length": 247.375, + "completions/mean_terminated_length": 247.375, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.3185043931007385, + "epoch": 0.215, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.168762445449829, + "learning_rate": 7.9e-07, + "loss": 0.1970764398574829, + "num_tokens": 363278.0, + "reward": -53.89249801635742, + "reward_std": 63.02619934082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -53.99250030517578, + "rewards/supergames_reward/std": 63.02619934082031, + "step": 43, + "step_time": 11.661934906995157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/max_terminated_length": 330.0, + "completions/mean_length": 212.5, + "completions/mean_terminated_length": 212.5, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4446268379688263, + "epoch": 0.22, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.305844306945801, + "learning_rate": 7.85e-07, + "loss": -0.06735197454690933, + "num_tokens": 374306.0, + "reward": 18.059999465942383, + "reward_std": 98.9417953491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 17.959999084472656, + "rewards/supergames_reward/std": 98.9417953491211, + "step": 44, + "step_time": 12.492453911982011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.0, + "completions/max_terminated_length": 316.0, + "completions/mean_length": 183.0, + "completions/mean_terminated_length": 183.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.5269614458084106, + "epoch": 0.225, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8006370067596436, + "learning_rate": 7.799999999999999e-07, + "loss": -0.08870815485715866, + "num_tokens": 382210.0, + "reward": -54.928749084472656, + "reward_std": 72.5127944946289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -55.02874755859375, + "rewards/supergames_reward/std": 72.5127944946289, + "step": 45, + "step_time": 11.7636467939883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.0, + "completions/max_terminated_length": 244.0, + "completions/mean_length": 136.625, + "completions/mean_terminated_length": 136.625, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.34820204973220825, + "epoch": 0.23, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.911736011505127, + "learning_rate": 7.75e-07, + "loss": 0.04397330805659294, + "num_tokens": 391583.0, + "reward": -19.10375213623047, + "reward_std": 23.967702865600586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -19.203750610351562, + "rewards/supergames_reward/std": 23.967702865600586, + "step": 46, + "step_time": 10.119426151999505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 160.375, + "completions/mean_terminated_length": 160.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.611914336681366, + "epoch": 0.235, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.571913242340088, + "learning_rate": 7.699999999999999e-07, + "loss": -0.044293053448200226, + "num_tokens": 399290.0, + "reward": -25.186250686645508, + "reward_std": 71.46305084228516, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.286251068115234, + "rewards/supergames_reward/std": 71.46305084228516, + "step": 47, + "step_time": 8.563868903991533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/max_terminated_length": 167.0, + "completions/mean_length": 111.125, + "completions/mean_terminated_length": 111.125, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.46585777401924133, + "epoch": 0.24, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.654109954833984, + "learning_rate": 7.65e-07, + "loss": -0.035151124000549316, + "num_tokens": 404707.0, + "reward": -0.408750057220459, + "reward_std": 31.939748764038086, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -0.5087499618530273, + "rewards/supergames_reward/std": 31.93975257873535, + "step": 48, + "step_time": 6.438482580007985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/max_terminated_length": 306.0, + "completions/mean_length": 216.625, + "completions/mean_terminated_length": 216.625, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.33054471015930176, + "epoch": 0.245, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.700303792953491, + "learning_rate": 7.599999999999999e-07, + "loss": -0.04393656551837921, + "num_tokens": 414736.0, + "reward": -97.26000213623047, + "reward_std": 1.6294406652450562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.36000061035156, + "rewards/supergames_reward/std": 1.629441738128662, + "step": 49, + "step_time": 11.636040106008295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 378.0, + "completions/max_terminated_length": 378.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.48711177706718445, + "epoch": 0.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9650564193725586, + "learning_rate": 7.55e-07, + "loss": -0.3529520332813263, + "num_tokens": 424795.0, + "reward": 47.408748626708984, + "reward_std": 72.72083282470703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 47.30875015258789, + "rewards/supergames_reward/std": 72.72083282470703, + "step": 50, + "step_time": 14.258096842997475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 267.625, + "completions/mean_terminated_length": 232.71429443359375, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4482860863208771, + "epoch": 0.255, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2998197078704834, + "learning_rate": 7.5e-07, + "loss": 0.2493654191493988, + "num_tokens": 436288.0, + "reward": 50.61125183105469, + "reward_std": 75.3349838256836, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.53624725341797, + "rewards/supergames_reward/std": 75.27783966064453, + "step": 51, + "step_time": 19.20827590499539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 176.0, + "completions/max_terminated_length": 176.0, + "completions/mean_length": 144.5, + "completions/mean_terminated_length": 144.5, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "entropy": 0.5378735661506653, + "epoch": 0.26, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.114461898803711, + "learning_rate": 7.45e-07, + "loss": 0.0036597400903701782, + "num_tokens": 441980.0, + "reward": -15.20625114440918, + "reward_std": 45.35526657104492, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.28125, + "rewards/supergames_reward/std": 45.301815032958984, + "step": 52, + "step_time": 6.660627231001854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/max_terminated_length": 136.0, + "completions/mean_length": 119.0, + "completions/mean_terminated_length": 119.0, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "entropy": 0.41796669363975525, + "epoch": 0.265, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.059505462646484, + "learning_rate": 7.4e-07, + "loss": -0.014973883517086506, + "num_tokens": 447468.0, + "reward": 15.170000076293945, + "reward_std": 46.658329010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 15.070000648498535, + "rewards/supergames_reward/std": 46.658329010009766, + "step": 53, + "step_time": 5.271571868011961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/max_terminated_length": 259.0, + "completions/mean_length": 191.125, + "completions/mean_terminated_length": 191.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.45421987771987915, + "epoch": 0.27, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4454867839813232, + "learning_rate": 7.35e-07, + "loss": -0.1128973588347435, + "num_tokens": 457301.0, + "reward": 38.89875030517578, + "reward_std": 62.1772575378418, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 38.79874801635742, + "rewards/supergames_reward/std": 62.1772575378418, + "step": 54, + "step_time": 10.524528659996577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/max_terminated_length": 319.0, + "completions/mean_length": 233.375, + "completions/mean_terminated_length": 233.375, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "entropy": 0.4449213445186615, + "epoch": 0.275, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2065398693084717, + "learning_rate": 7.3e-07, + "loss": 0.0016277075046673417, + "num_tokens": 467520.0, + "reward": 27.372499465942383, + "reward_std": 81.23126983642578, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.27250099182129, + "rewards/supergames_reward/std": 81.23126983642578, + "step": 55, + "step_time": 12.592280682991259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 357.0, + "completions/mean_length": 261.0, + "completions/mean_terminated_length": 225.1428680419922, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.506920337677002, + "epoch": 0.28, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.058388710021973, + "learning_rate": 7.249999999999999e-07, + "loss": 0.3289242386817932, + "num_tokens": 476056.0, + "reward": 47.05249786376953, + "reward_std": 80.97753143310547, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 46.977500915527344, + "rewards/supergames_reward/std": 80.9256362915039, + "step": 56, + "step_time": 18.845177220006008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.0, + "completions/max_terminated_length": 250.0, + "completions/mean_length": 192.75, + "completions/mean_terminated_length": 192.75, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "entropy": 0.5197336077690125, + "epoch": 0.285, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.230815410614014, + "learning_rate": 7.2e-07, + "loss": -0.15186546742916107, + "num_tokens": 484014.0, + "reward": -64.5050048828125, + "reward_std": 50.35145568847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -64.60499572753906, + "rewards/supergames_reward/std": 50.3514518737793, + "step": 57, + "step_time": 9.576232638006331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 249.0, + "completions/mean_terminated_length": 249.0, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "entropy": 0.42913514375686646, + "epoch": 0.29, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.8544528484344482, + "learning_rate": 7.149999999999999e-07, + "loss": -0.056405920535326004, + "num_tokens": 494390.0, + "reward": 40.123748779296875, + "reward_std": 84.164794921875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 40.02375030517578, + "rewards/supergames_reward/std": 84.164794921875, + "step": 58, + "step_time": 12.385353341000155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 138.0, + "completions/max_terminated_length": 138.0, + "completions/mean_length": 107.25, + "completions/mean_terminated_length": 107.25, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.34386488795280457, + "epoch": 0.295, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.6915154457092285, + "learning_rate": 7.1e-07, + "loss": 0.0865321159362793, + "num_tokens": 499816.0, + "reward": -8.244999885559082, + "reward_std": 4.681046962738037, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.345000267028809, + "rewards/supergames_reward/std": 4.681046962738037, + "step": 59, + "step_time": 5.42325339600211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/max_terminated_length": 187.0, + "completions/mean_length": 127.25, + "completions/mean_terminated_length": 127.25, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.5430045127868652, + "epoch": 0.3, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.635311126708984, + "learning_rate": 7.049999999999999e-07, + "loss": -0.05412375554442406, + "num_tokens": 505410.0, + "reward": 20.005001068115234, + "reward_std": 55.156005859375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 19.904998779296875, + "rewards/supergames_reward/std": 55.156005859375, + "step": 60, + "step_time": 7.068010902003152 + } + ], + "logging_steps": 1, + "max_steps": 200, + "num_input_tokens_seen": 505410, + "num_train_epochs": 1, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-60/training_args.bin b/checkpoint-60/training_args.bin new file mode 100644 index 0000000..730dec0 --- /dev/null +++ b/checkpoint-60/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691fe5be2d7d1d56160dae63866d136877a72efb30e15ff7b3249192f998a788 +size 7185 diff --git a/checkpoint-80/chat_template.jinja b/checkpoint-80/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/checkpoint-80/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-80/config.json b/checkpoint-80/config.json new file mode 100644 index 0000000..f57b09b --- /dev/null +++ b/checkpoint-80/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-80/generation_config.json b/checkpoint-80/generation_config.json new file mode 100644 index 0000000..1b2bba9 --- /dev/null +++ b/checkpoint-80/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/checkpoint-80/model.safetensors b/checkpoint-80/model.safetensors new file mode 100644 index 0000000..895468f --- /dev/null +++ b/checkpoint-80/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36c49868659b2f8b24ac9a86bf6e80c90079b3530402b89634f2af283a1ea452 +size 6174895536 diff --git a/checkpoint-80/optimizer.pt b/checkpoint-80/optimizer.pt new file mode 100644 index 0000000..81d9d05 --- /dev/null +++ b/checkpoint-80/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e39d224c64f43b5aacbde8ab914fec80a9f819231b78731c937ba3f159056817 +size 12350013801 diff --git a/checkpoint-80/rng_state.pth b/checkpoint-80/rng_state.pth new file mode 100644 index 0000000..a1fe3d4 --- /dev/null +++ b/checkpoint-80/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:772106dc1aba21ee6aaf31ea19134157618b434ed364c663ce63e5ff7344641c +size 14645 diff --git a/checkpoint-80/scheduler.pt b/checkpoint-80/scheduler.pt new file mode 100644 index 0000000..30fa8ce --- /dev/null +++ b/checkpoint-80/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0f1c734dfd575182ad7d0aca3bf6db0d25d9f1d262a3050bff25c7518303a48 +size 1465 diff --git a/checkpoint-80/tokenizer.json b/checkpoint-80/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-80/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-80/tokenizer_config.json b/checkpoint-80/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-80/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-80/trainer_state.json b/checkpoint-80/trainer_state.json new file mode 100644 index 0000000..b696646 --- /dev/null +++ b/checkpoint-80/trainer_state.json @@ -0,0 +1,2354 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.4, + "eval_steps": 500, + "global_step": 80, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 191.0, + "completions/mean_terminated_length": 191.0, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "entropy": 0.5566893219947815, + "epoch": 0.005, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.370361328125, + "learning_rate": 1e-06, + "loss": -0.24658073484897614, + "num_tokens": 10832.0, + "reward": -3.3000030517578125, + "reward_std": 85.62333679199219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -3.4000015258789062, + "rewards/supergames_reward/std": 85.62333679199219, + "step": 1, + "step_time": 22.779711072013015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/max_terminated_length": 216.0, + "completions/mean_length": 131.5, + "completions/mean_terminated_length": 131.5, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.6395841240882874, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.09774398803711, + "learning_rate": 9.95e-07, + "loss": 0.20567649602890015, + "num_tokens": 16404.0, + "reward": -12.422499656677246, + "reward_std": 7.134707450866699, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.522500038146973, + "rewards/supergames_reward/std": 7.134707927703857, + "step": 2, + "step_time": 9.960156448010821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/max_terminated_length": 287.0, + "completions/mean_length": 156.625, + "completions/mean_terminated_length": 156.625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "entropy": 0.562222421169281, + "epoch": 0.015, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.9053616523742676, + "learning_rate": 9.9e-07, + "loss": 0.09602774679660797, + "num_tokens": 26953.0, + "reward": -74.94999694824219, + "reward_std": 70.73094177246094, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -75.0, + "rewards/supergames_reward/std": 70.71067810058594, + "step": 3, + "step_time": 13.686320498993155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 154.125, + "completions/mean_terminated_length": 154.125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7323317527770996, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.933310031890869, + "learning_rate": 9.849999999999999e-07, + "loss": 0.28249427676200867, + "num_tokens": 36514.0, + "reward": -40.7599983215332, + "reward_std": 81.73140716552734, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -40.80999755859375, + "rewards/supergames_reward/std": 81.68995666503906, + "step": 4, + "step_time": 12.805880262021674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 205.25, + "completions/mean_terminated_length": 205.25, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.6167430281639099, + "epoch": 0.025, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.344135046005249, + "learning_rate": 9.8e-07, + "loss": 0.1433994024991989, + "num_tokens": 47476.0, + "reward": -97.6612548828125, + "reward_std": 6.3321428298950195, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.76124572753906, + "rewards/supergames_reward/std": 6.3321428298950195, + "step": 5, + "step_time": 18.789364666008623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/max_terminated_length": 276.0, + "completions/mean_length": 167.25, + "completions/mean_terminated_length": 167.25, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.7583790421485901, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.172371864318848, + "learning_rate": 9.75e-07, + "loss": -0.12127404659986496, + "num_tokens": 55246.0, + "reward": -58.687503814697266, + "reward_std": 58.64059829711914, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -58.76250457763672, + "rewards/supergames_reward/std": 58.620460510253906, + "step": 6, + "step_time": 12.948570273991209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/max_terminated_length": 261.0, + "completions/mean_length": 133.875, + "completions/mean_terminated_length": 133.875, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.7447654604911804, + "epoch": 0.035, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.428595066070557, + "learning_rate": 9.7e-07, + "loss": 0.10135584324598312, + "num_tokens": 60837.0, + "reward": -32.45000076293945, + "reward_std": 41.754486083984375, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -32.5, + "rewards/supergames_reward/std": 41.66190719604492, + "step": 7, + "step_time": 12.058315072004916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 414.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 209.125, + "completions/mean_terminated_length": 209.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.3860666751861572, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.806255578994751, + "learning_rate": 9.649999999999999e-07, + "loss": -0.33238139748573303, + "num_tokens": 70838.0, + "reward": -80.50375366210938, + "reward_std": 38.82380294799805, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -80.60375213623047, + "rewards/supergames_reward/std": 38.82379913330078, + "step": 8, + "step_time": 19.323370319994865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 247.125, + "completions/mean_terminated_length": 158.83334350585938, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.5816237330436707, + "epoch": 0.045, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3209943771362305, + "learning_rate": 9.6e-07, + "loss": 0.36752766370773315, + "num_tokens": 82135.0, + "reward": -27.688751220703125, + "reward_std": 82.8358154296875, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -27.738750457763672, + "rewards/supergames_reward/std": 82.78590393066406, + "step": 9, + "step_time": 21.948575104994234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/max_terminated_length": 240.0, + "completions/mean_length": 153.0, + "completions/mean_terminated_length": 153.0, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6301521062850952, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.125359058380127, + "learning_rate": 9.55e-07, + "loss": 0.06129350885748863, + "num_tokens": 89807.0, + "reward": -78.39624786376953, + "reward_std": 40.17042922973633, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -78.44625091552734, + "rewards/supergames_reward/std": 40.1396484375, + "step": 10, + "step_time": 9.098202993016457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/max_terminated_length": 133.0, + "completions/mean_length": 112.0, + "completions/mean_terminated_length": 112.0, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.5521990656852722, + "epoch": 0.055, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.616060256958008, + "learning_rate": 9.499999999999999e-07, + "loss": -0.006313305348157883, + "num_tokens": 95231.0, + "reward": -12.161249160766602, + "reward_std": 6.39578104019165, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.261249542236328, + "rewards/supergames_reward/std": 6.39578104019165, + "step": 11, + "step_time": 5.136311663984088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 214.875, + "completions/mean_terminated_length": 214.875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.5505905151367188, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.289583206176758, + "learning_rate": 9.45e-07, + "loss": -0.29802003502845764, + "num_tokens": 106238.0, + "reward": -91.66999816894531, + "reward_std": 23.2779541015625, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -91.77000427246094, + "rewards/supergames_reward/std": 23.277956008911133, + "step": 12, + "step_time": 14.945365622988902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 167.375, + "completions/mean_terminated_length": 167.375, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "entropy": 0.540415346622467, + "epoch": 0.065, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.858291149139404, + "learning_rate": 9.399999999999999e-07, + "loss": 0.24004128575325012, + "num_tokens": 116937.0, + "reward": -20.38249969482422, + "reward_std": 87.55204010009766, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.457500457763672, + "rewards/supergames_reward/std": 87.52605438232422, + "step": 13, + "step_time": 10.482285185018554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "entropy": 0.7237679958343506, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.083980560302734, + "learning_rate": 9.35e-07, + "loss": -0.19025824964046478, + "num_tokens": 124640.0, + "reward": -51.397499084472656, + "reward_std": 54.486454010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -51.49749755859375, + "rewards/supergames_reward/std": 54.486454010009766, + "step": 14, + "step_time": 9.597231683001155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/max_terminated_length": 236.0, + "completions/mean_length": 141.375, + "completions/mean_terminated_length": 141.375, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.754334568977356, + "epoch": 0.075, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.720428943634033, + "learning_rate": 9.3e-07, + "loss": -0.16830675303936005, + "num_tokens": 132219.0, + "reward": -56.226253509521484, + "reward_std": 60.115657806396484, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -56.32625198364258, + "rewards/supergames_reward/std": 60.11566162109375, + "step": 15, + "step_time": 8.998362872982398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.0, + "completions/max_terminated_length": 209.0, + "completions/mean_length": 136.375, + "completions/mean_terminated_length": 136.375, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.6280568838119507, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.9681806564331055, + "learning_rate": 9.25e-07, + "loss": -0.01910916529595852, + "num_tokens": 139726.0, + "reward": -68.24000549316406, + "reward_std": 52.238914489746094, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.33999633789062, + "rewards/supergames_reward/std": 52.238914489746094, + "step": 16, + "step_time": 8.034480294008972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 211.875, + "completions/mean_terminated_length": 211.875, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.899176836013794, + "epoch": 0.085, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.691746711730957, + "learning_rate": 9.2e-07, + "loss": 0.25513291358947754, + "num_tokens": 147869.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 17, + "step_time": 13.655792869016295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 387.0, + "completions/max_terminated_length": 387.0, + "completions/mean_length": 238.0, + "completions/mean_terminated_length": 238.0, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.5596873760223389, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6711347103118896, + "learning_rate": 9.15e-07, + "loss": -0.2628335654735565, + "num_tokens": 159101.0, + "reward": -24.900001525878906, + "reward_std": 103.50983428955078, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.0, + "rewards/supergames_reward/std": 103.50983428955078, + "step": 18, + "step_time": 14.784329420013819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 277.0, + "completions/mean_length": 219.75, + "completions/mean_terminated_length": 178.00001525878906, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.9245517253875732, + "epoch": 0.095, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.55502986907959, + "learning_rate": 9.1e-07, + "loss": 0.13060179352760315, + "num_tokens": 167299.0, + "reward": -86.89125061035156, + "reward_std": 36.95603561401367, + "rewards/json_format_reward/mean": 0.05000000074505806, + "rewards/json_format_reward/std": 0.09258200973272324, + "rewards/supergames_reward/mean": -86.94125366210938, + "rewards/supergames_reward/std": 36.93572235107422, + "step": 19, + "step_time": 18.832684700988466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/max_terminated_length": 267.0, + "completions/mean_length": 177.25, + "completions/mean_terminated_length": 177.25, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.7331254482269287, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.538497447967529, + "learning_rate": 9.05e-07, + "loss": -0.16418980062007904, + "num_tokens": 175141.0, + "reward": -68.52874755859375, + "reward_std": 58.48719787597656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.62875366210938, + "rewards/supergames_reward/std": 58.48720169067383, + "step": 20, + "step_time": 10.090975169994636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 190.625, + "completions/mean_terminated_length": 190.625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.6659425497055054, + "epoch": 0.105, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.485373497009277, + "learning_rate": 9e-07, + "loss": -0.10106432437896729, + "num_tokens": 183098.0, + "reward": -71.48875427246094, + "reward_std": 55.33565139770508, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -71.5887451171875, + "rewards/supergames_reward/std": 55.33565139770508, + "step": 21, + "step_time": 11.255001295008697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/max_terminated_length": 279.0, + "completions/mean_length": 179.0, + "completions/mean_terminated_length": 179.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.49365949630737305, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.829419136047363, + "learning_rate": 8.95e-07, + "loss": 0.10587231814861298, + "num_tokens": 192826.0, + "reward": -75.62000274658203, + "reward_std": 45.01959991455078, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -75.69499969482422, + "rewards/supergames_reward/std": 45.00411605834961, + "step": 22, + "step_time": 10.611246599000879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/max_terminated_length": 315.0, + "completions/mean_length": 212.625, + "completions/mean_terminated_length": 212.625, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.43159469962120056, + "epoch": 0.115, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.541125774383545, + "learning_rate": 8.9e-07, + "loss": -0.1681259125471115, + "num_tokens": 202815.0, + "reward": 23.67624855041504, + "reward_std": 67.74340057373047, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 23.576250076293945, + "rewards/supergames_reward/std": 67.743408203125, + "step": 23, + "step_time": 11.731771531980485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/max_terminated_length": 168.0, + "completions/mean_length": 101.625, + "completions/mean_terminated_length": 101.625, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.3972298502922058, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.8285417556762695, + "learning_rate": 8.85e-07, + "loss": -0.011585958302021027, + "num_tokens": 208188.0, + "reward": 58.849998474121094, + "reward_std": 56.9304084777832, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 58.75, + "rewards/supergames_reward/std": 56.9304084777832, + "step": 24, + "step_time": 6.187504950998118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 303.0, + "completions/max_terminated_length": 303.0, + "completions/mean_length": 165.375, + "completions/mean_terminated_length": 165.375, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "entropy": 0.6339899301528931, + "epoch": 0.125, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.823799133300781, + "learning_rate": 8.799999999999999e-07, + "loss": 0.05159185826778412, + "num_tokens": 215935.0, + "reward": -23.521251678466797, + "reward_std": 74.23067474365234, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -23.621248245239258, + "rewards/supergames_reward/std": 74.23067474365234, + "step": 25, + "step_time": 11.059416300005978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 349.0, + "completions/max_terminated_length": 349.0, + "completions/mean_length": 204.0, + "completions/mean_terminated_length": 204.0, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.6462895274162292, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.969292640686035, + "learning_rate": 8.75e-07, + "loss": 0.0115677984431386, + "num_tokens": 223999.0, + "reward": -74.8175048828125, + "reward_std": 40.94221115112305, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -74.89250183105469, + "rewards/supergames_reward/std": 40.924625396728516, + "step": 26, + "step_time": 12.666237785975682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 166.5, + "completions/mean_terminated_length": 166.5, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.4756850600242615, + "epoch": 0.135, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.915069103240967, + "learning_rate": 8.699999999999999e-07, + "loss": -0.13321346044540405, + "num_tokens": 233643.0, + "reward": -52.125, + "reward_std": 19.405282974243164, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -52.20000076293945, + "rewards/supergames_reward/std": 19.33464813232422, + "step": 27, + "step_time": 11.172539752995363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 137.125, + "completions/mean_terminated_length": 137.125, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.5834778547286987, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.146666526794434, + "learning_rate": 8.65e-07, + "loss": 0.060454584658145905, + "num_tokens": 241204.0, + "reward": -68.99250793457031, + "reward_std": 51.010581970214844, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -69.09249877929688, + "rewards/supergames_reward/std": 51.010581970214844, + "step": 28, + "step_time": 7.025046669004951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 182.375, + "completions/mean_terminated_length": 182.375, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.6368551254272461, + "epoch": 0.145, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.15535306930542, + "learning_rate": 8.599999999999999e-07, + "loss": -0.3323673903942108, + "num_tokens": 249087.0, + "reward": -44.17000198364258, + "reward_std": 80.5373306274414, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -44.27000045776367, + "rewards/supergames_reward/std": 80.5373306274414, + "step": 29, + "step_time": 12.079259724996518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 127.75, + "completions/mean_terminated_length": 127.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.4491196870803833, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.896186828613281, + "learning_rate": 8.55e-07, + "loss": -0.12887656688690186, + "num_tokens": 259445.0, + "reward": 57.446250915527344, + "reward_std": 38.78837585449219, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 57.34625244140625, + "rewards/supergames_reward/std": 38.78837966918945, + "step": 30, + "step_time": 7.483972909016302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/max_terminated_length": 166.0, + "completions/mean_length": 112.875, + "completions/mean_terminated_length": 112.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4858454465866089, + "epoch": 0.155, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.742906093597412, + "learning_rate": 8.499999999999999e-07, + "loss": 0.06930096447467804, + "num_tokens": 264924.0, + "reward": -21.174999237060547, + "reward_std": 31.89051628112793, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -21.25, + "rewards/supergames_reward/std": 31.819807052612305, + "step": 31, + "step_time": 6.1866529019898735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/max_terminated_length": 148.0, + "completions/mean_length": 118.125, + "completions/mean_terminated_length": 118.125, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.45428749918937683, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.663414478302002, + "learning_rate": 8.45e-07, + "loss": 0.05735419690608978, + "num_tokens": 270381.0, + "reward": 0.4687504768371582, + "reward_std": 38.66371154785156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 0.36875057220458984, + "rewards/supergames_reward/std": 38.6637077331543, + "step": 32, + "step_time": 5.577042757999152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 166.875, + "completions/mean_terminated_length": 166.875, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "entropy": 0.6107293367385864, + "epoch": 0.165, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.75006628036499, + "learning_rate": 8.399999999999999e-07, + "loss": -0.14760127663612366, + "num_tokens": 278156.0, + "reward": -87.84750366210938, + "reward_std": 26.84708023071289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -87.94749450683594, + "rewards/supergames_reward/std": 26.847076416015625, + "step": 33, + "step_time": 12.552876825997373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 360.0, + "completions/mean_length": 231.0, + "completions/mean_terminated_length": 190.85714721679688, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.6089653372764587, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.781267166137695, + "learning_rate": 8.349999999999999e-07, + "loss": 0.4294504225254059, + "num_tokens": 289364.0, + "reward": -99.92500305175781, + "reward_std": 0.07070883363485336, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -100.0, + "rewards/supergames_reward/std": 0.0, + "step": 34, + "step_time": 18.961819477990502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 134.125, + "completions/mean_terminated_length": 134.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46158918738365173, + "epoch": 0.175, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.016353607177734, + "learning_rate": 8.299999999999999e-07, + "loss": 0.09251818805932999, + "num_tokens": 298733.0, + "reward": -35.558753967285156, + "reward_std": 15.881204605102539, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -35.65875244140625, + "rewards/supergames_reward/std": 15.881203651428223, + "step": 35, + "step_time": 8.38491243700264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/max_terminated_length": 207.0, + "completions/mean_length": 156.875, + "completions/mean_terminated_length": 156.875, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "entropy": 0.5104788541793823, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.592893123626709, + "learning_rate": 8.249999999999999e-07, + "loss": -0.0410832017660141, + "num_tokens": 306420.0, + "reward": 5.4212493896484375, + "reward_std": 93.50404357910156, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 5.346250534057617, + "rewards/supergames_reward/std": 93.4718246459961, + "step": 36, + "step_time": 7.811868985998444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/max_terminated_length": 241.0, + "completions/mean_length": 178.0, + "completions/mean_terminated_length": 178.0, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.7527878880500793, + "epoch": 0.185, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8812546730041504, + "learning_rate": 8.199999999999999e-07, + "loss": 0.12539231777191162, + "num_tokens": 314252.0, + "reward": -89.58500671386719, + "reward_std": 29.256120681762695, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -89.66000366210938, + "rewards/supergames_reward/std": 29.24593734741211, + "step": 37, + "step_time": 8.959742914012168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/max_terminated_length": 199.0, + "completions/mean_length": 133.375, + "completions/mean_terminated_length": 133.375, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.3347901403903961, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.205414295196533, + "learning_rate": 8.149999999999999e-07, + "loss": -0.06991486996412277, + "num_tokens": 319839.0, + "reward": -14.498749732971191, + "reward_std": 13.007229804992676, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -14.598750114440918, + "rewards/supergames_reward/std": 13.007229804992676, + "step": 38, + "step_time": 7.377183554985095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/max_terminated_length": 181.0, + "completions/mean_length": 114.375, + "completions/mean_terminated_length": 114.375, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.4445965886116028, + "epoch": 0.195, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.561298847198486, + "learning_rate": 8.1e-07, + "loss": -0.06235164776444435, + "num_tokens": 325290.0, + "reward": -7.425000190734863, + "reward_std": 53.701602935791016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -7.5, + "rewards/supergames_reward/std": 53.652320861816406, + "step": 39, + "step_time": 6.753862089011818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 187.625, + "completions/mean_terminated_length": 187.625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.3950234353542328, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.5988712310791016, + "learning_rate": 8.05e-07, + "loss": -0.1381440907716751, + "num_tokens": 335119.0, + "reward": -33.68375015258789, + "reward_std": 42.069435119628906, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -33.783748626708984, + "rewards/supergames_reward/std": 42.069435119628906, + "step": 40, + "step_time": 13.718958162004128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 371.0, + "completions/mean_length": 251.125, + "completions/mean_terminated_length": 213.85714721679688, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.507067084312439, + "epoch": 0.205, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.3898658752441406, + "learning_rate": 8e-07, + "loss": 0.0078964838758111, + "num_tokens": 346464.0, + "reward": -86.48625183105469, + "reward_std": 38.020694732666016, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -86.56124877929688, + "rewards/supergames_reward/std": 38.010528564453125, + "step": 41, + "step_time": 18.90960379401804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 152.0, + "completions/max_terminated_length": 152.0, + "completions/mean_length": 116.375, + "completions/mean_terminated_length": 116.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.3801707923412323, + "epoch": 0.21, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.95e-07, + "loss": 0.0, + "num_tokens": 351955.0, + "reward": -9.899999618530273, + "reward_std": 0.0, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -10.0, + "rewards/supergames_reward/std": 0.0, + "step": 42, + "step_time": 5.711630532023264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/max_terminated_length": 308.0, + "completions/mean_length": 247.375, + "completions/mean_terminated_length": 247.375, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.3185043931007385, + "epoch": 0.215, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.168762445449829, + "learning_rate": 7.9e-07, + "loss": 0.1970764398574829, + "num_tokens": 363278.0, + "reward": -53.89249801635742, + "reward_std": 63.02619934082031, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -53.99250030517578, + "rewards/supergames_reward/std": 63.02619934082031, + "step": 43, + "step_time": 11.661934906995157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/max_terminated_length": 330.0, + "completions/mean_length": 212.5, + "completions/mean_terminated_length": 212.5, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4446268379688263, + "epoch": 0.22, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.305844306945801, + "learning_rate": 7.85e-07, + "loss": -0.06735197454690933, + "num_tokens": 374306.0, + "reward": 18.059999465942383, + "reward_std": 98.9417953491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 17.959999084472656, + "rewards/supergames_reward/std": 98.9417953491211, + "step": 44, + "step_time": 12.492453911982011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.0, + "completions/max_terminated_length": 316.0, + "completions/mean_length": 183.0, + "completions/mean_terminated_length": 183.0, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.5269614458084106, + "epoch": 0.225, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.8006370067596436, + "learning_rate": 7.799999999999999e-07, + "loss": -0.08870815485715866, + "num_tokens": 382210.0, + "reward": -54.928749084472656, + "reward_std": 72.5127944946289, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -55.02874755859375, + "rewards/supergames_reward/std": 72.5127944946289, + "step": 45, + "step_time": 11.7636467939883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.0, + "completions/max_terminated_length": 244.0, + "completions/mean_length": 136.625, + "completions/mean_terminated_length": 136.625, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.34820204973220825, + "epoch": 0.23, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.911736011505127, + "learning_rate": 7.75e-07, + "loss": 0.04397330805659294, + "num_tokens": 391583.0, + "reward": -19.10375213623047, + "reward_std": 23.967702865600586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -19.203750610351562, + "rewards/supergames_reward/std": 23.967702865600586, + "step": 46, + "step_time": 10.119426151999505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 160.375, + "completions/mean_terminated_length": 160.375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.611914336681366, + "epoch": 0.235, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.571913242340088, + "learning_rate": 7.699999999999999e-07, + "loss": -0.044293053448200226, + "num_tokens": 399290.0, + "reward": -25.186250686645508, + "reward_std": 71.46305084228516, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -25.286251068115234, + "rewards/supergames_reward/std": 71.46305084228516, + "step": 47, + "step_time": 8.563868903991533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/max_terminated_length": 167.0, + "completions/mean_length": 111.125, + "completions/mean_terminated_length": 111.125, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.46585777401924133, + "epoch": 0.24, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.654109954833984, + "learning_rate": 7.65e-07, + "loss": -0.035151124000549316, + "num_tokens": 404707.0, + "reward": -0.408750057220459, + "reward_std": 31.939748764038086, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -0.5087499618530273, + "rewards/supergames_reward/std": 31.93975257873535, + "step": 48, + "step_time": 6.438482580007985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/max_terminated_length": 306.0, + "completions/mean_length": 216.625, + "completions/mean_terminated_length": 216.625, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.33054471015930176, + "epoch": 0.245, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.700303792953491, + "learning_rate": 7.599999999999999e-07, + "loss": -0.04393656551837921, + "num_tokens": 414736.0, + "reward": -97.26000213623047, + "reward_std": 1.6294406652450562, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -97.36000061035156, + "rewards/supergames_reward/std": 1.629441738128662, + "step": 49, + "step_time": 11.636040106008295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 378.0, + "completions/max_terminated_length": 378.0, + "completions/mean_length": 218.375, + "completions/mean_terminated_length": 218.375, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.48711177706718445, + "epoch": 0.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9650564193725586, + "learning_rate": 7.55e-07, + "loss": -0.3529520332813263, + "num_tokens": 424795.0, + "reward": 47.408748626708984, + "reward_std": 72.72083282470703, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 47.30875015258789, + "rewards/supergames_reward/std": 72.72083282470703, + "step": 50, + "step_time": 14.258096842997475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 267.625, + "completions/mean_terminated_length": 232.71429443359375, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.4482860863208771, + "epoch": 0.255, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2998197078704834, + "learning_rate": 7.5e-07, + "loss": 0.2493654191493988, + "num_tokens": 436288.0, + "reward": 50.61125183105469, + "reward_std": 75.3349838256836, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 50.53624725341797, + "rewards/supergames_reward/std": 75.27783966064453, + "step": 51, + "step_time": 19.20827590499539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 176.0, + "completions/max_terminated_length": 176.0, + "completions/mean_length": 144.5, + "completions/mean_terminated_length": 144.5, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "entropy": 0.5378735661506653, + "epoch": 0.26, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.114461898803711, + "learning_rate": 7.45e-07, + "loss": 0.0036597400903701782, + "num_tokens": 441980.0, + "reward": -15.20625114440918, + "reward_std": 45.35526657104492, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.28125, + "rewards/supergames_reward/std": 45.301815032958984, + "step": 52, + "step_time": 6.660627231001854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/max_terminated_length": 136.0, + "completions/mean_length": 119.0, + "completions/mean_terminated_length": 119.0, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "entropy": 0.41796669363975525, + "epoch": 0.265, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.059505462646484, + "learning_rate": 7.4e-07, + "loss": -0.014973883517086506, + "num_tokens": 447468.0, + "reward": 15.170000076293945, + "reward_std": 46.658329010009766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 15.070000648498535, + "rewards/supergames_reward/std": 46.658329010009766, + "step": 53, + "step_time": 5.271571868011961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/max_terminated_length": 259.0, + "completions/mean_length": 191.125, + "completions/mean_terminated_length": 191.125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "entropy": 0.45421987771987915, + "epoch": 0.27, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4454867839813232, + "learning_rate": 7.35e-07, + "loss": -0.1128973588347435, + "num_tokens": 457301.0, + "reward": 38.89875030517578, + "reward_std": 62.1772575378418, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 38.79874801635742, + "rewards/supergames_reward/std": 62.1772575378418, + "step": 54, + "step_time": 10.524528659996577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/max_terminated_length": 319.0, + "completions/mean_length": 233.375, + "completions/mean_terminated_length": 233.375, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "entropy": 0.4449213445186615, + "epoch": 0.275, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2065398693084717, + "learning_rate": 7.3e-07, + "loss": 0.0016277075046673417, + "num_tokens": 467520.0, + "reward": 27.372499465942383, + "reward_std": 81.23126983642578, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 27.27250099182129, + "rewards/supergames_reward/std": 81.23126983642578, + "step": 55, + "step_time": 12.592280682991259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 357.0, + "completions/mean_length": 261.0, + "completions/mean_terminated_length": 225.1428680419922, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.506920337677002, + "epoch": 0.28, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.058388710021973, + "learning_rate": 7.249999999999999e-07, + "loss": 0.3289242386817932, + "num_tokens": 476056.0, + "reward": 47.05249786376953, + "reward_std": 80.97753143310547, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 46.977500915527344, + "rewards/supergames_reward/std": 80.9256362915039, + "step": 56, + "step_time": 18.845177220006008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.0, + "completions/max_terminated_length": 250.0, + "completions/mean_length": 192.75, + "completions/mean_terminated_length": 192.75, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "entropy": 0.5197336077690125, + "epoch": 0.285, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.230815410614014, + "learning_rate": 7.2e-07, + "loss": -0.15186546742916107, + "num_tokens": 484014.0, + "reward": -64.5050048828125, + "reward_std": 50.35145568847656, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -64.60499572753906, + "rewards/supergames_reward/std": 50.3514518737793, + "step": 57, + "step_time": 9.576232638006331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 249.0, + "completions/mean_terminated_length": 249.0, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "entropy": 0.42913514375686646, + "epoch": 0.29, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.8544528484344482, + "learning_rate": 7.149999999999999e-07, + "loss": -0.056405920535326004, + "num_tokens": 494390.0, + "reward": 40.123748779296875, + "reward_std": 84.164794921875, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 40.02375030517578, + "rewards/supergames_reward/std": 84.164794921875, + "step": 58, + "step_time": 12.385353341000155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 138.0, + "completions/max_terminated_length": 138.0, + "completions/mean_length": 107.25, + "completions/mean_terminated_length": 107.25, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.34386488795280457, + "epoch": 0.295, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.6915154457092285, + "learning_rate": 7.1e-07, + "loss": 0.0865321159362793, + "num_tokens": 499816.0, + "reward": -8.244999885559082, + "reward_std": 4.681046962738037, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.345000267028809, + "rewards/supergames_reward/std": 4.681046962738037, + "step": 59, + "step_time": 5.42325339600211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/max_terminated_length": 187.0, + "completions/mean_length": 127.25, + "completions/mean_terminated_length": 127.25, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.5430045127868652, + "epoch": 0.3, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.635311126708984, + "learning_rate": 7.049999999999999e-07, + "loss": -0.05412375554442406, + "num_tokens": 505410.0, + "reward": 20.005001068115234, + "reward_std": 55.156005859375, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 19.904998779296875, + "rewards/supergames_reward/std": 55.156005859375, + "step": 60, + "step_time": 7.068010902003152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/max_terminated_length": 286.0, + "completions/mean_length": 192.125, + "completions/mean_terminated_length": 192.125, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 0.4213869869709015, + "epoch": 0.305, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.4664618968963623, + "learning_rate": 7e-07, + "loss": -0.0495159812271595, + "num_tokens": 516283.0, + "reward": -47.212501525878906, + "reward_std": 91.2253646850586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -47.3125, + "rewards/supergames_reward/std": 91.2253646850586, + "step": 61, + "step_time": 11.176304235996213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 171.0, + "completions/mean_terminated_length": 171.0, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.4626811146736145, + "epoch": 0.31, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.949252128601074, + "learning_rate": 6.949999999999999e-07, + "loss": 0.192047581076622, + "num_tokens": 522251.0, + "reward": -15.092500686645508, + "reward_std": 49.82962417602539, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -15.167499542236328, + "rewards/supergames_reward/std": 49.78091049194336, + "step": 62, + "step_time": 12.075224861997413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 429.0, + "completions/mean_length": 283.5, + "completions/mean_terminated_length": 250.85714721679688, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.4814216196537018, + "epoch": 0.315, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.7918753623962402, + "learning_rate": 6.9e-07, + "loss": 0.15706156194210052, + "num_tokens": 533815.0, + "reward": -20.86625099182129, + "reward_std": 85.99250030517578, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -20.94124984741211, + "rewards/supergames_reward/std": 85.96620178222656, + "step": 63, + "step_time": 19.448832260008203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.0, + "completions/max_terminated_length": 189.0, + "completions/mean_length": 131.25, + "completions/mean_terminated_length": 131.25, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.4929414987564087, + "epoch": 0.32, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.774101734161377, + "learning_rate": 6.85e-07, + "loss": -0.060691747814416885, + "num_tokens": 539465.0, + "reward": -21.0674991607666, + "reward_std": 6.892730236053467, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -21.167499542236328, + "rewards/supergames_reward/std": 6.892730236053467, + "step": 64, + "step_time": 7.141569407976931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 454.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 281.375, + "completions/mean_terminated_length": 281.375, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "entropy": 0.45866572856903076, + "epoch": 0.325, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.092717409133911, + "learning_rate": 6.800000000000001e-07, + "loss": 0.11694959551095963, + "num_tokens": 551012.0, + "reward": 49.938751220703125, + "reward_std": 79.1865005493164, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 49.8387451171875, + "rewards/supergames_reward/std": 79.1865005493164, + "step": 65, + "step_time": 17.06170882002334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/max_terminated_length": 203.0, + "completions/mean_length": 128.625, + "completions/mean_terminated_length": 128.625, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.5005782246589661, + "epoch": 0.33, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.77616024017334, + "learning_rate": 6.75e-07, + "loss": 0.11242837458848953, + "num_tokens": 556577.0, + "reward": 48.875, + "reward_std": 49.13539123535156, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 48.775001525878906, + "rewards/supergames_reward/std": 49.13539123535156, + "step": 66, + "step_time": 7.695410129002994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 436.0, + "completions/max_terminated_length": 436.0, + "completions/mean_length": 281.875, + "completions/mean_terminated_length": 281.875, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "entropy": 0.507097065448761, + "epoch": 0.335, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.911184787750244, + "learning_rate": 6.7e-07, + "loss": -0.08151137083768845, + "num_tokens": 567120.0, + "reward": -12.886249542236328, + "reward_std": 84.04269409179688, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -12.986251831054688, + "rewards/supergames_reward/std": 84.04269409179688, + "step": 67, + "step_time": 16.409127942984924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 318.375, + "completions/mean_terminated_length": 290.71429443359375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.486402690410614, + "epoch": 0.34, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.9461607933044434, + "learning_rate": 6.65e-07, + "loss": 0.2442178875207901, + "num_tokens": 578995.0, + "reward": 2.8387489318847656, + "reward_std": 98.47579956054688, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 2.7637500762939453, + "rewards/supergames_reward/std": 98.44595336914062, + "step": 68, + "step_time": 19.32364999302081 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 291.0, + "completions/max_terminated_length": 291.0, + "completions/mean_length": 219.625, + "completions/mean_terminated_length": 219.625, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.37856727838516235, + "epoch": 0.345, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0204732418060303, + "learning_rate": 6.6e-07, + "loss": -0.08764916658401489, + "num_tokens": 589072.0, + "reward": -54.18375015258789, + "reward_std": 71.45012664794922, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -54.283748626708984, + "rewards/supergames_reward/std": 71.45013427734375, + "step": 69, + "step_time": 11.126611230982235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 305.0, + "completions/mean_length": 254.75, + "completions/mean_terminated_length": 218.00001525878906, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.46423614025115967, + "epoch": 0.35, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.0140154361724854, + "learning_rate": 6.55e-07, + "loss": 0.26797160506248474, + "num_tokens": 600438.0, + "reward": -32.04875183105469, + "reward_std": 69.86957550048828, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -32.12375259399414, + "rewards/supergames_reward/std": 69.84178161621094, + "step": 70, + "step_time": 19.12617294798838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 372.0, + "completions/max_terminated_length": 372.0, + "completions/mean_length": 194.125, + "completions/mean_terminated_length": 194.125, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.3427293300628662, + "epoch": 0.355, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.781383991241455, + "learning_rate": 6.5e-07, + "loss": 0.354524701833725, + "num_tokens": 610327.0, + "reward": -6.913749694824219, + "reward_std": 87.56071472167969, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": -6.988750457763672, + "rewards/supergames_reward/std": 87.53033447265625, + "step": 71, + "step_time": 14.041668199002743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/max_terminated_length": 197.0, + "completions/mean_length": 124.75, + "completions/mean_terminated_length": 124.75, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.4455118477344513, + "epoch": 0.36, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.726032733917236, + "learning_rate": 6.45e-07, + "loss": 0.18206289410591125, + "num_tokens": 615853.0, + "reward": 41.95124816894531, + "reward_std": 60.640811920166016, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 41.85124969482422, + "rewards/supergames_reward/std": 60.640811920166016, + "step": 72, + "step_time": 7.416493425989756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.0, + "completions/max_terminated_length": 323.0, + "completions/mean_length": 147.375, + "completions/mean_terminated_length": 147.375, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "entropy": 0.480733186006546, + "epoch": 0.365, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.853621482849121, + "learning_rate": 6.4e-07, + "loss": 0.04558124020695686, + "num_tokens": 621584.0, + "reward": 14.884998321533203, + "reward_std": 46.25857162475586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 14.78499984741211, + "rewards/supergames_reward/std": 46.25857162475586, + "step": 73, + "step_time": 11.95379540900467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 283.0, + "completions/max_terminated_length": 283.0, + "completions/mean_length": 188.125, + "completions/mean_terminated_length": 188.125, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.5526013374328613, + "epoch": 0.37, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.729883909225464, + "learning_rate": 6.35e-07, + "loss": 0.06268303096294403, + "num_tokens": 629545.0, + "reward": -68.05750274658203, + "reward_std": 59.44808578491211, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -68.15750122070312, + "rewards/supergames_reward/std": 59.44808578491211, + "step": 74, + "step_time": 10.64124580900534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 141.0, + "completions/max_terminated_length": 141.0, + "completions/mean_length": 117.375, + "completions/mean_terminated_length": 117.375, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.3744319677352905, + "epoch": 0.375, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.775154113769531, + "learning_rate": 6.3e-07, + "loss": -0.023284845054149628, + "num_tokens": 635052.0, + "reward": 3.9374990463256836, + "reward_std": 61.85771179199219, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 3.862499237060547, + "rewards/supergames_reward/std": 61.80967712402344, + "step": 75, + "step_time": 5.432648951013107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 157.0, + "completions/mean_terminated_length": 157.0, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4822140336036682, + "epoch": 0.38, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.715704917907715, + "learning_rate": 6.249999999999999e-07, + "loss": -0.04918142035603523, + "num_tokens": 644644.0, + "reward": 48.64125061035156, + "reward_std": 78.0205078125, + "rewards/json_format_reward/mean": 0.07500000298023224, + "rewards/json_format_reward/std": 0.0707106813788414, + "rewards/supergames_reward/mean": 48.56624984741211, + "rewards/supergames_reward/std": 77.96605682373047, + "step": 76, + "step_time": 10.348935816989979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/max_terminated_length": 380.0, + "completions/mean_length": 210.875, + "completions/mean_terminated_length": 210.875, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "entropy": 0.45722317695617676, + "epoch": 0.385, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6057567596435547, + "learning_rate": 6.2e-07, + "loss": 0.058352649211883545, + "num_tokens": 652771.0, + "reward": -8.283750534057617, + "reward_std": 77.68436431884766, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -8.383749008178711, + "rewards/supergames_reward/std": 77.68437194824219, + "step": 77, + "step_time": 14.142948755004909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/max_terminated_length": 324.0, + "completions/mean_length": 188.25, + "completions/mean_terminated_length": 188.25, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.33380764722824097, + "epoch": 0.39, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2239863872528076, + "learning_rate": 6.149999999999999e-07, + "loss": -0.04592633992433548, + "num_tokens": 662653.0, + "reward": 11.153749465942383, + "reward_std": 89.88329315185547, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 11.053749084472656, + "rewards/supergames_reward/std": 89.88329315185547, + "step": 78, + "step_time": 12.277474621019792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/max_terminated_length": 143.0, + "completions/mean_length": 108.875, + "completions/mean_terminated_length": 108.875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "entropy": 0.3518812656402588, + "epoch": 0.395, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.967776298522949, + "learning_rate": 6.1e-07, + "loss": -0.10800496488809586, + "num_tokens": 668076.0, + "reward": 70.33250427246094, + "reward_std": 49.9145393371582, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": 70.23249816894531, + "rewards/supergames_reward/std": 49.91453552246094, + "step": 79, + "step_time": 5.52714040101273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/max_terminated_length": 184.0, + "completions/mean_length": 152.125, + "completions/mean_terminated_length": 152.125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.4699896574020386, + "epoch": 0.4, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.561161994934082, + "learning_rate": 6.049999999999999e-07, + "loss": 0.09505834430456161, + "num_tokens": 675701.0, + "reward": -20.25625228881836, + "reward_std": 53.78542709350586, + "rewards/json_format_reward/mean": 0.10000000149011612, + "rewards/json_format_reward/std": 0.0, + "rewards/supergames_reward/mean": -20.35624885559082, + "rewards/supergames_reward/std": 53.78542709350586, + "step": 80, + "step_time": 7.155081019998761 + } + ], + "logging_steps": 1, + "max_steps": 200, + "num_input_tokens_seen": 675701, + "num_train_epochs": 1, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-80/training_args.bin b/checkpoint-80/training_args.bin new file mode 100644 index 0000000..730dec0 --- /dev/null +++ b/checkpoint-80/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691fe5be2d7d1d56160dae63866d136877a72efb30e15ff7b3249192f998a788 +size 7185 diff --git a/config.json b/config.json new file mode 100644 index 0000000..f57b09b --- /dev/null +++ b/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..1b2bba9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..73c5aaa --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d768f5bced7c176c364bd8caa06f355a13ba8e84bfa6afc5000b0943d729a1b0 +size 6174895536 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..730dec0 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691fe5be2d7d1d56160dae63866d136877a72efb30e15ff7b3249192f998a788 +size 7185 diff --git a/untrained_vs_trained.png b/untrained_vs_trained.png new file mode 100644 index 0000000000000000000000000000000000000000..345dcdafeceff8c22342576727e9e94c86bacb0a GIT binary patch literal 82257 zcmdqJgC`!D!~4>vc2bMA9r*Y$cm>vi9|tRS=H=lwr#*sx&><>Gne4I4I1 zY}oJ}^G`qGD<^k_1>g@c#|s*cDz+w$E(Z3-8{`cfuUpwVTA5$_&Dq%A!Q9qHfbZl9 zK2hG^%p4uBJBS@WZvCI{;Ip+iJ5CWnX;bMuN9nDc%dy%=zim=dz09&ZlXb`Tmwb}KyF#8FJ)-sv z<94rm#?$uOXZ8mOPqLJVD++hY3m^XB^HO1YX-}uVaS-u})uf$RTYzXEPu?D_B4{%vDm z!N&jf9UC^B*?Poo+kbuK=7IAc5B=9yHi+K+f8c^{x_3PJ{gzRt27U{gy?fHLl~etP zjT>o|M~)snnqTqz)}t!%zOMBNa^ckO4+F_hp8Phg@%_e)!_KERT&?KrGxMrhzZ+{D z|K1jUUe_!+6Y6U!CoP0ag`=kbyWjr0efv>gcB;op_s;WcY3X{U$r)vfmn7U~F8}rO z?%?D4l7)qZRSgY_?6RS%(?fN$GtG*YA79<;%D2_Gx%11irCc~ad!2jUg9i_$25S=R z2dk4T+tU{(z1LFi-o4wim3jRxUbL?n|L1nTHRn`}R_)9(t{Z&!?(&YGzIoyI_3I7K znMRq#_qVn=;vG~+4X<6>wR2}owrPU}y~t_tOC<|C<@JNT?SFi?F<8X5H*zb}8Gavw zPm8moMP4gKXV0Azvh6)xx-dlBFYcV`vyFxC!E0*w;q^`SIpe~=3;DwH=!p}T$Gh_t z^Lwc5vbFPrHDT)bh)uluG13pk{kF{e6cn6t8f}yRWiMaOLQiF-ObCzql{05<+R%&Q zombU-MfvdI1MbS>g{Ts{ejjPy zeNXWLJ!f0h3wLv`fA%%Ee^=r9HX6V2{dlry2JiV)vrXiW>6M(K+gHhGkF;kz{`1zY z?32luZIfUV5$U#h-O-_8-IYVZ)wDkLUQLn> z;~R7=*uW+qCDuA|^zX*T#)uOJf8w?3*t#>ySGDlHjfVabq1nI3A3k;JYxLs5^&9kF zp| z&$H_5_l^C$?F`43{el;rMp{mW<=ghv{}jG{Edx9LeFc{^>}gHjJ@`#CZ@!YI6<6lD zlzi>={i(S*vNj{FDG5q3(p;xc_ZYapeEBjoP`@@G1 z3y-GNM~cR5ll*$-A+yMp%xkX)hK4vEyuRkLzu}m6o>@SU zf?MPH6N~orwBcxX1#D@j#eKNaw+1Cu#$ouw2xgv8yYq8oYac_~D1}$F_ZCy2ndrs1_}WPrv=)=eO4~JEG!G^6@FGEl>OF zd9OI-nm0?cUkVQJ_xC?7A)%IIri8Pj!z$y~a&ISRq+b$Fvx%I!JX6%NJM)emM=1CG z{o7|4&8=mg?jhW2m!hMixm6Pbx^m28Cd-x-zI^$jP-%dTO7Z;sa-&7#v%f9{A3c2h z`0*dU|2|PG*TQY0a5x9YXLG*m+?eLW{X&;=En1b!JQjjE6%VPUX{jknt>olcby7df zMLK=`Y%uNl$G1PICMn19TDn&Ua}6y#cS-A^9F&=Vg&UNj_*gPoBlDSuhllEuGv5eV zv|MEI`ug<9kt6&i+b9%Dgn+5vkJ}EVa;qgD{q46idU|@?O3!Y0gtm1>{zEtoNQQVSXkKoU0lzk zAMR)L`%AT<6g2gOvM$YyHxEDdPM@F`r&zQmPtDJVVO_rc?z`C+-);Mpln~wY=MTR` zN}Bl9*I{Q0U45}ZEm@UAzpZCKa`{dSd=Ft_uGbxnIbF&D-1C)y*y0ka_i+2l7W2DXGr^cB5 zqV};7BDU#g|N6PAxmo4Vp+i4!-mF>|AvF87-+wf_L9#j1u#*3P@rTI@z9i1ujyKKj!RL7R>64k^i<{pJna=5b^UXJJ zP~FG#MBHXR3692l#P#H>rCjMu71>{e>$mvhE!65k|XsHBQhyQOis9HZcRi=*us%-f`vpEtLkZB162G}CVi^3aZv z@psnX8~pO{!2 ze7nB4(9}@f=<{Q!R3)p6qta*3o=u+o8s)agNT{2e8gB5e9A_7Coie)9ULDm&aaHua zckkrl{N&>yQ7)~V#7wJB{)B%0e48jGG!VxIrAO>c?0YRZ*5hP&#mAGi48qr|)SC`Z$n zMQl2Buv+poVWDR+F?GVB$MvI!CVg=tPNJYY8fIc&M%Q`1YD_^{d4tW?Y&{v}(tEO} zrzcXFAdJ?>z1a_z$O@IxGggyL;DIdAT;cs-gBj#_zG z|H;<%{Q27_ajGg_zkZ!$xcYjpen-nBT4rIo=h7U*reUN(YE5v|)zwvW)>e3Z=PBFo zT>o?a=BrodXFoNbd$9L-DBEs69gW>QSAyBb>BZ{p8Tw&t7tWtgG_3R!yZ-*>{k_L+ znx%19T2S@qsNj}usVAjuQQc60Rjjj76U^TTTDI|S*>~a$b>QR2XB!XQIP=3Vds z96!JRotA}{%x9v%MY+ejsG?8t?s>qcSCWWA`zWRo-Cm=PCej{yOgqb?s4DeTC|BuJ zHP7P52gj`>d31|%E`@NjGv+Ry3OlYZoZpzLDR4}`OlNka)uCeu?XM-zT3ff&t#!G! zym-wx{Wpb}iSwetiM8RO!Rtf9(b~#BzrZJ52NxiaC!xBAwD>VM^4BoR`*{ z-+X7Z&u2js->H_Q%nk%uV>OK$p2zf!V9KtemtT0W96FSaOD2^}vn*;%_{Qg}b&;Yf zRDyO%SK9O7!p5x!PGnR834Z!;houdzyS2z^RIr;S>Jb0q=6zQMzSUk`ULY@zy~aQd z67yOvU}IE=@Qie3CvYf4CMG=g>PosHV$+id)Ed!lfVNM%E#CH6>*OUlIbT3UTY;%( zudg+Jx9#*395+;VS+N_RpYZ9IrK0DXzwu>Kc>OC2O9mE+Rs%pnnjcR5aTgaC-9q~d zSaW{Eo+76d(pLaZ6U`c*DOjJQ`fO%;#%NBErQY9jEcyO!9&WV%V(0ND>^OD`3YvO` zUTMzNm*3*HYDI~=w4sv#3dNINfVP>S5XCPu;)}K)q$rt$ZIt}`tsl*8y#Kzv$!#!1 z-Kxy+!h=27B!lE<`YZiM$2zmAHhmSO&x$$>#qLk5x0w0VNRW7)DH4}L) zqfXhKH83z3TI?w*PdHF@VoP_Pczq#& zAA^#miQ+dXUgLfo&V$vv4RJ4oW1LrprPe3{@wL7MVIh@GCpP!j952t|>MQs5?tdG| zDr`)pQhopWWp9^`IDn4G5mwgmJf^*SO$DBX&}g(TFE_I6J9+Y?N%-yC-(w@Mo|vy( z)E5=Kr07<*FvOf)y1X#WY-eZ3Xj@pGZm^av1Zd0GlrpV9Kr@`3>$YXaip#euT)Xxx z({stWKP|tPKSIdTxpIjbI`HL`_{ZqkuKC$np|Zsh9-1W`7YEEQTti^1Qih&HKLEV2 zLgXof{1x0?rc(F$ynCi|W1Z%2qQke(r?yin{SH37wQ*ZlaIyX1{l-#v7uu9cyc|Wq z2f#1fSVBVL;Yq_cvHM*aq{)Tpdzs&GbqzOW@m{vOV|_^6x%TtiTwY( z@v~>oeDMSO($mv@USB(j4L{l9>ezyJ*^J^L^^Y8iT5Iri@M?b|;*AIA>w>Uqp8WMOps_U(?Ib6fi7RwNfB+AAt5 zItm>U=)x+ihcwHzCEVxaX1jMCI%MNuO%^OOsXNzFy}u<%Mc9cT$J0}9S(7FL%~$3w zT)4o5CM5!rXOOC$Uq4ZLoS(nx5%?BU23tQWhVzZF%@uP!8X15$HTz{lc{4?Qa4m1J zdNw;pff0S5U3jpc>bqC(#>=06`6b)jz46%v;h4VodFB(~va~sI=P@&@zA_25LNHP$ zAoF(`Y#uwI zyG66uIjxZI@ZpzcEs243iytcPBot*>_m*@f>K5n87XtE&^_X1oFWAe>Z0TUb#^kv; zLvhj<@$GJ(K_|8Wbnp6U>sE!_GPFDs=E;(2zY1$D9OJ)Dax_h)4ZkM`=*6LbsTxx# z^F4Q}FM3y;=%{(MTWmB zO+6i2>CYkpLQMuj-$ZC?YwJEvd81J)&_zUBSx2IM~77p)XOosb5dDEOn1)5cB^#x+`fG( z_aRDALci;WzxG<~h-spRzml+2R-Ixi82H%VZ%zHZsVLNIWmY{q5^y&YTUfAPH}l$G zA4Wzj+B%;b+@jl~gbUStU07gppBuC57y>*Duvp7FVmFCODvh#a)#c>iAaKZi>|{T# zS$HdpWF)siou+6=0cX{9JG+y6j_Zv*U&g+$itq41NgWH7m6H<$b+igaDb$XbpMeh1 z|Mbow{+fKt_IjJb;BpzN@8NSkr>xpXNVTw&TK(F;wLu04x=tvh*H-`9wQI>nIx@o6 zT}BF9EHgEHzuUO=6fbS6XV)>UmQ&uVCG;$m2ZCs7v)bG zn?e`2vnVE8XG2H-k@Y_(wxdX&i%v$>6k0IG|v0Q<+|<;zr)1>7|=6tf}wYIA$`@MgYC{0(+) zO%d@zA`(ku*%qByaVo|u%gd2AF3VrPOifKu*>IFJp_tUFdiZ@^S{emX4S4Wi#pa}l zh)C(zj}L|b-Dx2ZKk~<9SUio&yw_Idn#1qO_7i)C9M$__} zBJ-AT)%8v0&(c6i1+BXTXEO^{d^2uju|%=-4*Z(E9-EFT^aRfgoFrlX@1 zT4|T?6ak;OO;vH$4Px^W3=2^wWv_MhnecqC7Inr%gE)<65 z95qocl{)6R4cy4q%M0b@n8x!L1}o^OG*6GAaMQ>w>q_NK)qmyUo}J06NmLsufwb1t z4O%%I9=0&Mh@8m1JUzV_;;guFhqKmo3SoCO~5;!bKtt}`PSp%mFYVYWwa`xf<&$OrOI%o*HUh5+e5CUHuchG4j6MR*3 zhoMbXy3r4~J!%`u^^uwzb)IeL&4%M{(DxaA*A+AAU&4i&BEx)q?)73ZP#+nxS8w4rXl* z`O5-?7;rJZUC&(}h@md@27g8PlFU^9-tF6;g2$>F$IH`9%DlsH91Z|RuK2( zk3I%Rd3oi2*~5Do6;sH(iOnGj9H`fQQo>;Y1wgobzT(a{mq))^s3j;Iqz!x^TARVl z99eR?=o}5+pSNx;ivLu_k5dMLEsEe{tPB2&kcQ z{umo~{@#wEwafB6RopymiW60LAz2*LEm9@ux+;isHxpBmcD_y3iOt!y&|GTgt7N9C ze%5s2ou8mvdQC7GjF~SGC@$lk)xFAv65Y-AqjhK2Ymdp1k>otM2;21z^p#nq1V+HhCyd ztBE&{v5))G#{I+^%Nw{h${`?N8xo7sF$P6R(S?~P7A%*d933ps5%!vzcTpM z^TIX)!pmIdmb6fs9A)Bo0Q~B*1p4R$n}z|plG>nZJld46{H7spwl@b5m=<_|SPjVMr?{R(s|F}*C zoUzL+U;Wq~PaG2#7B2ofkPU0ccW#iCm34Nac+#RNmcn%ZSjx^{vqv;0IdK-X+B3^Hikw#yP_`zKw7bVxg)tKrvJy{7iH)?9!mT22C85 z<06JfM#c+{d2&(J-gSD&sb~yH24Wx&ZLM<*bi&5CKtT(r#^=r*+9c3z60A2P z?r2c=ZeC8s9aWE|%A3cHVy&cSCj0v2db9!Z(G}y+(Yu2wlK0gvUk;!vxy5!@aZe97 zsH6Do-m`~;a{u1F#uC?Q0wpNqlVFVHX`RAXzcI`6qa-R95v;vMnj6O>aSr9MTb3U6Jp)O}x zU-~j*G`AYauXJpoFw{zBsrqj4aMvNzGHtHzs<7jaRk~gBbRd90xk-e|DW)bVYFeHV zzHrDSN5QR6o1!^a%udNRscWj(CY1_-;u3Yo&Yig?b^E8?>|EBtUn{)F*~vbAxDnx9 z&TcSQ41zR9K7Krcb~)JaSf5mylq+efahG-%I|b=4L7H=PqjA+ppFeM8`B87`yq4(> zu_)LmC&@fgj0ja$~ONq4YTv|$(Jm|q;t=w=%ywo;$R z^us(GdNPnDm$2{^xGF>Q#D^^0XZ^-#xR6(`qyTo!Ga$tzua(6lY4(Ty{^}Pm-i`KJ zG$%#{?s_#CDm_v22~xaWFo8I z+P$qeY|GG>nqGlgWRX3Ae=9?6jO~sIghtFtpCgtI@C28jpek7rG`3jic=7v_Y@tqq zXRC!O?xh8BC`PBE=3gp`N`&}vi5!TL-qPaaD{0)`-fOCxj?8z>_QBSa(_gESNe`N+ zAMMiw{b?j@U|Q2e+B%z)wV6lTc<$@M(o%P-tf(@*F-AtfDgE5pvoN*hVtpGrmu?9R z5?3Bx*GnB@`N6{DkeKgrNY*be`*}Av8|-IVY)SFd<&=b;ZPwAa*|pAF)iIC%QT^*??Ma?4%v(IrEys|p-UMu3_``e)@sXV+9qD}SRceL~IRtZ;oOL#OY z^Y5Y18=ne=FeoNup@DLb`gD~qVObYP^Cmpxt9-+AE23Cu;pV7xox-LhxB6h&RZq|N ze{3A+?+>zx+QKY+DtD#$cc4t`IWEoyW4L@XRh*B_2Hpk^nWm^`)X$k43mjRSAfqcWcmqX%0O1S`$`Gy(7&>rmUCg21$H2j}x#De!k5 zHD}W6U>XtinpT5j<`A0Gov6eLOcqfCgmE!n0VR@8ryzj6+l9?lx!0FTAo$UvM5F{B zwL9QS2r%O9B#JZfVH`yBWxf{eWGY>S3?3;TXp{zS0P#bkSEBkq=rD|lV8eaJy1EV?j0)}D9;6{ z&F=KNvQNx0abXqaNE24VXthl}S(U4G*YNuOH`?<*<;4Bd(>p}ncW|wJbW&lZVHKjn zLneVYI((>}q3mMD=aC_>+ELLYyT4GSYNtBk8bJk*ly!6>A;|`&eKCp~$Xo>w)14%N)!4gVA+G1!5@_6)+(?}|=A(8OIDN+Z9 zoE#WU?4ApQVJ;=~;42w=QArFJPPs={vm;Masz?6Vcqjm1s_?YKP_1Kk?buOYU~!6o z=n}*i;c{7!4mE&G~t@VwPK0lV_jZeCg}o) z1@LXT`QQJt~?RDNcKT)(7+eb0sux&p<1#bl1Q3;W~c<~)jhP_cy#fukbXiDj@&9BlNYR<7dySineft|6oc)V$$cOLyV1Q>Qc_{{%UT7qT8b zdJ$JApWlmuH;X$v0(*>-TMwcGQ#8J?#Ccq+^z+}pI46AEg3APSjT|Z!$%exl4?)}< z7!6EUx9A>)bP8tM5(9y2*HHUgk?vD9ruX#Be6H9=s{wD$tl$!bPUuoPT`wNq^Psyg z4g{~WG##Cw73)P~6a=m1u@m!@SQID2fY{_q=X>259BF+et|}-u*DWGG`6j|EjE0o% z;5t+rKGGO-r%p8=&TSUh1@d5DBv=*Gid7 zAhHfsY=OZ`@GBwd`D3=_EAcLUp6xm|&G8Qpd3=6(;NCP?9RJ!DR~#5*nBw+-5s1T5 znIoyd$Qy?zm=Nt&t&_*G<_4_$?#b`r+xC-cadTIXz4&_Bcu{KJ@!UWw%jn)v!{g(W z#ihw|H9(Kq@osf;kWgQZGc^#eVTbOj(43&Cj{c;%mRWg-AX0lCk6A`@rCbrAYQ!)F zm*WBFG6_d<9YwyNq~Dsrau`p|&?eX^#KjP{?M=m*Ou>uZzkk08hn8qix&_yNhc~DM zVjC54L^vPTq&W;~i;m|{QTqutC;K}Yrwe*UJS0#q{W6atLE&=-h;pPx_NO%X$9WkyFD|#O;i`Fb} z)0I=TT!Obq#-S~mJ$LTh`j%}~*H@znTerkgc0%)$2e}JF&mk8F9%g|>!l$9K5h%*G znQj^FI>bbBnfD()B-bS7GLZ@~GX<|$4F-yy{R|NTb99g*Y#=~lc1U@&*tf-Mq)}rL zMsY5c5C|gIN*FSJkVuT^{l9^yU67F(MGeJYgrOi9bIh|VzNgq=Ar}#HtlQ38zA~px z?g&_=5^lwA>`SaBgAfTQT56fsE)3Su+Dj^pLV)Vz!}%}3b5z0!CLt!mWJp{UoB~LW z=*HIcA~o;Tg#>wf|BQ?b5)9(_BW?!OzyjHUOP4M+p~JNn*clSl-lDgp5CXK*$5)$3 zDgfqPU`k5L;@o%&bWYQwtLVlSSmX7579+i7%RwCu;?laiDe%!UDfRLU z)QCplM1ArKHkqs;aFHdXOKjfII@N|NFpk<`jm`p+CE^Ev{`qI+G_71HFNa;--A4h1x)8M@ z1PSXNK0Cq2B~*-L-vt)WnLs_S1!HIsdZ|~^A_#LN2Mj_8WWt0QWZOFA3=Zzw_b5gs zN(!3He%uR^T%}d(Ns1)U#}LY9{dyIJ`-Q}=;S$q$8owi7;|sU5Rb_%L&QpObjyKLml5u)z4z-u zl3*0{UiH97yVNw^kv5r@%4%->7k0uZBr`#ohuX*s&;u)x44OqXG{eTw%o`uO^6dO3 zNV^x%{-7;&vr)jd-uWFApGc%b86H8)TSWYjjRNy1hxUnl3=*eA_M&?3RVgYZ&xsQ! zT0eP%us2*5#~P6SJp+9=R+A5qIYQhe9e6YPgT~~Z3-@<5;xAGmWFvA)*|cd>grGSU zJ3Khq3U^2l)QcAAH{lFv=N+!CpWNj4sdD(-Ks+K!sP0;d2+I7IZ@VF0_CzkfL% z%gwxRUm~{aIWU)onYl5F=>8Eb5HY2uy&i{xGJQf6;WC=o!BBH}Hbi|V3D2hSR|!aK z^w}IBVgfd)KxBb>$ql=Q`%a$4H#fs{uKk|H{gB(Ir;b4lh=LIUPYx0GNrZny0&T!TJz&>p}g^#r9DhM0{zs zZ*5h*x3_m*=c8V4iEFOcikrljS38Fi|A3y@h+xBH!8qDpEbWF>rp6338}X1cq0FTH zA?Q_c;EyTb4kD8t)Eh<4r4^C_5pyFqtB_Jm7I~`XIM2zGO5{GDG?WfK{q-^sEYfaO zLweCWn>VbZqE+2T+5YZAZ&s;ouY|jJaT9R=6pCBRB<(KeeWWL4jZa3a^@QL(Z3|aBx4A zfs!0f6^XY$yYS0FNe>OM)hmHVE;R9yFf+Ej1wN@8^4|wTf7!gb#b*oi3~pn0slFEP zo?P2LJsghf7CDu^(9CjxmDAMqTrZ$1I$LN>W4pOk-27y{AE&V^IiGP)WDS$U&@4 z$E${?unzZfjb_q-tPtrPxUL`9-ItI5`h>J&BTk!2n)h6%sgTjz0LPu}Bz40c#|zztrt~0xN-v{fk^C`L=D_c4zQk=Ub%?yk@>)@+8@D?$m|55B(PK@AO;Bpwx(&TTS$3GdkHZh3{h!&a>pcr-9RT*?T$Y5VJO(X;QO#+Hm z%H^kLJ4-k9K|s+ys0Su6y{K`M=g}{?{xcd-w_NiRD;FrZB~|YP_#pU-9&p+a#DbosH#6Le~=AedM|#LIP;^ z?YI-{6y!G{=(_RA{6OBmK)-PCKNBuDeZ>(<1^S93QzHQ2>!C{23xt`ELhOZQ{hUi5 zxlWQ*Dcp7Jx(mv@k4f#Lr`Rz>rX4Kl3i{4-I^rD&M4r zE*cN?CdS(xC%YHf3$;J~_(OF2{_72=P|3hk9*<0u`T*yl;jg>vFS54hKcj0eB+Ot+ z5rCQNHfup5%0LDLq5}0nm{x-_LZ&tl#k&GE+Cg0Befltikp!bE3H;8j|Dgy%7tuXi zH<1^<&XT>)$6&yDT|-}gr}cj=YZt`ouLrpP@7IiW|L1f3`^Epi&C~tov;Mzj!0-Qq z7Zkt+sulS98fDhr&Mpw44l-;61(u-j!h<88har{V1Ohr#U#BAt5HZZj#>Q4-Si{}s z$%}DCmBEd$6(-c35iP90e7dt$`Vv7?2sMxd47z0vdLZV?xb@3Q4k0DF0<3^UYeUw! zaROYij#B?qO9*h(H{J;9`)B;KKtFbD*d5+{WO6$an>06EU{&v#$!l$rzdd;Uj0`Mj zOD>#j@LMfHj%D-fAYJQ@Y;ZC{r~z$Mu3mkD?nVU1Tt{{aI@zu87_RQj64c-5i%SEfqmPo}KMXd@~!NkbEt6XKLASPi<7u(sjTpv`1Q++1o#8HZRIB3Oe78=_W= z?%F=F5$r+*J(1+UFfXA*x);O@Lfy!$B>K+&lh_XJ5XyT8tbfh6s8O+}}K(g@K7gFjNxf zW=lfogNRp%J;BkAhc!%~;kp=sYKt8>@=&Dr?O>unI<$IWPj%QMZNM}wfWwp~C{wN2 z8=&EX_OZZ8HoudlD=-9C7;=mPepP`}G=l)f2${L5@qq7CxbN_FTXEEDG?fJ0M=s2d zG7DS1gLat$+>#7mhZ|87GJJ)0qlFHlik(DO0}3SrIi$jaW#<4Jhe}xoYe;(0fC=+G z|M!DKS!bi5(932=I<27LMvtHT(Cdu61mrJyVz&WSfXKySdB{vHH_i_7iW%5Z$O`tZ z(_yf5N)srvRdcz=@#{`bxr|~m&ql0uEKVXA0W}tCENzo7b`|L_H?5ixPlHdOM9py; zy@D~CLHqMi&_2C;3Y{ke(+J?EP$j>+V~o5(1`r2AKSwz@qygsYUS3?c#9x%A!WP2e zZzQk?;kII90{77FsxhR2NJPztpX@RgFv>wvLX;*9cLDnTddC=ntwivi6hIDYETYf9 z2L=X;?l5)6v_~uGyA2=gCds`q-%K1NRv$SSKvYl@cu1dC&xn@xKa_@{;^I8(4R3!U zt?Eo3fi7%ZAY@3aIce~*v7HBJ7FoMUkP#W)BwFJH3kB^k`1Lgut$)zCKRzk~V;IT- z|8&F^s~Q7jnC1oAt7gF+8b#-ACzG9kz8G$k#Tt^yP7)OY_D?A6YjeiL^|RPmHgXE_ zC3T3EgO~$)|Nbg%3Wv0r4A-Gny~PjXRH`6@ksY^l7zBk3Ig^23GGj_M9Vm*R=b~fr z;A4*|3Wi*FAJ@~tkg<8*3e<#0OI~nWdHp2>h<_W%DnkG&nFUQqNFZ1i+swQs^q5P_ z9IipgrsotvpSW0ZB*=-0Q~b^rsuH}iP&P8x`1|j_Guk#msDZ%4PG7ZIj|F(bn{tgN zvl7pvaQ(OjlHwV1K;gvp39kWL6zX<}r>8y#^aLMJz^icU6g&rhC8ItFnyIi#`@VzA z-;~ON(gxtD0rK?(K!wbGKx80x$2;urbR1+Z4i1hAJJ0Dl5iB~{ZiwU}Asf)CyN*SE zs|?}QY#MXFE;tfBJB{~Gf~BI6OM_cx-W;C-4eZ< zd>&$z;vDxnS$}uNhY2+!`~;H3=ctFPG;%FMD(s=Govgu3SiBZ7m}9)xR!x}PGtgg{ zQ6J^i@-gS)OXiu!fU)4yrGvw&S!g(pDyDh%xOq1$O-hCh~K8 zUftXIkkFoFDYqH3>@ggYEkxnzwsUe4BGJYe>_8tjL!5-I`HYJg`t0n59{P}QqMi&4 zp)%u+2+u-}v`yD9xNZ>9O@yq&RTz0t^6Af+FvP8m;zrc(i~ygENqwaG*TuznSH4PE zL-`XZP0hZ;z#L!%LI?81ufDkH*nRyC&9=-uukhOIpA$wgdDoLV-==NbnW~LjP{=1J z*kZu!_fc{5hFEjDKhY?!GSP;Ggm9n5Ia(y z>t#x((=pKdNp)NpX@pdL@E6rlTY8c4p-GZsF`*;1*Ce3_;wwxhYg`^v=Z~*Kf*EqA zmZ#c}6uqKPq9Npo&ZC-1lfhmbvu=+=zy5kv2NSdD=FA)TZ*5<107MyLBcvPy(h9kx zUJ_-M45x+ZxlzzZu7Fk7KbFeCNl!-6EP~sMf1W`$?rO>P`wDG7S#?}X3JRQJzT8w` zMe3m&@4vw_HY~A!iOWTd4m`-=m}b^_bkZ<`eQscexi`Li6rA@4DV{C?I*BfD;m23yX`yEo=n#b6l7*KBkp(5yR{6emP$91c)ZC@SrIfdmy8K zy{^Ltp=4del$=(nTP`faV-&(a(fF)Yh9Nk&ASnJDWf3UrVBSHbPXYpdr8HrdAcXx( z?>&MO&;}9oQ$j5e#QYaXxHNexMV9dc(U2MvD??1#G|{@^1X2A6v7`zj1vlPXs*SfK z`9%V_;j`3z2x2f{`bCVApD~DX^^W0&uM+P-o>`L}SP^}TKWu)oF9X9gG<6)YY)4uZ zdPlAeU5lh+p#M8zeDtq7cSey(C*U4WwjkdO^@FF?^Bk2(`UfT4R59iSA{z>dxiXF3PZmf%4{g3zsl{bO)t@}J>e(Pwbf{$~<(Hi?Mz0DYmb zcp#QV+h_w{K%vPcf#Gj|`1unVhKHKzG&`bpP|8b3-)kWbvJJFj6Ve?4c9Xi^CVRm! zc|(jl!lQH0^GJGz2vLw+*eOt@ThUb>{lOaINdz)HodQ-encgQFEBVOe+lgf4K3U2M z9b^Se0J$0spKcb^R{&oTO8$~G@?EJy zlhFGuuqWM+M1TuA17G?P>kci=rTqT>ewDiSI}RR9Be?BaCUOsI#V(TyLKVbrtZf`q zo(pk@k%Os&cd&U#=#1!2#C{@kx`-nr;|z#-e0>f(D{5#J8S6ykAqYrC&_a>qWJ!RO zWE=TKope7=?Ag6LF2C2UMrRI_!}w6MSWGjhk8T)>sdGY38xJxV&}|gnV5G&zx%Jx( zH{F8;6#s(p-^@D=A2wFY)>Et=c=ao+kg+^F#F!n7Qpxh$BaM~Xil*UsZ@U4&lR z54duoyQq0PX4X)hDQX;&lD!9xHK3%Of^Jv02-B!G!05(}8~bo*ob#ta8X^=EQ8rCZ zDU3F!lk$xr#wJfV&?2oW;a7kjPLQV}T;zv|Xo70u1f0Y{7sJ|VpKn3hpqrggV4&B=VW8%6u&br%;A zHYOZVe(*b$Ttaxd4&>L)t+2639E)L9mzb^gn5+&LjtuF&>yG|D z#CsIuu?nr*%=h`YZppJ~tuBD+Om{||(9ZUsLq~gj0XppxEgl;1pOs?CAi@IWihEAQvOiG7|r%Q zC*n-e&8V>;az}-Q+YAbEcf~wLgifApJP=ab70wjBb82j*Cl{Ab`#V(ODT_H@w zPLw%JS;-IwuOhG+?dKTM{kmlyn#7?NlOBD03(L`#s1!36N@k7+(PSKHepq@n>aJgZ z4fs&hCWbG$9H=aPh2R?O@IW?{HJmSuPEeFFS~6iKj(XaB4&w*Ft~K!c5PHeS`EZx9 zUYClmi#(qjt&5iE9$Nx2Z3Qc}Zc9Kq_|1WrHVB1nl^I6rIxqUh$4!(=+2xhnO_vzJ z;n(ITondD>csoz@n9ngAT|s+q8gJi^)LF$MbPwVZv(YD{oLV@^LG z%?(-z$uodfmaVP2BmUQA?G0j0xZ}r6kk#pS0oO;e5G8!?{CNyEdEBa#Z+ z(#rKJWZb0jtQY6@!2zdx#od6oR-sTFfQrY*)3Bi*)nsO7k_2aZ{+55^q6LP~E%?Qw z&^Qxdg?z%i9!YDEMuQ$|3zn6ev=hl3?4pZQOm!MIKMkBBiWDVZ+>MNN=PALfCUJDC z^_^F^{19+0yb#HF=drU z9Bkx2_LO-RW6-OvA#LS7TDx$0e$`SFnIZvzqu581S??6`91r+wWX5Y?aL~AUo2mSf zoc3*|fCx>?TYGuh*|YSg%;VUT(BhM-W0!f2qk7DMxp|zD#h|+ExrfI?z}bq&64N#r zT)_hxPCS0%4#_M^VXg>I0SHg!q)*75!ZR`CP0v+xeh=mphYe_kdqGTAIzJZ~UnSuT zrVkCh?c+7H~(TbrPR2T*SRTsoR6RHuA)FuAe zK+l()`B-yQMx4&EHq7Qdr~J72r=Q-Tw$)mj+|G>=MGT|DY0?Czj`(AtyqZ)SP4~VA z%zR&l3{vE=kVUuz+D{7J7+Iola>AT%#t#y#j*tkfR4y=wS8^NH8}NIt_fUxt)%ZmC z0C~U@W)K{P?qWi#`ZfDX=I>Na@z^nD$5(o5a#v=XqpLm`?b zFw@DSqUcU-{mx+FcVdF#2CT|MXUZihri-~_6gjyT=rm*L4mkU+8$=fpUGZsEpdm;3 z3AlbcxbiNa1d=@Q1flhOds=q%G}^9l(edW4hA5nVbkAfQr1c&*(Zi*#u1*3vHZ~;x z`;OpekQgGP5G@ot%&O}m;n5`91!cDRAy?X@J|5$RC-v#w8J(OddbPr8ELD(^VGE5! z%0c*pn0WV(H*2qv7|!)dNIdKnC-A%$oGFO!!VVGDcxxhe4&~%rF<;sKI5c=%LwqNx z=XvG_N2FXoMhhu0=32@??ea9kddR3D06T^*i6w@3Upu)%tTXT!p80hN1VN}KLC8~j zN})WMJf&zJ*22~r(d=H$dK*rBM{5x>0TEGtRWfv<&tPJjhcU6wTCuukHe{;Ho{guz zn^o*T%c!N)0yPZ4Dy5~4~5(LLM2@XNze20t=f%l+0NVPIx z8-%bG$1srAI*<5WlcB!^2cF>qZe$6XK_q*kRDv-Po!|V;)$!+;F29uByq#2_o9-b_ ziyi0^1S^y2$GTxgmE4+Kb(udzFIiF6yR7wWKba(qjkQx(+w(&@z0$CLVq@)>M4xF* zNhXm;0UR`(nu43`Ec3dEITyrzj8hSN1WA7~s+I&9n`+ar`Wb>_Y;xG{;SwBrlRr~) zy*pWXaZtal1UAqc@wEf zr{vLEXkRW~x*u1KC!-_P<{4vF%o`%t#~j`&BLzU1OT6OabVSuIQs=-XNeVe$XG-e% z+T!T(>~pN8VNP$oQw9`wU;>7`Y8Rdq1c*}`*oSc_ZqO^jmf_jHLnW$ftRG{aAkre$ z#y^qF7LtPoFn^isMogjO`2^uHKlI)ksQk^9|B(bE%_TIuJ9FMB0goBkIZZldHtQ@9 zT{}(>iDY5m$Al8;Qtf6{>@+$wlZ!c&;LW+m25Vg*oP@`AvubE1ggKK3mDP>5boqp2 zt2~Tt?F`jMRJYDy7Wrf|oK&I)5DOnQ;$L5j;jGb~q#{FF70jC}sTxnQU6N5h&_uh+ zy~{~Of$lDzT5eRe10hFsB$t#B#RYl|3n^arbkC%x=#>^B*b@T{nm|v|AF)hzN#@N# zCEM8XFcOAj#DIPB;|ZH<5;pLj6<7pJO)@TadMV6VQj{V2HqVk1hhbQXGU}(5^cROK zHuIK+1qDbBUn5gDq()2^lrK-!B<+|@9Yy^coe|A+K$#-w;)KLCrE&Q)k8yeixI53+`+N;7tkc@keTMD?Et*M_=ivi=a(tUBcB z4Ws&(1NycBR*dV*$B0C9nGe31t0t+53+6W&_vcc=FvR+-#lcePXs)7 zmJY_;GYHqE)**ZdAJBKT;^`X?>cZ`?0NGsP>6Ud zw4=ItOs3n5)uZk-+AuebPu>oRNs?z{V0{&Ak0%@ub9^gToh?~s>7U)F1kenS(B*OM z1}ZR7<=M;$b{!>^07#ll#G|E=$F@dExC?*hzK(of&>O^xqI3OUcQ($&uFW>EWjt0A zZB=v;$vl*Q_wG-$)t1z2Uz54_Bl*LiO;(y5d2Y!>5sSfZP$@>nF5egAWDU?>Q~l)UzBzY_DIvGJ_DFRIQ;| z?u|?i85F4)H!;?CHOY|ek-}3Eu4P+gGzblzYvCFlknY|Pp!tun)IU6&0GCa+Xhn6L ztZMr7{Jc6w#n~tbr5D7EYsh@SU}fAw?A57+pMs8OXytO(CHxe(CSPS>dtaW%U1h(K zDpc5k$;spED67JrJhYO|SgsGurErN9@U0(Q#=Wh z5VM5Jbm(3i{aWuE7bkiMHEpLOucvbTJDq`PUM~?ihV>({r3x5};mO$wiODR`n zWvWR-bQ;bKECjmAKn2W3;#k28XvRINHM->PIau$phC#`UckkZGRa%7vwX;vyw1o$! zA_8xfeE^RQ^822P9_=G)yyjw)^FN&A?G^HP(uJHJ496J@YRykL(qLqg_+xo9BKdVWCI~nEfC2-UBMiv+EirMSWr{PmCq@ zsDlL*42p;dm>A0dg3^&@0TB>Xs&o<`HL*KVlx_s+pdh_zY;+U^1O~7$APhxmBmLXw zrUVne_kY&6{`If*t@XQBj0()mec#u0&OUqZb1o==P^+H8{WNRl%wT|*P+evYwY$@g zm;`~EWj|St(15Cm3-OQf-P~w~Ic)V0?q1}g*8`+_T+iyz-p1DDAksB4Pm#r@Nv;4u zC94->7zK=E8>?M0gC&2G1YX3_UZDaMVufEsw{k{TnQr}W8gL!QNIRX8MX`=azcYP{UP{`ZMnc^aO5N3ADWAlO^x=4=CUJp-}n^+)2>9w@&z%uU?OLqd*rNhRVV%rh{0mR z5s36g>_#>x#xTpWWC_h8qb?9mCko+60>@M__tE@(%b?;17=4Fii0L-bDQQDKS1T(te8j*vQy_u{7esl3==Tq zMeD>z7|mB26QYFFeS>W*L+L*9<1hyrX}irh-L8&FRXx&GWPPK7^g13qMm zhuM_aWZY3lYC8sRMYBP$%xEA7b(clpOOwC;`YM7Mjm$c;wMX~r^o9c8P%60zwE*8F z9<7ZbCw?1-hXDt*t-XaZ?V8#|H>9FX$6x;kr5BBdPCB?Ia^@=4hnu^}#Ru*@6oJ^c zyq#tMfZ>wqzi5Dwho~P*NSDXIn77qr;Jr&wn{;OwMMY(mYn$Ir&9ptZge-8vp5#z=gX^D1wSo;(``0w|}_lY#EXEc(7z^Z!VAX zX8Fv*xQ8RImuONPY9sWOJa~fd;lcO|d5e5ObmzhClRSdwZi=>Q#8)53?&ff8Xc<4( z%E|_d7R)xCWI51`Qhi$9PE|;A`44ke1f2K@K9&*`yteOOlYXrT2wjG%7DI-3N{g-x znIn-bsfC>;l-mMlOj{gPL}Y6T98m5`*C2W>$+>J4MJOQ- z&8sxM4E!`6={Ixdn#RByv(5kmjpC#4l3m?4`2uFJ-djxu^fY7i7A%QZ0xVpRh*_Zr zF*yI;{smx2)*);8)eq{H?+Vx~U2qVY=nlx%2eq}et%-1DpNDIz<}k*5Ey?+A^8moibxzEuw}X5KVGG;%%-;7zqa3i931g(%?tDp@sdoW^Y3cX2hQ4E<}$6eU3= z$Tvw8E@o=>n6zd3*cx^7O?cdR@@NrK5!r)J!ANk}}{<|7Z4@QS?K;QH7%Xz0ml?Yh$6ceyMQLhN=hRJ*F{ z1hP(4D?ir2czGE)Ixj*?Zf($r*Kf6j=g7xW{sneTfoou=0ZCxHs{NWm!O)a0 z>@sDwEjxZIbDc{hb}u`Av3QR;rhLt<3+)|b*%bq&3=#~g=ti$yuC0I? z=JA%`-)L~2-=+rak+`Zsn8qz~qxDiGG_GSCReAJa=Ph{Lhp;stA{p&&s?_z31I-BQlYG~7{}lMi50acWpDran~$Ingdgw~x8C zKicVN^d+%LkfhQ9>|!t{B^P?ws4EPr_F=fxNQr%5!EzIM`s~|Z3q7t_%1@YwDifHO zZ0YF3^8lWwqKtYEC^8_2`_&9}uD9fBLn$doyP-O|w2(l<&`|5xeCG-I=ZBln$#;TQ z12#t3W2^h1%bZGXq*Wz~%An+zXla$(R5>(-hr(XMs$NuZ8%}ne*#|YLz@#V#$n`TQ zLRGRNVX{3^1d+NJZ4^egstNZLP$PSH70oGz-GYgISd1fc^s&<^s`4?PGpBt3zWNX< z$H{Yp(upSOLvW%V0WeTuNE;Lw=o@IhM+NDv)V2ZnijGY1LOOu?hs-twf%1li&hNLS zzQ>sRz0@2>?*p+u=aI04VVbkt8y=@3q1QE%C|t_pe1zfNH5g)2i_<5n4yj%4NwPhYmTMG*mKYk zfqqNhz7mZ8Ax4fe57>1dOD8&A02<9<tm)4<2kaz*KxpZliypsaM#4YFISQ ziHkARr@P7#Q>4iBk&BSI1w7dX+XamPT=Z}O4KY>!Uh`Hy1{~AC<)Qut`RmeC?0AnM zRg{xg3NfDn*?4`leKF>+aJ46Xd&0oO+UuiG+%x+@_N_av$i&BL$uvqg1Hi@w3b2lk z-oKbf#r!6a*-nSHY&i|idKU^+WXL_p_tr8C{`#KeP^L$~eF`R|dSzl5C0iBkVst); z-o{Ic!NqJ^~G{P9qZmu{i57 z(~ssn+kY|K@J>JB?sQq#lEy~3JHbVGhB`2S^E7em4_Kc0>)V6`taJOn+ip3DfgbpF z`*Spdg+w(B28%{R0gcSToQzUDCt%lrs#n9``BHAn*y%I~6sN&^(V_#3WeOZb zEjfs>x6r$jTU8~K^kCS_e^2(}*fBMiY1R{ZLN)Lt$xZ?aN0bfBqy{FD`belF4@tyS zYzG}QL|3zL?1Y`~!=a)RxP0~C>m;$1g~NYD zvPlHeHpN?Tg2BH9@2$l!EtvXLsqr0)&}!4>;qT;{jCGZ~i%o*y%vMEm&qdvoqy$H% zMeaz)$aS7q)o0)LLCz78lQRGc$Zgp@{0f(Ugqh3>;Tz5-D;bZzy{P9{;Sj2(<$1tG zc@_)*I}0Tpb-KU#J2ccJ|Wpn88@xgoIibGh2|})jjyhA}1w^v0xvm z;o7$O#h#I~hRAD!4lZ?|Fjvbn!`JNICp5Ms-whFTC5%1NPZb2tl>DL0M*wpLr*MPa za5Ifi^7Q1tntqN+H6myM$X>77KYZ;k<+i_Gv_N9YR#+M&rYuX^`dJ+LgTV-{o~>wk zW@PCx-|OzSWQO3f0{j*@G-M`BEEr$Joi-_ZHXNDo2())}sBPII#C9Hj;o;x2e_*xV z-;N(qhO~|zOzyM^*_vn)fPC$Y+D{L%f6Bty<9}js!cOkV3piy64wsw*cfydBz^Myg z|4Oj%!CD)B*%}fOfSS)&;OWFw;>4tK$$BuqtTn_!`oMA@Tgra_c+bPeDI6;kef1&#yY2t=4Ayh4A4xA+#C@;yA$K0ZAxS@CkRV2EHN=q?RXpy2%l5jAAPZ|0_u zIr=iz70AdiJUDZR$Pmesp%dOjg=-FbKzI`jwKTkud=8kQ5`v6C!~e;ASmVfF141?o zUxh7gyIu3}Nm8R%{QVqiZL>cI0QF*ZT5@n;36bX*??{6hP+!$IV1)uz1d2P_3g4&V;aqaTd;J~}+xBAk2K_X*F_i_?DJ}y%uY7Sja_GS~o0ILp* zCehjDY0KBbH(1b6#yem3g;f>V3DB89L<=_e=>4Mqx9TW)`mzBIIl*|aG-=FkI9)`s zu*YzalpaSNB=>qgov{GQ4!dEOQ~=uDH|p7Ak_4TNfwZF>e*x+j&bi2gzXisJM#~O^ z8JN9h(Sm=IDb4KqQ1d&@IWres$1#jgxifIys-BBBsO+^xl_FfEY{O&KO7`eF|B+Uo#(CH=Z+5j zkn{digp$v64VhY7p1Ex7>JYKV^31W?m#JuB`pR|3$TR-l;uRq zk3;jp9}r$*<=|mK2-TTwXqngK!mbGbcu;Fe1INsZ`sxT6=BFk&+=+u*c)a8LQ~H^~ z{<>0)H-ETk)6JF6ShTq&YG~)B6*eF487!2$t>U>jEGBpQ^R?xrQMKAqN2fnt@n%Tg zx3jw3=Lcg>L}}E`NxF&8JQe47uAEb??Sjp;QmYNyW5R123d*0`zcvQ2Mld5wQn zS4Fbmu5-@Pfe%k=+uwEGt(at9l{CZHQia1Abc1g}v=bauB-Sb>CY+&RCQzM~kEXvr zO|E7fZ56yp4D4ToTwn7#lJBq^Xo3l4o~)rkU2wp-zgOuA_-+C!QzOdeKN%DT;a(&b z3dFn5E9--Iw8@1An=DtWT-Lp>_7z`H6eij@ATy!x&B!foRX#sFEe&_vZ6`O6#qUhU z4w;ts^Nt;g92#~WVX)UWx=Xd}(Le*3yXP`*NnbQqW?+xPGw-I|GVeZNkg zvqp^VICSo4S9QbUrgy)%n3ZP6pU!AMLj!0W#S#)5r?WL zZV23Akanv)&`n`}dDQcxSF_54ZipW`;YO#e(8!|xBY?$bRf*A5*3)|0P_jt{6UPHx zy024D95mHPbgtK@qN)yq`bMTOLeRLw3LNT#09>U`Cu+Z!8|YNCcB}%F3quJ4%$&EI zJ3fePQGR{AT8YfAtVS@08%k=I+XIXFVzT1H49uQl7lR8ciPAzUyOons1eLsCxv2_g zo}|SMtO`vSv=WbK`!*&nZhils>R?&6h}In|9J2>KM?A2`d1x8CxOhUxy7U6rrz%)2 zY*josI)V+BCDfA|^$nL6j!qs35kce;tE$BTWB=Co;EVQSW*LMImnxSGP>(?UK zuS5B-1`oFNE2>sZ_Dqm7IwfY9ei_%&)W6e*k3(A$EZ%HtsZ89q-&?#1Yl5>s`@_&z z$5s3{`n2`QvrPQMhyNg3#@VCvQ7-Dzzw^%b z>SDRg5AKXdAwtrp9%d*}Jr0_?3!HNkVOP&&d+22AD#G9)*}$j#P4zbN={!uA(9;`~ zdbc0K7fw)^juABC52}H-uC+~lurof&)}?)uCRoEb-6w5Bq3RdYY(?XjL}-82XQ5ae z{`Ac}x!}Nyi>2#sE8RVLg;nzUuv?Ux9A<8n^fv>CaDqXfR>^M2p#iYYZcrK6s+dmD z;x!fRNZ^GaNwhk@9DnEO)X}zS>0lDE3cVs63z+6pkW#{Pv#4#vzP@p%h!=c zaZr^Fg@;;OsF5t+7QAFJlD0Z5d|<%8-yxV(s3>>pxRR2RkMYf$H-*`#V#3RWRVwq% z0t{-Vm1p!eZ&@gBwR25nV)@G(;t_fwBJEvy%b;>Tdh#gTwdc?4m4~)>uYlOjQO8Mx z7|tPA5Jwg}^^vM=_yX=#6P8N?>?g!u-wpvwm`w|B_G;~Yu5utuXRX<*j(LSgI*1Ax3&1Xsqujws zMmhZ_I>g5#Ffb6P5T(Np8W@gmnF5o06YMjb1(&vam;7{T;j=3z`(|ke1!Z3AjO22U z9Q`8nr%Mex7of?;gGDAkfS=>323Ui_)EIa-Z4g^{EV40I=$^K>d#b?p((mWbKgSX- z&;rRHTvcMVe5haZY`7xCv=~sB;XrCMnSo^qtSQK7M=;ghEU^HYu{xwgjsJ8Mqen8Cqm&=#)$}HZECGVxgBcdkI@a zJ9#HXh`k%_58bLN{0;J97;RWN1}@_e`jX|g6jBlWyWgmOKD z2_B>IMtX+)_}z3Fu4ZT21iLT? zlALUG4%*!l?+*|S3+EV({eoCUBDwD-7#nD}NJ3M^sltb0Zo?k;L+HIpl@Tn1*>^sS z$KvkiF5rIN)!ul-tkCGBdwi%fD7_fmuB8JEw5jxEaa#Vr&hM!l^% zo)fX}quVL8rFV>i8u`NiO=m-oUCVzr%YrCNcPF*$QgiC<{x#RC{XwBl@x@06a*p|< zmt6#~}-w;2RHWpWfE0$#3tDS2%X;HZ$7`WH;1_l!c>I74!a=`M8A= z`_QNmE|-QQCSZ|$ear%@;&Z2M6q^1-rd z5An5b7cGVlY8?5Qd;ZIgPfy-4;mdk0A-2=F7ZSQB`-F}QZ#5Y{?iVUPB1KUrcx1+f%#o^6qFvujj!~ahs0_JTz76dTv|Ni8#7&me$ z+y*q2WPb*AMFTut-K$qOprgSl0q?*)y`kRQ_pJo))O*f)mcF|`TR~~((>o6)NxIeB ztkGYq@@;65W=rn{iMs`1VLPWfM`eil$w#J6^$Gil7g3pUFmf^r(tFDV$5ZF{403ady*vFN^gK-Aj9TIil>d>2|xe zI?Dg?pif|iXw>=q>)30&2V#cBqD517tJm3H=zpo!^;Sw-Oe|>Sy&s5sG5aYqBTYch90!1;l#3XTBBP2Yr;W!$`v!p}WfGz5W#&HicX z$OuvV;|HyM5qjTG`f}pLCg6`mUjXKXiy9AI#s5L{WlWs5C)#nUO@HKR#%cs$P=$sfhYzJ<$LKa&oE!i&>}RpRp- zpMJYH%Q2O*?T^0R z5E!d}N1gM0@Kuamm8jm`YYLugV!z!ntx^5$anDs+KTZ;~Xe!xwP`U&D3~z!H<7c3> z5{yLT*?d&^+#7(t@ckqV#(MlJ6`A^iTZ3eQkKCgLIfVh&-iA%Fh~N2shMl~iD^2!u zLvFP1hQL5s7X{AxWT#M@s*C@cu6EsO7VRW70nG?mfXI^=rS^V2-73V)L6W{#RaNyW zwNZZ8^S~@+&0l^x)zuVsCqg6ZM*EzVtfHa71KkC4PQMN8i7!!VihuHmbLaZ)3&AXQ z@mBxEo=Hh6KE@przC0Ah9OE;#UWc~6UUWInOYGVT_B_c!>j?>pvX#}iozgRQygKm6 zc2j5a#OY4#QnS+I5z04~ZWCrV6b0C6*Ms`f`Gy%*GWtr#YO`Nl))R{3cA0;}3I0P^ z;nrUJ=664W6p-5S`fyo9G|RN{*A(S)hoJ2iJs)~ky*5k6fN3AI0nlE}t{=tEeqWIA zb#Yd+MA)3cYrR)d@~7@)r&=}sQ8QPLI%=e6JPEonv}ci<+kix)?yiABb0GB)5$o>G zRco$woJncswM4hNOjO{|`K&1RukkIp$u}V0Gy?lnEdB_#ADg}(YusD5D^TmMYuDt0 zL5qk)OYKOJQu9^IH>p-hS?qqV8d93NbxyQi`uA0DR_fhN&g?&)x_z_E?!Ze6Lsoq| z1ZWT$vE{~}XD^;r$2kd`tVPr%o1SZVTV~`t7G?B$*;!TxBd(AXi6YVyQ+24_4U`xm zVSwVjiBCq9gyDP16{RR>EXjXf-uAVo$lDtiQ@@!XS(-Qa?7G^DQm1z%y~cy8J%|o> zw3{R?BPv8UG+p;LPqqq}Yb8*X|#BQsckFT&Jw>jhp;@?)N9B%=`D!Ulk4=ntb-~A4~Uf z)HaH07J3x)3L55vTerX_X*0;{IvfRJ+hu2m6tjtBPmmHW&ZOH z0lqeGO3{QZJb$OwlrH_B_McwUq*E(A^VX7#OHr0b&)_hY{g ztY=ln$23JDA@$2I$3EBpJAcXZPzW)0iwO@8f1(g%1sD+5T%@7l${O9^O{oag3QL@&?_}DjYIM92;F1?q?6{)lRtGz0y;G#`^(!WaJ0P%CL*m9jy1? zl8B7%3C3M`2&fRwG67NV{&T<_LhCtVz z9~zJ&GxK!BSL>wL#jl9|V!8NA^J*#z~6=PF#gR& znV7ZI5pQy4p`CVP2JDI||28{0bwl=?tPhW9=q(a7+rq*kI(YX|T`&w`49R5{f)QK~l@g!KCKv0PeAC<&M3)=Z7_A91&xw z5TZIys{1#Os!5AWpWAIK$jQ`#7{uf0OO)c&)n6Z@du@IhMTk zS$vO_s6nfKS%*Sxgeop-o#$6MD*Qzx(|MUdy$*|&1|?P?=o!4D1P51XX=zWPeP(>F12~a1RnK-YTYo^_!js= z2aK}7%U_dFXUKx7B`8q6F}p}=(t=}Ks0dRH_fQS*4A1(I`WyqvtnW{lV}u`3LB$I7 z#uvsIlM3-h+TvUGV9M`AMfv3d)QJOnS>IG+JKe=l4BY&^(WjTvU3e!C(!v7`Hf|T% zbg?13v9~wlR7<#7Y-uDl^r!|5Dk(&NlouM4WFglbpDRMBd7ld_S?e~lCvZ+exO(pK zCkq8BW!f0c8&`h-d*s~e;mf^ya=P3`yqqF@Zp9d{QabpeE)Y%?2bAUUSb{uy{pG$# zI4lPYWCg1bsArWUjPgN+o}85zy~;hupN_@uD>rm=;TU1+Z|-c}BkxnnzWd?8&k(Ta zj`-VEhG)0a6$SmM+iUur`D<*V0=Vq`St+2My-#&>jGipC&(|^W_p~mQUO_u#0yC*@ zuQ4Y83j) z*jr|H__x)0s425SD#kb7@~sC)pO3aLKOFhECTGa+MN(&yGxJ{U*Vpdc6?7 zPqC%7%xlm{h{j8No3IIGwk3RhOjpnEv1#W+U5EVc?#uf|2BdQlLEMc}`+ zoH7#hu{QXKS=uJ+gWbR#M_n>AGi!2Q={ubMcDEI7?#MrnXM3S7tu>Wj3^$jAIuqh} z%ZByr=V=%ma|mbkm^;^^$A_OTD8Bj{H0QhPeMjy-+a8(GidG3t>BP?UnW&bcyB*Eb zLRQTATqrPl$-sPCx$5O5-JIZ7QRc%xr&%#-y1HqPX>34PM-5qNI7&!cp4MKc-+2{| zUOdBdmzl#Fv6FKi>{`ehC_dZKoaoDHMB~JxCfamCuV#=29n!dh&otxY|K}2NCUdw)ph3kqo zUq$o02<<;;?(xX+n>(Lw?E!maU1=I>44YSMwmH3S(5r8UKlr@!*k`gh{6yzfFG8u; zKnlt<(eqO1JAggyuvkWCG1sPKR|+ihs(3?v`)WzNgBbI(35I79#$G+w&-_Jd-$&&kUGb9ljYw+|11A=Cbh9a{tL3Z_7r>v2k2qHbcPS)}Jh zjAS}|${Y%k)5T3qVKn~{rXdrOjlqiZaz$hF5pum?bwA+AAHy)WTW4@ZvoRy*8j_Y> zLOsUStb4M+T1gIRP0zYOQX2oi$^tyg!MqM%it&hn!2L=9S92rB2%(^2XLU>24=qRC zeBm;?g*K18pPUeo^X*}k3B_3*6j3pk>KDhTF>LE@^>1$EB>QSxEshahRP;QWw3osJ zzmHQ_DAN{oSbuc23wZ=y%3YI9CRL2fS%Tc1MrcUnfLJlG3 z7Ks&y*c^oRnYyg8pX^dT7JD%f6L<9NDvTT9e<%G9EemUrsq_d_6A~bR)qzHuV0g{h zgAETeLf|;eESA&-JZJSl@zPAY2wpte>Ik$FYEXc%l2OUuBdwbh5e}67s^{~t^waH& zI42gb;xv37)DFps>Y_vAU|mE+n*@C-_s_p-Eq$iq+J2Rnc@%{Pp%EOYDL-kU&F;b8 z@~lHRI`8_%;|B(2jtMFE_HG0G#G`ZViZtZKx(%2I04pm6%E|iTnys=k3)7c`x+@Vuj}jx?G&gHq9L=?e2>w@IC^y(Vz}Ox<9s!*q7~rI|U_pY3)S z>a!WJtq$Hs2voU>0=xU^9p$m-aXWpD-3;bMRXKMQc23&F(PDx8RACkw&0(QmR=3ebF;;*+{rp*Dn_#!;uTI7`d25nbJ%P7t z(5I5o-23CCg&wRFh>}{#CglB>wld<>;m*fw@f>@W>KxP%N`4!EfVU|dM__^o@$vD| zL&s7N&gl+d^*p=e>M?c0i+vRz&>2Pb&iQ$gA^;gJaeUefm`&J$yBz{oQ_o|m%stnU zFGqfDd9*&J?ej8Y6vzeiQRoyDaHRCCsS-@b`7GB#BXH`%ZeGk4ck9XHwer!pN3s9r z#Z4mkFjg^`ctpqjAnz66q@^nmeHt*dQ8`>a;*$56>ut+i>Ep4}XhBYyE5WL=OtUb6 zFO7@x%>gzG4hsf}!>zEMb8*j2xhEbu*X=*E&1Gtt>jE%{($!3NeRpy*WKg=* zA_G>VS0WT9w4$~2M>Jh^_}2Etdh?tl^Su5?q_O|SV?=(O*MKI$Tx#I;hqDA)V8v$fjk zR3GyRddI)OU@RPb8)XoSCSOP&YC#0cSC(n1JagZDO^rZnxQR=KK@}omfLlYAdKk-U@04yASbe(Hkt{>@VjS|@e zUE2h^uIMswYeF%f*`4jCdUXd({ElzO)V%i#Fso7;+MPAnh){SB*c<@T`dhx1xezXW zP&TY9+<5a7?^cD{**ZtqyAE9@`D>E699_#4-=JQIglaFiCn8Z9R0}J<+DBqCCe4e*OJ4&&r@;hE%_sfS@)A~jr{bU zx4%0nBAwZ#bXS_^-C+)>-u(Sloth{>L=K2` z=80|-V=TJQzsQiIA6pst(@7^FzdRuCbD#LJ%z>7p4XAsHXi?V1(T{97HL~95@y(Yo z+KG}ZuoFYmAkJE&vqZhFefZl()^5*wF{?Y#iIpM=2?^H`{+Mz`rnW~Jg|RmZ^kb(| zLR52(@+p|y+pF-`Q?EgvoA*FUV&VpNJEB~)jb*sHh}AF>-7@adAd9QkF&lV`?6Z+a zylQ=N5~AUplm+xzcB_;AnZ;2SjK?l*=ZxgkBOJD)OBc z7A=#lkvD1USsh_^<4hy{rY?O@HKG|cXbYg^k%C4a;(w^f=;=<%$~e;hUl-Ot-?*QF zEkm0y1;bXgD(@){hW@UltLuyKR#!&%V9PJ__Qu9yKyZa4KYATlay&Mdlsmvc;xGg& z!$TC0G!T8P4f3H?p>)G&UCordo)k*9e4(LHcK~_JqBZ%=MducC!#wbpfy6n4sIJ0X zh+N>9tIIBKT*|h>T5iBN@|B~@Y-)75fXaR_ifoR!>-%$g7wCv_b{s-zbZ5<)#1~8H z1TYTL>YXE!bSrNUuIzl6AwSepWQy{0urF&!2GdTmaa@cas}UVs5aPYS2uRW5Kx?NT zJqQdx5w~L0(xC70j`o=N20pST2RQB8QG{Wu1~MgkZ+wHp}5B7`&vb7d@D#{Ek1K%jOW% zfy&#^()q9Y`~r3{4$?)#cWDhxsEAzPD`~Dt#%4h4SgMu5@0HnqC3L&+@XjGbIBigk z&x0c(EEn799zNCvenPMH;Br7zPMw<%O~Be~=K>Nm8@F70^r?ze^JOqQaNU82ze#4c zrSfAG(JG`REr+Ud!LfUvU5DF!+U9rns2UVArW?Cu$>gc6WNpG=zV5iTDnt*~_00}e zcHHAnjLWeOd||7Yl_E)ZWp|K*pcm$(>sZRs?_Dq)+>rMIwe@w_EO}s-Oekh?v7qvl zzwWQ~3YeJ$2w225xp`s3-j+&GYj<+*goM6dGvH+LFRuXfI5{Vt?mUWI-?c|yrAm>L z8v8Dbh<^tR8})Ljg%Rj^Sy{axYiJ!dcelQL!HuR>X;`b;HmVqnU#j(Lt`0y)5HSM;Uy?au2EVNapvd9@zaoHRMLJ;R} zIro0~q63sZH7c+7FDJkISYE_u^ynyDw08NQc>TKr z;EEc^dC;VOR=KN0{iQ&?Pg$_>Sc*AztO4PV&Bzk8(J>X#7hRim=v~M~ruQSzg< z2?k77=byyJ#pc?D=RKCNM=pWQK~SXS8R=IJ@p0C#P9Vtg&_eH{D4wxis@g^k(?c`q zCIZqGgog(BAaO+bg%G^C9YDG7(QI8Kqy;5Rktoh`8|vhjizC&nvih2?Cg^iYhj_WY zp?%{>mutV$7x&G=HnRl~r?Yq{^6Mc|t@2c;0FY*03b&QkDt5POofrMYmbs&up70!Y z1U)4x9nes_Z;?aY6~a0s6l|NL?}1?@b$r92-njZ2phES!Zsrw!>|}_mAm@m)2DgF2 zn6w7#SUp;Z3o-xK7fhl&jtEczJf@16RzwBX!`YhD*K#n05@99%=KQ*1?8Lj6?>GO* z?H_5xG-Y9=r#6K)89;qJIJqu97~kl3jNakSi&hziL1Do_PrBFPESv^9G@B3}-0Ab)%gYf#ep9$368@S=Z4X z>V1%ZHXLFPzMGDSw)cYa8li+I@Cyt_y826Q4#YSYqQ)a3NbuFXi@YzTD=HQF*eP= zExSm8`x+3S9B~K=Cz(ujJ8^IA2nb_ipHeCVTBh2LXz&s&x@NR@nqrqEL@hPbCNf=nF}40UV9yO6+ej_Vrz9-pMHs&>#0Ct(d70g~^j5o9KDKX~a{ zUI+dWN5v@>58S$fO=AT7;Pb>G(M|*`gK-_wy>D-7ITih#N*U-e__uL_ZKMx$q<;s9 zyXTs(Y+LfX;L1`5F(HxICH{U{w%@ibGrx&qUmbYY*>Dv4$JzEn-1j-^{APX%PODqz z+W&!k%{`4OZa%b6>0Fk@gYV8*S?ITA4h^(5P$(l33Wh>i$as0Mk;v5M6uEZCgSwjr z4tN(H_f-fsYe?E7{*J{IJF9U(oLnx|nC`r&qLH^hgDuyW1@O+HM+AW_pLRAqTtfW7 zPFJlg^^dytzggUXqE7|JGt#ukM)ReJXGT)sN4+a>hI9iGC zv(@Q<&)k*P{gtX{4H7r>&^TiXKR>aD%_vL)&HZ7CsAF_@TJYI3C`Io}h#tM0NysKB zpp_Buy(Hg(|J+;GZXS)(b?W~(C#Ddo8BI6u7O0y*r|55%Z8yTqLR?RvOg~fk+a_d= zT>K2Wcr3+jVoAyykpU?=uI@X*Et=c{(69#at(HolTX@)c7yc=FC8pBndRY>6X(X_E z;B)bP@s4%nS%bD5qsZvQ+%{K1nrC4Z^d+nJkLIjkQ;pQ-+#n;(^H3|s4B8{9jkkKx z<=27JE*cWb4HP-e_?`BUXvb2#{P3@^u(&S;Oo9SqOVP6fK&L^swpUXu{oS9e9CRbK zbFGVbh+jK#{keBs9x<>v@Ckx0$l++48t_Y*tCN#J)bXO?3v;9zH{Gj2=~q(2W9Zfl-P)MGgBtaDd| z2B)TOBCh*cmDY2R$!U*c_aj3VqBqDefXTN(Tli|Dul4j`HH`)H1JDSMi77la#{d`c zm=;QkG$>0GbU1n<)r_LM??*Q>c2XKPFcd0Z6k$@hg$9j&__(wy9Wd?b9Csewvz3<( z7>whga$`j+PM_F>va>^EzJnr7!GYrYsp6XGJDXN+&$}#zE#=$1MzJ{H6q|10Siom#5^gnypO@s`^ha)G-{p%R0LX5sDa0cHQi>( zkP7PVV^3?bXCKhTA&9b{qHpLGu=3RFIP{RW$UYs%fk3}y=o_5pb5?IQeLZaL*s3|{ zlm#@885WTXpvCs!V0xXpV&*j_jAl9FZYahP(~6KAa0J~Flz1`ik~7NPtRm6<3d4L0 zO&CYONF8}`8(@f)13HE53Q$}Rg;+~=&^zpF2^potP%xr+ICy2)V$${VC@^^XvIeFa zwZAhv;s2jWg3psx=TqH{@iNZDiqdZKtYmKD%4@^V=J_t#Nhqk$;&Y6^KQG46=GnHM zV(&~3;umCElMi0RX+}%`#^kBtqLE1RYX`fNj-Zgqi=oc9xtnYb!)q1~>=24I*HaQW z=<&(ze$t7S7{Xm33URSCJJ3PG?3mQ12nt=~KSG01`x*KE8=oL9olvL5!9^fu-$JJm z6U6`#3=|vcI%!0XLX;R|)XTWt3!QEy<~;809>D3!5`YTu1&I;}h@mcT)NhAA5g0(D zrxAq9%XC9Oa|di3hX8tXEo(64>>9iPb4KdazarU5Clm+e9Td0$igHje1rjgt27K;5;LC%x{0-vN6bbn2vYPv!p`w$Uw8A%+tlXUh6cJiDxo*z`T(T zhyp2K7a|eM6s6!#`hTX1ctH+Co&x7widjdPe<_eT<19O!_0qY-G>wX5F1CJJnNg)-Q#aLcTlMGCN zI4#HxMCcQ%$^>s$M9Gs5OUP6t{IySZltjnYi^ZZKBN4H$8ifwPAhg~>FeYr9Wri{u zkakX!?|%nF1|>*q@;CwQ5JFHu;XsSH%>H+JoKd0U_6M;8R0NO=O(9@ z<-LIR$zfp{qh9rl@Sv2hXP3iB`S!#?TNiJk+cl=+%B!Qm(d6WvUAnF#-pS0P2PiaI-OI8WOgYAoC+EXv0PE9UUOFLQ-iv^)En9m7ms;RpayWy}pYGIK z?=hrPhOX$@i&vD@AYQpIPE+>FL&LQ6JlqdUk>=XZP}yudQtvOeJO{_qF(hk zTWpN|&(e{6{eE&=F(n{r>{U&EaB^3|bvmG)McYYJnv`KS=fZ}{!wF@60G1+8yaFBz zOs&f#i8Fw?^4jUny-m7Xd(##qhMC~+zlArnY1Im^zt-@_jm_Ww-tfoHsXW`Lsq>0! zjkh|k>4H~E-*NG(iwy@_)N6|-cvOYD6#tYmt_DxTGP5qzF%f3d+QvK-3!R-8h75-p z&QKLYo?pnp5rR{A6miU4#5G4dDN*&r18)s(GF-axJ%>e=}RkiNsI_wa!%#X?d6=^uwBs+r0x zq!GX*A+kQ15|;C;uMJmP`Hlg# z`x!*n(*{*lQ&X#pn_5gv>CMFhD?vdQ5xq%Ymj+uEU#h9gLyXD>DD{FxYbdNuf>x=v z%GiwnkGKA36xM)G)*uFXAo-vf?}EV~@lWEVkl_+; zU1w@tcx;-xsNzwP?NWzrCUm`Jx--86X6hV7#K6dyR1vy1QV4jD-EhoQ1JOswFx~(y z8-0(&u{m|bdqakT3DuiSFBM3pqTP=4DGn#2($U8jOnH4)v>3~@8RHy?4#p+#sbj$7 zmI+bkGBObdjJ);jbQ)hEzIq}SN>$Zk*zcHE!h<{s3+$3iceV-jd3(0-5*>4${A))UuwcIF$Py7g3>CqZ&LJ;g# zCdR+;@M<}L_x|t_-okR26+VZ4@^oZG_9G1V4**l2aeqA{=k9YUed_2!<_yEZc5lFX z+MNG`7#3yAEI?Nh!|0e<^j(kEE+=g>$^}PZ_kWzt=Q&tzCCBf?vL@!Q4x< zJ#hL}uGXVa(C6vhXFjdAlSXQ}6u|40#d7J&=FVR5$B1d^+iJjw8-u0$mo-)(HucpS ze@1bz-Sf99FL+6JFIf^;*CAh7l{!N0=y+f@i7lSe7tp;uae*mmf) zKK;o(;U3`}XySIK_+a-)f40V^U%^&}-o<-dh4`nYEG^8IIPvZ!eA*@zFGjT@HgsS^ zk4T2oFI$I)$HbN0hO~JXK>2G#2FmPChz$XmjUQB|vZEiiJS9&5%SrOuOOm}=|E*XQ zVmO07>{NA~U!B0v#!%Vme}-)-{mKR*;Lm501#keAU>t;H%0E<42qUhPTs4 zfUD~yH4@NGR0hZ-^G=9h!+&Aeot(vRNAu$TJn5vRcYIsn#y97kLB!<6tg)!~ME6i_ z?3w~5jqc->B^cCFJ_8p|?+OOweRBFVP4Qt2?l0t&|^pf2jlyi=ZhC zS>~KNAM=~sw%;iAZ{Ac;<__~JD9|AfGLwTHlK`jN|a&7|c_Xtou zj_R3X?j0;*vNoVFO=<3fQ|^0qJE{S(v@+~PQagw-{#4ZhweS!|q~>KF1x3y=LTb$` z(8ZW)Ochi61PMh`qJdHjy9cm`OG#O#Le$Oz@XY~tz61g5?;ouMnYkMv9r{654jh4= zu+NzwlPe*x90$`KgR?4`f)H_z?A#2!8RBi= z!=jQC0IogKa3JDSbu`GXMPQS7G5$8i>& zq$R&)?g$X79&_2yKi`)7X1w)|AR8pD(q1#%RG3*kPEOmj5~}aYIHnySNN8OBMEiwr z4ifFUwDlpXa_fUmPRj*pr*6cw3~gCB4thm~hv!fy2+D+pQ27DVie2&;&%n>#rVbT{S_zPT(E}5T?ez$nz)(tX@jPX7FP4-q`V-8R1eBDB+(tD zAo3nll;D{g_U$FB1Nw4S82039k=#S#Z9e#j08;pYJ5F5T(d!8XULC`lD_ZrBKbnO> zuLAPfku8iI6xQHL#rgoNlBilbyPDZ|VdLFis&O<5 zQ<@?o+X10NQC;W7kc0rYiTFS6j$5}!sXd~v36#(t@}Xjj34?f7z*0jNl7RP-N%uH8 z@OLuUp@UAG!#bMLrAgVCnzBCiQ9x8u!w%WG*C8zs-I8~q90DuYwUl}v^Ef11G>QWj zk+_MnT=7Z>U5}7$&|mDNIGgqg@j{e^J3!1BV>SkxvrDRaJzCm!C~tiAA5tvx>W_l# zr53WYVnjIzg)JLlS}!4cVdE=;sUgz_sgRU*uH-S_HfywuRf9_j*^}D(bUwx9KHEM& z1oiPKj2iz;Y1v4?+O^c1S@uoipT||@O{>sTRS0$C+$Sht;0!&&+@u0h*(d?q5|!}( zV4>eEon!ty>{*LNUlHLFIwOJkcYAQ>L6v`Ryi_S-D=k3*F&B#s-9 zGXw)^NfaiJ9Eruu!$v@hX|F-(4^|h6ZZ`)k4(EQ&fLKrMz>yaLJgAus^!7xPGTp)8Ro>KVkNA!%DUr|$uS?kJy zFDJmQFUd_;41k@20}iMphV~GmAGzfC;XjT3h)~&f~sl_9I4+-tzXUE|o7@F#guaKo3gsTCl`s)k$z)Cd{q+|H)r4raJ@w60G@ z&{}d=2_4xApIZN)PQV8TL5`AkXUlY)bAo=+jKYg z4x#}~?<6Sdll8s!84_5jWxH&b8Jd58ikd2KKQGt%+3tYfA6oATPIv{f2J}q)()V(z ztfEJi;hBM7uS{ap^;x}rNs-&eDHw3>#{*c##$8ELhi`;RX(Bb`7{G#-@g$i89-8jsxi}kOjo447{c6?}OJACv#tU@^)HMEB{O# zF*n=pfu$06OdJlu!F&fgcAFs%%FK;`Hm<{JJg-*DZLnYCVg^V`@=6WDEP4pj_Hu>| z&v&=;lR&&}Jv}7{T>ujBKuAt*#lX6Qo(C0G-soQawtt^KtS=)?s~Ur>Q&!sZU0c_C zjB!Nf4?r0iP34(vd+iduGvHi;nv49L-IiU)D-GgWhx6e+ef7iTM@a}Jqhyhv6l7Pl zegW{6XOC~S({6M`>`*4!6&lepNB~6uVE!;CILA~{g9R&a(ZgYVT~}*(>D(3rCP+dD zoe@|h2ON(K-V4Zr{v_A2Y(v>B*az*%-9k(_CRXR+n4@Aq(V~nRoaVt{4!n){nQp4~ zLH@xJWi|w1f*J&vpzWq2{B=g_V7U-b(F^4)c;t^XKfE^1UcV1;tl>zz_hGE}2@2q}fT=n9a85WQJNB3KWfQD8P zu??+Qnq0?Zh(O_S2xl%ifO{wb6GG63d`71X7z=t7w|8XCDu`}INkj;qVv7>|wbYz| zmd`jmVQM$Qe7V)Snq06rMQ0S1jOkM)G6zO9?o?zG{ z+`1Spk5BN{Hh3aJlbFM6(-PMFDcR){o!@gD^ldifuEcAt*t{Ik76-4>5 z56jBR;3|~{1}-2U3?z0dW}>{1$M3^8b!O=5>I8^^YtbjV6J4w&2ujrIkBKSy=(w_i z<=Xq8B0Y$pDI?JE#HmRFda6Wk;k48()UFK1fM7p0`oh9?nf$`Uc0H?N;MRna5ruc< z`edrasTrDbDv|TlL4m0X21IXD-xwu^_*bXr*Tea4fng&CpQtwoN16h7L5$4e!7jq& zRkF7g_(~O_l=j7mWNXPakGx_9eECmxXxM^GN6KS+x}YicVip$GCs4ita1c4OhruV( z$fp!Q3UUGPKB3(M$k7D*;#1B9D#GhKyt419DSQT2JpD{OW!o{S4Xe7t2Ca!?T4MgyT34 zYPMksuCbD4mPf#-9fIaH(7!Oh{$Boc-n1G*+K+wKk5|2!{c+b2HE8gW@X%*zRmVF#7f7*Z&m8BvLZfSeJ?vA4%USUf`b(Ya}Zuw<+bG+;bKY?!` zyOa?u_+58~-Q6Q{v9|TB>8)!F(l-)$NdA?KkG}lDS!6a>m!KgGKU=3svv4blK(=ax zf82&tO(q9e1zqa>uss{~WOqR>kZUJRra5yhM7JE%!%) zj#bl4t`o{oQdJg!NE1D;50%dWb1q1&B-<>~qcJ(rjwHW2&GeLl6b};nxRWY3TIDfn+&AJdgVPC{tsic>@V3g#K}J`++rL@xKybvDKX5b@f8J)cKR6o7tm{{-6n zw++Hdes@A`q>LaN|7REC7tk}-WYpaXfoH6$LZdfyT62%yM>RBToGy`C4TftC(D|c_@cj4qa7R)@^ z5TCq}W89j!*!v%K;;fZ}Ie|s8LPlAD7b%Z$APNG)3lnoPrVVBLwBx;8M2npDFP;Vk_ zix}G{P_zNh4>IB!DUwEnn*KPhZ&`zuQ5q9N8>=7-V?z1?yA(K#Z4sWGyTwj1U>8_y zV6&#U;gA$`x|lBm*>D}kzxRV%w1)_LvozI&nr_;|7l&6tsbJ;cQQodhSUt5bIv)^> z3L5mkI}t;V(@s&62)WyC-`XohOC;>pt>qdTwwu<)THeoKNyCppMI=z1EGf{`DMVcf zp1WeGVI1|4Vf3Xix&R@OP`P25;;;IH>LsTAJf9eG;`3?Bh4n7__T6H((^`e;6g+a@ z#g=igqvzZaX1+ABtCa5xE)8(}?qfkgpu9Yr%1&lh3Ffw10|duBo2RI97D4#FNo#y+ z=m+7P|7PySufGmQN!uTWb%uWH+96~R8jV8SIv`(it^@6TvageSiSiKBR?X&sjEip1 z3|W#MGOYTGeJP+1>pl^|Wp1EjaW&~(EVxCA;>OMC5TwJxNW3T~80Tb7&>$M@Ua!)~ z;Ee{H5WW@(K5_xtZ`mCDuz26K%8>u(PL|nvOfu)+Om%zF>Iewp(o_z5CneX(+|101 zVkKxXnwLcb_=z|p&vfX9@0f*o;J+UCfgwKb2QNw=3MqH$YDy?(y0t+}2Ou!6L+6Mh z79xiQPgt%XQ;E7%iL7){T6Q6fk&3`hBBsSlfk7+e<^+Q)Zdm>i$vYJb*N&8x!$q+6EIwXO@?~o zgF|gY16j}UOYs7Ig?}Cg}>|>F<6bEKS8Qnm4XJW#qF{-*O{wlVCuE zI4vIl1EW*#2_C@AM$9mliQ4pW-S*0IB1uBt29Bo00Ri=qf(jN`8I87@o9zGm&sS^? z$kz?jQnlY1^lv^PW`r{%e2FyryuC8?A(>yE!gx|xfG#O*U;}(U`(mOIh-0+L4_F<< zcxyHXb&G>1pGGlYl+T~Tf+zEn>lgg$WV;rTm3k$aUK~UR>R_dk!4)p9`lD`ry3A}Q zJjp!&C>ks^qIO!2Q#*_Qsdmb0&lwIc|0j$G$oXON{Fk!4mo>mD?61>y{p=Ltq5w?{~5-fvwJ$x=uY5V=e*wE5!9 zM6&`uU=DgyllxJh6o6s!{4>Mf;*1v+ zbidAX!+7?634#}(Z8=fjh4I0}F!D`gx;<)uJZ9(-^Vx%5X^Wq9@>~lz#Tpv=%1N;c zVg|b%=NlW-cqpy_+p!+{$zIF9=Ic&9^U`207BT4syEx4>i|0ZMn9D{q2BW|JCbW`; zqFg5sv_=pu8bMzDulC+Ms_MI2_eG;g)TpsXG}y2pmM9i9f+jYWU8E@>MiB)R#D-D? zP3#3(ii!;jDjh{p1OZJ{M65`$P!z0yVxdVrpSf(=Z_da)_uT!*8RMRe{bt9)TEE}7 z%=ye`K66e6{~3F`e2xYJ+Fppg$sKaIO8&_tN*{HFq#vdIg+TYp*K3DbIGhU#>i)s; zi^QF)xkFVPC}|@io;|+f)WeRPaCw8zkF4D-^1)kbfC@KFoTy zSjt+^1<3PC@v~f1AxosOixjL4ExXWI?z>Oc15!L0UT#vQIHRMPAU?ONuhh8tCZWA&M^M`93d}*@?t*!O3lpg z+gG1gcUawe|MwUFcMa73Uz`K?2_mE;+@7;y{)tO1wKE=2y*V*?8w11b3 zyh6X~luF|vKdF{B5yv+J|G!KGp!1Gj_*&XCRs=30Uoccs8bT)|6AP&%)A;%BMp~*L zlz$cfLNh)|#VDoAz(nAq{*qVmH5KvU3qm3Eq*IM3;Czm#x=^ToEN|z-=_`e{LMqrZ z@tDWBSKRoWlENdCa04s1Te|Ol_TGdAKOTVGeofl&1k)}gd*(|pr44@+737=e&S60#xRVIUjm(8A>6WLml zdp3J6jci2oEM+J}oc%@au>t}u%t-}URPP&-DIODLi4GaJ{&5lt`wfmOnzhi2BSDUH z?&fqyvhSjWz)4y^zT$1)JY&x%+2)xpUCS;+&&4c@{Ip9?yC1r0{Vydd1|Cz9M8p0% zIelprgy4R{^8VH3r=}E%Oht;96+&uPFG@d6fwkuQqV_7>q3#RF>a;+PM z?SF?G6m5X0BZn?|B}x}z*R(DCpfhC^02Z1GN&iKpsIt`ZNW-e^q#;-xX<;dx1GaVl#lTrgZX8Y$pOv6@7Ybg;=NJy)Yc*o;u z<;t(V5swGS=O#Ro)F%R0i3@K?O&HP89Ndt$^F7r9$-h#XU5Fhl)9a)#Idy+F2`3n} zV{Uc7td2?We_IhtzaNemzL*j(DQe>^^C?rdN$nDHmN7y;BmI>s4e`&U>C7*f$|DRL zo*osaQe~-iyDx2Plk-*bmtv{6f&jTm#OTY!C8>8%DZrn&W>-v?Qd-$Rngh2IHo!Cb zQ!l{1bngqjT`5gYgpv|z!4TeD3Tu_t01k;R5e1#5ph82Gvq_VB$};7*iPunVV@Nto z|3az3SBU;}_Q6;g&XgZg{|~FWOJ{|2uwf==Y)d|0; zN3MtIdyZLdKKZ9VY1y}jt6Ki2wfWepDlhYY6P!Q3*-v=6Ej{%%hr=<=4nJ(TITHnB_-|*HF4jIgoYNOq2jP<1VIsH&WHd)DK;SE<5pO ztaFCbXi8QeoPBh@^?}0 zqH>*m?=AqmFDFM+RW)s7Y#S)gG|72${s$Q9*-@37Akf)I#Pmel>irnu*F|AOCQX{6 zh;V`%H=DwSsgJUxJo`Nf_5HXdZRs&2Hcc}TyezGQh1qRAWZ8mGD#Gm_k8u08i>~z% zlPyXn__1eXjFY4iOxiR_PP8#uKPKNSK_ZCQM!GA8LEl-BtNzet4VyY&bT511PpNnH zaZ}M8{bqdNZHE26;MOnSJV9Mmo>6K_Hbo2w%tK)dvlhIqT1Y_9M*}3k@+@B8OhwVa zuHA#(c_(tK$fv@M+TK2N>DUb+v=+XMOt(o?c1}{&<3sViwJRUp+xL0 z3##qEAN{m+Pwi_;~DntUZ#s{{V4?#QlF`*p!z}73eTuY>g~FZM|IQJmrvlI89yH9 zKuJ%MZ$CCO+F#&1@eg#b8Et5 zfA+c)SY0!;ddSH>muLK7owR#vfq6;KpA|6%2OZN4Pp|WOdFN2K=2|{Y=Z{fn)$bXx z-Ev1klG(36(lgWC>T_?#$IG9?WB3THmMf%2MacoSyg+_PQWF6Sk&_&Y7a=Rl7s^0J;D zt1o{B!k$K_TuU8ZX2m{OVui!eRM6DL2=1X-L4$hrZbSVXMK_QKRbS#T)UE9JmUctK z_|9(KyN`iH-=Uewk_9mrI4?tm|J#qe{iSd1)-?>do5=As88>c|^8>0Nd^(M8gvYoV zGav1@3#Hq!9s^I-c{a!?=MU!^5>6Mcy2%4R9Nq6rZ0diA>ffT>2^TM4?*7wHx0Ylh z!EDx#`6b*51S8}%Ok>yhzM$?gYpBXrR<8Z>c4|ZE0UE^6-XqMt>IOTV^^)0;yY*a` zu9|hyI`I&LJdfOsj4#jA4z-L)t0;vE4>a)ZsI_#3JjeMbbHAuDU!UYC+Ojg!j?Y-R z@Y#(}S$e*G@UqtjAkB6@ybGJqy?y&(GO1OoVVeQhiuWdXwMz%W$9ga^Fa}Cxk!5a0 z(VdA)-ab|+qUR0yhN+_1=G-b*2K+H1{78PZ3a({z`}_CmBsiV(&}Z!%wbL(f8HWx@ z2GLnH8lQaR>AC06=cc>U*E1cy?3_pM$%p<3UsG3}{YA^oA6RDj#flP)2h%3T%V0RE zS75e79><0;MJ6Lg{NVsu$aES7t#`J?uT{YogIa&vg0iVl)5eEKHB6;z|D_wgw7<~S zA2l$J{6rAqGE=>$F6~aJpvZz&4+}p7LHTAiwFR9HerWRc;g|C7c8C5qe`jJckhi~E z*@8%i%xTCXcIN6XY&&}L!TG%n;qMZrvnIeK9E`t9OwACnJ8 zxSzjmnELSC{r>dGc!B78@`ppvnYq{uKx;E(*>PgpLzov!zRbwl+F#R_S)xH!@o|U& zgXuaIOgK?*dMPzDw#Y-S?(QGFqWb+D*+}K$$5)!jW6ZjnmF|M`mz;+GkP z`4g5*1cw=NFnlOL(Tsg6EF6-u&I%tr-VQ>Ok-@L9{v}lXcR@x@x1}%EJ zT{Cp*$)^$NPf(lV!uT+F=N;m!X zdJGLNE&bl`TBz@kYK>nv7&P$vxpn7Otvfesde%T;`zX8Vl`DO(&RKmi-Fy3pUddfN zp4B#(ui-ViR;|kFp2^ZXt-|#q4f0}Cv_nQ4h6^(=Hs3n_72(Qc7G}zCBh)Gb@wW2u ziMq}?9`gCQy^^U4ymQGd~c5Z0Y6mQ>uSc$Czj(-{gvh|IBzF#@zV3syJ3ThbF zT3}@rgNao7qbIa@vn$FbI%-yK&2njP=VqA^R~G(Xcg^~{7tu~P{kj_mX)30jn7KN? znsSxr2nUt^48w9clTh{E&!hS_7Sz(Jjq{Bt`~R{RYJD52mxfJ&;cwow=^?)HIcP0r zUJWimQxP=z@S3;JZWd-vM+IRVI<5bOK5x)Vn33X=VUj7J2qX2+j!yNt=dI>u#7t@Y>Z|gmD1%1J-@)hh3=!D*sAn>gBFcK zUy%ljCAJ8TIgj93916DR0J6zZk|d6OI%T5X5ppcubalYJ~M|6tDg=1`&XC3)9+yMJ5w5oQOFY)aHE29)CZb9#P3~3z61| zO)aAq{6fWMIwJ32BwP0%{PdZ}&6DRXMNgtQ_|bx6vTpFlWxs zwY2*G@Oj}>|FHl=@%h-Qe&^xv|Ns1s>g23Uk)>zCE{N%7_Q%=q!}|V+nOV7V#42F%vH@dUro~-)@$nzzMgFs)*AS@ zw!@XW8Cx4fb!cU4I%7urq1lO^%O=(;t8JBCOYQvTb?V=G_V(v9&72y)K1|3uMtSJa zp~@jm{X0Ldz23VXKek|auZL?a&NnQ2{_tJ=+p|qw?f*m0jc1!Iw04hmx3o>T&3}Zs%=7Lzl3!+d z)VTX?>e>HNW>VyJi%sVNo=u9YL{V-sw4K|j|V}2A*$SV#_8tH;HIT{sblGBc%Wma zPVP&`s~Knw>E0@;zE?tSVFxvXSu#bD-mt%b8N%s`@Tl;Sn!(zD-fEhom}sh|=Fy!d zXo9mJKAhpvnET%T9-qULY4K40W4X?~=#|(+dU)aVy4G%gkVs#`iZop4V$!L7&Tgrb zqaM*~`nl)NyxaFY6sNYJb4sfyb$Rw=FVIesQ`;)A^R$yTn~8RgUAbaFA4UcLyA5cW zZBuW0uI?5@9~~6xO#5w{@TOYihQ%7DOpG41xXowgrTMJv3_M@d%(!gQf5`I1 z*hc#o$wy4xuEUxxA<3`Rx3A&Bg9rJD5HjqC@7}!|TuA|p=PBWPz^LqVI*udd$kVi|!q2a-}VQwj{)_IlJHXY8f7*;MaTgG8q?iuKq z29gOB=fWyaMzT@FW_Hy*O;FX{D(;+SpxgVmGLO5|gPdqwK?aV{nEZf*N`Q`W_^l~3 zW*jJurmTHC6vQ#Wbuh6 zYRp5Vk|wUB{Uj6NdO39$E!5S6@Zv!*SoOdC_7)h+`0T>jJ<-wMOW(9*iwW`+nzvc| zXiXUfJFwGNbj3p6+e5v+ zt8QMp{Df2Ps2DWqFZtt0q)69p9{zP2+pCk;AnI_OXB)YpZjJ=>T@p3#vHsyM^Jp6~ zUO9f>;MABFQ zzRVoO?Cb~d7k0Fb@1rQ89>(fT#`?ndE``sq7y}e0OqMr~Svr4}`_P)olbm>SP}cl= z5tm}BpaD#XA+}kQJ$sY(tBno!EAq;fE1K~GiRXtxnLe&^*=J@E+bNMgZZjYQ!k(3p z=__FC>YZLbL}QPx(;-C!4B*QMzfh~d9WrpdbT{&s@vn96QfM}fh+&*E7J)~->CO*n z65yk~bI_Jq(sBd$T^L7fY=)^Guav%77xFP&bCe;(_5d*b!V+%s18QQP{Fsz#r>{0p z!zAOYob~2akLkgX!VVXq5jKb}imIKltoB!5i9kxaZK@BLw@YjigUaS|mrX4;v~F$A z09i)*%3lE?jv>1m#@w+(KH8>v$TO{H?`^ZGswVZswQ<0Q`w_D@@)-eKnSAmJDKP8L zo;B$hdO&qs4K|h6elr>7ZfhU<(@cYuZA>B*xh+e3g*18~!-hNT>;NPERtcX)!mK zQ{CMtU@P4}XBT!~r8Ek&7(|uSqC^cg$G{cC97rTN942{TW(^Zpy7lPMW-d!oXue{BqCKLlh@nzu#EKA3qp#?a4xB_uFiGq zRECFkF%7d2l9MGXW{sD5>{!GNah(5-jIeyv{WS#y)O z^7vuWNt^Lvm_^EYW4~VhGw%ylawW-zPrw|e0{DY5tii+`QcWr|Pbqk*a9#KML{#D( zR4vl#Qj8NU>3m@>;GrgC+k7qJYt74!vrMkp9R_fKDC=i^<@*-BEX+7WMg-=X3eKn8 zcv#fN=||GM*{Ak>@*0TGU0+_CNL3ojhCQHEq^E)g=Gja9Dx{-fUNuLtr`hpggd>LB zqau`?FYtz`#4)`U>BS9(C!Y074K}q$$Rd85AitnInxi__aMkEc@v_QZdK$fTb;}~s zrL6v5E_zzqi4eFxgkZrMC#j%{MNyas=fPbSg3gG_ynyQJ3P(j^_#CX#A8df2 z_gEe67lzFa?x|h3{&Q%K+eEvO7*$0whHx?F9Tp)7Pq;Hy`a2kMk(9FjYXk+oJ_8^b z#t@`iFxq*UalcjYLZ7Q z`w`7$LQt=0)7vc{0$Cku<=5fE`!9TKAg4;{51z0ml_$VKpoO?)vlBB{X!JDv3$6G8 zjLa0WEC-2UA7`$3lQR;o(4?Vyze)Im&QgAvObB%Z7x0282|{E&>D)s!vwUw~_PRq} zgNvixR|X%9Q!54)*^aLCxt(h=@vJhc${{#VdN_oIuQz|vxl5O)E^7lij@)43l#P2^ zA^-z3Vmjn0pQ*ZGU0>~-fDu`7-REgzh`_2vMn>B;1L;^6mN06>h;BHr${f(%i;Pq| zU$&oT8%qIknz8+f$7wo=q1SWrd~q5U?=sdlZf8QWAU5(^>Z`&rOB#8$X|vfi>Y$hr zzH!R3Wl`wH6{+D?R{WMylhrp6ssjhHgj83zjvfEJd3dDV#?70JamY|&hloq<5yaMS z*s!>Ilx;>e`FsPV^TH}5-&4rKR1I(_OXHo4qZiv2`j;~wVmtlztFmjV{I+Q51rdKw zryJ!hl(XH=>m0!Hju(aZ?4(nE@N77Rbfou@i(eR`YQGgzu$*ra8kq*4(*S?J&8+%* znET4IfZki1r?eV4Km`}Klv#Pvx5hnn7GS5d0%k0Kc16z62|bU1@}@gwlfDSX?*wpp z)nQuL`os84EuwAAReVmT7h#G7NeR<(@Sq?DKl!L$DB;qJ7jKKt79;@phC8qTL0 zP;Q=}YDx6${^_?>h-3rFipc-+(n9y3S1|r@$g)Y06@aq9^5Lz@CT1M|jxHhr^q4SU zzbCU*C574HkYJLDm^|rLjeg(UF6nG=7>R)AR1ZzYmR9z6^H)aMYybjl_2@AUlseeG z{1o1QNqLd2z+T?l<|w=<`qU}q6c-3NCnyPg6qo0i)?VgqR3D}*(yww3kgP0$T@*|N za!AL1A13{*0JWKL9^~lopA*1rDaZnfaR_sv7pvpymT>`cCRv?tSX{U1;Wz?>msfms zjz!p94-#G-b7)#Qdp8a?Pq?A5bE`In&(AtHmE_YD0Cy3^$6d;%4_B7}0-=OR@ zekQGlDikX$SJtkpsZwxfdQ_G(sYn#rqZs;7?u^H zb6VuJ$dX!7PJM>2Ym>~;>$m9XIK|Gv@3LJB^~3CC@-{1;|q9724orQ+CzXfzIfWW6AzmH77cldT?(Ay88@Xj8j(ZKLq% zdWvKjVtOf8zov4LVy8y-w1dB{_HfU{f+~_Bcfu>$I&J=gy^Rln0H!E=Mdq^!TQJt$ ziE)_vcrLG#YqO0N&NZwt@#N~Hl8N*^xO5pNDEVlV%TW?(J6}G)Rg;Iy-R49HV3Ay- zUT@o>g-@;uNi8OWPq#WfG_#S~wWc|1s3dgdN{$oWnZM9cXPa*zYoo4XvuS{1 zRtQC(CR=jnq9%@B_T~_C%Y&db$10VvMZJHC?Um{mjxgZnLRr<3W z+TFSt2p0NWl3{+3SHYRvy0}evfA`*!-lCf*l~rDY_=_2)%lo0@V8K_ zz@$Cpzkjp!)vc(tZKPXI5iKZ!c&pugX05?7KhfRd95H0T0FyxrAK#)Rx|kGz9SO2% zv`LY(5*iiOAMDg@?v)<|1rv$1&@vo>;z7CL_vNPR5jmVtjQGWS`U~c(+<_6Awb_HA>zIL-w^MQdY%3l9gMI=3#?kn`kILO53vC1IR@Gt`CJkoPA z6rM2tm~1DXBXC>NoRWEY^ytxwxMu$TQI6l|)?9X$uvLy2@<12*(G}CCr|YuoV&@=> zWz(jSPmHU4I~KT@ax8a^}5Jf>uM2H5vqX}yq$YV%o z$mh1xqu|AbvYsiACGs$a|D{`+c50ZVxk&wKe;y<)4&JxLUhpzI*=)(oiOZ(_h$)_i zFttrFqbx>V-rbzTYY5O2%q3PG(x~ld}H(p|}MV^A`}H^_!^w-rpp|>>#~Fy8ZOiBshswv<{02Ly?VJ zlGZ+r;9^$YufINerC8Jf9IXkGZ{T$IXY8o7{V(Sww=LO^=@>6TV|SB~0D?w4PV`k} z=Z*9RYX<7;&bcv?6e*~ApkC$czohq2TursBJF?IFdMfEPc9SOQ@4KeNzAP1M~gL%n;mbnv;Vlq zSAE73JQ*7qrOa-YFX9KjoK#g1&|>xK)wdUWUzWd&^--Onv`}q@P=2fh1JrTwcx*Y~^q=b*$h`56O1d-c)Uw}`I_zpT|1hXpv z>{I%=ETN=A{G;`}EzX(d9=LMNmRv2~B`TAZIv7Ka_*W zSj5KG|L(i(3Ax_BTCAQ6olciv?WY%4ip?Qwxcw?7E~g~hJ|oCX7+?g`$AELw(Nf+< zw$(W?a{1F~4eml*c)8GS$ry%RWEcG8Dd8(?J9}=AKzF1VbIW20OpH)Pl2^oJ#BR$& zLeCl6E@}qJnA#kLM6~7Lmy^lv@02s*(#4Aw?}XYm;V22=Dy%8DCflw;&WzggnmXGxxs*uL(T(WfLmi`N({2^iHsTU(yU(dc^$Xzya$YdLS|LmR^RnbF4Y z-#(Fa^>*;PVrQ2r0wjQn-G&Sqa$%6%pQfG1JO_(T0*{@0FckE@q~Pvk(jsZZyvI0` zbNNf~ZmTJQ2%wjd#ib1`-rbAs_~Oq$|13fnECr%=7&Xd&8^}&8Q%=kp%@-LFC2ZZY+tMM-`sRNHYagpnym(8i>cLLZ;O(127qo5hE9}QqxEzUycl5T>Px2cYFVU;k z;1&LRIh(tE@4+gEz{s|2+0tf9t{JXGk2`r9QnF4$V`2fmm}h;!rush4ccYl4W$V_q z0y9c=It&<)Mp&Yc4)6qje(&0~AsQ2}X8fRh36Wd^;GGI+n2(XES2A+s$a_cyZ8*oL z(csW9xWY=??DzqnZj=Aq?~}rR~1jn#n8roTjlA5jQu(#_h`0O@l)J8KS#?_#*oC*xxz2naS*b;>&i;$R<5> z-Ki~mv|v@A3qxnloLK;_vRN9~!G_VfS?;jnaPU1eH7~*+gb?J=ULfAr_a1tgP%43> zJE*3W5P)gi>lpIGv3|WzoH(%*;@BQTCKPK4ckKZ{u310p^YLzztuC%#zy1Ua@jU`4 zM^Iw~VWau7Wy@AQjrDi?y2F$A5piYURGx~;iU&vOLSZ)N4Z93EeCSXQAbr}giS5$Ue;l;%B2~ik3EWE%;qPiaeo$OrHX-RW z)Yd&Pv?Isjjl;&FXRC<|%-1*V+ykC5qD##4=g&1QOQ9IkNMN`UWSo0IC+Xq4OOuaF zeu+C5%BauN@N|VGxPF(e=NLxsbnlN+{O5c?ts?p|_r3?I+Y%zC>8A@JzIFg!t;18+ zq+qH0h@rsd>RfP*U-o8ZMP6|DJRC?p{h8Fg6#OwzooKl^HP z){el!UM>JA(hQl^JPN^WE~hu~cx&wbTS@$yN@|xL7x1 z#A>u`+jgGcit<1)e|#DN^IaK?YMHLPqB4reCKK|--lhg3Q^O0Ocb`<|0J-qKCtP+6 zy)=HFpRUuN*RBx~&H**2@gTu*rP$VBF$ncr_1gQ;>5OT!ohzeKopUaRM(fD4rys$V z+#rEwSF2X7tkK>7Ds`J`b#dTtf7X%o=b3u5xSw+68Z>Tv#_-6PsqkYxt>T7UNbl?4 z$RnkdlP_jzqqxamfBiLt)Qb433d~5XmFcx!p{FGH)3%5@j+~&v#j%-|COZ~2uabeY zk-x64e-Gopd9#yR4bJf1(kKAued2hr&Jcqog^u4PK20kBv*4ia=~(EibwI~8 zg(FpH4YI8vD&*Fe>1j6;>lqmk&`n2&JagdeMCBz(1Op`B>BpT02cE_*B~AA@Qg2h) z19Xi8%|a1m9qfNjgc2Tk|f= zR*CiaR%xkQ5m&lsnf>6 zua(ob_f8d*>37P{j*myN+2ewJu6_)#!RO(Lg4aJ1#=NN!x^f4N80vUUv4oQ*@HHcp z!o1vm;Qv?dLu>$kEQI9%BRtaFS4vPOS$VeJd{t@+ zG{H=`;?l6K68ja#MXfI)!ocGUfHT}4J>Ft?#)g)dnh0s8b0IFuCh6j~d=0MzL4JVc z6Ku}|Q9mr-r~Rbkido|`{xGw6H@(uQlnt?#LYt8zMrgiwBIjdjw`_x}7wq^ff`yaP zt>c$mBUqOZ4mp>*psL5&-DPLjW$%47V32yVX0rspusO@m%+H9rEtiltAA5XvQ^V02 z6$ETeg2B@m1&}tJfply|)*R%<+1H3+|ubR3P)nkC=SCwg$sxQx~FI16N0qXG)|f ztqac1+j(XV^x!F6`vab69x`;r;B0~qJ8X9Z$*cyA8lA?=KY(mKSCJlSJ_j~|N{RwJ z*|?uBsqhzA!z}V z|9TL0hUA3(y@(@h$+iaV*4ghCXd;hG%>I&h8hdM;b7W#MfcGv4jpOM5xTDjoIhB26 zNJC3?^}MQct2~4t^4my+aG&@s1jR%el)jD*2E<3A4cyVs=7_b~G{VC%hK;=4;=+0> z-%fp0a<^C)RYAn-*9*+)NmrcRFDx;=_)_G;6CO1yR4^YaFk-~T*_-!>vQWrSDQ=$8 zVzj>ozB@5=(Dh#KE4DLWo!Uf`pBZB8dm`^s3ATS-U{@yO#x}YwW#iba!Ds_d){NH& z7$^Ms*I&(s91TI*xD=Z1KGa+6v#9&lmvES|UWa5wX)S!Ye#;(nS;*my^{km1)A!_9 zuc1Puz&lNY?U;}8Tl(}`>ped^hoPOhEWs2E@%T`WHEr(G%UD9Cr@9kXtr??p5}P@d z!-NLqniLYw$mXC(FoHWcv$7ZNJuTeIM91FUUUXT@|zrYE$w1P6BbDM{g9kWpBLb{4U6-V4uY=-EMQGR z@Fwt#ajo9`o)Xl%`sK4rFo1w?(tf)Epo2R*sOnrfaU>o!{l9wBR zLb5Qg3%*sZYZvO&HVYQuZix9HBbTd}5UJ5X(w71^MKK15-}aaw<0OID4JhCcc=CGF z&!C2Lg~OPSe;+XG@^=r2p2lPjXu7^eqZf%#(gWKI5{sh}L;U+a+SOUF6-yXXpe`#R z&G??`Hqw51B!Y@MDZe4VZ-#_q9ba&#Fj|=IYkzjX zg~hpLw+WnPi>ZL&y@{O~erJEO;suQ`oaalxMR94|g#0Ik@A0j=lPs6>L*cUe;Os;wA>aB}*<=E+u-qo|;AD8v~ zwQ2phr9}_3TdK#&JxB_0CiJs-ga|BfnQd z9`WpiXbrNzLbiQ~t#|(Fy!NqFEtK9QHxVoO6|NT+yQ}W2G28OrbWK-w9CUWw&|l^( zwLP)}V0Its(kH||r+mcd(dJ}XF#%$%REe41)vf~>xhoDG%W7(OmPx3GMSUC2Ai*-O=_Etad`#A5G z`5QqC_haYM$+~aY|e%KZAKP2}_~U$@w(=`@^y9 zFsJI~hF!aMX@;wB65FtJ+PqQhDYd{ly{N6a!Kye`J{esl2(0F{l$4%}a2*`VeIsVZ@mOKC3y zCb%P4v4qPo4bO;mVLdHYDL*Wi>$zvTy<8v2EGxkAl{6vl>T<81zxR(5dfZh);n-DlpHiqXl_*`JudWxs#;`2^z6^57*Q3BXggh_|6n8i2sN}oe^Br`%oQz2-EX8k`B z9d&Ep-uBHF5~KK&Q*Q4U(h7Dm*j(w4kSB_X73&gTqS2YX^bypM3syo09M?HL@59bG4CQ1WfJA*-wMC{975<-+>S-C$9rBkVvZNx$Fe>Cw07z5uXH0l#C%HW7_#e#^a8;8z37A_qr?# z$&B2(u_!6`&0X;$2vO1w{PLB~*fN8&Wqz$i?pk`2=p%i|z_VA(P*+Zq_q`i-M(%;R zUFw&s@6ZF#Zdne28LOR31~>^>o88C_%>_+ecO940qEXJWAsQ7R+z`@u1fofGnyC{v zlV-Q$ehNhbMBC%*D@g=Gx_J^SJNx($nwb$O3eE|136HYK7$IbjBrzcJVk>gO9g^Xj z(O-E3pkD3PHs89j&G5yGPYEryXOFh~go_CDY+3Uzhq2YiC9%Mj z3tQf6p|{gytBkvb(Q)mY4czh=gVu4DBCvdw#YYEuesf|Uop+_VE-Uj-fIf&ytVFvF z;bX*OQ*vsoeEen^Vx9$;(lvfwIu2iuEP_Z@TcNz+H#A(OsUx{vbX7Y%0Ah)+-GMU* zfJ)9s22zg0T9R|Io`@TO;FC!T8rIl2&VW8A-IM}bBL!PjC2#|bm8S?CLB8X~vWh}N zoSW5|g-^!l(Gyb=WKuv<=<3@4an9xMfBZ2N)SE{V%Mh9-eEAS=|A2XS4~VLXdz6ec z!3bS@VH{RN#Dvic9<738I6}9F>F?Y)D2L*r`r!LMFHEjM;@=mBCJWZOHeC{&!d$K) zql16w%&i$G2M1OmfHL?(%`i>Xlc9-cc%zfd0H>V#-KEz}CT3KeK#o)e! ztKMM*zs4M#+pw7j`N(`?VpE)fQTz&bNdp5BBB2?Mpv?$E_N<83I2+|;5&XW3KHFHy zN{VHw-@MxaDd3?m)*U{%7&gHkoG7XnsR`EHx6vYFSwQc;jbN~>b1o}Wcqi)~TE>?c>D5B>B3dZW9nDsbdBfk0TwL&h{4}#*?avay*6)I%tvmix|{_Rx8$) zJEhq2imnJ8>6SbR9MM;YfZVe4nC+_KA}D`rUcq`^N~_k}hUPWsOO@w;9@);eNeye+ znl0?wWr&8CW+2ZhxlvMMq5#8)U%A%&haUovh%1tT@Bt!<;@WNZucJNprw+zV5`4=#1|#7TT@+sDC_hc9{x8UN(_mnVx%~DhK`CVZC8A4eB1NYgH$5Zw zdy9N(7z~3Qu2IxUGQJ{2v1FmstOIdZcEKTUl`7NLcj80;WZBi*aa1Gu_>bXQ^Lw+KdYid=zOv-6H zR*eZQUp0;$igMCC_Qj*At>-Q*5Lt)F<=A7w z>0c^*Hx~8m|KTi1#qWn>tad)h^E+urEdB;ObP6l~0I_2zv<6X#knv;|3ia?OH~6)< zi7dBw~#94sJ-`z1JuF&;4U;mP-0+F2qzwQ!h#5!Qohw*habYc+ zH2F=EZIFC-`K{QBos6dvZiA}+t{0a6y(0R>L=wc-A1kCkKAdNpgiX(a+UH8jRSXi= z<^_c(h5#UhsIe6y4yWUnF^#jGUS^fZrZ&e*0 zYDtb(N>E9T}`3u&Pu;u2-&v1r*^idWf9ou0r~pwt*( zRT@U0D$6c<<+8OERXZR5hD+5A1B64IyXvLWwlJwvMk&`1d$3Q> z)dVa(1!G1ZgN}-6=a!uSu!nFdA z&WPHUJKplqM{D(e)?bWk9PUCO_&U96k)hvBT>D|qazj; zQ7rAP&ZM9X7KA3b&%|O}Lm)67x##=-;eYFopi*77Z}#DNKdqheuM<&Ex_K_PhB_6Q z`UF4yP&(z=vQRECI^qi~-}I*pY_phq&at2dkx-IgE~Zd75I$PJZPn(k4u)08??pB# zpoC@5gDzBRV%W#YK8SbY2FMw{x@TbcwQ-mp453gk&Mh`*|E-gQfff3T|J>rlOB-Ab zoU!JIi64!KtW%_tImE1wTzlN8x4-s;O;WZZT1LJHGuB-s;^1AwIJft1gDTr_T?WH- zN-Y(Ej<&Au-Jb(BkU+0n_pPXHrNRp>PJ4yOrv)K#=@fnh?AOMsEcrb1*kVto7+k}d0 zI^Il^2f(w-UP?`l$kyKW|I{9XO-uHZi1j~C*kd_iV=5CjV?Mk0>z5KkvB+{NG#0E{ z--O86a&qShAAE!RUI4Nz_903%uPW*UBo?_3kHyyR5tOiPU|ZRaN9zxiWm;vUgpnddmSn0 zbokR($jY`N@$(;^_I%{=9F62t{`!5pRd@deV$_(-v0MR)BDymkTW0x!T@JHjWy)k!qagKeLGJ?+1l)J-q4|M5-3#?*UVs-`v&oq(^V7GhuxFS=_v zwH=LFR7zFC3+km2y55F@F9rt(aN`08OxLu^cwz1P#Gnu`1Tw~;^6`;xa?0iZ3p|w= z5o4g${^8YjY|-b&Ed#gs6R7-Gm@>@n;pDRhiv(;+AOhLlSBLz=wV(SejJzDCOGV6W zdJ`X#vQ<&#vu|!c*;#}`wos7zDBaCb0Q$!x=+HwE_>W_s@^F{$d)M!W%|s)gD#~7g z@GQT$4$aQXXFjH-f>ah7q2)-?=1E8+0@^TR@|;-@n& z$U{0Y0L8~ZckRX2ZxS>7=E>GaU;XXf4L1EJ(CL|89l7BA+MF&UBoM^#YxiS3&piKpBxlrIhk<;>q z?es0=IW;?1d5v@@1hQx<{if?)HM7wx_tsP^R!QCt2WfZe~PhR?=+PpHZa zvN#(7A~}j0$v{cX(QKRg_3I~-@-`+9CGFQxir*%1N3YNedB-#FPW`>VSi1Io`hILl z^l=;7N4~ay_aFiN_K2Dhz@mnXlVUSJ=&Tp#qyuv9{K1*0J@s zU+{3cJ6+bEt5JM%EKQMt>m=&q?ikNdV49V>{u0}c}Nb_dR4g(gX zho#i<5uNLy2^xvEdJ=fL%3a&wmw!8wufNgoY2Zf~D$tQ|CIvCck9sarMMxP@cbI8% z#{s|&d(5-?;Kc!a=wULga_?!KZ9)(zWu_pH%QSNjuN)nv`*fb>!F|5jsiPPkKqU))|FXDP1Bt@VT@x!p7{7COTDwTT)Zw zS6D}jVD3YZN!R*n$=cx_Yu_y!%Qrjp6BGPycP%;-G9OQ;CS+Rqd zqX7K4(w}I0E<6T(J55NP7?Bo5uT=KF(N^_;HQy~IzPX}%Z5##0+$8R@3l+Ac9x0UO zHidz)=fYprGwHsL@?fx7Re$R;wSTig$q{NauV}qiDu_`6UAZP*LnTBz?^#Qtdnq=R z8h%ufQ?Soi2l}!6)#_iLatE37=FQr)(SDKq*g%*}1i7%{1G;RmGEgYis zf+Edt3X^*Wrxi(%FZ~syxhlbV<+4@=w*O|g47@({pnGwXJ84Y!At68^c0&k4obUCC zuc?j^jkY_rGlXc{Mg7Gs9xsi*qrdlx_H2g})UHPzN7quHpf?g5x4n&e1Nhw5|xPYY#A1#L9hXqr8!Ja}AHi8*ya zQ=s-zFbD{P^o5fmEcNpnUEk{8EaNa2h!f?dYvMfNv8@Q+rId#LLA~4=vMIEZ#0fN_Fqe_J8evJ|Nl)2MK{;dCs+YwrCmc|4Updd^i2|53|bnDiy|J^_nR{5`EtpmutPK#L@D%%G=zQ0}*e(uv^ z|4$5i1HzZBedbYG22*QHic`}4+@KrT!+%vD_OHfs3t{A?w)DBf7nmS>qK9O^KmOIO zxBgqxXqB-q``+v7Cr0^$_5bhd>GS|vWAM06GnvnI%C!U E0MO@ikpKVy literal 0 HcmV?d00001