From bf279aea9e48ca81b0d071baacff9406c2abffd4 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 28 Jul 2026 18:38:09 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: yapeichang/Llama-3.1-8B-RM8B Source: Original Platform --- .gitattributes | 36 + README.md | 44 + config.json | 35 + generation_config.json | 13 + latest | 1 + pytorch_model-00001-of-00007.bin | 3 + pytorch_model-00002-of-00007.bin | 3 + pytorch_model-00003-of-00007.bin | 3 + pytorch_model-00004-of-00007.bin | 3 + pytorch_model-00005-of-00007.bin | 3 + pytorch_model-00006-of-00007.bin | 3 + pytorch_model-00007-of-00007.bin | 3 + pytorch_model.bin.index.json | 298 + rng_state.pth | 3 + scheduler.pt | 3 + special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2072 ++ trainer_state.json | 31284 +++++++++++++++++++++++++++++ training_args.bin | 3 + zero_to_fp32.py | 760 + 21 files changed, 34599 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 latest create mode 100644 pytorch_model-00001-of-00007.bin create mode 100644 pytorch_model-00002-of-00007.bin create mode 100644 pytorch_model-00003-of-00007.bin create mode 100644 pytorch_model-00004-of-00007.bin create mode 100644 pytorch_model-00005-of-00007.bin create mode 100644 pytorch_model-00006-of-00007.bin create mode 100644 pytorch_model-00007-of-00007.bin create mode 100644 pytorch_model.bin.index.json create mode 100644 rng_state.pth create mode 100644 scheduler.pt create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 zero_to_fp32.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..a67ac04 --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +--- +license: llama3.1 +datasets: +- yapeichang/BLEUBERI-Tulu3-50k +- allenai/tulu-3-sft-mixture +base_model: +- yapeichang/Llama-3.1-8B +library_name: transformers +--- + +# Llama-3.1-8B-RM8B + +[[Paper](https://arxiv.org/pdf/2505.11080)] [[HF Collection](https://huggingface.co/collections/yapeichang/bleuberi-6840b3b9d02ff86c5878dafa)] [[Code](https://github.com/lilakk/BLEUBERI)] + +Authors: [Yapei Chang](https://lilakk.github.io/), [Yekyung Kim](https://mungg.github.io/), [Michael Krumdick](https://scholar.google.com/citations?user=nqf6-MwAAAAJ&hl=en), [Amir Zadeh](https://scholar.google.com/citations?user=MQFngiMAAAAJ&hl=en), [Chuan Li](https://scholar.google.com/citations?user=hoZesOwAAAAJ&hl=en), [Chris Tanner](https://www.chriswtanner.com/), [Mohit Iyyer](https://www.cs.umd.edu/~miyyer/) + +Contact: `yapeic@umd.edu` + +> **TLDR** > We extend RLVR beyond easily verifiable domains like math and code to the more open-ended setting of general instruction following. Surprisingly, we find that BLEU—a simple n-gram matching metric—when paired with high-quality references from strong LLMs, achieves human agreement comparable to 8B and 27B reward models on Chatbot Arena outputs. Based on this insight, we introduce BLEUBERI, which uses BLEU directly as a reward in GRPO training. BLEUBERI matches the performance of RM-guided GRPO across four instruction-following benchmarks and produces more factually grounded outputs, with human raters rating them on par with those from reward model-trained systems. + +## Model card + +

+ image +

+

+Model performance across four general instruction-following benchmarks. +

+ +This model corresponds to the Llama-3.1-8B, GRPO-RM row in the table. The RM used during training is [Skywork-RM-8B](https://huggingface.co/Skywork/Skywork-Reward-Llama-3.1-8B-v0.2). + +## Citation + +```bibtex +@misc{chang2025bleuberibleusurprisinglyeffective, + title={BLEUBERI: BLEU is a surprisingly effective reward for instruction following}, + author={Yapei Chang and Yekyung Kim and Michael Krumdick and Amir Zadeh and Chuan Li and Chris Tanner and Mohit Iyyer}, + year={2025}, + eprint={2505.11080}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2505.11080}, +} +``` \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..95917c4 --- /dev/null +++ b/config.json @@ -0,0 +1,35 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 8.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.3", + "use_cache": false, + "vocab_size": 128257 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..c6802f3 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128009 + ], + "pad_token_id": 128256, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.3" +} diff --git a/latest b/latest new file mode 100644 index 0000000..37b4190 --- /dev/null +++ b/latest @@ -0,0 +1 @@ +global_step1250 \ No newline at end of file diff --git a/pytorch_model-00001-of-00007.bin b/pytorch_model-00001-of-00007.bin new file mode 100644 index 0000000..0ecde21 --- /dev/null +++ b/pytorch_model-00001-of-00007.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a391398c7b05cbd439edcb6f97c5abe1590b225dc291ebe3938670e1369b0db +size 4886390289 diff --git a/pytorch_model-00002-of-00007.bin b/pytorch_model-00002-of-00007.bin new file mode 100644 index 0000000..ebcd822 --- /dev/null +++ b/pytorch_model-00002-of-00007.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:153fdf40c109055740ed402ea4e4ae64cbdc5b3f1d917cf66f0ccb4caa20ad0f +size 4831851519 diff --git a/pytorch_model-00003-of-00007.bin b/pytorch_model-00003-of-00007.bin new file mode 100644 index 0000000..89f6685 --- /dev/null +++ b/pytorch_model-00003-of-00007.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:510f3cc56e03d884585096ced254cadf6a595ee94fd329253a0e1ddffb4c838f +size 4999625131 diff --git a/pytorch_model-00004-of-00007.bin b/pytorch_model-00004-of-00007.bin new file mode 100644 index 0000000..a327028 --- /dev/null +++ b/pytorch_model-00004-of-00007.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5e45044bca155f178ccaad598204ad24abca3ff63f31ab137b395217de24e1f +size 4999625131 diff --git a/pytorch_model-00005-of-00007.bin b/pytorch_model-00005-of-00007.bin new file mode 100644 index 0000000..6e36d2f --- /dev/null +++ b/pytorch_model-00005-of-00007.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:515db69cedb1efada9cafcee5747eda33c6f915dd80ba6434d7c8f2ad383c5fc +size 4831851583 diff --git a/pytorch_model-00006-of-00007.bin b/pytorch_model-00006-of-00007.bin new file mode 100644 index 0000000..d79198d --- /dev/null +++ b/pytorch_model-00006-of-00007.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c047459d7db1ee9996f062e4943db11bee3ebfea4627b01e25af5ce2cd707b71 +size 4999625131 diff --git a/pytorch_model-00007-of-00007.bin b/pytorch_model-00007-of-00007.bin new file mode 100644 index 0000000..e436943 --- /dev/null +++ b/pytorch_model-00007-of-00007.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ca300faeadafd017a54ff3ee21fed0e6b8a404f790545185cf20e4e5fb80d7c +size 2572213225 diff --git a/pytorch_model.bin.index.json b/pytorch_model.bin.index.json new file mode 100644 index 0000000..25ce9e8 --- /dev/null +++ b/pytorch_model.bin.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121077760 + }, + "weight_map": { + "lm_head.weight": "pytorch_model-00007-of-00007.bin", + "model.embed_tokens.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.0.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.1.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.10.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.10.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.10.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.11.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.11.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.11.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.12.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.12.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.12.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.13.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.13.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.13.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.14.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.14.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.14.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.15.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.15.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.15.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.16.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.16.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.16.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.17.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.17.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.17.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.18.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.18.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.18.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.19.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.19.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.19.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.2.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.20.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.20.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.20.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin", + "model.layers.21.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.21.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.21.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.22.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.22.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.22.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.23.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.23.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.23.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.24.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.24.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.24.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.25.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.25.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.25.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin", + "model.layers.26.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.26.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.26.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.26.self_attn.k_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.27.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.27.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.27.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.27.self_attn.k_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.28.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.28.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.28.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.28.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.28.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.28.self_attn.k_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.28.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.28.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.29.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.29.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.29.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.29.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.29.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.29.self_attn.k_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.29.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.29.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.3.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.3.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.3.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin", + "model.layers.30.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.30.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.30.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.30.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.30.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.30.self_attn.k_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.30.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.30.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.31.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.31.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.31.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.31.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.31.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.31.self_attn.k_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.31.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.31.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin", + "model.layers.4.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.4.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.4.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.5.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.5.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.5.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.6.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.6.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.6.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.7.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.7.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.7.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.8.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.8.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.8.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin", + "model.layers.9.input_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.9.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.9.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin", + "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin", + "model.norm.weight": "pytorch_model-00007-of-00007.bin" + } +} diff --git a/rng_state.pth b/rng_state.pth new file mode 100644 index 0000000..46992b1 --- /dev/null +++ b/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:038abf078535ae453872358c4a1f17bb70914546be3fcc74b4729775f0ebd19c +size 14645 diff --git a/scheduler.pt b/scheduler.pt new file mode 100644 index 0000000..31fd304 --- /dev/null +++ b/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b9597faa4aa2da5d561bb5cd0a6f86d374da0ce999da80b7bddb43528f2dcb9 +size 1465 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..3dbf016 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..8d9ba1c --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3707f086f4c7d6e3bc76c1468bbd73e42d1641b1e40880a3b513ec3db4184a5f +size 17210261 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..fa31df8 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2072 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n {%- for arg_name, arg_val in tool_call.arguments | items %}\n {{- arg_name + '=\"' + arg_val + '\"' }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- endif %}\n {%- endfor %}\n {{- \")\" }}\n {%- else %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\" }}\n {%- endif %}\n {%- if builtin_tools is defined %}\n {#- This means we're in ipython mode #}\n {{- \"<|eom_id|>\" }}\n {%- else %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n {%- if message.content is mapping or message.content is iterable %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n", + "clean_up_tokenization_spaces": true, + "eos_token": "<|eot_id|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 128000, + "pad_token": "[PAD]", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..29c1c07 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,31284 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 1250, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 470.0, + "completions/mean_length": 276.6875, + "completions/mean_terminated_length": 184.60870361328125, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "epoch": 0.0008, + "grad_norm": 3.820082187652588, + "kl": 0.0003216266632080078, + "learning_rate": 0.0, + "loss": -0.1909, + "num_tokens": 15006.0, + "reward": -8.20977783203125, + "reward_std": 4.4750261306762695, + "rewards/rm_reward_func/mean": -8.20977783203125, + "rewards/rm_reward_func/std": 7.660107135772705, + "step": 1 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 424.0, + "completions/mean_length": 313.625, + "completions/mean_terminated_length": 159.3333282470703, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.0016, + "grad_norm": 4.830166339874268, + "kl": 0.0002695322036743164, + "learning_rate": 1.5873015873015872e-08, + "loss": 0.0694, + "num_tokens": 28018.0, + "reward": -11.44384765625, + "reward_std": 2.6692183017730713, + "rewards/rm_reward_func/mean": -11.44384765625, + "rewards/rm_reward_func/std": 6.70808744430542, + "step": 2 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 374.28125, + "completions/mean_terminated_length": 280.0526428222656, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.0024, + "grad_norm": 2.5839879512786865, + "kl": 0.0003209114074707031, + "learning_rate": 3.1746031746031744e-08, + "loss": -0.1093, + "num_tokens": 42787.0, + "reward": -13.593017578125, + "reward_std": 5.063211917877197, + "rewards/rm_reward_func/mean": -13.593017578125, + "rewards/rm_reward_func/std": 8.933624267578125, + "step": 3 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 467.0, + "completions/mean_length": 176.3125, + "completions/mean_terminated_length": 141.58621215820312, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.0032, + "grad_norm": 3.8852806091308594, + "kl": 0.00033855438232421875, + "learning_rate": 4.7619047619047613e-08, + "loss": -0.0693, + "num_tokens": 50509.0, + "reward": -8.043212890625, + "reward_std": 6.7915544509887695, + "rewards/rm_reward_func/mean": -8.043212890625, + "rewards/rm_reward_func/std": 9.116313934326172, + "step": 4 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 450.0, + "completions/mean_length": 240.59375, + "completions/mean_terminated_length": 212.51724243164062, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.004, + "grad_norm": 2.737971544265747, + "kl": 0.0002636909484863281, + "learning_rate": 6.349206349206349e-08, + "loss": -0.0144, + "num_tokens": 61880.0, + "reward": -7.558570861816406, + "reward_std": 4.206568717956543, + "rewards/rm_reward_func/mean": -7.558570861816406, + "rewards/rm_reward_func/std": 6.464176177978516, + "step": 5 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 449.0, + "completions/mean_length": 268.5, + "completions/mean_terminated_length": 187.33334350585938, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.0048, + "grad_norm": 2.704625368118286, + "kl": 0.00022745132446289062, + "learning_rate": 7.936507936507936e-08, + "loss": -0.0148, + "num_tokens": 73048.0, + "reward": -8.5592041015625, + "reward_std": 6.456365585327148, + "rewards/rm_reward_func/mean": -8.5592041015625, + "rewards/rm_reward_func/std": 6.686816692352295, + "step": 6 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 308.0, + "completions/mean_length": 255.34375, + "completions/mean_terminated_length": 169.7916717529297, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.0056, + "grad_norm": 3.0464067459106445, + "kl": 0.00023818016052246094, + "learning_rate": 9.523809523809523e-08, + "loss": 0.2187, + "num_tokens": 84043.0, + "reward": -10.86029052734375, + "reward_std": 5.81391716003418, + "rewards/rm_reward_func/mean": -10.86029052734375, + "rewards/rm_reward_func/std": 7.67104959487915, + "step": 7 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 370.15625, + "completions/mean_terminated_length": 245.0, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "epoch": 0.0064, + "grad_norm": 2.1677191257476807, + "kl": 0.0003154277801513672, + "learning_rate": 1.111111111111111e-07, + "loss": -0.0946, + "num_tokens": 97960.0, + "reward": -9.22998046875, + "reward_std": 4.78300666809082, + "rewards/rm_reward_func/mean": -9.22998046875, + "rewards/rm_reward_func/std": 15.388681411743164, + "step": 8 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 419.0, + "completions/mean_length": 355.5, + "completions/mean_terminated_length": 294.2608642578125, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.0072, + "grad_norm": 2.2008018493652344, + "kl": 0.00035500526428222656, + "learning_rate": 1.2698412698412698e-07, + "loss": 0.0485, + "num_tokens": 113944.0, + "reward": -7.944580078125, + "reward_std": 4.6264190673828125, + "rewards/rm_reward_func/mean": -7.944580078125, + "rewards/rm_reward_func/std": 7.673216342926025, + "step": 9 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 398.65625, + "completions/mean_terminated_length": 285.3125, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.008, + "grad_norm": 2.5219130516052246, + "kl": 0.0003647804260253906, + "learning_rate": 1.4285714285714285e-07, + "loss": -0.0682, + "num_tokens": 130357.0, + "reward": -14.8125, + "reward_std": 3.55131196975708, + "rewards/rm_reward_func/mean": -14.8125, + "rewards/rm_reward_func/std": 6.269670486450195, + "step": 10 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 512.0, + "completions/max_terminated_length": 468.0, + "completions/mean_length": 438.03125, + "completions/mean_terminated_length": 216.125, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "epoch": 0.0088, + "grad_norm": 2.1736643314361572, + "kl": 0.00029921531677246094, + "learning_rate": 1.5873015873015872e-07, + "loss": 0.2198, + "num_tokens": 148190.0, + "reward": -13.016815185546875, + "reward_std": 7.495743274688721, + "rewards/rm_reward_func/mean": -13.016815185546875, + "rewards/rm_reward_func/std": 8.30325698852539, + "step": 11 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 356.0, + "completions/mean_length": 178.5625, + "completions/mean_terminated_length": 144.0689697265625, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.0096, + "grad_norm": 4.596688747406006, + "kl": 0.00018262863159179688, + "learning_rate": 1.7460317460317458e-07, + "loss": 0.0833, + "num_tokens": 156104.0, + "reward": 0.974029541015625, + "reward_std": 4.125612735748291, + "rewards/rm_reward_func/mean": 0.974029541015625, + "rewards/rm_reward_func/std": 9.185401916503906, + "step": 12 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 404.0, + "completions/max_terminated_length": 404.0, + "completions/mean_length": 250.25, + "completions/mean_terminated_length": 250.25, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.0104, + "grad_norm": 3.2415318489074707, + "kl": 0.0003223419189453125, + "learning_rate": 1.9047619047619045e-07, + "loss": -0.1464, + "num_tokens": 166208.0, + "reward": -8.69970703125, + "reward_std": 4.659298419952393, + "rewards/rm_reward_func/mean": -8.69970703125, + "rewards/rm_reward_func/std": 5.567142963409424, + "step": 13 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 328.03125, + "completions/mean_terminated_length": 244.4091033935547, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.0112, + "grad_norm": 2.257594347000122, + "kl": 0.0003323554992675781, + "learning_rate": 2.0634920634920632e-07, + "loss": -0.0784, + "num_tokens": 178833.0, + "reward": -8.506607055664062, + "reward_std": 7.695546627044678, + "rewards/rm_reward_func/mean": -8.506607055664062, + "rewards/rm_reward_func/std": 8.626959800720215, + "step": 14 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 274.65625, + "completions/mean_terminated_length": 150.33334350585938, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "epoch": 0.012, + "grad_norm": 2.8894879817962646, + "kl": 0.00014698505401611328, + "learning_rate": 2.222222222222222e-07, + "loss": 0.1296, + "num_tokens": 192198.0, + "reward": -8.6026611328125, + "reward_std": 7.466651916503906, + "rewards/rm_reward_func/mean": -8.6026611328125, + "rewards/rm_reward_func/std": 8.928447723388672, + "step": 15 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 431.0, + "completions/max_terminated_length": 431.0, + "completions/mean_length": 163.53125, + "completions/mean_terminated_length": 163.53125, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.0128, + "grad_norm": 6.473876476287842, + "kl": 0.00042629241943359375, + "learning_rate": 2.3809523809523806e-07, + "loss": -0.1249, + "num_tokens": 203023.0, + "reward": -5.33013916015625, + "reward_std": 5.538287162780762, + "rewards/rm_reward_func/mean": -5.33013916015625, + "rewards/rm_reward_func/std": 8.5722074508667, + "step": 16 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 325.0625, + "completions/mean_terminated_length": 272.7200012207031, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.0136, + "grad_norm": 2.029867172241211, + "kl": 0.0003304481506347656, + "learning_rate": 2.5396825396825396e-07, + "loss": 0.0048, + "num_tokens": 215681.0, + "reward": -8.299209594726562, + "reward_std": 5.960148811340332, + "rewards/rm_reward_func/mean": -8.299209594726562, + "rewards/rm_reward_func/std": 9.681968688964844, + "step": 17 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 467.0, + "completions/mean_length": 309.46875, + "completions/mean_terminated_length": 217.4091033935547, + "completions/min_length": 15.0, + "completions/min_terminated_length": 15.0, + "epoch": 0.0144, + "grad_norm": 3.225783348083496, + "kl": 0.0002518892288208008, + "learning_rate": 2.698412698412698e-07, + "loss": 0.454, + "num_tokens": 228784.0, + "reward": -12.326904296875, + "reward_std": 6.084526062011719, + "rewards/rm_reward_func/mean": -12.326904296875, + "rewards/rm_reward_func/std": 10.45023250579834, + "step": 18 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 246.84375, + "completions/mean_terminated_length": 238.29031372070312, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.0152, + "grad_norm": 3.1062114238739014, + "kl": 0.00031566619873046875, + "learning_rate": 2.857142857142857e-07, + "loss": -0.0427, + "num_tokens": 238891.0, + "reward": -5.867828369140625, + "reward_std": 5.073713302612305, + "rewards/rm_reward_func/mean": -5.867828369140625, + "rewards/rm_reward_func/std": 6.714073181152344, + "step": 19 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 279.0625, + "completions/mean_terminated_length": 271.5483703613281, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "epoch": 0.016, + "grad_norm": 2.1619873046875, + "kl": 0.00023698806762695312, + "learning_rate": 3.0158730158730156e-07, + "loss": -0.1689, + "num_tokens": 249645.0, + "reward": -3.2251663208007812, + "reward_std": 4.3877034187316895, + "rewards/rm_reward_func/mean": -3.2251663208007812, + "rewards/rm_reward_func/std": 5.818446159362793, + "step": 20 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 282.40625, + "completions/mean_terminated_length": 258.6551818847656, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.0168, + "grad_norm": 4.3518900871276855, + "kl": 0.0003197193145751953, + "learning_rate": 3.1746031746031743e-07, + "loss": 0.1906, + "num_tokens": 261154.0, + "reward": -8.756103515625, + "reward_std": 3.641651153564453, + "rewards/rm_reward_func/mean": -8.756103515625, + "rewards/rm_reward_func/std": 4.865791320800781, + "step": 21 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 256.125, + "completions/mean_terminated_length": 184.47999572753906, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.0176, + "grad_norm": 3.306163787841797, + "kl": 0.00031304359436035156, + "learning_rate": 3.333333333333333e-07, + "loss": 0.1021, + "num_tokens": 272150.0, + "reward": -13.5908203125, + "reward_std": 7.745641708374023, + "rewards/rm_reward_func/mean": -13.5908203125, + "rewards/rm_reward_func/std": 8.889524459838867, + "step": 22 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 436.0, + "completions/mean_length": 355.59375, + "completions/mean_terminated_length": 261.75, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.0184, + "grad_norm": 2.356691837310791, + "kl": 0.0002713203430175781, + "learning_rate": 3.4920634920634917e-07, + "loss": -0.1426, + "num_tokens": 286265.0, + "reward": -8.7578125, + "reward_std": 4.64816951751709, + "rewards/rm_reward_func/mean": -8.7578125, + "rewards/rm_reward_func/std": 10.985777854919434, + "step": 23 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 238.65625, + "completions/mean_terminated_length": 147.5416717529297, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.0192, + "grad_norm": 5.42078161239624, + "kl": 0.0004291534423828125, + "learning_rate": 3.6507936507936504e-07, + "loss": 0.0164, + "num_tokens": 297494.0, + "reward": -6.644187927246094, + "reward_std": 5.260914325714111, + "rewards/rm_reward_func/mean": -6.644187927246094, + "rewards/rm_reward_func/std": 10.8004150390625, + "step": 24 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 231.4375, + "completions/mean_terminated_length": 202.41378784179688, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.02, + "grad_norm": 5.109860420227051, + "kl": 0.00030303001403808594, + "learning_rate": 3.809523809523809e-07, + "loss": 0.084, + "num_tokens": 309052.0, + "reward": -5.954833984375, + "reward_std": 5.726484298706055, + "rewards/rm_reward_func/mean": -5.954833984375, + "rewards/rm_reward_func/std": 7.126110553741455, + "step": 25 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 193.0, + "completions/mean_length": 240.5625, + "completions/mean_terminated_length": 77.70000457763672, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.0208, + "grad_norm": 3.4662835597991943, + "kl": 0.00029218196868896484, + "learning_rate": 3.968253968253968e-07, + "loss": 0.1907, + "num_tokens": 320334.0, + "reward": -14.75677490234375, + "reward_std": 4.248941421508789, + "rewards/rm_reward_func/mean": -14.75677490234375, + "rewards/rm_reward_func/std": 7.078120708465576, + "step": 26 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 450.09375, + "completions/mean_terminated_length": 379.933349609375, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "epoch": 0.0216, + "grad_norm": 1.6936448812484741, + "kl": 0.0002624988555908203, + "learning_rate": 4.1269841269841265e-07, + "loss": -0.0219, + "num_tokens": 340865.0, + "reward": -8.20947265625, + "reward_std": 6.251974582672119, + "rewards/rm_reward_func/mean": -8.20947265625, + "rewards/rm_reward_func/std": 6.31874942779541, + "step": 27 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 394.0, + "completions/max_terminated_length": 394.0, + "completions/mean_length": 167.78125, + "completions/mean_terminated_length": 167.78125, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.0224, + "grad_norm": 4.310393333435059, + "kl": 0.0003063678741455078, + "learning_rate": 4.285714285714285e-07, + "loss": 0.0078, + "num_tokens": 350994.0, + "reward": -4.7073516845703125, + "reward_std": 3.5501272678375244, + "rewards/rm_reward_func/mean": -4.7073516845703125, + "rewards/rm_reward_func/std": 11.039022445678711, + "step": 28 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 316.75, + "completions/mean_terminated_length": 199.60000610351562, + "completions/min_length": 26.0, + "completions/min_terminated_length": 26.0, + "epoch": 0.0232, + "grad_norm": 2.857746124267578, + "kl": 0.0002956390380859375, + "learning_rate": 4.444444444444444e-07, + "loss": 0.0622, + "num_tokens": 368186.0, + "reward": -8.3876953125, + "reward_std": 4.28533935546875, + "rewards/rm_reward_func/mean": -8.3876953125, + "rewards/rm_reward_func/std": 6.924084663391113, + "step": 29 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 453.0, + "completions/mean_length": 275.71875, + "completions/mean_terminated_length": 221.19232177734375, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.024, + "grad_norm": 2.670747995376587, + "kl": 0.0003662109375, + "learning_rate": 4.6031746031746025e-07, + "loss": 0.0619, + "num_tokens": 379457.0, + "reward": -8.928466796875, + "reward_std": 8.38475227355957, + "rewards/rm_reward_func/mean": -8.928466796875, + "rewards/rm_reward_func/std": 13.34780502319336, + "step": 30 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 470.0, + "completions/mean_length": 298.5, + "completions/mean_terminated_length": 152.42105102539062, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.0248, + "grad_norm": 2.7848010063171387, + "kl": 0.00027370452880859375, + "learning_rate": 4.761904761904761e-07, + "loss": 0.1278, + "num_tokens": 392985.0, + "reward": -14.843505859375, + "reward_std": 5.917230129241943, + "rewards/rm_reward_func/mean": -14.843505859375, + "rewards/rm_reward_func/std": 8.529240608215332, + "step": 31 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 258.9375, + "completions/mean_terminated_length": 174.58334350585938, + "completions/min_length": 17.0, + "completions/min_terminated_length": 17.0, + "epoch": 0.0256, + "grad_norm": 4.443420886993408, + "kl": 0.0003578662872314453, + "learning_rate": 4.92063492063492e-07, + "loss": 0.128, + "num_tokens": 405471.0, + "reward": -4.92132568359375, + "reward_std": 4.653083801269531, + "rewards/rm_reward_func/mean": -4.92132568359375, + "rewards/rm_reward_func/std": 13.973512649536133, + "step": 32 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 181.9375, + "completions/mean_terminated_length": 159.93333435058594, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "epoch": 0.0264, + "grad_norm": 4.213375091552734, + "kl": 0.0003600120544433594, + "learning_rate": 5.079365079365079e-07, + "loss": -0.1714, + "num_tokens": 415629.0, + "reward": -11.66253662109375, + "reward_std": 6.659981727600098, + "rewards/rm_reward_func/mean": -11.66253662109375, + "rewards/rm_reward_func/std": 6.839272975921631, + "step": 33 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 435.0, + "completions/mean_length": 236.0625, + "completions/mean_terminated_length": 158.8000030517578, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.0272, + "grad_norm": 3.4807815551757812, + "kl": 0.00034689903259277344, + "learning_rate": 5.238095238095238e-07, + "loss": -0.0034, + "num_tokens": 429039.0, + "reward": -8.47998046875, + "reward_std": 5.434866905212402, + "rewards/rm_reward_func/mean": -8.47998046875, + "rewards/rm_reward_func/std": 7.376414775848389, + "step": 34 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 336.46875, + "completions/mean_terminated_length": 244.52381896972656, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.028, + "grad_norm": 2.5712099075317383, + "kl": 0.0003306865692138672, + "learning_rate": 5.396825396825396e-07, + "loss": 0.1245, + "num_tokens": 443430.0, + "reward": -7.155723571777344, + "reward_std": 6.052016735076904, + "rewards/rm_reward_func/mean": -7.155723571777344, + "rewards/rm_reward_func/std": 8.14072322845459, + "step": 35 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 230.0, + "completions/mean_length": 251.21875, + "completions/mean_terminated_length": 72.78947448730469, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.0288, + "grad_norm": 3.459679126739502, + "kl": 0.0003533363342285156, + "learning_rate": 5.555555555555555e-07, + "loss": 0.027, + "num_tokens": 454285.0, + "reward": -9.2822265625, + "reward_std": 5.530088424682617, + "rewards/rm_reward_func/mean": -9.2822265625, + "rewards/rm_reward_func/std": 6.309089183807373, + "step": 36 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 426.0, + "completions/mean_length": 222.09375, + "completions/mean_terminated_length": 212.74192810058594, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.0296, + "grad_norm": 2.861426830291748, + "kl": 0.0002856254577636719, + "learning_rate": 5.714285714285714e-07, + "loss": -0.0668, + "num_tokens": 463344.0, + "reward": -7.4073486328125, + "reward_std": 6.660886287689209, + "rewards/rm_reward_func/mean": -7.4073486328125, + "rewards/rm_reward_func/std": 6.9068474769592285, + "step": 37 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 188.90625, + "completions/mean_terminated_length": 98.43999481201172, + "completions/min_length": 12.0, + "completions/min_terminated_length": 12.0, + "epoch": 0.0304, + "grad_norm": 5.866414546966553, + "kl": 0.0003485679626464844, + "learning_rate": 5.873015873015873e-07, + "loss": -0.2332, + "num_tokens": 475037.0, + "reward": -9.38623046875, + "reward_std": 3.1565802097320557, + "rewards/rm_reward_func/mean": -9.38623046875, + "rewards/rm_reward_func/std": 10.67844295501709, + "step": 38 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 404.0, + "completions/mean_length": 303.28125, + "completions/mean_terminated_length": 221.60870361328125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.0312, + "grad_norm": 2.542703151702881, + "kl": 0.00039768218994140625, + "learning_rate": 6.031746031746031e-07, + "loss": 0.076, + "num_tokens": 487278.0, + "reward": -8.93896484375, + "reward_std": 3.1694929599761963, + "rewards/rm_reward_func/mean": -8.93896484375, + "rewards/rm_reward_func/std": 8.222332000732422, + "step": 39 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 229.0625, + "completions/mean_terminated_length": 163.7692413330078, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.032, + "grad_norm": 6.272686004638672, + "kl": 0.0004992485046386719, + "learning_rate": 6.19047619047619e-07, + "loss": -0.0515, + "num_tokens": 498952.0, + "reward": -3.323486328125, + "reward_std": 5.879536151885986, + "rewards/rm_reward_func/mean": -3.323486328125, + "rewards/rm_reward_func/std": 8.639906883239746, + "step": 40 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 375.125, + "completions/mean_terminated_length": 321.5652160644531, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.0328, + "grad_norm": 2.0339434146881104, + "kl": 0.00039386749267578125, + "learning_rate": 6.349206349206349e-07, + "loss": -0.1041, + "num_tokens": 513076.0, + "reward": -4.98583984375, + "reward_std": 4.426126956939697, + "rewards/rm_reward_func/mean": -4.98583984375, + "rewards/rm_reward_func/std": 5.537177085876465, + "step": 41 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 397.0, + "completions/mean_length": 161.84375, + "completions/mean_terminated_length": 150.5483856201172, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.0336, + "grad_norm": 3.7768754959106445, + "kl": 0.0003666877746582031, + "learning_rate": 6.507936507936507e-07, + "loss": -0.0783, + "num_tokens": 523351.0, + "reward": -8.8109130859375, + "reward_std": 5.987880706787109, + "rewards/rm_reward_func/mean": -8.8109130859375, + "rewards/rm_reward_func/std": 6.827143669128418, + "step": 42 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 206.0, + "completions/mean_length": 207.6875, + "completions/mean_terminated_length": 106.25, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.0344, + "grad_norm": 4.852839469909668, + "kl": 0.0005655288696289062, + "learning_rate": 6.666666666666666e-07, + "loss": 0.0577, + "num_tokens": 532325.0, + "reward": -11.9395751953125, + "reward_std": 3.7294559478759766, + "rewards/rm_reward_func/mean": -11.9395751953125, + "rewards/rm_reward_func/std": 6.000690460205078, + "step": 43 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 345.21875, + "completions/mean_terminated_length": 339.8387145996094, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.0352, + "grad_norm": 2.18628191947937, + "kl": 0.0003714561462402344, + "learning_rate": 6.825396825396826e-07, + "loss": -0.0234, + "num_tokens": 546108.0, + "reward": -8.53497314453125, + "reward_std": 4.388243675231934, + "rewards/rm_reward_func/mean": -8.53497314453125, + "rewards/rm_reward_func/std": 7.794600486755371, + "step": 44 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 388.0, + "completions/mean_length": 305.90625, + "completions/mean_terminated_length": 197.95237731933594, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.036, + "grad_norm": 2.3126137256622314, + "kl": 0.0004107952117919922, + "learning_rate": 6.984126984126983e-07, + "loss": 0.0486, + "num_tokens": 559873.0, + "reward": -7.420013427734375, + "reward_std": 7.450569152832031, + "rewards/rm_reward_func/mean": -7.420013427734375, + "rewards/rm_reward_func/std": 10.354388236999512, + "step": 45 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 387.25, + "completions/mean_terminated_length": 321.9047546386719, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.0368, + "grad_norm": 2.344557762145996, + "kl": 0.00038242340087890625, + "learning_rate": 7.142857142857143e-07, + "loss": -0.011, + "num_tokens": 575289.0, + "reward": -4.837789535522461, + "reward_std": 7.047362327575684, + "rewards/rm_reward_func/mean": -4.837789535522461, + "rewards/rm_reward_func/std": 11.340645790100098, + "step": 46 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 233.0, + "completions/mean_length": 140.09375, + "completions/mean_terminated_length": 71.22222137451172, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "epoch": 0.0376, + "grad_norm": 5.920055389404297, + "kl": 0.0005631446838378906, + "learning_rate": 7.301587301587301e-07, + "loss": -0.1061, + "num_tokens": 586604.0, + "reward": -4.19140625, + "reward_std": 4.153356552124023, + "rewards/rm_reward_func/mean": -4.19140625, + "rewards/rm_reward_func/std": 7.545008182525635, + "step": 47 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 378.0, + "completions/max_terminated_length": 378.0, + "completions/mean_length": 163.6875, + "completions/mean_terminated_length": 163.6875, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.0384, + "grad_norm": 3.746752977371216, + "kl": 0.0006389617919921875, + "learning_rate": 7.46031746031746e-07, + "loss": -0.094, + "num_tokens": 593642.0, + "reward": -10.421875, + "reward_std": 5.012988567352295, + "rewards/rm_reward_func/mean": -10.421875, + "rewards/rm_reward_func/std": 6.197994232177734, + "step": 48 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 333.0625, + "completions/mean_terminated_length": 251.72727966308594, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.0392, + "grad_norm": 3.094766139984131, + "kl": 0.0006709098815917969, + "learning_rate": 7.619047619047618e-07, + "loss": 0.0426, + "num_tokens": 606588.0, + "reward": -7.983154296875, + "reward_std": 8.31886100769043, + "rewards/rm_reward_func/mean": -7.983154296875, + "rewards/rm_reward_func/std": 10.532109260559082, + "step": 49 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 229.5, + "completions/mean_terminated_length": 200.27586364746094, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.04, + "grad_norm": 3.4759812355041504, + "kl": 0.0006818771362304688, + "learning_rate": 7.777777777777778e-07, + "loss": 0.1175, + "num_tokens": 616556.0, + "reward": -6.1270751953125, + "reward_std": 8.114751815795898, + "rewards/rm_reward_func/mean": -6.1270751953125, + "rewards/rm_reward_func/std": 8.840689659118652, + "step": 50 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 397.0, + "completions/mean_length": 311.0625, + "completions/mean_terminated_length": 205.8095245361328, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.0408, + "grad_norm": 2.890885353088379, + "kl": 0.000766754150390625, + "learning_rate": 7.936507936507936e-07, + "loss": 0.0924, + "num_tokens": 628694.0, + "reward": -8.6982421875, + "reward_std": 3.657209873199463, + "rewards/rm_reward_func/mean": -8.6982421875, + "rewards/rm_reward_func/std": 13.067253112792969, + "step": 51 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 489.0, + "completions/mean_length": 283.28125, + "completions/mean_terminated_length": 163.4761962890625, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.0416, + "grad_norm": 2.9035823345184326, + "kl": 0.0004100799560546875, + "learning_rate": 8.095238095238095e-07, + "loss": 0.2123, + "num_tokens": 641223.0, + "reward": -5.83807373046875, + "reward_std": 4.900543689727783, + "rewards/rm_reward_func/mean": -5.83807373046875, + "rewards/rm_reward_func/std": 7.653470993041992, + "step": 52 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 425.0, + "completions/max_terminated_length": 425.0, + "completions/mean_length": 114.9375, + "completions/mean_terminated_length": 114.9375, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.0424, + "grad_norm": 10.671420097351074, + "kl": 0.0011627674102783203, + "learning_rate": 8.253968253968253e-07, + "loss": -0.1026, + "num_tokens": 647501.0, + "reward": -8.42413330078125, + "reward_std": 5.442896842956543, + "rewards/rm_reward_func/mean": -8.42413330078125, + "rewards/rm_reward_func/std": 8.57506275177002, + "step": 53 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 468.0, + "completions/mean_length": 252.53125, + "completions/mean_terminated_length": 179.87998962402344, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.0432, + "grad_norm": 3.140366315841675, + "kl": 0.0007600784301757812, + "learning_rate": 8.412698412698413e-07, + "loss": 0.1312, + "num_tokens": 660070.0, + "reward": -10.078125, + "reward_std": 4.523303985595703, + "rewards/rm_reward_func/mean": -10.078125, + "rewards/rm_reward_func/std": 6.438615322113037, + "step": 54 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 202.15625, + "completions/mean_terminated_length": 157.8928680419922, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.044, + "grad_norm": 4.555398941040039, + "kl": 0.0013036727905273438, + "learning_rate": 8.57142857142857e-07, + "loss": 0.0684, + "num_tokens": 669715.0, + "reward": -10.841552734375, + "reward_std": 8.45435619354248, + "rewards/rm_reward_func/mean": -10.841552734375, + "rewards/rm_reward_func/std": 9.872593879699707, + "step": 55 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 336.65625, + "completions/mean_terminated_length": 200.2777862548828, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "epoch": 0.0448, + "grad_norm": 2.7749080657958984, + "kl": 0.0010623931884765625, + "learning_rate": 8.73015873015873e-07, + "loss": -0.0338, + "num_tokens": 683152.0, + "reward": -1.090301513671875, + "reward_std": 3.921980142593384, + "rewards/rm_reward_func/mean": -1.090301513671875, + "rewards/rm_reward_func/std": 6.629665374755859, + "step": 56 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 320.03125, + "completions/mean_terminated_length": 244.91305541992188, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "epoch": 0.0456, + "grad_norm": 2.539121389389038, + "kl": 0.0009260177612304688, + "learning_rate": 8.888888888888888e-07, + "loss": 0.1425, + "num_tokens": 697105.0, + "reward": -7.415740966796875, + "reward_std": 5.969917297363281, + "rewards/rm_reward_func/mean": -7.415740966796875, + "rewards/rm_reward_func/std": 7.336832523345947, + "step": 57 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 330.71875, + "completions/mean_terminated_length": 206.68421936035156, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.0464, + "grad_norm": 2.4101502895355225, + "kl": 0.0011167526245117188, + "learning_rate": 9.047619047619047e-07, + "loss": -0.0899, + "num_tokens": 710128.0, + "reward": -6.55548095703125, + "reward_std": 5.115406036376953, + "rewards/rm_reward_func/mean": -6.55548095703125, + "rewards/rm_reward_func/std": 6.5659332275390625, + "step": 58 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 307.375, + "completions/mean_terminated_length": 239.1666717529297, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.0472, + "grad_norm": 2.872269630432129, + "kl": 0.0010232925415039062, + "learning_rate": 9.206349206349205e-07, + "loss": 0.014, + "num_tokens": 721980.0, + "reward": -5.94189453125, + "reward_std": 5.278933525085449, + "rewards/rm_reward_func/mean": -5.94189453125, + "rewards/rm_reward_func/std": 8.825187683105469, + "step": 59 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 270.4375, + "completions/mean_terminated_length": 235.9285888671875, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.048, + "grad_norm": 3.2065460681915283, + "kl": 0.0014667510986328125, + "learning_rate": 9.365079365079365e-07, + "loss": 0.2253, + "num_tokens": 733306.0, + "reward": -5.34954833984375, + "reward_std": 8.191361427307129, + "rewards/rm_reward_func/mean": -5.34954833984375, + "rewards/rm_reward_func/std": 10.043052673339844, + "step": 60 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 470.0, + "completions/mean_length": 302.96875, + "completions/mean_terminated_length": 254.73077392578125, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "epoch": 0.0488, + "grad_norm": 2.4722800254821777, + "kl": 0.0020198822021484375, + "learning_rate": 9.523809523809522e-07, + "loss": -0.0115, + "num_tokens": 745889.0, + "reward": -13.30328369140625, + "reward_std": 3.812379837036133, + "rewards/rm_reward_func/mean": -13.30328369140625, + "rewards/rm_reward_func/std": 8.161885261535645, + "step": 61 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 218.5625, + "completions/mean_terminated_length": 209.09677124023438, + "completions/min_length": 17.0, + "completions/min_terminated_length": 17.0, + "epoch": 0.0496, + "grad_norm": 3.010167360305786, + "kl": 0.0018796920776367188, + "learning_rate": 9.682539682539682e-07, + "loss": -0.2943, + "num_tokens": 754907.0, + "reward": -7.835418701171875, + "reward_std": 5.403800010681152, + "rewards/rm_reward_func/mean": -7.835418701171875, + "rewards/rm_reward_func/std": 8.523995399475098, + "step": 62 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 455.0, + "completions/mean_length": 331.15625, + "completions/mean_terminated_length": 248.95455932617188, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.0504, + "grad_norm": 2.819601058959961, + "kl": 0.0015172958374023438, + "learning_rate": 9.84126984126984e-07, + "loss": -0.0081, + "num_tokens": 768560.0, + "reward": -11.4364013671875, + "reward_std": 3.361478328704834, + "rewards/rm_reward_func/mean": -11.4364013671875, + "rewards/rm_reward_func/std": 6.750278472900391, + "step": 63 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 327.03125, + "completions/mean_terminated_length": 230.1428680419922, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "epoch": 0.0512, + "grad_norm": 3.7442524433135986, + "kl": 0.0022602081298828125, + "learning_rate": 1e-06, + "loss": -0.1989, + "num_tokens": 781345.0, + "reward": -8.39996337890625, + "reward_std": 4.888169288635254, + "rewards/rm_reward_func/mean": -8.39996337890625, + "rewards/rm_reward_func/std": 5.147318363189697, + "step": 64 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 268.40625, + "completions/mean_terminated_length": 252.16668701171875, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.052, + "grad_norm": 2.8628954887390137, + "kl": 0.0019245147705078125, + "learning_rate": 1e-06, + "loss": -0.0962, + "num_tokens": 792750.0, + "reward": -7.360107421875, + "reward_std": 5.436917781829834, + "rewards/rm_reward_func/mean": -7.360107421875, + "rewards/rm_reward_func/std": 11.22769832611084, + "step": 65 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 332.46875, + "completions/mean_terminated_length": 209.63157653808594, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.0528, + "grad_norm": 3.654360294342041, + "kl": 0.0015087127685546875, + "learning_rate": 1e-06, + "loss": -0.1237, + "num_tokens": 805853.0, + "reward": -9.6962890625, + "reward_std": 2.946159839630127, + "rewards/rm_reward_func/mean": -9.6962890625, + "rewards/rm_reward_func/std": 4.295775890350342, + "step": 66 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 402.0, + "completions/mean_terminated_length": 277.3333435058594, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.0536, + "grad_norm": 2.3473589420318604, + "kl": 0.002231597900390625, + "learning_rate": 1e-06, + "loss": -0.0437, + "num_tokens": 823173.0, + "reward": -8.37060546875, + "reward_std": 3.5887060165405273, + "rewards/rm_reward_func/mean": -8.37060546875, + "rewards/rm_reward_func/std": 4.726972579956055, + "step": 67 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 364.6875, + "completions/mean_terminated_length": 250.11111450195312, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.0544, + "grad_norm": 2.694181442260742, + "kl": 0.001918792724609375, + "learning_rate": 1e-06, + "loss": 0.2082, + "num_tokens": 837739.0, + "reward": -7.8717041015625, + "reward_std": 3.8687283992767334, + "rewards/rm_reward_func/mean": -7.8717041015625, + "rewards/rm_reward_func/std": 7.253809928894043, + "step": 68 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 312.46875, + "completions/mean_terminated_length": 256.6000061035156, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.0552, + "grad_norm": 2.733241558074951, + "kl": 0.002780914306640625, + "learning_rate": 1e-06, + "loss": -0.0181, + "num_tokens": 849434.0, + "reward": -6.937103271484375, + "reward_std": 3.667171001434326, + "rewards/rm_reward_func/mean": -6.937103271484375, + "rewards/rm_reward_func/std": 5.047467231750488, + "step": 69 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 336.71875, + "completions/mean_terminated_length": 278.29168701171875, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.056, + "grad_norm": 2.4156734943389893, + "kl": 0.002368927001953125, + "learning_rate": 1e-06, + "loss": -0.1443, + "num_tokens": 862321.0, + "reward": -4.07220458984375, + "reward_std": 10.273677825927734, + "rewards/rm_reward_func/mean": -4.07220458984375, + "rewards/rm_reward_func/std": 13.013540267944336, + "step": 70 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 419.0, + "completions/mean_length": 314.90625, + "completions/mean_terminated_length": 180.05262756347656, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.0568, + "grad_norm": 3.1217379570007324, + "kl": 0.00289154052734375, + "learning_rate": 1e-06, + "loss": -0.0551, + "num_tokens": 875374.0, + "reward": -6.78515625, + "reward_std": 4.401189804077148, + "rewards/rm_reward_func/mean": -6.78515625, + "rewards/rm_reward_func/std": 7.016132354736328, + "step": 71 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 353.125, + "completions/mean_terminated_length": 323.7037048339844, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.0576, + "grad_norm": 2.9577646255493164, + "kl": 0.003635406494140625, + "learning_rate": 1e-06, + "loss": -0.1052, + "num_tokens": 889138.0, + "reward": -3.15277099609375, + "reward_std": 5.309854507446289, + "rewards/rm_reward_func/mean": -3.15277099609375, + "rewards/rm_reward_func/std": 5.763915538787842, + "step": 72 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 466.0, + "completions/mean_length": 258.90625, + "completions/mean_terminated_length": 188.0399932861328, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.0584, + "grad_norm": 3.2543458938598633, + "kl": 0.0028629302978515625, + "learning_rate": 1e-06, + "loss": 0.2214, + "num_tokens": 900951.0, + "reward": -8.47222900390625, + "reward_std": 4.21871280670166, + "rewards/rm_reward_func/mean": -8.47222900390625, + "rewards/rm_reward_func/std": 5.818933486938477, + "step": 73 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 243.0, + "completions/mean_terminated_length": 153.33334350585938, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.0592, + "grad_norm": 3.68485426902771, + "kl": 0.0032253265380859375, + "learning_rate": 1e-06, + "loss": -0.0463, + "num_tokens": 911351.0, + "reward": -9.206695556640625, + "reward_std": 6.0724778175354, + "rewards/rm_reward_func/mean": -9.206695556640625, + "rewards/rm_reward_func/std": 7.073054313659668, + "step": 74 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 472.0, + "completions/mean_length": 289.78125, + "completions/mean_terminated_length": 238.50001525878906, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.06, + "grad_norm": 2.78497052192688, + "kl": 0.00466156005859375, + "learning_rate": 1e-06, + "loss": 0.2228, + "num_tokens": 923680.0, + "reward": -6.83343505859375, + "reward_std": 7.71270227432251, + "rewards/rm_reward_func/mean": -6.83343505859375, + "rewards/rm_reward_func/std": 10.820587158203125, + "step": 75 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.71875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 481.46875, + "completions/mean_terminated_length": 403.4444580078125, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.0608, + "grad_norm": 1.8886445760726929, + "kl": 0.002117633819580078, + "learning_rate": 1e-06, + "loss": 0.0654, + "num_tokens": 944191.0, + "reward": -10.4248046875, + "reward_std": 5.140851020812988, + "rewards/rm_reward_func/mean": -10.4248046875, + "rewards/rm_reward_func/std": 9.057696342468262, + "step": 76 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 459.0, + "completions/mean_length": 180.1875, + "completions/mean_terminated_length": 169.48387145996094, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.0616, + "grad_norm": 5.139257431030273, + "kl": 0.002521514892578125, + "learning_rate": 1e-06, + "loss": 0.1781, + "num_tokens": 952477.0, + "reward": -1.8204498291015625, + "reward_std": 4.737975120544434, + "rewards/rm_reward_func/mean": -1.8204498291015625, + "rewards/rm_reward_func/std": 12.699488639831543, + "step": 77 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 411.0, + "completions/mean_length": 212.5625, + "completions/mean_terminated_length": 181.58621215820312, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.0624, + "grad_norm": 3.0258443355560303, + "kl": 0.0031642913818359375, + "learning_rate": 1e-06, + "loss": 0.2051, + "num_tokens": 963415.0, + "reward": -5.3250732421875, + "reward_std": 4.80600643157959, + "rewards/rm_reward_func/mean": -5.3250732421875, + "rewards/rm_reward_func/std": 4.75626802444458, + "step": 78 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 343.53125, + "completions/mean_terminated_length": 277.60870361328125, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.0632, + "grad_norm": 2.744492292404175, + "kl": 0.00372314453125, + "learning_rate": 1e-06, + "loss": -0.0128, + "num_tokens": 977392.0, + "reward": -4.4423980712890625, + "reward_std": 5.815633773803711, + "rewards/rm_reward_func/mean": -4.4423980712890625, + "rewards/rm_reward_func/std": 10.560279846191406, + "step": 79 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 466.0, + "completions/mean_length": 312.25, + "completions/mean_terminated_length": 156.88888549804688, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.064, + "grad_norm": 2.8983473777770996, + "kl": 0.003662109375, + "learning_rate": 1e-06, + "loss": 0.042, + "num_tokens": 995120.0, + "reward": -12.873046875, + "reward_std": 3.767939805984497, + "rewards/rm_reward_func/mean": -12.873046875, + "rewards/rm_reward_func/std": 7.584336757659912, + "step": 80 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 413.0, + "completions/mean_length": 246.96875, + "completions/mean_terminated_length": 185.8076934814453, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.0648, + "grad_norm": 3.573094129562378, + "kl": 0.0052947998046875, + "learning_rate": 1e-06, + "loss": -0.1861, + "num_tokens": 1005943.0, + "reward": -5.154052734375, + "reward_std": 5.333109378814697, + "rewards/rm_reward_func/mean": -5.154052734375, + "rewards/rm_reward_func/std": 5.350836277008057, + "step": 81 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 470.0, + "completions/mean_length": 298.96875, + "completions/mean_terminated_length": 249.80770874023438, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.0656, + "grad_norm": 2.644007444381714, + "kl": 0.003917694091796875, + "learning_rate": 1e-06, + "loss": 0.0187, + "num_tokens": 1020238.0, + "reward": -9.076370239257812, + "reward_std": 4.40856409072876, + "rewards/rm_reward_func/mean": -9.076370239257812, + "rewards/rm_reward_func/std": 6.288516521453857, + "step": 82 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 465.0, + "completions/mean_length": 307.0, + "completions/mean_terminated_length": 199.61904907226562, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.0664, + "grad_norm": 2.8343067169189453, + "kl": 0.0068359375, + "learning_rate": 1e-06, + "loss": -0.0099, + "num_tokens": 1032262.0, + "reward": -7.667877197265625, + "reward_std": 5.629947185516357, + "rewards/rm_reward_func/mean": -7.667877197265625, + "rewards/rm_reward_func/std": 7.898858070373535, + "step": 83 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 465.0, + "completions/max_terminated_length": 465.0, + "completions/mean_length": 225.71875, + "completions/mean_terminated_length": 225.71875, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.0672, + "grad_norm": 2.8555967807769775, + "kl": 0.00545501708984375, + "learning_rate": 1e-06, + "loss": 0.0421, + "num_tokens": 1042485.0, + "reward": 2.5863037109375, + "reward_std": 3.6911566257476807, + "rewards/rm_reward_func/mean": 2.5863037109375, + "rewards/rm_reward_func/std": 10.012650489807129, + "step": 84 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 435.0, + "completions/mean_length": 219.59375, + "completions/mean_terminated_length": 210.16128540039062, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.068, + "grad_norm": 4.213369369506836, + "kl": 0.00635528564453125, + "learning_rate": 1e-06, + "loss": 0.0863, + "num_tokens": 1052608.0, + "reward": -3.3102264404296875, + "reward_std": 3.797806978225708, + "rewards/rm_reward_func/mean": -3.3102264404296875, + "rewards/rm_reward_func/std": 5.886042594909668, + "step": 85 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 289.65625, + "completions/mean_terminated_length": 266.6551818847656, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.0688, + "grad_norm": 3.4353296756744385, + "kl": 0.0052337646484375, + "learning_rate": 1e-06, + "loss": 0.0776, + "num_tokens": 1064949.0, + "reward": 2.492401123046875, + "reward_std": 7.244277000427246, + "rewards/rm_reward_func/mean": 2.492401123046875, + "rewards/rm_reward_func/std": 9.47168254852295, + "step": 86 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 236.28125, + "completions/mean_terminated_length": 196.8928680419922, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.0696, + "grad_norm": 9.39013385772705, + "kl": 0.00577545166015625, + "learning_rate": 1e-06, + "loss": -0.0612, + "num_tokens": 1074950.0, + "reward": -5.302734375, + "reward_std": 6.743742942810059, + "rewards/rm_reward_func/mean": -5.302734375, + "rewards/rm_reward_func/std": 8.008939743041992, + "step": 87 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 292.46875, + "completions/mean_terminated_length": 192.68182373046875, + "completions/min_length": 26.0, + "completions/min_terminated_length": 26.0, + "epoch": 0.0704, + "grad_norm": 3.4955945014953613, + "kl": 0.0057942867279052734, + "learning_rate": 1e-06, + "loss": 0.2027, + "num_tokens": 1090957.0, + "reward": -4.89019775390625, + "reward_std": 5.86878776550293, + "rewards/rm_reward_func/mean": -4.89019775390625, + "rewards/rm_reward_func/std": 11.281463623046875, + "step": 88 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 299.4375, + "completions/mean_terminated_length": 154.0, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.0712, + "grad_norm": 3.3145103454589844, + "kl": 0.00629425048828125, + "learning_rate": 1e-06, + "loss": 0.0759, + "num_tokens": 1103699.0, + "reward": -7.9558868408203125, + "reward_std": 3.8132681846618652, + "rewards/rm_reward_func/mean": -7.9558868408203125, + "rewards/rm_reward_func/std": 5.60645055770874, + "step": 89 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 462.0, + "completions/mean_length": 242.3125, + "completions/mean_terminated_length": 214.41378784179688, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.072, + "grad_norm": 3.746366500854492, + "kl": 0.007389068603515625, + "learning_rate": 1e-06, + "loss": 0.0806, + "num_tokens": 1114293.0, + "reward": 4.7412109375, + "reward_std": 6.3503098487854, + "rewards/rm_reward_func/mean": 4.7412109375, + "rewards/rm_reward_func/std": 12.657453536987305, + "step": 90 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 392.34375, + "completions/mean_terminated_length": 217.4615478515625, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.0728, + "grad_norm": 2.246333360671997, + "kl": 0.00717926025390625, + "learning_rate": 1e-06, + "loss": -0.0118, + "num_tokens": 1130400.0, + "reward": -6.076568603515625, + "reward_std": 4.733026027679443, + "rewards/rm_reward_func/mean": -6.076568603515625, + "rewards/rm_reward_func/std": 10.103605270385742, + "step": 91 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 402.0, + "completions/mean_length": 368.15625, + "completions/mean_terminated_length": 292.8095397949219, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "epoch": 0.0736, + "grad_norm": 2.1302545070648193, + "kl": 0.007269859313964844, + "learning_rate": 1e-06, + "loss": 0.1118, + "num_tokens": 1146957.0, + "reward": -7.0111083984375, + "reward_std": 6.281430244445801, + "rewards/rm_reward_func/mean": -7.0111083984375, + "rewards/rm_reward_func/std": 10.263130187988281, + "step": 92 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 339.0, + "completions/mean_terminated_length": 271.3043518066406, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.0744, + "grad_norm": 3.688321113586426, + "kl": 0.00998687744140625, + "learning_rate": 1e-06, + "loss": 0.0504, + "num_tokens": 1159773.0, + "reward": -3.8799972534179688, + "reward_std": 4.736905097961426, + "rewards/rm_reward_func/mean": -3.8799972534179688, + "rewards/rm_reward_func/std": 6.819652557373047, + "step": 93 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 484.0, + "completions/mean_length": 260.21875, + "completions/mean_terminated_length": 252.09677124023438, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.0752, + "grad_norm": 3.679842472076416, + "kl": 0.00600433349609375, + "learning_rate": 1e-06, + "loss": -0.1287, + "num_tokens": 1171236.0, + "reward": -5.69622802734375, + "reward_std": 3.642548084259033, + "rewards/rm_reward_func/mean": -5.69622802734375, + "rewards/rm_reward_func/std": 5.160977363586426, + "step": 94 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 376.25, + "completions/mean_terminated_length": 283.368408203125, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.076, + "grad_norm": 2.582540273666382, + "kl": 0.00710296630859375, + "learning_rate": 1e-06, + "loss": -0.1972, + "num_tokens": 1186820.0, + "reward": -8.5262451171875, + "reward_std": 3.4011950492858887, + "rewards/rm_reward_func/mean": -8.5262451171875, + "rewards/rm_reward_func/std": 7.117488384246826, + "step": 95 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 423.0, + "completions/mean_length": 178.6875, + "completions/mean_terminated_length": 167.93548583984375, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.0768, + "grad_norm": 4.087407112121582, + "kl": 0.0051116943359375, + "learning_rate": 1e-06, + "loss": -0.1202, + "num_tokens": 1198074.0, + "reward": -5.597259521484375, + "reward_std": 7.2116804122924805, + "rewards/rm_reward_func/mean": -5.597259521484375, + "rewards/rm_reward_func/std": 8.06857681274414, + "step": 96 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 284.03125, + "completions/mean_terminated_length": 208.0416717529297, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.0776, + "grad_norm": 9.104924201965332, + "kl": 0.00815582275390625, + "learning_rate": 1e-06, + "loss": -0.1018, + "num_tokens": 1211131.0, + "reward": -9.880126953125, + "reward_std": 6.673112869262695, + "rewards/rm_reward_func/mean": -9.880126953125, + "rewards/rm_reward_func/std": 7.890270233154297, + "step": 97 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 318.0, + "completions/mean_length": 260.78125, + "completions/mean_terminated_length": 177.0416717529297, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.0784, + "grad_norm": 5.061870098114014, + "kl": 0.0068988800048828125, + "learning_rate": 1e-06, + "loss": 0.1266, + "num_tokens": 1224396.0, + "reward": -9.5322265625, + "reward_std": 5.915492057800293, + "rewards/rm_reward_func/mean": -9.5322265625, + "rewards/rm_reward_func/std": 13.454625129699707, + "step": 98 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 312.5, + "completions/mean_terminated_length": 275.5555725097656, + "completions/min_length": 15.0, + "completions/min_terminated_length": 15.0, + "epoch": 0.0792, + "grad_norm": 7.630608081817627, + "kl": 0.008739471435546875, + "learning_rate": 1e-06, + "loss": -0.0647, + "num_tokens": 1239732.0, + "reward": 1.023895263671875, + "reward_std": 8.057183265686035, + "rewards/rm_reward_func/mean": 1.023895263671875, + "rewards/rm_reward_func/std": 10.845428466796875, + "step": 99 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 415.0, + "completions/max_terminated_length": 415.0, + "completions/mean_length": 252.8125, + "completions/mean_terminated_length": 252.8125, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.08, + "grad_norm": 3.1217799186706543, + "kl": 0.01123046875, + "learning_rate": 1e-06, + "loss": -0.0374, + "num_tokens": 1250262.0, + "reward": -8.40460205078125, + "reward_std": 3.330627679824829, + "rewards/rm_reward_func/mean": -8.40460205078125, + "rewards/rm_reward_func/std": 4.349534034729004, + "step": 100 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 329.0, + "completions/mean_length": 245.09375, + "completions/mean_terminated_length": 123.7727279663086, + "completions/min_length": 11.0, + "completions/min_terminated_length": 11.0, + "epoch": 0.0808, + "grad_norm": 4.933348655700684, + "kl": 0.0134124755859375, + "learning_rate": 1e-06, + "loss": 0.1239, + "num_tokens": 1260473.0, + "reward": -4.2023468017578125, + "reward_std": 4.068781852722168, + "rewards/rm_reward_func/mean": -4.2023468017578125, + "rewards/rm_reward_func/std": 4.7899885177612305, + "step": 101 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 316.03125, + "completions/mean_terminated_length": 270.8077087402344, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.0816, + "grad_norm": 5.183587551116943, + "kl": 0.01630401611328125, + "learning_rate": 1e-06, + "loss": -0.0734, + "num_tokens": 1272762.0, + "reward": -2.0743408203125, + "reward_std": 4.2065324783325195, + "rewards/rm_reward_func/mean": -2.0743408203125, + "rewards/rm_reward_func/std": 7.418477535247803, + "step": 102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 190.34375, + "completions/mean_terminated_length": 144.3928680419922, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.0824, + "grad_norm": 9.615796089172363, + "kl": 0.0133819580078125, + "learning_rate": 1e-06, + "loss": 0.2496, + "num_tokens": 1282085.0, + "reward": -2.1240234375, + "reward_std": 6.015683650970459, + "rewards/rm_reward_func/mean": -2.1240234375, + "rewards/rm_reward_func/std": 7.95778751373291, + "step": 103 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 318.75, + "completions/mean_terminated_length": 168.44444274902344, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.0832, + "grad_norm": 4.339044094085693, + "kl": 0.01284027099609375, + "learning_rate": 1e-06, + "loss": 0.0598, + "num_tokens": 1298069.0, + "reward": -1.1455078125, + "reward_std": 5.1649861335754395, + "rewards/rm_reward_func/mean": -1.1455078125, + "rewards/rm_reward_func/std": 7.434368133544922, + "step": 104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 463.78125, + "completions/mean_terminated_length": 371.727294921875, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "epoch": 0.084, + "grad_norm": 3.03131365776062, + "kl": 0.0128326416015625, + "learning_rate": 1e-06, + "loss": -0.0067, + "num_tokens": 1315710.0, + "reward": -4.12103271484375, + "reward_std": 5.871215343475342, + "rewards/rm_reward_func/mean": -4.12103271484375, + "rewards/rm_reward_func/std": 8.927309036254883, + "step": 105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 449.0, + "completions/mean_length": 384.53125, + "completions/mean_terminated_length": 297.3157958984375, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.0848, + "grad_norm": 2.725952625274658, + "kl": 0.010955810546875, + "learning_rate": 1e-06, + "loss": 0.0927, + "num_tokens": 1331855.0, + "reward": -1.3194122314453125, + "reward_std": 3.2353854179382324, + "rewards/rm_reward_func/mean": -1.3194122314453125, + "rewards/rm_reward_func/std": 4.594810485839844, + "step": 106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 315.5625, + "completions/mean_terminated_length": 250.08334350585938, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.0856, + "grad_norm": 29.571395874023438, + "kl": 0.010345458984375, + "learning_rate": 1e-06, + "loss": -0.1343, + "num_tokens": 1345001.0, + "reward": -6.830474853515625, + "reward_std": 4.087754726409912, + "rewards/rm_reward_func/mean": -6.830474853515625, + "rewards/rm_reward_func/std": 9.476208686828613, + "step": 107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 291.5625, + "completions/mean_terminated_length": 240.69232177734375, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.0864, + "grad_norm": 4.20673131942749, + "kl": 0.013458251953125, + "learning_rate": 1e-06, + "loss": 0.1504, + "num_tokens": 1357331.0, + "reward": -5.711700439453125, + "reward_std": 3.702240467071533, + "rewards/rm_reward_func/mean": -5.711700439453125, + "rewards/rm_reward_func/std": 7.0592451095581055, + "step": 108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 364.8125, + "completions/mean_terminated_length": 297.9090881347656, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.0872, + "grad_norm": 2.5211193561553955, + "kl": 0.0170440673828125, + "learning_rate": 1e-06, + "loss": -0.0797, + "num_tokens": 1371525.0, + "reward": 3.977081298828125, + "reward_std": 5.856342792510986, + "rewards/rm_reward_func/mean": 3.977081298828125, + "rewards/rm_reward_func/std": 12.513916969299316, + "step": 109 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 464.0, + "completions/mean_length": 319.65625, + "completions/mean_terminated_length": 255.5416717529297, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.088, + "grad_norm": 4.921560287475586, + "kl": 0.01397705078125, + "learning_rate": 1e-06, + "loss": 0.0172, + "num_tokens": 1384498.0, + "reward": -5.39752197265625, + "reward_std": 3.883021354675293, + "rewards/rm_reward_func/mean": -5.39752197265625, + "rewards/rm_reward_func/std": 11.68447208404541, + "step": 110 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 317.4375, + "completions/mean_terminated_length": 215.52381896972656, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.0888, + "grad_norm": 5.259191989898682, + "kl": 0.0244293212890625, + "learning_rate": 1e-06, + "loss": 0.0037, + "num_tokens": 1398792.0, + "reward": -9.126190185546875, + "reward_std": 3.321913719177246, + "rewards/rm_reward_func/mean": -9.126190185546875, + "rewards/rm_reward_func/std": 8.484477043151855, + "step": 111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 411.0, + "completions/mean_length": 360.4375, + "completions/mean_terminated_length": 208.875, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.0896, + "grad_norm": 13.804244995117188, + "kl": 0.0152130126953125, + "learning_rate": 1e-06, + "loss": 0.132, + "num_tokens": 1413758.0, + "reward": -12.7744140625, + "reward_std": 3.5870838165283203, + "rewards/rm_reward_func/mean": -12.7744140625, + "rewards/rm_reward_func/std": 4.53514289855957, + "step": 112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 399.8125, + "completions/mean_terminated_length": 312.5555725097656, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.0904, + "grad_norm": 2.4952642917633057, + "kl": 0.00986480712890625, + "learning_rate": 1e-06, + "loss": -0.0065, + "num_tokens": 1430304.0, + "reward": -8.034423828125, + "reward_std": 2.978208541870117, + "rewards/rm_reward_func/mean": -8.034423828125, + "rewards/rm_reward_func/std": 7.964836120605469, + "step": 113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 431.5, + "completions/mean_terminated_length": 297.3333435058594, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.0912, + "grad_norm": 3.2031984329223633, + "kl": 0.012176513671875, + "learning_rate": 1e-06, + "loss": -0.0707, + "num_tokens": 1446624.0, + "reward": -8.611083984375, + "reward_std": 3.2820539474487305, + "rewards/rm_reward_func/mean": -8.611083984375, + "rewards/rm_reward_func/std": 6.850192070007324, + "step": 114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 437.0, + "completions/mean_length": 244.6875, + "completions/mean_terminated_length": 104.66667175292969, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.092, + "grad_norm": 17.274948120117188, + "kl": 0.0135955810546875, + "learning_rate": 1e-06, + "loss": 0.3859, + "num_tokens": 1461166.0, + "reward": -5.8297882080078125, + "reward_std": 4.278718948364258, + "rewards/rm_reward_func/mean": -5.8297882080078125, + "rewards/rm_reward_func/std": 5.697154521942139, + "step": 115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 486.0, + "completions/mean_length": 312.5, + "completions/mean_terminated_length": 221.8181915283203, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.0928, + "grad_norm": 3.2622101306915283, + "kl": 0.01324462890625, + "learning_rate": 1e-06, + "loss": -0.178, + "num_tokens": 1474222.0, + "reward": -2.70849609375, + "reward_std": 5.48113489151001, + "rewards/rm_reward_func/mean": -2.70849609375, + "rewards/rm_reward_func/std": 13.66856575012207, + "step": 116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 210.09375, + "completions/mean_terminated_length": 140.42308044433594, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.0936, + "grad_norm": 12.358490943908691, + "kl": 0.02017974853515625, + "learning_rate": 1e-06, + "loss": 0.0062, + "num_tokens": 1487289.0, + "reward": -5.93212890625, + "reward_std": 5.732447147369385, + "rewards/rm_reward_func/mean": -5.93212890625, + "rewards/rm_reward_func/std": 10.107297897338867, + "step": 117 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 307.0, + "completions/mean_terminated_length": 184.0, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "epoch": 0.0944, + "grad_norm": 7.494678974151611, + "kl": 0.012603759765625, + "learning_rate": 1e-06, + "loss": -0.1111, + "num_tokens": 1501401.0, + "reward": -0.51397705078125, + "reward_std": 4.986804008483887, + "rewards/rm_reward_func/mean": -0.51397705078125, + "rewards/rm_reward_func/std": 6.126167297363281, + "step": 118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 221.59375, + "completions/mean_terminated_length": 202.23333740234375, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.0952, + "grad_norm": 5.377990245819092, + "kl": 0.0181732177734375, + "learning_rate": 1e-06, + "loss": -0.2771, + "num_tokens": 1512988.0, + "reward": -0.9085693359375, + "reward_std": 4.9990386962890625, + "rewards/rm_reward_func/mean": -0.9085693359375, + "rewards/rm_reward_func/std": 6.47190523147583, + "step": 119 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 467.0, + "completions/mean_length": 307.78125, + "completions/mean_terminated_length": 148.94444274902344, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.096, + "grad_norm": 5.832651138305664, + "kl": 0.01479339599609375, + "learning_rate": 1e-06, + "loss": 0.0782, + "num_tokens": 1530557.0, + "reward": -7.892578125, + "reward_std": 3.8970372676849365, + "rewards/rm_reward_func/mean": -7.892578125, + "rewards/rm_reward_func/std": 6.989450454711914, + "step": 120 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 275.4375, + "completions/mean_terminated_length": 220.84616088867188, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.0968, + "grad_norm": 2.5381200313568115, + "kl": 0.0143890380859375, + "learning_rate": 1e-06, + "loss": -0.1052, + "num_tokens": 1542539.0, + "reward": -1.7589111328125, + "reward_std": 3.9134409427642822, + "rewards/rm_reward_func/mean": -1.7589111328125, + "rewards/rm_reward_func/std": 9.491743087768555, + "step": 121 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 427.0, + "completions/mean_length": 336.03125, + "completions/mean_terminated_length": 215.63157653808594, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.0976, + "grad_norm": 4.144689559936523, + "kl": 0.014801025390625, + "learning_rate": 1e-06, + "loss": 0.0091, + "num_tokens": 1555908.0, + "reward": -3.1043167114257812, + "reward_std": 3.349581718444824, + "rewards/rm_reward_func/mean": -3.1043167114257812, + "rewards/rm_reward_func/std": 6.525447368621826, + "step": 122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 281.1875, + "completions/mean_terminated_length": 257.3103332519531, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.0984, + "grad_norm": 10.447542190551758, + "kl": 0.02520751953125, + "learning_rate": 1e-06, + "loss": 0.0106, + "num_tokens": 1568394.0, + "reward": 2.2360823154449463, + "reward_std": 4.762975692749023, + "rewards/rm_reward_func/mean": 2.2360823154449463, + "rewards/rm_reward_func/std": 9.465394973754883, + "step": 123 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 457.0, + "completions/mean_length": 282.90625, + "completions/mean_terminated_length": 240.48147583007812, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.0992, + "grad_norm": 66.1051025390625, + "kl": 0.0311279296875, + "learning_rate": 1e-06, + "loss": 0.0544, + "num_tokens": 1584383.0, + "reward": -4.30712890625, + "reward_std": 7.795045852661133, + "rewards/rm_reward_func/mean": -4.30712890625, + "rewards/rm_reward_func/std": 11.063386917114258, + "step": 124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 470.0, + "completions/mean_length": 276.84375, + "completions/mean_terminated_length": 252.51724243164062, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.1, + "grad_norm": 3.615330457687378, + "kl": 0.025054931640625, + "learning_rate": 1e-06, + "loss": 0.0065, + "num_tokens": 1595322.0, + "reward": -11.7393798828125, + "reward_std": 2.689405918121338, + "rewards/rm_reward_func/mean": -11.7393798828125, + "rewards/rm_reward_func/std": 13.393643379211426, + "step": 125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 312.03125, + "completions/mean_terminated_length": 207.2857208251953, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.1008, + "grad_norm": 23.150880813598633, + "kl": 0.0230255126953125, + "learning_rate": 1e-06, + "loss": -0.1352, + "num_tokens": 1609723.0, + "reward": -8.863861083984375, + "reward_std": 6.96150541305542, + "rewards/rm_reward_func/mean": -8.863861083984375, + "rewards/rm_reward_func/std": 9.106093406677246, + "step": 126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 392.78125, + "completions/mean_terminated_length": 321.25, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.1016, + "grad_norm": 3.9794867038726807, + "kl": 0.011749267578125, + "learning_rate": 1e-06, + "loss": 0.2709, + "num_tokens": 1626636.0, + "reward": -8.689453125, + "reward_std": 4.927318572998047, + "rewards/rm_reward_func/mean": -8.689453125, + "rewards/rm_reward_func/std": 7.077872276306152, + "step": 127 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 326.0, + "completions/mean_terminated_length": 264.0, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.1024, + "grad_norm": 3.827911376953125, + "kl": 0.0189971923828125, + "learning_rate": 1e-06, + "loss": -0.0156, + "num_tokens": 1639084.0, + "reward": -0.710693359375, + "reward_std": 6.628046035766602, + "rewards/rm_reward_func/mean": -0.710693359375, + "rewards/rm_reward_func/std": 9.962486267089844, + "step": 128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 366.8125, + "completions/mean_terminated_length": 310.0, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.1032, + "grad_norm": 2.512551784515381, + "kl": 0.0133056640625, + "learning_rate": 1e-06, + "loss": 0.1508, + "num_tokens": 1653750.0, + "reward": 2.409698486328125, + "reward_std": 8.444917678833008, + "rewards/rm_reward_func/mean": 2.409698486328125, + "rewards/rm_reward_func/std": 11.581578254699707, + "step": 129 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 466.0, + "completions/mean_length": 309.09375, + "completions/mean_terminated_length": 271.5185241699219, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.104, + "grad_norm": 4.3505144119262695, + "kl": 0.02154541015625, + "learning_rate": 1e-06, + "loss": -0.078, + "num_tokens": 1666745.0, + "reward": -0.958282470703125, + "reward_std": 5.802244663238525, + "rewards/rm_reward_func/mean": -0.958282470703125, + "rewards/rm_reward_func/std": 7.622645378112793, + "step": 130 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 390.0, + "completions/mean_length": 206.59375, + "completions/mean_terminated_length": 162.96429443359375, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.1048, + "grad_norm": 5.6502861976623535, + "kl": 0.01812744140625, + "learning_rate": 1e-06, + "loss": 0.1992, + "num_tokens": 1677804.0, + "reward": -6.44091796875, + "reward_std": 6.14309024810791, + "rewards/rm_reward_func/mean": -6.44091796875, + "rewards/rm_reward_func/std": 10.069220542907715, + "step": 131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 385.5, + "completions/mean_terminated_length": 309.6000061035156, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.1056, + "grad_norm": 2.8553056716918945, + "kl": 0.02423095703125, + "learning_rate": 1e-06, + "loss": -0.1016, + "num_tokens": 1693340.0, + "reward": 2.6444091796875, + "reward_std": 8.04133415222168, + "rewards/rm_reward_func/mean": 2.6444091796875, + "rewards/rm_reward_func/std": 10.943760871887207, + "step": 132 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 495.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 324.65625, + "completions/mean_terminated_length": 324.65625, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "epoch": 0.1064, + "grad_norm": 2.7388484477996826, + "kl": 0.0200958251953125, + "learning_rate": 1e-06, + "loss": 0.0034, + "num_tokens": 1705993.0, + "reward": -2.5941162109375, + "reward_std": 4.211709499359131, + "rewards/rm_reward_func/mean": -2.5941162109375, + "rewards/rm_reward_func/std": 6.4686055183410645, + "step": 133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 465.0, + "completions/mean_length": 254.71875, + "completions/mean_terminated_length": 207.07408142089844, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "epoch": 0.1072, + "grad_norm": 4.826857089996338, + "kl": 0.01715087890625, + "learning_rate": 1e-06, + "loss": 0.2512, + "num_tokens": 1716728.0, + "reward": -6.234710693359375, + "reward_std": 6.557104110717773, + "rewards/rm_reward_func/mean": -6.234710693359375, + "rewards/rm_reward_func/std": 8.982573509216309, + "step": 134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 395.0, + "completions/mean_length": 201.1875, + "completions/mean_terminated_length": 191.16128540039062, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.108, + "grad_norm": 4.799185752868652, + "kl": 0.0233612060546875, + "learning_rate": 1e-06, + "loss": -0.0819, + "num_tokens": 1726414.0, + "reward": -4.594329833984375, + "reward_std": 5.226075649261475, + "rewards/rm_reward_func/mean": -4.594329833984375, + "rewards/rm_reward_func/std": 6.881073474884033, + "step": 135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 460.0, + "completions/max_terminated_length": 460.0, + "completions/mean_length": 223.96875, + "completions/mean_terminated_length": 223.96875, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.1088, + "grad_norm": 9.70561408996582, + "kl": 0.025299072265625, + "learning_rate": 1e-06, + "loss": 0.053, + "num_tokens": 1736069.0, + "reward": -0.31732177734375, + "reward_std": 3.850168466567993, + "rewards/rm_reward_func/mean": -0.31732177734375, + "rewards/rm_reward_func/std": 7.10466194152832, + "step": 136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 477.0, + "completions/mean_length": 329.125, + "completions/mean_terminated_length": 204.0, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.1096, + "grad_norm": 2.982024669647217, + "kl": 0.0196685791015625, + "learning_rate": 1e-06, + "loss": 0.0927, + "num_tokens": 1749921.0, + "reward": -6.221199035644531, + "reward_std": 5.583423137664795, + "rewards/rm_reward_func/mean": -6.221199035644531, + "rewards/rm_reward_func/std": 7.993007659912109, + "step": 137 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 457.0, + "completions/mean_length": 215.84375, + "completions/mean_terminated_length": 173.5357208251953, + "completions/min_length": 13.0, + "completions/min_terminated_length": 13.0, + "epoch": 0.1104, + "grad_norm": 7.27449893951416, + "kl": 0.0246734619140625, + "learning_rate": 1e-06, + "loss": 0.0318, + "num_tokens": 1761748.0, + "reward": -2.1171817779541016, + "reward_std": 5.113093376159668, + "rewards/rm_reward_func/mean": -2.1171817779541016, + "rewards/rm_reward_func/std": 6.772409439086914, + "step": 138 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 308.84375, + "completions/mean_terminated_length": 241.125, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.1112, + "grad_norm": 5.053046226501465, + "kl": 0.0249481201171875, + "learning_rate": 1e-06, + "loss": 0.0379, + "num_tokens": 1775439.0, + "reward": -1.70648193359375, + "reward_std": 5.1250834465026855, + "rewards/rm_reward_func/mean": -1.70648193359375, + "rewards/rm_reward_func/std": 6.081603050231934, + "step": 139 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 411.90625, + "completions/mean_terminated_length": 351.8500061035156, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.112, + "grad_norm": 2.3874924182891846, + "kl": 0.0135955810546875, + "learning_rate": 1e-06, + "loss": 0.0458, + "num_tokens": 1792004.0, + "reward": 1.85693359375, + "reward_std": 5.751528263092041, + "rewards/rm_reward_func/mean": 1.85693359375, + "rewards/rm_reward_func/std": 7.656973361968994, + "step": 140 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 251.59375, + "completions/mean_terminated_length": 234.2333526611328, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "epoch": 0.1128, + "grad_norm": 3.846666097640991, + "kl": 0.0225677490234375, + "learning_rate": 1e-06, + "loss": 0.103, + "num_tokens": 1802663.0, + "reward": -2.373046875, + "reward_std": 6.561789035797119, + "rewards/rm_reward_func/mean": -2.373046875, + "rewards/rm_reward_func/std": 8.583110809326172, + "step": 141 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 269.46875, + "completions/mean_terminated_length": 213.50001525878906, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.1136, + "grad_norm": 3.5698037147521973, + "kl": 0.03497314453125, + "learning_rate": 1e-06, + "loss": -0.0293, + "num_tokens": 1814766.0, + "reward": 0.341033935546875, + "reward_std": 5.161829948425293, + "rewards/rm_reward_func/mean": 0.341033935546875, + "rewards/rm_reward_func/std": 7.019633769989014, + "step": 142 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 114.0, + "completions/mean_length": 273.8125, + "completions/mean_terminated_length": 63.64706039428711, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "epoch": 0.1144, + "grad_norm": 9.236984252929688, + "kl": 0.0199432373046875, + "learning_rate": 1e-06, + "loss": 0.3644, + "num_tokens": 1827504.0, + "reward": -11.159202575683594, + "reward_std": 6.791610240936279, + "rewards/rm_reward_func/mean": -11.159202575683594, + "rewards/rm_reward_func/std": 8.779293060302734, + "step": 143 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 476.4375, + "completions/mean_terminated_length": 369.75, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.1152, + "grad_norm": 2.5026986598968506, + "kl": 0.0117034912109375, + "learning_rate": 1e-06, + "loss": -0.0064, + "num_tokens": 1847678.0, + "reward": -4.88604736328125, + "reward_std": 4.872498035430908, + "rewards/rm_reward_func/mean": -4.88604736328125, + "rewards/rm_reward_func/std": 9.581089973449707, + "step": 144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 306.9375, + "completions/mean_terminated_length": 268.96295166015625, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.116, + "grad_norm": 2.9737255573272705, + "kl": 0.03043365478515625, + "learning_rate": 1e-06, + "loss": -0.0469, + "num_tokens": 1863420.0, + "reward": -2.0446929931640625, + "reward_std": 3.8031177520751953, + "rewards/rm_reward_func/mean": -2.0446929931640625, + "rewards/rm_reward_func/std": 5.832355976104736, + "step": 145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 349.59375, + "completions/mean_terminated_length": 223.2777862548828, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.1168, + "grad_norm": 4.583868503570557, + "kl": 0.01976776123046875, + "learning_rate": 1e-06, + "loss": 0.0255, + "num_tokens": 1876591.0, + "reward": -1.48095703125, + "reward_std": 4.935769081115723, + "rewards/rm_reward_func/mean": -1.48095703125, + "rewards/rm_reward_func/std": 16.34881019592285, + "step": 146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 407.25, + "completions/mean_terminated_length": 325.77777099609375, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.1176, + "grad_norm": 3.11322021484375, + "kl": 0.029876708984375, + "learning_rate": 1e-06, + "loss": -0.1804, + "num_tokens": 1892183.0, + "reward": -4.62982177734375, + "reward_std": 6.981845855712891, + "rewards/rm_reward_func/mean": -4.62982177734375, + "rewards/rm_reward_func/std": 14.073089599609375, + "step": 147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 291.21875, + "completions/mean_terminated_length": 240.2692413330078, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.1184, + "grad_norm": 3.6348979473114014, + "kl": 0.02703857421875, + "learning_rate": 1e-06, + "loss": -0.0011, + "num_tokens": 1905302.0, + "reward": -3.182891845703125, + "reward_std": 7.350401878356934, + "rewards/rm_reward_func/mean": -3.182891845703125, + "rewards/rm_reward_func/std": 10.496583938598633, + "step": 148 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 206.125, + "completions/mean_terminated_length": 174.48275756835938, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.1192, + "grad_norm": 6.732820987701416, + "kl": 0.0248565673828125, + "learning_rate": 1e-06, + "loss": 0.0798, + "num_tokens": 1915282.0, + "reward": -6.91180419921875, + "reward_std": 5.384483337402344, + "rewards/rm_reward_func/mean": -6.91180419921875, + "rewards/rm_reward_func/std": 8.211865425109863, + "step": 149 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 371.78125, + "completions/mean_terminated_length": 308.04547119140625, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.12, + "grad_norm": 3.473301887512207, + "kl": 0.02490234375, + "learning_rate": 1e-06, + "loss": -0.0287, + "num_tokens": 1929515.0, + "reward": -1.9688720703125, + "reward_std": 5.15925931930542, + "rewards/rm_reward_func/mean": -1.9688720703125, + "rewards/rm_reward_func/std": 8.88673210144043, + "step": 150 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 356.46875, + "completions/mean_terminated_length": 334.25, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.1208, + "grad_norm": 14.675175666809082, + "kl": 0.0251617431640625, + "learning_rate": 1e-06, + "loss": -0.0117, + "num_tokens": 1947602.0, + "reward": 3.12274169921875, + "reward_std": 5.56088399887085, + "rewards/rm_reward_func/mean": 3.12274169921875, + "rewards/rm_reward_func/std": 5.511745452880859, + "step": 151 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 408.75, + "completions/mean_terminated_length": 361.8182067871094, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.1216, + "grad_norm": 17.709932327270508, + "kl": 0.0260009765625, + "learning_rate": 1e-06, + "loss": 0.1719, + "num_tokens": 1964330.0, + "reward": 1.937957763671875, + "reward_std": 5.787369728088379, + "rewards/rm_reward_func/mean": 1.937957763671875, + "rewards/rm_reward_func/std": 7.468286037445068, + "step": 152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 412.78125, + "completions/mean_terminated_length": 389.8846435546875, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.1224, + "grad_norm": 3.4813315868377686, + "kl": 0.027984619140625, + "learning_rate": 1e-06, + "loss": -0.0986, + "num_tokens": 1981395.0, + "reward": 0.11110877990722656, + "reward_std": 4.848204612731934, + "rewards/rm_reward_func/mean": 0.11110877990722656, + "rewards/rm_reward_func/std": 8.962310791015625, + "step": 153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 384.375, + "completions/mean_terminated_length": 285.1111145019531, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.1232, + "grad_norm": 3.2246146202087402, + "kl": 0.022918701171875, + "learning_rate": 1e-06, + "loss": 0.0176, + "num_tokens": 1996591.0, + "reward": 1.41229248046875, + "reward_std": 7.246562957763672, + "rewards/rm_reward_func/mean": 1.41229248046875, + "rewards/rm_reward_func/std": 7.45635461807251, + "step": 154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 354.34375, + "completions/mean_terminated_length": 325.1481628417969, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.124, + "grad_norm": 3.3056445121765137, + "kl": 0.030120849609375, + "learning_rate": 1e-06, + "loss": -0.1102, + "num_tokens": 2010946.0, + "reward": 10.169677734375, + "reward_std": 4.897927284240723, + "rewards/rm_reward_func/mean": 10.169677734375, + "rewards/rm_reward_func/std": 14.160859107971191, + "step": 155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 261.9375, + "completions/mean_terminated_length": 215.629638671875, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.1248, + "grad_norm": 27.496681213378906, + "kl": 0.02239990234375, + "learning_rate": 1e-06, + "loss": -0.0082, + "num_tokens": 2022800.0, + "reward": -2.4217529296875, + "reward_std": 6.2300496101379395, + "rewards/rm_reward_func/mean": -2.4217529296875, + "rewards/rm_reward_func/std": 10.626413345336914, + "step": 156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 480.0, + "completions/mean_length": 346.875, + "completions/mean_terminated_length": 201.1764678955078, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.1256, + "grad_norm": 4.6230573654174805, + "kl": 0.021440505981445312, + "learning_rate": 1e-06, + "loss": 0.1942, + "num_tokens": 2036684.0, + "reward": -7.7816162109375, + "reward_std": 8.917112350463867, + "rewards/rm_reward_func/mean": -7.7816162109375, + "rewards/rm_reward_func/std": 14.053092956542969, + "step": 157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 392.03125, + "completions/mean_terminated_length": 369.8148193359375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.1264, + "grad_norm": 2.703683853149414, + "kl": 0.034088134765625, + "learning_rate": 1e-06, + "loss": -0.0831, + "num_tokens": 2051333.0, + "reward": 0.690460205078125, + "reward_std": 7.271797180175781, + "rewards/rm_reward_func/mean": 0.690460205078125, + "rewards/rm_reward_func/std": 11.116622924804688, + "step": 158 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 486.0, + "completions/mean_length": 276.875, + "completions/mean_terminated_length": 184.86956787109375, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.1272, + "grad_norm": 7.96703577041626, + "kl": 0.01966094970703125, + "learning_rate": 1e-06, + "loss": 0.2494, + "num_tokens": 2063065.0, + "reward": 5.6082763671875, + "reward_std": 6.509998798370361, + "rewards/rm_reward_func/mean": 5.6082763671875, + "rewards/rm_reward_func/std": 8.432168960571289, + "step": 159 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 380.1875, + "completions/mean_terminated_length": 290.0, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.128, + "grad_norm": 3.4324798583984375, + "kl": 0.033443450927734375, + "learning_rate": 1e-06, + "loss": 0.0201, + "num_tokens": 2077479.0, + "reward": -6.3060302734375, + "reward_std": 4.749573707580566, + "rewards/rm_reward_func/mean": -6.3060302734375, + "rewards/rm_reward_func/std": 13.343415260314941, + "step": 160 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 350.25, + "completions/mean_terminated_length": 296.3333435058594, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.1288, + "grad_norm": 2.8479349613189697, + "kl": 0.023101806640625, + "learning_rate": 1e-06, + "loss": -0.0853, + "num_tokens": 2091847.0, + "reward": -2.8158111572265625, + "reward_std": 3.8607840538024902, + "rewards/rm_reward_func/mean": -2.8158111572265625, + "rewards/rm_reward_func/std": 7.014427185058594, + "step": 161 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 326.90625, + "completions/mean_terminated_length": 242.77273559570312, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "epoch": 0.1296, + "grad_norm": 5.42257833480835, + "kl": 0.03411865234375, + "learning_rate": 1e-06, + "loss": 0.025, + "num_tokens": 2105620.0, + "reward": -8.8624267578125, + "reward_std": 3.4918594360351562, + "rewards/rm_reward_func/mean": -8.8624267578125, + "rewards/rm_reward_func/std": 6.637436866760254, + "step": 162 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 482.0, + "completions/mean_length": 197.5, + "completions/mean_terminated_length": 164.96551513671875, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.1304, + "grad_norm": 4.355756759643555, + "kl": 0.037811279296875, + "learning_rate": 1e-06, + "loss": -0.0119, + "num_tokens": 2114116.0, + "reward": -11.6080322265625, + "reward_std": 4.272393226623535, + "rewards/rm_reward_func/mean": -11.6080322265625, + "rewards/rm_reward_func/std": 6.417409896850586, + "step": 163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 276.53125, + "completions/mean_terminated_length": 210.59999084472656, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.1312, + "grad_norm": 3.7714273929595947, + "kl": 0.04034423828125, + "learning_rate": 1e-06, + "loss": 0.1036, + "num_tokens": 2125205.0, + "reward": -6.275434494018555, + "reward_std": 6.214200973510742, + "rewards/rm_reward_func/mean": -6.275434494018555, + "rewards/rm_reward_func/std": 8.869131088256836, + "step": 164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 432.0, + "completions/mean_length": 258.21875, + "completions/mean_terminated_length": 241.30001831054688, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.132, + "grad_norm": 7.6665167808532715, + "kl": 0.032684326171875, + "learning_rate": 1e-06, + "loss": -0.0494, + "num_tokens": 2135652.0, + "reward": -7.063232421875, + "reward_std": 4.540660858154297, + "rewards/rm_reward_func/mean": -7.063232421875, + "rewards/rm_reward_func/std": 7.4130377769470215, + "step": 165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 366.40625, + "completions/mean_terminated_length": 317.875, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.1328, + "grad_norm": 2.8710451126098633, + "kl": 0.0362548828125, + "learning_rate": 1e-06, + "loss": -0.0732, + "num_tokens": 2149737.0, + "reward": -1.594390869140625, + "reward_std": 5.7335591316223145, + "rewards/rm_reward_func/mean": -1.594390869140625, + "rewards/rm_reward_func/std": 10.367554664611816, + "step": 166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 370.6875, + "completions/mean_terminated_length": 315.39129638671875, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.1336, + "grad_norm": 2.312666416168213, + "kl": 0.01702880859375, + "learning_rate": 1e-06, + "loss": -0.113, + "num_tokens": 2165135.0, + "reward": 1.4799518585205078, + "reward_std": 6.825066566467285, + "rewards/rm_reward_func/mean": 1.4799518585205078, + "rewards/rm_reward_func/std": 11.376241683959961, + "step": 167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 485.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 189.3125, + "completions/mean_terminated_length": 189.3125, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "epoch": 0.1344, + "grad_norm": 11.69007396697998, + "kl": 0.06195068359375, + "learning_rate": 1e-06, + "loss": 0.0645, + "num_tokens": 2174529.0, + "reward": 4.023796081542969, + "reward_std": 5.484214782714844, + "rewards/rm_reward_func/mean": 4.023796081542969, + "rewards/rm_reward_func/std": 12.138504028320312, + "step": 168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 427.0, + "completions/mean_length": 280.0, + "completions/mean_terminated_length": 246.85714721679688, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.1352, + "grad_norm": 3.6376917362213135, + "kl": 0.037750244140625, + "learning_rate": 1e-06, + "loss": -0.0308, + "num_tokens": 2185737.0, + "reward": 3.1946182250976562, + "reward_std": 4.653729438781738, + "rewards/rm_reward_func/mean": 3.1946182250976562, + "rewards/rm_reward_func/std": 8.615864753723145, + "step": 169 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 288.53125, + "completions/mean_terminated_length": 265.4137878417969, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.136, + "grad_norm": 6.496363639831543, + "kl": 0.042877197265625, + "learning_rate": 1e-06, + "loss": -0.1146, + "num_tokens": 2197530.0, + "reward": -3.2793121337890625, + "reward_std": 5.921474456787109, + "rewards/rm_reward_func/mean": -3.2793121337890625, + "rewards/rm_reward_func/std": 9.333773612976074, + "step": 170 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 293.46875, + "completions/mean_terminated_length": 194.13636779785156, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.1368, + "grad_norm": 4.2207770347595215, + "kl": 0.029510498046875, + "learning_rate": 1e-06, + "loss": 0.271, + "num_tokens": 2210929.0, + "reward": -6.6259765625, + "reward_std": 6.164700984954834, + "rewards/rm_reward_func/mean": -6.6259765625, + "rewards/rm_reward_func/std": 8.203378677368164, + "step": 171 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 425.0, + "completions/mean_terminated_length": 348.23529052734375, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.1376, + "grad_norm": 2.683828592300415, + "kl": 0.02593994140625, + "learning_rate": 1e-06, + "loss": 0.092, + "num_tokens": 2229097.0, + "reward": -2.466796875, + "reward_std": 5.979694843292236, + "rewards/rm_reward_func/mean": -2.466796875, + "rewards/rm_reward_func/std": 21.459537506103516, + "step": 172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 312.15625, + "completions/mean_terminated_length": 207.4761962890625, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.1384, + "grad_norm": 4.745087146759033, + "kl": 0.027069091796875, + "learning_rate": 1e-06, + "loss": 0.2947, + "num_tokens": 2244958.0, + "reward": 0.202301025390625, + "reward_std": 7.154237270355225, + "rewards/rm_reward_func/mean": 0.202301025390625, + "rewards/rm_reward_func/std": 12.688895225524902, + "step": 173 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 447.09375, + "completions/mean_terminated_length": 373.5333557128906, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.1392, + "grad_norm": 11.718962669372559, + "kl": 0.0233154296875, + "learning_rate": 1e-06, + "loss": -0.0934, + "num_tokens": 2262017.0, + "reward": 5.55755615234375, + "reward_std": 6.223611354827881, + "rewards/rm_reward_func/mean": 5.55755615234375, + "rewards/rm_reward_func/std": 9.582088470458984, + "step": 174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 287.4375, + "completions/mean_terminated_length": 280.19354248046875, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.14, + "grad_norm": 4.201067924499512, + "kl": 0.060211181640625, + "learning_rate": 1e-06, + "loss": -0.0737, + "num_tokens": 2275175.0, + "reward": -4.629188537597656, + "reward_std": 5.972632884979248, + "rewards/rm_reward_func/mean": -4.629188537597656, + "rewards/rm_reward_func/std": 9.620247840881348, + "step": 175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 315.46875, + "completions/mean_terminated_length": 270.1153869628906, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.1408, + "grad_norm": 36.3863639831543, + "kl": 0.034271240234375, + "learning_rate": 1e-06, + "loss": -0.0361, + "num_tokens": 2288718.0, + "reward": 2.839324951171875, + "reward_std": 5.17338752746582, + "rewards/rm_reward_func/mean": 2.839324951171875, + "rewards/rm_reward_func/std": 6.9854583740234375, + "step": 176 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 459.3125, + "completions/mean_terminated_length": 412.8235168457031, + "completions/min_length": 284.0, + "completions/min_terminated_length": 284.0, + "epoch": 0.1416, + "grad_norm": 2.37556529045105, + "kl": 0.02569580078125, + "learning_rate": 1e-06, + "loss": -0.0249, + "num_tokens": 2305264.0, + "reward": 4.6612548828125, + "reward_std": 4.835301399230957, + "rewards/rm_reward_func/mean": 4.6612548828125, + "rewards/rm_reward_func/std": 6.540696620941162, + "step": 177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 486.0, + "completions/mean_length": 335.90625, + "completions/mean_terminated_length": 277.2083435058594, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.1424, + "grad_norm": 3.222507953643799, + "kl": 0.023162841796875, + "learning_rate": 1e-06, + "loss": 0.0366, + "num_tokens": 2320461.0, + "reward": 0.611053466796875, + "reward_std": 8.012971878051758, + "rewards/rm_reward_func/mean": 0.611053466796875, + "rewards/rm_reward_func/std": 21.763654708862305, + "step": 178 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 301.0, + "completions/mean_length": 276.0625, + "completions/mean_terminated_length": 197.4166717529297, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.1432, + "grad_norm": 3.5161032676696777, + "kl": 0.0360260009765625, + "learning_rate": 1e-06, + "loss": -0.0888, + "num_tokens": 2331991.0, + "reward": -10.7362060546875, + "reward_std": 3.1551408767700195, + "rewards/rm_reward_func/mean": -10.7362060546875, + "rewards/rm_reward_func/std": 7.49362850189209, + "step": 179 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 237.0625, + "completions/mean_terminated_length": 145.4166717529297, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.144, + "grad_norm": 34.31532287597656, + "kl": 0.02911376953125, + "learning_rate": 1e-06, + "loss": 0.3252, + "num_tokens": 2342809.0, + "reward": -6.35968017578125, + "reward_std": 4.969329833984375, + "rewards/rm_reward_func/mean": -6.35968017578125, + "rewards/rm_reward_func/std": 10.100662231445312, + "step": 180 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 444.0, + "completions/max_terminated_length": 444.0, + "completions/mean_length": 174.9375, + "completions/mean_terminated_length": 174.9375, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "epoch": 0.1448, + "grad_norm": 11.092037200927734, + "kl": 0.059326171875, + "learning_rate": 1e-06, + "loss": -0.0305, + "num_tokens": 2351343.0, + "reward": 3.357177734375, + "reward_std": 7.95058012008667, + "rewards/rm_reward_func/mean": 3.357177734375, + "rewards/rm_reward_func/std": 9.181958198547363, + "step": 181 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 473.0, + "completions/mean_length": 221.625, + "completions/mean_terminated_length": 167.8518524169922, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.1456, + "grad_norm": 10.811847686767578, + "kl": 0.04290771484375, + "learning_rate": 1e-06, + "loss": -0.1376, + "num_tokens": 2362931.0, + "reward": -2.74267578125, + "reward_std": 5.7764482498168945, + "rewards/rm_reward_func/mean": -2.74267578125, + "rewards/rm_reward_func/std": 14.296358108520508, + "step": 182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 350.90625, + "completions/mean_terminated_length": 266.5238037109375, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.1464, + "grad_norm": 11.304616928100586, + "kl": 0.02972412109375, + "learning_rate": 1e-06, + "loss": -0.0109, + "num_tokens": 2376720.0, + "reward": -2.41259765625, + "reward_std": 4.233550071716309, + "rewards/rm_reward_func/mean": -2.41259765625, + "rewards/rm_reward_func/std": 8.830628395080566, + "step": 183 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 388.46875, + "completions/mean_terminated_length": 359.9615478515625, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "epoch": 0.1472, + "grad_norm": 17.827436447143555, + "kl": 0.04998779296875, + "learning_rate": 1e-06, + "loss": -0.0211, + "num_tokens": 2391807.0, + "reward": 5.77008056640625, + "reward_std": 6.17415189743042, + "rewards/rm_reward_func/mean": 5.77008056640625, + "rewards/rm_reward_func/std": 7.23347806930542, + "step": 184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 329.5, + "completions/mean_terminated_length": 258.08697509765625, + "completions/min_length": 26.0, + "completions/min_terminated_length": 26.0, + "epoch": 0.148, + "grad_norm": 20.43731117248535, + "kl": 0.037933349609375, + "learning_rate": 1e-06, + "loss": 0.2097, + "num_tokens": 2406519.0, + "reward": -1.623046875, + "reward_std": 10.683743476867676, + "rewards/rm_reward_func/mean": -1.623046875, + "rewards/rm_reward_func/std": 14.652113914489746, + "step": 185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 397.8125, + "completions/mean_terminated_length": 345.9090881347656, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.1488, + "grad_norm": 3.0334293842315674, + "kl": 0.0233306884765625, + "learning_rate": 1e-06, + "loss": 0.0335, + "num_tokens": 2421993.0, + "reward": 8.384445190429688, + "reward_std": 11.130109786987305, + "rewards/rm_reward_func/mean": 8.384445190429688, + "rewards/rm_reward_func/std": 12.353397369384766, + "step": 186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 363.875, + "completions/mean_terminated_length": 233.1764678955078, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.1496, + "grad_norm": 2.8142430782318115, + "kl": 0.03497314453125, + "learning_rate": 1e-06, + "loss": 0.1352, + "num_tokens": 2435917.0, + "reward": -1.4100570678710938, + "reward_std": 7.787389755249023, + "rewards/rm_reward_func/mean": -1.4100570678710938, + "rewards/rm_reward_func/std": 9.113836288452148, + "step": 187 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 266.53125, + "completions/mean_terminated_length": 250.16668701171875, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "epoch": 0.1504, + "grad_norm": 8.15317440032959, + "kl": 0.0330810546875, + "learning_rate": 1e-06, + "loss": -0.0168, + "num_tokens": 2446750.0, + "reward": -1.41357421875, + "reward_std": 6.009856224060059, + "rewards/rm_reward_func/mean": -1.41357421875, + "rewards/rm_reward_func/std": 7.865617752075195, + "step": 188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 282.34375, + "completions/mean_terminated_length": 162.04762268066406, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "epoch": 0.1512, + "grad_norm": 86.80154418945312, + "kl": 0.05029296875, + "learning_rate": 1e-06, + "loss": 0.0314, + "num_tokens": 2458649.0, + "reward": -3.796173095703125, + "reward_std": 4.965305328369141, + "rewards/rm_reward_func/mean": -3.796173095703125, + "rewards/rm_reward_func/std": 7.862216949462891, + "step": 189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 337.875, + "completions/mean_terminated_length": 246.6666717529297, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.152, + "grad_norm": 3.135280132293701, + "kl": 0.037353515625, + "learning_rate": 1e-06, + "loss": -0.1424, + "num_tokens": 2471605.0, + "reward": 4.255180358886719, + "reward_std": 5.963335990905762, + "rewards/rm_reward_func/mean": 4.255180358886719, + "rewards/rm_reward_func/std": 7.225700855255127, + "step": 190 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 327.96875, + "completions/mean_terminated_length": 217.5500030517578, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.1528, + "grad_norm": 3.981858730316162, + "kl": 0.0426025390625, + "learning_rate": 1e-06, + "loss": 0.187, + "num_tokens": 2484356.0, + "reward": -6.214874267578125, + "reward_std": 4.893754482269287, + "rewards/rm_reward_func/mean": -6.214874267578125, + "rewards/rm_reward_func/std": 11.553343772888184, + "step": 191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 363.5625, + "completions/mean_terminated_length": 305.478271484375, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.1536, + "grad_norm": 5.650016784667969, + "kl": 0.03961181640625, + "learning_rate": 1e-06, + "loss": 0.2227, + "num_tokens": 2499622.0, + "reward": -3.061126708984375, + "reward_std": 7.803011417388916, + "rewards/rm_reward_func/mean": -3.061126708984375, + "rewards/rm_reward_func/std": 11.103745460510254, + "step": 192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 282.375, + "completions/mean_terminated_length": 239.8518524169922, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.1544, + "grad_norm": 4.78141975402832, + "kl": 0.041748046875, + "learning_rate": 1e-06, + "loss": 0.0997, + "num_tokens": 2511370.0, + "reward": -4.6614990234375, + "reward_std": 3.844748020172119, + "rewards/rm_reward_func/mean": -4.6614990234375, + "rewards/rm_reward_func/std": 6.409038066864014, + "step": 193 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 218.28125, + "completions/mean_terminated_length": 120.375, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.1552, + "grad_norm": 15.153929710388184, + "kl": 0.03350830078125, + "learning_rate": 1e-06, + "loss": 0.1686, + "num_tokens": 2524115.0, + "reward": 1.6598663330078125, + "reward_std": 6.3394975662231445, + "rewards/rm_reward_func/mean": 1.6598663330078125, + "rewards/rm_reward_func/std": 11.99753475189209, + "step": 194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 158.875, + "completions/mean_terminated_length": 108.42857360839844, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "epoch": 0.156, + "grad_norm": 7.861037731170654, + "kl": 0.0665283203125, + "learning_rate": 1e-06, + "loss": -0.0664, + "num_tokens": 2531239.0, + "reward": -6.14581298828125, + "reward_std": 6.690445423126221, + "rewards/rm_reward_func/mean": -6.14581298828125, + "rewards/rm_reward_func/std": 10.1651029586792, + "step": 195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 329.625, + "completions/mean_terminated_length": 317.4666748046875, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.1568, + "grad_norm": 8.710630416870117, + "kl": 0.04339599609375, + "learning_rate": 1e-06, + "loss": 0.0139, + "num_tokens": 2543811.0, + "reward": 7.0421142578125, + "reward_std": 7.355923175811768, + "rewards/rm_reward_func/mean": 7.0421142578125, + "rewards/rm_reward_func/std": 15.240861892700195, + "step": 196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 467.0, + "completions/max_terminated_length": 467.0, + "completions/mean_length": 174.8125, + "completions/mean_terminated_length": 174.8125, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.1576, + "grad_norm": 4.834583759307861, + "kl": 0.03778076171875, + "learning_rate": 1e-06, + "loss": 0.0676, + "num_tokens": 2553517.0, + "reward": 0.12786865234375, + "reward_std": 3.288187265396118, + "rewards/rm_reward_func/mean": 0.12786865234375, + "rewards/rm_reward_func/std": 5.167597770690918, + "step": 197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 301.75, + "completions/mean_terminated_length": 280.0, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.1584, + "grad_norm": 3.8911170959472656, + "kl": 0.037628173828125, + "learning_rate": 1e-06, + "loss": 0.0381, + "num_tokens": 2565581.0, + "reward": 8.29638671875, + "reward_std": 5.519439697265625, + "rewards/rm_reward_func/mean": 8.29638671875, + "rewards/rm_reward_func/std": 12.758336067199707, + "step": 198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 317.46875, + "completions/mean_terminated_length": 252.625, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.1592, + "grad_norm": 4.481633186340332, + "kl": 0.0205078125, + "learning_rate": 1e-06, + "loss": 0.0399, + "num_tokens": 2578404.0, + "reward": -3.97637939453125, + "reward_std": 6.747166633605957, + "rewards/rm_reward_func/mean": -3.97637939453125, + "rewards/rm_reward_func/std": 13.583218574523926, + "step": 199 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 334.21875, + "completions/mean_terminated_length": 195.94444274902344, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.16, + "grad_norm": 2.933378219604492, + "kl": 0.0440826416015625, + "learning_rate": 1e-06, + "loss": 0.0151, + "num_tokens": 2591595.0, + "reward": -2.1455078125, + "reward_std": 2.6753859519958496, + "rewards/rm_reward_func/mean": -2.1455078125, + "rewards/rm_reward_func/std": 7.882194519042969, + "step": 200 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 391.9375, + "completions/mean_terminated_length": 358.3199768066406, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.1608, + "grad_norm": 4.800445079803467, + "kl": 0.02825927734375, + "learning_rate": 1e-06, + "loss": 0.0658, + "num_tokens": 2606889.0, + "reward": 2.6326370239257812, + "reward_std": 7.309446334838867, + "rewards/rm_reward_func/mean": 2.6326370239257812, + "rewards/rm_reward_func/std": 9.031261444091797, + "step": 201 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 456.0, + "completions/mean_length": 322.4375, + "completions/mean_terminated_length": 287.3333435058594, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.1616, + "grad_norm": 6.8586320877075195, + "kl": 0.07305908203125, + "learning_rate": 1e-06, + "loss": 0.0351, + "num_tokens": 2623223.0, + "reward": 0.64892578125, + "reward_std": 3.7500052452087402, + "rewards/rm_reward_func/mean": 0.64892578125, + "rewards/rm_reward_func/std": 11.14384937286377, + "step": 202 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 310.0, + "completions/mean_terminated_length": 230.95652770996094, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.1624, + "grad_norm": 4.2218337059021, + "kl": 0.039154052734375, + "learning_rate": 1e-06, + "loss": 0.1215, + "num_tokens": 2638695.0, + "reward": -3.1102294921875, + "reward_std": 10.1006498336792, + "rewards/rm_reward_func/mean": -3.1102294921875, + "rewards/rm_reward_func/std": 10.23782730102539, + "step": 203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 470.0, + "completions/mean_length": 271.6875, + "completions/mean_terminated_length": 191.58334350585938, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.1632, + "grad_norm": 4.688625335693359, + "kl": 0.0411376953125, + "learning_rate": 1e-06, + "loss": 0.2514, + "num_tokens": 2652157.0, + "reward": 2.2646484375, + "reward_std": 7.363420486450195, + "rewards/rm_reward_func/mean": 2.2646484375, + "rewards/rm_reward_func/std": 11.030182838439941, + "step": 204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 473.0, + "completions/mean_length": 306.90625, + "completions/mean_terminated_length": 166.57894897460938, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.164, + "grad_norm": 4.533467769622803, + "kl": 0.02496337890625, + "learning_rate": 1e-06, + "loss": -0.0195, + "num_tokens": 2664242.0, + "reward": -4.059173583984375, + "reward_std": 5.695614337921143, + "rewards/rm_reward_func/mean": -4.059173583984375, + "rewards/rm_reward_func/std": 7.963764190673828, + "step": 205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 321.875, + "completions/mean_terminated_length": 247.478271484375, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.1648, + "grad_norm": 4.341475009918213, + "kl": 0.0380401611328125, + "learning_rate": 1e-06, + "loss": 0.0705, + "num_tokens": 2677966.0, + "reward": -6.3792724609375, + "reward_std": 8.557640075683594, + "rewards/rm_reward_func/mean": -6.3792724609375, + "rewards/rm_reward_func/std": 10.657180786132812, + "step": 206 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 254.1875, + "completions/mean_terminated_length": 227.51724243164062, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.1656, + "grad_norm": 18.00819206237793, + "kl": 0.02423095703125, + "learning_rate": 1e-06, + "loss": -0.1429, + "num_tokens": 2688732.0, + "reward": -7.324462890625, + "reward_std": 5.273411750793457, + "rewards/rm_reward_func/mean": -7.324462890625, + "rewards/rm_reward_func/std": 13.72179126739502, + "step": 207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 219.53125, + "completions/mean_terminated_length": 200.03334045410156, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.1664, + "grad_norm": 9.372909545898438, + "kl": 0.04443359375, + "learning_rate": 1e-06, + "loss": 0.1745, + "num_tokens": 2700973.0, + "reward": -3.869039535522461, + "reward_std": 4.182497978210449, + "rewards/rm_reward_func/mean": -3.869039535522461, + "rewards/rm_reward_func/std": 5.315476417541504, + "step": 208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 381.875, + "completions/mean_terminated_length": 313.71429443359375, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.1672, + "grad_norm": 14.11308765411377, + "kl": 0.0384521484375, + "learning_rate": 1e-06, + "loss": -0.0134, + "num_tokens": 2715521.0, + "reward": 7.0210418701171875, + "reward_std": 9.68466854095459, + "rewards/rm_reward_func/mean": 7.0210418701171875, + "rewards/rm_reward_func/std": 17.186697006225586, + "step": 209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 259.0, + "completions/mean_terminated_length": 174.6666717529297, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.168, + "grad_norm": 4.891456604003906, + "kl": 0.071197509765625, + "learning_rate": 1e-06, + "loss": -0.0402, + "num_tokens": 2725753.0, + "reward": -1.91473388671875, + "reward_std": 5.077539920806885, + "rewards/rm_reward_func/mean": -1.91473388671875, + "rewards/rm_reward_func/std": 8.266571044921875, + "step": 210 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 413.21875, + "completions/mean_terminated_length": 268.8461608886719, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.1688, + "grad_norm": 3.1043269634246826, + "kl": 0.01953125, + "learning_rate": 1e-06, + "loss": 0.1178, + "num_tokens": 2742696.0, + "reward": -9.068359375, + "reward_std": 7.59498405456543, + "rewards/rm_reward_func/mean": -9.068359375, + "rewards/rm_reward_func/std": 10.789223670959473, + "step": 211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 359.5, + "completions/mean_terminated_length": 255.15789794921875, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.1696, + "grad_norm": 5.556434631347656, + "kl": 0.03485107421875, + "learning_rate": 1e-06, + "loss": 0.0683, + "num_tokens": 2758216.0, + "reward": 2.6542510986328125, + "reward_std": 4.50178337097168, + "rewards/rm_reward_func/mean": 2.6542510986328125, + "rewards/rm_reward_func/std": 7.546914100646973, + "step": 212 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 250.25, + "completions/mean_terminated_length": 223.1724090576172, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.1704, + "grad_norm": 4.8082275390625, + "kl": 0.03082275390625, + "learning_rate": 1e-06, + "loss": 0.0538, + "num_tokens": 2768344.0, + "reward": 0.75689697265625, + "reward_std": 5.467959880828857, + "rewards/rm_reward_func/mean": 0.75689697265625, + "rewards/rm_reward_func/std": 8.289709091186523, + "step": 213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 473.0, + "completions/mean_length": 377.5625, + "completions/mean_terminated_length": 307.1428527832031, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.1712, + "grad_norm": 3.0155625343322754, + "kl": 0.032958984375, + "learning_rate": 1e-06, + "loss": -0.0965, + "num_tokens": 2782498.0, + "reward": 0.0223388671875, + "reward_std": 4.46353006362915, + "rewards/rm_reward_func/mean": 0.0223388671875, + "rewards/rm_reward_func/std": 11.193925857543945, + "step": 214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 203.6875, + "completions/mean_terminated_length": 171.79310607910156, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.172, + "grad_norm": 5.511065483093262, + "kl": 0.0542755126953125, + "learning_rate": 1e-06, + "loss": 0.3333, + "num_tokens": 2792208.0, + "reward": -2.86065673828125, + "reward_std": 5.38389778137207, + "rewards/rm_reward_func/mean": -2.86065673828125, + "rewards/rm_reward_func/std": 11.51030158996582, + "step": 215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 397.84375, + "completions/mean_terminated_length": 329.3500061035156, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.1728, + "grad_norm": 3.1028032302856445, + "kl": 0.040283203125, + "learning_rate": 1e-06, + "loss": -0.0738, + "num_tokens": 2807251.0, + "reward": 5.0045166015625, + "reward_std": 5.947755813598633, + "rewards/rm_reward_func/mean": 5.0045166015625, + "rewards/rm_reward_func/std": 6.689188480377197, + "step": 216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 473.0, + "completions/mean_length": 267.875, + "completions/mean_terminated_length": 199.51998901367188, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.1736, + "grad_norm": 12.485280990600586, + "kl": 0.043365478515625, + "learning_rate": 1e-06, + "loss": -0.0573, + "num_tokens": 2822775.0, + "reward": -3.88580322265625, + "reward_std": 5.136831283569336, + "rewards/rm_reward_func/mean": -3.88580322265625, + "rewards/rm_reward_func/std": 11.440207481384277, + "step": 217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 242.875, + "completions/mean_terminated_length": 153.1666717529297, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.1744, + "grad_norm": 8.996060371398926, + "kl": 0.0584716796875, + "learning_rate": 1e-06, + "loss": 0.3665, + "num_tokens": 2833379.0, + "reward": -2.662109375, + "reward_std": 7.626079082489014, + "rewards/rm_reward_func/mean": -2.662109375, + "rewards/rm_reward_func/std": 9.784847259521484, + "step": 218 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 413.40625, + "completions/mean_terminated_length": 301.66668701171875, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.1752, + "grad_norm": 3.5558414459228516, + "kl": 0.02426910400390625, + "learning_rate": 1e-06, + "loss": 0.0093, + "num_tokens": 2849992.0, + "reward": -1.936492919921875, + "reward_std": 7.031126976013184, + "rewards/rm_reward_func/mean": -1.936492919921875, + "rewards/rm_reward_func/std": 12.036778450012207, + "step": 219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 357.40625, + "completions/mean_terminated_length": 305.875, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.176, + "grad_norm": 2.7269515991210938, + "kl": 0.047210693359375, + "learning_rate": 1e-06, + "loss": 0.0937, + "num_tokens": 2865101.0, + "reward": 2.5566253662109375, + "reward_std": 5.476223945617676, + "rewards/rm_reward_func/mean": 2.5566253662109375, + "rewards/rm_reward_func/std": 11.695185661315918, + "step": 220 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 289.53125, + "completions/mean_terminated_length": 274.70001220703125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.1768, + "grad_norm": 5.313303470611572, + "kl": 0.05035400390625, + "learning_rate": 1e-06, + "loss": 0.295, + "num_tokens": 2877494.0, + "reward": 9.74462890625, + "reward_std": 7.827642917633057, + "rewards/rm_reward_func/mean": 9.74462890625, + "rewards/rm_reward_func/std": 13.538431167602539, + "step": 221 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 327.1875, + "completions/mean_terminated_length": 254.86956787109375, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.1776, + "grad_norm": 4.54102087020874, + "kl": 0.0721435546875, + "learning_rate": 1e-06, + "loss": -0.0192, + "num_tokens": 2890372.0, + "reward": 4.88897705078125, + "reward_std": 3.758185386657715, + "rewards/rm_reward_func/mean": 4.88897705078125, + "rewards/rm_reward_func/std": 7.395822525024414, + "step": 222 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 263.65625, + "completions/mean_terminated_length": 237.96551513671875, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.1784, + "grad_norm": 4.374941825866699, + "kl": 0.082183837890625, + "learning_rate": 1e-06, + "loss": 0.0815, + "num_tokens": 2901145.0, + "reward": -4.588043212890625, + "reward_std": 7.087510585784912, + "rewards/rm_reward_func/mean": -4.588043212890625, + "rewards/rm_reward_func/std": 14.971769332885742, + "step": 223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 441.0, + "completions/mean_length": 228.15625, + "completions/mean_terminated_length": 198.79310607910156, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.1792, + "grad_norm": 3.8707516193389893, + "kl": 0.0631103515625, + "learning_rate": 1e-06, + "loss": -0.0906, + "num_tokens": 2911422.0, + "reward": 0.30474853515625, + "reward_std": 6.519827842712402, + "rewards/rm_reward_func/mean": 0.30474853515625, + "rewards/rm_reward_func/std": 7.540925979614258, + "step": 224 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 325.0, + "completions/mean_length": 119.1875, + "completions/mean_terminated_length": 106.51612854003906, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.18, + "grad_norm": 8.087342262268066, + "kl": 0.083251953125, + "learning_rate": 1e-06, + "loss": 0.2222, + "num_tokens": 2920028.0, + "reward": -4.3948974609375, + "reward_std": 5.361183166503906, + "rewards/rm_reward_func/mean": -4.3948974609375, + "rewards/rm_reward_func/std": 10.915719032287598, + "step": 225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 403.0, + "completions/mean_length": 262.78125, + "completions/mean_terminated_length": 205.2692413330078, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.1808, + "grad_norm": 5.448842525482178, + "kl": 0.05535888671875, + "learning_rate": 1e-06, + "loss": 0.1111, + "num_tokens": 2930717.0, + "reward": 1.3837890625, + "reward_std": 7.229766845703125, + "rewards/rm_reward_func/mean": 1.3837890625, + "rewards/rm_reward_func/std": 12.284672737121582, + "step": 226 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 274.03125, + "completions/mean_terminated_length": 240.0357208251953, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.1816, + "grad_norm": 5.66386604309082, + "kl": 0.0574951171875, + "learning_rate": 1e-06, + "loss": 0.2228, + "num_tokens": 2942294.0, + "reward": -2.87408447265625, + "reward_std": 8.044215202331543, + "rewards/rm_reward_func/mean": -2.87408447265625, + "rewards/rm_reward_func/std": 10.464455604553223, + "step": 227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 367.0, + "completions/max_terminated_length": 367.0, + "completions/mean_length": 184.78125, + "completions/mean_terminated_length": 184.78125, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.1824, + "grad_norm": 6.5997467041015625, + "kl": 0.0521240234375, + "learning_rate": 1e-06, + "loss": 0.0741, + "num_tokens": 2953559.0, + "reward": -5.8045654296875, + "reward_std": 4.83258056640625, + "rewards/rm_reward_func/mean": -5.8045654296875, + "rewards/rm_reward_func/std": 9.505318641662598, + "step": 228 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 293.0625, + "completions/mean_terminated_length": 252.51852416992188, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.1832, + "grad_norm": 3.3603475093841553, + "kl": 0.0386962890625, + "learning_rate": 1e-06, + "loss": 0.0101, + "num_tokens": 2966665.0, + "reward": -2.287109375, + "reward_std": 4.542160987854004, + "rewards/rm_reward_func/mean": -2.287109375, + "rewards/rm_reward_func/std": 8.938611030578613, + "step": 229 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 109.0, + "completions/mean_length": 171.875, + "completions/mean_terminated_length": 58.5, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.184, + "grad_norm": 5.757355213165283, + "kl": 0.051544189453125, + "learning_rate": 1e-06, + "loss": -0.0941, + "num_tokens": 2975517.0, + "reward": -1.37554931640625, + "reward_std": 6.1020684242248535, + "rewards/rm_reward_func/mean": -1.37554931640625, + "rewards/rm_reward_func/std": 9.515417098999023, + "step": 230 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 373.875, + "completions/mean_terminated_length": 319.8260803222656, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.1848, + "grad_norm": 2.7484848499298096, + "kl": 0.03887939453125, + "learning_rate": 1e-06, + "loss": -0.047, + "num_tokens": 2989801.0, + "reward": -6.4609375, + "reward_std": 4.946435928344727, + "rewards/rm_reward_func/mean": -6.4609375, + "rewards/rm_reward_func/std": 7.9586381912231445, + "step": 231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 410.0, + "completions/mean_length": 218.03125, + "completions/mean_terminated_length": 163.59259033203125, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "epoch": 0.1856, + "grad_norm": 4.303516864776611, + "kl": 0.0555419921875, + "learning_rate": 1e-06, + "loss": 0.1213, + "num_tokens": 2999034.0, + "reward": -3.778125762939453, + "reward_std": 6.124919891357422, + "rewards/rm_reward_func/mean": -3.778125762939453, + "rewards/rm_reward_func/std": 6.401005744934082, + "step": 232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 341.71875, + "completions/mean_terminated_length": 324.10345458984375, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "epoch": 0.1864, + "grad_norm": 2.5016844272613525, + "kl": 0.0266265869140625, + "learning_rate": 1e-06, + "loss": -0.0613, + "num_tokens": 3012305.0, + "reward": 0.1171875, + "reward_std": 6.036954402923584, + "rewards/rm_reward_func/mean": 0.1171875, + "rewards/rm_reward_func/std": 8.2446870803833, + "step": 233 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 325.0625, + "completions/mean_terminated_length": 240.09091186523438, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.1872, + "grad_norm": 3.126168966293335, + "kl": 0.033935546875, + "learning_rate": 1e-06, + "loss": 0.088, + "num_tokens": 3024995.0, + "reward": -2.0553665161132812, + "reward_std": 4.089500427246094, + "rewards/rm_reward_func/mean": -2.0553665161132812, + "rewards/rm_reward_func/std": 6.073705196380615, + "step": 234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 348.03125, + "completions/mean_terminated_length": 331.0689697265625, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.188, + "grad_norm": 2.8671622276306152, + "kl": 0.03570556640625, + "learning_rate": 1e-06, + "loss": -0.0666, + "num_tokens": 3040716.0, + "reward": 5.6767578125, + "reward_std": 5.897713661193848, + "rewards/rm_reward_func/mean": 5.6767578125, + "rewards/rm_reward_func/std": 9.497367858886719, + "step": 235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 178.53125, + "completions/mean_terminated_length": 144.03448486328125, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.1888, + "grad_norm": 6.242638111114502, + "kl": 0.05078125, + "learning_rate": 1e-06, + "loss": 0.0829, + "num_tokens": 3050005.0, + "reward": -10.707763671875, + "reward_std": 5.651987552642822, + "rewards/rm_reward_func/mean": -10.707763671875, + "rewards/rm_reward_func/std": 7.248280048370361, + "step": 236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 501.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 233.0, + "completions/mean_terminated_length": 233.0, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.1896, + "grad_norm": 3.2471871376037598, + "kl": 0.02520751953125, + "learning_rate": 1e-06, + "loss": -0.0609, + "num_tokens": 3060493.0, + "reward": -0.5721435546875, + "reward_std": 6.302328586578369, + "rewards/rm_reward_func/mean": -0.5721435546875, + "rewards/rm_reward_func/std": 10.102531433105469, + "step": 237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 453.0, + "completions/mean_terminated_length": 354.66668701171875, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "epoch": 0.1904, + "grad_norm": 2.5366711616516113, + "kl": 0.02734375, + "learning_rate": 1e-06, + "loss": 0.0203, + "num_tokens": 3077549.0, + "reward": -0.6923828125, + "reward_std": 4.651421546936035, + "rewards/rm_reward_func/mean": -0.6923828125, + "rewards/rm_reward_func/std": 6.494197368621826, + "step": 238 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 480.0, + "completions/mean_length": 312.65625, + "completions/mean_terminated_length": 266.65386962890625, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.1912, + "grad_norm": 3.5506675243377686, + "kl": 0.038726806640625, + "learning_rate": 1e-06, + "loss": -0.0479, + "num_tokens": 3090450.0, + "reward": -0.1416015625, + "reward_std": 5.028796672821045, + "rewards/rm_reward_func/mean": -0.1416015625, + "rewards/rm_reward_func/std": 7.697935581207275, + "step": 239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 342.75, + "completions/mean_terminated_length": 265.81817626953125, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.192, + "grad_norm": 4.870519161224365, + "kl": 0.02532958984375, + "learning_rate": 1e-06, + "loss": 0.0788, + "num_tokens": 3105794.0, + "reward": -0.5222930908203125, + "reward_std": 7.223459243774414, + "rewards/rm_reward_func/mean": -0.5222930908203125, + "rewards/rm_reward_func/std": 8.785179138183594, + "step": 240 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 459.0, + "completions/mean_length": 328.71875, + "completions/mean_terminated_length": 294.77777099609375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.1928, + "grad_norm": 2.8499858379364014, + "kl": 0.036407470703125, + "learning_rate": 1e-06, + "loss": -0.0227, + "num_tokens": 3119169.0, + "reward": -0.429351806640625, + "reward_std": 4.58777379989624, + "rewards/rm_reward_func/mean": -0.429351806640625, + "rewards/rm_reward_func/std": 6.2184648513793945, + "step": 241 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 417.875, + "completions/mean_terminated_length": 323.75, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.1936, + "grad_norm": 3.2108466625213623, + "kl": 0.0294189453125, + "learning_rate": 1e-06, + "loss": 0.0375, + "num_tokens": 3137933.0, + "reward": -4.9796905517578125, + "reward_std": 5.5578694343566895, + "rewards/rm_reward_func/mean": -4.9796905517578125, + "rewards/rm_reward_func/std": 14.907642364501953, + "step": 242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 221.375, + "completions/mean_terminated_length": 167.55555725097656, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.1944, + "grad_norm": 4.164768695831299, + "kl": 0.061614990234375, + "learning_rate": 1e-06, + "loss": -0.1704, + "num_tokens": 3147577.0, + "reward": -3.182586669921875, + "reward_std": 5.680960655212402, + "rewards/rm_reward_func/mean": -3.182586669921875, + "rewards/rm_reward_func/std": 9.807361602783203, + "step": 243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 308.59375, + "completions/mean_terminated_length": 270.9259338378906, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.1952, + "grad_norm": 4.503950119018555, + "kl": 0.0273284912109375, + "learning_rate": 1e-06, + "loss": -0.1201, + "num_tokens": 3162324.0, + "reward": -5.913942337036133, + "reward_std": 5.996950626373291, + "rewards/rm_reward_func/mean": -5.913942337036133, + "rewards/rm_reward_func/std": 8.33896255493164, + "step": 244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 261.0, + "completions/mean_terminated_length": 252.90321350097656, + "completions/min_length": 15.0, + "completions/min_terminated_length": 15.0, + "epoch": 0.196, + "grad_norm": 32.10102081298828, + "kl": 0.049560546875, + "learning_rate": 1e-06, + "loss": -0.0312, + "num_tokens": 3176892.0, + "reward": 5.21258544921875, + "reward_std": 7.160707473754883, + "rewards/rm_reward_func/mean": 5.21258544921875, + "rewards/rm_reward_func/std": 9.029263496398926, + "step": 245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 350.0, + "completions/mean_length": 174.59375, + "completions/mean_terminated_length": 139.6896514892578, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.1968, + "grad_norm": 7.7096686363220215, + "kl": 0.06005859375, + "learning_rate": 1e-06, + "loss": 0.4931, + "num_tokens": 3185263.0, + "reward": -0.90380859375, + "reward_std": 8.060064315795898, + "rewards/rm_reward_func/mean": -0.90380859375, + "rewards/rm_reward_func/std": 9.48470687866211, + "step": 246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 252.09375, + "completions/mean_terminated_length": 225.20689392089844, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.1976, + "grad_norm": 3.6530377864837646, + "kl": 0.057586669921875, + "learning_rate": 1e-06, + "loss": -0.0903, + "num_tokens": 3196322.0, + "reward": -0.84429931640625, + "reward_std": 3.671651840209961, + "rewards/rm_reward_func/mean": -0.84429931640625, + "rewards/rm_reward_func/std": 5.721739292144775, + "step": 247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 216.875, + "completions/mean_terminated_length": 148.7692413330078, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "epoch": 0.1984, + "grad_norm": 9.012897491455078, + "kl": 0.060882568359375, + "learning_rate": 1e-06, + "loss": -0.0163, + "num_tokens": 3206374.0, + "reward": -1.3134765625, + "reward_std": 1.9922480583190918, + "rewards/rm_reward_func/mean": -1.3134765625, + "rewards/rm_reward_func/std": 9.720147132873535, + "step": 248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 221.75, + "completions/mean_terminated_length": 202.40000915527344, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.1992, + "grad_norm": 4.076515197753906, + "kl": 0.05389404296875, + "learning_rate": 1e-06, + "loss": 0.0904, + "num_tokens": 3217438.0, + "reward": 5.044475555419922, + "reward_std": 5.064403533935547, + "rewards/rm_reward_func/mean": 5.044475555419922, + "rewards/rm_reward_func/std": 11.514410972595215, + "step": 249 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 408.25, + "completions/mean_terminated_length": 327.5555725097656, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.2, + "grad_norm": 2.5681052207946777, + "kl": 0.027313232421875, + "learning_rate": 1e-06, + "loss": 0.0815, + "num_tokens": 3233350.0, + "reward": -2.93603515625, + "reward_std": 3.7280993461608887, + "rewards/rm_reward_func/mean": -2.93603515625, + "rewards/rm_reward_func/std": 12.866559028625488, + "step": 250 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 336.5625, + "completions/mean_terminated_length": 267.9130554199219, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.2008, + "grad_norm": 3.0375123023986816, + "kl": 0.045684814453125, + "learning_rate": 1e-06, + "loss": -0.0519, + "num_tokens": 3248000.0, + "reward": -8.931640625, + "reward_std": 7.065740585327148, + "rewards/rm_reward_func/mean": -8.931640625, + "rewards/rm_reward_func/std": 10.82422924041748, + "step": 251 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 238.0, + "completions/mean_terminated_length": 187.25926208496094, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "epoch": 0.2016, + "grad_norm": 5.324024677276611, + "kl": 0.057220458984375, + "learning_rate": 1e-06, + "loss": 0.158, + "num_tokens": 3259632.0, + "reward": -5.93310546875, + "reward_std": 4.27932071685791, + "rewards/rm_reward_func/mean": -5.93310546875, + "rewards/rm_reward_func/std": 12.716781616210938, + "step": 252 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 321.0, + "completions/mean_length": 147.09375, + "completions/mean_terminated_length": 122.76667022705078, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "epoch": 0.2024, + "grad_norm": 4.747439861297607, + "kl": 0.0233154296875, + "learning_rate": 1e-06, + "loss": -0.0654, + "num_tokens": 3266675.0, + "reward": -6.288330078125, + "reward_std": 5.211904525756836, + "rewards/rm_reward_func/mean": -6.288330078125, + "rewards/rm_reward_func/std": 9.554975509643555, + "step": 253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 319.34375, + "completions/mean_terminated_length": 283.6666564941406, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.2032, + "grad_norm": 3.346832513809204, + "kl": 0.0314178466796875, + "learning_rate": 1e-06, + "loss": 0.0012, + "num_tokens": 3280838.0, + "reward": -1.9285697937011719, + "reward_std": 6.478157043457031, + "rewards/rm_reward_func/mean": -1.9285697937011719, + "rewards/rm_reward_func/std": 10.842815399169922, + "step": 254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 365.46875, + "completions/mean_terminated_length": 360.7419128417969, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "epoch": 0.204, + "grad_norm": 2.939826011657715, + "kl": 0.04241943359375, + "learning_rate": 1e-06, + "loss": -0.0565, + "num_tokens": 3294621.0, + "reward": 10.9114990234375, + "reward_std": 7.665554523468018, + "rewards/rm_reward_func/mean": 10.9114990234375, + "rewards/rm_reward_func/std": 11.80000114440918, + "step": 255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 238.4375, + "completions/mean_terminated_length": 238.4375, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.2048, + "grad_norm": 3.935816764831543, + "kl": 0.0316162109375, + "learning_rate": 1e-06, + "loss": -0.1044, + "num_tokens": 3304547.0, + "reward": -8.1998291015625, + "reward_std": 5.198800086975098, + "rewards/rm_reward_func/mean": -8.1998291015625, + "rewards/rm_reward_func/std": 8.181034088134766, + "step": 256 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 332.09375, + "completions/mean_terminated_length": 313.4827575683594, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.2056, + "grad_norm": 3.129162311553955, + "kl": 0.049652099609375, + "learning_rate": 1e-06, + "loss": 0.0076, + "num_tokens": 3318494.0, + "reward": 5.046142578125, + "reward_std": 4.183533191680908, + "rewards/rm_reward_func/mean": 5.046142578125, + "rewards/rm_reward_func/std": 7.549215793609619, + "step": 257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 254.0, + "completions/mean_terminated_length": 168.0, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.2064, + "grad_norm": 14.513141632080078, + "kl": 0.0689849853515625, + "learning_rate": 1e-06, + "loss": -0.0949, + "num_tokens": 3331886.0, + "reward": -0.634033203125, + "reward_std": 4.15193510055542, + "rewards/rm_reward_func/mean": -0.634033203125, + "rewards/rm_reward_func/std": 10.710453033447266, + "step": 258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 484.0, + "completions/mean_length": 345.96875, + "completions/mean_terminated_length": 290.625, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.2072, + "grad_norm": 3.5183660984039307, + "kl": 0.0648193359375, + "learning_rate": 1e-06, + "loss": 0.1997, + "num_tokens": 3345461.0, + "reward": 2.46270751953125, + "reward_std": 8.18614673614502, + "rewards/rm_reward_func/mean": 2.46270751953125, + "rewards/rm_reward_func/std": 9.014900207519531, + "step": 259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 181.75, + "completions/mean_terminated_length": 181.75, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.208, + "grad_norm": 4.466091632843018, + "kl": 0.07086181640625, + "learning_rate": 1e-06, + "loss": 0.0176, + "num_tokens": 3356853.0, + "reward": -4.78826904296875, + "reward_std": 4.506010055541992, + "rewards/rm_reward_func/mean": -4.78826904296875, + "rewards/rm_reward_func/std": 9.68130111694336, + "step": 260 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 436.0, + "completions/mean_length": 277.53125, + "completions/mean_terminated_length": 185.78260803222656, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.2088, + "grad_norm": 4.487113952636719, + "kl": 0.04511070251464844, + "learning_rate": 1e-06, + "loss": -0.2554, + "num_tokens": 3371294.0, + "reward": -5.722412109375, + "reward_std": 3.237701654434204, + "rewards/rm_reward_func/mean": -5.722412109375, + "rewards/rm_reward_func/std": 7.949067115783691, + "step": 261 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 509.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 186.78125, + "completions/mean_terminated_length": 186.78125, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.2096, + "grad_norm": 5.796142101287842, + "kl": 0.08880615234375, + "learning_rate": 1e-06, + "loss": -0.1566, + "num_tokens": 3379743.0, + "reward": -4.72845458984375, + "reward_std": 7.553511619567871, + "rewards/rm_reward_func/mean": -4.72845458984375, + "rewards/rm_reward_func/std": 11.902534484863281, + "step": 262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 199.625, + "completions/mean_terminated_length": 141.7777862548828, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.2104, + "grad_norm": 7.342236042022705, + "kl": 0.06292724609375, + "learning_rate": 1e-06, + "loss": -0.1884, + "num_tokens": 3390747.0, + "reward": 8.73486328125, + "reward_std": 5.577756404876709, + "rewards/rm_reward_func/mean": 8.73486328125, + "rewards/rm_reward_func/std": 9.168144226074219, + "step": 263 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 284.9375, + "completions/mean_terminated_length": 242.88888549804688, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "epoch": 0.2112, + "grad_norm": 4.4385223388671875, + "kl": 0.0706787109375, + "learning_rate": 1e-06, + "loss": 0.1751, + "num_tokens": 3401873.0, + "reward": 4.10955810546875, + "reward_std": 7.785256862640381, + "rewards/rm_reward_func/mean": 4.10955810546875, + "rewards/rm_reward_func/std": 9.873637199401855, + "step": 264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 309.5625, + "completions/mean_terminated_length": 217.5454559326172, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.212, + "grad_norm": 4.364110469818115, + "kl": 0.05230712890625, + "learning_rate": 1e-06, + "loss": -0.1079, + "num_tokens": 3414907.0, + "reward": -8.052734375, + "reward_std": 4.879029273986816, + "rewards/rm_reward_func/mean": -8.052734375, + "rewards/rm_reward_func/std": 6.013888835906982, + "step": 265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 284.5, + "completions/mean_terminated_length": 148.0, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.2128, + "grad_norm": 6.178150653839111, + "kl": 0.0651397705078125, + "learning_rate": 1e-06, + "loss": 0.2324, + "num_tokens": 3426979.0, + "reward": -0.2777862548828125, + "reward_std": 4.882081985473633, + "rewards/rm_reward_func/mean": -0.2777862548828125, + "rewards/rm_reward_func/std": 12.670929908752441, + "step": 266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 245.78125, + "completions/mean_terminated_length": 171.239990234375, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "epoch": 0.2136, + "grad_norm": 7.719261169433594, + "kl": 0.05291748046875, + "learning_rate": 1e-06, + "loss": 0.0783, + "num_tokens": 3437388.0, + "reward": -0.2825927734375, + "reward_std": 5.749314308166504, + "rewards/rm_reward_func/mean": -0.2825927734375, + "rewards/rm_reward_func/std": 7.607658386230469, + "step": 267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 438.0, + "completions/mean_length": 264.0, + "completions/mean_terminated_length": 228.57144165039062, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.2144, + "grad_norm": 10.929339408874512, + "kl": 0.05267333984375, + "learning_rate": 1e-06, + "loss": -0.0666, + "num_tokens": 3450292.0, + "reward": -1.218994140625, + "reward_std": 3.6973087787628174, + "rewards/rm_reward_func/mean": -1.218994140625, + "rewards/rm_reward_func/std": 10.225122451782227, + "step": 268 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 428.0, + "completions/mean_length": 312.0, + "completions/mean_terminated_length": 245.33334350585938, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.2152, + "grad_norm": 3.8709096908569336, + "kl": 0.03293609619140625, + "learning_rate": 1e-06, + "loss": 0.118, + "num_tokens": 3463604.0, + "reward": -4.028564453125, + "reward_std": 5.055566787719727, + "rewards/rm_reward_func/mean": -4.028564453125, + "rewards/rm_reward_func/std": 13.777055740356445, + "step": 269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 382.0, + "completions/mean_length": 154.9375, + "completions/mean_terminated_length": 143.4193572998047, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.216, + "grad_norm": 4.488786220550537, + "kl": 0.09271240234375, + "learning_rate": 1e-06, + "loss": -0.0088, + "num_tokens": 3473114.0, + "reward": 0.6698150634765625, + "reward_std": 2.86566162109375, + "rewards/rm_reward_func/mean": 0.6698150634765625, + "rewards/rm_reward_func/std": 4.058006286621094, + "step": 270 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 424.0, + "completions/mean_length": 255.3125, + "completions/mean_terminated_length": 169.75, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.2168, + "grad_norm": 6.03592586517334, + "kl": 0.0821533203125, + "learning_rate": 1e-06, + "loss": -0.0379, + "num_tokens": 3485300.0, + "reward": -5.101806640625, + "reward_std": 2.251300573348999, + "rewards/rm_reward_func/mean": -5.101806640625, + "rewards/rm_reward_func/std": 12.82911491394043, + "step": 271 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 174.53125, + "completions/mean_terminated_length": 163.64515686035156, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.2176, + "grad_norm": 5.9694294929504395, + "kl": 0.07928466796875, + "learning_rate": 1e-06, + "loss": 0.1671, + "num_tokens": 3495549.0, + "reward": 1.263671875, + "reward_std": 7.210736274719238, + "rewards/rm_reward_func/mean": 1.263671875, + "rewards/rm_reward_func/std": 12.690206527709961, + "step": 272 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 275.59375, + "completions/mean_terminated_length": 221.03846740722656, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.2184, + "grad_norm": 7.359996795654297, + "kl": 0.0755615234375, + "learning_rate": 1e-06, + "loss": -0.0083, + "num_tokens": 3510912.0, + "reward": 9.77337646484375, + "reward_std": 7.382805824279785, + "rewards/rm_reward_func/mean": 9.77337646484375, + "rewards/rm_reward_func/std": 12.175554275512695, + "step": 273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 482.0, + "completions/mean_length": 209.53125, + "completions/mean_terminated_length": 166.32144165039062, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.2192, + "grad_norm": 4.633665561676025, + "kl": 0.06842041015625, + "learning_rate": 1e-06, + "loss": 0.0266, + "num_tokens": 3521105.0, + "reward": -6.2216796875, + "reward_std": 11.70566177368164, + "rewards/rm_reward_func/mean": -6.2216796875, + "rewards/rm_reward_func/std": 14.380496978759766, + "step": 274 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 308.375, + "completions/mean_terminated_length": 251.36000061035156, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.22, + "grad_norm": 9.020089149475098, + "kl": 0.05230712890625, + "learning_rate": 1e-06, + "loss": 0.0549, + "num_tokens": 3534301.0, + "reward": -1.356536865234375, + "reward_std": 6.025744438171387, + "rewards/rm_reward_func/mean": -1.356536865234375, + "rewards/rm_reward_func/std": 11.09672737121582, + "step": 275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 215.46875, + "completions/mean_terminated_length": 205.90321350097656, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.2208, + "grad_norm": 3.9002256393432617, + "kl": 0.039642333984375, + "learning_rate": 1e-06, + "loss": 0.0256, + "num_tokens": 3544108.0, + "reward": 1.199432373046875, + "reward_std": 4.459294319152832, + "rewards/rm_reward_func/mean": 1.199432373046875, + "rewards/rm_reward_func/std": 5.9614973068237305, + "step": 276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 409.0, + "completions/mean_terminated_length": 347.20001220703125, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.2216, + "grad_norm": 2.783045768737793, + "kl": 0.045440673828125, + "learning_rate": 1e-06, + "loss": -0.0599, + "num_tokens": 3559324.0, + "reward": 5.524658203125, + "reward_std": 7.193890571594238, + "rewards/rm_reward_func/mean": 5.524658203125, + "rewards/rm_reward_func/std": 16.028470993041992, + "step": 277 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 452.0, + "completions/mean_length": 208.5625, + "completions/mean_terminated_length": 177.1724090576172, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.2224, + "grad_norm": 4.956029415130615, + "kl": 0.06280517578125, + "learning_rate": 1e-06, + "loss": -0.0151, + "num_tokens": 3568326.0, + "reward": -1.5927276611328125, + "reward_std": 3.9969515800476074, + "rewards/rm_reward_func/mean": -1.5927276611328125, + "rewards/rm_reward_func/std": 9.469999313354492, + "step": 278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 261.375, + "completions/mean_terminated_length": 147.4545440673828, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.2232, + "grad_norm": 3.7816061973571777, + "kl": 0.062408447265625, + "learning_rate": 1e-06, + "loss": -0.2091, + "num_tokens": 3579634.0, + "reward": -7.9661865234375, + "reward_std": 5.55967378616333, + "rewards/rm_reward_func/mean": -7.9661865234375, + "rewards/rm_reward_func/std": 10.989361763000488, + "step": 279 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 314.46875, + "completions/mean_terminated_length": 195.9499969482422, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.224, + "grad_norm": 3.4097347259521484, + "kl": 0.06787109375, + "learning_rate": 1e-06, + "loss": -0.0373, + "num_tokens": 3592225.0, + "reward": -2.29248046875, + "reward_std": 6.151001930236816, + "rewards/rm_reward_func/mean": -2.29248046875, + "rewards/rm_reward_func/std": 10.61819839477539, + "step": 280 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 346.0, + "completions/mean_length": 225.625, + "completions/mean_terminated_length": 159.53846740722656, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.2248, + "grad_norm": 3.473419427871704, + "kl": 0.08502197265625, + "learning_rate": 1e-06, + "loss": -0.0459, + "num_tokens": 3601253.0, + "reward": -2.1300048828125, + "reward_std": 5.648019790649414, + "rewards/rm_reward_func/mean": -2.1300048828125, + "rewards/rm_reward_func/std": 6.02969217300415, + "step": 281 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 227.625, + "completions/mean_terminated_length": 148.0, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.2256, + "grad_norm": 14.872659683227539, + "kl": 0.08856201171875, + "learning_rate": 1e-06, + "loss": -0.037, + "num_tokens": 3615729.0, + "reward": 3.3761444091796875, + "reward_std": 2.2959201335906982, + "rewards/rm_reward_func/mean": 3.3761444091796875, + "rewards/rm_reward_func/std": 5.013526916503906, + "step": 282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 282.375, + "completions/mean_terminated_length": 239.8518524169922, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.2264, + "grad_norm": 4.269347190856934, + "kl": 0.05474853515625, + "learning_rate": 1e-06, + "loss": -0.0225, + "num_tokens": 3626917.0, + "reward": 5.6048431396484375, + "reward_std": 4.628182411193848, + "rewards/rm_reward_func/mean": 5.6048431396484375, + "rewards/rm_reward_func/std": 8.509807586669922, + "step": 283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/max_terminated_length": 266.0, + "completions/mean_length": 95.375, + "completions/mean_terminated_length": 95.375, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.2272, + "grad_norm": 7.475558757781982, + "kl": 0.13861083984375, + "learning_rate": 1e-06, + "loss": 0.0557, + "num_tokens": 3633521.0, + "reward": 1.9283256530761719, + "reward_std": 1.3042426109313965, + "rewards/rm_reward_func/mean": 1.9283256530761719, + "rewards/rm_reward_func/std": 3.519742250442505, + "step": 284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 194.8125, + "completions/mean_terminated_length": 121.61538696289062, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.228, + "grad_norm": 5.4495086669921875, + "kl": 0.1224365234375, + "learning_rate": 1e-06, + "loss": -0.0286, + "num_tokens": 3642939.0, + "reward": 1.588470458984375, + "reward_std": 2.9584450721740723, + "rewards/rm_reward_func/mean": 1.588470458984375, + "rewards/rm_reward_func/std": 8.191596031188965, + "step": 285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 224.5625, + "completions/mean_terminated_length": 215.29031372070312, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.2288, + "grad_norm": 3.4381351470947266, + "kl": 0.0645751953125, + "learning_rate": 1e-06, + "loss": 0.0433, + "num_tokens": 3653277.0, + "reward": -6.94952392578125, + "reward_std": 3.9081485271453857, + "rewards/rm_reward_func/mean": -6.94952392578125, + "rewards/rm_reward_func/std": 9.789095878601074, + "step": 286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 277.375, + "completions/mean_terminated_length": 253.10345458984375, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.2296, + "grad_norm": 2.947699785232544, + "kl": 0.07073974609375, + "learning_rate": 1e-06, + "loss": -0.0916, + "num_tokens": 3664473.0, + "reward": -3.729705810546875, + "reward_std": 6.659585952758789, + "rewards/rm_reward_func/mean": -3.729705810546875, + "rewards/rm_reward_func/std": 9.191458702087402, + "step": 287 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 453.0, + "completions/mean_length": 217.15625, + "completions/mean_terminated_length": 207.64515686035156, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.2304, + "grad_norm": 6.4276814460754395, + "kl": 0.10601806640625, + "learning_rate": 1e-06, + "loss": 0.3546, + "num_tokens": 3675502.0, + "reward": 1.8604736328125, + "reward_std": 5.1977338790893555, + "rewards/rm_reward_func/mean": 1.8604736328125, + "rewards/rm_reward_func/std": 6.9971723556518555, + "step": 288 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 461.0, + "completions/mean_length": 304.46875, + "completions/mean_terminated_length": 235.2916717529297, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.2312, + "grad_norm": 3.1732778549194336, + "kl": 0.0634765625, + "learning_rate": 1e-06, + "loss": -0.0507, + "num_tokens": 3688813.0, + "reward": 2.79443359375, + "reward_std": 5.335002899169922, + "rewards/rm_reward_func/mean": 2.79443359375, + "rewards/rm_reward_func/std": 9.30492115020752, + "step": 289 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 501.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 171.0625, + "completions/mean_terminated_length": 171.0625, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.232, + "grad_norm": 11.23501968383789, + "kl": 0.07696533203125, + "learning_rate": 1e-06, + "loss": -0.0426, + "num_tokens": 3698575.0, + "reward": 7.9627685546875, + "reward_std": 3.578082323074341, + "rewards/rm_reward_func/mean": 7.9627685546875, + "rewards/rm_reward_func/std": 10.838711738586426, + "step": 290 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 315.625, + "completions/mean_terminated_length": 238.78260803222656, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.2328, + "grad_norm": 5.33342981338501, + "kl": 0.0753173828125, + "learning_rate": 1e-06, + "loss": -0.1215, + "num_tokens": 3710707.0, + "reward": 0.0675048828125, + "reward_std": 6.095547199249268, + "rewards/rm_reward_func/mean": 0.0675048828125, + "rewards/rm_reward_func/std": 8.877041816711426, + "step": 291 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 293.0, + "completions/max_terminated_length": 293.0, + "completions/mean_length": 101.1875, + "completions/mean_terminated_length": 101.1875, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.2336, + "grad_norm": 4.05195951461792, + "kl": 0.118408203125, + "learning_rate": 1e-06, + "loss": -0.0076, + "num_tokens": 3717129.0, + "reward": 2.620485305786133, + "reward_std": 1.9676257371902466, + "rewards/rm_reward_func/mean": 2.620485305786133, + "rewards/rm_reward_func/std": 4.160001754760742, + "step": 292 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 471.0, + "completions/mean_length": 186.375, + "completions/mean_terminated_length": 164.6666717529297, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.2344, + "grad_norm": 2.865370273590088, + "kl": 0.09393310546875, + "learning_rate": 1e-06, + "loss": 0.1097, + "num_tokens": 3725957.0, + "reward": 1.41033935546875, + "reward_std": 3.00130033493042, + "rewards/rm_reward_func/mean": 1.41033935546875, + "rewards/rm_reward_func/std": 9.897608757019043, + "step": 293 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 328.1875, + "completions/mean_terminated_length": 231.90476989746094, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.2352, + "grad_norm": 3.5064573287963867, + "kl": 0.068359375, + "learning_rate": 1e-06, + "loss": -0.1071, + "num_tokens": 3740923.0, + "reward": -3.520263671875, + "reward_std": 12.347634315490723, + "rewards/rm_reward_func/mean": -3.520263671875, + "rewards/rm_reward_func/std": 15.20130729675293, + "step": 294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 274.03125, + "completions/mean_terminated_length": 194.70834350585938, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.236, + "grad_norm": 5.17435884475708, + "kl": 0.0858154296875, + "learning_rate": 1e-06, + "loss": 0.2784, + "num_tokens": 3751876.0, + "reward": 0.765625, + "reward_std": 6.9244384765625, + "rewards/rm_reward_func/mean": 0.765625, + "rewards/rm_reward_func/std": 14.027642250061035, + "step": 295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 200.25, + "completions/mean_terminated_length": 155.71429443359375, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.2368, + "grad_norm": 6.697838306427002, + "kl": 0.0771484375, + "learning_rate": 1e-06, + "loss": 0.232, + "num_tokens": 3761100.0, + "reward": 6.63671875, + "reward_std": 5.848393440246582, + "rewards/rm_reward_func/mean": 6.63671875, + "rewards/rm_reward_func/std": 12.636683464050293, + "step": 296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 239.28125, + "completions/mean_terminated_length": 162.9199981689453, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.2376, + "grad_norm": 2.993525743484497, + "kl": 0.09796142578125, + "learning_rate": 1e-06, + "loss": -0.0469, + "num_tokens": 3773725.0, + "reward": 0.16357421875, + "reward_std": 4.376680850982666, + "rewards/rm_reward_func/mean": 0.16357421875, + "rewards/rm_reward_func/std": 10.634230613708496, + "step": 297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 293.59375, + "completions/mean_terminated_length": 279.0333557128906, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.2384, + "grad_norm": 7.341770172119141, + "kl": 0.09283447265625, + "learning_rate": 1e-06, + "loss": 0.2676, + "num_tokens": 3786040.0, + "reward": 3.809326171875, + "reward_std": 3.955899238586426, + "rewards/rm_reward_func/mean": 3.809326171875, + "rewards/rm_reward_func/std": 4.149126052856445, + "step": 298 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 362.9375, + "completions/mean_terminated_length": 328.5384826660156, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.2392, + "grad_norm": 2.8502986431121826, + "kl": 0.05450439453125, + "learning_rate": 1e-06, + "loss": -0.0865, + "num_tokens": 3800550.0, + "reward": -2.6076202392578125, + "reward_std": 6.401535987854004, + "rewards/rm_reward_func/mean": -2.6076202392578125, + "rewards/rm_reward_func/std": 12.927457809448242, + "step": 299 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 333.0, + "completions/mean_terminated_length": 299.85186767578125, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.24, + "grad_norm": 3.4198484420776367, + "kl": 0.087646484375, + "learning_rate": 1e-06, + "loss": -0.0911, + "num_tokens": 3813398.0, + "reward": 10.27294921875, + "reward_std": 8.61447525024414, + "rewards/rm_reward_func/mean": 10.27294921875, + "rewards/rm_reward_func/std": 12.064411163330078, + "step": 300 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 315.0, + "completions/mean_length": 226.9375, + "completions/mean_terminated_length": 174.1481475830078, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.2408, + "grad_norm": 20.889190673828125, + "kl": 0.064208984375, + "learning_rate": 1e-06, + "loss": 0.2754, + "num_tokens": 3824964.0, + "reward": 4.1443328857421875, + "reward_std": 4.807868003845215, + "rewards/rm_reward_func/mean": 4.1443328857421875, + "rewards/rm_reward_func/std": 10.203561782836914, + "step": 301 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 355.03125, + "completions/mean_terminated_length": 293.60870361328125, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.2416, + "grad_norm": 3.480130910873413, + "kl": 0.0916748046875, + "learning_rate": 1e-06, + "loss": 0.1964, + "num_tokens": 3839741.0, + "reward": 0.181640625, + "reward_std": 6.749973773956299, + "rewards/rm_reward_func/mean": 0.181640625, + "rewards/rm_reward_func/std": 8.411787986755371, + "step": 302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 469.0, + "completions/mean_length": 285.96875, + "completions/mean_terminated_length": 167.57142639160156, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.2424, + "grad_norm": 5.95459508895874, + "kl": 0.073211669921875, + "learning_rate": 1e-06, + "loss": 0.0174, + "num_tokens": 3852788.0, + "reward": -7.36474609375, + "reward_std": 2.4573302268981934, + "rewards/rm_reward_func/mean": -7.36474609375, + "rewards/rm_reward_func/std": 10.699016571044922, + "step": 303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 293.40625, + "completions/mean_terminated_length": 242.9615478515625, + "completions/min_length": 14.0, + "completions/min_terminated_length": 14.0, + "epoch": 0.2432, + "grad_norm": 3.7543785572052, + "kl": 0.0772705078125, + "learning_rate": 1e-06, + "loss": 0.2814, + "num_tokens": 3864457.0, + "reward": 4.44287109375, + "reward_std": 8.432929039001465, + "rewards/rm_reward_func/mean": 4.44287109375, + "rewards/rm_reward_func/std": 20.573001861572266, + "step": 304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 198.5, + "completions/mean_terminated_length": 188.3870849609375, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.244, + "grad_norm": 3.925734281539917, + "kl": 0.0897216796875, + "learning_rate": 1e-06, + "loss": 0.0502, + "num_tokens": 3874617.0, + "reward": -5.2659912109375, + "reward_std": 3.777980327606201, + "rewards/rm_reward_func/mean": -5.2659912109375, + "rewards/rm_reward_func/std": 7.426268577575684, + "step": 305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 472.0, + "completions/mean_length": 257.0, + "completions/mean_terminated_length": 248.77418518066406, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.2448, + "grad_norm": 5.314541816711426, + "kl": 0.0880126953125, + "learning_rate": 1e-06, + "loss": -0.1034, + "num_tokens": 3885481.0, + "reward": -1.97320556640625, + "reward_std": 4.8526153564453125, + "rewards/rm_reward_func/mean": -1.97320556640625, + "rewards/rm_reward_func/std": 8.424205780029297, + "step": 306 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 464.0, + "completions/max_terminated_length": 464.0, + "completions/mean_length": 241.5, + "completions/mean_terminated_length": 241.5, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.2456, + "grad_norm": 3.4764163494110107, + "kl": 0.1007080078125, + "learning_rate": 1e-06, + "loss": 0.0276, + "num_tokens": 3895369.0, + "reward": 14.22119140625, + "reward_std": 3.785148859024048, + "rewards/rm_reward_func/mean": 14.22119140625, + "rewards/rm_reward_func/std": 10.576017379760742, + "step": 307 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 371.0, + "completions/mean_length": 297.125, + "completions/mean_terminated_length": 225.5, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.2464, + "grad_norm": 3.061163902282715, + "kl": 0.042938232421875, + "learning_rate": 1e-06, + "loss": 0.0031, + "num_tokens": 3907725.0, + "reward": -2.1278076171875, + "reward_std": 2.4199328422546387, + "rewards/rm_reward_func/mean": -2.1278076171875, + "rewards/rm_reward_func/std": 5.059024333953857, + "step": 308 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 461.0, + "completions/mean_length": 273.6875, + "completions/mean_terminated_length": 266.0, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.2472, + "grad_norm": 3.6154208183288574, + "kl": 0.08709716796875, + "learning_rate": 1e-06, + "loss": -0.0281, + "num_tokens": 3918539.0, + "reward": 5.895008087158203, + "reward_std": 3.205272912979126, + "rewards/rm_reward_func/mean": 5.895008087158203, + "rewards/rm_reward_func/std": 7.15859317779541, + "step": 309 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 410.0, + "completions/mean_length": 283.6875, + "completions/mean_terminated_length": 231.00001525878906, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "epoch": 0.248, + "grad_norm": 2.8838815689086914, + "kl": 0.035888671875, + "learning_rate": 1e-06, + "loss": -0.0877, + "num_tokens": 3932985.0, + "reward": -2.539306640625, + "reward_std": 4.815282344818115, + "rewards/rm_reward_func/mean": -2.539306640625, + "rewards/rm_reward_func/std": 8.142019271850586, + "step": 310 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 295.1875, + "completions/mean_terminated_length": 288.19354248046875, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.2488, + "grad_norm": 3.6053807735443115, + "kl": 0.07330322265625, + "learning_rate": 1e-06, + "loss": -0.0069, + "num_tokens": 3944983.0, + "reward": 5.669269561767578, + "reward_std": 5.6011061668396, + "rewards/rm_reward_func/mean": 5.669269561767578, + "rewards/rm_reward_func/std": 10.164783477783203, + "step": 311 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 277.4375, + "completions/mean_terminated_length": 261.8000183105469, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.2496, + "grad_norm": 4.000564098358154, + "kl": 0.091552734375, + "learning_rate": 1e-06, + "loss": 0.0693, + "num_tokens": 3957221.0, + "reward": 6.09881591796875, + "reward_std": 6.87520170211792, + "rewards/rm_reward_func/mean": 6.09881591796875, + "rewards/rm_reward_func/std": 14.669084548950195, + "step": 312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 439.0, + "completions/mean_length": 151.09375, + "completions/mean_terminated_length": 113.75862121582031, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.2504, + "grad_norm": 3.9963266849517822, + "kl": 0.10784912109375, + "learning_rate": 1e-06, + "loss": -0.0074, + "num_tokens": 3966792.0, + "reward": 2.5313186645507812, + "reward_std": 3.022167682647705, + "rewards/rm_reward_func/mean": 2.5313186645507812, + "rewards/rm_reward_func/std": 6.7323784828186035, + "step": 313 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 353.5, + "completions/mean_terminated_length": 300.66668701171875, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.2512, + "grad_norm": 7.695732116699219, + "kl": 0.05596923828125, + "learning_rate": 1e-06, + "loss": -0.145, + "num_tokens": 3980432.0, + "reward": -0.392333984375, + "reward_std": 5.235438346862793, + "rewards/rm_reward_func/mean": -0.392333984375, + "rewards/rm_reward_func/std": 11.274943351745605, + "step": 314 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 447.0, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 226.90625, + "completions/mean_terminated_length": 226.90625, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.252, + "grad_norm": 19.040664672851562, + "kl": 0.09423828125, + "learning_rate": 1e-06, + "loss": -0.0073, + "num_tokens": 3993069.0, + "reward": 3.00018310546875, + "reward_std": 2.8838400840759277, + "rewards/rm_reward_func/mean": 3.00018310546875, + "rewards/rm_reward_func/std": 4.894301414489746, + "step": 315 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 191.78125, + "completions/mean_terminated_length": 181.4516143798828, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.2528, + "grad_norm": 12.309869766235352, + "kl": 0.119140625, + "learning_rate": 1e-06, + "loss": 0.2599, + "num_tokens": 4004622.0, + "reward": 5.98388671875, + "reward_std": 5.505092144012451, + "rewards/rm_reward_func/mean": 5.98388671875, + "rewards/rm_reward_func/std": 7.081192970275879, + "step": 316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 239.34375, + "completions/mean_terminated_length": 211.13792419433594, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.2536, + "grad_norm": 5.658858299255371, + "kl": 0.09930419921875, + "learning_rate": 1e-06, + "loss": -0.0048, + "num_tokens": 4015681.0, + "reward": -1.23388671875, + "reward_std": 4.671802520751953, + "rewards/rm_reward_func/mean": -1.23388671875, + "rewards/rm_reward_func/std": 10.551041603088379, + "step": 317 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 251.28125, + "completions/mean_terminated_length": 191.11538696289062, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.2544, + "grad_norm": 3.543397903442383, + "kl": 0.0836181640625, + "learning_rate": 1e-06, + "loss": -0.0158, + "num_tokens": 4026234.0, + "reward": -0.675323486328125, + "reward_std": 2.8310508728027344, + "rewards/rm_reward_func/mean": -0.675323486328125, + "rewards/rm_reward_func/std": 6.245419502258301, + "step": 318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 178.8125, + "completions/mean_terminated_length": 178.8125, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.2552, + "grad_norm": 4.103335380554199, + "kl": 0.115234375, + "learning_rate": 1e-06, + "loss": 0.049, + "num_tokens": 4035396.0, + "reward": 3.646728515625, + "reward_std": 3.5850768089294434, + "rewards/rm_reward_func/mean": 3.646728515625, + "rewards/rm_reward_func/std": 9.643186569213867, + "step": 319 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.0, + "completions/max_terminated_length": 229.0, + "completions/mean_length": 116.25, + "completions/mean_terminated_length": 116.25, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.256, + "grad_norm": 3.8824329376220703, + "kl": 0.14990234375, + "learning_rate": 1e-06, + "loss": -0.0005, + "num_tokens": 4041572.0, + "reward": -1.460205078125, + "reward_std": 2.3325414657592773, + "rewards/rm_reward_func/mean": -1.460205078125, + "rewards/rm_reward_func/std": 8.575539588928223, + "step": 320 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 319.9375, + "completions/mean_terminated_length": 255.9166717529297, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.2568, + "grad_norm": 8.91545295715332, + "kl": 0.07806396484375, + "learning_rate": 1e-06, + "loss": 0.0173, + "num_tokens": 4054298.0, + "reward": 2.38446044921875, + "reward_std": 3.706024169921875, + "rewards/rm_reward_func/mean": 2.38446044921875, + "rewards/rm_reward_func/std": 6.616323471069336, + "step": 321 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 269.375, + "completions/mean_terminated_length": 224.44444274902344, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.2576, + "grad_norm": 4.994580268859863, + "kl": 0.15380859375, + "learning_rate": 1e-06, + "loss": 0.0178, + "num_tokens": 4065390.0, + "reward": 7.4681549072265625, + "reward_std": 3.772608518600464, + "rewards/rm_reward_func/mean": 7.4681549072265625, + "rewards/rm_reward_func/std": 8.73366641998291, + "step": 322 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 306.84375, + "completions/mean_terminated_length": 238.45834350585938, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.2584, + "grad_norm": 6.513478755950928, + "kl": 0.08917236328125, + "learning_rate": 1e-06, + "loss": 0.4623, + "num_tokens": 4079945.0, + "reward": 1.5455856323242188, + "reward_std": 8.046390533447266, + "rewards/rm_reward_func/mean": 1.5455856323242188, + "rewards/rm_reward_func/std": 9.18606185913086, + "step": 323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 308.5625, + "completions/mean_terminated_length": 295.0000305175781, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.2592, + "grad_norm": 3.7793478965759277, + "kl": 0.08447265625, + "learning_rate": 1e-06, + "loss": -0.0834, + "num_tokens": 4091579.0, + "reward": 2.5294189453125, + "reward_std": 5.412877559661865, + "rewards/rm_reward_func/mean": 2.5294189453125, + "rewards/rm_reward_func/std": 6.231657028198242, + "step": 324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 264.53125, + "completions/mean_terminated_length": 218.70370483398438, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.26, + "grad_norm": 15.98507022857666, + "kl": 0.10076904296875, + "learning_rate": 1e-06, + "loss": 0.2493, + "num_tokens": 4105692.0, + "reward": 3.5554351806640625, + "reward_std": 5.774120330810547, + "rewards/rm_reward_func/mean": 3.5554351806640625, + "rewards/rm_reward_func/std": 7.647456169128418, + "step": 325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 323.625, + "completions/mean_terminated_length": 304.137939453125, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.2608, + "grad_norm": 5.084076881408691, + "kl": 0.070556640625, + "learning_rate": 1e-06, + "loss": -0.157, + "num_tokens": 4119744.0, + "reward": 6.6622314453125, + "reward_std": 5.713457107543945, + "rewards/rm_reward_func/mean": 6.6622314453125, + "rewards/rm_reward_func/std": 9.382472038269043, + "step": 326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 469.0, + "completions/mean_length": 380.6875, + "completions/mean_terminated_length": 301.8999938964844, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.2616, + "grad_norm": 7.414204120635986, + "kl": 0.09124755859375, + "learning_rate": 1e-06, + "loss": -0.1516, + "num_tokens": 4134790.0, + "reward": -4.882232666015625, + "reward_std": 4.5240888595581055, + "rewards/rm_reward_func/mean": -4.882232666015625, + "rewards/rm_reward_func/std": 9.277235984802246, + "step": 327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 295.09375, + "completions/mean_terminated_length": 254.92593383789062, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.2624, + "grad_norm": 25.824256896972656, + "kl": 0.07318115234375, + "learning_rate": 1e-06, + "loss": 0.0039, + "num_tokens": 4149569.0, + "reward": 5.16973876953125, + "reward_std": 6.8412652015686035, + "rewards/rm_reward_func/mean": 5.16973876953125, + "rewards/rm_reward_func/std": 9.498034477233887, + "step": 328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 410.5, + "completions/mean_terminated_length": 364.3636474609375, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.2632, + "grad_norm": 2.756131172180176, + "kl": 0.0433349609375, + "learning_rate": 1e-06, + "loss": -0.1067, + "num_tokens": 4164521.0, + "reward": 0.5872802734375, + "reward_std": 5.443880558013916, + "rewards/rm_reward_func/mean": 0.5872802734375, + "rewards/rm_reward_func/std": 6.557459354400635, + "step": 329 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 302.65625, + "completions/mean_terminated_length": 263.8888854980469, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.264, + "grad_norm": 5.988248348236084, + "kl": 0.10546875, + "learning_rate": 1e-06, + "loss": -0.078, + "num_tokens": 4176206.0, + "reward": -2.9931640625, + "reward_std": 5.075013160705566, + "rewards/rm_reward_func/mean": -2.9931640625, + "rewards/rm_reward_func/std": 7.403041839599609, + "step": 330 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 316.4375, + "completions/mean_terminated_length": 227.5454559326172, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.2648, + "grad_norm": 8.037940979003906, + "kl": 0.087493896484375, + "learning_rate": 1e-06, + "loss": -0.0889, + "num_tokens": 4188604.0, + "reward": -0.03961181640625, + "reward_std": 5.210146903991699, + "rewards/rm_reward_func/mean": -0.03961181640625, + "rewards/rm_reward_func/std": 6.811439514160156, + "step": 331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 352.6875, + "completions/mean_terminated_length": 342.0666809082031, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.2656, + "grad_norm": 3.725177049636841, + "kl": 0.088134765625, + "learning_rate": 1e-06, + "loss": -0.0404, + "num_tokens": 4201738.0, + "reward": 2.9521484375, + "reward_std": 6.857073783874512, + "rewards/rm_reward_func/mean": 2.9521484375, + "rewards/rm_reward_func/std": 7.544101238250732, + "step": 332 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 386.8125, + "completions/mean_terminated_length": 345.0833435058594, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "epoch": 0.2664, + "grad_norm": 2.696730613708496, + "kl": 0.084686279296875, + "learning_rate": 1e-06, + "loss": -0.0032, + "num_tokens": 4216772.0, + "reward": 5.23828125, + "reward_std": 5.675407409667969, + "rewards/rm_reward_func/mean": 5.23828125, + "rewards/rm_reward_func/std": 6.8515472412109375, + "step": 333 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 417.0, + "completions/mean_length": 214.78125, + "completions/mean_terminated_length": 205.19354248046875, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.2672, + "grad_norm": 4.98583459854126, + "kl": 0.0863037109375, + "learning_rate": 1e-06, + "loss": -0.0115, + "num_tokens": 4229349.0, + "reward": -2.3466796875, + "reward_std": 5.685585021972656, + "rewards/rm_reward_func/mean": -2.3466796875, + "rewards/rm_reward_func/std": 7.332208633422852, + "step": 334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 408.78125, + "completions/mean_terminated_length": 236.75, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.268, + "grad_norm": 3.633908987045288, + "kl": 0.0657958984375, + "learning_rate": 1e-06, + "loss": 0.0563, + "num_tokens": 4247326.0, + "reward": 1.4497833251953125, + "reward_std": 5.859368801116943, + "rewards/rm_reward_func/mean": 1.4497833251953125, + "rewards/rm_reward_func/std": 7.945054054260254, + "step": 335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 398.375, + "completions/mean_terminated_length": 377.3333435058594, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.2688, + "grad_norm": 6.791285514831543, + "kl": 0.0994873046875, + "learning_rate": 1e-06, + "loss": -0.0843, + "num_tokens": 4263082.0, + "reward": 1.28485107421875, + "reward_std": 5.6787285804748535, + "rewards/rm_reward_func/mean": 1.28485107421875, + "rewards/rm_reward_func/std": 12.219388961791992, + "step": 336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 452.0, + "completions/mean_length": 312.9375, + "completions/mean_terminated_length": 299.66668701171875, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.2696, + "grad_norm": 3.929457902908325, + "kl": 0.0523681640625, + "learning_rate": 1e-06, + "loss": 0.014, + "num_tokens": 4275208.0, + "reward": 7.05194091796875, + "reward_std": 5.769165992736816, + "rewards/rm_reward_func/mean": 7.05194091796875, + "rewards/rm_reward_func/std": 16.998645782470703, + "step": 337 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 456.34375, + "completions/mean_terminated_length": 422.95001220703125, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "epoch": 0.2704, + "grad_norm": 4.277588844299316, + "kl": 0.04718017578125, + "learning_rate": 1e-06, + "loss": -0.0212, + "num_tokens": 4293051.0, + "reward": -1.598388671875, + "reward_std": 5.49388313293457, + "rewards/rm_reward_func/mean": -1.598388671875, + "rewards/rm_reward_func/std": 19.015661239624023, + "step": 338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 259.65625, + "completions/mean_terminated_length": 251.51612854003906, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.2712, + "grad_norm": 26.45867919921875, + "kl": 0.097412109375, + "learning_rate": 1e-06, + "loss": 0.1121, + "num_tokens": 4309008.0, + "reward": 1.324951171875, + "reward_std": 5.575651168823242, + "rewards/rm_reward_func/mean": 1.324951171875, + "rewards/rm_reward_func/std": 11.64552116394043, + "step": 339 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 341.15625, + "completions/mean_terminated_length": 284.2083435058594, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.272, + "grad_norm": 4.051329135894775, + "kl": 0.10674476623535156, + "learning_rate": 1e-06, + "loss": -0.0366, + "num_tokens": 4325749.0, + "reward": 12.0419921875, + "reward_std": 3.590223550796509, + "rewards/rm_reward_func/mean": 12.0419921875, + "rewards/rm_reward_func/std": 17.978811264038086, + "step": 340 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 418.0, + "completions/mean_length": 245.375, + "completions/mean_terminated_length": 217.79310607910156, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.2728, + "grad_norm": 8.272236824035645, + "kl": 0.07733154296875, + "learning_rate": 1e-06, + "loss": 0.3771, + "num_tokens": 4337033.0, + "reward": 0.50048828125, + "reward_std": 5.649558067321777, + "rewards/rm_reward_func/mean": 0.50048828125, + "rewards/rm_reward_func/std": 5.990384101867676, + "step": 341 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 333.78125, + "completions/mean_terminated_length": 264.0434875488281, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.2736, + "grad_norm": 3.4469985961914062, + "kl": 0.06951904296875, + "learning_rate": 1e-06, + "loss": 0.0304, + "num_tokens": 4349922.0, + "reward": 3.2762298583984375, + "reward_std": 8.944576263427734, + "rewards/rm_reward_func/mean": 3.2762298583984375, + "rewards/rm_reward_func/std": 14.431233406066895, + "step": 342 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 140.0, + "completions/max_terminated_length": 140.0, + "completions/mean_length": 68.78125, + "completions/mean_terminated_length": 68.78125, + "completions/min_length": 14.0, + "completions/min_terminated_length": 14.0, + "epoch": 0.2744, + "grad_norm": 12.23254680633545, + "kl": 0.1826171875, + "learning_rate": 1e-06, + "loss": 0.1112, + "num_tokens": 4355779.0, + "reward": -2.1884765625, + "reward_std": 3.757051944732666, + "rewards/rm_reward_func/mean": -2.1884765625, + "rewards/rm_reward_func/std": 9.511407852172852, + "step": 343 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 284.78125, + "completions/mean_terminated_length": 232.34616088867188, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.2752, + "grad_norm": 5.746858596801758, + "kl": 0.10980224609375, + "learning_rate": 1e-06, + "loss": -0.1269, + "num_tokens": 4368244.0, + "reward": 0.2880859375, + "reward_std": 3.1070127487182617, + "rewards/rm_reward_func/mean": 0.2880859375, + "rewards/rm_reward_func/std": 14.436668395996094, + "step": 344 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 388.0, + "completions/max_terminated_length": 388.0, + "completions/mean_length": 158.59375, + "completions/mean_terminated_length": 158.59375, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.276, + "grad_norm": 6.810152053833008, + "kl": 0.12841796875, + "learning_rate": 1e-06, + "loss": -0.1169, + "num_tokens": 4378415.0, + "reward": 2.64385986328125, + "reward_std": 4.024966716766357, + "rewards/rm_reward_func/mean": 2.64385986328125, + "rewards/rm_reward_func/std": 8.710803985595703, + "step": 345 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 486.0, + "completions/mean_length": 300.15625, + "completions/mean_terminated_length": 189.1904754638672, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.2768, + "grad_norm": 4.056140422821045, + "kl": 0.13568115234375, + "learning_rate": 1e-06, + "loss": 0.0213, + "num_tokens": 4390156.0, + "reward": 8.573406219482422, + "reward_std": 4.77297306060791, + "rewards/rm_reward_func/mean": 8.573406219482422, + "rewards/rm_reward_func/std": 5.832368850708008, + "step": 346 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 387.3125, + "completions/mean_terminated_length": 345.75, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "epoch": 0.2776, + "grad_norm": 3.6153457164764404, + "kl": 0.0836181640625, + "learning_rate": 1e-06, + "loss": -0.0447, + "num_tokens": 4405446.0, + "reward": 7.1898193359375, + "reward_std": 6.768939971923828, + "rewards/rm_reward_func/mean": 7.1898193359375, + "rewards/rm_reward_func/std": 12.574838638305664, + "step": 347 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 436.0, + "completions/mean_length": 319.53125, + "completions/mean_terminated_length": 204.0500030517578, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.2784, + "grad_norm": 38.247314453125, + "kl": 0.071136474609375, + "learning_rate": 1e-06, + "loss": -0.0091, + "num_tokens": 4419767.0, + "reward": -6.6529541015625, + "reward_std": 5.824492454528809, + "rewards/rm_reward_func/mean": -6.6529541015625, + "rewards/rm_reward_func/std": 8.597187042236328, + "step": 348 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 477.0, + "completions/mean_length": 318.875, + "completions/mean_terminated_length": 298.89654541015625, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.2792, + "grad_norm": 4.058663845062256, + "kl": 0.082733154296875, + "learning_rate": 1e-06, + "loss": -0.0386, + "num_tokens": 4432003.0, + "reward": -0.1033935546875, + "reward_std": 8.34597396850586, + "rewards/rm_reward_func/mean": -0.1033935546875, + "rewards/rm_reward_func/std": 15.697948455810547, + "step": 349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 368.59375, + "completions/mean_terminated_length": 282.5500183105469, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.28, + "grad_norm": 27.752031326293945, + "kl": 0.07891845703125, + "learning_rate": 1e-06, + "loss": -0.047, + "num_tokens": 4449870.0, + "reward": 0.47021484375, + "reward_std": 6.845122337341309, + "rewards/rm_reward_func/mean": 0.47021484375, + "rewards/rm_reward_func/std": 11.682084083557129, + "step": 350 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 242.75, + "completions/mean_terminated_length": 234.06451416015625, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.2808, + "grad_norm": 4.997708797454834, + "kl": 0.1334228515625, + "learning_rate": 1e-06, + "loss": -0.0635, + "num_tokens": 4462158.0, + "reward": 7.95166015625, + "reward_std": 8.451520919799805, + "rewards/rm_reward_func/mean": 7.95166015625, + "rewards/rm_reward_func/std": 13.134011268615723, + "step": 351 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 266.6875, + "completions/mean_terminated_length": 170.69564819335938, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.2816, + "grad_norm": 11.950886726379395, + "kl": 0.12762451171875, + "learning_rate": 1e-06, + "loss": 0.0629, + "num_tokens": 4473372.0, + "reward": 2.12890625, + "reward_std": 10.61873722076416, + "rewards/rm_reward_func/mean": 2.12890625, + "rewards/rm_reward_func/std": 17.77381706237793, + "step": 352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 315.84375, + "completions/mean_terminated_length": 260.91998291015625, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.2824, + "grad_norm": 9.539039611816406, + "kl": 0.135009765625, + "learning_rate": 1e-06, + "loss": 0.2418, + "num_tokens": 4486855.0, + "reward": 10.563232421875, + "reward_std": 8.17442512512207, + "rewards/rm_reward_func/mean": 10.563232421875, + "rewards/rm_reward_func/std": 15.37407398223877, + "step": 353 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 388.25, + "completions/mean_terminated_length": 353.6000061035156, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.2832, + "grad_norm": 3.476372003555298, + "kl": 0.1109619140625, + "learning_rate": 1e-06, + "loss": -0.0289, + "num_tokens": 4501519.0, + "reward": 19.090087890625, + "reward_std": 6.380127906799316, + "rewards/rm_reward_func/mean": 19.090087890625, + "rewards/rm_reward_func/std": 11.411051750183105, + "step": 354 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 316.96875, + "completions/mean_terminated_length": 262.3599853515625, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.284, + "grad_norm": 5.603827953338623, + "kl": 0.129150390625, + "learning_rate": 1e-06, + "loss": -0.0998, + "num_tokens": 4514398.0, + "reward": -2.05572509765625, + "reward_std": 5.069465637207031, + "rewards/rm_reward_func/mean": -2.05572509765625, + "rewards/rm_reward_func/std": 7.577589988708496, + "step": 355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 369.90625, + "completions/mean_terminated_length": 330.1199951171875, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.2848, + "grad_norm": 3.7223684787750244, + "kl": 0.1007080078125, + "learning_rate": 1e-06, + "loss": 0.011, + "num_tokens": 4528347.0, + "reward": 1.4289093017578125, + "reward_std": 5.715549468994141, + "rewards/rm_reward_func/mean": 1.4289093017578125, + "rewards/rm_reward_func/std": 14.303176879882812, + "step": 356 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 335.28125, + "completions/mean_terminated_length": 323.5000305175781, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.2856, + "grad_norm": 4.0376200675964355, + "kl": 0.0750732421875, + "learning_rate": 1e-06, + "loss": -0.0575, + "num_tokens": 4541772.0, + "reward": 6.256103515625, + "reward_std": 8.777978897094727, + "rewards/rm_reward_func/mean": 6.256103515625, + "rewards/rm_reward_func/std": 14.211292266845703, + "step": 357 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 278.75, + "completions/mean_terminated_length": 235.55555725097656, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.2864, + "grad_norm": 4.349524021148682, + "kl": 0.197265625, + "learning_rate": 1e-06, + "loss": 0.0664, + "num_tokens": 4552908.0, + "reward": 5.4974365234375, + "reward_std": 5.0523176193237305, + "rewards/rm_reward_func/mean": 5.4974365234375, + "rewards/rm_reward_func/std": 15.988303184509277, + "step": 358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 484.0, + "completions/mean_length": 248.75, + "completions/mean_terminated_length": 240.258056640625, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.2872, + "grad_norm": 4.672142028808594, + "kl": 0.200927734375, + "learning_rate": 1e-06, + "loss": 0.0244, + "num_tokens": 4563372.0, + "reward": -1.47747802734375, + "reward_std": 4.878537654876709, + "rewards/rm_reward_func/mean": -1.47747802734375, + "rewards/rm_reward_func/std": 9.86972713470459, + "step": 359 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 277.15625, + "completions/mean_terminated_length": 211.39999389648438, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.288, + "grad_norm": 6.796772480010986, + "kl": 0.0882568359375, + "learning_rate": 1e-06, + "loss": -0.0351, + "num_tokens": 4574377.0, + "reward": 1.7371826171875, + "reward_std": 6.135521411895752, + "rewards/rm_reward_func/mean": 1.7371826171875, + "rewards/rm_reward_func/std": 14.885226249694824, + "step": 360 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 347.6875, + "completions/mean_terminated_length": 336.73333740234375, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.2888, + "grad_norm": 7.4351806640625, + "kl": 0.1314697265625, + "learning_rate": 1e-06, + "loss": -0.1156, + "num_tokens": 4587791.0, + "reward": -0.7554969787597656, + "reward_std": 5.259791851043701, + "rewards/rm_reward_func/mean": -0.7554969787597656, + "rewards/rm_reward_func/std": 12.53715991973877, + "step": 361 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 321.59375, + "completions/mean_terminated_length": 294.39288330078125, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.2896, + "grad_norm": 5.127910614013672, + "kl": 0.08160400390625, + "learning_rate": 1e-06, + "loss": -0.0329, + "num_tokens": 4600466.0, + "reward": 1.8590850830078125, + "reward_std": 7.74429988861084, + "rewards/rm_reward_func/mean": 1.8590850830078125, + "rewards/rm_reward_func/std": 14.955947875976562, + "step": 362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 460.0, + "completions/mean_length": 169.59375, + "completions/mean_terminated_length": 158.5483856201172, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.2904, + "grad_norm": 4.208690643310547, + "kl": 0.1861572265625, + "learning_rate": 1e-06, + "loss": 0.0933, + "num_tokens": 4609349.0, + "reward": 3.2529525756835938, + "reward_std": 2.592770576477051, + "rewards/rm_reward_func/mean": 3.2529525756835938, + "rewards/rm_reward_func/std": 9.945049285888672, + "step": 363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 419.96875, + "completions/mean_terminated_length": 364.75, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.2912, + "grad_norm": 3.5521671772003174, + "kl": 0.079864501953125, + "learning_rate": 1e-06, + "loss": -0.1582, + "num_tokens": 4626428.0, + "reward": 2.2109375, + "reward_std": 9.090686798095703, + "rewards/rm_reward_func/mean": 2.2109375, + "rewards/rm_reward_func/std": 24.739288330078125, + "step": 364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 237.53125, + "completions/mean_terminated_length": 209.13792419433594, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.292, + "grad_norm": 6.295807838439941, + "kl": 0.051605224609375, + "learning_rate": 1e-06, + "loss": 0.0554, + "num_tokens": 4639565.0, + "reward": -10.0103759765625, + "reward_std": 5.414639472961426, + "rewards/rm_reward_func/mean": -10.0103759765625, + "rewards/rm_reward_func/std": 11.643550872802734, + "step": 365 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 335.84375, + "completions/mean_terminated_length": 286.5199890136719, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.2928, + "grad_norm": 4.325115203857422, + "kl": 0.0908203125, + "learning_rate": 1e-06, + "loss": -0.0157, + "num_tokens": 4652608.0, + "reward": 13.1593017578125, + "reward_std": 7.079737663269043, + "rewards/rm_reward_func/mean": 13.1593017578125, + "rewards/rm_reward_func/std": 17.48756980895996, + "step": 366 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 364.25, + "completions/mean_terminated_length": 275.6000061035156, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.2936, + "grad_norm": 4.418249130249023, + "kl": 0.169189453125, + "learning_rate": 1e-06, + "loss": 0.1931, + "num_tokens": 4667984.0, + "reward": 5.2593994140625, + "reward_std": 7.445150375366211, + "rewards/rm_reward_func/mean": 5.2593994140625, + "rewards/rm_reward_func/std": 19.743183135986328, + "step": 367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 297.8125, + "completions/mean_terminated_length": 258.1481628417969, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.2944, + "grad_norm": 3.9477925300598145, + "kl": 0.167724609375, + "learning_rate": 1e-06, + "loss": -0.0239, + "num_tokens": 4681170.0, + "reward": 7.892822265625, + "reward_std": 4.728097915649414, + "rewards/rm_reward_func/mean": 7.892822265625, + "rewards/rm_reward_func/std": 5.967565536499023, + "step": 368 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 281.28125, + "completions/mean_terminated_length": 238.55555725097656, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.2952, + "grad_norm": 6.2598443031311035, + "kl": 0.108642578125, + "learning_rate": 1e-06, + "loss": -0.2548, + "num_tokens": 4692299.0, + "reward": -7.61279296875, + "reward_std": 4.482115745544434, + "rewards/rm_reward_func/mean": -7.61279296875, + "rewards/rm_reward_func/std": 9.240117073059082, + "step": 369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 442.0, + "completions/mean_length": 204.84375, + "completions/mean_terminated_length": 194.9354705810547, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.296, + "grad_norm": 4.286995887756348, + "kl": 0.1629638671875, + "learning_rate": 1e-06, + "loss": -0.0786, + "num_tokens": 4701822.0, + "reward": 9.278564453125, + "reward_std": 7.017640590667725, + "rewards/rm_reward_func/mean": 9.278564453125, + "rewards/rm_reward_func/std": 11.16659164428711, + "step": 370 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 137.71875, + "completions/mean_terminated_length": 125.64515686035156, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.2968, + "grad_norm": 6.7198004722595215, + "kl": 0.17926025390625, + "learning_rate": 1e-06, + "loss": 0.0636, + "num_tokens": 4709437.0, + "reward": 1.2930221557617188, + "reward_std": 2.7805871963500977, + "rewards/rm_reward_func/mean": 1.2930221557617188, + "rewards/rm_reward_func/std": 6.2812652587890625, + "step": 371 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 127.0, + "completions/max_terminated_length": 127.0, + "completions/mean_length": 67.65625, + "completions/mean_terminated_length": 67.65625, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.2976, + "grad_norm": 7.943471908569336, + "kl": 0.2227783203125, + "learning_rate": 1e-06, + "loss": -0.0281, + "num_tokens": 4716018.0, + "reward": 0.7178955078125, + "reward_std": 2.993560314178467, + "rewards/rm_reward_func/mean": 0.7178955078125, + "rewards/rm_reward_func/std": 7.339946746826172, + "step": 372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 326.53125, + "completions/mean_terminated_length": 314.16668701171875, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.2984, + "grad_norm": 2.898266315460205, + "kl": 0.1676025390625, + "learning_rate": 1e-06, + "loss": 0.0155, + "num_tokens": 4729731.0, + "reward": 8.0377197265625, + "reward_std": 4.975411891937256, + "rewards/rm_reward_func/mean": 8.0377197265625, + "rewards/rm_reward_func/std": 11.102714538574219, + "step": 373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 314.59375, + "completions/mean_terminated_length": 224.8636474609375, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.2992, + "grad_norm": 5.358304023742676, + "kl": 0.1610107421875, + "learning_rate": 1e-06, + "loss": 0.197, + "num_tokens": 4742030.0, + "reward": 5.0345458984375, + "reward_std": 6.491810321807861, + "rewards/rm_reward_func/mean": 5.0345458984375, + "rewards/rm_reward_func/std": 8.370210647583008, + "step": 374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 426.375, + "completions/mean_terminated_length": 350.8235168457031, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.3, + "grad_norm": 4.6803483963012695, + "kl": 0.0802001953125, + "learning_rate": 1e-06, + "loss": -0.0817, + "num_tokens": 4761410.0, + "reward": 10.879684448242188, + "reward_std": 8.08679485321045, + "rewards/rm_reward_func/mean": 10.879684448242188, + "rewards/rm_reward_func/std": 10.522982597351074, + "step": 375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 342.0, + "completions/mean_length": 204.09375, + "completions/mean_terminated_length": 194.16128540039062, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.3008, + "grad_norm": 3.7969260215759277, + "kl": 0.1573486328125, + "learning_rate": 1e-06, + "loss": -0.011, + "num_tokens": 4770525.0, + "reward": 9.822998046875, + "reward_std": 6.23846435546875, + "rewards/rm_reward_func/mean": 9.822998046875, + "rewards/rm_reward_func/std": 9.928614616394043, + "step": 376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 231.34375, + "completions/mean_terminated_length": 212.6333465576172, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.3016, + "grad_norm": 4.173020839691162, + "kl": 0.240234375, + "learning_rate": 1e-06, + "loss": -0.0314, + "num_tokens": 4780584.0, + "reward": 7.1978607177734375, + "reward_std": 5.048998832702637, + "rewards/rm_reward_func/mean": 7.1978607177734375, + "rewards/rm_reward_func/std": 9.145500183105469, + "step": 377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 312.96875, + "completions/mean_terminated_length": 246.625, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.3024, + "grad_norm": 3.5591626167297363, + "kl": 0.175537109375, + "learning_rate": 1e-06, + "loss": 0.0296, + "num_tokens": 4792983.0, + "reward": 1.46685791015625, + "reward_std": 3.8416523933410645, + "rewards/rm_reward_func/mean": 1.46685791015625, + "rewards/rm_reward_func/std": 8.133792877197266, + "step": 378 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 497.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 240.4375, + "completions/mean_terminated_length": 240.4375, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.3032, + "grad_norm": 4.513542652130127, + "kl": 0.148193359375, + "learning_rate": 1e-06, + "loss": -0.0096, + "num_tokens": 4803181.0, + "reward": 3.9730072021484375, + "reward_std": 3.2463040351867676, + "rewards/rm_reward_func/mean": 3.9730072021484375, + "rewards/rm_reward_func/std": 7.680938243865967, + "step": 379 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 274.78125, + "completions/mean_terminated_length": 250.2413787841797, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.304, + "grad_norm": 4.1872639656066895, + "kl": 0.088134765625, + "learning_rate": 1e-06, + "loss": 0.2139, + "num_tokens": 4816062.0, + "reward": 8.164306640625, + "reward_std": 9.427093505859375, + "rewards/rm_reward_func/mean": 8.164306640625, + "rewards/rm_reward_func/std": 11.66413402557373, + "step": 380 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 318.28125, + "completions/mean_terminated_length": 242.478271484375, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.3048, + "grad_norm": 4.635010242462158, + "kl": 0.1771240234375, + "learning_rate": 1e-06, + "loss": -0.0366, + "num_tokens": 4828711.0, + "reward": 7.5330810546875, + "reward_std": 4.666987419128418, + "rewards/rm_reward_func/mean": 7.5330810546875, + "rewards/rm_reward_func/std": 6.908311367034912, + "step": 381 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 274.90625, + "completions/mean_terminated_length": 220.19232177734375, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.3056, + "grad_norm": 4.149857044219971, + "kl": 0.168212890625, + "learning_rate": 1e-06, + "loss": 0.0295, + "num_tokens": 4839428.0, + "reward": -3.975341796875, + "reward_std": 4.726168632507324, + "rewards/rm_reward_func/mean": -3.975341796875, + "rewards/rm_reward_func/std": 9.026304244995117, + "step": 382 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 276.0, + "completions/mean_terminated_length": 209.9199981689453, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.3064, + "grad_norm": 3.875955820083618, + "kl": 0.13720703125, + "learning_rate": 1e-06, + "loss": 0.0275, + "num_tokens": 4850612.0, + "reward": -5.6435546875, + "reward_std": 3.571234941482544, + "rewards/rm_reward_func/mean": -5.6435546875, + "rewards/rm_reward_func/std": 16.429338455200195, + "step": 383 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 227.34375, + "completions/mean_terminated_length": 208.36668395996094, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.3072, + "grad_norm": 3.766765832901001, + "kl": 0.154052734375, + "learning_rate": 1e-06, + "loss": 0.1151, + "num_tokens": 4862007.0, + "reward": 9.3204345703125, + "reward_std": 5.31584358215332, + "rewards/rm_reward_func/mean": 9.3204345703125, + "rewards/rm_reward_func/std": 12.34428596496582, + "step": 384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 274.90625, + "completions/mean_terminated_length": 220.19232177734375, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.308, + "grad_norm": 3.00742506980896, + "kl": 0.134521484375, + "learning_rate": 1e-06, + "loss": -0.0583, + "num_tokens": 4873196.0, + "reward": -3.42340087890625, + "reward_std": 3.529475450515747, + "rewards/rm_reward_func/mean": -3.42340087890625, + "rewards/rm_reward_func/std": 14.037924766540527, + "step": 385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 375.53125, + "completions/mean_terminated_length": 293.6499938964844, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.3088, + "grad_norm": 4.225069046020508, + "kl": 0.15325927734375, + "learning_rate": 1e-06, + "loss": -0.1237, + "num_tokens": 4889389.0, + "reward": 10.78369140625, + "reward_std": 9.285670280456543, + "rewards/rm_reward_func/mean": 10.78369140625, + "rewards/rm_reward_func/std": 19.006528854370117, + "step": 386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 340.40625, + "completions/mean_terminated_length": 262.4090881347656, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.3096, + "grad_norm": 2.9847397804260254, + "kl": 0.14892578125, + "learning_rate": 1e-06, + "loss": 0.0144, + "num_tokens": 4903578.0, + "reward": 9.98577880859375, + "reward_std": 4.354106426239014, + "rewards/rm_reward_func/mean": 9.98577880859375, + "rewards/rm_reward_func/std": 9.918432235717773, + "step": 387 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 282.96875, + "completions/mean_terminated_length": 259.2758483886719, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.3104, + "grad_norm": 3.2981696128845215, + "kl": 0.138427734375, + "learning_rate": 1e-06, + "loss": 0.0006, + "num_tokens": 4915585.0, + "reward": 6.951171875, + "reward_std": 4.37175178527832, + "rewards/rm_reward_func/mean": 6.951171875, + "rewards/rm_reward_func/std": 16.657833099365234, + "step": 388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 282.0625, + "completions/mean_terminated_length": 258.2758483886719, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.3112, + "grad_norm": 4.533021450042725, + "kl": 0.1182861328125, + "learning_rate": 1e-06, + "loss": -0.1073, + "num_tokens": 4929283.0, + "reward": 1.020263671875, + "reward_std": 5.0986127853393555, + "rewards/rm_reward_func/mean": 1.020263671875, + "rewards/rm_reward_func/std": 10.977892875671387, + "step": 389 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 252.9375, + "completions/mean_terminated_length": 193.1538543701172, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.312, + "grad_norm": 4.486575603485107, + "kl": 0.15936279296875, + "learning_rate": 1e-06, + "loss": -0.0062, + "num_tokens": 4941017.0, + "reward": -1.0895843505859375, + "reward_std": 5.964107990264893, + "rewards/rm_reward_func/mean": -1.0895843505859375, + "rewards/rm_reward_func/std": 11.993568420410156, + "step": 390 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 404.875, + "completions/mean_terminated_length": 380.15386962890625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.3128, + "grad_norm": 3.2242486476898193, + "kl": 0.097900390625, + "learning_rate": 1e-06, + "loss": 0.0056, + "num_tokens": 4955989.0, + "reward": 17.81597900390625, + "reward_std": 8.450302124023438, + "rewards/rm_reward_func/mean": 17.81597900390625, + "rewards/rm_reward_func/std": 14.226698875427246, + "step": 391 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 275.53125, + "completions/mean_terminated_length": 220.9615478515625, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.3136, + "grad_norm": 3.7851226329803467, + "kl": 0.231781005859375, + "learning_rate": 1e-06, + "loss": 0.0319, + "num_tokens": 4967710.0, + "reward": 13.55126953125, + "reward_std": 4.914253234863281, + "rewards/rm_reward_func/mean": 13.55126953125, + "rewards/rm_reward_func/std": 15.68505859375, + "step": 392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 208.75, + "completions/mean_terminated_length": 188.53334045410156, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.3144, + "grad_norm": 3.546250104904175, + "kl": 0.248779296875, + "learning_rate": 1e-06, + "loss": 0.1134, + "num_tokens": 4977470.0, + "reward": 6.796875, + "reward_std": 4.045927047729492, + "rewards/rm_reward_func/mean": 6.796875, + "rewards/rm_reward_func/std": 9.776312828063965, + "step": 393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 285.1875, + "completions/mean_terminated_length": 252.7857208251953, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.3152, + "grad_norm": 8.546547889709473, + "kl": 0.190673828125, + "learning_rate": 1e-06, + "loss": 0.2501, + "num_tokens": 4990396.0, + "reward": 12.95703125, + "reward_std": 6.034456729888916, + "rewards/rm_reward_func/mean": 12.95703125, + "rewards/rm_reward_func/std": 11.617748260498047, + "step": 394 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 449.0, + "completions/max_terminated_length": 449.0, + "completions/mean_length": 294.84375, + "completions/mean_terminated_length": 294.84375, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.316, + "grad_norm": 3.5022923946380615, + "kl": 0.162109375, + "learning_rate": 1e-06, + "loss": -0.0858, + "num_tokens": 5002815.0, + "reward": 14.046142578125, + "reward_std": 6.6521525382995605, + "rewards/rm_reward_func/mean": 14.046142578125, + "rewards/rm_reward_func/std": 15.969138145446777, + "step": 395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 406.75, + "completions/mean_terminated_length": 334.7368469238281, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "epoch": 0.3168, + "grad_norm": 3.3227157592773438, + "kl": 0.09228515625, + "learning_rate": 1e-06, + "loss": -0.0137, + "num_tokens": 5021671.0, + "reward": 3.28466796875, + "reward_std": 4.573013782501221, + "rewards/rm_reward_func/mean": 3.28466796875, + "rewards/rm_reward_func/std": 10.546538352966309, + "step": 396 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 480.0, + "completions/max_terminated_length": 480.0, + "completions/mean_length": 199.3125, + "completions/mean_terminated_length": 199.3125, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.3176, + "grad_norm": 8.376974105834961, + "kl": 0.1419677734375, + "learning_rate": 1e-06, + "loss": 0.01, + "num_tokens": 5033201.0, + "reward": 3.65576171875, + "reward_std": 4.244373321533203, + "rewards/rm_reward_func/mean": 3.65576171875, + "rewards/rm_reward_func/std": 15.917009353637695, + "step": 397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 303.9375, + "completions/mean_terminated_length": 303.9375, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.3184, + "grad_norm": 3.5459303855895996, + "kl": 0.10992431640625, + "learning_rate": 1e-06, + "loss": 0.0089, + "num_tokens": 5044935.0, + "reward": 5.091552734375, + "reward_std": 4.916255950927734, + "rewards/rm_reward_func/mean": 5.091552734375, + "rewards/rm_reward_func/std": 14.6499605178833, + "step": 398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 317.90625, + "completions/mean_terminated_length": 281.96295166015625, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.3192, + "grad_norm": 3.4768733978271484, + "kl": 0.1575927734375, + "learning_rate": 1e-06, + "loss": 0.0603, + "num_tokens": 5058220.0, + "reward": 2.6343994140625, + "reward_std": 6.248598098754883, + "rewards/rm_reward_func/mean": 2.6343994140625, + "rewards/rm_reward_func/std": 12.041556358337402, + "step": 399 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 495.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 312.09375, + "completions/mean_terminated_length": 312.09375, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.32, + "grad_norm": 3.6274449825286865, + "kl": 0.13006591796875, + "learning_rate": 1e-06, + "loss": 0.0541, + "num_tokens": 5070567.0, + "reward": 6.393798828125, + "reward_std": 4.916718006134033, + "rewards/rm_reward_func/mean": 6.393798828125, + "rewards/rm_reward_func/std": 14.114630699157715, + "step": 400 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 120.0, + "completions/max_terminated_length": 120.0, + "completions/mean_length": 77.9375, + "completions/mean_terminated_length": 77.9375, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.3208, + "grad_norm": 4.715888023376465, + "kl": 0.33203125, + "learning_rate": 1e-06, + "loss": -0.0051, + "num_tokens": 5077253.0, + "reward": 2.888671875, + "reward_std": 3.143357038497925, + "rewards/rm_reward_func/mean": 2.888671875, + "rewards/rm_reward_func/std": 10.683290481567383, + "step": 401 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 297.0, + "completions/mean_length": 140.5625, + "completions/mean_terminated_length": 115.80000305175781, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "epoch": 0.3216, + "grad_norm": 7.256206512451172, + "kl": 0.289306640625, + "learning_rate": 1e-06, + "loss": 0.432, + "num_tokens": 5085031.0, + "reward": 2.5806884765625, + "reward_std": 4.735476016998291, + "rewards/rm_reward_func/mean": 2.5806884765625, + "rewards/rm_reward_func/std": 8.712172508239746, + "step": 402 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 465.0, + "completions/mean_length": 286.125, + "completions/mean_terminated_length": 262.75860595703125, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.3224, + "grad_norm": 6.090017795562744, + "kl": 0.133056640625, + "learning_rate": 1e-06, + "loss": -0.2161, + "num_tokens": 5097227.0, + "reward": 3.853515625, + "reward_std": 5.223775863647461, + "rewards/rm_reward_func/mean": 3.853515625, + "rewards/rm_reward_func/std": 14.418701171875, + "step": 403 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 303.3125, + "completions/mean_terminated_length": 281.7241516113281, + "completions/min_length": 26.0, + "completions/min_terminated_length": 26.0, + "epoch": 0.3232, + "grad_norm": 4.603776931762695, + "kl": 0.0992431640625, + "learning_rate": 1e-06, + "loss": 0.0173, + "num_tokens": 5109629.0, + "reward": 6.81072998046875, + "reward_std": 10.010828018188477, + "rewards/rm_reward_func/mean": 6.81072998046875, + "rewards/rm_reward_func/std": 19.7315673828125, + "step": 404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 327.40625, + "completions/mean_terminated_length": 265.875, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.324, + "grad_norm": 3.5028727054595947, + "kl": 0.10601806640625, + "learning_rate": 1e-06, + "loss": -0.067, + "num_tokens": 5122570.0, + "reward": -3.7301025390625, + "reward_std": 4.875129222869873, + "rewards/rm_reward_func/mean": -3.7301025390625, + "rewards/rm_reward_func/std": 7.149316787719727, + "step": 405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 360.21875, + "completions/mean_terminated_length": 280.71429443359375, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.3248, + "grad_norm": 5.7773871421813965, + "kl": 0.1090087890625, + "learning_rate": 1e-06, + "loss": -0.3303, + "num_tokens": 5136473.0, + "reward": 0.67950439453125, + "reward_std": 9.863033294677734, + "rewards/rm_reward_func/mean": 0.67950439453125, + "rewards/rm_reward_func/std": 14.524174690246582, + "step": 406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 461.0, + "completions/mean_length": 238.84375, + "completions/mean_terminated_length": 162.36000061035156, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.3256, + "grad_norm": 2.8504834175109863, + "kl": 0.235107421875, + "learning_rate": 1e-06, + "loss": -0.0181, + "num_tokens": 5147908.0, + "reward": 5.07373046875, + "reward_std": 3.009080648422241, + "rewards/rm_reward_func/mean": 5.07373046875, + "rewards/rm_reward_func/std": 7.276069641113281, + "step": 407 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 342.375, + "completions/mean_terminated_length": 285.8333435058594, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.3264, + "grad_norm": 6.275087833404541, + "kl": 0.151123046875, + "learning_rate": 1e-06, + "loss": 0.237, + "num_tokens": 5161120.0, + "reward": 6.7158203125, + "reward_std": 6.726596832275391, + "rewards/rm_reward_func/mean": 6.7158203125, + "rewards/rm_reward_func/std": 7.3756632804870605, + "step": 408 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 417.15625, + "completions/mean_terminated_length": 390.6000061035156, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "epoch": 0.3272, + "grad_norm": 3.3832011222839355, + "kl": 0.0989990234375, + "learning_rate": 1e-06, + "loss": 0.0316, + "num_tokens": 5178797.0, + "reward": 6.41094970703125, + "reward_std": 5.119528293609619, + "rewards/rm_reward_func/mean": 6.41094970703125, + "rewards/rm_reward_func/std": 7.976118564605713, + "step": 409 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 425.0, + "completions/mean_length": 293.25, + "completions/mean_terminated_length": 286.19354248046875, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.328, + "grad_norm": 3.2517099380493164, + "kl": 0.17236328125, + "learning_rate": 1e-06, + "loss": 0.0392, + "num_tokens": 5190149.0, + "reward": 9.473388671875, + "reward_std": 6.672955513000488, + "rewards/rm_reward_func/mean": 9.473388671875, + "rewards/rm_reward_func/std": 11.397151947021484, + "step": 410 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 407.0, + "completions/mean_length": 304.125, + "completions/mean_terminated_length": 209.63636779785156, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.3288, + "grad_norm": 6.965356349945068, + "kl": 0.15704345703125, + "learning_rate": 1e-06, + "loss": 0.2693, + "num_tokens": 5203601.0, + "reward": -2.846466064453125, + "reward_std": 3.277106761932373, + "rewards/rm_reward_func/mean": -2.846466064453125, + "rewards/rm_reward_func/std": 8.159171104431152, + "step": 411 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 452.0, + "completions/mean_length": 329.3125, + "completions/mean_terminated_length": 268.41668701171875, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.3296, + "grad_norm": 3.6799185276031494, + "kl": 0.079345703125, + "learning_rate": 1e-06, + "loss": -0.0095, + "num_tokens": 5217035.0, + "reward": -4.873779296875, + "reward_std": 5.421168327331543, + "rewards/rm_reward_func/mean": -4.873779296875, + "rewards/rm_reward_func/std": 10.083694458007812, + "step": 412 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 466.0, + "completions/mean_length": 382.3125, + "completions/mean_terminated_length": 358.2962951660156, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.3304, + "grad_norm": 3.278068780899048, + "kl": 0.1441650390625, + "learning_rate": 1e-06, + "loss": 0.0341, + "num_tokens": 5232501.0, + "reward": 10.257858276367188, + "reward_std": 8.294170379638672, + "rewards/rm_reward_func/mean": 10.257858276367188, + "rewards/rm_reward_func/std": 13.173551559448242, + "step": 413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 482.0, + "completions/mean_length": 304.84375, + "completions/mean_terminated_length": 235.7916717529297, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.3312, + "grad_norm": 3.0338048934936523, + "kl": 0.15618896484375, + "learning_rate": 1e-06, + "loss": -0.1137, + "num_tokens": 5244808.0, + "reward": 5.3134765625, + "reward_std": 6.2857160568237305, + "rewards/rm_reward_func/mean": 5.3134765625, + "rewards/rm_reward_func/std": 16.0675106048584, + "step": 414 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 249.125, + "completions/mean_terminated_length": 200.44444274902344, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.332, + "grad_norm": 6.477999687194824, + "kl": 0.2847900390625, + "learning_rate": 1e-06, + "loss": 0.1968, + "num_tokens": 5256652.0, + "reward": 9.817138671875, + "reward_std": 10.444900512695312, + "rewards/rm_reward_func/mean": 9.817138671875, + "rewards/rm_reward_func/std": 11.969996452331543, + "step": 415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 409.21875, + "completions/mean_terminated_length": 347.5500183105469, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "epoch": 0.3328, + "grad_norm": 3.6648788452148438, + "kl": 0.2205810546875, + "learning_rate": 1e-06, + "loss": -0.0097, + "num_tokens": 5273531.0, + "reward": -2.630218505859375, + "reward_std": 3.172177314758301, + "rewards/rm_reward_func/mean": -2.630218505859375, + "rewards/rm_reward_func/std": 7.456665515899658, + "step": 416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 338.96875, + "completions/mean_terminated_length": 314.25, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.3336, + "grad_norm": 4.057036876678467, + "kl": 0.1800537109375, + "learning_rate": 1e-06, + "loss": 0.0165, + "num_tokens": 5288122.0, + "reward": 7.453369140625, + "reward_std": 4.4758758544921875, + "rewards/rm_reward_func/mean": 7.453369140625, + "rewards/rm_reward_func/std": 8.864148139953613, + "step": 417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 271.8125, + "completions/mean_terminated_length": 264.06451416015625, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.3344, + "grad_norm": 7.1819305419921875, + "kl": 0.20086669921875, + "learning_rate": 1e-06, + "loss": 0.258, + "num_tokens": 5302668.0, + "reward": 4.56103515625, + "reward_std": 5.861100673675537, + "rewards/rm_reward_func/mean": 4.56103515625, + "rewards/rm_reward_func/std": 10.66663932800293, + "step": 418 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 449.9375, + "completions/mean_terminated_length": 421.727294921875, + "completions/min_length": 332.0, + "completions/min_terminated_length": 332.0, + "epoch": 0.3352, + "grad_norm": 4.249541282653809, + "kl": 0.2069091796875, + "learning_rate": 1e-06, + "loss": 0.0362, + "num_tokens": 5319242.0, + "reward": 11.88818359375, + "reward_std": 10.299996376037598, + "rewards/rm_reward_func/mean": 11.88818359375, + "rewards/rm_reward_func/std": 14.682938575744629, + "step": 419 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 301.375, + "completions/mean_terminated_length": 294.58062744140625, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.336, + "grad_norm": 4.146368980407715, + "kl": 0.2005615234375, + "learning_rate": 1e-06, + "loss": 0.1198, + "num_tokens": 5334606.0, + "reward": 1.5439453125, + "reward_std": 5.175732612609863, + "rewards/rm_reward_func/mean": 1.5439453125, + "rewards/rm_reward_func/std": 11.174506187438965, + "step": 420 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 240.125, + "completions/mean_terminated_length": 222.00001525878906, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.3368, + "grad_norm": 5.178651332855225, + "kl": 0.252685546875, + "learning_rate": 1e-06, + "loss": 0.3182, + "num_tokens": 5345146.0, + "reward": -5.8485107421875, + "reward_std": 10.920194625854492, + "rewards/rm_reward_func/mean": -5.8485107421875, + "rewards/rm_reward_func/std": 11.926911354064941, + "step": 421 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 258.625, + "completions/mean_terminated_length": 222.4285888671875, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.3376, + "grad_norm": 7.027132511138916, + "kl": 0.424560546875, + "learning_rate": 1e-06, + "loss": 0.2645, + "num_tokens": 5356774.0, + "reward": 10.3265380859375, + "reward_std": 7.619661331176758, + "rewards/rm_reward_func/mean": 10.3265380859375, + "rewards/rm_reward_func/std": 16.47081184387207, + "step": 422 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 366.71875, + "completions/mean_terminated_length": 333.19232177734375, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "epoch": 0.3384, + "grad_norm": 5.974679470062256, + "kl": 0.1844482421875, + "learning_rate": 1e-06, + "loss": 0.0469, + "num_tokens": 5370701.0, + "reward": 20.43310546875, + "reward_std": 9.503026008605957, + "rewards/rm_reward_func/mean": 20.43310546875, + "rewards/rm_reward_func/std": 18.206256866455078, + "step": 423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 346.90625, + "completions/mean_terminated_length": 300.67999267578125, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.3392, + "grad_norm": 4.590559482574463, + "kl": 0.110626220703125, + "learning_rate": 1e-06, + "loss": 0.0688, + "num_tokens": 5384434.0, + "reward": 1.7897424697875977, + "reward_std": 6.459235191345215, + "rewards/rm_reward_func/mean": 1.7897424697875977, + "rewards/rm_reward_func/std": 15.98327350616455, + "step": 424 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 423.15625, + "completions/mean_terminated_length": 354.0555725097656, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.34, + "grad_norm": 16.818695068359375, + "kl": 0.2874755859375, + "learning_rate": 1e-06, + "loss": -0.0521, + "num_tokens": 5400303.0, + "reward": 3.7025909423828125, + "reward_std": 7.56202507019043, + "rewards/rm_reward_func/mean": 3.7025909423828125, + "rewards/rm_reward_func/std": 20.20536231994629, + "step": 425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 353.71875, + "completions/mean_terminated_length": 281.7727355957031, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.3408, + "grad_norm": 5.755769729614258, + "kl": 0.47705078125, + "learning_rate": 1e-06, + "loss": 0.2143, + "num_tokens": 5413798.0, + "reward": -4.92926025390625, + "reward_std": 6.647279739379883, + "rewards/rm_reward_func/mean": -4.92926025390625, + "rewards/rm_reward_func/std": 9.619757652282715, + "step": 426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 436.0, + "completions/mean_length": 363.4375, + "completions/mean_terminated_length": 261.78948974609375, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "epoch": 0.3416, + "grad_norm": 5.360143661499023, + "kl": 0.496826171875, + "learning_rate": 1e-06, + "loss": 0.1156, + "num_tokens": 5427236.0, + "reward": -9.527587890625, + "reward_std": 7.441409111022949, + "rewards/rm_reward_func/mean": -9.527587890625, + "rewards/rm_reward_func/std": 8.035903930664062, + "step": 427 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 399.0, + "completions/mean_length": 301.53125, + "completions/mean_terminated_length": 191.2857208251953, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.3424, + "grad_norm": 5.262341022491455, + "kl": 0.5040283203125, + "learning_rate": 1e-06, + "loss": 0.1718, + "num_tokens": 5441501.0, + "reward": -8.2998046875, + "reward_std": 4.3596343994140625, + "rewards/rm_reward_func/mean": -8.2998046875, + "rewards/rm_reward_func/std": 10.656414031982422, + "step": 428 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 498.40625, + "completions/mean_terminated_length": 457.625, + "completions/min_length": 409.0, + "completions/min_terminated_length": 409.0, + "epoch": 0.3432, + "grad_norm": 5.260901927947998, + "kl": 0.877197265625, + "learning_rate": 1e-06, + "loss": 0.0611, + "num_tokens": 5460434.0, + "reward": -9.208251953125, + "reward_std": 7.153204917907715, + "rewards/rm_reward_func/mean": -9.208251953125, + "rewards/rm_reward_func/std": 9.89172077178955, + "step": 429 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 246.0625, + "completions/mean_terminated_length": 208.07144165039062, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.344, + "grad_norm": 7.583652496337891, + "kl": 0.421142578125, + "learning_rate": 1e-06, + "loss": 0.2763, + "num_tokens": 5470404.0, + "reward": -1.38763427734375, + "reward_std": 8.258423805236816, + "rewards/rm_reward_func/mean": -1.38763427734375, + "rewards/rm_reward_func/std": 14.190618515014648, + "step": 430 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 375.84375, + "completions/mean_terminated_length": 176.84616088867188, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "epoch": 0.3448, + "grad_norm": 7.52949857711792, + "kl": 1.322265625, + "learning_rate": 1e-06, + "loss": 0.3466, + "num_tokens": 5486727.0, + "reward": -10.94189453125, + "reward_std": 8.947233200073242, + "rewards/rm_reward_func/mean": -10.94189453125, + "rewards/rm_reward_func/std": 11.468242645263672, + "step": 431 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 373.0, + "completions/mean_length": 336.09375, + "completions/mean_terminated_length": 160.1875, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.3456, + "grad_norm": 6.60636568069458, + "kl": 1.8037109375, + "learning_rate": 1e-06, + "loss": 0.1765, + "num_tokens": 5503626.0, + "reward": -11.5931396484375, + "reward_std": 7.840054035186768, + "rewards/rm_reward_func/mean": -11.5931396484375, + "rewards/rm_reward_func/std": 13.185185432434082, + "step": 432 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 371.0, + "completions/mean_length": 373.71875, + "completions/mean_terminated_length": 143.25, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.3464, + "grad_norm": 20.934152603149414, + "kl": 1.482421875, + "learning_rate": 1e-06, + "loss": 0.4161, + "num_tokens": 5518233.0, + "reward": -11.6781005859375, + "reward_std": 9.535715103149414, + "rewards/rm_reward_func/mean": -11.6781005859375, + "rewards/rm_reward_func/std": 12.062625885009766, + "step": 433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 415.0, + "completions/mean_length": 397.625, + "completions/mean_terminated_length": 207.0, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.3472, + "grad_norm": 6.227436542510986, + "kl": 1.304443359375, + "learning_rate": 1e-06, + "loss": 0.1463, + "num_tokens": 5535941.0, + "reward": -11.416015625, + "reward_std": 10.354272842407227, + "rewards/rm_reward_func/mean": -11.416015625, + "rewards/rm_reward_func/std": 10.621146202087402, + "step": 434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.8125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 434.0, + "completions/mean_length": 450.34375, + "completions/mean_terminated_length": 183.1666717529297, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.348, + "grad_norm": 24.91790199279785, + "kl": 2.39453125, + "learning_rate": 1e-06, + "loss": 0.1695, + "num_tokens": 5553336.0, + "reward": -16.6943359375, + "reward_std": 5.182467460632324, + "rewards/rm_reward_func/mean": -16.6943359375, + "rewards/rm_reward_func/std": 7.9551005363464355, + "step": 435 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.84375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 496.6875, + "completions/mean_terminated_length": 414.0, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "epoch": 0.3488, + "grad_norm": 7.014784812927246, + "kl": 2.12548828125, + "learning_rate": 1e-06, + "loss": 0.1083, + "num_tokens": 5571502.0, + "reward": -18.14990234375, + "reward_std": 7.876721382141113, + "rewards/rm_reward_func/mean": -18.14990234375, + "rewards/rm_reward_func/std": 9.936715126037598, + "step": 436 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.78125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 453.0, + "completions/mean_length": 435.375, + "completions/mean_terminated_length": 161.71429443359375, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.3496, + "grad_norm": 6.28053617477417, + "kl": 3.0546875, + "learning_rate": 1e-06, + "loss": 0.3596, + "num_tokens": 5588674.0, + "reward": -16.22265625, + "reward_std": 9.346332550048828, + "rewards/rm_reward_func/mean": -16.22265625, + "rewards/rm_reward_func/std": 12.737253189086914, + "step": 437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.84375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 431.0, + "completions/mean_length": 487.4375, + "completions/mean_terminated_length": 354.8000183105469, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "epoch": 0.3504, + "grad_norm": 6.686891078948975, + "kl": 2.892578125, + "learning_rate": 1e-06, + "loss": 0.1816, + "num_tokens": 5608344.0, + "reward": -13.66796875, + "reward_std": 10.213824272155762, + "rewards/rm_reward_func/mean": -13.66796875, + "rewards/rm_reward_func/std": 13.559255599975586, + "step": 438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 475.03125, + "completions/mean_terminated_length": 404.4545593261719, + "completions/min_length": 330.0, + "completions/min_terminated_length": 330.0, + "epoch": 0.3512, + "grad_norm": 4.943912506103516, + "kl": 1.7698974609375, + "learning_rate": 1e-06, + "loss": 0.091, + "num_tokens": 5629521.0, + "reward": -12.60577392578125, + "reward_std": 10.764892578125, + "rewards/rm_reward_func/mean": -12.60577392578125, + "rewards/rm_reward_func/std": 13.864947319030762, + "step": 439 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 449.28125, + "completions/mean_terminated_length": 311.3000183105469, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.352, + "grad_norm": 10.61517333984375, + "kl": 3.8173828125, + "learning_rate": 1e-06, + "loss": 0.1559, + "num_tokens": 5647154.0, + "reward": -16.4229736328125, + "reward_std": 4.779942989349365, + "rewards/rm_reward_func/mean": -16.4229736328125, + "rewards/rm_reward_func/std": 6.662553787231445, + "step": 440 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 459.0, + "completions/mean_length": 438.65625, + "completions/mean_terminated_length": 298.6363830566406, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.3528, + "grad_norm": 9.699241638183594, + "kl": 3.1298828125, + "learning_rate": 1e-06, + "loss": 0.2842, + "num_tokens": 5664383.0, + "reward": -12.2291259765625, + "reward_std": 8.05883502960205, + "rewards/rm_reward_func/mean": -12.2291259765625, + "rewards/rm_reward_func/std": 13.489725112915039, + "step": 441 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 356.9375, + "completions/mean_terminated_length": 98.5, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.3536, + "grad_norm": 8.036911010742188, + "kl": 3.30029296875, + "learning_rate": 1e-06, + "loss": 0.2722, + "num_tokens": 5678285.0, + "reward": -16.7509765625, + "reward_std": 7.531893730163574, + "rewards/rm_reward_func/mean": -16.7509765625, + "rewards/rm_reward_func/std": 9.410063743591309, + "step": 442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 484.0, + "completions/mean_length": 502.78125, + "completions/mean_terminated_length": 438.25, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "epoch": 0.3544, + "grad_norm": 11.907139778137207, + "kl": 4.3046875, + "learning_rate": 1e-06, + "loss": 0.1862, + "num_tokens": 5696630.0, + "reward": -15.4208984375, + "reward_std": 7.1098833084106445, + "rewards/rm_reward_func/mean": -15.4208984375, + "rewards/rm_reward_func/std": 17.015018463134766, + "step": 443 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.71875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 423.71875, + "completions/mean_terminated_length": 198.11111450195312, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.3552, + "grad_norm": 131.54908752441406, + "kl": 3.455078125, + "learning_rate": 1e-06, + "loss": 0.1955, + "num_tokens": 5713213.0, + "reward": -18.561279296875, + "reward_std": 7.316192626953125, + "rewards/rm_reward_func/mean": -18.561279296875, + "rewards/rm_reward_func/std": 9.157906532287598, + "step": 444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.78125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 465.0, + "completions/mean_length": 462.34375, + "completions/mean_terminated_length": 285.0, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.356, + "grad_norm": 5.364108085632324, + "kl": 3.9921875, + "learning_rate": 1e-06, + "loss": 0.2971, + "num_tokens": 5729992.0, + "reward": -8.744140625, + "reward_std": 15.03476333618164, + "rewards/rm_reward_func/mean": -8.744140625, + "rewards/rm_reward_func/std": 19.18375587463379, + "step": 445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 434.6875, + "completions/mean_terminated_length": 202.75, + "completions/min_length": 17.0, + "completions/min_terminated_length": 17.0, + "epoch": 0.3568, + "grad_norm": 8.373677253723145, + "kl": 4.99609375, + "learning_rate": 1e-06, + "loss": 0.2947, + "num_tokens": 5747238.0, + "reward": -14.09228515625, + "reward_std": 10.04998779296875, + "rewards/rm_reward_func/mean": -14.09228515625, + "rewards/rm_reward_func/std": 12.171612739562988, + "step": 446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 425.25, + "completions/mean_terminated_length": 298.4615478515625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.3576, + "grad_norm": 6.68599271774292, + "kl": 3.321533203125, + "learning_rate": 1e-06, + "loss": 0.3343, + "num_tokens": 5764894.0, + "reward": -3.4609375, + "reward_std": 9.75390625, + "rewards/rm_reward_func/mean": -3.4609375, + "rewards/rm_reward_func/std": 23.642847061157227, + "step": 447 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 430.0, + "completions/mean_length": 314.0625, + "completions/mean_terminated_length": 210.38095092773438, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.3584, + "grad_norm": 16.456398010253906, + "kl": 2.2950439453125, + "learning_rate": 1e-06, + "loss": 0.3137, + "num_tokens": 5777040.0, + "reward": -8.64706802368164, + "reward_std": 6.216998100280762, + "rewards/rm_reward_func/mean": -8.64706802368164, + "rewards/rm_reward_func/std": 15.454483032226562, + "step": 448 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 356.21875, + "completions/mean_terminated_length": 249.63157653808594, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.3592, + "grad_norm": 9.984318733215332, + "kl": 0.9708251953125, + "learning_rate": 1e-06, + "loss": 0.1167, + "num_tokens": 5791887.0, + "reward": 2.573716163635254, + "reward_std": 8.430018424987793, + "rewards/rm_reward_func/mean": 2.573716163635254, + "rewards/rm_reward_func/std": 14.201119422912598, + "step": 449 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 406.09375, + "completions/mean_terminated_length": 370.79168701171875, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.36, + "grad_norm": 7.163238525390625, + "kl": 0.5322265625, + "learning_rate": 1e-06, + "loss": 0.0106, + "num_tokens": 5807138.0, + "reward": -2.3760223388671875, + "reward_std": 7.770718574523926, + "rewards/rm_reward_func/mean": -2.3760223388671875, + "rewards/rm_reward_func/std": 10.865222930908203, + "step": 450 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 373.0, + "completions/mean_length": 329.5, + "completions/mean_terminated_length": 168.47059631347656, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.3608, + "grad_norm": 17.861509323120117, + "kl": 5.4453125, + "learning_rate": 1e-06, + "loss": 0.5361, + "num_tokens": 5822826.0, + "reward": -9.7861328125, + "reward_std": 11.642084121704102, + "rewards/rm_reward_func/mean": -9.7861328125, + "rewards/rm_reward_func/std": 16.064666748046875, + "step": 451 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 445.1875, + "completions/mean_terminated_length": 333.8333435058594, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.3616, + "grad_norm": 37.739498138427734, + "kl": 4.68994140625, + "learning_rate": 1e-06, + "loss": 0.2919, + "num_tokens": 5839992.0, + "reward": -1.2283172607421875, + "reward_std": 15.66540813446045, + "rewards/rm_reward_func/mean": -1.2283172607421875, + "rewards/rm_reward_func/std": 21.28140640258789, + "step": 452 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 413.0, + "completions/mean_terminated_length": 268.3077087402344, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.3624, + "grad_norm": 15.843510627746582, + "kl": 5.333740234375, + "learning_rate": 1e-06, + "loss": 0.2438, + "num_tokens": 5857648.0, + "reward": -10.1357421875, + "reward_std": 7.069613456726074, + "rewards/rm_reward_func/mean": -10.1357421875, + "rewards/rm_reward_func/std": 11.879778861999512, + "step": 453 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.84375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 430.0, + "completions/mean_length": 492.78125, + "completions/mean_terminated_length": 389.0, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "epoch": 0.3632, + "grad_norm": 7.660159587860107, + "kl": 6.46484375, + "learning_rate": 1e-06, + "loss": 0.3101, + "num_tokens": 5875665.0, + "reward": -10.20697021484375, + "reward_std": 12.935501098632812, + "rewards/rm_reward_func/mean": -10.20697021484375, + "rewards/rm_reward_func/std": 14.417232513427734, + "step": 454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 267.0, + "completions/mean_length": 411.5, + "completions/mean_terminated_length": 190.40000915527344, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.364, + "grad_norm": 12.648509979248047, + "kl": 8.0, + "learning_rate": 1e-06, + "loss": 0.625, + "num_tokens": 5891249.0, + "reward": -18.5673828125, + "reward_std": 8.711212158203125, + "rewards/rm_reward_func/mean": -18.5673828125, + "rewards/rm_reward_func/std": 9.04566478729248, + "step": 455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 403.0, + "completions/mean_length": 350.40625, + "completions/mean_terminated_length": 188.8125, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.3648, + "grad_norm": 6.931374549865723, + "kl": 7.671875, + "learning_rate": 1e-06, + "loss": 0.6147, + "num_tokens": 5906094.0, + "reward": -11.6011962890625, + "reward_std": 11.895530700683594, + "rewards/rm_reward_func/mean": -11.6011962890625, + "rewards/rm_reward_func/std": 14.706385612487793, + "step": 456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 473.0, + "completions/mean_length": 411.875, + "completions/mean_terminated_length": 298.4000244140625, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.3656, + "grad_norm": 7.6649980545043945, + "kl": 6.799072265625, + "learning_rate": 1e-06, + "loss": 0.4754, + "num_tokens": 5921498.0, + "reward": -13.50347900390625, + "reward_std": 9.386650085449219, + "rewards/rm_reward_func/mean": -13.50347900390625, + "rewards/rm_reward_func/std": 14.427614212036133, + "step": 457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 412.59375, + "completions/mean_terminated_length": 335.27777099609375, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "epoch": 0.3664, + "grad_norm": 14.270245552062988, + "kl": 3.159912109375, + "learning_rate": 1e-06, + "loss": 0.2522, + "num_tokens": 5939749.0, + "reward": 0.822509765625, + "reward_std": 10.554486274719238, + "rewards/rm_reward_func/mean": 0.822509765625, + "rewards/rm_reward_func/std": 19.273405075073242, + "step": 458 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 436.84375, + "completions/mean_terminated_length": 391.75, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.3672, + "grad_norm": 7.888670921325684, + "kl": 3.0789794921875, + "learning_rate": 1e-06, + "loss": 0.1706, + "num_tokens": 5957832.0, + "reward": -1.209869384765625, + "reward_std": 4.854766368865967, + "rewards/rm_reward_func/mean": -1.209869384765625, + "rewards/rm_reward_func/std": 25.843124389648438, + "step": 459 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 477.0, + "completions/mean_length": 439.1875, + "completions/mean_terminated_length": 317.8333435058594, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.368, + "grad_norm": 17.467594146728516, + "kl": 6.8984375, + "learning_rate": 1e-06, + "loss": 0.4475, + "num_tokens": 5975382.0, + "reward": -8.3388671875, + "reward_std": 10.500482559204102, + "rewards/rm_reward_func/mean": -8.3388671875, + "rewards/rm_reward_func/std": 15.440128326416016, + "step": 460 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 373.3125, + "completions/mean_terminated_length": 334.47998046875, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.3688, + "grad_norm": 19.844507217407227, + "kl": 2.7052001953125, + "learning_rate": 1e-06, + "loss": 0.379, + "num_tokens": 5990128.0, + "reward": 8.0970458984375, + "reward_std": 15.225787162780762, + "rewards/rm_reward_func/mean": 8.0970458984375, + "rewards/rm_reward_func/std": 18.16506004333496, + "step": 461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 446.375, + "completions/mean_terminated_length": 412.0, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "epoch": 0.3696, + "grad_norm": 7.17181921005249, + "kl": 0.4322509765625, + "learning_rate": 1e-06, + "loss": 0.0593, + "num_tokens": 6007132.0, + "reward": 8.548095703125, + "reward_std": 7.690432548522949, + "rewards/rm_reward_func/mean": 8.548095703125, + "rewards/rm_reward_func/std": 8.413957595825195, + "step": 462 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 439.125, + "completions/mean_terminated_length": 356.5333557128906, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.3704, + "grad_norm": 5.347095012664795, + "kl": 6.09375, + "learning_rate": 1e-06, + "loss": 0.3736, + "num_tokens": 6025032.0, + "reward": 1.619140625, + "reward_std": 15.041004180908203, + "rewards/rm_reward_func/mean": 1.619140625, + "rewards/rm_reward_func/std": 28.54711151123047, + "step": 463 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 247.9375, + "completions/mean_terminated_length": 239.41934204101562, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.3712, + "grad_norm": 10.885205268859863, + "kl": 1.0289306640625, + "learning_rate": 1e-06, + "loss": 0.0564, + "num_tokens": 6035230.0, + "reward": 15.275390625, + "reward_std": 8.497432708740234, + "rewards/rm_reward_func/mean": 15.275390625, + "rewards/rm_reward_func/std": 14.570096015930176, + "step": 464 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 345.34375, + "completions/mean_terminated_length": 314.4814758300781, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.372, + "grad_norm": 16.305341720581055, + "kl": 2.215576171875, + "learning_rate": 1e-06, + "loss": 0.2026, + "num_tokens": 6048537.0, + "reward": 1.36273193359375, + "reward_std": 9.692459106445312, + "rewards/rm_reward_func/mean": 1.36273193359375, + "rewards/rm_reward_func/std": 17.84818458557129, + "step": 465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 407.4375, + "completions/mean_terminated_length": 302.875, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.3728, + "grad_norm": 7.994433879852295, + "kl": 3.80224609375, + "learning_rate": 1e-06, + "loss": 0.2663, + "num_tokens": 6064695.0, + "reward": 3.369873046875, + "reward_std": 8.856344223022461, + "rewards/rm_reward_func/mean": 3.369873046875, + "rewards/rm_reward_func/std": 22.26304817199707, + "step": 466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 458.0, + "completions/mean_length": 290.5, + "completions/mean_terminated_length": 283.3548278808594, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.3736, + "grad_norm": 6.062528133392334, + "kl": 0.313720703125, + "learning_rate": 1e-06, + "loss": 0.0317, + "num_tokens": 6077287.0, + "reward": 5.29248046875, + "reward_std": 8.455047607421875, + "rewards/rm_reward_func/mean": 5.29248046875, + "rewards/rm_reward_func/std": 13.72189998626709, + "step": 467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 396.28125, + "completions/mean_terminated_length": 326.8500061035156, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.3744, + "grad_norm": 13.018776893615723, + "kl": 3.998291015625, + "learning_rate": 1e-06, + "loss": 0.3904, + "num_tokens": 6092112.0, + "reward": 0.782562255859375, + "reward_std": 9.454818725585938, + "rewards/rm_reward_func/mean": 0.782562255859375, + "rewards/rm_reward_func/std": 14.342537879943848, + "step": 468 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 457.0, + "completions/mean_terminated_length": 428.19049072265625, + "completions/min_length": 363.0, + "completions/min_terminated_length": 363.0, + "epoch": 0.3752, + "grad_norm": 7.1996026039123535, + "kl": 3.93896484375, + "learning_rate": 1e-06, + "loss": 0.1778, + "num_tokens": 6109216.0, + "reward": 5.27410888671875, + "reward_std": 14.364347457885742, + "rewards/rm_reward_func/mean": 5.27410888671875, + "rewards/rm_reward_func/std": 22.558135986328125, + "step": 469 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 405.4375, + "completions/mean_terminated_length": 322.5555725097656, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.376, + "grad_norm": 19.028276443481445, + "kl": 8.43359375, + "learning_rate": 1e-06, + "loss": 0.5417, + "num_tokens": 6124742.0, + "reward": -0.470703125, + "reward_std": 12.874818801879883, + "rewards/rm_reward_func/mean": -0.470703125, + "rewards/rm_reward_func/std": 20.95646095275879, + "step": 470 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 398.0, + "completions/mean_length": 392.8125, + "completions/mean_terminated_length": 311.2631530761719, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "epoch": 0.3768, + "grad_norm": 28.145797729492188, + "kl": 6.830322265625, + "learning_rate": 1e-06, + "loss": 0.4005, + "num_tokens": 6141808.0, + "reward": -3.615478515625, + "reward_std": 9.015436172485352, + "rewards/rm_reward_func/mean": -3.615478515625, + "rewards/rm_reward_func/std": 18.243576049804688, + "step": 471 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 417.125, + "completions/mean_terminated_length": 374.0, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.3776, + "grad_norm": 41.16624450683594, + "kl": 7.7880859375, + "learning_rate": 1e-06, + "loss": 0.3244, + "num_tokens": 6157436.0, + "reward": 10.93359375, + "reward_std": 15.283958435058594, + "rewards/rm_reward_func/mean": 10.93359375, + "rewards/rm_reward_func/std": 27.59185028076172, + "step": 472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 351.3125, + "completions/mean_terminated_length": 278.2727355957031, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "epoch": 0.3784, + "grad_norm": 19.67943572998047, + "kl": 7.505615234375, + "learning_rate": 1e-06, + "loss": 0.536, + "num_tokens": 6173246.0, + "reward": 1.380859375, + "reward_std": 14.768939971923828, + "rewards/rm_reward_func/mean": 1.380859375, + "rewards/rm_reward_func/std": 18.74730682373047, + "step": 473 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 482.0, + "completions/mean_length": 406.28125, + "completions/mean_terminated_length": 333.9473571777344, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.3792, + "grad_norm": 14.080571174621582, + "kl": 6.677734375, + "learning_rate": 1e-06, + "loss": 0.3526, + "num_tokens": 6189295.0, + "reward": -8.592529296875, + "reward_std": 8.344656944274902, + "rewards/rm_reward_func/mean": -8.592529296875, + "rewards/rm_reward_func/std": 14.625808715820312, + "step": 474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 372.5625, + "completions/mean_terminated_length": 326.0833435058594, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.38, + "grad_norm": 55.990806579589844, + "kl": 3.190673828125, + "learning_rate": 1e-06, + "loss": 0.3699, + "num_tokens": 6204681.0, + "reward": 10.92529296875, + "reward_std": 17.398414611816406, + "rewards/rm_reward_func/mean": 10.92529296875, + "rewards/rm_reward_func/std": 21.6011905670166, + "step": 475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 473.0, + "completions/mean_length": 336.34375, + "completions/mean_terminated_length": 303.8148193359375, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.3808, + "grad_norm": 26.353031158447266, + "kl": 3.114013671875, + "learning_rate": 1e-06, + "loss": 0.2881, + "num_tokens": 6217324.0, + "reward": -5.37896728515625, + "reward_std": 7.184835433959961, + "rewards/rm_reward_func/mean": -5.37896728515625, + "rewards/rm_reward_func/std": 18.45599365234375, + "step": 476 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 417.75, + "completions/mean_terminated_length": 396.0, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "epoch": 0.3816, + "grad_norm": 8.078795433044434, + "kl": 3.979248046875, + "learning_rate": 1e-06, + "loss": 0.2362, + "num_tokens": 6233940.0, + "reward": 8.0015869140625, + "reward_std": 12.765542984008789, + "rewards/rm_reward_func/mean": 8.0015869140625, + "rewards/rm_reward_func/std": 16.40848159790039, + "step": 477 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 393.375, + "completions/mean_terminated_length": 322.20001220703125, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "epoch": 0.3824, + "grad_norm": 16.03205108642578, + "kl": 7.90283203125, + "learning_rate": 1e-06, + "loss": 0.3989, + "num_tokens": 6249568.0, + "reward": -4.557373046875, + "reward_std": 9.42959213256836, + "rewards/rm_reward_func/mean": -4.557373046875, + "rewards/rm_reward_func/std": 19.618621826171875, + "step": 478 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 391.4375, + "completions/mean_terminated_length": 328.28570556640625, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.3832, + "grad_norm": 21.212261199951172, + "kl": 7.78076171875, + "learning_rate": 1e-06, + "loss": 0.364, + "num_tokens": 6264534.0, + "reward": 0.4144287109375, + "reward_std": 8.546712875366211, + "rewards/rm_reward_func/mean": 0.4144287109375, + "rewards/rm_reward_func/std": 16.865026473999023, + "step": 479 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 370.65625, + "completions/mean_terminated_length": 323.54168701171875, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "epoch": 0.384, + "grad_norm": 25.12664031982422, + "kl": 6.88818359375, + "learning_rate": 1e-06, + "loss": 0.3655, + "num_tokens": 6278235.0, + "reward": 4.96453857421875, + "reward_std": 17.571796417236328, + "rewards/rm_reward_func/mean": 4.96453857421875, + "rewards/rm_reward_func/std": 19.67218589782715, + "step": 480 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 209.0, + "completions/mean_length": 390.875, + "completions/mean_terminated_length": 124.4000015258789, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.3848, + "grad_norm": 79.35224914550781, + "kl": 21.0, + "learning_rate": 1e-06, + "loss": 1.0686, + "num_tokens": 6294447.0, + "reward": -10.9208984375, + "reward_std": 13.471847534179688, + "rewards/rm_reward_func/mean": -10.9208984375, + "rewards/rm_reward_func/std": 27.3441219329834, + "step": 481 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 229.0, + "completions/mean_length": 404.0, + "completions/mean_terminated_length": 166.40000915527344, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.3856, + "grad_norm": 132.53387451171875, + "kl": 13.1297607421875, + "learning_rate": 1e-06, + "loss": 0.7436, + "num_tokens": 6311167.0, + "reward": -12.4658203125, + "reward_std": 8.332673072814941, + "rewards/rm_reward_func/mean": -12.4658203125, + "rewards/rm_reward_func/std": 14.378026962280273, + "step": 482 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 486.0, + "completions/mean_length": 379.8125, + "completions/mean_terminated_length": 263.1764831542969, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.3864, + "grad_norm": 17.89412498474121, + "kl": 7.271484375, + "learning_rate": 1e-06, + "loss": 0.4685, + "num_tokens": 6325729.0, + "reward": -2.525604248046875, + "reward_std": 10.174657821655273, + "rewards/rm_reward_func/mean": -2.525604248046875, + "rewards/rm_reward_func/std": 19.22846221923828, + "step": 483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 355.34375, + "completions/mean_terminated_length": 326.3333435058594, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.3872, + "grad_norm": 10.339418411254883, + "kl": 5.09521484375, + "learning_rate": 1e-06, + "loss": 0.349, + "num_tokens": 6339132.0, + "reward": 0.1297607421875, + "reward_std": 12.494827270507812, + "rewards/rm_reward_func/mean": 0.1297607421875, + "rewards/rm_reward_func/std": 17.290435791015625, + "step": 484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 388.0, + "completions/mean_length": 372.75, + "completions/mean_terminated_length": 169.23077392578125, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.388, + "grad_norm": 12.62641429901123, + "kl": 8.357177734375, + "learning_rate": 1e-06, + "loss": 0.6359, + "num_tokens": 6357132.0, + "reward": -7.014617919921875, + "reward_std": 11.388704299926758, + "rewards/rm_reward_func/mean": -7.014617919921875, + "rewards/rm_reward_func/std": 11.901640892028809, + "step": 485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 450.0, + "completions/mean_length": 307.125, + "completions/mean_terminated_length": 214.0, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.3888, + "grad_norm": 13.34914779663086, + "kl": 6.413330078125, + "learning_rate": 1e-06, + "loss": 0.5975, + "num_tokens": 6369224.0, + "reward": -6.581207275390625, + "reward_std": 14.152790069580078, + "rewards/rm_reward_func/mean": -6.581207275390625, + "rewards/rm_reward_func/std": 17.590269088745117, + "step": 486 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 422.0625, + "completions/mean_terminated_length": 368.1000061035156, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "epoch": 0.3896, + "grad_norm": 15.647866249084473, + "kl": 5.72802734375, + "learning_rate": 1e-06, + "loss": 0.3176, + "num_tokens": 6385186.0, + "reward": 2.65093994140625, + "reward_std": 8.202743530273438, + "rewards/rm_reward_func/mean": 2.65093994140625, + "rewards/rm_reward_func/std": 15.850854873657227, + "step": 487 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 359.40625, + "completions/mean_terminated_length": 279.4761962890625, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.3904, + "grad_norm": 43.07307434082031, + "kl": 6.449462890625, + "learning_rate": 1e-06, + "loss": 0.5931, + "num_tokens": 6400463.0, + "reward": -1.673583984375, + "reward_std": 10.083917617797852, + "rewards/rm_reward_func/mean": -1.673583984375, + "rewards/rm_reward_func/std": 13.745939254760742, + "step": 488 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 394.0, + "completions/mean_length": 399.375, + "completions/mean_terminated_length": 286.75, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.3912, + "grad_norm": 91.3353042602539, + "kl": 7.009765625, + "learning_rate": 1e-06, + "loss": 0.4609, + "num_tokens": 6415571.0, + "reward": -7.83154296875, + "reward_std": 9.295122146606445, + "rewards/rm_reward_func/mean": -7.83154296875, + "rewards/rm_reward_func/std": 16.444271087646484, + "step": 489 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 484.0, + "completions/mean_length": 362.71875, + "completions/mean_terminated_length": 328.2692565917969, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.392, + "grad_norm": 24.025558471679688, + "kl": 3.818359375, + "learning_rate": 1e-06, + "loss": 0.3076, + "num_tokens": 6429994.0, + "reward": 12.959228515625, + "reward_std": 14.361933708190918, + "rewards/rm_reward_func/mean": 12.959228515625, + "rewards/rm_reward_func/std": 26.099708557128906, + "step": 490 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 419.0, + "completions/mean_length": 348.15625, + "completions/mean_terminated_length": 184.3125, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.3928, + "grad_norm": 49.87342071533203, + "kl": 9.898193359375, + "learning_rate": 1e-06, + "loss": 0.8155, + "num_tokens": 6449151.0, + "reward": -8.174560546875, + "reward_std": 11.404885292053223, + "rewards/rm_reward_func/mean": -8.174560546875, + "rewards/rm_reward_func/std": 15.167132377624512, + "step": 491 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 308.5, + "completions/mean_terminated_length": 270.8148193359375, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.3936, + "grad_norm": 42.138450622558594, + "kl": 4.73828125, + "learning_rate": 1e-06, + "loss": 0.5528, + "num_tokens": 6463335.0, + "reward": 11.81298828125, + "reward_std": 15.574457168579102, + "rewards/rm_reward_func/mean": 11.81298828125, + "rewards/rm_reward_func/std": 20.5043888092041, + "step": 492 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 486.0, + "completions/mean_length": 285.34375, + "completions/mean_terminated_length": 166.61904907226562, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.3944, + "grad_norm": 122.83219146728516, + "kl": 9.7734375, + "learning_rate": 1e-06, + "loss": 0.9872, + "num_tokens": 6475618.0, + "reward": -2.90985107421875, + "reward_std": 13.12482738494873, + "rewards/rm_reward_func/mean": -2.90985107421875, + "rewards/rm_reward_func/std": 14.76265811920166, + "step": 493 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 352.0625, + "completions/mean_terminated_length": 256.1000061035156, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.3952, + "grad_norm": 18.307031631469727, + "kl": 11.32470703125, + "learning_rate": 1e-06, + "loss": 0.8083, + "num_tokens": 6489324.0, + "reward": -12.8271484375, + "reward_std": 8.004693984985352, + "rewards/rm_reward_func/mean": -12.8271484375, + "rewards/rm_reward_func/std": 11.452200889587402, + "step": 494 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 361.25, + "completions/mean_terminated_length": 302.2608642578125, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.396, + "grad_norm": 24.6053409576416, + "kl": 6.4337158203125, + "learning_rate": 1e-06, + "loss": 0.2939, + "num_tokens": 6507252.0, + "reward": -5.3721923828125, + "reward_std": 7.6330060958862305, + "rewards/rm_reward_func/mean": -5.3721923828125, + "rewards/rm_reward_func/std": 12.24431324005127, + "step": 495 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 278.3125, + "completions/mean_terminated_length": 200.4166717529297, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.3968, + "grad_norm": 15.430408477783203, + "kl": 4.6640625, + "learning_rate": 1e-06, + "loss": 0.4849, + "num_tokens": 6521006.0, + "reward": 11.46609115600586, + "reward_std": 17.760250091552734, + "rewards/rm_reward_func/mean": 11.46609115600586, + "rewards/rm_reward_func/std": 25.725770950317383, + "step": 496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 470.03125, + "completions/mean_terminated_length": 377.70001220703125, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.3976, + "grad_norm": 171.98805236816406, + "kl": 15.0703125, + "learning_rate": 1e-06, + "loss": 0.7248, + "num_tokens": 6539287.0, + "reward": -14.515625, + "reward_std": 12.340795516967773, + "rewards/rm_reward_func/mean": -14.515625, + "rewards/rm_reward_func/std": 17.410518646240234, + "step": 497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 479.625, + "completions/mean_terminated_length": 408.3999938964844, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "epoch": 0.3984, + "grad_norm": 38.708248138427734, + "kl": 13.7109375, + "learning_rate": 1e-06, + "loss": 0.6117, + "num_tokens": 6556987.0, + "reward": -11.59375, + "reward_std": 13.794965744018555, + "rewards/rm_reward_func/mean": -11.59375, + "rewards/rm_reward_func/std": 23.78890609741211, + "step": 498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 421.0, + "completions/mean_length": 409.3125, + "completions/mean_terminated_length": 238.1666717529297, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.3992, + "grad_norm": 25.57811164855957, + "kl": 14.84375, + "learning_rate": 1e-06, + "loss": 0.8756, + "num_tokens": 6572293.0, + "reward": -6.9130859375, + "reward_std": 14.296964645385742, + "rewards/rm_reward_func/mean": -6.9130859375, + "rewards/rm_reward_func/std": 21.80596351623535, + "step": 499 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 378.0, + "completions/mean_terminated_length": 273.77777099609375, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.4, + "grad_norm": 70.98407745361328, + "kl": 7.545654296875, + "learning_rate": 1e-06, + "loss": 0.5359, + "num_tokens": 6586981.0, + "reward": -2.8328094482421875, + "reward_std": 7.248239040374756, + "rewards/rm_reward_func/mean": -2.8328094482421875, + "rewards/rm_reward_func/std": 12.194914817810059, + "step": 500 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.8125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 496.78125, + "completions/mean_terminated_length": 430.8333435058594, + "completions/min_length": 373.0, + "completions/min_terminated_length": 373.0, + "epoch": 0.4008, + "grad_norm": 9.479290008544922, + "kl": 6.978515625, + "learning_rate": 1e-06, + "loss": 0.3081, + "num_tokens": 6606366.0, + "reward": -7.8212890625, + "reward_std": 11.259090423583984, + "rewards/rm_reward_func/mean": -7.8212890625, + "rewards/rm_reward_func/std": 18.525226593017578, + "step": 501 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 385.9375, + "completions/mean_terminated_length": 274.70587158203125, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.4016, + "grad_norm": 42.01311492919922, + "kl": 5.5234375, + "learning_rate": 1e-06, + "loss": 0.5541, + "num_tokens": 6621700.0, + "reward": -0.1533203125, + "reward_std": 14.605756759643555, + "rewards/rm_reward_func/mean": -0.1533203125, + "rewards/rm_reward_func/std": 23.270185470581055, + "step": 502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.71875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 436.625, + "completions/mean_terminated_length": 244.0, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "epoch": 0.4024, + "grad_norm": 20.074079513549805, + "kl": 7.421875, + "learning_rate": 1e-06, + "loss": 0.5378, + "num_tokens": 6637896.0, + "reward": -12.885040283203125, + "reward_std": 12.589856147766113, + "rewards/rm_reward_func/mean": -12.885040283203125, + "rewards/rm_reward_func/std": 16.110185623168945, + "step": 503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 231.0, + "completions/mean_length": 243.875, + "completions/mean_terminated_length": 103.42857360839844, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.4032, + "grad_norm": 24.548263549804688, + "kl": 8.2744140625, + "learning_rate": 1e-06, + "loss": 0.9126, + "num_tokens": 6648780.0, + "reward": -6.2499847412109375, + "reward_std": 12.00403118133545, + "rewards/rm_reward_func/mean": -6.2499847412109375, + "rewards/rm_reward_func/std": 13.340556144714355, + "step": 504 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 460.0625, + "completions/mean_terminated_length": 384.15386962890625, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "epoch": 0.404, + "grad_norm": 17.74894905090332, + "kl": 4.763671875, + "learning_rate": 1e-06, + "loss": 0.2654, + "num_tokens": 6666302.0, + "reward": -6.497314453125, + "reward_std": 12.197681427001953, + "rewards/rm_reward_func/mean": -6.497314453125, + "rewards/rm_reward_func/std": 15.69522476196289, + "step": 505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 426.0, + "completions/mean_length": 376.0, + "completions/mean_terminated_length": 201.1428680419922, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.4048, + "grad_norm": 13.425418853759766, + "kl": 10.6875, + "learning_rate": 1e-06, + "loss": 0.8323, + "num_tokens": 6683206.0, + "reward": -5.591552734375, + "reward_std": 16.38616943359375, + "rewards/rm_reward_func/mean": -5.591552734375, + "rewards/rm_reward_func/std": 19.70439910888672, + "step": 506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 437.09375, + "completions/mean_terminated_length": 272.3000183105469, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.4056, + "grad_norm": 23.703475952148438, + "kl": 9.7734375, + "learning_rate": 1e-06, + "loss": 0.5968, + "num_tokens": 6700297.0, + "reward": -4.63623046875, + "reward_std": 19.530458450317383, + "rewards/rm_reward_func/mean": -4.63623046875, + "rewards/rm_reward_func/std": 19.052122116088867, + "step": 507 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 435.0, + "completions/mean_length": 324.25, + "completions/mean_terminated_length": 158.58824157714844, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.4064, + "grad_norm": 28.222517013549805, + "kl": 9.54296875, + "learning_rate": 1e-06, + "loss": 0.5761, + "num_tokens": 6715521.0, + "reward": -10.552978515625, + "reward_std": 8.230770111083984, + "rewards/rm_reward_func/mean": -10.552978515625, + "rewards/rm_reward_func/std": 15.682586669921875, + "step": 508 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.4072, + "grad_norm": 38.62055969238281, + "kl": 15.5, + "learning_rate": 1e-06, + "loss": 0.6205, + "num_tokens": 6733761.0, + "reward": -21.97564697265625, + "reward_std": 5.978219985961914, + "rewards/rm_reward_func/mean": -21.97564697265625, + "rewards/rm_reward_func/std": 7.307789325714111, + "step": 509 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 366.53125, + "completions/mean_terminated_length": 253.38888549804688, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.408, + "grad_norm": 25.734407424926758, + "kl": 8.80078125, + "learning_rate": 1e-06, + "loss": 0.6397, + "num_tokens": 6749586.0, + "reward": -2.8720703125, + "reward_std": 11.287141799926758, + "rewards/rm_reward_func/mean": -2.8720703125, + "rewards/rm_reward_func/std": 18.167095184326172, + "step": 510 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 449.0, + "completions/mean_length": 436.15625, + "completions/mean_terminated_length": 325.3077087402344, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "epoch": 0.4088, + "grad_norm": 21.79092025756836, + "kl": 8.068603515625, + "learning_rate": 1e-06, + "loss": 0.3422, + "num_tokens": 6765663.0, + "reward": -13.787574768066406, + "reward_std": 6.30387020111084, + "rewards/rm_reward_func/mean": -13.787574768066406, + "rewards/rm_reward_func/std": 17.804723739624023, + "step": 511 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.71875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 420.71875, + "completions/mean_terminated_length": 187.44444274902344, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.4096, + "grad_norm": 161.0446319580078, + "kl": 9.1796875, + "learning_rate": 1e-06, + "loss": 0.6176, + "num_tokens": 6782334.0, + "reward": -11.8310546875, + "reward_std": 9.512861251831055, + "rewards/rm_reward_func/mean": -11.8310546875, + "rewards/rm_reward_func/std": 16.280780792236328, + "step": 512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 397.84375, + "completions/mean_terminated_length": 231.00001525878906, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.4104, + "grad_norm": 12.210495948791504, + "kl": 4.85791015625, + "learning_rate": 1e-06, + "loss": 0.4964, + "num_tokens": 6798361.0, + "reward": 0.423828125, + "reward_std": 10.785526275634766, + "rewards/rm_reward_func/mean": 0.423828125, + "rewards/rm_reward_func/std": 16.22878074645996, + "step": 513 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 384.9375, + "completions/mean_terminated_length": 298.0, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.4112, + "grad_norm": 27.765371322631836, + "kl": 5.994140625, + "learning_rate": 1e-06, + "loss": 0.5379, + "num_tokens": 6813559.0, + "reward": -0.2978515625, + "reward_std": 15.280010223388672, + "rewards/rm_reward_func/mean": -0.2978515625, + "rewards/rm_reward_func/std": 18.968994140625, + "step": 514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 512.0, + "completions/max_terminated_length": 373.0, + "completions/mean_length": 456.125, + "completions/mean_terminated_length": 288.5, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.412, + "grad_norm": 17.61673927307129, + "kl": 7.875, + "learning_rate": 1e-06, + "loss": 0.4006, + "num_tokens": 6832075.0, + "reward": -14.86328125, + "reward_std": 6.095500946044922, + "rewards/rm_reward_func/mean": -14.86328125, + "rewards/rm_reward_func/std": 16.34231948852539, + "step": 515 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.8125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 480.84375, + "completions/mean_terminated_length": 345.8333435058594, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.4128, + "grad_norm": 17.899005889892578, + "kl": 6.27734375, + "learning_rate": 1e-06, + "loss": 0.2511, + "num_tokens": 6849974.0, + "reward": -14.375091552734375, + "reward_std": 6.907338619232178, + "rewards/rm_reward_func/mean": -14.375091552734375, + "rewards/rm_reward_func/std": 9.020212173461914, + "step": 516 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 378.34375, + "completions/mean_terminated_length": 206.50001525878906, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.4136, + "grad_norm": 25.587743759155273, + "kl": 5.07421875, + "learning_rate": 1e-06, + "loss": 0.3087, + "num_tokens": 6866065.0, + "reward": -9.5302734375, + "reward_std": 5.052003860473633, + "rewards/rm_reward_func/mean": -9.5302734375, + "rewards/rm_reward_func/std": 10.77525520324707, + "step": 517 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 383.625, + "completions/mean_terminated_length": 196.0, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.4144, + "grad_norm": 28.90671730041504, + "kl": 4.85546875, + "learning_rate": 1e-06, + "loss": 0.5353, + "num_tokens": 6883749.0, + "reward": -7.447265625, + "reward_std": 11.519471168518066, + "rewards/rm_reward_func/mean": -7.447265625, + "rewards/rm_reward_func/std": 13.736445426940918, + "step": 518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.71875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 460.6875, + "completions/mean_terminated_length": 329.5555725097656, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "epoch": 0.4152, + "grad_norm": 6.363600254058838, + "kl": 5.875, + "learning_rate": 1e-06, + "loss": 0.3459, + "num_tokens": 6901867.0, + "reward": -8.869140625, + "reward_std": 12.279427528381348, + "rewards/rm_reward_func/mean": -8.869140625, + "rewards/rm_reward_func/std": 18.361650466918945, + "step": 519 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 426.90625, + "completions/mean_terminated_length": 317.5, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.416, + "grad_norm": 12.628585815429688, + "kl": 3.195068359375, + "learning_rate": 1e-06, + "loss": 0.2452, + "num_tokens": 6918720.0, + "reward": -3.0980224609375, + "reward_std": 8.710382461547852, + "rewards/rm_reward_func/mean": -3.0980224609375, + "rewards/rm_reward_func/std": 11.913897514343262, + "step": 520 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 423.0, + "completions/mean_length": 296.28125, + "completions/mean_terminated_length": 166.85000610351562, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.4168, + "grad_norm": 17.954702377319336, + "kl": 5.048828125, + "learning_rate": 1e-06, + "loss": 0.4875, + "num_tokens": 6931665.0, + "reward": 0.85546875, + "reward_std": 10.385427474975586, + "rewards/rm_reward_func/mean": 0.85546875, + "rewards/rm_reward_func/std": 15.259444236755371, + "step": 521 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 409.40625, + "completions/mean_terminated_length": 277.5, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.4176, + "grad_norm": 19.831439971923828, + "kl": 6.078125, + "learning_rate": 1e-06, + "loss": 0.4218, + "num_tokens": 6946854.0, + "reward": -6.93914794921875, + "reward_std": 13.028332710266113, + "rewards/rm_reward_func/mean": -6.93914794921875, + "rewards/rm_reward_func/std": 14.581646919250488, + "step": 522 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 438.09375, + "completions/mean_terminated_length": 364.1875, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.4184, + "grad_norm": 30.67502212524414, + "kl": 4.91064453125, + "learning_rate": 1e-06, + "loss": 0.2753, + "num_tokens": 6963185.0, + "reward": -6.296905517578125, + "reward_std": 5.502427101135254, + "rewards/rm_reward_func/mean": -6.296905517578125, + "rewards/rm_reward_func/std": 12.138696670532227, + "step": 523 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 322.0, + "completions/mean_length": 346.78125, + "completions/mean_terminated_length": 134.35714721679688, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.4192, + "grad_norm": 33.20919418334961, + "kl": 6.63623046875, + "learning_rate": 1e-06, + "loss": 0.4747, + "num_tokens": 6976906.0, + "reward": -11.229736328125, + "reward_std": 4.540736198425293, + "rewards/rm_reward_func/mean": -11.229736328125, + "rewards/rm_reward_func/std": 11.639115333557129, + "step": 524 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 370.03125, + "completions/mean_terminated_length": 343.7407531738281, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.42, + "grad_norm": 9.042720794677734, + "kl": 0.87451171875, + "learning_rate": 1e-06, + "loss": 0.174, + "num_tokens": 6990763.0, + "reward": 4.61993408203125, + "reward_std": 7.443538665771484, + "rewards/rm_reward_func/mean": 4.61993408203125, + "rewards/rm_reward_func/std": 12.067349433898926, + "step": 525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 399.65625, + "completions/mean_terminated_length": 152.5, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.4208, + "grad_norm": 23.3360538482666, + "kl": 12.09375, + "learning_rate": 1e-06, + "loss": 0.8084, + "num_tokens": 7008024.0, + "reward": -11.912353515625, + "reward_std": 9.951939582824707, + "rewards/rm_reward_func/mean": -11.912353515625, + "rewards/rm_reward_func/std": 15.001555442810059, + "step": 526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 364.9375, + "completions/mean_terminated_length": 287.9047546386719, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.4216, + "grad_norm": 11.807538986206055, + "kl": 3.63671875, + "learning_rate": 1e-06, + "loss": 0.2078, + "num_tokens": 7023238.0, + "reward": 1.132568359375, + "reward_std": 11.639840126037598, + "rewards/rm_reward_func/mean": 1.132568359375, + "rewards/rm_reward_func/std": 15.697065353393555, + "step": 527 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 325.0, + "completions/mean_length": 308.46875, + "completions/mean_terminated_length": 150.1666717529297, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.4224, + "grad_norm": 10.746874809265137, + "kl": 7.41259765625, + "learning_rate": 1e-06, + "loss": 0.4985, + "num_tokens": 7035693.0, + "reward": -4.9210205078125, + "reward_std": 7.971255302429199, + "rewards/rm_reward_func/mean": -4.9210205078125, + "rewards/rm_reward_func/std": 17.77606773376465, + "step": 528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 450.3125, + "completions/mean_terminated_length": 360.15386962890625, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "epoch": 0.4232, + "grad_norm": 18.503520965576172, + "kl": 9.25390625, + "learning_rate": 1e-06, + "loss": 0.4482, + "num_tokens": 7054207.0, + "reward": 1.451171875, + "reward_std": 22.573057174682617, + "rewards/rm_reward_func/mean": 1.451171875, + "rewards/rm_reward_func/std": 27.66277313232422, + "step": 529 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.84375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 328.0, + "completions/mean_length": 467.78125, + "completions/mean_terminated_length": 229.0, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.424, + "grad_norm": 47.28842544555664, + "kl": 14.5625, + "learning_rate": 1e-06, + "loss": 0.7504, + "num_tokens": 7072056.0, + "reward": -19.1044921875, + "reward_std": 12.893132209777832, + "rewards/rm_reward_func/mean": -19.1044921875, + "rewards/rm_reward_func/std": 14.44466781616211, + "step": 530 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.8125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 424.0, + "completions/mean_length": 474.28125, + "completions/mean_terminated_length": 310.8333435058594, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "epoch": 0.4248, + "grad_norm": 44.54637908935547, + "kl": 8.7578125, + "learning_rate": 1e-06, + "loss": 0.4787, + "num_tokens": 7089625.0, + "reward": -19.9476318359375, + "reward_std": 8.834146499633789, + "rewards/rm_reward_func/mean": -19.9476318359375, + "rewards/rm_reward_func/std": 10.349533081054688, + "step": 531 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 366.28125, + "completions/mean_terminated_length": 178.92857360839844, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.4256, + "grad_norm": 522.36865234375, + "kl": 6.591796875, + "learning_rate": 1e-06, + "loss": 0.6736, + "num_tokens": 7105522.0, + "reward": -1.458984375, + "reward_std": 9.65514850616455, + "rewards/rm_reward_func/mean": -1.458984375, + "rewards/rm_reward_func/std": 23.94460678100586, + "step": 532 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 390.0, + "completions/mean_length": 395.84375, + "completions/mean_terminated_length": 202.25, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.4264, + "grad_norm": 39.247413635253906, + "kl": 7.3671875, + "learning_rate": 1e-06, + "loss": 0.569, + "num_tokens": 7120965.0, + "reward": -13.78955078125, + "reward_std": 11.25367546081543, + "rewards/rm_reward_func/mean": -13.78955078125, + "rewards/rm_reward_func/std": 15.153644561767578, + "step": 533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 418.6875, + "completions/mean_terminated_length": 282.3077087402344, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.4272, + "grad_norm": 21.322107315063477, + "kl": 4.0390625, + "learning_rate": 1e-06, + "loss": 0.2022, + "num_tokens": 7136419.0, + "reward": -12.512939453125, + "reward_std": 14.791849136352539, + "rewards/rm_reward_func/mean": -12.512939453125, + "rewards/rm_reward_func/std": 16.32183074951172, + "step": 534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 441.0, + "completions/mean_length": 434.34375, + "completions/mean_terminated_length": 263.5, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.428, + "grad_norm": 18.7730770111084, + "kl": 5.703125, + "learning_rate": 1e-06, + "loss": 0.4185, + "num_tokens": 7152758.0, + "reward": -8.214715957641602, + "reward_std": 9.016678810119629, + "rewards/rm_reward_func/mean": -8.214715957641602, + "rewards/rm_reward_func/std": 10.963976860046387, + "step": 535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 449.0, + "completions/mean_length": 364.15625, + "completions/mean_terminated_length": 275.45001220703125, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.4288, + "grad_norm": 440.05767822265625, + "kl": 5.39599609375, + "learning_rate": 1e-06, + "loss": 0.5068, + "num_tokens": 7167107.0, + "reward": -0.919921875, + "reward_std": 10.649211883544922, + "rewards/rm_reward_func/mean": -0.919921875, + "rewards/rm_reward_func/std": 23.648954391479492, + "step": 536 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 404.0, + "completions/mean_length": 369.59375, + "completions/mean_terminated_length": 208.20001220703125, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.4296, + "grad_norm": 12.493600845336914, + "kl": 5.7001953125, + "learning_rate": 1e-06, + "loss": 0.29, + "num_tokens": 7181822.0, + "reward": -3.5146484375, + "reward_std": 7.881195068359375, + "rewards/rm_reward_func/mean": -3.5146484375, + "rewards/rm_reward_func/std": 15.348002433776855, + "step": 537 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 305.0, + "completions/mean_length": 395.65625, + "completions/mean_terminated_length": 173.5454559326172, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.4304, + "grad_norm": 57.40653991699219, + "kl": 9.09375, + "learning_rate": 1e-06, + "loss": 0.7079, + "num_tokens": 7201579.0, + "reward": -9.9658203125, + "reward_std": 12.702253341674805, + "rewards/rm_reward_func/mean": -9.9658203125, + "rewards/rm_reward_func/std": 15.439082145690918, + "step": 538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 296.0, + "completions/mean_length": 341.8125, + "completions/mean_terminated_length": 93.0769271850586, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.4312, + "grad_norm": 15.874788284301758, + "kl": 9.1171875, + "learning_rate": 1e-06, + "loss": 0.6786, + "num_tokens": 7218213.0, + "reward": -14.662109375, + "reward_std": 6.063241004943848, + "rewards/rm_reward_func/mean": -14.662109375, + "rewards/rm_reward_func/std": 12.366585731506348, + "step": 539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.84375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 466.03125, + "completions/mean_terminated_length": 217.8000030517578, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.432, + "grad_norm": 16.715585708618164, + "kl": 8.3671875, + "learning_rate": 1e-06, + "loss": 0.3327, + "num_tokens": 7235414.0, + "reward": -15.6708984375, + "reward_std": 11.589529991149902, + "rewards/rm_reward_func/mean": -15.6708984375, + "rewards/rm_reward_func/std": 12.419350624084473, + "step": 540 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 445.1875, + "completions/mean_terminated_length": 369.4666748046875, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "epoch": 0.4328, + "grad_norm": 5.2816667556762695, + "kl": 4.4296875, + "learning_rate": 1e-06, + "loss": 0.2283, + "num_tokens": 7252700.0, + "reward": -0.094482421875, + "reward_std": 17.288135528564453, + "rewards/rm_reward_func/mean": -0.094482421875, + "rewards/rm_reward_func/std": 24.340282440185547, + "step": 541 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 378.9375, + "completions/mean_terminated_length": 287.8947448730469, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.4336, + "grad_norm": 20.250680923461914, + "kl": 4.130859375, + "learning_rate": 1e-06, + "loss": 0.4542, + "num_tokens": 7267650.0, + "reward": -4.664306640625, + "reward_std": 12.11115837097168, + "rewards/rm_reward_func/mean": -4.664306640625, + "rewards/rm_reward_func/std": 14.438556671142578, + "step": 542 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 387.53125, + "completions/mean_terminated_length": 149.90908813476562, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.4344, + "grad_norm": 24.06378173828125, + "kl": 4.3828125, + "learning_rate": 1e-06, + "loss": 0.4955, + "num_tokens": 7282563.0, + "reward": -3.005126953125, + "reward_std": 11.770925521850586, + "rewards/rm_reward_func/mean": -3.005126953125, + "rewards/rm_reward_func/std": 15.770807266235352, + "step": 543 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 264.90625, + "completions/mean_terminated_length": 168.21739196777344, + "completions/min_length": 26.0, + "completions/min_terminated_length": 26.0, + "epoch": 0.4352, + "grad_norm": 27.67669677734375, + "kl": 3.830078125, + "learning_rate": 1e-06, + "loss": 0.5483, + "num_tokens": 7293816.0, + "reward": -2.7064208984375, + "reward_std": 13.272294998168945, + "rewards/rm_reward_func/mean": -2.7064208984375, + "rewards/rm_reward_func/std": 16.109525680541992, + "step": 544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 311.9375, + "completions/mean_terminated_length": 221.0, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.436, + "grad_norm": 20.555767059326172, + "kl": 3.757080078125, + "learning_rate": 1e-06, + "loss": 0.5049, + "num_tokens": 7307534.0, + "reward": -0.9964599609375, + "reward_std": 8.262689590454102, + "rewards/rm_reward_func/mean": -0.9964599609375, + "rewards/rm_reward_func/std": 15.29664421081543, + "step": 545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 466.0, + "completions/mean_length": 331.0625, + "completions/mean_terminated_length": 236.2857208251953, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.4368, + "grad_norm": 19.920595169067383, + "kl": 3.9521484375, + "learning_rate": 1e-06, + "loss": 0.0243, + "num_tokens": 7327104.0, + "reward": -12.331245422363281, + "reward_std": 5.995415687561035, + "rewards/rm_reward_func/mean": -12.331245422363281, + "rewards/rm_reward_func/std": 11.88155746459961, + "step": 546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 444.3125, + "completions/mean_terminated_length": 357.2857360839844, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.4376, + "grad_norm": 21.573915481567383, + "kl": 4.890625, + "learning_rate": 1e-06, + "loss": 0.3193, + "num_tokens": 7344522.0, + "reward": -5.7215576171875, + "reward_std": 17.706233978271484, + "rewards/rm_reward_func/mean": -5.7215576171875, + "rewards/rm_reward_func/std": 20.520912170410156, + "step": 547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 460.0, + "completions/mean_length": 403.375, + "completions/mean_terminated_length": 294.75, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.4384, + "grad_norm": 18.524627685546875, + "kl": 5.6640625, + "learning_rate": 1e-06, + "loss": 0.368, + "num_tokens": 7363686.0, + "reward": -8.279296875, + "reward_std": 14.83357048034668, + "rewards/rm_reward_func/mean": -8.279296875, + "rewards/rm_reward_func/std": 16.11411476135254, + "step": 548 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 251.96875, + "completions/mean_terminated_length": 214.82144165039062, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.4392, + "grad_norm": 17.9769344329834, + "kl": 3.47265625, + "learning_rate": 1e-06, + "loss": 0.5011, + "num_tokens": 7374869.0, + "reward": -9.139419555664062, + "reward_std": 6.864280700683594, + "rewards/rm_reward_func/mean": -9.139419555664062, + "rewards/rm_reward_func/std": 13.617587089538574, + "step": 549 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 391.0, + "completions/mean_length": 335.46875, + "completions/mean_terminated_length": 243.0, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.44, + "grad_norm": 12.295485496520996, + "kl": 6.3740234375, + "learning_rate": 1e-06, + "loss": 0.5446, + "num_tokens": 7388164.0, + "reward": -0.984375, + "reward_std": 13.594259262084961, + "rewards/rm_reward_func/mean": -0.984375, + "rewards/rm_reward_func/std": 17.934398651123047, + "step": 550 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 326.4375, + "completions/mean_terminated_length": 182.11111450195312, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.4408, + "grad_norm": 9.252522468566895, + "kl": 9.421875, + "learning_rate": 1e-06, + "loss": 0.8192, + "num_tokens": 7401826.0, + "reward": -3.5927734375, + "reward_std": 14.570196151733398, + "rewards/rm_reward_func/mean": -3.5927734375, + "rewards/rm_reward_func/std": 18.126157760620117, + "step": 551 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 377.0, + "completions/mean_length": 281.34375, + "completions/mean_terminated_length": 160.52381896972656, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.4416, + "grad_norm": 8.966374397277832, + "kl": 9.5185546875, + "learning_rate": 1e-06, + "loss": 0.6778, + "num_tokens": 7414197.0, + "reward": -1.685546875, + "reward_std": 9.854778289794922, + "rewards/rm_reward_func/mean": -1.685546875, + "rewards/rm_reward_func/std": 13.30488395690918, + "step": 552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 398.0, + "completions/mean_length": 359.75, + "completions/mean_terminated_length": 164.0, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.4424, + "grad_norm": 21.584428787231445, + "kl": 13.09375, + "learning_rate": 1e-06, + "loss": 0.9478, + "num_tokens": 7433813.0, + "reward": -11.2989501953125, + "reward_std": 9.919830322265625, + "rewards/rm_reward_func/mean": -11.2989501953125, + "rewards/rm_reward_func/std": 11.97014045715332, + "step": 553 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 466.0, + "completions/mean_length": 406.59375, + "completions/mean_terminated_length": 230.9166717529297, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.4432, + "grad_norm": 63.04143524169922, + "kl": 14.703125, + "learning_rate": 1e-06, + "loss": 0.7666, + "num_tokens": 7449552.0, + "reward": -16.6484375, + "reward_std": 8.551742553710938, + "rewards/rm_reward_func/mean": -16.6484375, + "rewards/rm_reward_func/std": 16.584556579589844, + "step": 554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 380.5625, + "completions/mean_terminated_length": 264.5882263183594, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.444, + "grad_norm": 592.711669921875, + "kl": 17.265625, + "learning_rate": 1e-06, + "loss": 1.0223, + "num_tokens": 7465730.0, + "reward": -6.50244140625, + "reward_std": 14.185935974121094, + "rewards/rm_reward_func/mean": -6.50244140625, + "rewards/rm_reward_func/std": 18.073198318481445, + "step": 555 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 438.0, + "completions/mean_length": 319.03125, + "completions/mean_terminated_length": 274.5, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.4448, + "grad_norm": 13.58010482788086, + "kl": 7.970703125, + "learning_rate": 1e-06, + "loss": 0.3343, + "num_tokens": 7478107.0, + "reward": -8.147216796875, + "reward_std": 11.794363021850586, + "rewards/rm_reward_func/mean": -8.147216796875, + "rewards/rm_reward_func/std": 15.557478904724121, + "step": 556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 273.84375, + "completions/mean_terminated_length": 239.82144165039062, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.4456, + "grad_norm": 25.78789520263672, + "kl": 7.234619140625, + "learning_rate": 1e-06, + "loss": 0.3103, + "num_tokens": 7492406.0, + "reward": 1.95947265625, + "reward_std": 10.006596565246582, + "rewards/rm_reward_func/mean": 1.95947265625, + "rewards/rm_reward_func/std": 15.28310489654541, + "step": 557 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 225.6875, + "completions/mean_terminated_length": 172.6666717529297, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.4464, + "grad_norm": 16.13897705078125, + "kl": 12.6171875, + "learning_rate": 1e-06, + "loss": 1.0411, + "num_tokens": 7504660.0, + "reward": 0.7138671875, + "reward_std": 15.415876388549805, + "rewards/rm_reward_func/mean": 0.7138671875, + "rewards/rm_reward_func/std": 21.385021209716797, + "step": 558 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 348.0, + "completions/mean_terminated_length": 249.60000610351562, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.4472, + "grad_norm": 16.69782829284668, + "kl": 9.87890625, + "learning_rate": 1e-06, + "loss": 0.6521, + "num_tokens": 7518204.0, + "reward": 0.142578125, + "reward_std": 15.55780029296875, + "rewards/rm_reward_func/mean": 0.142578125, + "rewards/rm_reward_func/std": 18.24248695373535, + "step": 559 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 486.0, + "completions/mean_length": 212.71875, + "completions/mean_terminated_length": 169.96429443359375, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.448, + "grad_norm": 96.58130645751953, + "kl": 6.24658203125, + "learning_rate": 1e-06, + "loss": 0.5224, + "num_tokens": 7527819.0, + "reward": 1.61529541015625, + "reward_std": 13.240062713623047, + "rewards/rm_reward_func/mean": 1.61529541015625, + "rewards/rm_reward_func/std": 14.038684844970703, + "step": 560 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 239.09375, + "completions/mean_terminated_length": 210.86207580566406, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.4488, + "grad_norm": 44.059059143066406, + "kl": 11.875, + "learning_rate": 1e-06, + "loss": 0.4508, + "num_tokens": 7538014.0, + "reward": -12.23046875, + "reward_std": 13.231216430664062, + "rewards/rm_reward_func/mean": -12.23046875, + "rewards/rm_reward_func/std": 17.87043571472168, + "step": 561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 441.0, + "completions/mean_length": 162.28125, + "completions/mean_terminated_length": 126.10344696044922, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.4496, + "grad_norm": 23.391313552856445, + "kl": 6.7119140625, + "learning_rate": 1e-06, + "loss": 0.3591, + "num_tokens": 7546287.0, + "reward": -9.3154296875, + "reward_std": 6.772148132324219, + "rewards/rm_reward_func/mean": -9.3154296875, + "rewards/rm_reward_func/std": 11.018516540527344, + "step": 562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 366.1875, + "completions/mean_terminated_length": 317.5833435058594, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "epoch": 0.4504, + "grad_norm": 41.109046936035156, + "kl": 7.4140625, + "learning_rate": 1e-06, + "loss": 0.3541, + "num_tokens": 7560213.0, + "reward": -8.3702392578125, + "reward_std": 10.981157302856445, + "rewards/rm_reward_func/mean": -8.3702392578125, + "rewards/rm_reward_func/std": 13.88586711883545, + "step": 563 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 339.5, + "completions/mean_terminated_length": 291.1999816894531, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.4512, + "grad_norm": 34.89735412597656, + "kl": 4.27685546875, + "learning_rate": 1e-06, + "loss": 0.0249, + "num_tokens": 7574949.0, + "reward": 1.029541015625, + "reward_std": 12.96914291381836, + "rewards/rm_reward_func/mean": 1.029541015625, + "rewards/rm_reward_func/std": 22.506608963012695, + "step": 564 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 272.375, + "completions/mean_terminated_length": 217.07693481445312, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.452, + "grad_norm": 10.21695613861084, + "kl": 5.6484375, + "learning_rate": 1e-06, + "loss": 0.3193, + "num_tokens": 7586177.0, + "reward": -7.305419921875, + "reward_std": 8.39257526397705, + "rewards/rm_reward_func/mean": -7.305419921875, + "rewards/rm_reward_func/std": 10.208649635314941, + "step": 565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 196.34375, + "completions/mean_terminated_length": 163.6896514892578, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.4528, + "grad_norm": 74.78276062011719, + "kl": 3.7001953125, + "learning_rate": 1e-06, + "loss": -0.0352, + "num_tokens": 7595588.0, + "reward": -7.03125, + "reward_std": 6.5332746505737305, + "rewards/rm_reward_func/mean": -7.03125, + "rewards/rm_reward_func/std": 15.433568000793457, + "step": 566 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 303.4375, + "completions/mean_terminated_length": 289.5333557128906, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.4536, + "grad_norm": 212.21656799316406, + "kl": 2.8095703125, + "learning_rate": 1e-06, + "loss": -0.0082, + "num_tokens": 7607330.0, + "reward": -1.4860076904296875, + "reward_std": 13.193567276000977, + "rewards/rm_reward_func/mean": -1.4860076904296875, + "rewards/rm_reward_func/std": 17.839399337768555, + "step": 567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 236.78125, + "completions/mean_terminated_length": 173.2692413330078, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "epoch": 0.4544, + "grad_norm": 22.839019775390625, + "kl": 4.947509765625, + "learning_rate": 1e-06, + "loss": 0.3036, + "num_tokens": 7619147.0, + "reward": -4.39599609375, + "reward_std": 10.989629745483398, + "rewards/rm_reward_func/mean": -4.39599609375, + "rewards/rm_reward_func/std": 14.866265296936035, + "step": 568 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 261.25, + "completions/mean_terminated_length": 253.16128540039062, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.4552, + "grad_norm": 10.727757453918457, + "kl": 2.3701171875, + "learning_rate": 1e-06, + "loss": -0.1091, + "num_tokens": 7629659.0, + "reward": 2.4486083984375, + "reward_std": 13.246370315551758, + "rewards/rm_reward_func/mean": 2.4486083984375, + "rewards/rm_reward_func/std": 20.726552963256836, + "step": 569 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 495.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 218.90625, + "completions/mean_terminated_length": 218.90625, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.456, + "grad_norm": 12.797094345092773, + "kl": 2.15869140625, + "learning_rate": 1e-06, + "loss": 0.3033, + "num_tokens": 7641048.0, + "reward": 14.328125, + "reward_std": 6.0160746574401855, + "rewards/rm_reward_func/mean": 14.328125, + "rewards/rm_reward_func/std": 12.738174438476562, + "step": 570 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 443.0, + "completions/mean_length": 239.65625, + "completions/mean_terminated_length": 189.22222900390625, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "epoch": 0.4568, + "grad_norm": 28.028533935546875, + "kl": 6.15625, + "learning_rate": 1e-06, + "loss": 0.1429, + "num_tokens": 7651101.0, + "reward": -17.66259765625, + "reward_std": 4.737618446350098, + "rewards/rm_reward_func/mean": -17.66259765625, + "rewards/rm_reward_func/std": 7.526523590087891, + "step": 571 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 244.9375, + "completions/mean_terminated_length": 183.3076934814453, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "epoch": 0.4576, + "grad_norm": 28.566667556762695, + "kl": 4.779541015625, + "learning_rate": 1e-06, + "loss": 0.31, + "num_tokens": 7661483.0, + "reward": -11.293212890625, + "reward_std": 4.30594539642334, + "rewards/rm_reward_func/mean": -11.293212890625, + "rewards/rm_reward_func/std": 9.476552963256836, + "step": 572 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 372.5, + "completions/mean_terminated_length": 299.4285888671875, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.4584, + "grad_norm": 9.447068214416504, + "kl": 3.060302734375, + "learning_rate": 1e-06, + "loss": 0.061, + "num_tokens": 7679483.0, + "reward": -9.2685546875, + "reward_std": 7.054027080535889, + "rewards/rm_reward_func/mean": -9.2685546875, + "rewards/rm_reward_func/std": 12.366097450256348, + "step": 573 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 337.125, + "completions/mean_terminated_length": 304.7407531738281, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.4592, + "grad_norm": 11.237780570983887, + "kl": 4.16357421875, + "learning_rate": 1e-06, + "loss": 0.166, + "num_tokens": 7692759.0, + "reward": -8.243682861328125, + "reward_std": 9.03434944152832, + "rewards/rm_reward_func/mean": -8.243682861328125, + "rewards/rm_reward_func/std": 10.684294700622559, + "step": 574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 259.28125, + "completions/mean_terminated_length": 160.3913116455078, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "epoch": 0.46, + "grad_norm": 26.71776008605957, + "kl": 2.917724609375, + "learning_rate": 1e-06, + "loss": 0.0122, + "num_tokens": 7704504.0, + "reward": -0.0777587890625, + "reward_std": 7.303743839263916, + "rewards/rm_reward_func/mean": -0.0777587890625, + "rewards/rm_reward_func/std": 13.445627212524414, + "step": 575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 329.5625, + "completions/mean_terminated_length": 278.47998046875, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.4608, + "grad_norm": 29.287355422973633, + "kl": 3.5146484375, + "learning_rate": 1e-06, + "loss": 0.1701, + "num_tokens": 7718426.0, + "reward": -9.46527099609375, + "reward_std": 5.940766334533691, + "rewards/rm_reward_func/mean": -9.46527099609375, + "rewards/rm_reward_func/std": 8.79275131225586, + "step": 576 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 271.375, + "completions/mean_terminated_length": 226.8148193359375, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.4616, + "grad_norm": 31.825645446777344, + "kl": 1.86328125, + "learning_rate": 1e-06, + "loss": -0.0067, + "num_tokens": 7731526.0, + "reward": 7.0263671875, + "reward_std": 8.148383140563965, + "rewards/rm_reward_func/mean": 7.0263671875, + "rewards/rm_reward_func/std": 15.413996696472168, + "step": 577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 470.0, + "completions/mean_length": 321.90625, + "completions/mean_terminated_length": 302.2413635253906, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.4624, + "grad_norm": 9.535029411315918, + "kl": 3.79296875, + "learning_rate": 1e-06, + "loss": 0.0741, + "num_tokens": 7746387.0, + "reward": -4.50048828125, + "reward_std": 10.548614501953125, + "rewards/rm_reward_func/mean": -4.50048828125, + "rewards/rm_reward_func/std": 18.594907760620117, + "step": 578 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 201.5, + "completions/mean_terminated_length": 98.0, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.4632, + "grad_norm": 8.471213340759277, + "kl": 2.734375, + "learning_rate": 1e-06, + "loss": -0.1795, + "num_tokens": 7755707.0, + "reward": -7.9576416015625, + "reward_std": 9.555851936340332, + "rewards/rm_reward_func/mean": -7.9576416015625, + "rewards/rm_reward_func/std": 11.002127647399902, + "step": 579 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 220.96875, + "completions/mean_terminated_length": 190.86207580566406, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.464, + "grad_norm": 8.996735572814941, + "kl": 1.244140625, + "learning_rate": 1e-06, + "loss": -0.0699, + "num_tokens": 7765434.0, + "reward": 3.6036376953125, + "reward_std": 8.55752944946289, + "rewards/rm_reward_func/mean": 3.6036376953125, + "rewards/rm_reward_func/std": 18.499204635620117, + "step": 580 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 346.0, + "completions/mean_terminated_length": 281.0434875488281, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.4648, + "grad_norm": 40.83759689331055, + "kl": 0.305419921875, + "learning_rate": 1e-06, + "loss": 0.1, + "num_tokens": 7778234.0, + "reward": 4.37225341796875, + "reward_std": 7.295747756958008, + "rewards/rm_reward_func/mean": 4.37225341796875, + "rewards/rm_reward_func/std": 8.913907051086426, + "step": 581 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 456.0, + "completions/mean_length": 305.96875, + "completions/mean_terminated_length": 237.2916717529297, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.4656, + "grad_norm": 37.06733703613281, + "kl": 2.0625, + "learning_rate": 1e-06, + "loss": -0.0452, + "num_tokens": 7791017.0, + "reward": -5.02978515625, + "reward_std": 5.757181167602539, + "rewards/rm_reward_func/mean": -5.02978515625, + "rewards/rm_reward_func/std": 9.750814437866211, + "step": 582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 238.90625, + "completions/mean_terminated_length": 199.8928680419922, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.4664, + "grad_norm": 34.94654846191406, + "kl": 1.6552734375, + "learning_rate": 1e-06, + "loss": -0.1114, + "num_tokens": 7803718.0, + "reward": 1.8824462890625, + "reward_std": 10.042098999023438, + "rewards/rm_reward_func/mean": 1.8824462890625, + "rewards/rm_reward_func/std": 14.9249906539917, + "step": 583 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 289.09375, + "completions/mean_terminated_length": 257.25, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.4672, + "grad_norm": 4.869449138641357, + "kl": 0.384765625, + "learning_rate": 1e-06, + "loss": -0.0789, + "num_tokens": 7815969.0, + "reward": 8.9342041015625, + "reward_std": 9.483150482177734, + "rewards/rm_reward_func/mean": 8.9342041015625, + "rewards/rm_reward_func/std": 18.35944938659668, + "step": 584 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 462.03125, + "completions/mean_terminated_length": 366.6363830566406, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.468, + "grad_norm": 5.1822285652160645, + "kl": 0.37060546875, + "learning_rate": 1e-06, + "loss": -0.0113, + "num_tokens": 7833826.0, + "reward": 7.9248046875, + "reward_std": 7.081142425537109, + "rewards/rm_reward_func/mean": 7.9248046875, + "rewards/rm_reward_func/std": 15.302535057067871, + "step": 585 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 304.0, + "completions/mean_length": 131.4375, + "completions/mean_terminated_length": 119.16128540039062, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.4688, + "grad_norm": 18.64679718017578, + "kl": 3.083984375, + "learning_rate": 1e-06, + "loss": 0.2691, + "num_tokens": 7841144.0, + "reward": 5.0889892578125, + "reward_std": 6.059321880340576, + "rewards/rm_reward_func/mean": 5.0889892578125, + "rewards/rm_reward_func/std": 9.212634086608887, + "step": 586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 335.84375, + "completions/mean_terminated_length": 310.6785888671875, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.4696, + "grad_norm": 8.921401023864746, + "kl": 2.075439453125, + "learning_rate": 1e-06, + "loss": -0.1186, + "num_tokens": 7853947.0, + "reward": -0.1695556640625, + "reward_std": 9.398602485656738, + "rewards/rm_reward_func/mean": -0.1695556640625, + "rewards/rm_reward_func/std": 15.199664115905762, + "step": 587 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 301.5, + "completions/mean_terminated_length": 271.4285888671875, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "epoch": 0.4704, + "grad_norm": 49.62898635864258, + "kl": 0.86962890625, + "learning_rate": 1e-06, + "loss": -0.058, + "num_tokens": 7865547.0, + "reward": -1.8533935546875, + "reward_std": 7.218017578125, + "rewards/rm_reward_func/mean": -1.8533935546875, + "rewards/rm_reward_func/std": 9.405008316040039, + "step": 588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 333.375, + "completions/mean_terminated_length": 333.375, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.4712, + "grad_norm": 15.543201446533203, + "kl": 0.32958984375, + "learning_rate": 1e-06, + "loss": -0.0359, + "num_tokens": 7879767.0, + "reward": -5.37200927734375, + "reward_std": 3.4672133922576904, + "rewards/rm_reward_func/mean": -5.37200927734375, + "rewards/rm_reward_func/std": 7.8242692947387695, + "step": 589 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 377.84375, + "completions/mean_terminated_length": 340.2799987792969, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.472, + "grad_norm": 7.237336158752441, + "kl": 0.4638671875, + "learning_rate": 1e-06, + "loss": -0.0329, + "num_tokens": 7894162.0, + "reward": 6.933837890625, + "reward_std": 7.349064826965332, + "rewards/rm_reward_func/mean": 6.933837890625, + "rewards/rm_reward_func/std": 12.662103652954102, + "step": 590 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 270.09375, + "completions/mean_terminated_length": 245.0689697265625, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.4728, + "grad_norm": 9.466930389404297, + "kl": 1.3511962890625, + "learning_rate": 1e-06, + "loss": 0.1344, + "num_tokens": 7905197.0, + "reward": -5.54095458984375, + "reward_std": 5.511334419250488, + "rewards/rm_reward_func/mean": -5.54095458984375, + "rewards/rm_reward_func/std": 9.380617141723633, + "step": 591 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 187.5625, + "completions/mean_terminated_length": 127.48148345947266, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "epoch": 0.4736, + "grad_norm": 10.78067684173584, + "kl": 0.6640625, + "learning_rate": 1e-06, + "loss": -0.0794, + "num_tokens": 7913271.0, + "reward": 4.76513671875, + "reward_std": 7.13428258895874, + "rewards/rm_reward_func/mean": 4.76513671875, + "rewards/rm_reward_func/std": 9.546843528747559, + "step": 592 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 297.8125, + "completions/mean_terminated_length": 214.0, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.4744, + "grad_norm": 5.899567604064941, + "kl": 0.46875, + "learning_rate": 1e-06, + "loss": -0.1007, + "num_tokens": 7926073.0, + "reward": 1.1571044921875, + "reward_std": 7.337251663208008, + "rewards/rm_reward_func/mean": 1.1571044921875, + "rewards/rm_reward_func/std": 10.56173038482666, + "step": 593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 446.40625, + "completions/mean_terminated_length": 395.3888854980469, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "epoch": 0.4752, + "grad_norm": 41.35441589355469, + "kl": 2.393310546875, + "learning_rate": 1e-06, + "loss": 0.12, + "num_tokens": 7942310.0, + "reward": -0.016571044921875, + "reward_std": 9.10225772857666, + "rewards/rm_reward_func/mean": -0.016571044921875, + "rewards/rm_reward_func/std": 14.169840812683105, + "step": 594 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 468.0, + "completions/mean_length": 234.15625, + "completions/mean_terminated_length": 215.6333465576172, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.476, + "grad_norm": 13.656573295593262, + "kl": 6.201904296875, + "learning_rate": 1e-06, + "loss": 0.7573, + "num_tokens": 7954699.0, + "reward": 6.20806884765625, + "reward_std": 9.577255249023438, + "rewards/rm_reward_func/mean": 6.20806884765625, + "rewards/rm_reward_func/std": 14.503737449645996, + "step": 595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 455.46875, + "completions/mean_terminated_length": 398.9375, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "epoch": 0.4768, + "grad_norm": 5.957422256469727, + "kl": 2.294677734375, + "learning_rate": 1e-06, + "loss": 0.071, + "num_tokens": 7973066.0, + "reward": -7.954833984375, + "reward_std": 6.558610916137695, + "rewards/rm_reward_func/mean": -7.954833984375, + "rewards/rm_reward_func/std": 9.106958389282227, + "step": 596 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 506.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 269.96875, + "completions/mean_terminated_length": 269.96875, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.4776, + "grad_norm": 8.821357727050781, + "kl": 0.6884765625, + "learning_rate": 1e-06, + "loss": 0.0222, + "num_tokens": 7984641.0, + "reward": 6.18316650390625, + "reward_std": 5.323254585266113, + "rewards/rm_reward_func/mean": 6.18316650390625, + "rewards/rm_reward_func/std": 14.378352165222168, + "step": 597 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 231.8125, + "completions/mean_terminated_length": 222.77418518066406, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.4784, + "grad_norm": 34.12923812866211, + "kl": 7.343505859375, + "learning_rate": 1e-06, + "loss": 0.3296, + "num_tokens": 7995091.0, + "reward": 2.2025146484375, + "reward_std": 7.2968525886535645, + "rewards/rm_reward_func/mean": 2.2025146484375, + "rewards/rm_reward_func/std": 13.586483001708984, + "step": 598 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 302.03125, + "completions/mean_terminated_length": 280.3103332519531, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.4792, + "grad_norm": 6.906167507171631, + "kl": 0.790283203125, + "learning_rate": 1e-06, + "loss": -0.1121, + "num_tokens": 8007900.0, + "reward": 4.49267578125, + "reward_std": 6.870022296905518, + "rewards/rm_reward_func/mean": 4.49267578125, + "rewards/rm_reward_func/std": 9.993231773376465, + "step": 599 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 248.53125, + "completions/mean_terminated_length": 210.8928680419922, + "completions/min_length": 17.0, + "completions/min_terminated_length": 17.0, + "epoch": 0.48, + "grad_norm": 31.199399948120117, + "kl": 5.575927734375, + "learning_rate": 1e-06, + "loss": 0.4506, + "num_tokens": 8022373.0, + "reward": -0.75750732421875, + "reward_std": 8.437265396118164, + "rewards/rm_reward_func/mean": -0.75750732421875, + "rewards/rm_reward_func/std": 14.56766128540039, + "step": 600 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 446.0, + "completions/mean_length": 286.75, + "completions/mean_terminated_length": 234.7692413330078, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.4808, + "grad_norm": 61.780311584472656, + "kl": 6.131103515625, + "learning_rate": 1e-06, + "loss": 0.1509, + "num_tokens": 8034205.0, + "reward": 8.10595703125, + "reward_std": 10.480352401733398, + "rewards/rm_reward_func/mean": 8.10595703125, + "rewards/rm_reward_func/std": 14.536432266235352, + "step": 601 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 477.0, + "completions/mean_length": 346.59375, + "completions/mean_terminated_length": 259.952392578125, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.4816, + "grad_norm": 213.77684020996094, + "kl": 21.95849609375, + "learning_rate": 1e-06, + "loss": 1.2326, + "num_tokens": 8048464.0, + "reward": -6.7283935546875, + "reward_std": 6.977231979370117, + "rewards/rm_reward_func/mean": -6.7283935546875, + "rewards/rm_reward_func/std": 14.559057235717773, + "step": 602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 360.71875, + "completions/mean_terminated_length": 318.3599853515625, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.4824, + "grad_norm": 13.767047882080078, + "kl": 8.62158203125, + "learning_rate": 1e-06, + "loss": 0.4078, + "num_tokens": 8063103.0, + "reward": -1.645751953125, + "reward_std": 12.11515998840332, + "rewards/rm_reward_func/mean": -1.645751953125, + "rewards/rm_reward_func/std": 15.272088050842285, + "step": 603 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 322.0, + "completions/mean_terminated_length": 278.15386962890625, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.4832, + "grad_norm": 24.88362693786621, + "kl": 8.8076171875, + "learning_rate": 1e-06, + "loss": 0.2726, + "num_tokens": 8075391.0, + "reward": -6.1571044921875, + "reward_std": 18.25802230834961, + "rewards/rm_reward_func/mean": -6.1571044921875, + "rewards/rm_reward_func/std": 19.966968536376953, + "step": 604 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 453.0, + "completions/mean_length": 218.71875, + "completions/mean_terminated_length": 176.82144165039062, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.484, + "grad_norm": 11.096504211425781, + "kl": 3.28759765625, + "learning_rate": 1e-06, + "loss": 0.169, + "num_tokens": 8086822.0, + "reward": 6.87744140625, + "reward_std": 7.8228960037231445, + "rewards/rm_reward_func/mean": 6.87744140625, + "rewards/rm_reward_func/std": 12.922216415405273, + "step": 605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 420.03125, + "completions/mean_terminated_length": 398.8077087402344, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "epoch": 0.4848, + "grad_norm": 10.59483528137207, + "kl": 3.473876953125, + "learning_rate": 1e-06, + "loss": 0.0768, + "num_tokens": 8106231.0, + "reward": 15.840576171875, + "reward_std": 15.33732795715332, + "rewards/rm_reward_func/mean": 15.840576171875, + "rewards/rm_reward_func/std": 20.74690818786621, + "step": 606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 429.15625, + "completions/mean_terminated_length": 385.76190185546875, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.4856, + "grad_norm": 9.831181526184082, + "kl": 4.218505859375, + "learning_rate": 1e-06, + "loss": 0.1415, + "num_tokens": 8124388.0, + "reward": -1.52099609375, + "reward_std": 13.66183853149414, + "rewards/rm_reward_func/mean": -1.52099609375, + "rewards/rm_reward_func/std": 18.234188079833984, + "step": 607 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 358.75, + "completions/mean_terminated_length": 323.3846130371094, + "completions/min_length": 14.0, + "completions/min_terminated_length": 14.0, + "epoch": 0.4864, + "grad_norm": 10.595935821533203, + "kl": 2.94921875, + "learning_rate": 1e-06, + "loss": -0.0267, + "num_tokens": 8138108.0, + "reward": 5.33203125, + "reward_std": 11.863018035888672, + "rewards/rm_reward_func/mean": 5.33203125, + "rewards/rm_reward_func/std": 24.679624557495117, + "step": 608 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 460.0, + "completions/mean_length": 217.8125, + "completions/mean_terminated_length": 208.32257080078125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.4872, + "grad_norm": 22.514549255371094, + "kl": 5.7451171875, + "learning_rate": 1e-06, + "loss": 0.3125, + "num_tokens": 8150726.0, + "reward": 0.619384765625, + "reward_std": 10.594083786010742, + "rewards/rm_reward_func/mean": 0.619384765625, + "rewards/rm_reward_func/std": 15.584603309631348, + "step": 609 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 392.0, + "completions/mean_length": 236.28125, + "completions/mean_terminated_length": 207.7586212158203, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.488, + "grad_norm": 25.03614616394043, + "kl": 8.80859375, + "learning_rate": 1e-06, + "loss": 0.7636, + "num_tokens": 8162679.0, + "reward": 4.9461517333984375, + "reward_std": 10.481914520263672, + "rewards/rm_reward_func/mean": 4.9461517333984375, + "rewards/rm_reward_func/std": 14.789458274841309, + "step": 610 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 405.28125, + "completions/mean_terminated_length": 363.5217590332031, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.4888, + "grad_norm": 9.863654136657715, + "kl": 4.548828125, + "learning_rate": 1e-06, + "loss": 0.1808, + "num_tokens": 8177880.0, + "reward": 1.1624755859375, + "reward_std": 12.410948753356934, + "rewards/rm_reward_func/mean": 1.1624755859375, + "rewards/rm_reward_func/std": 17.47865867614746, + "step": 611 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 382.3125, + "completions/mean_terminated_length": 358.2962951660156, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.4896, + "grad_norm": 16.073301315307617, + "kl": 3.01123046875, + "learning_rate": 1e-06, + "loss": 0.0967, + "num_tokens": 8192650.0, + "reward": 0.911651611328125, + "reward_std": 8.299646377563477, + "rewards/rm_reward_func/mean": 0.911651611328125, + "rewards/rm_reward_func/std": 14.285723686218262, + "step": 612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 450.0, + "completions/mean_length": 341.65625, + "completions/mean_terminated_length": 275.0, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.4904, + "grad_norm": 37.6219596862793, + "kl": 9.759765625, + "learning_rate": 1e-06, + "loss": 0.5727, + "num_tokens": 8205711.0, + "reward": -1.9541015625, + "reward_std": 9.328568458557129, + "rewards/rm_reward_func/mean": -1.9541015625, + "rewards/rm_reward_func/std": 21.1943302154541, + "step": 613 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 267.03125, + "completions/mean_terminated_length": 185.375, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.4912, + "grad_norm": 153.5972900390625, + "kl": 7.2421875, + "learning_rate": 1e-06, + "loss": 0.3504, + "num_tokens": 8218824.0, + "reward": -9.59063720703125, + "reward_std": 9.781373023986816, + "rewards/rm_reward_func/mean": -9.59063720703125, + "rewards/rm_reward_func/std": 11.414146423339844, + "step": 614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 188.53125, + "completions/mean_terminated_length": 166.9666748046875, + "completions/min_length": 12.0, + "completions/min_terminated_length": 12.0, + "epoch": 0.492, + "grad_norm": 28.16669464111328, + "kl": 5.5712890625, + "learning_rate": 1e-06, + "loss": 0.1814, + "num_tokens": 8228137.0, + "reward": -3.1532211303710938, + "reward_std": 12.097888946533203, + "rewards/rm_reward_func/mean": -3.1532211303710938, + "rewards/rm_reward_func/std": 20.61545181274414, + "step": 615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 313.53125, + "completions/mean_terminated_length": 235.86956787109375, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "epoch": 0.4928, + "grad_norm": 40.259830474853516, + "kl": 10.94140625, + "learning_rate": 1e-06, + "loss": 0.5348, + "num_tokens": 8241418.0, + "reward": -5.6474609375, + "reward_std": 11.151725769042969, + "rewards/rm_reward_func/mean": -5.6474609375, + "rewards/rm_reward_func/std": 13.004457473754883, + "step": 616 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 265.28125, + "completions/mean_terminated_length": 230.0357208251953, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "epoch": 0.4936, + "grad_norm": 9.916903495788574, + "kl": 5.4296875, + "learning_rate": 1e-06, + "loss": 0.1988, + "num_tokens": 8251955.0, + "reward": -0.9298095703125, + "reward_std": 15.462509155273438, + "rewards/rm_reward_func/mean": -0.9298095703125, + "rewards/rm_reward_func/std": 16.872419357299805, + "step": 617 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 471.0, + "completions/mean_length": 204.09375, + "completions/mean_terminated_length": 172.2413787841797, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.4944, + "grad_norm": 63.2779655456543, + "kl": 8.203125, + "learning_rate": 1e-06, + "loss": 0.2805, + "num_tokens": 8264246.0, + "reward": -4.11572265625, + "reward_std": 13.06856918334961, + "rewards/rm_reward_func/mean": -4.11572265625, + "rewards/rm_reward_func/std": 14.47628116607666, + "step": 618 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 346.78125, + "completions/mean_terminated_length": 247.65000915527344, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "epoch": 0.4952, + "grad_norm": 44.88002014160156, + "kl": 9.136474609375, + "learning_rate": 1e-06, + "loss": 0.5063, + "num_tokens": 8277079.0, + "reward": -9.4381103515625, + "reward_std": 5.765872955322266, + "rewards/rm_reward_func/mean": -9.4381103515625, + "rewards/rm_reward_func/std": 11.600118637084961, + "step": 619 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 480.0, + "completions/mean_length": 293.15625, + "completions/mean_terminated_length": 270.5172424316406, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.496, + "grad_norm": 37.431556701660156, + "kl": 2.302734375, + "learning_rate": 1e-06, + "loss": 0.0574, + "num_tokens": 8288420.0, + "reward": -2.6103515625, + "reward_std": 13.910895347595215, + "rewards/rm_reward_func/mean": -2.6103515625, + "rewards/rm_reward_func/std": 16.85130500793457, + "step": 620 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 473.0, + "completions/mean_length": 258.84375, + "completions/mean_terminated_length": 222.6785888671875, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.4968, + "grad_norm": 13.83285903930664, + "kl": 7.1640625, + "learning_rate": 1e-06, + "loss": 0.2555, + "num_tokens": 8298487.0, + "reward": -6.125762939453125, + "reward_std": 8.449478149414062, + "rewards/rm_reward_func/mean": -6.125762939453125, + "rewards/rm_reward_func/std": 9.483611106872559, + "step": 621 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 435.0, + "completions/mean_length": 191.71875, + "completions/mean_terminated_length": 170.36666870117188, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.4976, + "grad_norm": 12.294360160827637, + "kl": 5.13671875, + "learning_rate": 1e-06, + "loss": 0.2798, + "num_tokens": 8310150.0, + "reward": 7.74169921875, + "reward_std": 11.26545238494873, + "rewards/rm_reward_func/mean": 7.74169921875, + "rewards/rm_reward_func/std": 15.538458824157715, + "step": 622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 461.0, + "completions/mean_length": 234.125, + "completions/mean_terminated_length": 170.0, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.4984, + "grad_norm": 7.7052130699157715, + "kl": 2.001953125, + "learning_rate": 1e-06, + "loss": 0.0997, + "num_tokens": 8321794.0, + "reward": 6.1068115234375, + "reward_std": 6.466403961181641, + "rewards/rm_reward_func/mean": 6.1068115234375, + "rewards/rm_reward_func/std": 10.666815757751465, + "step": 623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 489.0, + "completions/mean_length": 271.15625, + "completions/mean_terminated_length": 226.55555725097656, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.4992, + "grad_norm": 37.59298324584961, + "kl": 6.85546875, + "learning_rate": 1e-06, + "loss": 0.4438, + "num_tokens": 8333623.0, + "reward": -11.0943603515625, + "reward_std": 6.584903717041016, + "rewards/rm_reward_func/mean": -11.0943603515625, + "rewards/rm_reward_func/std": 11.544700622558594, + "step": 624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 282.40625, + "completions/mean_terminated_length": 205.875, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.5, + "grad_norm": 50.945106506347656, + "kl": 7.80419921875, + "learning_rate": 1e-06, + "loss": 0.1727, + "num_tokens": 8346988.0, + "reward": -10.51025390625, + "reward_std": 4.640988349914551, + "rewards/rm_reward_func/mean": -10.51025390625, + "rewards/rm_reward_func/std": 19.07770538330078, + "step": 625 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 276.5625, + "completions/mean_terminated_length": 260.8666687011719, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.5008, + "grad_norm": 8.402812957763672, + "kl": 3.9267578125, + "learning_rate": 1e-06, + "loss": 0.0878, + "num_tokens": 8358310.0, + "reward": 10.74951171875, + "reward_std": 11.159141540527344, + "rewards/rm_reward_func/mean": 10.74951171875, + "rewards/rm_reward_func/std": 24.454740524291992, + "step": 626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 314.59375, + "completions/mean_terminated_length": 294.17242431640625, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.5016, + "grad_norm": 7.214862823486328, + "kl": 3.716796875, + "learning_rate": 1e-06, + "loss": 0.1032, + "num_tokens": 8371249.0, + "reward": -1.5455322265625, + "reward_std": 13.805252075195312, + "rewards/rm_reward_func/mean": -1.5455322265625, + "rewards/rm_reward_func/std": 15.483540534973145, + "step": 627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 311.90625, + "completions/mean_terminated_length": 255.87998962402344, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.5024, + "grad_norm": 5.6593217849731445, + "kl": 2.5107421875, + "learning_rate": 1e-06, + "loss": -0.0017, + "num_tokens": 8383638.0, + "reward": 5.8603515625, + "reward_std": 11.585649490356445, + "rewards/rm_reward_func/mean": 5.8603515625, + "rewards/rm_reward_func/std": 14.266550064086914, + "step": 628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 272.90625, + "completions/mean_terminated_length": 238.75001525878906, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.5032, + "grad_norm": 11.517411231994629, + "kl": 6.453125, + "learning_rate": 1e-06, + "loss": 0.2772, + "num_tokens": 8394563.0, + "reward": -4.296142578125, + "reward_std": 15.2503023147583, + "rewards/rm_reward_func/mean": -4.296142578125, + "rewards/rm_reward_func/std": 15.96473217010498, + "step": 629 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 310.53125, + "completions/mean_terminated_length": 281.75, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "epoch": 0.504, + "grad_norm": 6.986127853393555, + "kl": 1.2666015625, + "learning_rate": 1e-06, + "loss": -0.0682, + "num_tokens": 8410244.0, + "reward": 1.4921875, + "reward_std": 4.761030673980713, + "rewards/rm_reward_func/mean": 1.4921875, + "rewards/rm_reward_func/std": 14.25729751586914, + "step": 630 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 302.84375, + "completions/mean_terminated_length": 288.9000244140625, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.5048, + "grad_norm": 7.896935939788818, + "kl": 4.4833984375, + "learning_rate": 1e-06, + "loss": 0.0858, + "num_tokens": 8424207.0, + "reward": 5.5654296875, + "reward_std": 8.17580509185791, + "rewards/rm_reward_func/mean": 5.5654296875, + "rewards/rm_reward_func/std": 20.969762802124023, + "step": 631 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 271.1875, + "completions/mean_terminated_length": 246.27586364746094, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.5056, + "grad_norm": 10.68520450592041, + "kl": 1.8544921875, + "learning_rate": 1e-06, + "loss": 0.1494, + "num_tokens": 8436013.0, + "reward": 1.7723541259765625, + "reward_std": 9.819549560546875, + "rewards/rm_reward_func/mean": 1.7723541259765625, + "rewards/rm_reward_func/std": 15.363565444946289, + "step": 632 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 327.875, + "completions/mean_terminated_length": 315.6000061035156, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.5064, + "grad_norm": 8.58014965057373, + "kl": 1.8515625, + "learning_rate": 1e-06, + "loss": 0.1069, + "num_tokens": 8449793.0, + "reward": 6.67364501953125, + "reward_std": 11.383968353271484, + "rewards/rm_reward_func/mean": 6.67364501953125, + "rewards/rm_reward_func/std": 13.357749938964844, + "step": 633 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 450.0, + "completions/mean_length": 272.25, + "completions/mean_terminated_length": 205.1199951171875, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.5072, + "grad_norm": 9.852457046508789, + "kl": 2.81201171875, + "learning_rate": 1e-06, + "loss": 0.135, + "num_tokens": 8460897.0, + "reward": -4.5697021484375, + "reward_std": 5.767214775085449, + "rewards/rm_reward_func/mean": -4.5697021484375, + "rewards/rm_reward_func/std": 9.804206848144531, + "step": 634 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 277.375, + "completions/mean_terminated_length": 253.10345458984375, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.508, + "grad_norm": 13.91521167755127, + "kl": 4.9462890625, + "learning_rate": 1e-06, + "loss": 0.1343, + "num_tokens": 8472789.0, + "reward": 0.547698974609375, + "reward_std": 12.389558792114258, + "rewards/rm_reward_func/mean": 0.547698974609375, + "rewards/rm_reward_func/std": 21.48679542541504, + "step": 635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 246.15625, + "completions/mean_terminated_length": 196.92593383789062, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.5088, + "grad_norm": 10.53650188446045, + "kl": 2.0712890625, + "learning_rate": 1e-06, + "loss": 0.1688, + "num_tokens": 8484914.0, + "reward": -1.3759765625, + "reward_std": 2.3449654579162598, + "rewards/rm_reward_func/mean": -1.3759765625, + "rewards/rm_reward_func/std": 12.400309562683105, + "step": 636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 333.875, + "completions/mean_terminated_length": 264.1739196777344, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.5096, + "grad_norm": 24.877025604248047, + "kl": 6.34228515625, + "learning_rate": 1e-06, + "loss": 0.2245, + "num_tokens": 8500902.0, + "reward": -9.4365234375, + "reward_std": 3.848327159881592, + "rewards/rm_reward_func/mean": -9.4365234375, + "rewards/rm_reward_func/std": 13.026070594787598, + "step": 637 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 410.40625, + "completions/mean_terminated_length": 364.227294921875, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.5104, + "grad_norm": 8.95622730255127, + "kl": 1.86279296875, + "learning_rate": 1e-06, + "loss": -0.0112, + "num_tokens": 8515907.0, + "reward": 1.7091064453125, + "reward_std": 10.488420486450195, + "rewards/rm_reward_func/mean": 1.7091064453125, + "rewards/rm_reward_func/std": 15.33460807800293, + "step": 638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 274.0625, + "completions/mean_terminated_length": 207.44000244140625, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.5112, + "grad_norm": 13.682821273803711, + "kl": 5.75537109375, + "learning_rate": 1e-06, + "loss": 0.0707, + "num_tokens": 8528813.0, + "reward": -12.8310546875, + "reward_std": 8.058960914611816, + "rewards/rm_reward_func/mean": -12.8310546875, + "rewards/rm_reward_func/std": 10.044072151184082, + "step": 639 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 252.125, + "completions/mean_terminated_length": 215.00001525878906, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.512, + "grad_norm": 10.187383651733398, + "kl": 3.99658203125, + "learning_rate": 1e-06, + "loss": 0.1334, + "num_tokens": 8539441.0, + "reward": 1.024169921875, + "reward_std": 7.485830783843994, + "rewards/rm_reward_func/mean": 1.024169921875, + "rewards/rm_reward_func/std": 11.878993034362793, + "step": 640 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 197.96875, + "completions/mean_terminated_length": 139.8148193359375, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.5128, + "grad_norm": 6.773480415344238, + "kl": 4.6123046875, + "learning_rate": 1e-06, + "loss": 0.2292, + "num_tokens": 8549136.0, + "reward": -1.02978515625, + "reward_std": 5.95222282409668, + "rewards/rm_reward_func/mean": -1.02978515625, + "rewards/rm_reward_func/std": 9.468921661376953, + "step": 641 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 480.0, + "completions/mean_length": 213.53125, + "completions/mean_terminated_length": 170.8928680419922, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.5136, + "grad_norm": 6.013530254364014, + "kl": 0.498046875, + "learning_rate": 1e-06, + "loss": 0.0151, + "num_tokens": 8560257.0, + "reward": 8.07672119140625, + "reward_std": 5.212985038757324, + "rewards/rm_reward_func/mean": 8.07672119140625, + "rewards/rm_reward_func/std": 9.585315704345703, + "step": 642 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 419.125, + "completions/mean_terminated_length": 393.1199951171875, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.5144, + "grad_norm": 6.766341686248779, + "kl": 1.09765625, + "learning_rate": 1e-06, + "loss": 0.0954, + "num_tokens": 8575533.0, + "reward": 14.398681640625, + "reward_std": 10.005803108215332, + "rewards/rm_reward_func/mean": 14.398681640625, + "rewards/rm_reward_func/std": 12.649359703063965, + "step": 643 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 221.21875, + "completions/mean_terminated_length": 167.37037658691406, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.5152, + "grad_norm": 15.727855682373047, + "kl": 0.90576171875, + "learning_rate": 1e-06, + "loss": -0.0352, + "num_tokens": 8587596.0, + "reward": 7.632568359375, + "reward_std": 3.306748867034912, + "rewards/rm_reward_func/mean": 7.632568359375, + "rewards/rm_reward_func/std": 9.895013809204102, + "step": 644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 465.0, + "completions/mean_length": 252.40625, + "completions/mean_terminated_length": 235.10000610351562, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.516, + "grad_norm": 20.60538673400879, + "kl": 1.96630859375, + "learning_rate": 1e-06, + "loss": -0.0003, + "num_tokens": 8599505.0, + "reward": 5.3416748046875, + "reward_std": 11.507960319519043, + "rewards/rm_reward_func/mean": 5.3416748046875, + "rewards/rm_reward_func/std": 17.1043758392334, + "step": 645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 469.0, + "completions/mean_length": 281.15625, + "completions/mean_terminated_length": 77.47058868408203, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.5168, + "grad_norm": 10.928936004638672, + "kl": 1.356201171875, + "learning_rate": 1e-06, + "loss": 0.0367, + "num_tokens": 8610998.0, + "reward": -1.3593597412109375, + "reward_std": 3.8079264163970947, + "rewards/rm_reward_func/mean": -1.3593597412109375, + "rewards/rm_reward_func/std": 8.6522216796875, + "step": 646 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 486.0, + "completions/mean_length": 362.59375, + "completions/mean_terminated_length": 312.79168701171875, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.5176, + "grad_norm": 7.342215538024902, + "kl": 2.76806640625, + "learning_rate": 1e-06, + "loss": 0.1712, + "num_tokens": 8625361.0, + "reward": -2.8857421875, + "reward_std": 11.833799362182617, + "rewards/rm_reward_func/mean": -2.8857421875, + "rewards/rm_reward_func/std": 15.419763565063477, + "step": 647 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 392.90625, + "completions/mean_terminated_length": 321.45001220703125, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.5184, + "grad_norm": 16.402130126953125, + "kl": 4.22412109375, + "learning_rate": 1e-06, + "loss": 0.1936, + "num_tokens": 8639934.0, + "reward": 0.00299072265625, + "reward_std": 5.563260078430176, + "rewards/rm_reward_func/mean": 0.00299072265625, + "rewards/rm_reward_func/std": 17.13874053955078, + "step": 648 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 484.0, + "completions/mean_length": 174.40625, + "completions/mean_terminated_length": 163.51612854003906, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.5192, + "grad_norm": 8.888167381286621, + "kl": 0.8353271484375, + "learning_rate": 1e-06, + "loss": -0.0946, + "num_tokens": 8652491.0, + "reward": 0.316680908203125, + "reward_std": 4.313920021057129, + "rewards/rm_reward_func/mean": 0.316680908203125, + "rewards/rm_reward_func/std": 10.923198699951172, + "step": 649 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 290.03125, + "completions/mean_terminated_length": 189.13636779785156, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "epoch": 0.52, + "grad_norm": 6.159624099731445, + "kl": 1.846435546875, + "learning_rate": 1e-06, + "loss": 0.007, + "num_tokens": 8665452.0, + "reward": 4.464111328125, + "reward_std": 7.780835151672363, + "rewards/rm_reward_func/mean": 4.464111328125, + "rewards/rm_reward_func/std": 10.548760414123535, + "step": 650 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 265.6875, + "completions/mean_terminated_length": 208.84616088867188, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.5208, + "grad_norm": 11.240917205810547, + "kl": 1.3095703125, + "learning_rate": 1e-06, + "loss": 0.063, + "num_tokens": 8676746.0, + "reward": 2.96875, + "reward_std": 5.190629005432129, + "rewards/rm_reward_func/mean": 2.96875, + "rewards/rm_reward_func/std": 12.547528266906738, + "step": 651 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 453.0, + "completions/mean_length": 299.84375, + "completions/mean_terminated_length": 134.8333282470703, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.5216, + "grad_norm": 9.644030570983887, + "kl": 0.29296875, + "learning_rate": 1e-06, + "loss": -0.0588, + "num_tokens": 8689357.0, + "reward": 0.384521484375, + "reward_std": 3.9550719261169434, + "rewards/rm_reward_func/mean": 0.384521484375, + "rewards/rm_reward_func/std": 12.727936744689941, + "step": 652 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 347.9375, + "completions/mean_terminated_length": 317.5555725097656, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.5224, + "grad_norm": 10.710493087768555, + "kl": 0.42578125, + "learning_rate": 1e-06, + "loss": 0.0054, + "num_tokens": 8702963.0, + "reward": 10.232421875, + "reward_std": 6.635782241821289, + "rewards/rm_reward_func/mean": 10.232421875, + "rewards/rm_reward_func/std": 9.094783782958984, + "step": 653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 470.0, + "completions/mean_length": 320.875, + "completions/mean_terminated_length": 301.10345458984375, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "epoch": 0.5232, + "grad_norm": 6.3006792068481445, + "kl": 0.252197265625, + "learning_rate": 1e-06, + "loss": 0.0086, + "num_tokens": 8716151.0, + "reward": 8.225341796875, + "reward_std": 7.599928855895996, + "rewards/rm_reward_func/mean": 8.225341796875, + "rewards/rm_reward_func/std": 19.037904739379883, + "step": 654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 375.875, + "completions/mean_terminated_length": 294.20001220703125, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.524, + "grad_norm": 8.997413635253906, + "kl": 0.244140625, + "learning_rate": 1e-06, + "loss": 0.0777, + "num_tokens": 8732867.0, + "reward": 4.292236328125, + "reward_std": 7.305292129516602, + "rewards/rm_reward_func/mean": 4.292236328125, + "rewards/rm_reward_func/std": 11.710954666137695, + "step": 655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 357.53125, + "completions/mean_terminated_length": 297.08697509765625, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.5248, + "grad_norm": 6.956607818603516, + "kl": 0.75, + "learning_rate": 1e-06, + "loss": 0.0366, + "num_tokens": 8747420.0, + "reward": 11.81103515625, + "reward_std": 6.258151054382324, + "rewards/rm_reward_func/mean": 11.81103515625, + "rewards/rm_reward_func/std": 15.444485664367676, + "step": 656 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 507.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 360.75, + "completions/mean_terminated_length": 360.75, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.5256, + "grad_norm": 6.72127103805542, + "kl": 0.552490234375, + "learning_rate": 1e-06, + "loss": 0.0153, + "num_tokens": 8760988.0, + "reward": 10.35595703125, + "reward_std": 5.374138832092285, + "rewards/rm_reward_func/mean": 10.35595703125, + "rewards/rm_reward_func/std": 6.271159648895264, + "step": 657 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 286.84375, + "completions/mean_terminated_length": 223.79998779296875, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.5264, + "grad_norm": 7.279812335968018, + "kl": 0.7529296875, + "learning_rate": 1e-06, + "loss": 0.0554, + "num_tokens": 8772311.0, + "reward": 3.248779296875, + "reward_std": 4.097195625305176, + "rewards/rm_reward_func/mean": 3.248779296875, + "rewards/rm_reward_func/std": 11.073434829711914, + "step": 658 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 444.15625, + "completions/mean_terminated_length": 384.29412841796875, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "epoch": 0.5272, + "grad_norm": 7.7773356437683105, + "kl": 0.760986328125, + "learning_rate": 1e-06, + "loss": -0.0174, + "num_tokens": 8788708.0, + "reward": -2.1377792358398438, + "reward_std": 4.090878486633301, + "rewards/rm_reward_func/mean": -2.1377792358398438, + "rewards/rm_reward_func/std": 10.053132057189941, + "step": 659 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 417.59375, + "completions/mean_terminated_length": 391.1600036621094, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "epoch": 0.528, + "grad_norm": 6.207853317260742, + "kl": 0.521240234375, + "learning_rate": 1e-06, + "loss": 0.0471, + "num_tokens": 8804327.0, + "reward": 3.51171875, + "reward_std": 6.670271873474121, + "rewards/rm_reward_func/mean": 3.51171875, + "rewards/rm_reward_func/std": 13.217809677124023, + "step": 660 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 222.25, + "completions/mean_terminated_length": 141.1199951171875, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.5288, + "grad_norm": 3.570781946182251, + "kl": 2.3623046875, + "learning_rate": 1e-06, + "loss": 0.0811, + "num_tokens": 8815471.0, + "reward": 3.49066162109375, + "reward_std": 3.8420963287353516, + "rewards/rm_reward_func/mean": 3.49066162109375, + "rewards/rm_reward_func/std": 9.580389976501465, + "step": 661 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 459.0, + "completions/mean_length": 334.34375, + "completions/mean_terminated_length": 275.125, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.5296, + "grad_norm": 16.329147338867188, + "kl": 0.8173828125, + "learning_rate": 1e-06, + "loss": 0.0685, + "num_tokens": 8828386.0, + "reward": 1.876220703125, + "reward_std": 7.676386833190918, + "rewards/rm_reward_func/mean": 1.876220703125, + "rewards/rm_reward_func/std": 12.502883911132812, + "step": 662 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 398.625, + "completions/mean_terminated_length": 330.6000061035156, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.5304, + "grad_norm": 6.071362018585205, + "kl": 1.19775390625, + "learning_rate": 1e-06, + "loss": 0.1563, + "num_tokens": 8844302.0, + "reward": 1.69091796875, + "reward_std": 8.782705307006836, + "rewards/rm_reward_func/mean": 1.69091796875, + "rewards/rm_reward_func/std": 12.565754890441895, + "step": 663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 232.6875, + "completions/mean_terminated_length": 223.6774139404297, + "completions/min_length": 26.0, + "completions/min_terminated_length": 26.0, + "epoch": 0.5312, + "grad_norm": 11.870681762695312, + "kl": 4.712890625, + "learning_rate": 1e-06, + "loss": 0.2744, + "num_tokens": 8855052.0, + "reward": -1.134765625, + "reward_std": 4.481464385986328, + "rewards/rm_reward_func/mean": -1.134765625, + "rewards/rm_reward_func/std": 17.394426345825195, + "step": 664 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 358.21875, + "completions/mean_terminated_length": 353.258056640625, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "epoch": 0.532, + "grad_norm": 6.449156284332275, + "kl": 2.161865234375, + "learning_rate": 1e-06, + "loss": 0.0442, + "num_tokens": 8868555.0, + "reward": 2.504180908203125, + "reward_std": 8.275325775146484, + "rewards/rm_reward_func/mean": 2.504180908203125, + "rewards/rm_reward_func/std": 15.419231414794922, + "step": 665 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 412.40625, + "completions/mean_terminated_length": 373.4347839355469, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.5328, + "grad_norm": 15.121233940124512, + "kl": 4.3203125, + "learning_rate": 1e-06, + "loss": 0.0551, + "num_tokens": 8886344.0, + "reward": 3.4033203125, + "reward_std": 12.214666366577148, + "rewards/rm_reward_func/mean": 3.4033203125, + "rewards/rm_reward_func/std": 17.266735076904297, + "step": 666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 277.125, + "completions/mean_terminated_length": 222.92308044433594, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.5336, + "grad_norm": 20.014434814453125, + "kl": 5.343505859375, + "learning_rate": 1e-06, + "loss": 0.1802, + "num_tokens": 8900948.0, + "reward": -5.2738800048828125, + "reward_std": 6.429419994354248, + "rewards/rm_reward_func/mean": -5.2738800048828125, + "rewards/rm_reward_func/std": 11.191460609436035, + "step": 667 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 286.21875, + "completions/mean_terminated_length": 223.0, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.5344, + "grad_norm": 46.5359001159668, + "kl": 16.328125, + "learning_rate": 1e-06, + "loss": 0.6287, + "num_tokens": 8912667.0, + "reward": -19.37646484375, + "reward_std": 8.034518241882324, + "rewards/rm_reward_func/mean": -19.37646484375, + "rewards/rm_reward_func/std": 11.764345169067383, + "step": 668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 374.09375, + "completions/mean_terminated_length": 354.39288330078125, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.5352, + "grad_norm": 8.606017112731934, + "kl": 4.453125, + "learning_rate": 1e-06, + "loss": 0.1118, + "num_tokens": 8926590.0, + "reward": 5.37371826171875, + "reward_std": 12.284038543701172, + "rewards/rm_reward_func/mean": 5.37371826171875, + "rewards/rm_reward_func/std": 20.101335525512695, + "step": 669 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 341.5625, + "completions/mean_terminated_length": 323.9310302734375, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.536, + "grad_norm": 20.169708251953125, + "kl": 2.861328125, + "learning_rate": 1e-06, + "loss": 0.3412, + "num_tokens": 8941640.0, + "reward": 8.84033203125, + "reward_std": 7.4539666175842285, + "rewards/rm_reward_func/mean": 8.84033203125, + "rewards/rm_reward_func/std": 11.035738945007324, + "step": 670 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 315.84375, + "completions/mean_terminated_length": 250.45834350585938, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.5368, + "grad_norm": 8.888484001159668, + "kl": 1.6533203125, + "learning_rate": 1e-06, + "loss": -0.0764, + "num_tokens": 8955819.0, + "reward": 1.8431396484375, + "reward_std": 9.139184951782227, + "rewards/rm_reward_func/mean": 1.8431396484375, + "rewards/rm_reward_func/std": 12.700114250183105, + "step": 671 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 273.5625, + "completions/mean_terminated_length": 265.8709716796875, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.5376, + "grad_norm": 10.354080200195312, + "kl": 1.771484375, + "learning_rate": 1e-06, + "loss": 0.1823, + "num_tokens": 8966917.0, + "reward": 3.476806640625, + "reward_std": 4.19705867767334, + "rewards/rm_reward_func/mean": 3.476806640625, + "rewards/rm_reward_func/std": 17.924257278442383, + "step": 672 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 100.0, + "completions/mean_terminated_length": 86.70967102050781, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.5384, + "grad_norm": 28.584192276000977, + "kl": 4.08642578125, + "learning_rate": 1e-06, + "loss": 0.4196, + "num_tokens": 8974605.0, + "reward": 1.3251953125, + "reward_std": 7.180523872375488, + "rewards/rm_reward_func/mean": 1.3251953125, + "rewards/rm_reward_func/std": 14.241349220275879, + "step": 673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 406.0, + "completions/mean_length": 226.21875, + "completions/mean_terminated_length": 130.95834350585938, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.5392, + "grad_norm": 35.00379943847656, + "kl": 4.3994140625, + "learning_rate": 1e-06, + "loss": 0.1642, + "num_tokens": 8986524.0, + "reward": -5.6318359375, + "reward_std": 3.752939224243164, + "rewards/rm_reward_func/mean": -5.6318359375, + "rewards/rm_reward_func/std": 12.199244499206543, + "step": 674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 456.0, + "completions/mean_length": 313.5625, + "completions/mean_terminated_length": 223.3636474609375, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.54, + "grad_norm": 41.50772476196289, + "kl": 8.8515625, + "learning_rate": 1e-06, + "loss": 0.2941, + "num_tokens": 8999078.0, + "reward": -8.677978515625, + "reward_std": 4.428235054016113, + "rewards/rm_reward_func/mean": -8.677978515625, + "rewards/rm_reward_func/std": 14.32275104522705, + "step": 675 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 366.0, + "completions/mean_length": 183.375, + "completions/mean_terminated_length": 122.51851654052734, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.5408, + "grad_norm": 7.0692219734191895, + "kl": 4.337890625, + "learning_rate": 1e-06, + "loss": 0.1818, + "num_tokens": 9007402.0, + "reward": -1.382568359375, + "reward_std": 6.2474846839904785, + "rewards/rm_reward_func/mean": -1.382568359375, + "rewards/rm_reward_func/std": 9.534759521484375, + "step": 676 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 376.6875, + "completions/mean_terminated_length": 357.3571472167969, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.5416, + "grad_norm": 52.67711639404297, + "kl": 6.74609375, + "learning_rate": 1e-06, + "loss": 0.187, + "num_tokens": 9021656.0, + "reward": 0.373046875, + "reward_std": 13.075346946716309, + "rewards/rm_reward_func/mean": 0.373046875, + "rewards/rm_reward_func/std": 22.373414993286133, + "step": 677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 340.0625, + "completions/mean_terminated_length": 222.42105102539062, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.5424, + "grad_norm": 8.724742889404297, + "kl": 2.99169921875, + "learning_rate": 1e-06, + "loss": 0.0752, + "num_tokens": 9037954.0, + "reward": 3.04296875, + "reward_std": 12.51136589050293, + "rewards/rm_reward_func/mean": 3.04296875, + "rewards/rm_reward_func/std": 18.065021514892578, + "step": 678 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 190.28125, + "completions/mean_terminated_length": 144.32144165039062, + "completions/min_length": 26.0, + "completions/min_terminated_length": 26.0, + "epoch": 0.5432, + "grad_norm": 17.94440269470215, + "kl": 4.4853515625, + "learning_rate": 1e-06, + "loss": 0.4292, + "num_tokens": 9046651.0, + "reward": 0.7139892578125, + "reward_std": 3.5457606315612793, + "rewards/rm_reward_func/mean": 0.7139892578125, + "rewards/rm_reward_func/std": 9.750553131103516, + "step": 679 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 312.09375, + "completions/mean_terminated_length": 298.7666931152344, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.544, + "grad_norm": 21.662830352783203, + "kl": 7.321533203125, + "learning_rate": 1e-06, + "loss": 0.1919, + "num_tokens": 9059910.0, + "reward": -3.553955078125, + "reward_std": 10.071688652038574, + "rewards/rm_reward_func/mean": -3.553955078125, + "rewards/rm_reward_func/std": 14.8821382522583, + "step": 680 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 266.25, + "completions/mean_terminated_length": 220.74073791503906, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.5448, + "grad_norm": 6.978208065032959, + "kl": 4.47412109375, + "learning_rate": 1e-06, + "loss": 0.1302, + "num_tokens": 9070510.0, + "reward": -1.48779296875, + "reward_std": 9.718238830566406, + "rewards/rm_reward_func/mean": -1.48779296875, + "rewards/rm_reward_func/std": 15.631418228149414, + "step": 681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 252.6875, + "completions/mean_terminated_length": 215.6428680419922, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.5456, + "grad_norm": 132.93753051757812, + "kl": 9.15625, + "learning_rate": 1e-06, + "loss": 0.2597, + "num_tokens": 9080252.0, + "reward": -10.84326171875, + "reward_std": 9.151596069335938, + "rewards/rm_reward_func/mean": -10.84326171875, + "rewards/rm_reward_func/std": 14.070874214172363, + "step": 682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 434.0, + "completions/mean_length": 145.9375, + "completions/mean_terminated_length": 121.53334045410156, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.5464, + "grad_norm": 22.839073181152344, + "kl": 5.984375, + "learning_rate": 1e-06, + "loss": 0.1499, + "num_tokens": 9088962.0, + "reward": -9.11358642578125, + "reward_std": 12.002496719360352, + "rewards/rm_reward_func/mean": -9.11358642578125, + "rewards/rm_reward_func/std": 16.55123519897461, + "step": 683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 238.875, + "completions/mean_terminated_length": 210.6206817626953, + "completions/min_length": 26.0, + "completions/min_terminated_length": 26.0, + "epoch": 0.5472, + "grad_norm": 37.41764831542969, + "kl": 5.50390625, + "learning_rate": 1e-06, + "loss": 0.0492, + "num_tokens": 9098534.0, + "reward": -3.8291015625, + "reward_std": 8.362621307373047, + "rewards/rm_reward_func/mean": -3.8291015625, + "rewards/rm_reward_func/std": 15.430756568908691, + "step": 684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 464.0, + "completions/mean_length": 230.5625, + "completions/mean_terminated_length": 221.48387145996094, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.548, + "grad_norm": 19.959774017333984, + "kl": 5.6953125, + "learning_rate": 1e-06, + "loss": 0.0816, + "num_tokens": 9108744.0, + "reward": -6.86328125, + "reward_std": 15.26154613494873, + "rewards/rm_reward_func/mean": -6.86328125, + "rewards/rm_reward_func/std": 18.045209884643555, + "step": 685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 371.625, + "completions/mean_terminated_length": 345.629638671875, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.5488, + "grad_norm": 10.441740989685059, + "kl": 4.70703125, + "learning_rate": 1e-06, + "loss": 0.2121, + "num_tokens": 9123308.0, + "reward": 0.5693359375, + "reward_std": 7.536800384521484, + "rewards/rm_reward_func/mean": 0.5693359375, + "rewards/rm_reward_func/std": 17.190340042114258, + "step": 686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 184.9375, + "completions/mean_terminated_length": 151.10345458984375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.5496, + "grad_norm": 12.261507987976074, + "kl": 2.86083984375, + "learning_rate": 1e-06, + "loss": 0.0786, + "num_tokens": 9132018.0, + "reward": 5.0924072265625, + "reward_std": 3.7133607864379883, + "rewards/rm_reward_func/mean": 5.0924072265625, + "rewards/rm_reward_func/std": 8.903958320617676, + "step": 687 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 493.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 254.96875, + "completions/mean_terminated_length": 254.96875, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.5504, + "grad_norm": 11.390403747558594, + "kl": 3.8935546875, + "learning_rate": 1e-06, + "loss": -0.1216, + "num_tokens": 9143177.0, + "reward": -8.523193359375, + "reward_std": 11.859602928161621, + "rewards/rm_reward_func/mean": -8.523193359375, + "rewards/rm_reward_func/std": 12.442675590515137, + "step": 688 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 407.0, + "completions/mean_length": 254.90625, + "completions/mean_terminated_length": 237.7666778564453, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "epoch": 0.5512, + "grad_norm": 9.756564140319824, + "kl": 4.615234375, + "learning_rate": 1e-06, + "loss": 0.1961, + "num_tokens": 9155374.0, + "reward": -8.00146484375, + "reward_std": 10.193147659301758, + "rewards/rm_reward_func/mean": -8.00146484375, + "rewards/rm_reward_func/std": 11.081028938293457, + "step": 689 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 344.90625, + "completions/mean_terminated_length": 279.5217590332031, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.552, + "grad_norm": 12.181934356689453, + "kl": 3.39208984375, + "learning_rate": 1e-06, + "loss": 0.0899, + "num_tokens": 9170707.0, + "reward": -6.829345703125, + "reward_std": 5.473711013793945, + "rewards/rm_reward_func/mean": -6.829345703125, + "rewards/rm_reward_func/std": 10.798709869384766, + "step": 690 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 484.0, + "completions/mean_length": 332.0625, + "completions/mean_terminated_length": 320.0666809082031, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.5528, + "grad_norm": 10.006911277770996, + "kl": 1.677978515625, + "learning_rate": 1e-06, + "loss": -0.0336, + "num_tokens": 9184165.0, + "reward": 3.05224609375, + "reward_std": 7.134219169616699, + "rewards/rm_reward_func/mean": 3.05224609375, + "rewards/rm_reward_func/std": 13.809707641601562, + "step": 691 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 250.5625, + "completions/mean_terminated_length": 202.1481475830078, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.5536, + "grad_norm": 6.326221942901611, + "kl": 0.51953125, + "learning_rate": 1e-06, + "loss": -0.0631, + "num_tokens": 9195959.0, + "reward": 12.587890625, + "reward_std": 4.309545040130615, + "rewards/rm_reward_func/mean": 12.587890625, + "rewards/rm_reward_func/std": 6.763890266418457, + "step": 692 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 217.6875, + "completions/mean_terminated_length": 208.19354248046875, + "completions/min_length": 12.0, + "completions/min_terminated_length": 12.0, + "epoch": 0.5544, + "grad_norm": 15.060230255126953, + "kl": 2.18994140625, + "learning_rate": 1e-06, + "loss": 0.1014, + "num_tokens": 9206501.0, + "reward": -7.5948486328125, + "reward_std": 6.410397529602051, + "rewards/rm_reward_func/mean": -7.5948486328125, + "rewards/rm_reward_func/std": 9.005189895629883, + "step": 693 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 444.0, + "completions/mean_length": 206.0, + "completions/mean_terminated_length": 196.1290283203125, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "epoch": 0.5552, + "grad_norm": 12.398327827453613, + "kl": 1.31103515625, + "learning_rate": 1e-06, + "loss": -0.1765, + "num_tokens": 9218037.0, + "reward": -7.2659912109375, + "reward_std": 6.091501712799072, + "rewards/rm_reward_func/mean": -7.2659912109375, + "rewards/rm_reward_func/std": 11.163793563842773, + "step": 694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 345.84375, + "completions/mean_terminated_length": 322.1071472167969, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.556, + "grad_norm": 6.776838779449463, + "kl": 2.171875, + "learning_rate": 1e-06, + "loss": 0.0345, + "num_tokens": 9231384.0, + "reward": -5.34033203125, + "reward_std": 5.828690052032471, + "rewards/rm_reward_func/mean": -5.34033203125, + "rewards/rm_reward_func/std": 6.941495418548584, + "step": 695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 266.84375, + "completions/mean_terminated_length": 221.44444274902344, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.5568, + "grad_norm": 6.661186695098877, + "kl": 0.462158203125, + "learning_rate": 1e-06, + "loss": 0.0817, + "num_tokens": 9244219.0, + "reward": 3.683349609375, + "reward_std": 6.5144147872924805, + "rewards/rm_reward_func/mean": 3.683349609375, + "rewards/rm_reward_func/std": 10.859166145324707, + "step": 696 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 407.0, + "completions/mean_terminated_length": 382.7692565917969, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.5576, + "grad_norm": 16.05687141418457, + "kl": 0.77197265625, + "learning_rate": 1e-06, + "loss": -0.0237, + "num_tokens": 9260139.0, + "reward": 3.64599609375, + "reward_std": 6.319300651550293, + "rewards/rm_reward_func/mean": 3.64599609375, + "rewards/rm_reward_func/std": 14.248506546020508, + "step": 697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 395.09375, + "completions/mean_terminated_length": 349.34783935546875, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.5584, + "grad_norm": 7.179235458374023, + "kl": 1.25146484375, + "learning_rate": 1e-06, + "loss": -0.0007, + "num_tokens": 9275854.0, + "reward": 3.7479248046875, + "reward_std": 7.614123821258545, + "rewards/rm_reward_func/mean": 3.7479248046875, + "rewards/rm_reward_func/std": 17.118871688842773, + "step": 698 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 320.0, + "completions/mean_length": 225.90625, + "completions/mean_terminated_length": 172.92593383789062, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.5592, + "grad_norm": 17.386568069458008, + "kl": 0.8544921875, + "learning_rate": 1e-06, + "loss": -0.137, + "num_tokens": 9286667.0, + "reward": -11.1297607421875, + "reward_std": 6.347601890563965, + "rewards/rm_reward_func/mean": -11.1297607421875, + "rewards/rm_reward_func/std": 6.936679363250732, + "step": 699 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 433.0, + "completions/max_terminated_length": 433.0, + "completions/mean_length": 210.59375, + "completions/mean_terminated_length": 210.59375, + "completions/min_length": 15.0, + "completions/min_terminated_length": 15.0, + "epoch": 0.56, + "grad_norm": 13.88813591003418, + "kl": 1.533203125, + "learning_rate": 1e-06, + "loss": 0.082, + "num_tokens": 9296486.0, + "reward": -1.9166259765625, + "reward_std": 6.144941806793213, + "rewards/rm_reward_func/mean": -1.9166259765625, + "rewards/rm_reward_func/std": 11.042118072509766, + "step": 700 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 417.0, + "completions/mean_length": 235.3125, + "completions/mean_terminated_length": 216.86668395996094, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.5608, + "grad_norm": 14.585112571716309, + "kl": 1.34814453125, + "learning_rate": 1e-06, + "loss": -0.0184, + "num_tokens": 9306088.0, + "reward": -1.9686279296875, + "reward_std": 6.677350997924805, + "rewards/rm_reward_func/mean": -1.9686279296875, + "rewards/rm_reward_func/std": 8.160258293151855, + "step": 701 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 452.0, + "completions/mean_length": 243.34375, + "completions/mean_terminated_length": 225.433349609375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.5616, + "grad_norm": 16.384502410888672, + "kl": 1.71630859375, + "learning_rate": 1e-06, + "loss": 0.2464, + "num_tokens": 9319155.0, + "reward": 0.691162109375, + "reward_std": 6.911756992340088, + "rewards/rm_reward_func/mean": 0.691162109375, + "rewards/rm_reward_func/std": 17.84705924987793, + "step": 702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 382.0, + "completions/max_terminated_length": 382.0, + "completions/mean_length": 148.78125, + "completions/mean_terminated_length": 148.78125, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.5624, + "grad_norm": 6.983405113220215, + "kl": 0.526123046875, + "learning_rate": 1e-06, + "loss": -0.1855, + "num_tokens": 9326356.0, + "reward": -2.7777099609375, + "reward_std": 3.3739712238311768, + "rewards/rm_reward_func/mean": -2.7777099609375, + "rewards/rm_reward_func/std": 7.716010570526123, + "step": 703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 366.71875, + "completions/mean_terminated_length": 357.0333557128906, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.5632, + "grad_norm": 10.195340156555176, + "kl": 1.0078125, + "learning_rate": 1e-06, + "loss": -0.0249, + "num_tokens": 9340835.0, + "reward": 5.092041015625, + "reward_std": 8.344895362854004, + "rewards/rm_reward_func/mean": 5.092041015625, + "rewards/rm_reward_func/std": 13.963737487792969, + "step": 704 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 430.0, + "completions/mean_length": 267.59375, + "completions/mean_terminated_length": 259.70965576171875, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "epoch": 0.564, + "grad_norm": 15.919843673706055, + "kl": 1.554931640625, + "learning_rate": 1e-06, + "loss": 0.159, + "num_tokens": 9352078.0, + "reward": -2.619140625, + "reward_std": 7.604259490966797, + "rewards/rm_reward_func/mean": -2.619140625, + "rewards/rm_reward_func/std": 10.991021156311035, + "step": 705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 407.375, + "completions/mean_terminated_length": 366.4347839355469, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.5648, + "grad_norm": 12.950475692749023, + "kl": 2.766845703125, + "learning_rate": 1e-06, + "loss": 0.068, + "num_tokens": 9367250.0, + "reward": 2.364013671875, + "reward_std": 8.381866455078125, + "rewards/rm_reward_func/mean": 2.364013671875, + "rewards/rm_reward_func/std": 12.802971839904785, + "step": 706 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 298.75, + "completions/mean_terminated_length": 259.2592468261719, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.5656, + "grad_norm": 6.933451175689697, + "kl": 0.818603515625, + "learning_rate": 1e-06, + "loss": 0.0176, + "num_tokens": 9379698.0, + "reward": 4.7874755859375, + "reward_std": 7.199830532073975, + "rewards/rm_reward_func/mean": 4.7874755859375, + "rewards/rm_reward_func/std": 9.206077575683594, + "step": 707 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 441.0, + "completions/max_terminated_length": 441.0, + "completions/mean_length": 232.8125, + "completions/mean_terminated_length": 232.8125, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.5664, + "grad_norm": 6.320394992828369, + "kl": 0.52197265625, + "learning_rate": 1e-06, + "loss": -0.1093, + "num_tokens": 9389684.0, + "reward": 13.0057373046875, + "reward_std": 8.226985931396484, + "rewards/rm_reward_func/mean": 13.0057373046875, + "rewards/rm_reward_func/std": 11.692206382751465, + "step": 708 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 468.0, + "completions/mean_length": 290.46875, + "completions/mean_terminated_length": 275.70001220703125, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.5672, + "grad_norm": 9.328205108642578, + "kl": 2.07763671875, + "learning_rate": 1e-06, + "loss": 0.1139, + "num_tokens": 9400939.0, + "reward": 2.2265625, + "reward_std": 8.862954139709473, + "rewards/rm_reward_func/mean": 2.2265625, + "rewards/rm_reward_func/std": 12.553840637207031, + "step": 709 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 286.625, + "completions/mean_terminated_length": 223.51998901367188, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.568, + "grad_norm": 12.409536361694336, + "kl": 4.6474609375, + "learning_rate": 1e-06, + "loss": 0.2891, + "num_tokens": 9412575.0, + "reward": 3.2001953125, + "reward_std": 6.288336753845215, + "rewards/rm_reward_func/mean": 3.2001953125, + "rewards/rm_reward_func/std": 10.550227165222168, + "step": 710 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 338.90625, + "completions/mean_terminated_length": 314.1785888671875, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.5688, + "grad_norm": 5.891862869262695, + "kl": 1.07666015625, + "learning_rate": 1e-06, + "loss": 0.0507, + "num_tokens": 9427844.0, + "reward": 7.5908203125, + "reward_std": 3.023407459259033, + "rewards/rm_reward_func/mean": 7.5908203125, + "rewards/rm_reward_func/std": 12.348587036132812, + "step": 711 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 338.59375, + "completions/mean_terminated_length": 259.7727355957031, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.5696, + "grad_norm": 5.630682468414307, + "kl": 2.202880859375, + "learning_rate": 1e-06, + "loss": 0.0399, + "num_tokens": 9441263.0, + "reward": 4.940185546875, + "reward_std": 8.606001853942871, + "rewards/rm_reward_func/mean": 4.940185546875, + "rewards/rm_reward_func/std": 15.963677406311035, + "step": 712 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 381.65625, + "completions/mean_terminated_length": 191.1538543701172, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.5704, + "grad_norm": 21.119258880615234, + "kl": 10.06396484375, + "learning_rate": 1e-06, + "loss": 0.3982, + "num_tokens": 9456948.0, + "reward": -4.12841796875, + "reward_std": 10.323966026306152, + "rewards/rm_reward_func/mean": -4.12841796875, + "rewards/rm_reward_func/std": 16.72847557067871, + "step": 713 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 382.0625, + "completions/mean_terminated_length": 281.0, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.5712, + "grad_norm": 7.876568794250488, + "kl": 5.89892578125, + "learning_rate": 1e-06, + "loss": 0.1956, + "num_tokens": 9471470.0, + "reward": -1.296875, + "reward_std": 11.687822341918945, + "rewards/rm_reward_func/mean": -1.296875, + "rewards/rm_reward_func/std": 17.725196838378906, + "step": 714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 301.6875, + "completions/mean_terminated_length": 219.3913116455078, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.572, + "grad_norm": 16.49651527404785, + "kl": 6.4150390625, + "learning_rate": 1e-06, + "loss": 0.2308, + "num_tokens": 9486132.0, + "reward": -4.572021484375, + "reward_std": 6.965249061584473, + "rewards/rm_reward_func/mean": -4.572021484375, + "rewards/rm_reward_func/std": 12.301454544067383, + "step": 715 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 383.8125, + "completions/mean_terminated_length": 306.8999938964844, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.5728, + "grad_norm": 7.270954608917236, + "kl": 4.827392578125, + "learning_rate": 1e-06, + "loss": 0.2095, + "num_tokens": 9503854.0, + "reward": 20.08734130859375, + "reward_std": 11.994423866271973, + "rewards/rm_reward_func/mean": 20.08734130859375, + "rewards/rm_reward_func/std": 29.330028533935547, + "step": 716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 394.40625, + "completions/mean_terminated_length": 361.47998046875, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.5736, + "grad_norm": 5.99873161315918, + "kl": 5.353271484375, + "learning_rate": 1e-06, + "loss": 0.1456, + "num_tokens": 9519059.0, + "reward": 0.1048583984375, + "reward_std": 15.651195526123047, + "rewards/rm_reward_func/mean": 0.1048583984375, + "rewards/rm_reward_func/std": 15.79875373840332, + "step": 717 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 340.53125, + "completions/mean_terminated_length": 300.9615478515625, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.5744, + "grad_norm": 10.691807746887207, + "kl": 6.0263671875, + "learning_rate": 1e-06, + "loss": 0.1929, + "num_tokens": 9532652.0, + "reward": 3.922882080078125, + "reward_std": 8.720820426940918, + "rewards/rm_reward_func/mean": 3.922882080078125, + "rewards/rm_reward_func/std": 20.64432716369629, + "step": 718 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 238.09375, + "completions/mean_terminated_length": 161.39999389648438, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.5752, + "grad_norm": 5.596479892730713, + "kl": 2.0068359375, + "learning_rate": 1e-06, + "loss": 0.0605, + "num_tokens": 9545783.0, + "reward": 6.065826416015625, + "reward_std": 3.9478113651275635, + "rewards/rm_reward_func/mean": 6.065826416015625, + "rewards/rm_reward_func/std": 7.903535842895508, + "step": 719 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 393.5, + "completions/mean_terminated_length": 381.2413635253906, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "epoch": 0.576, + "grad_norm": 7.530872344970703, + "kl": 4.125, + "learning_rate": 1e-06, + "loss": 0.0967, + "num_tokens": 9561407.0, + "reward": 8.341751098632812, + "reward_std": 13.022964477539062, + "rewards/rm_reward_func/mean": 8.341751098632812, + "rewards/rm_reward_func/std": 16.08083724975586, + "step": 720 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 346.71875, + "completions/mean_terminated_length": 260.1428527832031, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.5768, + "grad_norm": 12.534733772277832, + "kl": 4.27685546875, + "learning_rate": 1e-06, + "loss": 0.1581, + "num_tokens": 9574310.0, + "reward": -3.28759765625, + "reward_std": 5.179610252380371, + "rewards/rm_reward_func/mean": -3.28759765625, + "rewards/rm_reward_func/std": 12.57066822052002, + "step": 721 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 365.28125, + "completions/mean_terminated_length": 307.86956787109375, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.5776, + "grad_norm": 8.022375106811523, + "kl": 1.400390625, + "learning_rate": 1e-06, + "loss": 0.0708, + "num_tokens": 9589439.0, + "reward": 5.868088722229004, + "reward_std": 7.8726677894592285, + "rewards/rm_reward_func/mean": 5.868088722229004, + "rewards/rm_reward_func/std": 16.735122680664062, + "step": 722 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 419.90625, + "completions/mean_terminated_length": 406.7500305175781, + "completions/min_length": 284.0, + "completions/min_terminated_length": 284.0, + "epoch": 0.5784, + "grad_norm": 6.199563980102539, + "kl": 1.158935546875, + "learning_rate": 1e-06, + "loss": 0.0303, + "num_tokens": 9604860.0, + "reward": 7.0489501953125, + "reward_std": 6.722829818725586, + "rewards/rm_reward_func/mean": 7.0489501953125, + "rewards/rm_reward_func/std": 10.408683776855469, + "step": 723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 410.5625, + "completions/mean_terminated_length": 364.4545593261719, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.5792, + "grad_norm": 8.32470703125, + "kl": 2.0205078125, + "learning_rate": 1e-06, + "loss": 0.0497, + "num_tokens": 9620638.0, + "reward": 8.731201171875, + "reward_std": 13.764381408691406, + "rewards/rm_reward_func/mean": 8.731201171875, + "rewards/rm_reward_func/std": 16.106142044067383, + "step": 724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 333.1875, + "completions/mean_terminated_length": 239.52381896972656, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.58, + "grad_norm": 9.582738876342773, + "kl": 2.6728515625, + "learning_rate": 1e-06, + "loss": 0.1381, + "num_tokens": 9633836.0, + "reward": 0.17033767700195312, + "reward_std": 8.874990463256836, + "rewards/rm_reward_func/mean": 0.17033767700195312, + "rewards/rm_reward_func/std": 9.867877960205078, + "step": 725 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 358.40625, + "completions/mean_terminated_length": 277.952392578125, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.5808, + "grad_norm": 5.19415807723999, + "kl": 2.4541015625, + "learning_rate": 1e-06, + "loss": 0.138, + "num_tokens": 9647841.0, + "reward": 3.939697265625, + "reward_std": 5.236965179443359, + "rewards/rm_reward_func/mean": 3.939697265625, + "rewards/rm_reward_func/std": 12.523097038269043, + "step": 726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 385.40625, + "completions/mean_terminated_length": 298.78948974609375, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.5816, + "grad_norm": 7.742301940917969, + "kl": 2.68505859375, + "learning_rate": 1e-06, + "loss": 0.2645, + "num_tokens": 9663086.0, + "reward": 1.2738037109375, + "reward_std": 6.155429363250732, + "rewards/rm_reward_func/mean": 1.2738037109375, + "rewards/rm_reward_func/std": 7.7377028465271, + "step": 727 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 158.0, + "completions/max_terminated_length": 158.0, + "completions/mean_length": 79.90625, + "completions/mean_terminated_length": 79.90625, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.5824, + "grad_norm": 7.791232585906982, + "kl": 1.28955078125, + "learning_rate": 1e-06, + "loss": 0.0041, + "num_tokens": 9670771.0, + "reward": -0.384521484375, + "reward_std": 2.3221030235290527, + "rewards/rm_reward_func/mean": -0.384521484375, + "rewards/rm_reward_func/std": 5.91448974609375, + "step": 728 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 437.34375, + "completions/mean_terminated_length": 386.2631530761719, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.5832, + "grad_norm": 6.162810325622559, + "kl": 2.515625, + "learning_rate": 1e-06, + "loss": 0.0724, + "num_tokens": 9687254.0, + "reward": 4.739013671875, + "reward_std": 13.861799240112305, + "rewards/rm_reward_func/mean": 4.739013671875, + "rewards/rm_reward_func/std": 15.850342750549316, + "step": 729 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 445.0, + "completions/mean_length": 365.15625, + "completions/mean_terminated_length": 307.6956481933594, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.584, + "grad_norm": 8.422097206115723, + "kl": 3.58056640625, + "learning_rate": 1e-06, + "loss": 0.0715, + "num_tokens": 9700803.0, + "reward": -7.97607421875, + "reward_std": 8.740215301513672, + "rewards/rm_reward_func/mean": -7.97607421875, + "rewards/rm_reward_func/std": 10.935211181640625, + "step": 730 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 473.875, + "completions/mean_terminated_length": 451.0, + "completions/min_length": 347.0, + "completions/min_terminated_length": 347.0, + "epoch": 0.5848, + "grad_norm": 9.889822006225586, + "kl": 0.578125, + "learning_rate": 1e-06, + "loss": 0.0484, + "num_tokens": 9718783.0, + "reward": 15.586181640625, + "reward_std": 11.35405445098877, + "rewards/rm_reward_func/mean": 15.586181640625, + "rewards/rm_reward_func/std": 11.300307273864746, + "step": 731 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 292.59375, + "completions/mean_terminated_length": 269.89654541015625, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.5856, + "grad_norm": 7.809492111206055, + "kl": 0.862060546875, + "learning_rate": 1e-06, + "loss": 0.0524, + "num_tokens": 9732722.0, + "reward": 8.6011962890625, + "reward_std": 3.8772120475769043, + "rewards/rm_reward_func/mean": 8.6011962890625, + "rewards/rm_reward_func/std": 7.693234920501709, + "step": 732 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 441.21875, + "completions/mean_terminated_length": 392.78948974609375, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "epoch": 0.5864, + "grad_norm": 7.425821781158447, + "kl": 3.7294921875, + "learning_rate": 1e-06, + "loss": 0.1576, + "num_tokens": 9749089.0, + "reward": 9.70703125, + "reward_std": 12.684297561645508, + "rewards/rm_reward_func/mean": 9.70703125, + "rewards/rm_reward_func/std": 14.209367752075195, + "step": 733 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 470.0, + "completions/mean_length": 310.65625, + "completions/mean_terminated_length": 231.86956787109375, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.5872, + "grad_norm": 24.46310806274414, + "kl": 5.77197265625, + "learning_rate": 1e-06, + "loss": 0.2606, + "num_tokens": 9761590.0, + "reward": -5.23046875, + "reward_std": 6.721203804016113, + "rewards/rm_reward_func/mean": -5.23046875, + "rewards/rm_reward_func/std": 17.831560134887695, + "step": 734 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 351.03125, + "completions/mean_terminated_length": 288.0434875488281, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.588, + "grad_norm": 4.2976603507995605, + "kl": 1.31982421875, + "learning_rate": 1e-06, + "loss": 0.0227, + "num_tokens": 9775831.0, + "reward": 10.992095947265625, + "reward_std": 7.566171646118164, + "rewards/rm_reward_func/mean": 10.992095947265625, + "rewards/rm_reward_func/std": 9.95695686340332, + "step": 735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 489.0, + "completions/mean_length": 261.46875, + "completions/mean_terminated_length": 225.6785888671875, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.5888, + "grad_norm": 7.958250999450684, + "kl": 2.50341796875, + "learning_rate": 1e-06, + "loss": 0.137, + "num_tokens": 9786494.0, + "reward": 0.07373046875, + "reward_std": 6.209069728851318, + "rewards/rm_reward_func/mean": 0.07373046875, + "rewards/rm_reward_func/std": 12.3759765625, + "step": 736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 330.8125, + "completions/mean_terminated_length": 324.9677429199219, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.5896, + "grad_norm": 5.071633815765381, + "kl": 1.260986328125, + "learning_rate": 1e-06, + "loss": -0.0572, + "num_tokens": 9799144.0, + "reward": 9.56390380859375, + "reward_std": 10.431290626525879, + "rewards/rm_reward_func/mean": 9.56390380859375, + "rewards/rm_reward_func/std": 19.839946746826172, + "step": 737 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 380.25, + "completions/mean_terminated_length": 328.6956481933594, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.5904, + "grad_norm": 10.128765106201172, + "kl": 3.7099609375, + "learning_rate": 1e-06, + "loss": 0.1282, + "num_tokens": 9816376.0, + "reward": 7.971923828125, + "reward_std": 5.655712127685547, + "rewards/rm_reward_func/mean": 7.971923828125, + "rewards/rm_reward_func/std": 19.648178100585938, + "step": 738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 320.96875, + "completions/mean_terminated_length": 276.8846130371094, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.5912, + "grad_norm": 12.91031551361084, + "kl": 1.35986328125, + "learning_rate": 1e-06, + "loss": 0.0801, + "num_tokens": 9832231.0, + "reward": 14.126220703125, + "reward_std": 8.189908981323242, + "rewards/rm_reward_func/mean": 14.126220703125, + "rewards/rm_reward_func/std": 11.697257995605469, + "step": 739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 344.125, + "completions/mean_terminated_length": 288.16668701171875, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.592, + "grad_norm": 6.711599826812744, + "kl": 6.6611328125, + "learning_rate": 1e-06, + "loss": 0.4934, + "num_tokens": 9846835.0, + "reward": 3.853271484375, + "reward_std": 11.102495193481445, + "rewards/rm_reward_func/mean": 3.853271484375, + "rewards/rm_reward_func/std": 13.937686920166016, + "step": 740 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 369.5, + "completions/mean_terminated_length": 294.8571472167969, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.5928, + "grad_norm": 9.946474075317383, + "kl": 3.537353515625, + "learning_rate": 1e-06, + "loss": 0.2248, + "num_tokens": 9863235.0, + "reward": 5.8260498046875, + "reward_std": 8.534289360046387, + "rewards/rm_reward_func/mean": 5.8260498046875, + "rewards/rm_reward_func/std": 14.377345085144043, + "step": 741 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 261.8125, + "completions/mean_terminated_length": 204.07693481445312, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.5936, + "grad_norm": 34.741207122802734, + "kl": 16.79248046875, + "learning_rate": 1e-06, + "loss": 0.9002, + "num_tokens": 9875661.0, + "reward": -8.799308776855469, + "reward_std": 6.605631351470947, + "rewards/rm_reward_func/mean": -8.799308776855469, + "rewards/rm_reward_func/std": 15.544058799743652, + "step": 742 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 390.09375, + "completions/mean_terminated_length": 316.95001220703125, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.5944, + "grad_norm": 7.391372203826904, + "kl": 0.4296875, + "learning_rate": 1e-06, + "loss": 0.0217, + "num_tokens": 9891272.0, + "reward": 8.581817626953125, + "reward_std": 3.363267183303833, + "rewards/rm_reward_func/mean": 8.581817626953125, + "rewards/rm_reward_func/std": 12.903508186340332, + "step": 743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 262.15625, + "completions/mean_terminated_length": 254.09677124023438, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.5952, + "grad_norm": 6.578904151916504, + "kl": 5.13134765625, + "learning_rate": 1e-06, + "loss": 0.4582, + "num_tokens": 9904589.0, + "reward": 18.150390625, + "reward_std": 6.425145626068115, + "rewards/rm_reward_func/mean": 18.150390625, + "rewards/rm_reward_func/std": 10.988823890686035, + "step": 744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 450.0, + "completions/mean_length": 300.03125, + "completions/mean_terminated_length": 251.11538696289062, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.596, + "grad_norm": 26.791982650756836, + "kl": 5.4140625, + "learning_rate": 1e-06, + "loss": 0.1141, + "num_tokens": 9918158.0, + "reward": -5.0845947265625, + "reward_std": 7.70248556137085, + "rewards/rm_reward_func/mean": -5.0845947265625, + "rewards/rm_reward_func/std": 11.328791618347168, + "step": 745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 461.0, + "completions/mean_length": 314.125, + "completions/mean_terminated_length": 268.4615478515625, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.5968, + "grad_norm": 10.60112476348877, + "kl": 2.5537109375, + "learning_rate": 1e-06, + "loss": -0.0021, + "num_tokens": 9930418.0, + "reward": 3.98583984375, + "reward_std": 7.434886932373047, + "rewards/rm_reward_func/mean": 3.98583984375, + "rewards/rm_reward_func/std": 15.446136474609375, + "step": 746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 408.5625, + "completions/mean_terminated_length": 397.862060546875, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "epoch": 0.5976, + "grad_norm": 7.734127998352051, + "kl": 1.279541015625, + "learning_rate": 1e-06, + "loss": 0.0852, + "num_tokens": 9947396.0, + "reward": 8.797119140625, + "reward_std": 9.704774856567383, + "rewards/rm_reward_func/mean": 8.797119140625, + "rewards/rm_reward_func/std": 14.60503101348877, + "step": 747 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 305.9375, + "completions/mean_terminated_length": 299.2903137207031, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.5984, + "grad_norm": 30.35190200805664, + "kl": 10.0810546875, + "learning_rate": 1e-06, + "loss": 0.2479, + "num_tokens": 9959546.0, + "reward": -7.094024658203125, + "reward_std": 9.547418594360352, + "rewards/rm_reward_func/mean": -7.094024658203125, + "rewards/rm_reward_func/std": 12.000333786010742, + "step": 748 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 285.0, + "completions/mean_terminated_length": 277.6773986816406, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.5992, + "grad_norm": 9.810269355773926, + "kl": 0.603515625, + "learning_rate": 1e-06, + "loss": 0.057, + "num_tokens": 9970658.0, + "reward": 2.1044921875, + "reward_std": 3.5760817527770996, + "rewards/rm_reward_func/mean": 2.1044921875, + "rewards/rm_reward_func/std": 6.474411487579346, + "step": 749 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 356.4375, + "completions/mean_terminated_length": 295.5652160644531, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.6, + "grad_norm": 14.325955390930176, + "kl": 3.8134765625, + "learning_rate": 1e-06, + "loss": 0.0913, + "num_tokens": 9984464.0, + "reward": 4.713623046875, + "reward_std": 8.757728576660156, + "rewards/rm_reward_func/mean": 4.713623046875, + "rewards/rm_reward_func/std": 16.11822509765625, + "step": 750 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 297.5625, + "completions/mean_terminated_length": 275.3793029785156, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.6008, + "grad_norm": 44.704063415527344, + "kl": 3.189453125, + "learning_rate": 1e-06, + "loss": 0.2095, + "num_tokens": 9997474.0, + "reward": 10.037109375, + "reward_std": 13.35189151763916, + "rewards/rm_reward_func/mean": 10.037109375, + "rewards/rm_reward_func/std": 21.836666107177734, + "step": 751 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 264.59375, + "completions/mean_terminated_length": 256.6128845214844, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.6016, + "grad_norm": 7.517630100250244, + "kl": 4.232177734375, + "learning_rate": 1e-06, + "loss": 0.215, + "num_tokens": 10008461.0, + "reward": 0.819580078125, + "reward_std": 10.651954650878906, + "rewards/rm_reward_func/mean": 0.819580078125, + "rewards/rm_reward_func/std": 15.250157356262207, + "step": 752 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 293.25, + "completions/mean_terminated_length": 278.66668701171875, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.6024, + "grad_norm": 10.266236305236816, + "kl": 4.54296875, + "learning_rate": 1e-06, + "loss": 0.0359, + "num_tokens": 10020021.0, + "reward": -3.8173828125, + "reward_std": 9.468101501464844, + "rewards/rm_reward_func/mean": -3.8173828125, + "rewards/rm_reward_func/std": 13.068405151367188, + "step": 753 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 477.0, + "completions/mean_length": 192.4375, + "completions/mean_terminated_length": 146.7857208251953, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.6032, + "grad_norm": 9.960123062133789, + "kl": 5.95947265625, + "learning_rate": 1e-06, + "loss": 0.0333, + "num_tokens": 10030379.0, + "reward": -3.7373046875, + "reward_std": 10.038860321044922, + "rewards/rm_reward_func/mean": -3.7373046875, + "rewards/rm_reward_func/std": 15.353011131286621, + "step": 754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 317.65625, + "completions/mean_terminated_length": 241.60870361328125, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.604, + "grad_norm": 6.909921169281006, + "kl": 1.919921875, + "learning_rate": 1e-06, + "loss": 0.0276, + "num_tokens": 10043216.0, + "reward": 1.073760986328125, + "reward_std": 6.802268028259277, + "rewards/rm_reward_func/mean": 1.073760986328125, + "rewards/rm_reward_func/std": 10.724485397338867, + "step": 755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 486.0, + "completions/mean_length": 322.78125, + "completions/mean_terminated_length": 310.16668701171875, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.6048, + "grad_norm": 4.838641166687012, + "kl": 1.0595703125, + "learning_rate": 1e-06, + "loss": -0.0381, + "num_tokens": 10057593.0, + "reward": 16.49951171875, + "reward_std": 8.707597732543945, + "rewards/rm_reward_func/mean": 16.49951171875, + "rewards/rm_reward_func/std": 13.02664566040039, + "step": 756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 232.6875, + "completions/mean_terminated_length": 223.6774139404297, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.6056, + "grad_norm": 6.288021087646484, + "kl": 2.422119140625, + "learning_rate": 1e-06, + "loss": 0.0256, + "num_tokens": 10068959.0, + "reward": 7.3639373779296875, + "reward_std": 7.31492805480957, + "rewards/rm_reward_func/mean": 7.3639373779296875, + "rewards/rm_reward_func/std": 11.444160461425781, + "step": 757 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 461.0, + "completions/mean_length": 257.375, + "completions/mean_terminated_length": 240.40000915527344, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.6064, + "grad_norm": 12.485620498657227, + "kl": 1.30322265625, + "learning_rate": 1e-06, + "loss": -0.256, + "num_tokens": 10079523.0, + "reward": 8.375, + "reward_std": 11.750480651855469, + "rewards/rm_reward_func/mean": 8.375, + "rewards/rm_reward_func/std": 15.094318389892578, + "step": 758 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 359.3125, + "completions/mean_terminated_length": 308.41668701171875, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.6072, + "grad_norm": 67.01472473144531, + "kl": 0.41552734375, + "learning_rate": 1e-06, + "loss": -0.0451, + "num_tokens": 10096813.0, + "reward": 8.5546875, + "reward_std": 4.309829235076904, + "rewards/rm_reward_func/mean": 8.5546875, + "rewards/rm_reward_func/std": 15.106447219848633, + "step": 759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 387.09375, + "completions/mean_terminated_length": 352.1199951171875, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "epoch": 0.608, + "grad_norm": 7.828210830688477, + "kl": 1.158203125, + "learning_rate": 1e-06, + "loss": 0.0712, + "num_tokens": 10112304.0, + "reward": 2.0579833984375, + "reward_std": 6.627885341644287, + "rewards/rm_reward_func/mean": 2.0579833984375, + "rewards/rm_reward_func/std": 9.508418083190918, + "step": 760 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 299.96875, + "completions/mean_terminated_length": 240.59999084472656, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.6088, + "grad_norm": 32.08409118652344, + "kl": 9.375, + "learning_rate": 1e-06, + "loss": 0.0282, + "num_tokens": 10124279.0, + "reward": -8.2109375, + "reward_std": 14.024812698364258, + "rewards/rm_reward_func/mean": -8.2109375, + "rewards/rm_reward_func/std": 16.273033142089844, + "step": 761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 267.03125, + "completions/mean_terminated_length": 185.375, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.6096, + "grad_norm": 9.463338851928711, + "kl": 0.489013671875, + "learning_rate": 1e-06, + "loss": 0.0646, + "num_tokens": 10135368.0, + "reward": 4.3076171875, + "reward_std": 3.226278781890869, + "rewards/rm_reward_func/mean": 4.3076171875, + "rewards/rm_reward_func/std": 11.224862098693848, + "step": 762 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 210.40625, + "completions/mean_terminated_length": 210.40625, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.6104, + "grad_norm": 15.222582817077637, + "kl": 2.01904296875, + "learning_rate": 1e-06, + "loss": 0.1217, + "num_tokens": 10144741.0, + "reward": -5.0877685546875, + "reward_std": 11.080349922180176, + "rewards/rm_reward_func/mean": -5.0877685546875, + "rewards/rm_reward_func/std": 13.31748104095459, + "step": 763 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 414.3125, + "completions/mean_terminated_length": 386.9599914550781, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "epoch": 0.6112, + "grad_norm": 13.568563461303711, + "kl": 3.62890625, + "learning_rate": 1e-06, + "loss": 0.0975, + "num_tokens": 10160167.0, + "reward": 6.3486328125, + "reward_std": 12.485017776489258, + "rewards/rm_reward_func/mean": 6.3486328125, + "rewards/rm_reward_func/std": 13.953908920288086, + "step": 764 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 378.0, + "completions/max_terminated_length": 378.0, + "completions/mean_length": 162.1875, + "completions/mean_terminated_length": 162.1875, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.612, + "grad_norm": 19.0142765045166, + "kl": 2.60107421875, + "learning_rate": 1e-06, + "loss": 0.0385, + "num_tokens": 10170917.0, + "reward": -2.573974609375, + "reward_std": 3.245800495147705, + "rewards/rm_reward_func/mean": -2.573974609375, + "rewards/rm_reward_func/std": 13.304573059082031, + "step": 765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 436.625, + "completions/mean_terminated_length": 351.20001220703125, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.6128, + "grad_norm": 24.08083724975586, + "kl": 2.4443359375, + "learning_rate": 1e-06, + "loss": 0.0583, + "num_tokens": 10190281.0, + "reward": 2.33367919921875, + "reward_std": 6.308701992034912, + "rewards/rm_reward_func/mean": 2.33367919921875, + "rewards/rm_reward_func/std": 15.869560241699219, + "step": 766 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 345.46875, + "completions/mean_terminated_length": 280.3043518066406, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.6136, + "grad_norm": 9.395527839660645, + "kl": 2.9775390625, + "learning_rate": 1e-06, + "loss": 0.0412, + "num_tokens": 10205904.0, + "reward": -1.7017822265625, + "reward_std": 6.9784321784973145, + "rewards/rm_reward_func/mean": -1.7017822265625, + "rewards/rm_reward_func/std": 11.290467262268066, + "step": 767 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 262.09375, + "completions/mean_terminated_length": 131.1904754638672, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.6144, + "grad_norm": 6.257786750793457, + "kl": 1.263671875, + "learning_rate": 1e-06, + "loss": -0.0265, + "num_tokens": 10219307.0, + "reward": 0.228759765625, + "reward_std": 3.712106466293335, + "rewards/rm_reward_func/mean": 0.228759765625, + "rewards/rm_reward_func/std": 7.532781600952148, + "step": 768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 469.0, + "completions/mean_length": 219.875, + "completions/mean_terminated_length": 200.40000915527344, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.6152, + "grad_norm": 12.303415298461914, + "kl": 1.0126953125, + "learning_rate": 1e-06, + "loss": -0.1391, + "num_tokens": 10229703.0, + "reward": 1.8739013671875, + "reward_std": 9.617498397827148, + "rewards/rm_reward_func/mean": 1.8739013671875, + "rewards/rm_reward_func/std": 12.91628360748291, + "step": 769 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 399.0625, + "completions/mean_terminated_length": 367.44000244140625, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.616, + "grad_norm": 7.800823211669922, + "kl": 2.70703125, + "learning_rate": 1e-06, + "loss": 0.0288, + "num_tokens": 10244537.0, + "reward": 12.45703125, + "reward_std": 13.960775375366211, + "rewards/rm_reward_func/mean": 12.45703125, + "rewards/rm_reward_func/std": 23.239635467529297, + "step": 770 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 414.78125, + "completions/mean_terminated_length": 376.7391357421875, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "epoch": 0.6168, + "grad_norm": 23.5693302154541, + "kl": 1.23095703125, + "learning_rate": 1e-06, + "loss": 0.0611, + "num_tokens": 10260738.0, + "reward": 7.40087890625, + "reward_std": 7.460350036621094, + "rewards/rm_reward_func/mean": 7.40087890625, + "rewards/rm_reward_func/std": 15.638055801391602, + "step": 771 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 442.0, + "completions/mean_length": 293.3125, + "completions/mean_terminated_length": 262.0714416503906, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.6176, + "grad_norm": 24.51900291442871, + "kl": 1.25, + "learning_rate": 1e-06, + "loss": -0.0135, + "num_tokens": 10275532.0, + "reward": 7.166015625, + "reward_std": 10.221585273742676, + "rewards/rm_reward_func/mean": 7.166015625, + "rewards/rm_reward_func/std": 18.162363052368164, + "step": 772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 351.6875, + "completions/mean_terminated_length": 341.0000305175781, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.6184, + "grad_norm": 11.453636169433594, + "kl": 0.533447265625, + "learning_rate": 1e-06, + "loss": -0.0055, + "num_tokens": 10288858.0, + "reward": 10.736896514892578, + "reward_std": 6.508322715759277, + "rewards/rm_reward_func/mean": 10.736896514892578, + "rewards/rm_reward_func/std": 15.74055290222168, + "step": 773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 489.0, + "completions/mean_length": 187.375, + "completions/mean_terminated_length": 165.73333740234375, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.6192, + "grad_norm": 11.30697250366211, + "kl": 2.5537109375, + "learning_rate": 1e-06, + "loss": 0.0897, + "num_tokens": 10299606.0, + "reward": -3.98828125, + "reward_std": 3.1018943786621094, + "rewards/rm_reward_func/mean": -3.98828125, + "rewards/rm_reward_func/std": 13.498686790466309, + "step": 774 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 507.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 319.6875, + "completions/mean_terminated_length": 319.6875, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.62, + "grad_norm": 13.508712768554688, + "kl": 5.896484375, + "learning_rate": 1e-06, + "loss": 0.2158, + "num_tokens": 10312068.0, + "reward": -2.490478515625, + "reward_std": 13.193625450134277, + "rewards/rm_reward_func/mean": -2.490478515625, + "rewards/rm_reward_func/std": 20.5009765625, + "step": 775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 284.3125, + "completions/mean_terminated_length": 220.55999755859375, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.6208, + "grad_norm": 30.201000213623047, + "kl": 3.12548828125, + "learning_rate": 1e-06, + "loss": 0.0509, + "num_tokens": 10324806.0, + "reward": 6.44482421875, + "reward_std": 9.712287902832031, + "rewards/rm_reward_func/mean": 6.44482421875, + "rewards/rm_reward_func/std": 14.30795669555664, + "step": 776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 299.1875, + "completions/mean_terminated_length": 292.32257080078125, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.6216, + "grad_norm": 33.898277282714844, + "kl": 3.952392578125, + "learning_rate": 1e-06, + "loss": 0.0502, + "num_tokens": 10337852.0, + "reward": 0.22265625, + "reward_std": 9.971935272216797, + "rewards/rm_reward_func/mean": 0.22265625, + "rewards/rm_reward_func/std": 19.976945877075195, + "step": 777 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 470.0, + "completions/mean_length": 257.09375, + "completions/mean_terminated_length": 248.87095642089844, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.6224, + "grad_norm": 7.3413543701171875, + "kl": 1.251708984375, + "learning_rate": 1e-06, + "loss": 0.005, + "num_tokens": 10350439.0, + "reward": 5.71923828125, + "reward_std": 5.034487724304199, + "rewards/rm_reward_func/mean": 5.71923828125, + "rewards/rm_reward_func/std": 12.428587913513184, + "step": 778 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 206.75, + "completions/mean_terminated_length": 196.90321350097656, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "epoch": 0.6232, + "grad_norm": 15.728623390197754, + "kl": 5.38427734375, + "learning_rate": 1e-06, + "loss": 0.185, + "num_tokens": 10360047.0, + "reward": -2.06689453125, + "reward_std": 5.258633613586426, + "rewards/rm_reward_func/mean": -2.06689453125, + "rewards/rm_reward_func/std": 14.16489315032959, + "step": 779 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 181.03125, + "completions/mean_terminated_length": 170.35482788085938, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.624, + "grad_norm": 9.44076919555664, + "kl": 0.400390625, + "learning_rate": 1e-06, + "loss": 0.0051, + "num_tokens": 10369952.0, + "reward": 6.325927734375, + "reward_std": 2.282184600830078, + "rewards/rm_reward_func/mean": 6.325927734375, + "rewards/rm_reward_func/std": 5.712595462799072, + "step": 780 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 298.90625, + "completions/mean_terminated_length": 292.0322570800781, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.6248, + "grad_norm": 7.031983375549316, + "kl": 3.04150390625, + "learning_rate": 1e-06, + "loss": -0.0902, + "num_tokens": 10384941.0, + "reward": 12.3427734375, + "reward_std": 17.103315353393555, + "rewards/rm_reward_func/mean": 12.3427734375, + "rewards/rm_reward_func/std": 21.697189331054688, + "step": 781 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 345.53125, + "completions/mean_terminated_length": 334.433349609375, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.6256, + "grad_norm": 12.419363021850586, + "kl": 1.218994140625, + "learning_rate": 1e-06, + "loss": -0.085, + "num_tokens": 10403062.0, + "reward": 9.69482421875, + "reward_std": 10.32385540008545, + "rewards/rm_reward_func/mean": 9.69482421875, + "rewards/rm_reward_func/std": 19.279569625854492, + "step": 782 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 271.0625, + "completions/mean_terminated_length": 236.6428680419922, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.6264, + "grad_norm": 13.157743453979492, + "kl": 1.40673828125, + "learning_rate": 1e-06, + "loss": 0.0223, + "num_tokens": 10414368.0, + "reward": 5.6474609375, + "reward_std": 5.8091206550598145, + "rewards/rm_reward_func/mean": 5.6474609375, + "rewards/rm_reward_func/std": 13.779108047485352, + "step": 783 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 284.125, + "completions/mean_terminated_length": 268.933349609375, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.6272, + "grad_norm": 4.267988681793213, + "kl": 0.6849365234375, + "learning_rate": 1e-06, + "loss": 0.0464, + "num_tokens": 10426644.0, + "reward": 8.6116943359375, + "reward_std": 8.114376068115234, + "rewards/rm_reward_func/mean": 8.6116943359375, + "rewards/rm_reward_func/std": 13.790817260742188, + "step": 784 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 310.46875, + "completions/mean_terminated_length": 231.60870361328125, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.628, + "grad_norm": 26.008434295654297, + "kl": 1.810791015625, + "learning_rate": 1e-06, + "loss": 0.0316, + "num_tokens": 10443571.0, + "reward": 7.016845703125, + "reward_std": 8.47242259979248, + "rewards/rm_reward_func/mean": 7.016845703125, + "rewards/rm_reward_func/std": 13.28702449798584, + "step": 785 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 244.15625, + "completions/mean_terminated_length": 216.44827270507812, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.6288, + "grad_norm": 18.478044509887695, + "kl": 6.23046875, + "learning_rate": 1e-06, + "loss": 0.1698, + "num_tokens": 10453696.0, + "reward": 2.2125244140625, + "reward_std": 8.634559631347656, + "rewards/rm_reward_func/mean": 2.2125244140625, + "rewards/rm_reward_func/std": 14.19184684753418, + "step": 786 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 280.875, + "completions/mean_terminated_length": 238.07408142089844, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.6296, + "grad_norm": 8.919168472290039, + "kl": 0.99560546875, + "learning_rate": 1e-06, + "loss": -0.0556, + "num_tokens": 10467428.0, + "reward": -6.05810546875, + "reward_std": 5.291695594787598, + "rewards/rm_reward_func/mean": -6.05810546875, + "rewards/rm_reward_func/std": 14.077133178710938, + "step": 787 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 350.96875, + "completions/mean_terminated_length": 297.29168701171875, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.6304, + "grad_norm": 10.0316743850708, + "kl": 4.890625, + "learning_rate": 1e-06, + "loss": -0.1151, + "num_tokens": 10480643.0, + "reward": -2.29998779296875, + "reward_std": 18.708683013916016, + "rewards/rm_reward_func/mean": -2.29998779296875, + "rewards/rm_reward_func/std": 21.16217041015625, + "step": 788 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 298.5, + "completions/mean_terminated_length": 284.2666931152344, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "epoch": 0.6312, + "grad_norm": 21.06656837463379, + "kl": 6.18212890625, + "learning_rate": 1e-06, + "loss": 0.3144, + "num_tokens": 10492979.0, + "reward": -6.215576171875, + "reward_std": 7.852952003479004, + "rewards/rm_reward_func/mean": -6.215576171875, + "rewards/rm_reward_func/std": 14.924491882324219, + "step": 789 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 495.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 184.15625, + "completions/mean_terminated_length": 184.15625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.632, + "grad_norm": 6.202207565307617, + "kl": 1.20654296875, + "learning_rate": 1e-06, + "loss": 0.0211, + "num_tokens": 10503632.0, + "reward": 7.91845703125, + "reward_std": 6.895031452178955, + "rewards/rm_reward_func/mean": 7.91845703125, + "rewards/rm_reward_func/std": 13.781877517700195, + "step": 790 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 200.8125, + "completions/mean_terminated_length": 168.6206817626953, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.6328, + "grad_norm": 19.258525848388672, + "kl": 2.4609375, + "learning_rate": 1e-06, + "loss": -0.188, + "num_tokens": 10512882.0, + "reward": 0.01904296875, + "reward_std": 13.03593635559082, + "rewards/rm_reward_func/mean": 0.01904296875, + "rewards/rm_reward_func/std": 16.95264434814453, + "step": 791 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 376.71875, + "completions/mean_terminated_length": 331.625, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.6336, + "grad_norm": 15.64734172821045, + "kl": 5.783203125, + "learning_rate": 1e-06, + "loss": 0.1589, + "num_tokens": 10527289.0, + "reward": 0.9697265625, + "reward_std": 10.092090606689453, + "rewards/rm_reward_func/mean": 0.9697265625, + "rewards/rm_reward_func/std": 18.045101165771484, + "step": 792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 442.59375, + "completions/mean_terminated_length": 419.4583435058594, + "completions/min_length": 307.0, + "completions/min_terminated_length": 307.0, + "epoch": 0.6344, + "grad_norm": 5.187870025634766, + "kl": 2.06005859375, + "learning_rate": 1e-06, + "loss": 0.0698, + "num_tokens": 10543972.0, + "reward": 14.57086181640625, + "reward_std": 9.486396789550781, + "rewards/rm_reward_func/mean": 14.57086181640625, + "rewards/rm_reward_func/std": 18.273242950439453, + "step": 793 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 387.5625, + "completions/mean_terminated_length": 364.5185241699219, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.6352, + "grad_norm": 14.29277229309082, + "kl": 4.53125, + "learning_rate": 1e-06, + "loss": 0.1114, + "num_tokens": 10560582.0, + "reward": -0.26617431640625, + "reward_std": 8.553365707397461, + "rewards/rm_reward_func/mean": -0.26617431640625, + "rewards/rm_reward_func/std": 10.436263084411621, + "step": 794 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 236.59375, + "completions/mean_terminated_length": 227.7096710205078, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.636, + "grad_norm": 7.5032501220703125, + "kl": 1.154296875, + "learning_rate": 1e-06, + "loss": 0.0065, + "num_tokens": 10571985.0, + "reward": 7.23681640625, + "reward_std": 8.498395919799805, + "rewards/rm_reward_func/mean": 7.23681640625, + "rewards/rm_reward_func/std": 13.58292007446289, + "step": 795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 239.15625, + "completions/mean_terminated_length": 162.75999450683594, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.6368, + "grad_norm": 21.288700103759766, + "kl": 6.2919921875, + "learning_rate": 1e-06, + "loss": 0.1455, + "num_tokens": 10582014.0, + "reward": -8.60565185546875, + "reward_std": 5.36991548538208, + "rewards/rm_reward_func/mean": -8.60565185546875, + "rewards/rm_reward_func/std": 15.003857612609863, + "step": 796 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 247.03125, + "completions/mean_terminated_length": 209.17857360839844, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.6376, + "grad_norm": 7.441081523895264, + "kl": 2.571533203125, + "learning_rate": 1e-06, + "loss": 0.0626, + "num_tokens": 10592679.0, + "reward": 2.77825927734375, + "reward_std": 2.551882028579712, + "rewards/rm_reward_func/mean": 2.77825927734375, + "rewards/rm_reward_func/std": 8.97035026550293, + "step": 797 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 176.28125, + "completions/mean_terminated_length": 165.4516143798828, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.6384, + "grad_norm": 6.360789775848389, + "kl": 0.4638671875, + "learning_rate": 1e-06, + "loss": 0.0344, + "num_tokens": 10604672.0, + "reward": 15.427978515625, + "reward_std": 2.812357187271118, + "rewards/rm_reward_func/mean": 15.427978515625, + "rewards/rm_reward_func/std": 14.900362014770508, + "step": 798 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 338.125, + "completions/mean_terminated_length": 259.0909118652344, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.6392, + "grad_norm": 6.797297954559326, + "kl": 0.995361328125, + "learning_rate": 1e-06, + "loss": 0.0529, + "num_tokens": 10621628.0, + "reward": 0.404541015625, + "reward_std": 7.615126609802246, + "rewards/rm_reward_func/mean": 0.404541015625, + "rewards/rm_reward_func/std": 12.74864673614502, + "step": 799 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 421.46875, + "completions/mean_terminated_length": 374.0476379394531, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.64, + "grad_norm": 9.847874641418457, + "kl": 2.7900390625, + "learning_rate": 1e-06, + "loss": 0.0953, + "num_tokens": 10638051.0, + "reward": 3.163330078125, + "reward_std": 10.612391471862793, + "rewards/rm_reward_func/mean": 3.163330078125, + "rewards/rm_reward_func/std": 15.437744140625, + "step": 800 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 455.0, + "completions/mean_length": 216.84375, + "completions/mean_terminated_length": 148.73077392578125, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.6408, + "grad_norm": 10.988313674926758, + "kl": 3.015625, + "learning_rate": 1e-06, + "loss": 0.1434, + "num_tokens": 10647630.0, + "reward": -3.7789306640625, + "reward_std": 6.626611232757568, + "rewards/rm_reward_func/mean": -3.7789306640625, + "rewards/rm_reward_func/std": 10.867609977722168, + "step": 801 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 401.0625, + "completions/mean_terminated_length": 364.0833435058594, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.6416, + "grad_norm": 6.588342666625977, + "kl": 1.3544921875, + "learning_rate": 1e-06, + "loss": -0.0717, + "num_tokens": 10665272.0, + "reward": 3.876220703125, + "reward_std": 8.485553741455078, + "rewards/rm_reward_func/mean": 3.876220703125, + "rewards/rm_reward_func/std": 11.902339935302734, + "step": 802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 264.9375, + "completions/mean_terminated_length": 256.9677429199219, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.6424, + "grad_norm": 16.689680099487305, + "kl": 4.962890625, + "learning_rate": 1e-06, + "loss": 0.2397, + "num_tokens": 10676294.0, + "reward": -3.4169921875, + "reward_std": 6.7452592849731445, + "rewards/rm_reward_func/mean": -3.4169921875, + "rewards/rm_reward_func/std": 19.948022842407227, + "step": 803 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 485.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 270.46875, + "completions/mean_terminated_length": 270.46875, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.6432, + "grad_norm": 6.953851222991943, + "kl": 1.05712890625, + "learning_rate": 1e-06, + "loss": -0.0594, + "num_tokens": 10690029.0, + "reward": -0.96844482421875, + "reward_std": 4.365692138671875, + "rewards/rm_reward_func/mean": -0.96844482421875, + "rewards/rm_reward_func/std": 6.488677024841309, + "step": 804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 277.65625, + "completions/mean_terminated_length": 270.0967712402344, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.644, + "grad_norm": 6.296433448791504, + "kl": 1.06787109375, + "learning_rate": 1e-06, + "loss": -0.0623, + "num_tokens": 10701498.0, + "reward": 1.8621826171875, + "reward_std": 4.512002468109131, + "rewards/rm_reward_func/mean": 1.8621826171875, + "rewards/rm_reward_func/std": 13.205521583557129, + "step": 805 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 211.0625, + "completions/mean_terminated_length": 201.35482788085938, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.6448, + "grad_norm": 6.924914836883545, + "kl": 2.736328125, + "learning_rate": 1e-06, + "loss": 0.0839, + "num_tokens": 10712828.0, + "reward": 4.1123046875, + "reward_std": 2.182391405105591, + "rewards/rm_reward_func/mean": 4.1123046875, + "rewards/rm_reward_func/std": 6.567470073699951, + "step": 806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 256.65625, + "completions/mean_terminated_length": 209.37037658691406, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.6456, + "grad_norm": 18.669151306152344, + "kl": 6.09765625, + "learning_rate": 1e-06, + "loss": 0.2974, + "num_tokens": 10724233.0, + "reward": -15.416748046875, + "reward_std": 7.353342056274414, + "rewards/rm_reward_func/mean": -15.416748046875, + "rewards/rm_reward_func/std": 11.860098838806152, + "step": 807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 482.0, + "completions/mean_length": 396.25, + "completions/mean_terminated_length": 326.8000183105469, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.6464, + "grad_norm": 6.7639875411987305, + "kl": 4.123046875, + "learning_rate": 1e-06, + "loss": 0.1232, + "num_tokens": 10739529.0, + "reward": -0.310546875, + "reward_std": 9.28040885925293, + "rewards/rm_reward_func/mean": -0.310546875, + "rewards/rm_reward_func/std": 11.789945602416992, + "step": 808 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 353.40625, + "completions/mean_terminated_length": 324.03704833984375, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.6472, + "grad_norm": 11.955965042114258, + "kl": 5.74609375, + "learning_rate": 1e-06, + "loss": 0.123, + "num_tokens": 10756438.0, + "reward": 1.25439453125, + "reward_std": 12.41511058807373, + "rewards/rm_reward_func/mean": 1.25439453125, + "rewards/rm_reward_func/std": 13.135236740112305, + "step": 809 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 262.03125, + "completions/mean_terminated_length": 236.1724090576172, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.648, + "grad_norm": 6.4916911125183105, + "kl": 4.392578125, + "learning_rate": 1e-06, + "loss": 0.1702, + "num_tokens": 10767359.0, + "reward": -5.23974609375, + "reward_std": 5.122437477111816, + "rewards/rm_reward_func/mean": -5.23974609375, + "rewards/rm_reward_func/std": 10.06690502166748, + "step": 810 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 358.96875, + "completions/mean_terminated_length": 330.629638671875, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.6488, + "grad_norm": 38.71643829345703, + "kl": 10.5546875, + "learning_rate": 1e-06, + "loss": 0.3489, + "num_tokens": 10780798.0, + "reward": -9.0185546875, + "reward_std": 5.921327590942383, + "rewards/rm_reward_func/mean": -9.0185546875, + "rewards/rm_reward_func/std": 9.659137725830078, + "step": 811 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 368.40625, + "completions/mean_terminated_length": 358.8333435058594, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.6496, + "grad_norm": 8.951486587524414, + "kl": 3.515625, + "learning_rate": 1e-06, + "loss": -0.0127, + "num_tokens": 10795059.0, + "reward": -1.08935546875, + "reward_std": 11.639703750610352, + "rewards/rm_reward_func/mean": -1.08935546875, + "rewards/rm_reward_func/std": 14.727822303771973, + "step": 812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 480.0, + "completions/mean_length": 289.21875, + "completions/mean_terminated_length": 237.80770874023438, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.6504, + "grad_norm": 21.51280403137207, + "kl": 4.89453125, + "learning_rate": 1e-06, + "loss": 0.0933, + "num_tokens": 10806426.0, + "reward": -5.33056640625, + "reward_std": 8.251094818115234, + "rewards/rm_reward_func/mean": -5.33056640625, + "rewards/rm_reward_func/std": 14.58669376373291, + "step": 813 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 325.90625, + "completions/mean_terminated_length": 273.79998779296875, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.6512, + "grad_norm": 14.707463264465332, + "kl": 6.0205078125, + "learning_rate": 1e-06, + "loss": 0.1921, + "num_tokens": 10820431.0, + "reward": -2.914306640625, + "reward_std": 8.024243354797363, + "rewards/rm_reward_func/mean": -2.914306640625, + "rewards/rm_reward_func/std": 14.138604164123535, + "step": 814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 264.0625, + "completions/mean_terminated_length": 256.06451416015625, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.652, + "grad_norm": 9.099730491638184, + "kl": 2.73681640625, + "learning_rate": 1e-06, + "loss": 0.0878, + "num_tokens": 10831561.0, + "reward": -2.70703125, + "reward_std": 6.87619686126709, + "rewards/rm_reward_func/mean": -2.70703125, + "rewards/rm_reward_func/std": 13.68471908569336, + "step": 815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 276.15625, + "completions/mean_terminated_length": 260.433349609375, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.6528, + "grad_norm": 9.196378707885742, + "kl": 4.783203125, + "learning_rate": 1e-06, + "loss": 0.0501, + "num_tokens": 10843046.0, + "reward": -3.22216796875, + "reward_std": 6.319431781768799, + "rewards/rm_reward_func/mean": -3.22216796875, + "rewards/rm_reward_func/std": 8.792978286743164, + "step": 816 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 403.0, + "completions/max_terminated_length": 403.0, + "completions/mean_length": 156.75, + "completions/mean_terminated_length": 156.75, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "epoch": 0.6536, + "grad_norm": 7.186251163482666, + "kl": 5.7412109375, + "learning_rate": 1e-06, + "loss": 0.248, + "num_tokens": 10851550.0, + "reward": 1.57763671875, + "reward_std": 8.453566551208496, + "rewards/rm_reward_func/mean": 1.57763671875, + "rewards/rm_reward_func/std": 19.387672424316406, + "step": 817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 292.71875, + "completions/mean_terminated_length": 270.03448486328125, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.6544, + "grad_norm": 9.593155860900879, + "kl": 1.69677734375, + "learning_rate": 1e-06, + "loss": 0.0271, + "num_tokens": 10865029.0, + "reward": 5.674560546875, + "reward_std": 6.335400581359863, + "rewards/rm_reward_func/mean": 5.674560546875, + "rewards/rm_reward_func/std": 13.289620399475098, + "step": 818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 442.0, + "completions/mean_length": 311.0, + "completions/mean_terminated_length": 244.0, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.6552, + "grad_norm": 9.963946342468262, + "kl": 1.982421875, + "learning_rate": 1e-06, + "loss": 0.0487, + "num_tokens": 10877253.0, + "reward": -0.123199462890625, + "reward_std": 6.386770248413086, + "rewards/rm_reward_func/mean": -0.123199462890625, + "rewards/rm_reward_func/std": 10.461974143981934, + "step": 819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 426.0, + "completions/mean_length": 358.9375, + "completions/mean_terminated_length": 307.91668701171875, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.656, + "grad_norm": 6.363739013671875, + "kl": 2.058837890625, + "learning_rate": 1e-06, + "loss": 0.1152, + "num_tokens": 10893083.0, + "reward": -0.5570068359375, + "reward_std": 6.5521240234375, + "rewards/rm_reward_func/mean": -0.5570068359375, + "rewards/rm_reward_func/std": 18.305418014526367, + "step": 820 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 319.75, + "completions/mean_terminated_length": 299.862060546875, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.6568, + "grad_norm": 8.368305206298828, + "kl": 3.38720703125, + "learning_rate": 1e-06, + "loss": -0.0173, + "num_tokens": 10906779.0, + "reward": 4.049560546875, + "reward_std": 14.061559677124023, + "rewards/rm_reward_func/mean": 4.049560546875, + "rewards/rm_reward_func/std": 21.243824005126953, + "step": 821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 486.0, + "completions/mean_length": 324.6875, + "completions/mean_terminated_length": 290.0, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.6576, + "grad_norm": 8.28355598449707, + "kl": 2.156005859375, + "learning_rate": 1e-06, + "loss": -0.1195, + "num_tokens": 10919833.0, + "reward": -12.810791015625, + "reward_std": 5.67118501663208, + "rewards/rm_reward_func/mean": -12.810791015625, + "rewards/rm_reward_func/std": 7.540193557739258, + "step": 822 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 424.0, + "completions/max_terminated_length": 424.0, + "completions/mean_length": 167.9375, + "completions/mean_terminated_length": 167.9375, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "epoch": 0.6584, + "grad_norm": 7.692931175231934, + "kl": 4.6162109375, + "learning_rate": 1e-06, + "loss": 0.2633, + "num_tokens": 10932023.0, + "reward": -4.395263671875, + "reward_std": 5.285364151000977, + "rewards/rm_reward_func/mean": -4.395263671875, + "rewards/rm_reward_func/std": 9.415786743164062, + "step": 823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 262.84375, + "completions/mean_terminated_length": 193.0800018310547, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "epoch": 0.6592, + "grad_norm": 11.74027156829834, + "kl": 0.53271484375, + "learning_rate": 1e-06, + "loss": -0.1976, + "num_tokens": 10943026.0, + "reward": 4.1466064453125, + "reward_std": 7.172249794006348, + "rewards/rm_reward_func/mean": 4.1466064453125, + "rewards/rm_reward_func/std": 11.245109558105469, + "step": 824 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 314.4375, + "completions/mean_terminated_length": 237.13043212890625, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.66, + "grad_norm": 4.412758827209473, + "kl": 0.615966796875, + "learning_rate": 1e-06, + "loss": 0.0277, + "num_tokens": 10957904.0, + "reward": -2.131103515625, + "reward_std": 4.323854446411133, + "rewards/rm_reward_func/mean": -2.131103515625, + "rewards/rm_reward_func/std": 8.868707656860352, + "step": 825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 462.0, + "completions/mean_length": 349.3125, + "completions/mean_terminated_length": 319.1851806640625, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.6608, + "grad_norm": 8.348759651184082, + "kl": 1.59619140625, + "learning_rate": 1e-06, + "loss": 0.065, + "num_tokens": 10971842.0, + "reward": 0.430908203125, + "reward_std": 5.89361047744751, + "rewards/rm_reward_func/mean": 0.430908203125, + "rewards/rm_reward_func/std": 11.598136901855469, + "step": 826 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 261.21875, + "completions/mean_terminated_length": 261.21875, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.6616, + "grad_norm": 10.604769706726074, + "kl": 1.353515625, + "learning_rate": 1e-06, + "loss": -0.1187, + "num_tokens": 10982777.0, + "reward": 1.006195068359375, + "reward_std": 13.28079891204834, + "rewards/rm_reward_func/mean": 1.006195068359375, + "rewards/rm_reward_func/std": 14.09341049194336, + "step": 827 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 333.65625, + "completions/mean_terminated_length": 321.7666931152344, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.6624, + "grad_norm": 6.49513053894043, + "kl": 2.354248046875, + "learning_rate": 1e-06, + "loss": 0.1536, + "num_tokens": 10996190.0, + "reward": 4.603271484375, + "reward_std": 7.740581035614014, + "rewards/rm_reward_func/mean": 4.603271484375, + "rewards/rm_reward_func/std": 18.76112937927246, + "step": 828 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 470.0, + "completions/mean_length": 340.15625, + "completions/mean_terminated_length": 282.875, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "epoch": 0.6632, + "grad_norm": 9.52800464630127, + "kl": 2.73828125, + "learning_rate": 1e-06, + "loss": -0.0569, + "num_tokens": 11009467.0, + "reward": 3.36962890625, + "reward_std": 10.40433120727539, + "rewards/rm_reward_func/mean": 3.36962890625, + "rewards/rm_reward_func/std": 13.418220520019531, + "step": 829 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 240.65625, + "completions/mean_terminated_length": 222.56668090820312, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.664, + "grad_norm": 10.561025619506836, + "kl": 2.5478515625, + "learning_rate": 1e-06, + "loss": 0.0396, + "num_tokens": 11020232.0, + "reward": 8.3240966796875, + "reward_std": 10.61776351928711, + "rewards/rm_reward_func/mean": 8.3240966796875, + "rewards/rm_reward_func/std": 14.970850944519043, + "step": 830 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 330.1875, + "completions/mean_terminated_length": 279.2799987792969, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.6648, + "grad_norm": 11.870400428771973, + "kl": 3.390625, + "learning_rate": 1e-06, + "loss": 0.0118, + "num_tokens": 11033358.0, + "reward": 5.705322265625, + "reward_std": 7.648243427276611, + "rewards/rm_reward_func/mean": 5.705322265625, + "rewards/rm_reward_func/std": 19.251300811767578, + "step": 831 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 383.34375, + "completions/mean_terminated_length": 370.03448486328125, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.6656, + "grad_norm": 7.473958969116211, + "kl": 0.41357421875, + "learning_rate": 1e-06, + "loss": 0.0041, + "num_tokens": 11050633.0, + "reward": 9.1138916015625, + "reward_std": 4.864926338195801, + "rewards/rm_reward_func/mean": 9.1138916015625, + "rewards/rm_reward_func/std": 15.038235664367676, + "step": 832 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 295.90625, + "completions/mean_terminated_length": 288.93548583984375, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.6664, + "grad_norm": 8.113051414489746, + "kl": 1.4638671875, + "learning_rate": 1e-06, + "loss": 0.0144, + "num_tokens": 11063022.0, + "reward": 3.6669921875, + "reward_std": 7.224357604980469, + "rewards/rm_reward_func/mean": 3.6669921875, + "rewards/rm_reward_func/std": 12.741311073303223, + "step": 833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 489.0, + "completions/mean_length": 336.53125, + "completions/mean_terminated_length": 330.8709716796875, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "epoch": 0.6672, + "grad_norm": 6.7608513832092285, + "kl": 0.941162109375, + "learning_rate": 1e-06, + "loss": -0.0642, + "num_tokens": 11076839.0, + "reward": 0.627685546875, + "reward_std": 4.866849422454834, + "rewards/rm_reward_func/mean": 0.627685546875, + "rewards/rm_reward_func/std": 9.038817405700684, + "step": 834 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 283.34375, + "completions/mean_terminated_length": 268.1000061035156, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.668, + "grad_norm": 22.115230560302734, + "kl": 7.388671875, + "learning_rate": 1e-06, + "loss": 0.1979, + "num_tokens": 11091042.0, + "reward": 0.4773712158203125, + "reward_std": 9.4666109085083, + "rewards/rm_reward_func/mean": 0.4773712158203125, + "rewards/rm_reward_func/std": 18.32492446899414, + "step": 835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 397.0, + "completions/mean_length": 184.3125, + "completions/mean_terminated_length": 162.4666748046875, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.6688, + "grad_norm": 31.762920379638672, + "kl": 12.359375, + "learning_rate": 1e-06, + "loss": 0.5706, + "num_tokens": 11102388.0, + "reward": -8.64892578125, + "reward_std": 7.599771499633789, + "rewards/rm_reward_func/mean": -8.64892578125, + "rewards/rm_reward_func/std": 9.485981941223145, + "step": 836 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 424.0, + "completions/mean_length": 242.21875, + "completions/mean_terminated_length": 192.25926208496094, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.6696, + "grad_norm": 27.026330947875977, + "kl": 9.296875, + "learning_rate": 1e-06, + "loss": 0.2068, + "num_tokens": 11115539.0, + "reward": -7.0478515625, + "reward_std": 10.2100248336792, + "rewards/rm_reward_func/mean": -7.0478515625, + "rewards/rm_reward_func/std": 18.10556411743164, + "step": 837 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 494.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 272.75, + "completions/mean_terminated_length": 272.75, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.6704, + "grad_norm": 8.460334777832031, + "kl": 2.99609375, + "learning_rate": 1e-06, + "loss": -0.0436, + "num_tokens": 11126803.0, + "reward": 3.97998046875, + "reward_std": 8.513212203979492, + "rewards/rm_reward_func/mean": 3.97998046875, + "rewards/rm_reward_func/std": 12.565332412719727, + "step": 838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 388.0, + "completions/max_terminated_length": 388.0, + "completions/mean_length": 158.90625, + "completions/mean_terminated_length": 158.90625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.6712, + "grad_norm": 7.5811028480529785, + "kl": 0.560546875, + "learning_rate": 1e-06, + "loss": 0.0078, + "num_tokens": 11135056.0, + "reward": 10.41888427734375, + "reward_std": 2.0775063037872314, + "rewards/rm_reward_func/mean": 10.41888427734375, + "rewards/rm_reward_func/std": 4.920321941375732, + "step": 839 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 194.75, + "completions/mean_terminated_length": 173.60000610351562, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.672, + "grad_norm": 17.789752960205078, + "kl": 7.279296875, + "learning_rate": 1e-06, + "loss": 0.2592, + "num_tokens": 11143560.0, + "reward": -6.46221923828125, + "reward_std": 8.128403663635254, + "rewards/rm_reward_func/mean": -6.46221923828125, + "rewards/rm_reward_func/std": 8.697508811950684, + "step": 840 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 324.78125, + "completions/mean_terminated_length": 318.7419128417969, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.6728, + "grad_norm": 7.485666751861572, + "kl": 1.71435546875, + "learning_rate": 1e-06, + "loss": 0.0202, + "num_tokens": 11158129.0, + "reward": 14.0751953125, + "reward_std": 8.069329261779785, + "rewards/rm_reward_func/mean": 14.0751953125, + "rewards/rm_reward_func/std": 10.917937278747559, + "step": 841 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 448.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 282.53125, + "completions/mean_terminated_length": 282.53125, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.6736, + "grad_norm": 8.429314613342285, + "kl": 3.7744140625, + "learning_rate": 1e-06, + "loss": 0.1521, + "num_tokens": 11170770.0, + "reward": 8.86328125, + "reward_std": 12.041173934936523, + "rewards/rm_reward_func/mean": 8.86328125, + "rewards/rm_reward_func/std": 16.136621475219727, + "step": 842 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 335.0, + "completions/mean_length": 147.4375, + "completions/mean_terminated_length": 109.72413635253906, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.6744, + "grad_norm": 7.667215347290039, + "kl": 2.34375, + "learning_rate": 1e-06, + "loss": 0.0167, + "num_tokens": 11182616.0, + "reward": 9.590576171875, + "reward_std": 2.2192811965942383, + "rewards/rm_reward_func/mean": 9.590576171875, + "rewards/rm_reward_func/std": 11.049510955810547, + "step": 843 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 389.0, + "completions/mean_length": 146.96875, + "completions/mean_terminated_length": 135.19354248046875, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.6752, + "grad_norm": 28.290157318115234, + "kl": 6.166015625, + "learning_rate": 1e-06, + "loss": 0.2478, + "num_tokens": 11192607.0, + "reward": -7.3682861328125, + "reward_std": 4.421894073486328, + "rewards/rm_reward_func/mean": -7.3682861328125, + "rewards/rm_reward_func/std": 13.098028182983398, + "step": 844 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 437.0, + "completions/mean_length": 278.875, + "completions/mean_terminated_length": 271.3548278808594, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.676, + "grad_norm": 18.575132369995117, + "kl": 5.462890625, + "learning_rate": 1e-06, + "loss": 0.1575, + "num_tokens": 11204379.0, + "reward": -2.055908203125, + "reward_std": 6.175776481628418, + "rewards/rm_reward_func/mean": -2.055908203125, + "rewards/rm_reward_func/std": 10.646092414855957, + "step": 845 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 335.0, + "completions/max_terminated_length": 335.0, + "completions/mean_length": 123.96875, + "completions/mean_terminated_length": 123.96875, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.6768, + "grad_norm": 14.692407608032227, + "kl": 4.3916015625, + "learning_rate": 1e-06, + "loss": 0.1037, + "num_tokens": 11213498.0, + "reward": 2.0, + "reward_std": 3.80605149269104, + "rewards/rm_reward_func/mean": 2.0, + "rewards/rm_reward_func/std": 9.591236114501953, + "step": 846 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 285.75, + "completions/mean_terminated_length": 278.45159912109375, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.6776, + "grad_norm": 9.11420726776123, + "kl": 1.35791015625, + "learning_rate": 1e-06, + "loss": -0.1391, + "num_tokens": 11224962.0, + "reward": 7.869140625, + "reward_std": 11.650513648986816, + "rewards/rm_reward_func/mean": 7.869140625, + "rewards/rm_reward_func/std": 14.315463066101074, + "step": 847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 411.0, + "completions/max_terminated_length": 411.0, + "completions/mean_length": 196.5625, + "completions/mean_terminated_length": 196.5625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.6784, + "grad_norm": 9.939024925231934, + "kl": 0.9248046875, + "learning_rate": 1e-06, + "loss": -0.0194, + "num_tokens": 11233572.0, + "reward": 7.11181640625, + "reward_std": 5.81515645980835, + "rewards/rm_reward_func/mean": 7.11181640625, + "rewards/rm_reward_func/std": 10.113432884216309, + "step": 848 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 374.0, + "completions/max_terminated_length": 374.0, + "completions/mean_length": 218.65625, + "completions/mean_terminated_length": 218.65625, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.6792, + "grad_norm": 14.37784194946289, + "kl": 2.833984375, + "learning_rate": 1e-06, + "loss": -0.0176, + "num_tokens": 11242921.0, + "reward": -6.9840850830078125, + "reward_std": 5.020229816436768, + "rewards/rm_reward_func/mean": -6.9840850830078125, + "rewards/rm_reward_func/std": 8.650106430053711, + "step": 849 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 252.0, + "completions/max_terminated_length": 252.0, + "completions/mean_length": 133.5, + "completions/mean_terminated_length": 133.5, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.68, + "grad_norm": 27.497337341308594, + "kl": 2.4345703125, + "learning_rate": 1e-06, + "loss": 0.0773, + "num_tokens": 11250937.0, + "reward": 6.5244140625, + "reward_std": 7.013378143310547, + "rewards/rm_reward_func/mean": 6.5244140625, + "rewards/rm_reward_func/std": 11.999058723449707, + "step": 850 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 397.40625, + "completions/mean_terminated_length": 381.0357360839844, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "epoch": 0.6808, + "grad_norm": 6.854437351226807, + "kl": 0.42431640625, + "learning_rate": 1e-06, + "loss": -0.0064, + "num_tokens": 11266398.0, + "reward": 6.41265869140625, + "reward_std": 3.7932474613189697, + "rewards/rm_reward_func/mean": 6.41265869140625, + "rewards/rm_reward_func/std": 17.261632919311523, + "step": 851 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 500.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 158.09375, + "completions/mean_terminated_length": 158.09375, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.6816, + "grad_norm": 10.595882415771484, + "kl": 3.65625, + "learning_rate": 1e-06, + "loss": 0.0449, + "num_tokens": 11274801.0, + "reward": -12.980224609375, + "reward_std": 5.851085662841797, + "rewards/rm_reward_func/mean": -12.980224609375, + "rewards/rm_reward_func/std": 13.411259651184082, + "step": 852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 232.75, + "completions/mean_terminated_length": 223.74192810058594, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.6824, + "grad_norm": 11.931048393249512, + "kl": 4.76171875, + "learning_rate": 1e-06, + "loss": 0.1431, + "num_tokens": 11284161.0, + "reward": -9.20654296875, + "reward_std": 10.515345573425293, + "rewards/rm_reward_func/mean": -9.20654296875, + "rewards/rm_reward_func/std": 18.860841751098633, + "step": 853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 253.5625, + "completions/mean_terminated_length": 236.33334350585938, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.6832, + "grad_norm": 13.002371788024902, + "kl": 1.6796875, + "learning_rate": 1e-06, + "loss": 0.0015, + "num_tokens": 11294715.0, + "reward": 5.0428466796875, + "reward_std": 6.735762596130371, + "rewards/rm_reward_func/mean": 5.0428466796875, + "rewards/rm_reward_func/std": 12.883460998535156, + "step": 854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 482.0, + "completions/mean_length": 225.5625, + "completions/mean_terminated_length": 206.4666748046875, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.684, + "grad_norm": 15.005594253540039, + "kl": 2.9814453125, + "learning_rate": 1e-06, + "loss": -0.0264, + "num_tokens": 11304629.0, + "reward": -3.63232421875, + "reward_std": 7.6953206062316895, + "rewards/rm_reward_func/mean": -3.63232421875, + "rewards/rm_reward_func/std": 14.550228118896484, + "step": 855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 424.0, + "completions/mean_length": 222.5625, + "completions/mean_terminated_length": 155.7692413330078, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.6848, + "grad_norm": 13.886627197265625, + "kl": 2.037109375, + "learning_rate": 1e-06, + "loss": -0.0007, + "num_tokens": 11317831.0, + "reward": -3.196533203125, + "reward_std": 2.4364099502563477, + "rewards/rm_reward_func/mean": -3.196533203125, + "rewards/rm_reward_func/std": 9.435223579406738, + "step": 856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 473.0, + "completions/max_terminated_length": 473.0, + "completions/mean_length": 276.1875, + "completions/mean_terminated_length": 276.1875, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.6856, + "grad_norm": 42.20450210571289, + "kl": 2.49951171875, + "learning_rate": 1e-06, + "loss": 0.085, + "num_tokens": 11331389.0, + "reward": -5.261474609375, + "reward_std": 6.263424873352051, + "rewards/rm_reward_func/mean": -5.261474609375, + "rewards/rm_reward_func/std": 8.193347930908203, + "step": 857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 399.0, + "completions/max_terminated_length": 399.0, + "completions/mean_length": 256.1875, + "completions/mean_terminated_length": 256.1875, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.6864, + "grad_norm": 11.973814010620117, + "kl": 0.7158203125, + "learning_rate": 1e-06, + "loss": 0.0198, + "num_tokens": 11341651.0, + "reward": 8.2965087890625, + "reward_std": 7.567012786865234, + "rewards/rm_reward_func/mean": 8.2965087890625, + "rewards/rm_reward_func/std": 12.142066955566406, + "step": 858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 231.15625, + "completions/mean_terminated_length": 202.10345458984375, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.6872, + "grad_norm": 5.184216499328613, + "kl": 1.4384765625, + "learning_rate": 1e-06, + "loss": 0.0217, + "num_tokens": 11351872.0, + "reward": 6.35546875, + "reward_std": 2.562025785446167, + "rewards/rm_reward_func/mean": 6.35546875, + "rewards/rm_reward_func/std": 8.541465759277344, + "step": 859 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 454.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 188.5625, + "completions/mean_terminated_length": 188.5625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.688, + "grad_norm": 8.177552223205566, + "kl": 2.0556640625, + "learning_rate": 1e-06, + "loss": 0.0713, + "num_tokens": 11361914.0, + "reward": -1.439453125, + "reward_std": 3.191701889038086, + "rewards/rm_reward_func/mean": -1.439453125, + "rewards/rm_reward_func/std": 13.077706336975098, + "step": 860 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 168.21875, + "completions/mean_terminated_length": 168.21875, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.6888, + "grad_norm": 12.462203979492188, + "kl": 0.489501953125, + "learning_rate": 1e-06, + "loss": -0.0841, + "num_tokens": 11371129.0, + "reward": 0.892578125, + "reward_std": 6.40195369720459, + "rewards/rm_reward_func/mean": 0.892578125, + "rewards/rm_reward_func/std": 15.08119010925293, + "step": 861 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 496.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 218.28125, + "completions/mean_terminated_length": 218.28125, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.6896, + "grad_norm": 9.378438949584961, + "kl": 1.27490234375, + "learning_rate": 1e-06, + "loss": 0.0262, + "num_tokens": 11381746.0, + "reward": 8.6956787109375, + "reward_std": 7.514378547668457, + "rewards/rm_reward_func/mean": 8.6956787109375, + "rewards/rm_reward_func/std": 12.007038116455078, + "step": 862 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 452.0, + "completions/max_terminated_length": 452.0, + "completions/mean_length": 251.5, + "completions/mean_terminated_length": 251.5, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.6904, + "grad_norm": 9.017190933227539, + "kl": 0.43896484375, + "learning_rate": 1e-06, + "loss": -0.0491, + "num_tokens": 11395986.0, + "reward": 4.80322265625, + "reward_std": 2.939964771270752, + "rewards/rm_reward_func/mean": 4.80322265625, + "rewards/rm_reward_func/std": 9.644649505615234, + "step": 863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 288.875, + "completions/mean_terminated_length": 281.6773986816406, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.6912, + "grad_norm": 8.287468910217285, + "kl": 1.0576171875, + "learning_rate": 1e-06, + "loss": -0.0984, + "num_tokens": 11407550.0, + "reward": -3.62890625, + "reward_std": 5.023155689239502, + "rewards/rm_reward_func/mean": -3.62890625, + "rewards/rm_reward_func/std": 15.647568702697754, + "step": 864 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 451.0, + "completions/mean_length": 234.59375, + "completions/mean_terminated_length": 225.64515686035156, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.692, + "grad_norm": 12.379044532775879, + "kl": 1.384765625, + "learning_rate": 1e-06, + "loss": -0.0319, + "num_tokens": 11419385.0, + "reward": 1.34521484375, + "reward_std": 5.130009174346924, + "rewards/rm_reward_func/mean": 1.34521484375, + "rewards/rm_reward_func/std": 16.12451171875, + "step": 865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 402.0, + "completions/mean_length": 298.53125, + "completions/mean_terminated_length": 249.2692413330078, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.6928, + "grad_norm": 8.24583911895752, + "kl": 2.95263671875, + "learning_rate": 1e-06, + "loss": 0.0369, + "num_tokens": 11434746.0, + "reward": -10.24560546875, + "reward_std": 6.283100128173828, + "rewards/rm_reward_func/mean": -10.24560546875, + "rewards/rm_reward_func/std": 11.58389949798584, + "step": 866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 402.0, + "completions/max_terminated_length": 402.0, + "completions/mean_length": 245.84375, + "completions/mean_terminated_length": 245.84375, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.6936, + "grad_norm": 11.311857223510742, + "kl": 1.7294921875, + "learning_rate": 1e-06, + "loss": 0.0011, + "num_tokens": 11445405.0, + "reward": -2.562255859375, + "reward_std": 4.00094747543335, + "rewards/rm_reward_func/mean": -2.562255859375, + "rewards/rm_reward_func/std": 5.972076416015625, + "step": 867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 219.96875, + "completions/mean_terminated_length": 189.7586212158203, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.6944, + "grad_norm": 21.208097457885742, + "kl": 4.55859375, + "learning_rate": 1e-06, + "loss": 0.1824, + "num_tokens": 11455276.0, + "reward": 0.225830078125, + "reward_std": 9.15473461151123, + "rewards/rm_reward_func/mean": 0.225830078125, + "rewards/rm_reward_func/std": 12.977889060974121, + "step": 868 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 295.71875, + "completions/mean_terminated_length": 255.6666717529297, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.6952, + "grad_norm": 61.403076171875, + "kl": 2.2958984375, + "learning_rate": 1e-06, + "loss": 0.0541, + "num_tokens": 11466739.0, + "reward": -4.762786865234375, + "reward_std": 12.194247245788574, + "rewards/rm_reward_func/mean": -4.762786865234375, + "rewards/rm_reward_func/std": 13.474750518798828, + "step": 869 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 387.0, + "completions/max_terminated_length": 387.0, + "completions/mean_length": 194.40625, + "completions/mean_terminated_length": 194.40625, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.696, + "grad_norm": 11.072949409484863, + "kl": 2.7587890625, + "learning_rate": 1e-06, + "loss": 0.1886, + "num_tokens": 11477680.0, + "reward": -2.43963623046875, + "reward_std": 2.318965435028076, + "rewards/rm_reward_func/mean": -2.43963623046875, + "rewards/rm_reward_func/std": 8.259891510009766, + "step": 870 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 359.0, + "completions/max_terminated_length": 359.0, + "completions/mean_length": 120.15625, + "completions/mean_terminated_length": 120.15625, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.6968, + "grad_norm": 22.81490135192871, + "kl": 2.748046875, + "learning_rate": 1e-06, + "loss": 0.1335, + "num_tokens": 11484381.0, + "reward": -4.06182861328125, + "reward_std": 3.879387855529785, + "rewards/rm_reward_func/mean": -4.06182861328125, + "rewards/rm_reward_func/std": 8.522046089172363, + "step": 871 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 470.0, + "completions/max_terminated_length": 470.0, + "completions/mean_length": 184.25, + "completions/mean_terminated_length": 184.25, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.6976, + "grad_norm": 21.19168472290039, + "kl": 1.7607421875, + "learning_rate": 1e-06, + "loss": 0.0231, + "num_tokens": 11493045.0, + "reward": -6.307861328125, + "reward_std": 5.177760601043701, + "rewards/rm_reward_func/mean": -6.307861328125, + "rewards/rm_reward_func/std": 17.660837173461914, + "step": 872 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 491.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 302.21875, + "completions/mean_terminated_length": 302.21875, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.6984, + "grad_norm": 13.353934288024902, + "kl": 0.684326171875, + "learning_rate": 1e-06, + "loss": -0.0657, + "num_tokens": 11505260.0, + "reward": 6.2265625, + "reward_std": 4.352338790893555, + "rewards/rm_reward_func/mean": 6.2265625, + "rewards/rm_reward_func/std": 12.638473510742188, + "step": 873 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 469.0, + "completions/max_terminated_length": 469.0, + "completions/mean_length": 275.75, + "completions/mean_terminated_length": 275.75, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.6992, + "grad_norm": 10.98495864868164, + "kl": 2.4599609375, + "learning_rate": 1e-06, + "loss": 0.0102, + "num_tokens": 11518476.0, + "reward": -2.28125, + "reward_std": 8.162097930908203, + "rewards/rm_reward_func/mean": -2.28125, + "rewards/rm_reward_func/std": 16.660022735595703, + "step": 874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 360.09375, + "completions/mean_terminated_length": 309.4583435058594, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.7, + "grad_norm": 10.327630043029785, + "kl": 0.825439453125, + "learning_rate": 1e-06, + "loss": 0.0071, + "num_tokens": 11532407.0, + "reward": 5.72357177734375, + "reward_std": 6.062471389770508, + "rewards/rm_reward_func/mean": 5.72357177734375, + "rewards/rm_reward_func/std": 8.450596809387207, + "step": 875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 261.65625, + "completions/mean_terminated_length": 253.5806427001953, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.7008, + "grad_norm": 12.771651268005371, + "kl": 0.5576171875, + "learning_rate": 1e-06, + "loss": -0.3164, + "num_tokens": 11544620.0, + "reward": -0.7434616088867188, + "reward_std": 7.011323928833008, + "rewards/rm_reward_func/mean": -0.7434616088867188, + "rewards/rm_reward_func/std": 9.20909595489502, + "step": 876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 449.0, + "completions/mean_length": 357.53125, + "completions/mean_terminated_length": 306.04168701171875, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.7016, + "grad_norm": 6.507169246673584, + "kl": 0.88671875, + "learning_rate": 1e-06, + "loss": 0.0546, + "num_tokens": 11558797.0, + "reward": 2.79498291015625, + "reward_std": 4.3798723220825195, + "rewards/rm_reward_func/mean": 2.79498291015625, + "rewards/rm_reward_func/std": 11.152603149414062, + "step": 877 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 459.0, + "completions/mean_length": 337.21875, + "completions/mean_terminated_length": 325.5666809082031, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.7024, + "grad_norm": 7.747379302978516, + "kl": 2.21875, + "learning_rate": 1e-06, + "loss": -0.0081, + "num_tokens": 11572412.0, + "reward": -0.7412109375, + "reward_std": 10.57952880859375, + "rewards/rm_reward_func/mean": -0.7412109375, + "rewards/rm_reward_func/std": 18.887123107910156, + "step": 878 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 467.0, + "completions/mean_length": 245.84375, + "completions/mean_terminated_length": 218.3103485107422, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.7032, + "grad_norm": 12.370904922485352, + "kl": 1.8740234375, + "learning_rate": 1e-06, + "loss": 0.0823, + "num_tokens": 11583719.0, + "reward": -5.319202423095703, + "reward_std": 8.250015258789062, + "rewards/rm_reward_func/mean": -5.319202423095703, + "rewards/rm_reward_func/std": 13.415038108825684, + "step": 879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 256.78125, + "completions/mean_terminated_length": 239.7666778564453, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.704, + "grad_norm": 6.51011848449707, + "kl": 0.8095703125, + "learning_rate": 1e-06, + "loss": 0.0124, + "num_tokens": 11594712.0, + "reward": 12.4833984375, + "reward_std": 4.730257034301758, + "rewards/rm_reward_func/mean": 12.4833984375, + "rewards/rm_reward_func/std": 8.233878135681152, + "step": 880 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 372.90625, + "completions/mean_terminated_length": 368.4193420410156, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.7048, + "grad_norm": 6.078239440917969, + "kl": 0.372802734375, + "learning_rate": 1e-06, + "loss": -0.0806, + "num_tokens": 11608573.0, + "reward": 10.44952392578125, + "reward_std": 5.383957386016846, + "rewards/rm_reward_func/mean": 10.44952392578125, + "rewards/rm_reward_func/std": 14.256927490234375, + "step": 881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 276.0625, + "completions/mean_terminated_length": 251.65516662597656, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.7056, + "grad_norm": 13.18306827545166, + "kl": 3.087646484375, + "learning_rate": 1e-06, + "loss": 0.2504, + "num_tokens": 11622719.0, + "reward": 2.23345947265625, + "reward_std": 10.839166641235352, + "rewards/rm_reward_func/mean": 2.23345947265625, + "rewards/rm_reward_func/std": 13.140902519226074, + "step": 882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 79.0, + "completions/mean_length": 174.28125, + "completions/mean_terminated_length": 61.708335876464844, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "epoch": 0.7064, + "grad_norm": 9.840855598449707, + "kl": 0.5, + "learning_rate": 1e-06, + "loss": -0.0233, + "num_tokens": 11631200.0, + "reward": 2.555927276611328, + "reward_std": 1.7101624011993408, + "rewards/rm_reward_func/mean": 2.555927276611328, + "rewards/rm_reward_func/std": 6.12503719329834, + "step": 883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 397.53125, + "completions/mean_terminated_length": 385.6896667480469, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.7072, + "grad_norm": 6.078858852386475, + "kl": 0.498046875, + "learning_rate": 1e-06, + "loss": -0.0905, + "num_tokens": 11647873.0, + "reward": 2.686767578125, + "reward_std": 7.326623439788818, + "rewards/rm_reward_func/mean": 2.686767578125, + "rewards/rm_reward_func/std": 11.423783302307129, + "step": 884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 377.34375, + "completions/mean_terminated_length": 339.6399841308594, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.708, + "grad_norm": 10.225689888000488, + "kl": 2.7353515625, + "learning_rate": 1e-06, + "loss": 0.0753, + "num_tokens": 11662996.0, + "reward": 2.117431640625, + "reward_std": 9.586727142333984, + "rewards/rm_reward_func/mean": 2.117431640625, + "rewards/rm_reward_func/std": 12.49357795715332, + "step": 885 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 482.0, + "completions/mean_length": 235.8125, + "completions/mean_terminated_length": 196.35714721679688, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "epoch": 0.7088, + "grad_norm": 17.94410514831543, + "kl": 3.6484375, + "learning_rate": 1e-06, + "loss": 0.1815, + "num_tokens": 11672782.0, + "reward": -1.3177490234375, + "reward_std": 9.219121932983398, + "rewards/rm_reward_func/mean": -1.3177490234375, + "rewards/rm_reward_func/std": 11.736974716186523, + "step": 886 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 326.125, + "completions/mean_terminated_length": 264.16668701171875, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.7096, + "grad_norm": 11.345696449279785, + "kl": 5.15625, + "learning_rate": 1e-06, + "loss": 0.192, + "num_tokens": 11690626.0, + "reward": 4.96630859375, + "reward_std": 6.32305908203125, + "rewards/rm_reward_func/mean": 4.96630859375, + "rewards/rm_reward_func/std": 13.138326644897461, + "step": 887 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 408.0, + "completions/max_terminated_length": 408.0, + "completions/mean_length": 187.46875, + "completions/mean_terminated_length": 187.46875, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.7104, + "grad_norm": 21.07513427734375, + "kl": 0.44580078125, + "learning_rate": 1e-06, + "loss": -0.0441, + "num_tokens": 11699697.0, + "reward": 3.6982421875, + "reward_std": 3.48081636428833, + "rewards/rm_reward_func/mean": 3.6982421875, + "rewards/rm_reward_func/std": 14.544777870178223, + "step": 888 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 460.0, + "completions/mean_length": 328.15625, + "completions/mean_terminated_length": 322.2257995605469, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.7112, + "grad_norm": 7.2959303855896, + "kl": 3.615478515625, + "learning_rate": 1e-06, + "loss": 0.152, + "num_tokens": 11712518.0, + "reward": 3.5261077880859375, + "reward_std": 9.151225090026855, + "rewards/rm_reward_func/mean": 3.5261077880859375, + "rewards/rm_reward_func/std": 11.036227226257324, + "step": 889 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 467.0, + "completions/mean_length": 306.21875, + "completions/mean_terminated_length": 268.1111145019531, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.712, + "grad_norm": 7.229876518249512, + "kl": 4.206298828125, + "learning_rate": 1e-06, + "loss": 0.1888, + "num_tokens": 11726501.0, + "reward": 5.666015625, + "reward_std": 12.855541229248047, + "rewards/rm_reward_func/mean": 5.666015625, + "rewards/rm_reward_func/std": 19.8636531829834, + "step": 890 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 465.0, + "completions/mean_length": 177.25, + "completions/mean_terminated_length": 166.4516143798828, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.7128, + "grad_norm": 11.873459815979004, + "kl": 4.671875, + "learning_rate": 1e-06, + "loss": 0.1815, + "num_tokens": 11735293.0, + "reward": -2.13092041015625, + "reward_std": 2.0769262313842773, + "rewards/rm_reward_func/mean": -2.13092041015625, + "rewards/rm_reward_func/std": 10.466885566711426, + "step": 891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 430.71875, + "completions/mean_terminated_length": 375.1052551269531, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "epoch": 0.7136, + "grad_norm": 15.590665817260742, + "kl": 9.9453125, + "learning_rate": 1e-06, + "loss": 0.3228, + "num_tokens": 11751772.0, + "reward": -10.29931640625, + "reward_std": 10.349671363830566, + "rewards/rm_reward_func/mean": -10.29931640625, + "rewards/rm_reward_func/std": 14.615714073181152, + "step": 892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 477.0, + "completions/mean_length": 215.03125, + "completions/mean_terminated_length": 172.60714721679688, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.7144, + "grad_norm": 23.91371726989746, + "kl": 7.96875, + "learning_rate": 1e-06, + "loss": 0.3347, + "num_tokens": 11762333.0, + "reward": -2.80712890625, + "reward_std": 3.1941628456115723, + "rewards/rm_reward_func/mean": -2.80712890625, + "rewards/rm_reward_func/std": 15.444784164428711, + "step": 893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 337.59375, + "completions/mean_terminated_length": 218.26315307617188, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.7152, + "grad_norm": 15.916248321533203, + "kl": 10.64453125, + "learning_rate": 1e-06, + "loss": 0.4986, + "num_tokens": 11777384.0, + "reward": -6.3330078125, + "reward_std": 8.051085472106934, + "rewards/rm_reward_func/mean": -6.3330078125, + "rewards/rm_reward_func/std": 13.273330688476562, + "step": 894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 423.625, + "completions/mean_terminated_length": 389.0434875488281, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.716, + "grad_norm": 27.08871078491211, + "kl": 11.65234375, + "learning_rate": 1e-06, + "loss": 0.4485, + "num_tokens": 11792972.0, + "reward": -5.36322021484375, + "reward_std": 8.532169342041016, + "rewards/rm_reward_func/mean": -5.36322021484375, + "rewards/rm_reward_func/std": 14.15001106262207, + "step": 895 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 302.5, + "completions/mean_terminated_length": 232.6666717529297, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.7168, + "grad_norm": 15.366225242614746, + "kl": 8.49609375, + "learning_rate": 1e-06, + "loss": 0.2878, + "num_tokens": 11804820.0, + "reward": -3.9163665771484375, + "reward_std": 10.814449310302734, + "rewards/rm_reward_func/mean": -3.9163665771484375, + "rewards/rm_reward_func/std": 13.626547813415527, + "step": 896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 459.0, + "completions/mean_length": 301.5625, + "completions/mean_terminated_length": 262.59259033203125, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.7176, + "grad_norm": 16.501802444458008, + "kl": 6.5625, + "learning_rate": 1e-06, + "loss": 0.3279, + "num_tokens": 11817710.0, + "reward": -13.09716796875, + "reward_std": 3.4331793785095215, + "rewards/rm_reward_func/mean": -13.09716796875, + "rewards/rm_reward_func/std": 7.296655178070068, + "step": 897 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 383.15625, + "completions/mean_terminated_length": 305.8500061035156, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.7184, + "grad_norm": 53.010684967041016, + "kl": 7.34423828125, + "learning_rate": 1e-06, + "loss": 0.2151, + "num_tokens": 11835283.0, + "reward": -2.9622802734375, + "reward_std": 7.178778648376465, + "rewards/rm_reward_func/mean": -2.9622802734375, + "rewards/rm_reward_func/std": 15.090096473693848, + "step": 898 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 429.0, + "completions/mean_length": 188.46875, + "completions/mean_terminated_length": 178.03225708007812, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.7192, + "grad_norm": 12.646398544311523, + "kl": 2.4697265625, + "learning_rate": 1e-06, + "loss": 0.0837, + "num_tokens": 11844258.0, + "reward": -5.5667266845703125, + "reward_std": 3.9434971809387207, + "rewards/rm_reward_func/mean": -5.5667266845703125, + "rewards/rm_reward_func/std": 9.585037231445312, + "step": 899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 374.71875, + "completions/mean_terminated_length": 336.2799987792969, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.72, + "grad_norm": 14.73233413696289, + "kl": 4.44140625, + "learning_rate": 1e-06, + "loss": 0.2017, + "num_tokens": 11858337.0, + "reward": 4.2802734375, + "reward_std": 4.078566074371338, + "rewards/rm_reward_func/mean": 4.2802734375, + "rewards/rm_reward_func/std": 18.690641403198242, + "step": 900 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 297.0625, + "completions/mean_terminated_length": 236.87998962402344, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.7208, + "grad_norm": 12.702629089355469, + "kl": 1.1796875, + "learning_rate": 1e-06, + "loss": 0.0394, + "num_tokens": 11871475.0, + "reward": -1.2879791259765625, + "reward_std": 9.910463333129883, + "rewards/rm_reward_func/mean": -1.2879791259765625, + "rewards/rm_reward_func/std": 13.349971771240234, + "step": 901 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 477.0, + "completions/mean_length": 292.21875, + "completions/mean_terminated_length": 260.8214416503906, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.7216, + "grad_norm": 4.130387306213379, + "kl": 1.59130859375, + "learning_rate": 1e-06, + "loss": 0.0345, + "num_tokens": 11884650.0, + "reward": -2.958984375, + "reward_std": 4.643342971801758, + "rewards/rm_reward_func/mean": -2.958984375, + "rewards/rm_reward_func/std": 10.972875595092773, + "step": 902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 469.0, + "completions/mean_length": 226.59375, + "completions/mean_terminated_length": 146.67999267578125, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.7224, + "grad_norm": 7.769096851348877, + "kl": 3.14453125, + "learning_rate": 1e-06, + "loss": 0.1549, + "num_tokens": 11894645.0, + "reward": -5.154296875, + "reward_std": 1.5148166418075562, + "rewards/rm_reward_func/mean": -5.154296875, + "rewards/rm_reward_func/std": 8.897757530212402, + "step": 903 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 368.5625, + "completions/mean_terminated_length": 335.4615478515625, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.7232, + "grad_norm": 8.09273910522461, + "kl": 2.3486328125, + "learning_rate": 1e-06, + "loss": 0.146, + "num_tokens": 11908447.0, + "reward": -3.269775390625, + "reward_std": 5.145322799682617, + "rewards/rm_reward_func/mean": -3.269775390625, + "rewards/rm_reward_func/std": 9.46324634552002, + "step": 904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 311.125, + "completions/mean_terminated_length": 264.76922607421875, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.724, + "grad_norm": 8.956731796264648, + "kl": 2.220703125, + "learning_rate": 1e-06, + "loss": 0.0198, + "num_tokens": 11921539.0, + "reward": -2.416015625, + "reward_std": 7.836517333984375, + "rewards/rm_reward_func/mean": -2.416015625, + "rewards/rm_reward_func/std": 19.40036392211914, + "step": 905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 403.78125, + "completions/mean_terminated_length": 354.5909118652344, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.7248, + "grad_norm": 5.295882701873779, + "kl": 3.423828125, + "learning_rate": 1e-06, + "loss": 0.1755, + "num_tokens": 11936668.0, + "reward": -7.770965576171875, + "reward_std": 7.4591522216796875, + "rewards/rm_reward_func/mean": -7.770965576171875, + "rewards/rm_reward_func/std": 9.990768432617188, + "step": 906 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 367.15625, + "completions/mean_terminated_length": 254.5, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.7256, + "grad_norm": 9.633946418762207, + "kl": 2.376953125, + "learning_rate": 1e-06, + "loss": 0.0554, + "num_tokens": 11952129.0, + "reward": -9.96728515625, + "reward_std": 5.492212772369385, + "rewards/rm_reward_func/mean": -9.96728515625, + "rewards/rm_reward_func/std": 7.280327320098877, + "step": 907 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 175.5, + "completions/mean_terminated_length": 127.42857360839844, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.7264, + "grad_norm": 4.560550212860107, + "kl": 0.552734375, + "learning_rate": 1e-06, + "loss": 0.0122, + "num_tokens": 11962097.0, + "reward": 7.42626953125, + "reward_std": 3.087636709213257, + "rewards/rm_reward_func/mean": 7.42626953125, + "rewards/rm_reward_func/std": 7.106700897216797, + "step": 908 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 299.21875, + "completions/mean_terminated_length": 285.0333557128906, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.7272, + "grad_norm": 7.450979232788086, + "kl": 1.3125, + "learning_rate": 1e-06, + "loss": 0.0609, + "num_tokens": 11974520.0, + "reward": 3.33935546875, + "reward_std": 8.265787124633789, + "rewards/rm_reward_func/mean": 3.33935546875, + "rewards/rm_reward_func/std": 13.333514213562012, + "step": 909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 366.6875, + "completions/mean_terminated_length": 309.8260803222656, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.728, + "grad_norm": 8.965705871582031, + "kl": 1.3427734375, + "learning_rate": 1e-06, + "loss": 0.0475, + "num_tokens": 11989574.0, + "reward": -3.5350265502929688, + "reward_std": 7.181827068328857, + "rewards/rm_reward_func/mean": -3.5350265502929688, + "rewards/rm_reward_func/std": 8.107301712036133, + "step": 910 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 325.25, + "completions/mean_terminated_length": 290.6666564941406, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.7288, + "grad_norm": 6.053767681121826, + "kl": 0.9969482421875, + "learning_rate": 1e-06, + "loss": 0.0931, + "num_tokens": 12003822.0, + "reward": -4.5394287109375, + "reward_std": 6.439982891082764, + "rewards/rm_reward_func/mean": -4.5394287109375, + "rewards/rm_reward_func/std": 12.76736831665039, + "step": 911 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 380.8125, + "completions/mean_terminated_length": 265.058837890625, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.7296, + "grad_norm": 13.645872116088867, + "kl": 1.7490234375, + "learning_rate": 1e-06, + "loss": 0.0605, + "num_tokens": 12019832.0, + "reward": -1.08984375, + "reward_std": 7.622467994689941, + "rewards/rm_reward_func/mean": -1.08984375, + "rewards/rm_reward_func/std": 14.811065673828125, + "step": 912 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 312.78125, + "completions/mean_terminated_length": 266.8077087402344, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.7304, + "grad_norm": 9.30956745147705, + "kl": 1.2353515625, + "learning_rate": 1e-06, + "loss": -0.0088, + "num_tokens": 12032193.0, + "reward": 1.819305419921875, + "reward_std": 11.730962753295898, + "rewards/rm_reward_func/mean": 1.819305419921875, + "rewards/rm_reward_func/std": 17.3555908203125, + "step": 913 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 403.78125, + "completions/mean_terminated_length": 367.7083435058594, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "epoch": 0.7312, + "grad_norm": 8.707047462463379, + "kl": 1.447509765625, + "learning_rate": 1e-06, + "loss": -0.0823, + "num_tokens": 12047786.0, + "reward": 1.6616268157958984, + "reward_std": 7.0725531578063965, + "rewards/rm_reward_func/mean": 1.6616268157958984, + "rewards/rm_reward_func/std": 11.841200828552246, + "step": 914 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 325.09375, + "completions/mean_terminated_length": 305.75860595703125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.732, + "grad_norm": 6.918102741241455, + "kl": 1.4365234375, + "learning_rate": 1e-06, + "loss": 0.0435, + "num_tokens": 12061229.0, + "reward": -0.952178955078125, + "reward_std": 10.511100769042969, + "rewards/rm_reward_func/mean": -0.952178955078125, + "rewards/rm_reward_func/std": 13.577516555786133, + "step": 915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 302.4375, + "completions/mean_terminated_length": 272.5, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.7328, + "grad_norm": 16.212295532226562, + "kl": 1.50927734375, + "learning_rate": 1e-06, + "loss": 0.2198, + "num_tokens": 12073931.0, + "reward": -2.2662353515625, + "reward_std": 6.384040832519531, + "rewards/rm_reward_func/mean": -2.2662353515625, + "rewards/rm_reward_func/std": 8.968281745910645, + "step": 916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 220.5625, + "completions/mean_terminated_length": 201.1333465576172, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.7336, + "grad_norm": 6.973321914672852, + "kl": 0.521484375, + "learning_rate": 1e-06, + "loss": -0.0016, + "num_tokens": 12084549.0, + "reward": 6.8271484375, + "reward_std": 3.8583192825317383, + "rewards/rm_reward_func/mean": 6.8271484375, + "rewards/rm_reward_func/std": 11.556774139404297, + "step": 917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 432.0, + "completions/max_terminated_length": 432.0, + "completions/mean_length": 193.75, + "completions/mean_terminated_length": 193.75, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.7344, + "grad_norm": 11.093525886535645, + "kl": 0.890625, + "learning_rate": 1e-06, + "loss": -0.0353, + "num_tokens": 12093829.0, + "reward": 7.5962066650390625, + "reward_std": 5.227684497833252, + "rewards/rm_reward_func/mean": 7.5962066650390625, + "rewards/rm_reward_func/std": 7.640827178955078, + "step": 918 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 263.03125, + "completions/mean_terminated_length": 246.433349609375, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.7352, + "grad_norm": 8.780437469482422, + "kl": 1.28955078125, + "learning_rate": 1e-06, + "loss": -0.0192, + "num_tokens": 12105222.0, + "reward": 0.42479705810546875, + "reward_std": 5.950567722320557, + "rewards/rm_reward_func/mean": 0.42479705810546875, + "rewards/rm_reward_func/std": 10.092032432556152, + "step": 919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 482.0, + "completions/mean_length": 341.21875, + "completions/mean_terminated_length": 284.29168701171875, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.736, + "grad_norm": 18.315034866333008, + "kl": 2.19921875, + "learning_rate": 1e-06, + "loss": 0.3037, + "num_tokens": 12118829.0, + "reward": 7.9627838134765625, + "reward_std": 6.95778226852417, + "rewards/rm_reward_func/mean": 7.9627838134765625, + "rewards/rm_reward_func/std": 21.903451919555664, + "step": 920 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 388.34375, + "completions/mean_terminated_length": 339.95654296875, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.7368, + "grad_norm": 10.440993309020996, + "kl": 0.66943359375, + "learning_rate": 1e-06, + "loss": 0.1228, + "num_tokens": 12133104.0, + "reward": 8.855255126953125, + "reward_std": 10.921690940856934, + "rewards/rm_reward_func/mean": 8.855255126953125, + "rewards/rm_reward_func/std": 17.32367515563965, + "step": 921 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 327.875, + "completions/mean_terminated_length": 285.3846130371094, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.7376, + "grad_norm": 13.36120891571045, + "kl": 1.40380859375, + "learning_rate": 1e-06, + "loss": -0.0399, + "num_tokens": 12145796.0, + "reward": -1.751861572265625, + "reward_std": 5.661749839782715, + "rewards/rm_reward_func/mean": -1.751861572265625, + "rewards/rm_reward_func/std": 7.456704616546631, + "step": 922 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 396.90625, + "completions/mean_terminated_length": 370.3461608886719, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.7384, + "grad_norm": 7.57150936126709, + "kl": 4.10546875, + "learning_rate": 1e-06, + "loss": 0.0999, + "num_tokens": 12160545.0, + "reward": 2.98095703125, + "reward_std": 12.732685089111328, + "rewards/rm_reward_func/mean": 2.98095703125, + "rewards/rm_reward_func/std": 16.23927879333496, + "step": 923 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 403.375, + "completions/mean_terminated_length": 360.86956787109375, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.7392, + "grad_norm": 17.749086380004883, + "kl": 3.669921875, + "learning_rate": 1e-06, + "loss": 0.1162, + "num_tokens": 12175453.0, + "reward": -4.994476318359375, + "reward_std": 6.3849382400512695, + "rewards/rm_reward_func/mean": -4.994476318359375, + "rewards/rm_reward_func/std": 7.732586860656738, + "step": 924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 375.03125, + "completions/mean_terminated_length": 349.6666564941406, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.74, + "grad_norm": 10.675846099853516, + "kl": 2.8515625, + "learning_rate": 1e-06, + "loss": 0.0318, + "num_tokens": 12189734.0, + "reward": -0.42730712890625, + "reward_std": 8.244186401367188, + "rewards/rm_reward_func/mean": -0.42730712890625, + "rewards/rm_reward_func/std": 11.274063110351562, + "step": 925 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 321.875, + "completions/mean_terminated_length": 258.5, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.7408, + "grad_norm": 20.539968490600586, + "kl": 7.5546875, + "learning_rate": 1e-06, + "loss": 0.1656, + "num_tokens": 12202114.0, + "reward": -17.49755859375, + "reward_std": 5.544781684875488, + "rewards/rm_reward_func/mean": -17.49755859375, + "rewards/rm_reward_func/std": 8.136129379272461, + "step": 926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 215.375, + "completions/mean_terminated_length": 146.92308044433594, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.7416, + "grad_norm": 16.78243637084961, + "kl": 1.171875, + "learning_rate": 1e-06, + "loss": 0.0322, + "num_tokens": 12214726.0, + "reward": -3.51953125, + "reward_std": 2.579955577850342, + "rewards/rm_reward_func/mean": -3.51953125, + "rewards/rm_reward_func/std": 12.52515697479248, + "step": 927 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 469.0, + "completions/mean_length": 340.34375, + "completions/mean_terminated_length": 292.2799987792969, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.7424, + "grad_norm": 34.499900817871094, + "kl": 9.70751953125, + "learning_rate": 1e-06, + "loss": 0.3566, + "num_tokens": 12227649.0, + "reward": -9.71435546875, + "reward_std": 4.9702467918396, + "rewards/rm_reward_func/mean": -9.71435546875, + "rewards/rm_reward_func/std": 13.78485107421875, + "step": 928 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 340.21875, + "completions/mean_terminated_length": 308.40740966796875, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.7432, + "grad_norm": 6.942431449890137, + "kl": 4.47021484375, + "learning_rate": 1e-06, + "loss": 0.0818, + "num_tokens": 12240512.0, + "reward": -4.043701171875, + "reward_std": 9.750587463378906, + "rewards/rm_reward_func/mean": -4.043701171875, + "rewards/rm_reward_func/std": 10.625539779663086, + "step": 929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 423.0, + "completions/mean_length": 287.4375, + "completions/mean_terminated_length": 212.58334350585938, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.744, + "grad_norm": 31.259197235107422, + "kl": 9.36083984375, + "learning_rate": 1e-06, + "loss": 0.3652, + "num_tokens": 12253814.0, + "reward": -10.76513671875, + "reward_std": 2.81522274017334, + "rewards/rm_reward_func/mean": -10.76513671875, + "rewards/rm_reward_func/std": 10.83386516571045, + "step": 930 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 469.0, + "completions/mean_length": 214.125, + "completions/mean_terminated_length": 194.2666778564453, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.7448, + "grad_norm": 33.200462341308594, + "kl": 10.28125, + "learning_rate": 1e-06, + "loss": 0.465, + "num_tokens": 12265218.0, + "reward": -19.12255859375, + "reward_std": 3.9576473236083984, + "rewards/rm_reward_func/mean": -19.12255859375, + "rewards/rm_reward_func/std": 7.913651466369629, + "step": 931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 276.65625, + "completions/mean_terminated_length": 269.06451416015625, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.7456, + "grad_norm": 22.295116424560547, + "kl": 6.87109375, + "learning_rate": 1e-06, + "loss": 0.1678, + "num_tokens": 12277919.0, + "reward": -6.76416015625, + "reward_std": 9.437423706054688, + "rewards/rm_reward_func/mean": -6.76416015625, + "rewards/rm_reward_func/std": 13.566336631774902, + "step": 932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 321.875, + "completions/mean_terminated_length": 294.71429443359375, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.7464, + "grad_norm": 38.953163146972656, + "kl": 10.8125, + "learning_rate": 1e-06, + "loss": 0.3992, + "num_tokens": 12290051.0, + "reward": -13.24609375, + "reward_std": 5.0346527099609375, + "rewards/rm_reward_func/mean": -13.24609375, + "rewards/rm_reward_func/std": 5.71676778793335, + "step": 933 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 240.125, + "completions/mean_terminated_length": 231.35482788085938, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.7472, + "grad_norm": 16.947662353515625, + "kl": 5.8046875, + "learning_rate": 1e-06, + "loss": 0.1648, + "num_tokens": 12299375.0, + "reward": -9.80364990234375, + "reward_std": 7.922030448913574, + "rewards/rm_reward_func/mean": -9.80364990234375, + "rewards/rm_reward_func/std": 8.047533988952637, + "step": 934 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 334.5, + "completions/mean_terminated_length": 265.0434875488281, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.748, + "grad_norm": 20.085559844970703, + "kl": 3.55517578125, + "learning_rate": 1e-06, + "loss": 0.0963, + "num_tokens": 12313439.0, + "reward": -4.38671875, + "reward_std": 5.369791030883789, + "rewards/rm_reward_func/mean": -4.38671875, + "rewards/rm_reward_func/std": 12.364655494689941, + "step": 935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 190.5625, + "completions/mean_terminated_length": 190.5625, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.7488, + "grad_norm": 19.231365203857422, + "kl": 5.12109375, + "learning_rate": 1e-06, + "loss": 0.3493, + "num_tokens": 12322225.0, + "reward": -9.181640625, + "reward_std": 6.0393853187561035, + "rewards/rm_reward_func/mean": -9.181640625, + "rewards/rm_reward_func/std": 11.844439506530762, + "step": 936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 376.625, + "completions/mean_terminated_length": 331.5, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.7496, + "grad_norm": 12.45418930053711, + "kl": 6.169921875, + "learning_rate": 1e-06, + "loss": 0.336, + "num_tokens": 12338125.0, + "reward": -4.42236328125, + "reward_std": 8.240564346313477, + "rewards/rm_reward_func/mean": -4.42236328125, + "rewards/rm_reward_func/std": 17.537168502807617, + "step": 937 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 455.0, + "completions/mean_length": 244.09375, + "completions/mean_terminated_length": 216.37930297851562, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.7504, + "grad_norm": 15.027655601501465, + "kl": 2.1923828125, + "learning_rate": 1e-06, + "loss": 0.0436, + "num_tokens": 12351256.0, + "reward": -4.74468994140625, + "reward_std": 5.563222885131836, + "rewards/rm_reward_func/mean": -4.74468994140625, + "rewards/rm_reward_func/std": 10.182250022888184, + "step": 938 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 455.0, + "completions/mean_length": 273.71875, + "completions/mean_terminated_length": 249.0689697265625, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.7512, + "grad_norm": 6.310357570648193, + "kl": 2.48291015625, + "learning_rate": 1e-06, + "loss": 0.1097, + "num_tokens": 12362543.0, + "reward": -3.814697265625, + "reward_std": 5.3505682945251465, + "rewards/rm_reward_func/mean": -3.814697265625, + "rewards/rm_reward_func/std": 7.993979454040527, + "step": 939 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 170.59375, + "completions/mean_terminated_length": 147.83334350585938, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.752, + "grad_norm": 10.239607810974121, + "kl": 1.9765625, + "learning_rate": 1e-06, + "loss": 0.2201, + "num_tokens": 12370810.0, + "reward": 6.2581787109375, + "reward_std": 5.198629379272461, + "rewards/rm_reward_func/mean": 6.2581787109375, + "rewards/rm_reward_func/std": 11.37491512298584, + "step": 940 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 358.1875, + "completions/mean_terminated_length": 342.2758483886719, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.7528, + "grad_norm": 6.657660961151123, + "kl": 2.30615234375, + "learning_rate": 1e-06, + "loss": 0.0582, + "num_tokens": 12384584.0, + "reward": 1.087982177734375, + "reward_std": 10.667032241821289, + "rewards/rm_reward_func/mean": 1.087982177734375, + "rewards/rm_reward_func/std": 15.517912864685059, + "step": 941 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 487.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 243.21875, + "completions/mean_terminated_length": 243.21875, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.7536, + "grad_norm": 7.709587574005127, + "kl": 2.3125, + "learning_rate": 1e-06, + "loss": 0.123, + "num_tokens": 12396471.0, + "reward": -9.607421875, + "reward_std": 2.2891464233398438, + "rewards/rm_reward_func/mean": -9.607421875, + "rewards/rm_reward_func/std": 11.87063217163086, + "step": 942 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 330.40625, + "completions/mean_terminated_length": 296.77777099609375, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.7544, + "grad_norm": 9.007935523986816, + "kl": 0.546142578125, + "learning_rate": 1e-06, + "loss": 0.0883, + "num_tokens": 12412252.0, + "reward": 3.67486572265625, + "reward_std": 3.7311360836029053, + "rewards/rm_reward_func/mean": 3.67486572265625, + "rewards/rm_reward_func/std": 8.098952293395996, + "step": 943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 447.0, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 180.96875, + "completions/mean_terminated_length": 180.96875, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.7552, + "grad_norm": 7.097393989562988, + "kl": 1.11328125, + "learning_rate": 1e-06, + "loss": 0.1134, + "num_tokens": 12421491.0, + "reward": 3.06976318359375, + "reward_std": 3.404435873031616, + "rewards/rm_reward_func/mean": 3.06976318359375, + "rewards/rm_reward_func/std": 5.2598958015441895, + "step": 944 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 225.625, + "completions/mean_terminated_length": 206.53334045410156, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.756, + "grad_norm": 6.910060405731201, + "kl": 0.96484375, + "learning_rate": 1e-06, + "loss": -0.0059, + "num_tokens": 12432391.0, + "reward": 1.87451171875, + "reward_std": 6.181256294250488, + "rewards/rm_reward_func/mean": 1.87451171875, + "rewards/rm_reward_func/std": 9.241499900817871, + "step": 945 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 264.9375, + "completions/mean_terminated_length": 168.26087951660156, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.7568, + "grad_norm": 18.712726593017578, + "kl": 0.5518798828125, + "learning_rate": 1e-06, + "loss": 0.1766, + "num_tokens": 12446741.0, + "reward": -6.08203125, + "reward_std": 6.159107685089111, + "rewards/rm_reward_func/mean": -6.08203125, + "rewards/rm_reward_func/std": 11.603202819824219, + "step": 946 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 298.78125, + "completions/mean_terminated_length": 276.7241516113281, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.7576, + "grad_norm": 6.926180362701416, + "kl": 0.4462890625, + "learning_rate": 1e-06, + "loss": 0.0145, + "num_tokens": 12458686.0, + "reward": 6.00732421875, + "reward_std": 3.4000492095947266, + "rewards/rm_reward_func/mean": 6.00732421875, + "rewards/rm_reward_func/std": 8.406200408935547, + "step": 947 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 415.0, + "completions/mean_length": 269.1875, + "completions/mean_terminated_length": 234.50001525878906, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.7584, + "grad_norm": 7.784516334533691, + "kl": 2.767578125, + "learning_rate": 1e-06, + "loss": -0.0341, + "num_tokens": 12469732.0, + "reward": -5.19873046875, + "reward_std": 12.44931697845459, + "rewards/rm_reward_func/mean": -5.19873046875, + "rewards/rm_reward_func/std": 13.344382286071777, + "step": 948 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 468.0, + "completions/mean_length": 263.5, + "completions/mean_terminated_length": 246.933349609375, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.7592, + "grad_norm": 7.466695785522461, + "kl": 1.8359375, + "learning_rate": 1e-06, + "loss": 0.0924, + "num_tokens": 12483748.0, + "reward": -2.9599609375, + "reward_std": 8.385852813720703, + "rewards/rm_reward_func/mean": -2.9599609375, + "rewards/rm_reward_func/std": 15.177970886230469, + "step": 949 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 242.59375, + "completions/mean_terminated_length": 233.90321350097656, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.76, + "grad_norm": 14.014596939086914, + "kl": 1.61474609375, + "learning_rate": 1e-06, + "loss": -0.0958, + "num_tokens": 12494015.0, + "reward": 1.893310546875, + "reward_std": 7.712099075317383, + "rewards/rm_reward_func/mean": 1.893310546875, + "rewards/rm_reward_func/std": 14.392072677612305, + "step": 950 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 370.1875, + "completions/mean_terminated_length": 314.6956481933594, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.7608, + "grad_norm": 9.24715805053711, + "kl": 1.931640625, + "learning_rate": 1e-06, + "loss": 0.0885, + "num_tokens": 12508949.0, + "reward": -0.7783203125, + "reward_std": 8.970268249511719, + "rewards/rm_reward_func/mean": -0.7783203125, + "rewards/rm_reward_func/std": 15.02639102935791, + "step": 951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 297.5625, + "completions/mean_terminated_length": 290.6451416015625, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "epoch": 0.7616, + "grad_norm": 13.282108306884766, + "kl": 2.541015625, + "learning_rate": 1e-06, + "loss": 0.0898, + "num_tokens": 12522735.0, + "reward": 3.421630859375, + "reward_std": 8.37659740447998, + "rewards/rm_reward_func/mean": 3.421630859375, + "rewards/rm_reward_func/std": 12.819450378417969, + "step": 952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 328.65625, + "completions/mean_terminated_length": 322.7419128417969, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.7624, + "grad_norm": 9.377727508544922, + "kl": 1.41259765625, + "learning_rate": 1e-06, + "loss": -0.0102, + "num_tokens": 12535252.0, + "reward": 4.1376953125, + "reward_std": 6.576689720153809, + "rewards/rm_reward_func/mean": 4.1376953125, + "rewards/rm_reward_func/std": 16.938846588134766, + "step": 953 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 379.1875, + "completions/mean_terminated_length": 360.21429443359375, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.7632, + "grad_norm": 13.483288764953613, + "kl": 1.65625, + "learning_rate": 1e-06, + "loss": -0.0113, + "num_tokens": 12549658.0, + "reward": 4.1842041015625, + "reward_std": 11.937593460083008, + "rewards/rm_reward_func/mean": 4.1842041015625, + "rewards/rm_reward_func/std": 16.32895851135254, + "step": 954 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 351.65625, + "completions/mean_terminated_length": 340.9666748046875, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.764, + "grad_norm": 7.8393707275390625, + "kl": 2.38818359375, + "learning_rate": 1e-06, + "loss": 0.0489, + "num_tokens": 12563167.0, + "reward": -6.7680511474609375, + "reward_std": 6.109908103942871, + "rewards/rm_reward_func/mean": -6.7680511474609375, + "rewards/rm_reward_func/std": 9.325031280517578, + "step": 955 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 507.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 253.34375, + "completions/mean_terminated_length": 253.34375, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.7648, + "grad_norm": 6.62860631942749, + "kl": 0.927734375, + "learning_rate": 1e-06, + "loss": -0.0399, + "num_tokens": 12576426.0, + "reward": -0.5888671875, + "reward_std": 4.022240161895752, + "rewards/rm_reward_func/mean": -0.5888671875, + "rewards/rm_reward_func/std": 6.941095352172852, + "step": 956 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 290.53125, + "completions/mean_terminated_length": 267.6206970214844, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.7656, + "grad_norm": 9.683281898498535, + "kl": 4.0126953125, + "learning_rate": 1e-06, + "loss": 0.0822, + "num_tokens": 12587875.0, + "reward": -11.738689422607422, + "reward_std": 5.651496887207031, + "rewards/rm_reward_func/mean": -11.738689422607422, + "rewards/rm_reward_func/std": 10.464712142944336, + "step": 957 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 452.0, + "completions/mean_length": 278.40625, + "completions/mean_terminated_length": 254.2413787841797, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.7664, + "grad_norm": 13.09759521484375, + "kl": 2.10693359375, + "learning_rate": 1e-06, + "loss": 0.0637, + "num_tokens": 12599120.0, + "reward": -3.7507591247558594, + "reward_std": 5.601696014404297, + "rewards/rm_reward_func/mean": -3.7507591247558594, + "rewards/rm_reward_func/std": 7.122979640960693, + "step": 958 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 385.4375, + "completions/mean_terminated_length": 362.0, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.7672, + "grad_norm": 17.111623764038086, + "kl": 2.60302734375, + "learning_rate": 1e-06, + "loss": 0.0545, + "num_tokens": 12614662.0, + "reward": 11.8017578125, + "reward_std": 12.20899772644043, + "rewards/rm_reward_func/mean": 11.8017578125, + "rewards/rm_reward_func/std": 16.13597297668457, + "step": 959 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 285.59375, + "completions/mean_terminated_length": 270.5, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.768, + "grad_norm": 18.32476806640625, + "kl": 2.861328125, + "learning_rate": 1e-06, + "loss": 0.1135, + "num_tokens": 12628177.0, + "reward": 10.161865234375, + "reward_std": 9.773149490356445, + "rewards/rm_reward_func/mean": 10.161865234375, + "rewards/rm_reward_func/std": 18.04802131652832, + "step": 960 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 314.96875, + "completions/mean_terminated_length": 294.5862121582031, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.7688, + "grad_norm": 11.534658432006836, + "kl": 2.671875, + "learning_rate": 1e-06, + "loss": 0.1081, + "num_tokens": 12640928.0, + "reward": -3.29248046875, + "reward_std": 11.981504440307617, + "rewards/rm_reward_func/mean": -3.29248046875, + "rewards/rm_reward_func/std": 14.41004753112793, + "step": 961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 430.0, + "completions/max_terminated_length": 430.0, + "completions/mean_length": 194.5, + "completions/mean_terminated_length": 194.5, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.7696, + "grad_norm": 6.043729782104492, + "kl": 0.4736328125, + "learning_rate": 1e-06, + "loss": -0.0156, + "num_tokens": 12653056.0, + "reward": -7.85498046875, + "reward_std": 1.4552382230758667, + "rewards/rm_reward_func/mean": -7.85498046875, + "rewards/rm_reward_func/std": 5.594080924987793, + "step": 962 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 284.84375, + "completions/mean_terminated_length": 269.70001220703125, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.7704, + "grad_norm": 9.446344375610352, + "kl": 1.1533203125, + "learning_rate": 1e-06, + "loss": 0.0328, + "num_tokens": 12664139.0, + "reward": 1.72314453125, + "reward_std": 7.9739813804626465, + "rewards/rm_reward_func/mean": 1.72314453125, + "rewards/rm_reward_func/std": 14.244680404663086, + "step": 963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 287.34375, + "completions/mean_terminated_length": 280.0967712402344, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.7712, + "grad_norm": 9.243260383605957, + "kl": 0.39892578125, + "learning_rate": 1e-06, + "loss": -0.0251, + "num_tokens": 12675110.0, + "reward": 13.383621215820312, + "reward_std": 4.858243942260742, + "rewards/rm_reward_func/mean": 13.383621215820312, + "rewards/rm_reward_func/std": 12.968853950500488, + "step": 964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 404.0, + "completions/mean_length": 262.125, + "completions/mean_terminated_length": 192.1599884033203, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.772, + "grad_norm": 13.435283660888672, + "kl": 4.7197265625, + "learning_rate": 1e-06, + "loss": 0.2661, + "num_tokens": 12689778.0, + "reward": -4.897491455078125, + "reward_std": 4.347750663757324, + "rewards/rm_reward_func/mean": -4.897491455078125, + "rewards/rm_reward_func/std": 11.386714935302734, + "step": 965 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 386.4375, + "completions/mean_terminated_length": 344.5833435058594, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "epoch": 0.7728, + "grad_norm": 6.507021903991699, + "kl": 1.4296875, + "learning_rate": 1e-06, + "loss": -0.0111, + "num_tokens": 12705064.0, + "reward": 13.21240234375, + "reward_std": 10.414816856384277, + "rewards/rm_reward_func/mean": 13.21240234375, + "rewards/rm_reward_func/std": 14.156932830810547, + "step": 966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 473.0, + "completions/max_terminated_length": 473.0, + "completions/mean_length": 306.90625, + "completions/mean_terminated_length": 306.90625, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.7736, + "grad_norm": 6.9047393798828125, + "kl": 2.828125, + "learning_rate": 1e-06, + "loss": -0.0205, + "num_tokens": 12716701.0, + "reward": 2.09149169921875, + "reward_std": 10.416152954101562, + "rewards/rm_reward_func/mean": 2.09149169921875, + "rewards/rm_reward_func/std": 12.270896911621094, + "step": 967 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 289.03125, + "completions/mean_terminated_length": 281.8387145996094, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.7744, + "grad_norm": 7.523399353027344, + "kl": 2.68798828125, + "learning_rate": 1e-06, + "loss": 0.0547, + "num_tokens": 12729430.0, + "reward": 2.884521484375, + "reward_std": 5.321774482727051, + "rewards/rm_reward_func/mean": 2.884521484375, + "rewards/rm_reward_func/std": 15.252958297729492, + "step": 968 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 423.0, + "completions/mean_length": 210.21875, + "completions/mean_terminated_length": 200.48387145996094, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.7752, + "grad_norm": 30.35150909423828, + "kl": 11.3837890625, + "learning_rate": 1e-06, + "loss": 0.6159, + "num_tokens": 12740117.0, + "reward": -4.417724609375, + "reward_std": 6.602893352508545, + "rewards/rm_reward_func/mean": -4.417724609375, + "rewards/rm_reward_func/std": 15.255743980407715, + "step": 969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 353.03125, + "completions/mean_terminated_length": 336.5862121582031, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.776, + "grad_norm": 8.360908508300781, + "kl": 2.9921875, + "learning_rate": 1e-06, + "loss": 0.1711, + "num_tokens": 12753542.0, + "reward": -0.7406158447265625, + "reward_std": 4.496933937072754, + "rewards/rm_reward_func/mean": -0.7406158447265625, + "rewards/rm_reward_func/std": 11.36292839050293, + "step": 970 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 360.15625, + "completions/mean_terminated_length": 291.1363830566406, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.7768, + "grad_norm": 10.487771987915039, + "kl": 0.97509765625, + "learning_rate": 1e-06, + "loss": 0.0275, + "num_tokens": 12767451.0, + "reward": 5.02734375, + "reward_std": 7.548166275024414, + "rewards/rm_reward_func/mean": 5.02734375, + "rewards/rm_reward_func/std": 21.371307373046875, + "step": 971 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 348.0625, + "completions/mean_terminated_length": 331.10345458984375, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.7776, + "grad_norm": 6.754936695098877, + "kl": 2.9931640625, + "learning_rate": 1e-06, + "loss": 0.0756, + "num_tokens": 12780813.0, + "reward": 0.3992156982421875, + "reward_std": 5.77485466003418, + "rewards/rm_reward_func/mean": 0.3992156982421875, + "rewards/rm_reward_func/std": 6.508362770080566, + "step": 972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 477.0, + "completions/mean_length": 350.78125, + "completions/mean_terminated_length": 345.58062744140625, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.7784, + "grad_norm": 6.786907196044922, + "kl": 1.45751953125, + "learning_rate": 1e-06, + "loss": -0.0457, + "num_tokens": 12797526.0, + "reward": 14.609710693359375, + "reward_std": 11.176668167114258, + "rewards/rm_reward_func/mean": 14.609710693359375, + "rewards/rm_reward_func/std": 18.939729690551758, + "step": 973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 307.34375, + "completions/mean_terminated_length": 250.0399932861328, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.7792, + "grad_norm": 9.214887619018555, + "kl": 2.23046875, + "learning_rate": 1e-06, + "loss": 0.0533, + "num_tokens": 12811017.0, + "reward": -0.3729248046875, + "reward_std": 4.81978702545166, + "rewards/rm_reward_func/mean": -0.3729248046875, + "rewards/rm_reward_func/std": 6.530143737792969, + "step": 974 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 307.28125, + "completions/mean_terminated_length": 293.63336181640625, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.78, + "grad_norm": 28.01193618774414, + "kl": 11.6064453125, + "learning_rate": 1e-06, + "loss": 0.4855, + "num_tokens": 12823162.0, + "reward": -6.5583343505859375, + "reward_std": 6.375615119934082, + "rewards/rm_reward_func/mean": -6.5583343505859375, + "rewards/rm_reward_func/std": 13.018743515014648, + "step": 975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 359.5, + "completions/mean_terminated_length": 331.2592468261719, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.7808, + "grad_norm": 16.22539710998535, + "kl": 6.41015625, + "learning_rate": 1e-06, + "loss": 0.1488, + "num_tokens": 12837546.0, + "reward": -6.0869140625, + "reward_std": 5.604630470275879, + "rewards/rm_reward_func/mean": -6.0869140625, + "rewards/rm_reward_func/std": 11.859865188598633, + "step": 976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 297.25, + "completions/mean_terminated_length": 237.1199951171875, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.7816, + "grad_norm": 43.490577697753906, + "kl": 7.90283203125, + "learning_rate": 1e-06, + "loss": 0.1253, + "num_tokens": 12850946.0, + "reward": -4.4287109375, + "reward_std": 7.8558669090271, + "rewards/rm_reward_func/mean": -4.4287109375, + "rewards/rm_reward_func/std": 18.278430938720703, + "step": 977 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 333.0, + "completions/max_terminated_length": 333.0, + "completions/mean_length": 167.125, + "completions/mean_terminated_length": 167.125, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.7824, + "grad_norm": 9.18833065032959, + "kl": 2.99072265625, + "learning_rate": 1e-06, + "loss": 0.2724, + "num_tokens": 12858686.0, + "reward": 2.391876220703125, + "reward_std": 3.7164864540100098, + "rewards/rm_reward_func/mean": 2.391876220703125, + "rewards/rm_reward_func/std": 9.332040786743164, + "step": 978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 340.0625, + "completions/mean_terminated_length": 334.51611328125, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "epoch": 0.7832, + "grad_norm": 6.65054988861084, + "kl": 0.5810546875, + "learning_rate": 1e-06, + "loss": -0.0442, + "num_tokens": 12872840.0, + "reward": 8.1917724609375, + "reward_std": 7.095458984375, + "rewards/rm_reward_func/mean": 8.1917724609375, + "rewards/rm_reward_func/std": 12.081109046936035, + "step": 979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 403.0, + "completions/mean_terminated_length": 382.8148193359375, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.784, + "grad_norm": 9.431859970092773, + "kl": 7.0546875, + "learning_rate": 1e-06, + "loss": 0.2173, + "num_tokens": 12888208.0, + "reward": 2.475616455078125, + "reward_std": 8.312936782836914, + "rewards/rm_reward_func/mean": 2.475616455078125, + "rewards/rm_reward_func/std": 13.530970573425293, + "step": 980 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 407.84375, + "completions/mean_terminated_length": 383.8077087402344, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.7848, + "grad_norm": 6.22176456451416, + "kl": 3.5087890625, + "learning_rate": 1e-06, + "loss": 0.0194, + "num_tokens": 12907155.0, + "reward": 4.936767578125, + "reward_std": 9.09925651550293, + "rewards/rm_reward_func/mean": 4.936767578125, + "rewards/rm_reward_func/std": 20.013263702392578, + "step": 981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 320.96875, + "completions/mean_terminated_length": 301.2069091796875, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.7856, + "grad_norm": 5.159428119659424, + "kl": 1.8134765625, + "learning_rate": 1e-06, + "loss": 0.0296, + "num_tokens": 12921042.0, + "reward": 9.160308837890625, + "reward_std": 9.23448371887207, + "rewards/rm_reward_func/mean": 9.160308837890625, + "rewards/rm_reward_func/std": 13.724321365356445, + "step": 982 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 426.0, + "completions/max_terminated_length": 426.0, + "completions/mean_length": 257.75, + "completions/mean_terminated_length": 257.75, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.7864, + "grad_norm": 10.37830924987793, + "kl": 1.292724609375, + "learning_rate": 1e-06, + "loss": -0.0107, + "num_tokens": 12932818.0, + "reward": 3.912353515625, + "reward_std": 6.3594069480896, + "rewards/rm_reward_func/mean": 3.912353515625, + "rewards/rm_reward_func/std": 14.007357597351074, + "step": 983 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 390.03125, + "completions/mean_terminated_length": 251.80001831054688, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.7872, + "grad_norm": 8.068547248840332, + "kl": 0.436279296875, + "learning_rate": 1e-06, + "loss": 0.0255, + "num_tokens": 12951635.0, + "reward": 4.229248046875, + "reward_std": 4.562774658203125, + "rewards/rm_reward_func/mean": 4.229248046875, + "rewards/rm_reward_func/std": 10.442963600158691, + "step": 984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 262.125, + "completions/mean_terminated_length": 204.4615478515625, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.788, + "grad_norm": 11.331951141357422, + "kl": 0.728759765625, + "learning_rate": 1e-06, + "loss": 0.0525, + "num_tokens": 12965559.0, + "reward": 3.13623046875, + "reward_std": 7.865469932556152, + "rewards/rm_reward_func/mean": 3.13623046875, + "rewards/rm_reward_func/std": 14.55949592590332, + "step": 985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 462.0, + "completions/mean_length": 294.71875, + "completions/mean_terminated_length": 287.70965576171875, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.7888, + "grad_norm": 12.841826438903809, + "kl": 0.88623046875, + "learning_rate": 1e-06, + "loss": 0.1087, + "num_tokens": 12977742.0, + "reward": 5.2115478515625, + "reward_std": 3.0502023696899414, + "rewards/rm_reward_func/mean": 5.2115478515625, + "rewards/rm_reward_func/std": 14.679587364196777, + "step": 986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 475.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 222.34375, + "completions/mean_terminated_length": 222.34375, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.7896, + "grad_norm": 6.139382839202881, + "kl": 1.00048828125, + "learning_rate": 1e-06, + "loss": 0.0219, + "num_tokens": 12987129.0, + "reward": 1.97021484375, + "reward_std": 4.622243881225586, + "rewards/rm_reward_func/mean": 1.97021484375, + "rewards/rm_reward_func/std": 9.068449020385742, + "step": 987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 506.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 268.15625, + "completions/mean_terminated_length": 268.15625, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.7904, + "grad_norm": 9.928857803344727, + "kl": 1.89697265625, + "learning_rate": 1e-06, + "loss": -0.1183, + "num_tokens": 12999926.0, + "reward": -0.8626174926757812, + "reward_std": 6.4827094078063965, + "rewards/rm_reward_func/mean": -0.8626174926757812, + "rewards/rm_reward_func/std": 8.545820236206055, + "step": 988 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 446.03125, + "completions/mean_terminated_length": 436.6071472167969, + "completions/min_length": 260.0, + "completions/min_terminated_length": 260.0, + "epoch": 0.7912, + "grad_norm": 8.559710502624512, + "kl": 2.8173828125, + "learning_rate": 1e-06, + "loss": 0.1298, + "num_tokens": 13017151.0, + "reward": -2.3729248046875, + "reward_std": 6.979788303375244, + "rewards/rm_reward_func/mean": -2.3729248046875, + "rewards/rm_reward_func/std": 10.80506706237793, + "step": 989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 488.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 360.53125, + "completions/mean_terminated_length": 360.53125, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "epoch": 0.792, + "grad_norm": 14.2783784866333, + "kl": 4.57421875, + "learning_rate": 1e-06, + "loss": 0.2056, + "num_tokens": 13031040.0, + "reward": 3.993865966796875, + "reward_std": 10.918571472167969, + "rewards/rm_reward_func/mean": 3.993865966796875, + "rewards/rm_reward_func/std": 11.101519584655762, + "step": 990 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 418.0, + "completions/mean_length": 253.21875, + "completions/mean_terminated_length": 205.29629516601562, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.7928, + "grad_norm": 12.58895492553711, + "kl": 4.3525390625, + "learning_rate": 1e-06, + "loss": 0.1975, + "num_tokens": 13041079.0, + "reward": -6.41998291015625, + "reward_std": 8.907421112060547, + "rewards/rm_reward_func/mean": -6.41998291015625, + "rewards/rm_reward_func/std": 10.190556526184082, + "step": 991 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 399.0, + "completions/max_terminated_length": 399.0, + "completions/mean_length": 209.8125, + "completions/mean_terminated_length": 209.8125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.7936, + "grad_norm": 8.097042083740234, + "kl": 4.4404296875, + "learning_rate": 1e-06, + "loss": 0.1805, + "num_tokens": 13053145.0, + "reward": 5.662109375, + "reward_std": 10.280261993408203, + "rewards/rm_reward_func/mean": 5.662109375, + "rewards/rm_reward_func/std": 12.116209030151367, + "step": 992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 193.875, + "completions/mean_terminated_length": 172.6666717529297, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.7944, + "grad_norm": 16.431880950927734, + "kl": 3.3447265625, + "learning_rate": 1e-06, + "loss": 0.1053, + "num_tokens": 13064117.0, + "reward": 2.54443359375, + "reward_std": 3.539630889892578, + "rewards/rm_reward_func/mean": 2.54443359375, + "rewards/rm_reward_func/std": 13.015660285949707, + "step": 993 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 325.3125, + "completions/mean_terminated_length": 240.45455932617188, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.7952, + "grad_norm": 5.3214592933654785, + "kl": 2.08642578125, + "learning_rate": 1e-06, + "loss": 0.1091, + "num_tokens": 13080759.0, + "reward": 5.818359375, + "reward_std": 6.74766731262207, + "rewards/rm_reward_func/mean": 5.818359375, + "rewards/rm_reward_func/std": 13.01656723022461, + "step": 994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 440.0, + "completions/mean_length": 398.0625, + "completions/mean_terminated_length": 360.0833435058594, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.796, + "grad_norm": 7.939972400665283, + "kl": 3.1787109375, + "learning_rate": 1e-06, + "loss": 0.0815, + "num_tokens": 13096265.0, + "reward": -3.901641845703125, + "reward_std": 7.492610931396484, + "rewards/rm_reward_func/mean": -3.901641845703125, + "rewards/rm_reward_func/std": 12.713387489318848, + "step": 995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 433.0, + "completions/max_terminated_length": 433.0, + "completions/mean_length": 216.65625, + "completions/mean_terminated_length": 216.65625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.7968, + "grad_norm": 17.098487854003906, + "kl": 3.54638671875, + "learning_rate": 1e-06, + "loss": 0.0526, + "num_tokens": 13109534.0, + "reward": -1.665863037109375, + "reward_std": 4.5015339851379395, + "rewards/rm_reward_func/mean": -1.665863037109375, + "rewards/rm_reward_func/std": 8.465290069580078, + "step": 996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 354.09375, + "completions/mean_terminated_length": 317.65386962890625, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.7976, + "grad_norm": 9.26326847076416, + "kl": 0.84375, + "learning_rate": 1e-06, + "loss": 0.0522, + "num_tokens": 13124065.0, + "reward": 9.808670043945312, + "reward_std": 6.167551040649414, + "rewards/rm_reward_func/mean": 9.808670043945312, + "rewards/rm_reward_func/std": 7.601739406585693, + "step": 997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 466.0, + "completions/mean_length": 373.6875, + "completions/mean_terminated_length": 364.4666748046875, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "epoch": 0.7984, + "grad_norm": 9.506736755371094, + "kl": 7.11279296875, + "learning_rate": 1e-06, + "loss": 0.3447, + "num_tokens": 13138111.0, + "reward": -5.72540283203125, + "reward_std": 7.122351169586182, + "rewards/rm_reward_func/mean": -5.72540283203125, + "rewards/rm_reward_func/std": 7.666444301605225, + "step": 998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 406.0, + "completions/mean_length": 242.625, + "completions/mean_terminated_length": 167.1999969482422, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.7992, + "grad_norm": 11.145411491394043, + "kl": 1.321533203125, + "learning_rate": 1e-06, + "loss": 0.0769, + "num_tokens": 13149155.0, + "reward": 4.50341796875, + "reward_std": 3.2994678020477295, + "rewards/rm_reward_func/mean": 4.50341796875, + "rewards/rm_reward_func/std": 8.315497398376465, + "step": 999 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 479.65625, + "completions/mean_terminated_length": 432.3846435546875, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "epoch": 0.8, + "grad_norm": 10.525664329528809, + "kl": 3.92578125, + "learning_rate": 1e-06, + "loss": 0.121, + "num_tokens": 13169528.0, + "reward": -4.1995849609375, + "reward_std": 10.804488182067871, + "rewards/rm_reward_func/mean": -4.1995849609375, + "rewards/rm_reward_func/std": 12.529898643493652, + "step": 1000 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 489.0, + "completions/mean_length": 345.59375, + "completions/mean_terminated_length": 269.9545593261719, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.8008, + "grad_norm": 8.35184383392334, + "kl": 2.693359375, + "learning_rate": 1e-06, + "loss": 0.0605, + "num_tokens": 13182515.0, + "reward": 0.7723388671875, + "reward_std": 5.575720310211182, + "rewards/rm_reward_func/mean": 0.7723388671875, + "rewards/rm_reward_func/std": 9.116813659667969, + "step": 1001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 395.625, + "completions/mean_terminated_length": 379.0000305175781, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "epoch": 0.8016, + "grad_norm": 9.205164909362793, + "kl": 3.8544921875, + "learning_rate": 1e-06, + "loss": 0.1071, + "num_tokens": 13197319.0, + "reward": 12.017333984375, + "reward_std": 12.300132751464844, + "rewards/rm_reward_func/mean": 12.017333984375, + "rewards/rm_reward_func/std": 18.185022354125977, + "step": 1002 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 391.875, + "completions/mean_terminated_length": 344.86956787109375, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.8024, + "grad_norm": 7.878791332244873, + "kl": 2.18896484375, + "learning_rate": 1e-06, + "loss": 0.0596, + "num_tokens": 13212147.0, + "reward": -5.649444580078125, + "reward_std": 5.377230167388916, + "rewards/rm_reward_func/mean": -5.649444580078125, + "rewards/rm_reward_func/std": 7.007201671600342, + "step": 1003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 304.0, + "completions/max_terminated_length": 304.0, + "completions/mean_length": 119.34375, + "completions/mean_terminated_length": 119.34375, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.8032, + "grad_norm": 6.185940742492676, + "kl": 1.81640625, + "learning_rate": 1e-06, + "loss": 0.068, + "num_tokens": 13220862.0, + "reward": 4.4765625, + "reward_std": 0.8225632309913635, + "rewards/rm_reward_func/mean": 4.4765625, + "rewards/rm_reward_func/std": 10.800799369812012, + "step": 1004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 359.25, + "completions/mean_terminated_length": 299.478271484375, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.804, + "grad_norm": 38.11021041870117, + "kl": 4.56787109375, + "learning_rate": 1e-06, + "loss": 0.0942, + "num_tokens": 13235742.0, + "reward": -5.1068115234375, + "reward_std": 9.743478775024414, + "rewards/rm_reward_func/mean": -5.1068115234375, + "rewards/rm_reward_func/std": 13.0623197555542, + "step": 1005 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 473.0, + "completions/mean_length": 213.75, + "completions/mean_terminated_length": 204.1290283203125, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.8048, + "grad_norm": 13.769661903381348, + "kl": 1.0166015625, + "learning_rate": 1e-06, + "loss": -0.0556, + "num_tokens": 13245302.0, + "reward": -6.537109375, + "reward_std": 4.325830459594727, + "rewards/rm_reward_func/mean": -6.537109375, + "rewards/rm_reward_func/std": 9.237016677856445, + "step": 1006 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 281.875, + "completions/mean_terminated_length": 266.5333557128906, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.8056, + "grad_norm": 13.841572761535645, + "kl": 5.92578125, + "learning_rate": 1e-06, + "loss": 0.1448, + "num_tokens": 13256546.0, + "reward": -1.900146484375, + "reward_std": 8.712725639343262, + "rewards/rm_reward_func/mean": -1.900146484375, + "rewards/rm_reward_func/std": 12.52529525756836, + "step": 1007 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 501.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 309.78125, + "completions/mean_terminated_length": 309.78125, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.8064, + "grad_norm": 21.372207641601562, + "kl": 4.873291015625, + "learning_rate": 1e-06, + "loss": 0.0049, + "num_tokens": 13268731.0, + "reward": 5.0888671875, + "reward_std": 5.007624626159668, + "rewards/rm_reward_func/mean": 5.0888671875, + "rewards/rm_reward_func/std": 22.30028533935547, + "step": 1008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 314.96875, + "completions/mean_terminated_length": 286.8214416503906, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.8072, + "grad_norm": 25.30167579650879, + "kl": 9.71826171875, + "learning_rate": 1e-06, + "loss": 0.4105, + "num_tokens": 13285794.0, + "reward": -8.27099609375, + "reward_std": 7.645841598510742, + "rewards/rm_reward_func/mean": -8.27099609375, + "rewards/rm_reward_func/std": 16.137731552124023, + "step": 1009 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 410.0, + "completions/mean_length": 273.875, + "completions/mean_terminated_length": 266.19354248046875, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.808, + "grad_norm": 8.614705085754395, + "kl": 0.37060546875, + "learning_rate": 1e-06, + "loss": -0.0065, + "num_tokens": 13299302.0, + "reward": 20.390045166015625, + "reward_std": 5.194117546081543, + "rewards/rm_reward_func/mean": 20.390045166015625, + "rewards/rm_reward_func/std": 17.19424057006836, + "step": 1010 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 342.6875, + "completions/mean_terminated_length": 303.6153869628906, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8088, + "grad_norm": 4.921807289123535, + "kl": 1.82275390625, + "learning_rate": 1e-06, + "loss": 0.0481, + "num_tokens": 13313140.0, + "reward": 4.9627838134765625, + "reward_std": 6.066807270050049, + "rewards/rm_reward_func/mean": 4.9627838134765625, + "rewards/rm_reward_func/std": 9.324835777282715, + "step": 1011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 455.0, + "completions/mean_length": 245.84375, + "completions/mean_terminated_length": 171.3199920654297, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.8096, + "grad_norm": 19.971662521362305, + "kl": 3.2412109375, + "learning_rate": 1e-06, + "loss": 0.0408, + "num_tokens": 13325503.0, + "reward": -3.34765625, + "reward_std": 4.555385589599609, + "rewards/rm_reward_func/mean": -3.34765625, + "rewards/rm_reward_func/std": 15.34835433959961, + "step": 1012 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 193.03125, + "completions/mean_terminated_length": 193.03125, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.8104, + "grad_norm": 11.546648025512695, + "kl": 1.3896484375, + "learning_rate": 1e-06, + "loss": 0.0518, + "num_tokens": 13334680.0, + "reward": -4.6224365234375, + "reward_std": 3.3871943950653076, + "rewards/rm_reward_func/mean": -4.6224365234375, + "rewards/rm_reward_func/std": 9.441699981689453, + "step": 1013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 431.0, + "completions/max_terminated_length": 431.0, + "completions/mean_length": 322.84375, + "completions/mean_terminated_length": 322.84375, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.8112, + "grad_norm": 6.9250359535217285, + "kl": 1.911376953125, + "learning_rate": 1e-06, + "loss": 0.0922, + "num_tokens": 13347131.0, + "reward": 4.8478851318359375, + "reward_std": 8.108898162841797, + "rewards/rm_reward_func/mean": 4.8478851318359375, + "rewards/rm_reward_func/std": 15.027045249938965, + "step": 1014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 232.625, + "completions/mean_terminated_length": 192.71429443359375, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.812, + "grad_norm": 35.626102447509766, + "kl": 0.970703125, + "learning_rate": 1e-06, + "loss": 0.0813, + "num_tokens": 13360111.0, + "reward": 0.5765380859375, + "reward_std": 6.022714614868164, + "rewards/rm_reward_func/mean": 0.5765380859375, + "rewards/rm_reward_func/std": 14.149896621704102, + "step": 1015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 380.0, + "completions/mean_length": 352.46875, + "completions/mean_terminated_length": 299.29168701171875, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "epoch": 0.8128, + "grad_norm": 9.04706859588623, + "kl": 0.98681640625, + "learning_rate": 1e-06, + "loss": 0.0161, + "num_tokens": 13373950.0, + "reward": -5.08154296875, + "reward_std": 3.3396553993225098, + "rewards/rm_reward_func/mean": -5.08154296875, + "rewards/rm_reward_func/std": 7.902137756347656, + "step": 1016 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 414.5, + "completions/mean_terminated_length": 392.0, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "epoch": 0.8136, + "grad_norm": 7.317805290222168, + "kl": 0.775390625, + "learning_rate": 1e-06, + "loss": -0.0302, + "num_tokens": 13391822.0, + "reward": 5.2294158935546875, + "reward_std": 6.866477012634277, + "rewards/rm_reward_func/mean": 5.2294158935546875, + "rewards/rm_reward_func/std": 10.840312957763672, + "step": 1017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 267.6875, + "completions/mean_terminated_length": 222.44444274902344, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.8144, + "grad_norm": 8.47047233581543, + "kl": 0.93603515625, + "learning_rate": 1e-06, + "loss": 0.0759, + "num_tokens": 13402404.0, + "reward": 0.318359375, + "reward_std": 2.553496837615967, + "rewards/rm_reward_func/mean": 0.318359375, + "rewards/rm_reward_func/std": 9.620682716369629, + "step": 1018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 509.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 298.75, + "completions/mean_terminated_length": 298.75, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.8152, + "grad_norm": 10.068121910095215, + "kl": 0.60400390625, + "learning_rate": 1e-06, + "loss": 0.0192, + "num_tokens": 13414804.0, + "reward": -0.441314697265625, + "reward_std": 2.4171152114868164, + "rewards/rm_reward_func/mean": -0.441314697265625, + "rewards/rm_reward_func/std": 6.881007671356201, + "step": 1019 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 498.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 343.5, + "completions/mean_terminated_length": 343.5, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.816, + "grad_norm": 10.040746688842773, + "kl": 3.17138671875, + "learning_rate": 1e-06, + "loss": 0.005, + "num_tokens": 13428028.0, + "reward": 8.162225723266602, + "reward_std": 8.07081413269043, + "rewards/rm_reward_func/mean": 8.162225723266602, + "rewards/rm_reward_func/std": 20.003002166748047, + "step": 1020 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 482.0, + "completions/mean_length": 391.78125, + "completions/mean_terminated_length": 351.7083435058594, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.8168, + "grad_norm": 10.024792671203613, + "kl": 2.970703125, + "learning_rate": 1e-06, + "loss": 0.0447, + "num_tokens": 13443005.0, + "reward": -0.19919586181640625, + "reward_std": 7.3775715827941895, + "rewards/rm_reward_func/mean": -0.19919586181640625, + "rewards/rm_reward_func/std": 14.476346015930176, + "step": 1021 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 340.09375, + "completions/mean_terminated_length": 334.5483703613281, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.8176, + "grad_norm": 7.288039684295654, + "kl": 0.75927734375, + "learning_rate": 1e-06, + "loss": 0.001, + "num_tokens": 13456536.0, + "reward": 9.702880859375, + "reward_std": 5.229021072387695, + "rewards/rm_reward_func/mean": 9.702880859375, + "rewards/rm_reward_func/std": 9.884917259216309, + "step": 1022 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 433.0, + "completions/max_terminated_length": 433.0, + "completions/mean_length": 216.71875, + "completions/mean_terminated_length": 216.71875, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.8184, + "grad_norm": 5.475179672241211, + "kl": 0.51953125, + "learning_rate": 1e-06, + "loss": 0.0019, + "num_tokens": 13466887.0, + "reward": 9.95068359375, + "reward_std": 3.0980048179626465, + "rewards/rm_reward_func/mean": 9.95068359375, + "rewards/rm_reward_func/std": 5.0042338371276855, + "step": 1023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 334.65625, + "completions/mean_terminated_length": 301.8148193359375, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.8192, + "grad_norm": 9.030261039733887, + "kl": 0.4716796875, + "learning_rate": 1e-06, + "loss": -0.0265, + "num_tokens": 13480036.0, + "reward": 2.835418701171875, + "reward_std": 5.5337724685668945, + "rewards/rm_reward_func/mean": 2.835418701171875, + "rewards/rm_reward_func/std": 8.441008567810059, + "step": 1024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 472.0, + "completions/mean_length": 232.0625, + "completions/mean_terminated_length": 223.03225708007812, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.82, + "grad_norm": 13.575851440429688, + "kl": 1.310546875, + "learning_rate": 1e-06, + "loss": 0.0396, + "num_tokens": 13491534.0, + "reward": 4.1461181640625, + "reward_std": 4.509701728820801, + "rewards/rm_reward_func/mean": 4.1461181640625, + "rewards/rm_reward_func/std": 10.166535377502441, + "step": 1025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 408.0, + "completions/max_terminated_length": 408.0, + "completions/mean_length": 250.875, + "completions/mean_terminated_length": 250.875, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.8208, + "grad_norm": 10.076627731323242, + "kl": 3.59375, + "learning_rate": 1e-06, + "loss": -0.0269, + "num_tokens": 13505082.0, + "reward": 2.3536376953125, + "reward_std": 10.482784271240234, + "rewards/rm_reward_func/mean": 2.3536376953125, + "rewards/rm_reward_func/std": 24.783512115478516, + "step": 1026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 411.0, + "completions/max_terminated_length": 411.0, + "completions/mean_length": 260.21875, + "completions/mean_terminated_length": 260.21875, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.8216, + "grad_norm": 27.619287490844727, + "kl": 2.212890625, + "learning_rate": 1e-06, + "loss": 0.0159, + "num_tokens": 13515673.0, + "reward": 0.1591796875, + "reward_std": 5.797090530395508, + "rewards/rm_reward_func/mean": 0.1591796875, + "rewards/rm_reward_func/std": 17.642372131347656, + "step": 1027 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 446.0, + "completions/mean_length": 341.03125, + "completions/mean_terminated_length": 316.6071472167969, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.8224, + "grad_norm": 8.583765029907227, + "kl": 2.4423828125, + "learning_rate": 1e-06, + "loss": 0.072, + "num_tokens": 13531434.0, + "reward": 7.218017578125, + "reward_std": 3.8263235092163086, + "rewards/rm_reward_func/mean": 7.218017578125, + "rewards/rm_reward_func/std": 22.30869483947754, + "step": 1028 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 301.0, + "completions/mean_terminated_length": 294.19354248046875, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.8232, + "grad_norm": 17.819374084472656, + "kl": 0.42431640625, + "learning_rate": 1e-06, + "loss": 0.0075, + "num_tokens": 13543170.0, + "reward": 13.765625, + "reward_std": 3.121476173400879, + "rewards/rm_reward_func/mean": 13.765625, + "rewards/rm_reward_func/std": 17.44659996032715, + "step": 1029 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 335.4375, + "completions/mean_terminated_length": 255.18182373046875, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.824, + "grad_norm": 12.179718971252441, + "kl": 1.9833984375, + "learning_rate": 1e-06, + "loss": 0.048, + "num_tokens": 13558328.0, + "reward": -1.099609375, + "reward_std": 2.019794464111328, + "rewards/rm_reward_func/mean": -1.099609375, + "rewards/rm_reward_func/std": 11.247794151306152, + "step": 1030 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 319.6875, + "completions/mean_terminated_length": 255.58334350585938, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.8248, + "grad_norm": 5.296995162963867, + "kl": 0.63330078125, + "learning_rate": 1e-06, + "loss": -0.0521, + "num_tokens": 13571750.0, + "reward": 8.6865234375, + "reward_std": 5.767685890197754, + "rewards/rm_reward_func/mean": 8.6865234375, + "rewards/rm_reward_func/std": 10.389171600341797, + "step": 1031 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 244.0625, + "completions/mean_terminated_length": 216.34483337402344, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.8256, + "grad_norm": 8.053586959838867, + "kl": 2.3212890625, + "learning_rate": 1e-06, + "loss": 0.034, + "num_tokens": 13581456.0, + "reward": -4.644775390625, + "reward_std": 6.1426496505737305, + "rewards/rm_reward_func/mean": -4.644775390625, + "rewards/rm_reward_func/std": 9.984515190124512, + "step": 1032 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 285.1875, + "completions/mean_terminated_length": 261.7241516113281, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.8264, + "grad_norm": 6.782697677612305, + "kl": 0.65087890625, + "learning_rate": 1e-06, + "loss": 0.0033, + "num_tokens": 13593006.0, + "reward": 0.996185302734375, + "reward_std": 2.4062700271606445, + "rewards/rm_reward_func/mean": 0.996185302734375, + "rewards/rm_reward_func/std": 5.100530624389648, + "step": 1033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 303.96875, + "completions/mean_terminated_length": 297.258056640625, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.8272, + "grad_norm": 11.949031829833984, + "kl": 3.359375, + "learning_rate": 1e-06, + "loss": 0.0168, + "num_tokens": 13604669.0, + "reward": -4.284423828125, + "reward_std": 13.229473114013672, + "rewards/rm_reward_func/mean": -4.284423828125, + "rewards/rm_reward_func/std": 17.82453727722168, + "step": 1034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 431.15625, + "completions/mean_terminated_length": 404.2083435058594, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "epoch": 0.828, + "grad_norm": 6.652060031890869, + "kl": 1.0, + "learning_rate": 1e-06, + "loss": 0.0325, + "num_tokens": 13620410.0, + "reward": 1.5401611328125, + "reward_std": 7.661131858825684, + "rewards/rm_reward_func/mean": 1.5401611328125, + "rewards/rm_reward_func/std": 9.722722053527832, + "step": 1035 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 262.53125, + "completions/mean_terminated_length": 226.8928680419922, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.8288, + "grad_norm": 12.548855781555176, + "kl": 3.18408203125, + "learning_rate": 1e-06, + "loss": 0.1386, + "num_tokens": 13632595.0, + "reward": 0.58056640625, + "reward_std": 2.6908631324768066, + "rewards/rm_reward_func/mean": 0.58056640625, + "rewards/rm_reward_func/std": 9.962352752685547, + "step": 1036 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 465.0, + "completions/mean_length": 333.3125, + "completions/mean_terminated_length": 327.5483703613281, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.8296, + "grad_norm": 7.802600383758545, + "kl": 3.310546875, + "learning_rate": 1e-06, + "loss": 0.081, + "num_tokens": 13648397.0, + "reward": -1.08026123046875, + "reward_std": 7.778307914733887, + "rewards/rm_reward_func/mean": -1.08026123046875, + "rewards/rm_reward_func/std": 9.398695945739746, + "step": 1037 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 468.0, + "completions/max_terminated_length": 468.0, + "completions/mean_length": 261.5, + "completions/mean_terminated_length": 261.5, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.8304, + "grad_norm": 13.804716110229492, + "kl": 1.47998046875, + "learning_rate": 1e-06, + "loss": 0.0421, + "num_tokens": 13659029.0, + "reward": -1.4334716796875, + "reward_std": 3.4379241466522217, + "rewards/rm_reward_func/mean": -1.4334716796875, + "rewards/rm_reward_func/std": 13.476388931274414, + "step": 1038 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 397.59375, + "completions/mean_terminated_length": 376.40740966796875, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.8312, + "grad_norm": 11.404579162597656, + "kl": 1.0322265625, + "learning_rate": 1e-06, + "loss": -0.0507, + "num_tokens": 13675640.0, + "reward": 4.508544921875, + "reward_std": 11.643041610717773, + "rewards/rm_reward_func/mean": 4.508544921875, + "rewards/rm_reward_func/std": 12.394889831542969, + "step": 1039 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 358.3125, + "completions/mean_terminated_length": 315.2799987792969, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.832, + "grad_norm": 7.375335216522217, + "kl": 1.384765625, + "learning_rate": 1e-06, + "loss": 0.0147, + "num_tokens": 13689202.0, + "reward": 0.5595703125, + "reward_std": 5.928117752075195, + "rewards/rm_reward_func/mean": 0.5595703125, + "rewards/rm_reward_func/std": 8.306135177612305, + "step": 1040 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 442.28125, + "completions/mean_terminated_length": 400.45001220703125, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "epoch": 0.8328, + "grad_norm": 103.06358337402344, + "kl": 5.14306640625, + "learning_rate": 1e-06, + "loss": 0.2078, + "num_tokens": 13706795.0, + "reward": -7.9290771484375, + "reward_std": 5.579919815063477, + "rewards/rm_reward_func/mean": -7.9290771484375, + "rewards/rm_reward_func/std": 9.49826717376709, + "step": 1041 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 460.0, + "completions/max_terminated_length": 460.0, + "completions/mean_length": 267.0625, + "completions/mean_terminated_length": 267.0625, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "epoch": 0.8336, + "grad_norm": 13.862227439880371, + "kl": 2.5947265625, + "learning_rate": 1e-06, + "loss": -0.0647, + "num_tokens": 13717565.0, + "reward": -8.9283447265625, + "reward_std": 2.9016993045806885, + "rewards/rm_reward_func/mean": -8.9283447265625, + "rewards/rm_reward_func/std": 7.622008800506592, + "step": 1042 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 388.78125, + "completions/mean_terminated_length": 376.03448486328125, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "epoch": 0.8344, + "grad_norm": 10.778160095214844, + "kl": 1.15380859375, + "learning_rate": 1e-06, + "loss": 0.0711, + "num_tokens": 13735566.0, + "reward": 7.612060546875, + "reward_std": 7.548920631408691, + "rewards/rm_reward_func/mean": 7.612060546875, + "rewards/rm_reward_func/std": 13.313557624816895, + "step": 1043 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 371.40625, + "completions/mean_terminated_length": 366.8709716796875, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.8352, + "grad_norm": 8.611907958984375, + "kl": 3.005126953125, + "learning_rate": 1e-06, + "loss": 0.0158, + "num_tokens": 13749723.0, + "reward": 3.6009521484375, + "reward_std": 10.527230262756348, + "rewards/rm_reward_func/mean": 3.6009521484375, + "rewards/rm_reward_func/std": 13.915834426879883, + "step": 1044 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 301.0, + "completions/mean_terminated_length": 279.17242431640625, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.836, + "grad_norm": 19.481334686279297, + "kl": 6.16796875, + "learning_rate": 1e-06, + "loss": 0.1613, + "num_tokens": 13765139.0, + "reward": -4.6524658203125, + "reward_std": 3.753931999206543, + "rewards/rm_reward_func/mean": -4.6524658203125, + "rewards/rm_reward_func/std": 11.036279678344727, + "step": 1045 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 272.84375, + "completions/mean_terminated_length": 228.55555725097656, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.8368, + "grad_norm": 18.560142517089844, + "kl": 7.0703125, + "learning_rate": 1e-06, + "loss": 0.2253, + "num_tokens": 13776190.0, + "reward": -9.101318359375, + "reward_std": 8.66612434387207, + "rewards/rm_reward_func/mean": -9.101318359375, + "rewards/rm_reward_func/std": 10.977357864379883, + "step": 1046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 369.15625, + "completions/mean_terminated_length": 329.1600036621094, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.8376, + "grad_norm": 14.926949501037598, + "kl": 0.888671875, + "learning_rate": 1e-06, + "loss": -0.0087, + "num_tokens": 13789795.0, + "reward": 0.6767578125, + "reward_std": 8.464103698730469, + "rewards/rm_reward_func/mean": 0.6767578125, + "rewards/rm_reward_func/std": 14.696283340454102, + "step": 1047 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 358.0, + "completions/mean_terminated_length": 336.0, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.8384, + "grad_norm": 14.153160095214844, + "kl": 3.96533203125, + "learning_rate": 1e-06, + "loss": 0.0663, + "num_tokens": 13805603.0, + "reward": -7.618194580078125, + "reward_std": 5.023479461669922, + "rewards/rm_reward_func/mean": -7.618194580078125, + "rewards/rm_reward_func/std": 13.050457000732422, + "step": 1048 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 377.9375, + "completions/mean_terminated_length": 369.0000305175781, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.8392, + "grad_norm": 9.757351875305176, + "kl": 2.5224609375, + "learning_rate": 1e-06, + "loss": 0.0443, + "num_tokens": 13822233.0, + "reward": 3.964599609375, + "reward_std": 10.471548080444336, + "rewards/rm_reward_func/mean": 3.964599609375, + "rewards/rm_reward_func/std": 16.459672927856445, + "step": 1049 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 310.1875, + "completions/mean_terminated_length": 281.3571472167969, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.84, + "grad_norm": 12.101102828979492, + "kl": 3.15673828125, + "learning_rate": 1e-06, + "loss": -0.0473, + "num_tokens": 13833951.0, + "reward": -2.974365234375, + "reward_std": 9.494062423706055, + "rewards/rm_reward_func/mean": -2.974365234375, + "rewards/rm_reward_func/std": 17.267580032348633, + "step": 1050 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 234.0625, + "completions/mean_terminated_length": 194.35714721679688, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.8408, + "grad_norm": 6.150353908538818, + "kl": 2.26953125, + "learning_rate": 1e-06, + "loss": 0.0698, + "num_tokens": 13844073.0, + "reward": -1.613037109375, + "reward_std": 4.632699489593506, + "rewards/rm_reward_func/mean": -1.613037109375, + "rewards/rm_reward_func/std": 8.01832103729248, + "step": 1051 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 359.6875, + "completions/mean_terminated_length": 317.03997802734375, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.8416, + "grad_norm": 9.147581100463867, + "kl": 3.4599609375, + "learning_rate": 1e-06, + "loss": 0.1188, + "num_tokens": 13858791.0, + "reward": -1.190673828125, + "reward_std": 5.757209300994873, + "rewards/rm_reward_func/mean": -1.190673828125, + "rewards/rm_reward_func/std": 9.849881172180176, + "step": 1052 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 333.71875, + "completions/mean_terminated_length": 263.9565124511719, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.8424, + "grad_norm": 12.606836318969727, + "kl": 3.59375, + "learning_rate": 1e-06, + "loss": 0.1426, + "num_tokens": 13873742.0, + "reward": -8.640029907226562, + "reward_std": 7.198362350463867, + "rewards/rm_reward_func/mean": -8.640029907226562, + "rewards/rm_reward_func/std": 10.380491256713867, + "step": 1053 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 469.0, + "completions/mean_length": 230.84375, + "completions/mean_terminated_length": 221.77418518066406, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8432, + "grad_norm": 6.97705078125, + "kl": 2.47314453125, + "learning_rate": 1e-06, + "loss": 0.173, + "num_tokens": 13883497.0, + "reward": -2.553131103515625, + "reward_std": 3.148016929626465, + "rewards/rm_reward_func/mean": -2.553131103515625, + "rewards/rm_reward_func/std": 8.437515258789062, + "step": 1054 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 325.25, + "completions/mean_terminated_length": 213.1999969482422, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.844, + "grad_norm": 7.593416213989258, + "kl": 0.8896484375, + "learning_rate": 1e-06, + "loss": 0.0335, + "num_tokens": 13897529.0, + "reward": 6.3282470703125, + "reward_std": 3.7554547786712646, + "rewards/rm_reward_func/mean": 6.3282470703125, + "rewards/rm_reward_func/std": 13.18265438079834, + "step": 1055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 338.25, + "completions/mean_terminated_length": 289.6000061035156, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8448, + "grad_norm": 7.9972734451293945, + "kl": 1.67822265625, + "learning_rate": 1e-06, + "loss": 0.0365, + "num_tokens": 13910961.0, + "reward": 9.41973876953125, + "reward_std": 5.808597564697266, + "rewards/rm_reward_func/mean": 9.41973876953125, + "rewards/rm_reward_func/std": 13.108800888061523, + "step": 1056 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 378.15625, + "completions/mean_terminated_length": 317.31817626953125, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.8456, + "grad_norm": 9.818436622619629, + "kl": 2.041015625, + "learning_rate": 1e-06, + "loss": 0.0809, + "num_tokens": 13925710.0, + "reward": 4.3310546875, + "reward_std": 6.725727081298828, + "rewards/rm_reward_func/mean": 4.3310546875, + "rewards/rm_reward_func/std": 8.763275146484375, + "step": 1057 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 465.0, + "completions/mean_length": 377.5, + "completions/mean_terminated_length": 352.59259033203125, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.8464, + "grad_norm": 11.298351287841797, + "kl": 0.86474609375, + "learning_rate": 1e-06, + "loss": 0.1117, + "num_tokens": 13939478.0, + "reward": 4.32666015625, + "reward_std": 8.901585578918457, + "rewards/rm_reward_func/mean": 4.32666015625, + "rewards/rm_reward_func/std": 16.733640670776367, + "step": 1058 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 472.0, + "completions/mean_length": 346.40625, + "completions/mean_terminated_length": 300.0400085449219, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.8472, + "grad_norm": 7.649352550506592, + "kl": 2.932861328125, + "learning_rate": 1e-06, + "loss": -0.004, + "num_tokens": 13955987.0, + "reward": 10.8458251953125, + "reward_std": 10.861635208129883, + "rewards/rm_reward_func/mean": 10.8458251953125, + "rewards/rm_reward_func/std": 20.45244598388672, + "step": 1059 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 251.59375, + "completions/mean_terminated_length": 203.37037658691406, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.848, + "grad_norm": 14.20515251159668, + "kl": 1.2626953125, + "learning_rate": 1e-06, + "loss": -0.1202, + "num_tokens": 13967622.0, + "reward": 3.55889892578125, + "reward_std": 3.865619659423828, + "rewards/rm_reward_func/mean": 3.55889892578125, + "rewards/rm_reward_func/std": 14.522747039794922, + "step": 1060 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 447.0, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 204.1875, + "completions/mean_terminated_length": 204.1875, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8488, + "grad_norm": 5.686960220336914, + "kl": 0.53759765625, + "learning_rate": 1e-06, + "loss": -0.0044, + "num_tokens": 13976516.0, + "reward": 10.380859375, + "reward_std": 1.672519326210022, + "rewards/rm_reward_func/mean": 10.380859375, + "rewards/rm_reward_func/std": 4.958350658416748, + "step": 1061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 455.0, + "completions/max_terminated_length": 455.0, + "completions/mean_length": 294.875, + "completions/mean_terminated_length": 294.875, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.8496, + "grad_norm": 6.868638038635254, + "kl": 0.47802734375, + "learning_rate": 1e-06, + "loss": -0.0463, + "num_tokens": 13987656.0, + "reward": -3.4047164916992188, + "reward_std": 2.7297439575195312, + "rewards/rm_reward_func/mean": -3.4047164916992188, + "rewards/rm_reward_func/std": 8.868703842163086, + "step": 1062 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 484.0, + "completions/mean_length": 354.15625, + "completions/mean_terminated_length": 309.9599914550781, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.8504, + "grad_norm": 10.120816230773926, + "kl": 0.4521484375, + "learning_rate": 1e-06, + "loss": 0.0789, + "num_tokens": 14001477.0, + "reward": 10.3330078125, + "reward_std": 5.511585235595703, + "rewards/rm_reward_func/mean": 10.3330078125, + "rewards/rm_reward_func/std": 11.582442283630371, + "step": 1063 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 383.15625, + "completions/mean_terminated_length": 347.0799865722656, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8512, + "grad_norm": 8.560920715332031, + "kl": 1.580078125, + "learning_rate": 1e-06, + "loss": 0.1501, + "num_tokens": 14016578.0, + "reward": 15.5166015625, + "reward_std": 7.954902648925781, + "rewards/rm_reward_func/mean": 15.5166015625, + "rewards/rm_reward_func/std": 15.997610092163086, + "step": 1064 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 219.71875, + "completions/mean_terminated_length": 177.96429443359375, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.852, + "grad_norm": 9.301816940307617, + "kl": 0.8408203125, + "learning_rate": 1e-06, + "loss": 0.0325, + "num_tokens": 14026161.0, + "reward": 0.931640625, + "reward_std": 2.5424585342407227, + "rewards/rm_reward_func/mean": 0.931640625, + "rewards/rm_reward_func/std": 9.632967948913574, + "step": 1065 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 511.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 292.34375, + "completions/mean_terminated_length": 292.34375, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8528, + "grad_norm": 12.27116584777832, + "kl": 1.8857421875, + "learning_rate": 1e-06, + "loss": -0.0008, + "num_tokens": 14038460.0, + "reward": 5.19140625, + "reward_std": 2.821535110473633, + "rewards/rm_reward_func/mean": 5.19140625, + "rewards/rm_reward_func/std": 11.627610206604004, + "step": 1066 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 349.5625, + "completions/mean_terminated_length": 344.32257080078125, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.8536, + "grad_norm": 11.470945358276367, + "kl": 0.71337890625, + "learning_rate": 1e-06, + "loss": -0.0169, + "num_tokens": 14051606.0, + "reward": 3.7967529296875, + "reward_std": 6.249090194702148, + "rewards/rm_reward_func/mean": 3.7967529296875, + "rewards/rm_reward_func/std": 6.5110979080200195, + "step": 1067 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 312.90625, + "completions/mean_terminated_length": 299.63336181640625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8544, + "grad_norm": 10.407036781311035, + "kl": 0.72705078125, + "learning_rate": 1e-06, + "loss": 0.0446, + "num_tokens": 14063891.0, + "reward": 16.3878173828125, + "reward_std": 8.335765838623047, + "rewards/rm_reward_func/mean": 16.3878173828125, + "rewards/rm_reward_func/std": 16.84733009338379, + "step": 1068 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 281.96875, + "completions/mean_terminated_length": 266.63336181640625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8552, + "grad_norm": 7.953994274139404, + "kl": 0.6884765625, + "learning_rate": 1e-06, + "loss": 0.0595, + "num_tokens": 14074970.0, + "reward": 3.028076171875, + "reward_std": 4.655251979827881, + "rewards/rm_reward_func/mean": 3.028076171875, + "rewards/rm_reward_func/std": 14.149062156677246, + "step": 1069 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 300.375, + "completions/mean_terminated_length": 261.1851806640625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.856, + "grad_norm": 5.069062232971191, + "kl": 0.404541015625, + "learning_rate": 1e-06, + "loss": 0.0521, + "num_tokens": 14088094.0, + "reward": 1.788726806640625, + "reward_std": 5.874140739440918, + "rewards/rm_reward_func/mean": 1.788726806640625, + "rewards/rm_reward_func/std": 10.132195472717285, + "step": 1070 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 198.875, + "completions/mean_terminated_length": 188.77418518066406, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.8568, + "grad_norm": 11.527831077575684, + "kl": 1.47314453125, + "learning_rate": 1e-06, + "loss": 0.1007, + "num_tokens": 14097274.0, + "reward": -1.9715576171875, + "reward_std": 2.8827223777770996, + "rewards/rm_reward_func/mean": -1.9715576171875, + "rewards/rm_reward_func/std": 5.209477424621582, + "step": 1071 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 449.09375, + "completions/mean_terminated_length": 411.3500061035156, + "completions/min_length": 336.0, + "completions/min_terminated_length": 336.0, + "epoch": 0.8576, + "grad_norm": 6.973728656768799, + "kl": 0.34130859375, + "learning_rate": 1e-06, + "loss": 0.012, + "num_tokens": 14115125.0, + "reward": 8.21722412109375, + "reward_std": 5.218785285949707, + "rewards/rm_reward_func/mean": 8.21722412109375, + "rewards/rm_reward_func/std": 12.76237964630127, + "step": 1072 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 368.53125, + "completions/mean_terminated_length": 312.39129638671875, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8584, + "grad_norm": 6.441647052764893, + "kl": 0.60107421875, + "learning_rate": 1e-06, + "loss": 0.0165, + "num_tokens": 14129646.0, + "reward": 8.17431640625, + "reward_std": 4.927266597747803, + "rewards/rm_reward_func/mean": 8.17431640625, + "rewards/rm_reward_func/std": 10.222173690795898, + "step": 1073 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 328.34375, + "completions/mean_terminated_length": 267.125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8592, + "grad_norm": 5.157811164855957, + "kl": 1.52880859375, + "learning_rate": 1e-06, + "loss": 0.0403, + "num_tokens": 14142593.0, + "reward": 3.81683349609375, + "reward_std": 3.48171329498291, + "rewards/rm_reward_func/mean": 3.81683349609375, + "rewards/rm_reward_func/std": 9.174951553344727, + "step": 1074 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 347.3125, + "completions/mean_terminated_length": 272.4545593261719, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.86, + "grad_norm": 5.949108600616455, + "kl": 0.45263671875, + "learning_rate": 1e-06, + "loss": 0.0243, + "num_tokens": 14155811.0, + "reward": 14.94580078125, + "reward_std": 4.586613655090332, + "rewards/rm_reward_func/mean": 14.94580078125, + "rewards/rm_reward_func/std": 6.567379951477051, + "step": 1075 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 350.25, + "completions/mean_terminated_length": 312.923095703125, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.8608, + "grad_norm": 7.930327415466309, + "kl": 0.419921875, + "learning_rate": 1e-06, + "loss": -0.0041, + "num_tokens": 14168835.0, + "reward": 2.20947265625, + "reward_std": 6.557795524597168, + "rewards/rm_reward_func/mean": 2.20947265625, + "rewards/rm_reward_func/std": 8.4446382522583, + "step": 1076 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 386.21875, + "completions/mean_terminated_length": 344.29168701171875, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "epoch": 0.8616, + "grad_norm": 8.628700256347656, + "kl": 0.5888671875, + "learning_rate": 1e-06, + "loss": -0.0536, + "num_tokens": 14183226.0, + "reward": 10.73797607421875, + "reward_std": 5.025568008422852, + "rewards/rm_reward_func/mean": 10.73797607421875, + "rewards/rm_reward_func/std": 8.780284881591797, + "step": 1077 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 453.21875, + "completions/mean_terminated_length": 426.5, + "completions/min_length": 354.0, + "completions/min_terminated_length": 354.0, + "epoch": 0.8624, + "grad_norm": 6.511486053466797, + "kl": 0.40283203125, + "learning_rate": 1e-06, + "loss": 0.033, + "num_tokens": 14200257.0, + "reward": 15.1558837890625, + "reward_std": 4.903788089752197, + "rewards/rm_reward_func/mean": 15.1558837890625, + "rewards/rm_reward_func/std": 12.219951629638672, + "step": 1078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 311.78125, + "completions/mean_terminated_length": 305.32257080078125, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.8632, + "grad_norm": 9.137861251831055, + "kl": 1.89208984375, + "learning_rate": 1e-06, + "loss": 0.0506, + "num_tokens": 14212794.0, + "reward": 3.208251953125, + "reward_std": 6.694586753845215, + "rewards/rm_reward_func/mean": 3.208251953125, + "rewards/rm_reward_func/std": 14.088113784790039, + "step": 1079 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 248.1875, + "completions/mean_terminated_length": 230.60000610351562, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.864, + "grad_norm": 8.597902297973633, + "kl": 3.4228515625, + "learning_rate": 1e-06, + "loss": 0.1484, + "num_tokens": 14223552.0, + "reward": -1.8214111328125, + "reward_std": 4.2023773193359375, + "rewards/rm_reward_func/mean": -1.8214111328125, + "rewards/rm_reward_func/std": 9.527159690856934, + "step": 1080 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 369.84375, + "completions/mean_terminated_length": 272.5789489746094, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8648, + "grad_norm": 5.029150485992432, + "kl": 0.5234375, + "learning_rate": 1e-06, + "loss": 0.0322, + "num_tokens": 14238843.0, + "reward": -7.0312652587890625, + "reward_std": 2.070448637008667, + "rewards/rm_reward_func/mean": -7.0312652587890625, + "rewards/rm_reward_func/std": 11.313837051391602, + "step": 1081 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 444.0, + "completions/max_terminated_length": 444.0, + "completions/mean_length": 282.59375, + "completions/mean_terminated_length": 282.59375, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.8656, + "grad_norm": 16.06825828552246, + "kl": 5.65869140625, + "learning_rate": 1e-06, + "loss": 0.2783, + "num_tokens": 14251942.0, + "reward": 3.9051666259765625, + "reward_std": 4.887936592102051, + "rewards/rm_reward_func/mean": 3.9051666259765625, + "rewards/rm_reward_func/std": 14.745894432067871, + "step": 1082 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 458.0, + "completions/mean_length": 306.90625, + "completions/mean_terminated_length": 300.2903137207031, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.8664, + "grad_norm": 7.322081089019775, + "kl": 3.85546875, + "learning_rate": 1e-06, + "loss": 0.1572, + "num_tokens": 14267451.0, + "reward": 8.35272216796875, + "reward_std": 9.634117126464844, + "rewards/rm_reward_func/mean": 8.35272216796875, + "rewards/rm_reward_func/std": 19.10032844543457, + "step": 1083 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 345.5, + "completions/mean_terminated_length": 340.1290283203125, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.8672, + "grad_norm": 12.525562286376953, + "kl": 5.53955078125, + "learning_rate": 1e-06, + "loss": 0.4021, + "num_tokens": 14285251.0, + "reward": 14.5098876953125, + "reward_std": 6.451245307922363, + "rewards/rm_reward_func/mean": 14.5098876953125, + "rewards/rm_reward_func/std": 12.280783653259277, + "step": 1084 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 227.1875, + "completions/mean_terminated_length": 197.72413635253906, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.868, + "grad_norm": 12.410992622375488, + "kl": 6.01171875, + "learning_rate": 1e-06, + "loss": 0.2138, + "num_tokens": 14295113.0, + "reward": 2.096435546875, + "reward_std": 6.907596588134766, + "rewards/rm_reward_func/mean": 2.096435546875, + "rewards/rm_reward_func/std": 11.79339599609375, + "step": 1085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 263.28125, + "completions/mean_terminated_length": 246.70001220703125, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.8688, + "grad_norm": 7.831367015838623, + "kl": 0.89599609375, + "learning_rate": 1e-06, + "loss": -0.026, + "num_tokens": 14308474.0, + "reward": 2.3360595703125, + "reward_std": 6.861347675323486, + "rewards/rm_reward_func/mean": 2.3360595703125, + "rewards/rm_reward_func/std": 9.401144027709961, + "step": 1086 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 246.40625, + "completions/mean_terminated_length": 237.8386993408203, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.8696, + "grad_norm": 12.294774055480957, + "kl": 3.60009765625, + "learning_rate": 1e-06, + "loss": 0.0805, + "num_tokens": 14319327.0, + "reward": 1.0987548828125, + "reward_std": 5.113190650939941, + "rewards/rm_reward_func/mean": 1.0987548828125, + "rewards/rm_reward_func/std": 7.818641185760498, + "step": 1087 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 269.0, + "completions/mean_terminated_length": 261.1612854003906, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.8704, + "grad_norm": 13.56505298614502, + "kl": 4.18359375, + "learning_rate": 1e-06, + "loss": 0.114, + "num_tokens": 14331231.0, + "reward": 1.3173828125, + "reward_std": 7.998728275299072, + "rewards/rm_reward_func/mean": 1.3173828125, + "rewards/rm_reward_func/std": 18.137277603149414, + "step": 1088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 472.0, + "completions/mean_length": 299.0625, + "completions/mean_terminated_length": 249.92308044433594, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8712, + "grad_norm": 5.182938575744629, + "kl": 1.154296875, + "learning_rate": 1e-06, + "loss": 0.0724, + "num_tokens": 14343945.0, + "reward": 9.21612548828125, + "reward_std": 8.317728042602539, + "rewards/rm_reward_func/mean": 9.21612548828125, + "rewards/rm_reward_func/std": 16.426794052124023, + "step": 1089 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 403.0, + "completions/max_terminated_length": 403.0, + "completions/mean_length": 185.0, + "completions/mean_terminated_length": 185.0, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "epoch": 0.872, + "grad_norm": 68.98828125, + "kl": 4.005859375, + "learning_rate": 1e-06, + "loss": 0.2527, + "num_tokens": 14354649.0, + "reward": -9.63653564453125, + "reward_std": 3.627918243408203, + "rewards/rm_reward_func/mean": -9.63653564453125, + "rewards/rm_reward_func/std": 8.9398775100708, + "step": 1090 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 488.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 313.59375, + "completions/mean_terminated_length": 313.59375, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.8728, + "grad_norm": 13.77872085571289, + "kl": 5.103515625, + "learning_rate": 1e-06, + "loss": 0.2633, + "num_tokens": 14367212.0, + "reward": 0.75341796875, + "reward_std": 5.394412040710449, + "rewards/rm_reward_func/mean": 0.75341796875, + "rewards/rm_reward_func/std": 13.423392295837402, + "step": 1091 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 239.0, + "completions/mean_length": 228.75, + "completions/mean_terminated_length": 134.33334350585938, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8736, + "grad_norm": 11.227128028869629, + "kl": 1.6025390625, + "learning_rate": 1e-06, + "loss": 0.1076, + "num_tokens": 14378772.0, + "reward": -0.025146484375, + "reward_std": 2.094247341156006, + "rewards/rm_reward_func/mean": -0.025146484375, + "rewards/rm_reward_func/std": 8.626964569091797, + "step": 1092 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 347.5, + "completions/mean_terminated_length": 283.13043212890625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8744, + "grad_norm": 6.2191619873046875, + "kl": 3.33203125, + "learning_rate": 1e-06, + "loss": 0.1357, + "num_tokens": 14394876.0, + "reward": 16.056884765625, + "reward_std": 9.725163459777832, + "rewards/rm_reward_func/mean": 16.056884765625, + "rewards/rm_reward_func/std": 21.10055923461914, + "step": 1093 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 407.0, + "completions/max_terminated_length": 407.0, + "completions/mean_length": 238.3125, + "completions/mean_terminated_length": 238.3125, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.8752, + "grad_norm": 9.5230073928833, + "kl": 3.8193359375, + "learning_rate": 1e-06, + "loss": 0.0945, + "num_tokens": 14404414.0, + "reward": -4.79345703125, + "reward_std": 5.990765571594238, + "rewards/rm_reward_func/mean": -4.79345703125, + "rewards/rm_reward_func/std": 13.47055721282959, + "step": 1094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 236.90625, + "completions/mean_terminated_length": 185.9629669189453, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.876, + "grad_norm": 7.131078243255615, + "kl": 1.12109375, + "learning_rate": 1e-06, + "loss": 0.0133, + "num_tokens": 14415475.0, + "reward": 0.505615234375, + "reward_std": 3.531787395477295, + "rewards/rm_reward_func/mean": 0.505615234375, + "rewards/rm_reward_func/std": 6.195098876953125, + "step": 1095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 386.0, + "completions/max_terminated_length": 386.0, + "completions/mean_length": 217.75, + "completions/mean_terminated_length": 217.75, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8768, + "grad_norm": 7.909564971923828, + "kl": 1.53125, + "learning_rate": 1e-06, + "loss": 0.0247, + "num_tokens": 14425171.0, + "reward": -1.14208984375, + "reward_std": 1.705171823501587, + "rewards/rm_reward_func/mean": -1.14208984375, + "rewards/rm_reward_func/std": 10.815631866455078, + "step": 1096 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 265.15625, + "completions/mean_terminated_length": 182.875, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.8776, + "grad_norm": 4.670645236968994, + "kl": 2.76171875, + "learning_rate": 1e-06, + "loss": 0.1654, + "num_tokens": 14438568.0, + "reward": 10.848876953125, + "reward_std": 4.882187366485596, + "rewards/rm_reward_func/mean": 10.848876953125, + "rewards/rm_reward_func/std": 13.137846946716309, + "step": 1097 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 390.0, + "completions/mean_length": 317.75, + "completions/mean_terminated_length": 253.0, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.8784, + "grad_norm": 11.463107109069824, + "kl": 3.96484375, + "learning_rate": 1e-06, + "loss": 0.1702, + "num_tokens": 14451368.0, + "reward": 4.6419677734375, + "reward_std": 3.999204397201538, + "rewards/rm_reward_func/mean": 4.6419677734375, + "rewards/rm_reward_func/std": 16.210906982421875, + "step": 1098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 488.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 302.15625, + "completions/mean_terminated_length": 302.15625, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.8792, + "grad_norm": 21.74379539489746, + "kl": 7.875, + "learning_rate": 1e-06, + "loss": 0.225, + "num_tokens": 14463477.0, + "reward": -5.63720703125, + "reward_std": 9.232072830200195, + "rewards/rm_reward_func/mean": -5.63720703125, + "rewards/rm_reward_func/std": 18.789886474609375, + "step": 1099 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 299.25, + "completions/mean_terminated_length": 259.85186767578125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.88, + "grad_norm": 6.47990608215332, + "kl": 5.8642578125, + "learning_rate": 1e-06, + "loss": 0.1774, + "num_tokens": 14476309.0, + "reward": -0.43701171875, + "reward_std": 10.200658798217773, + "rewards/rm_reward_func/mean": -0.43701171875, + "rewards/rm_reward_func/std": 14.860036849975586, + "step": 1100 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 499.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 105.0, + "completions/mean_terminated_length": 105.0, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.8808, + "grad_norm": 5.006459712982178, + "kl": 1.625, + "learning_rate": 1e-06, + "loss": -0.0308, + "num_tokens": 14483213.0, + "reward": 1.8359375, + "reward_std": 0.8900178074836731, + "rewards/rm_reward_func/mean": 1.8359375, + "rewards/rm_reward_func/std": 11.928114891052246, + "step": 1101 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 280.28125, + "completions/mean_terminated_length": 264.8333435058594, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8816, + "grad_norm": 8.10487174987793, + "kl": 2.853515625, + "learning_rate": 1e-06, + "loss": 0.0011, + "num_tokens": 14495190.0, + "reward": 5.513671875, + "reward_std": 8.272500991821289, + "rewards/rm_reward_func/mean": 5.513671875, + "rewards/rm_reward_func/std": 16.234622955322266, + "step": 1102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 239.5, + "completions/mean_terminated_length": 230.7096710205078, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8824, + "grad_norm": 3.8931567668914795, + "kl": 0.5224609375, + "learning_rate": 1e-06, + "loss": 0.0255, + "num_tokens": 14505822.0, + "reward": 11.3662109375, + "reward_std": 2.4082765579223633, + "rewards/rm_reward_func/mean": 11.3662109375, + "rewards/rm_reward_func/std": 5.015256404876709, + "step": 1103 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 237.28125, + "completions/mean_terminated_length": 198.0357208251953, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8832, + "grad_norm": 10.963964462280273, + "kl": 4.1640625, + "learning_rate": 1e-06, + "loss": 0.1059, + "num_tokens": 14518447.0, + "reward": 1.65234375, + "reward_std": 3.4185004234313965, + "rewards/rm_reward_func/mean": 1.65234375, + "rewards/rm_reward_func/std": 13.96972370147705, + "step": 1104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 469.0, + "completions/max_terminated_length": 469.0, + "completions/mean_length": 252.53125, + "completions/mean_terminated_length": 252.53125, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.884, + "grad_norm": 7.281085014343262, + "kl": 4.080078125, + "learning_rate": 1e-06, + "loss": 0.212, + "num_tokens": 14529264.0, + "reward": -5.43353271484375, + "reward_std": 5.849520206451416, + "rewards/rm_reward_func/mean": -5.43353271484375, + "rewards/rm_reward_func/std": 9.325867652893066, + "step": 1105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 492.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 341.4375, + "completions/mean_terminated_length": 341.4375, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "epoch": 0.8848, + "grad_norm": 7.2115478515625, + "kl": 0.4150390625, + "learning_rate": 1e-06, + "loss": 0.0024, + "num_tokens": 14543294.0, + "reward": 7.579345703125, + "reward_std": 4.300886154174805, + "rewards/rm_reward_func/mean": 7.579345703125, + "rewards/rm_reward_func/std": 5.634270191192627, + "step": 1106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 450.0, + "completions/max_terminated_length": 450.0, + "completions/mean_length": 281.21875, + "completions/mean_terminated_length": 281.21875, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8856, + "grad_norm": 11.654593467712402, + "kl": 1.85693359375, + "learning_rate": 1e-06, + "loss": 0.0569, + "num_tokens": 14554965.0, + "reward": 2.6845703125, + "reward_std": 5.363699436187744, + "rewards/rm_reward_func/mean": 2.6845703125, + "rewards/rm_reward_func/std": 11.830452919006348, + "step": 1107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 509.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 215.78125, + "completions/mean_terminated_length": 215.78125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8864, + "grad_norm": 7.386474132537842, + "kl": 0.47412109375, + "learning_rate": 1e-06, + "loss": -0.0309, + "num_tokens": 14564774.0, + "reward": 16.75201416015625, + "reward_std": 2.713106155395508, + "rewards/rm_reward_func/mean": 16.75201416015625, + "rewards/rm_reward_func/std": 14.134072303771973, + "step": 1108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 278.9375, + "completions/mean_terminated_length": 245.6428680419922, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8872, + "grad_norm": 8.896984100341797, + "kl": 0.71337890625, + "learning_rate": 1e-06, + "loss": -0.012, + "num_tokens": 14576156.0, + "reward": 3.846038818359375, + "reward_std": 4.625146389007568, + "rewards/rm_reward_func/mean": 3.846038818359375, + "rewards/rm_reward_func/std": 7.2643961906433105, + "step": 1109 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 401.84375, + "completions/mean_terminated_length": 386.1071472167969, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "epoch": 0.888, + "grad_norm": 9.240114212036133, + "kl": 1.05859375, + "learning_rate": 1e-06, + "loss": 0.0088, + "num_tokens": 14591575.0, + "reward": 8.68145751953125, + "reward_std": 6.467904567718506, + "rewards/rm_reward_func/mean": 8.68145751953125, + "rewards/rm_reward_func/std": 9.515111923217773, + "step": 1110 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 397.84375, + "completions/mean_terminated_length": 345.9545593261719, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "epoch": 0.8888, + "grad_norm": 6.770890235900879, + "kl": 2.41015625, + "learning_rate": 1e-06, + "loss": 0.0554, + "num_tokens": 14606562.0, + "reward": -2.921142578125, + "reward_std": 5.699788570404053, + "rewards/rm_reward_func/mean": -2.921142578125, + "rewards/rm_reward_func/std": 17.231367111206055, + "step": 1111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 480.0, + "completions/mean_length": 187.0, + "completions/mean_terminated_length": 96.0, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8896, + "grad_norm": 4.088595390319824, + "kl": 0.5556640625, + "learning_rate": 1e-06, + "loss": 0.025, + "num_tokens": 14618274.0, + "reward": 4.689453125, + "reward_std": 0.7515532970428467, + "rewards/rm_reward_func/mean": 4.689453125, + "rewards/rm_reward_func/std": 11.583274841308594, + "step": 1112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 283.46875, + "completions/mean_terminated_length": 179.59091186523438, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8904, + "grad_norm": 5.539703369140625, + "kl": 2.7138671875, + "learning_rate": 1e-06, + "loss": 0.1386, + "num_tokens": 14631761.0, + "reward": 2.4609222412109375, + "reward_std": 4.746338844299316, + "rewards/rm_reward_func/mean": 2.4609222412109375, + "rewards/rm_reward_func/std": 9.370748519897461, + "step": 1113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 264.59375, + "completions/mean_terminated_length": 248.10000610351562, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.8912, + "grad_norm": 16.168067932128906, + "kl": 1.09033203125, + "learning_rate": 1e-06, + "loss": -0.1252, + "num_tokens": 14643388.0, + "reward": 1.837890625, + "reward_std": 7.579433441162109, + "rewards/rm_reward_func/mean": 1.837890625, + "rewards/rm_reward_func/std": 13.530681610107422, + "step": 1114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 415.03125, + "completions/mean_terminated_length": 364.23809814453125, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.892, + "grad_norm": 15.032100677490234, + "kl": 4.005859375, + "learning_rate": 1e-06, + "loss": 0.101, + "num_tokens": 14662877.0, + "reward": -12.3818359375, + "reward_std": 5.061200141906738, + "rewards/rm_reward_func/mean": -12.3818359375, + "rewards/rm_reward_func/std": 8.946840286254883, + "step": 1115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 241.1875, + "completions/mean_terminated_length": 213.1724090576172, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.8928, + "grad_norm": 12.395576477050781, + "kl": 3.50927734375, + "learning_rate": 1e-06, + "loss": 0.3018, + "num_tokens": 14673483.0, + "reward": -4.934326171875, + "reward_std": 6.060213088989258, + "rewards/rm_reward_func/mean": -4.934326171875, + "rewards/rm_reward_func/std": 12.644729614257812, + "step": 1116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 316.15625, + "completions/mean_terminated_length": 250.875, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.8936, + "grad_norm": 9.258816719055176, + "kl": 4.6328125, + "learning_rate": 1e-06, + "loss": 0.1356, + "num_tokens": 14685808.0, + "reward": -9.6767578125, + "reward_std": 4.323359489440918, + "rewards/rm_reward_func/mean": -9.6767578125, + "rewards/rm_reward_func/std": 4.797229290008545, + "step": 1117 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 347.75, + "completions/mean_terminated_length": 336.8000183105469, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.8944, + "grad_norm": 13.36823558807373, + "kl": 3.09423828125, + "learning_rate": 1e-06, + "loss": 0.0864, + "num_tokens": 14698928.0, + "reward": 3.083251953125, + "reward_std": 6.492587089538574, + "rewards/rm_reward_func/mean": 3.083251953125, + "rewards/rm_reward_func/std": 15.465557098388672, + "step": 1118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 344.90625, + "completions/mean_terminated_length": 327.6206970214844, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.8952, + "grad_norm": 16.696882247924805, + "kl": 1.796875, + "learning_rate": 1e-06, + "loss": 0.026, + "num_tokens": 14714893.0, + "reward": 4.2606201171875, + "reward_std": 6.969727516174316, + "rewards/rm_reward_func/mean": 4.2606201171875, + "rewards/rm_reward_func/std": 21.33582305908203, + "step": 1119 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 372.0, + "completions/mean_length": 211.4375, + "completions/mean_terminated_length": 201.74192810058594, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.896, + "grad_norm": 36.645877838134766, + "kl": 3.154296875, + "learning_rate": 1e-06, + "loss": 0.3064, + "num_tokens": 14726907.0, + "reward": -5.021240234375, + "reward_std": 5.98872184753418, + "rewards/rm_reward_func/mean": -5.021240234375, + "rewards/rm_reward_func/std": 11.189291000366211, + "step": 1120 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 410.0, + "completions/max_terminated_length": 410.0, + "completions/mean_length": 206.65625, + "completions/mean_terminated_length": 206.65625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8968, + "grad_norm": 5.673826217651367, + "kl": 0.5107421875, + "learning_rate": 1e-06, + "loss": -0.0006, + "num_tokens": 14739032.0, + "reward": 10.79296875, + "reward_std": 1.6997040510177612, + "rewards/rm_reward_func/mean": 10.79296875, + "rewards/rm_reward_func/std": 6.225403785705566, + "step": 1121 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 308.21875, + "completions/mean_terminated_length": 270.4814758300781, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8976, + "grad_norm": 9.43205451965332, + "kl": 4.9072265625, + "learning_rate": 1e-06, + "loss": 0.214, + "num_tokens": 14751527.0, + "reward": -3.093505859375, + "reward_std": 5.163877487182617, + "rewards/rm_reward_func/mean": -3.093505859375, + "rewards/rm_reward_func/std": 11.953996658325195, + "step": 1122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 342.0, + "completions/mean_length": 182.8125, + "completions/mean_terminated_length": 135.7857208251953, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8984, + "grad_norm": 12.99282169342041, + "kl": 2.005859375, + "learning_rate": 1e-06, + "loss": 0.2826, + "num_tokens": 14761121.0, + "reward": 0.334228515625, + "reward_std": 4.173687934875488, + "rewards/rm_reward_func/mean": 0.334228515625, + "rewards/rm_reward_func/std": 6.381908416748047, + "step": 1123 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 185.0625, + "completions/mean_terminated_length": 124.51851654052734, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8992, + "grad_norm": 9.33836841583252, + "kl": 0.52001953125, + "learning_rate": 1e-06, + "loss": 0.022, + "num_tokens": 14770411.0, + "reward": 7.060302734375, + "reward_std": 1.1086238622665405, + "rewards/rm_reward_func/mean": 7.060302734375, + "rewards/rm_reward_func/std": 5.903942584991455, + "step": 1124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 345.09375, + "completions/mean_terminated_length": 321.25, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.9, + "grad_norm": 10.884320259094238, + "kl": 2.587890625, + "learning_rate": 1e-06, + "loss": 0.0485, + "num_tokens": 14786686.0, + "reward": 11.99560546875, + "reward_std": 7.212575912475586, + "rewards/rm_reward_func/mean": 11.99560546875, + "rewards/rm_reward_func/std": 22.268768310546875, + "step": 1125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 448.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 232.03125, + "completions/mean_terminated_length": 232.03125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9008, + "grad_norm": 9.929396629333496, + "kl": 1.83203125, + "learning_rate": 1e-06, + "loss": 0.0425, + "num_tokens": 14795951.0, + "reward": -1.327392578125, + "reward_std": 3.5014753341674805, + "rewards/rm_reward_func/mean": -1.327392578125, + "rewards/rm_reward_func/std": 10.548651695251465, + "step": 1126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 410.625, + "completions/mean_terminated_length": 357.5238037109375, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "epoch": 0.9016, + "grad_norm": 10.775675773620605, + "kl": 1.833740234375, + "learning_rate": 1e-06, + "loss": 0.0623, + "num_tokens": 14812659.0, + "reward": 6.20263671875, + "reward_std": 6.819557189941406, + "rewards/rm_reward_func/mean": 6.20263671875, + "rewards/rm_reward_func/std": 21.737092971801758, + "step": 1127 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 370.125, + "completions/mean_terminated_length": 244.94117736816406, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9024, + "grad_norm": 8.568568229675293, + "kl": 1.802734375, + "learning_rate": 1e-06, + "loss": 0.0395, + "num_tokens": 14826751.0, + "reward": -0.14697265625, + "reward_std": 5.660521507263184, + "rewards/rm_reward_func/mean": -0.14697265625, + "rewards/rm_reward_func/std": 9.781434059143066, + "step": 1128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 389.375, + "completions/mean_terminated_length": 333.6363830566406, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.9032, + "grad_norm": 14.49055004119873, + "kl": 1.3701171875, + "learning_rate": 1e-06, + "loss": 0.1125, + "num_tokens": 14842283.0, + "reward": -4.70123291015625, + "reward_std": 5.61661434173584, + "rewards/rm_reward_func/mean": -4.70123291015625, + "rewards/rm_reward_func/std": 6.281442165374756, + "step": 1129 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 377.125, + "completions/mean_terminated_length": 357.8571472167969, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.904, + "grad_norm": 8.899121284484863, + "kl": 0.798828125, + "learning_rate": 1e-06, + "loss": -0.0292, + "num_tokens": 14856479.0, + "reward": 11.397480010986328, + "reward_std": 5.290931701660156, + "rewards/rm_reward_func/mean": 11.397480010986328, + "rewards/rm_reward_func/std": 16.51535415649414, + "step": 1130 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 404.15625, + "completions/mean_terminated_length": 361.95654296875, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.9048, + "grad_norm": 20.565969467163086, + "kl": 2.8994140625, + "learning_rate": 1e-06, + "loss": -0.0021, + "num_tokens": 14872308.0, + "reward": -4.064208984375, + "reward_std": 9.170465469360352, + "rewards/rm_reward_func/mean": -4.064208984375, + "rewards/rm_reward_func/std": 11.234378814697266, + "step": 1131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 282.34375, + "completions/mean_terminated_length": 267.0333557128906, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9056, + "grad_norm": 38.7857551574707, + "kl": 1.5107421875, + "learning_rate": 1e-06, + "loss": 0.0238, + "num_tokens": 14884023.0, + "reward": 4.152587890625, + "reward_std": 8.475656509399414, + "rewards/rm_reward_func/mean": 4.152587890625, + "rewards/rm_reward_func/std": 13.332249641418457, + "step": 1132 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 248.625, + "completions/mean_terminated_length": 221.37930297851562, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "epoch": 0.9064, + "grad_norm": 13.033285140991211, + "kl": 1.998046875, + "learning_rate": 1e-06, + "loss": 0.0814, + "num_tokens": 14894971.0, + "reward": 2.286590576171875, + "reward_std": 8.568852424621582, + "rewards/rm_reward_func/mean": 2.286590576171875, + "rewards/rm_reward_func/std": 21.08578109741211, + "step": 1133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 348.375, + "completions/mean_terminated_length": 343.0967712402344, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.9072, + "grad_norm": 13.447713851928711, + "kl": 5.14453125, + "learning_rate": 1e-06, + "loss": 0.2624, + "num_tokens": 14908463.0, + "reward": -6.699335098266602, + "reward_std": 6.876185417175293, + "rewards/rm_reward_func/mean": -6.699335098266602, + "rewards/rm_reward_func/std": 8.872479438781738, + "step": 1134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 366.78125, + "completions/mean_terminated_length": 339.8888854980469, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.908, + "grad_norm": 28.221643447875977, + "kl": 3.7958984375, + "learning_rate": 1e-06, + "loss": 0.0659, + "num_tokens": 14922760.0, + "reward": -6.485260009765625, + "reward_std": 7.134948253631592, + "rewards/rm_reward_func/mean": -6.485260009765625, + "rewards/rm_reward_func/std": 11.682751655578613, + "step": 1135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 482.0, + "completions/mean_length": 265.59375, + "completions/mean_terminated_length": 208.73077392578125, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.9088, + "grad_norm": 101.20088195800781, + "kl": 2.15234375, + "learning_rate": 1e-06, + "loss": -0.1044, + "num_tokens": 14937771.0, + "reward": 0.34814453125, + "reward_std": 8.503646850585938, + "rewards/rm_reward_func/mean": 0.34814453125, + "rewards/rm_reward_func/std": 12.46761703491211, + "step": 1136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 467.0, + "completions/mean_length": 288.5625, + "completions/mean_terminated_length": 273.66668701171875, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9096, + "grad_norm": 10.237065315246582, + "kl": 1.3642578125, + "learning_rate": 1e-06, + "loss": 0.0369, + "num_tokens": 14951197.0, + "reward": 0.99951171875, + "reward_std": 5.672191143035889, + "rewards/rm_reward_func/mean": 0.99951171875, + "rewards/rm_reward_func/std": 10.137231826782227, + "step": 1137 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 261.21875, + "completions/mean_terminated_length": 225.3928680419922, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "epoch": 0.9104, + "grad_norm": 19.81197166442871, + "kl": 4.25390625, + "learning_rate": 1e-06, + "loss": 0.088, + "num_tokens": 14961988.0, + "reward": -4.6865234375, + "reward_std": 8.24875259399414, + "rewards/rm_reward_func/mean": -4.6865234375, + "rewards/rm_reward_func/std": 21.321937561035156, + "step": 1138 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 308.875, + "completions/mean_terminated_length": 287.862060546875, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.9112, + "grad_norm": 17.55968475341797, + "kl": 4.6123046875, + "learning_rate": 1e-06, + "loss": 0.1737, + "num_tokens": 14974224.0, + "reward": -4.378509521484375, + "reward_std": 5.992432594299316, + "rewards/rm_reward_func/mean": -4.378509521484375, + "rewards/rm_reward_func/std": 11.200332641601562, + "step": 1139 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 454.5, + "completions/mean_terminated_length": 389.3333435058594, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.912, + "grad_norm": 86.18330383300781, + "kl": 3.326171875, + "learning_rate": 1e-06, + "loss": 0.0577, + "num_tokens": 14992072.0, + "reward": -0.056640625, + "reward_std": 9.960199356079102, + "rewards/rm_reward_func/mean": -0.056640625, + "rewards/rm_reward_func/std": 13.414643287658691, + "step": 1140 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 504.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 271.09375, + "completions/mean_terminated_length": 271.09375, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9128, + "grad_norm": 25.556337356567383, + "kl": 5.330078125, + "learning_rate": 1e-06, + "loss": 0.2298, + "num_tokens": 15002867.0, + "reward": -7.34228515625, + "reward_std": 4.103965759277344, + "rewards/rm_reward_func/mean": -7.34228515625, + "rewards/rm_reward_func/std": 8.186371803283691, + "step": 1141 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 479.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 199.3125, + "completions/mean_terminated_length": 199.3125, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.9136, + "grad_norm": 33.93816375732422, + "kl": 5.087890625, + "learning_rate": 1e-06, + "loss": 0.1551, + "num_tokens": 15011485.0, + "reward": -7.401123046875, + "reward_std": 7.4005818367004395, + "rewards/rm_reward_func/mean": -7.401123046875, + "rewards/rm_reward_func/std": 17.179683685302734, + "step": 1142 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 250.84375, + "completions/mean_terminated_length": 190.57693481445312, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9144, + "grad_norm": 22.07563591003418, + "kl": 4.1875, + "learning_rate": 1e-06, + "loss": 0.069, + "num_tokens": 15025976.0, + "reward": 0.76611328125, + "reward_std": 4.296019077301025, + "rewards/rm_reward_func/mean": 0.76611328125, + "rewards/rm_reward_func/std": 12.173991203308105, + "step": 1143 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 435.0, + "completions/max_terminated_length": 435.0, + "completions/mean_length": 278.59375, + "completions/mean_terminated_length": 278.59375, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.9152, + "grad_norm": 22.454143524169922, + "kl": 2.92822265625, + "learning_rate": 1e-06, + "loss": -0.0074, + "num_tokens": 15041539.0, + "reward": -3.9213485717773438, + "reward_std": 7.493886470794678, + "rewards/rm_reward_func/mean": -3.9213485717773438, + "rewards/rm_reward_func/std": 12.379457473754883, + "step": 1144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 385.40625, + "completions/mean_terminated_length": 343.2083435058594, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.916, + "grad_norm": 27.009363174438477, + "kl": 4.2822265625, + "learning_rate": 1e-06, + "loss": 0.1094, + "num_tokens": 15056096.0, + "reward": -11.80712890625, + "reward_std": 8.341408729553223, + "rewards/rm_reward_func/mean": -11.80712890625, + "rewards/rm_reward_func/std": 11.495277404785156, + "step": 1145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 266.65625, + "completions/mean_terminated_length": 258.7419128417969, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9168, + "grad_norm": 43.986976623535156, + "kl": 1.9150390625, + "learning_rate": 1e-06, + "loss": -0.0023, + "num_tokens": 15066469.0, + "reward": 4.2705078125, + "reward_std": 8.840106964111328, + "rewards/rm_reward_func/mean": 4.2705078125, + "rewards/rm_reward_func/std": 16.68376922607422, + "step": 1146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 482.0, + "completions/mean_length": 333.5625, + "completions/mean_terminated_length": 300.5185241699219, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.9176, + "grad_norm": 10.10371208190918, + "kl": 1.15283203125, + "learning_rate": 1e-06, + "loss": -0.0467, + "num_tokens": 15084015.0, + "reward": 10.65411376953125, + "reward_std": 9.344249725341797, + "rewards/rm_reward_func/mean": 10.65411376953125, + "rewards/rm_reward_func/std": 16.203916549682617, + "step": 1147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 311.8125, + "completions/mean_terminated_length": 274.7407531738281, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.9184, + "grad_norm": 36.31003952026367, + "kl": 3.45703125, + "learning_rate": 1e-06, + "loss": 0.0783, + "num_tokens": 15096569.0, + "reward": -10.46044921875, + "reward_std": 8.902938842773438, + "rewards/rm_reward_func/mean": -10.46044921875, + "rewards/rm_reward_func/std": 14.822985649108887, + "step": 1148 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 440.59375, + "completions/mean_terminated_length": 391.7368469238281, + "completions/min_length": 273.0, + "completions/min_terminated_length": 273.0, + "epoch": 0.9192, + "grad_norm": 14.747289657592773, + "kl": 1.51904296875, + "learning_rate": 1e-06, + "loss": 0.0563, + "num_tokens": 15113956.0, + "reward": 6.316192626953125, + "reward_std": 6.612483501434326, + "rewards/rm_reward_func/mean": 6.316192626953125, + "rewards/rm_reward_func/std": 10.599157333374023, + "step": 1149 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 488.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 261.78125, + "completions/mean_terminated_length": 261.78125, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.92, + "grad_norm": 21.25505828857422, + "kl": 3.0498046875, + "learning_rate": 1e-06, + "loss": 0.0598, + "num_tokens": 15124293.0, + "reward": -5.06103515625, + "reward_std": 5.579965591430664, + "rewards/rm_reward_func/mean": -5.06103515625, + "rewards/rm_reward_func/std": 10.378743171691895, + "step": 1150 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 484.0, + "completions/mean_length": 267.84375, + "completions/mean_terminated_length": 222.629638671875, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.9208, + "grad_norm": 25.507362365722656, + "kl": 0.8896484375, + "learning_rate": 1e-06, + "loss": -0.1392, + "num_tokens": 15135672.0, + "reward": 1.32666015625, + "reward_std": 8.384624481201172, + "rewards/rm_reward_func/mean": 1.32666015625, + "rewards/rm_reward_func/std": 11.488581657409668, + "step": 1151 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 356.6875, + "completions/mean_terminated_length": 263.5, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9216, + "grad_norm": 9.581064224243164, + "kl": 1.3369140625, + "learning_rate": 1e-06, + "loss": 0.037, + "num_tokens": 15150006.0, + "reward": -1.42645263671875, + "reward_std": 4.21787691116333, + "rewards/rm_reward_func/mean": -1.42645263671875, + "rewards/rm_reward_func/std": 8.071844100952148, + "step": 1152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 506.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 222.625, + "completions/mean_terminated_length": 222.625, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.9224, + "grad_norm": 18.28358268737793, + "kl": 0.68798828125, + "learning_rate": 1e-06, + "loss": 0.0422, + "num_tokens": 15159922.0, + "reward": -3.984771728515625, + "reward_std": 4.933001518249512, + "rewards/rm_reward_func/mean": -3.984771728515625, + "rewards/rm_reward_func/std": 5.989495277404785, + "step": 1153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 261.90625, + "completions/mean_terminated_length": 253.8386993408203, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9232, + "grad_norm": 16.853899002075195, + "kl": 1.552734375, + "learning_rate": 1e-06, + "loss": -0.0088, + "num_tokens": 15170879.0, + "reward": 4.74188232421875, + "reward_std": 8.369260787963867, + "rewards/rm_reward_func/mean": 4.74188232421875, + "rewards/rm_reward_func/std": 15.880949974060059, + "step": 1154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 277.875, + "completions/mean_terminated_length": 223.84616088867188, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.924, + "grad_norm": 8.054681777954102, + "kl": 1.3564453125, + "learning_rate": 1e-06, + "loss": 0.0452, + "num_tokens": 15182435.0, + "reward": -0.05224609375, + "reward_std": 5.243035316467285, + "rewards/rm_reward_func/mean": -0.05224609375, + "rewards/rm_reward_func/std": 11.808974266052246, + "step": 1155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 370.6875, + "completions/mean_terminated_length": 331.1199951171875, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.9248, + "grad_norm": 12.995421409606934, + "kl": 0.4638671875, + "learning_rate": 1e-06, + "loss": 0.075, + "num_tokens": 15196273.0, + "reward": -0.77685546875, + "reward_std": 3.534580945968628, + "rewards/rm_reward_func/mean": -0.77685546875, + "rewards/rm_reward_func/std": 10.869222640991211, + "step": 1156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 375.25, + "completions/mean_terminated_length": 303.6190490722656, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.9256, + "grad_norm": 29.573589324951172, + "kl": 2.072265625, + "learning_rate": 1e-06, + "loss": 0.0377, + "num_tokens": 15210377.0, + "reward": 3.455078125, + "reward_std": 9.187593460083008, + "rewards/rm_reward_func/mean": 3.455078125, + "rewards/rm_reward_func/std": 15.553760528564453, + "step": 1157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 482.0, + "completions/mean_length": 393.75, + "completions/mean_terminated_length": 347.478271484375, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.9264, + "grad_norm": 10.407255172729492, + "kl": 1.662109375, + "learning_rate": 1e-06, + "loss": 0.0592, + "num_tokens": 15225153.0, + "reward": -4.851356506347656, + "reward_std": 8.531854629516602, + "rewards/rm_reward_func/mean": -4.851356506347656, + "rewards/rm_reward_func/std": 12.067245483398438, + "step": 1158 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 483.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 228.5, + "completions/mean_terminated_length": 228.5, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9272, + "grad_norm": 22.906204223632812, + "kl": 0.4560546875, + "learning_rate": 1e-06, + "loss": -0.2539, + "num_tokens": 15236201.0, + "reward": 4.116214752197266, + "reward_std": 7.407968521118164, + "rewards/rm_reward_func/mean": 4.116214752197266, + "rewards/rm_reward_func/std": 12.722545623779297, + "step": 1159 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 351.09375, + "completions/mean_terminated_length": 313.9615478515625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.928, + "grad_norm": 17.148536682128906, + "kl": 0.40380859375, + "learning_rate": 1e-06, + "loss": 0.0482, + "num_tokens": 15250796.0, + "reward": 10.2752685546875, + "reward_std": 4.209094047546387, + "rewards/rm_reward_func/mean": 10.2752685546875, + "rewards/rm_reward_func/std": 11.812329292297363, + "step": 1160 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 407.0, + "completions/max_terminated_length": 407.0, + "completions/mean_length": 171.5, + "completions/mean_terminated_length": 171.5, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.9288, + "grad_norm": 42.1457633972168, + "kl": 1.83056640625, + "learning_rate": 1e-06, + "loss": 0.2306, + "num_tokens": 15258540.0, + "reward": 0.6884765625, + "reward_std": 7.095283508300781, + "rewards/rm_reward_func/mean": 0.6884765625, + "rewards/rm_reward_func/std": 11.873205184936523, + "step": 1161 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 369.46875, + "completions/mean_terminated_length": 313.6956481933594, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9296, + "grad_norm": 7.2761640548706055, + "kl": 2.3427734375, + "learning_rate": 1e-06, + "loss": 0.0776, + "num_tokens": 15272827.0, + "reward": 9.095703125, + "reward_std": 6.575770378112793, + "rewards/rm_reward_func/mean": 9.095703125, + "rewards/rm_reward_func/std": 15.32244873046875, + "step": 1162 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 497.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 166.8125, + "completions/mean_terminated_length": 166.8125, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.9304, + "grad_norm": 10.371278762817383, + "kl": 0.509765625, + "learning_rate": 1e-06, + "loss": -0.0377, + "num_tokens": 15280869.0, + "reward": 5.901123046875, + "reward_std": 2.3234264850616455, + "rewards/rm_reward_func/mean": 5.901123046875, + "rewards/rm_reward_func/std": 10.951936721801758, + "step": 1163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 296.21875, + "completions/mean_terminated_length": 211.78260803222656, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9312, + "grad_norm": 7.607293605804443, + "kl": 2.140625, + "learning_rate": 1e-06, + "loss": -0.0091, + "num_tokens": 15292900.0, + "reward": -0.714080810546875, + "reward_std": 6.104308128356934, + "rewards/rm_reward_func/mean": -0.714080810546875, + "rewards/rm_reward_func/std": 9.522336959838867, + "step": 1164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 332.71875, + "completions/mean_terminated_length": 291.3461608886719, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.932, + "grad_norm": 6.345226764678955, + "kl": 1.32861328125, + "learning_rate": 1e-06, + "loss": 0.0391, + "num_tokens": 15306323.0, + "reward": 9.87939453125, + "reward_std": 8.231972694396973, + "rewards/rm_reward_func/mean": 9.87939453125, + "rewards/rm_reward_func/std": 16.97296142578125, + "step": 1165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 296.71875, + "completions/mean_terminated_length": 274.4482727050781, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9328, + "grad_norm": 11.910536766052246, + "kl": 3.41845703125, + "learning_rate": 1e-06, + "loss": -0.0196, + "num_tokens": 15318378.0, + "reward": -2.5654296875, + "reward_std": 8.650764465332031, + "rewards/rm_reward_func/mean": -2.5654296875, + "rewards/rm_reward_func/std": 14.586143493652344, + "step": 1166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 370.0625, + "completions/mean_terminated_length": 337.3077087402344, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9336, + "grad_norm": 5.885568618774414, + "kl": 0.9931640625, + "learning_rate": 1e-06, + "loss": 0.0021, + "num_tokens": 15334220.0, + "reward": 14.67633056640625, + "reward_std": 8.415434837341309, + "rewards/rm_reward_func/mean": 14.67633056640625, + "rewards/rm_reward_func/std": 19.035816192626953, + "step": 1167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 397.21875, + "completions/mean_terminated_length": 389.5666809082031, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.9344, + "grad_norm": 11.35747241973877, + "kl": 2.22021484375, + "learning_rate": 1e-06, + "loss": 0.0588, + "num_tokens": 15349371.0, + "reward": 7.377197265625, + "reward_std": 6.683224678039551, + "rewards/rm_reward_func/mean": 7.377197265625, + "rewards/rm_reward_func/std": 13.624530792236328, + "step": 1168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 369.375, + "completions/mean_terminated_length": 349.0000305175781, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.9352, + "grad_norm": 28.526676177978516, + "kl": 4.8515625, + "learning_rate": 1e-06, + "loss": 0.1177, + "num_tokens": 15363063.0, + "reward": 3.8070831298828125, + "reward_std": 14.380706787109375, + "rewards/rm_reward_func/mean": 3.8070831298828125, + "rewards/rm_reward_func/std": 15.08162784576416, + "step": 1169 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 500.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 277.78125, + "completions/mean_terminated_length": 277.78125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.936, + "grad_norm": 7.368760585784912, + "kl": 0.72314453125, + "learning_rate": 1e-06, + "loss": 0.0015, + "num_tokens": 15373880.0, + "reward": 2.238525390625, + "reward_std": 2.461265802383423, + "rewards/rm_reward_func/mean": 2.238525390625, + "rewards/rm_reward_func/std": 6.371425628662109, + "step": 1170 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 371.5, + "completions/mean_terminated_length": 324.66668701171875, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.9368, + "grad_norm": 23.937978744506836, + "kl": 2.81396484375, + "learning_rate": 1e-06, + "loss": 0.0015, + "num_tokens": 15387984.0, + "reward": -3.2255859375, + "reward_std": 8.331253051757812, + "rewards/rm_reward_func/mean": -3.2255859375, + "rewards/rm_reward_func/std": 10.779619216918945, + "step": 1171 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 378.65625, + "completions/mean_terminated_length": 318.04547119140625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.9376, + "grad_norm": 15.15499496459961, + "kl": 4.59619140625, + "learning_rate": 1e-06, + "loss": 0.0643, + "num_tokens": 15402765.0, + "reward": -3.593994140625, + "reward_std": 6.05653190612793, + "rewards/rm_reward_func/mean": -3.593994140625, + "rewards/rm_reward_func/std": 10.450895309448242, + "step": 1172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 506.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 241.40625, + "completions/mean_terminated_length": 241.40625, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.9384, + "grad_norm": 10.372428894042969, + "kl": 2.1416015625, + "learning_rate": 1e-06, + "loss": 0.0091, + "num_tokens": 15417634.0, + "reward": 6.179931640625, + "reward_std": 7.2046685218811035, + "rewards/rm_reward_func/mean": 6.179931640625, + "rewards/rm_reward_func/std": 8.590556144714355, + "step": 1173 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 276.90625, + "completions/mean_terminated_length": 222.6538543701172, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9392, + "grad_norm": 14.381688117980957, + "kl": 1.92138671875, + "learning_rate": 1e-06, + "loss": 0.0851, + "num_tokens": 15429623.0, + "reward": -2.0927734375, + "reward_std": 4.637411117553711, + "rewards/rm_reward_func/mean": -2.0927734375, + "rewards/rm_reward_func/std": 13.167999267578125, + "step": 1174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 280.53125, + "completions/mean_terminated_length": 265.1000061035156, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.94, + "grad_norm": 32.04814529418945, + "kl": 7.716796875, + "learning_rate": 1e-06, + "loss": 0.1969, + "num_tokens": 15440720.0, + "reward": -1.5761604309082031, + "reward_std": 14.394807815551758, + "rewards/rm_reward_func/mean": -1.5761604309082031, + "rewards/rm_reward_func/std": 17.306682586669922, + "step": 1175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 365.46875, + "completions/mean_terminated_length": 331.65386962890625, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.9408, + "grad_norm": 6.667887210845947, + "kl": 1.29052734375, + "learning_rate": 1e-06, + "loss": -0.0015, + "num_tokens": 15454551.0, + "reward": 3.3128662109375, + "reward_std": 7.290994644165039, + "rewards/rm_reward_func/mean": 3.3128662109375, + "rewards/rm_reward_func/std": 11.585569381713867, + "step": 1176 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 372.25, + "completions/mean_terminated_length": 317.5652160644531, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9416, + "grad_norm": 6.514392375946045, + "kl": 3.54052734375, + "learning_rate": 1e-06, + "loss": 0.1126, + "num_tokens": 15469031.0, + "reward": 15.22454833984375, + "reward_std": 8.186134338378906, + "rewards/rm_reward_func/mean": 15.22454833984375, + "rewards/rm_reward_func/std": 20.967836380004883, + "step": 1177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 352.0, + "completions/max_terminated_length": 352.0, + "completions/mean_length": 128.28125, + "completions/mean_terminated_length": 128.28125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9424, + "grad_norm": 57.319156646728516, + "kl": 0.97265625, + "learning_rate": 1e-06, + "loss": 0.0197, + "num_tokens": 15475856.0, + "reward": 4.0428466796875, + "reward_std": 1.0468058586120605, + "rewards/rm_reward_func/mean": 4.0428466796875, + "rewards/rm_reward_func/std": 8.96422004699707, + "step": 1178 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 466.0, + "completions/max_terminated_length": 466.0, + "completions/mean_length": 257.375, + "completions/mean_terminated_length": 257.375, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.9432, + "grad_norm": 37.18678283691406, + "kl": 4.50732421875, + "learning_rate": 1e-06, + "loss": 0.1144, + "num_tokens": 15490860.0, + "reward": -0.7977094650268555, + "reward_std": 9.967428207397461, + "rewards/rm_reward_func/mean": -0.7977094650268555, + "rewards/rm_reward_func/std": 12.24376106262207, + "step": 1179 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 274.75, + "completions/mean_terminated_length": 208.3199920654297, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.944, + "grad_norm": 11.060836791992188, + "kl": 2.35205078125, + "learning_rate": 1e-06, + "loss": 0.0886, + "num_tokens": 15502980.0, + "reward": 4.240234375, + "reward_std": 2.008146047592163, + "rewards/rm_reward_func/mean": 4.240234375, + "rewards/rm_reward_func/std": 12.89011287689209, + "step": 1180 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 487.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 224.71875, + "completions/mean_terminated_length": 224.71875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.9448, + "grad_norm": 25.126611709594727, + "kl": 3.6806640625, + "learning_rate": 1e-06, + "loss": 0.0654, + "num_tokens": 15512067.0, + "reward": -1.59136962890625, + "reward_std": 6.693836212158203, + "rewards/rm_reward_func/mean": -1.59136962890625, + "rewards/rm_reward_func/std": 13.480401039123535, + "step": 1181 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 310.40625, + "completions/mean_terminated_length": 296.9666748046875, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9456, + "grad_norm": 6.820868968963623, + "kl": 2.0498046875, + "learning_rate": 1e-06, + "loss": 0.0904, + "num_tokens": 15527256.0, + "reward": 11.24267578125, + "reward_std": 6.773592472076416, + "rewards/rm_reward_func/mean": 11.24267578125, + "rewards/rm_reward_func/std": 16.010009765625, + "step": 1182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 324.3125, + "completions/mean_terminated_length": 271.7599792480469, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9464, + "grad_norm": 5.817383289337158, + "kl": 1.25830078125, + "learning_rate": 1e-06, + "loss": 0.0173, + "num_tokens": 15540034.0, + "reward": 12.9453125, + "reward_std": 6.853506088256836, + "rewards/rm_reward_func/mean": 12.9453125, + "rewards/rm_reward_func/std": 11.050248146057129, + "step": 1183 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 394.96875, + "completions/mean_terminated_length": 223.92308044433594, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9472, + "grad_norm": 11.657125473022461, + "kl": 1.68115234375, + "learning_rate": 1e-06, + "loss": 0.0629, + "num_tokens": 15556841.0, + "reward": 1.7704925537109375, + "reward_std": 4.207978248596191, + "rewards/rm_reward_func/mean": 1.7704925537109375, + "rewards/rm_reward_func/std": 6.741135120391846, + "step": 1184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 461.0, + "completions/mean_length": 329.9375, + "completions/mean_terminated_length": 303.9285888671875, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.948, + "grad_norm": 26.10344696044922, + "kl": 4.341796875, + "learning_rate": 1e-06, + "loss": 0.2526, + "num_tokens": 15570263.0, + "reward": 7.865570068359375, + "reward_std": 6.319413185119629, + "rewards/rm_reward_func/mean": 7.865570068359375, + "rewards/rm_reward_func/std": 15.726142883300781, + "step": 1185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 461.0, + "completions/mean_length": 335.1875, + "completions/mean_terminated_length": 276.25, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9488, + "grad_norm": 6.295200824737549, + "kl": 0.4013671875, + "learning_rate": 1e-06, + "loss": 0.0074, + "num_tokens": 15584085.0, + "reward": 7.826171875, + "reward_std": 1.844343900680542, + "rewards/rm_reward_func/mean": 7.826171875, + "rewards/rm_reward_func/std": 16.900344848632812, + "step": 1186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 311.96875, + "completions/mean_terminated_length": 255.95999145507812, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9496, + "grad_norm": 7.316650867462158, + "kl": 0.916015625, + "learning_rate": 1e-06, + "loss": -0.0087, + "num_tokens": 15598652.0, + "reward": -1.8782958984375, + "reward_std": 4.917675495147705, + "rewards/rm_reward_func/mean": -1.8782958984375, + "rewards/rm_reward_func/std": 8.429694175720215, + "step": 1187 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 484.0, + "completions/mean_length": 367.78125, + "completions/mean_terminated_length": 358.16668701171875, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "epoch": 0.9504, + "grad_norm": 27.917741775512695, + "kl": 1.7177734375, + "learning_rate": 1e-06, + "loss": 0.0232, + "num_tokens": 15612501.0, + "reward": 11.6102294921875, + "reward_std": 8.347414016723633, + "rewards/rm_reward_func/mean": 11.6102294921875, + "rewards/rm_reward_func/std": 13.412553787231445, + "step": 1188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 443.46875, + "completions/mean_terminated_length": 420.625, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "epoch": 0.9512, + "grad_norm": 19.72421646118164, + "kl": 3.552734375, + "learning_rate": 1e-06, + "loss": 0.1278, + "num_tokens": 15628892.0, + "reward": 5.533203125, + "reward_std": 10.962414741516113, + "rewards/rm_reward_func/mean": 5.533203125, + "rewards/rm_reward_func/std": 24.766939163208008, + "step": 1189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 296.40625, + "completions/mean_terminated_length": 289.45159912109375, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.952, + "grad_norm": 8.21200180053711, + "kl": 1.095703125, + "learning_rate": 1e-06, + "loss": 0.0381, + "num_tokens": 15642537.0, + "reward": 6.253631591796875, + "reward_std": 4.122445106506348, + "rewards/rm_reward_func/mean": 6.253631591796875, + "rewards/rm_reward_func/std": 6.488741874694824, + "step": 1190 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 486.0, + "completions/mean_length": 342.65625, + "completions/mean_terminated_length": 331.3666687011719, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.9528, + "grad_norm": 7.0987677574157715, + "kl": 2.54150390625, + "learning_rate": 1e-06, + "loss": 0.0774, + "num_tokens": 15657294.0, + "reward": 4.48828125, + "reward_std": 5.943986892700195, + "rewards/rm_reward_func/mean": 4.48828125, + "rewards/rm_reward_func/std": 18.430313110351562, + "step": 1191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 337.0625, + "completions/mean_terminated_length": 296.69232177734375, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.9536, + "grad_norm": 10.321151733398438, + "kl": 0.74853515625, + "learning_rate": 1e-06, + "loss": 0.0207, + "num_tokens": 15670624.0, + "reward": 8.99795150756836, + "reward_std": 6.07137393951416, + "rewards/rm_reward_func/mean": 8.99795150756836, + "rewards/rm_reward_func/std": 11.28697395324707, + "step": 1192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 406.46875, + "completions/mean_terminated_length": 365.1739196777344, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.9544, + "grad_norm": 75.35392761230469, + "kl": 8.7998046875, + "learning_rate": 1e-06, + "loss": 0.3095, + "num_tokens": 15688439.0, + "reward": -8.990234375, + "reward_std": 6.966933250427246, + "rewards/rm_reward_func/mean": -8.990234375, + "rewards/rm_reward_func/std": 18.05649757385254, + "step": 1193 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 239.8125, + "completions/mean_terminated_length": 211.65516662597656, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9552, + "grad_norm": 15.314480781555176, + "kl": 1.806640625, + "learning_rate": 1e-06, + "loss": 0.0248, + "num_tokens": 15700577.0, + "reward": 3.8753662109375, + "reward_std": 4.899765491485596, + "rewards/rm_reward_func/mean": 3.8753662109375, + "rewards/rm_reward_func/std": 8.12557315826416, + "step": 1194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 362.15625, + "completions/mean_terminated_length": 327.5769348144531, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.956, + "grad_norm": 78.24580383300781, + "kl": 8.9765625, + "learning_rate": 1e-06, + "loss": 0.2552, + "num_tokens": 15714486.0, + "reward": -6.42059326171875, + "reward_std": 9.736968994140625, + "rewards/rm_reward_func/mean": -6.42059326171875, + "rewards/rm_reward_func/std": 14.20551586151123, + "step": 1195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 397.0, + "completions/mean_length": 280.46875, + "completions/mean_terminated_length": 189.86956787109375, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9568, + "grad_norm": 31.360624313354492, + "kl": 3.84814453125, + "learning_rate": 1e-06, + "loss": 0.19, + "num_tokens": 15726301.0, + "reward": 1.2685317993164062, + "reward_std": 4.518694877624512, + "rewards/rm_reward_func/mean": 1.2685317993164062, + "rewards/rm_reward_func/std": 8.930562973022461, + "step": 1196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 274.03125, + "completions/mean_terminated_length": 240.0357208251953, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9576, + "grad_norm": 7.44788122177124, + "kl": 0.564453125, + "learning_rate": 1e-06, + "loss": 0.0323, + "num_tokens": 15738126.0, + "reward": 8.062255859375, + "reward_std": 1.784210443496704, + "rewards/rm_reward_func/mean": 8.062255859375, + "rewards/rm_reward_func/std": 6.24770450592041, + "step": 1197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 430.0, + "completions/max_terminated_length": 430.0, + "completions/mean_length": 200.1875, + "completions/mean_terminated_length": 200.1875, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9584, + "grad_norm": 46.06965637207031, + "kl": 6.3017578125, + "learning_rate": 1e-06, + "loss": 0.2246, + "num_tokens": 15746660.0, + "reward": -1.352294921875, + "reward_std": 7.574354648590088, + "rewards/rm_reward_func/mean": -1.352294921875, + "rewards/rm_reward_func/std": 14.081011772155762, + "step": 1198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 303.0625, + "completions/mean_terminated_length": 264.370361328125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9592, + "grad_norm": 13.53447437286377, + "kl": 3.17578125, + "learning_rate": 1e-06, + "loss": 0.0633, + "num_tokens": 15761318.0, + "reward": 6.076904296875, + "reward_std": 8.05277156829834, + "rewards/rm_reward_func/mean": 6.076904296875, + "rewards/rm_reward_func/std": 10.448375701904297, + "step": 1199 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 372.71875, + "completions/mean_terminated_length": 358.3103332519531, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.96, + "grad_norm": 82.56756591796875, + "kl": 6.08251953125, + "learning_rate": 1e-06, + "loss": 0.1814, + "num_tokens": 15777317.0, + "reward": 0.806884765625, + "reward_std": 8.0141019821167, + "rewards/rm_reward_func/mean": 0.806884765625, + "rewards/rm_reward_func/std": 20.149206161499023, + "step": 1200 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 80.0, + "completions/mean_length": 134.0, + "completions/mean_terminated_length": 80.0, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9608, + "grad_norm": 80.49437713623047, + "kl": 2.533203125, + "learning_rate": 1e-06, + "loss": 0.3195, + "num_tokens": 15786189.0, + "reward": 2.772796630859375, + "reward_std": 2.7184181213378906, + "rewards/rm_reward_func/mean": 2.772796630859375, + "rewards/rm_reward_func/std": 8.184410095214844, + "step": 1201 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 355.71875, + "completions/mean_terminated_length": 326.77777099609375, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9616, + "grad_norm": 22.120071411132812, + "kl": 0.51708984375, + "learning_rate": 1e-06, + "loss": 0.0185, + "num_tokens": 15802404.0, + "reward": 19.8399658203125, + "reward_std": 4.591503620147705, + "rewards/rm_reward_func/mean": 19.8399658203125, + "rewards/rm_reward_func/std": 21.975711822509766, + "step": 1202 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 399.125, + "completions/mean_terminated_length": 383.0000305175781, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.9624, + "grad_norm": 13.387299537658691, + "kl": 4.68603515625, + "learning_rate": 1e-06, + "loss": 0.2318, + "num_tokens": 15818008.0, + "reward": 5.8818359375, + "reward_std": 6.727931976318359, + "rewards/rm_reward_func/mean": 5.8818359375, + "rewards/rm_reward_func/std": 23.548294067382812, + "step": 1203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 284.125, + "completions/mean_terminated_length": 241.92593383789062, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9632, + "grad_norm": 5.043156147003174, + "kl": 2.41845703125, + "learning_rate": 1e-06, + "loss": 0.096, + "num_tokens": 15830828.0, + "reward": 2.5513916015625, + "reward_std": 6.220531463623047, + "rewards/rm_reward_func/mean": 2.5513916015625, + "rewards/rm_reward_func/std": 13.680570602416992, + "step": 1204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 477.0, + "completions/mean_length": 379.25, + "completions/mean_terminated_length": 365.5172424316406, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.964, + "grad_norm": 19.86156463623047, + "kl": 4.646484375, + "learning_rate": 1e-06, + "loss": 0.19, + "num_tokens": 15844940.0, + "reward": 12.568359375, + "reward_std": 12.276927947998047, + "rewards/rm_reward_func/mean": 12.568359375, + "rewards/rm_reward_func/std": 18.882339477539062, + "step": 1205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 472.0, + "completions/mean_length": 244.0625, + "completions/mean_terminated_length": 226.20001220703125, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.9648, + "grad_norm": 35.16064453125, + "kl": 3.6748046875, + "learning_rate": 1e-06, + "loss": -0.0097, + "num_tokens": 15854710.0, + "reward": 4.755615234375, + "reward_std": 10.025222778320312, + "rewards/rm_reward_func/mean": 4.755615234375, + "rewards/rm_reward_func/std": 17.34681510925293, + "step": 1206 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 480.0, + "completions/mean_length": 330.71875, + "completions/mean_terminated_length": 297.1481628417969, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.9656, + "grad_norm": 20.283857345581055, + "kl": 4.10986328125, + "learning_rate": 1e-06, + "loss": 0.1873, + "num_tokens": 15869749.0, + "reward": -1.422607421875, + "reward_std": 7.128761291503906, + "rewards/rm_reward_func/mean": -1.422607421875, + "rewards/rm_reward_func/std": 8.89996337890625, + "step": 1207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 461.0, + "completions/max_terminated_length": 461.0, + "completions/mean_length": 228.8125, + "completions/mean_terminated_length": 228.8125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9664, + "grad_norm": 14.01342487335205, + "kl": 1.6328125, + "learning_rate": 1e-06, + "loss": 0.0612, + "num_tokens": 15879927.0, + "reward": 5.265869140625, + "reward_std": 2.066286087036133, + "rewards/rm_reward_func/mean": 5.265869140625, + "rewards/rm_reward_func/std": 7.653271675109863, + "step": 1208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 509.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 301.34375, + "completions/mean_terminated_length": 301.34375, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9672, + "grad_norm": 16.027812957763672, + "kl": 2.18359375, + "learning_rate": 1e-06, + "loss": -0.007, + "num_tokens": 15895018.0, + "reward": 5.47796630859375, + "reward_std": 9.107280731201172, + "rewards/rm_reward_func/mean": 5.47796630859375, + "rewards/rm_reward_func/std": 13.736273765563965, + "step": 1209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 452.0, + "completions/mean_length": 366.75, + "completions/mean_terminated_length": 326.0799865722656, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "epoch": 0.968, + "grad_norm": 6.4775848388671875, + "kl": 0.8310546875, + "learning_rate": 1e-06, + "loss": -0.1347, + "num_tokens": 15909162.0, + "reward": 4.66473388671875, + "reward_std": 10.003836631774902, + "rewards/rm_reward_func/mean": 4.66473388671875, + "rewards/rm_reward_func/std": 11.560744285583496, + "step": 1210 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 442.0, + "completions/max_terminated_length": 442.0, + "completions/mean_length": 250.40625, + "completions/mean_terminated_length": 250.40625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9688, + "grad_norm": 9.306504249572754, + "kl": 1.1865234375, + "learning_rate": 1e-06, + "loss": 0.0649, + "num_tokens": 15920199.0, + "reward": 11.21533203125, + "reward_std": 4.61986780166626, + "rewards/rm_reward_func/mean": 11.21533203125, + "rewards/rm_reward_func/std": 11.726616859436035, + "step": 1211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 428.0, + "completions/mean_length": 276.8125, + "completions/mean_terminated_length": 243.21429443359375, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9696, + "grad_norm": 29.228607177734375, + "kl": 1.453125, + "learning_rate": 1e-06, + "loss": 0.0867, + "num_tokens": 15932337.0, + "reward": -3.7728271484375, + "reward_std": 2.3056063652038574, + "rewards/rm_reward_func/mean": -3.7728271484375, + "rewards/rm_reward_func/std": 5.636190891265869, + "step": 1212 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 242.71875, + "completions/mean_terminated_length": 204.25001525878906, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.9704, + "grad_norm": 28.853511810302734, + "kl": 2.189453125, + "learning_rate": 1e-06, + "loss": 0.0766, + "num_tokens": 15942880.0, + "reward": 4.451416015625, + "reward_std": 4.814798355102539, + "rewards/rm_reward_func/mean": 4.451416015625, + "rewards/rm_reward_func/std": 7.7672343254089355, + "step": 1213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 307.4375, + "completions/mean_terminated_length": 260.23077392578125, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.9712, + "grad_norm": 8.984874725341797, + "kl": 0.439697265625, + "learning_rate": 1e-06, + "loss": 0.0709, + "num_tokens": 15954926.0, + "reward": 2.17816162109375, + "reward_std": 5.897883415222168, + "rewards/rm_reward_func/mean": 2.17816162109375, + "rewards/rm_reward_func/std": 6.458736419677734, + "step": 1214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 421.0, + "completions/max_terminated_length": 421.0, + "completions/mean_length": 164.15625, + "completions/mean_terminated_length": 164.15625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.972, + "grad_norm": 15.127213478088379, + "kl": 0.443115234375, + "learning_rate": 1e-06, + "loss": 0.0297, + "num_tokens": 15965971.0, + "reward": 8.07522964477539, + "reward_std": 1.5865893363952637, + "rewards/rm_reward_func/mean": 8.07522964477539, + "rewards/rm_reward_func/std": 9.159050941467285, + "step": 1215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 255.9375, + "completions/mean_terminated_length": 247.6774139404297, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.9728, + "grad_norm": 42.37367630004883, + "kl": 2.08447265625, + "learning_rate": 1e-06, + "loss": 0.0359, + "num_tokens": 15979769.0, + "reward": 1.78619384765625, + "reward_std": 2.9838078022003174, + "rewards/rm_reward_func/mean": 1.78619384765625, + "rewards/rm_reward_func/std": 11.067867279052734, + "step": 1216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 346.59375, + "completions/mean_terminated_length": 271.4090881347656, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9736, + "grad_norm": 9.83594799041748, + "kl": 4.3447265625, + "learning_rate": 1e-06, + "loss": 0.1791, + "num_tokens": 15996100.0, + "reward": -1.2491302490234375, + "reward_std": 3.7429025173187256, + "rewards/rm_reward_func/mean": -1.2491302490234375, + "rewards/rm_reward_func/std": 8.419785499572754, + "step": 1217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 353.59375, + "completions/mean_terminated_length": 343.0333557128906, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.9744, + "grad_norm": 10.141213417053223, + "kl": 2.3017578125, + "learning_rate": 1e-06, + "loss": 0.0357, + "num_tokens": 16009191.0, + "reward": 3.6845703125, + "reward_std": 9.749015808105469, + "rewards/rm_reward_func/mean": 3.6845703125, + "rewards/rm_reward_func/std": 15.83315658569336, + "step": 1218 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 451.40625, + "completions/mean_terminated_length": 419.66668701171875, + "completions/min_length": 304.0, + "completions/min_terminated_length": 304.0, + "epoch": 0.9752, + "grad_norm": 6.541053771972656, + "kl": 0.66943359375, + "learning_rate": 1e-06, + "loss": 0.0511, + "num_tokens": 16026156.0, + "reward": 10.817626953125, + "reward_std": 10.195655822753906, + "rewards/rm_reward_func/mean": 10.817626953125, + "rewards/rm_reward_func/std": 14.443761825561523, + "step": 1219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 509.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 308.6875, + "completions/mean_terminated_length": 308.6875, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.976, + "grad_norm": 7.883213996887207, + "kl": 0.72705078125, + "learning_rate": 1e-06, + "loss": -0.0421, + "num_tokens": 16040322.0, + "reward": 2.47705078125, + "reward_std": 7.948477745056152, + "rewards/rm_reward_func/mean": 2.47705078125, + "rewards/rm_reward_func/std": 14.740900039672852, + "step": 1220 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 231.6875, + "completions/mean_terminated_length": 231.6875, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.9768, + "grad_norm": 16.915742874145508, + "kl": 1.87890625, + "learning_rate": 1e-06, + "loss": 0.0358, + "num_tokens": 16050816.0, + "reward": -4.7006988525390625, + "reward_std": 6.505680561065674, + "rewards/rm_reward_func/mean": -4.7006988525390625, + "rewards/rm_reward_func/std": 9.603677749633789, + "step": 1221 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 248.0, + "completions/mean_terminated_length": 210.2857208251953, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9776, + "grad_norm": 7.012750625610352, + "kl": 0.8623046875, + "learning_rate": 1e-06, + "loss": 0.0782, + "num_tokens": 16061744.0, + "reward": 12.907470703125, + "reward_std": 4.333094596862793, + "rewards/rm_reward_func/mean": 12.907470703125, + "rewards/rm_reward_func/std": 8.020941734313965, + "step": 1222 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 228.8125, + "completions/mean_terminated_length": 228.8125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9784, + "grad_norm": 6.7300801277160645, + "kl": 0.89794921875, + "learning_rate": 1e-06, + "loss": 0.0436, + "num_tokens": 16072642.0, + "reward": -0.694091796875, + "reward_std": 3.24855637550354, + "rewards/rm_reward_func/mean": -0.694091796875, + "rewards/rm_reward_func/std": 10.3416166305542, + "step": 1223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 339.0, + "completions/max_terminated_length": 339.0, + "completions/mean_length": 175.375, + "completions/mean_terminated_length": 175.375, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9792, + "grad_norm": 5.84846305847168, + "kl": 1.20361328125, + "learning_rate": 1e-06, + "loss": 0.0497, + "num_tokens": 16080534.0, + "reward": 4.77056884765625, + "reward_std": 2.0563995838165283, + "rewards/rm_reward_func/mean": 4.77056884765625, + "rewards/rm_reward_func/std": 4.059889316558838, + "step": 1224 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 284.625, + "completions/mean_terminated_length": 269.4666748046875, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.98, + "grad_norm": 8.25778865814209, + "kl": 2.14990234375, + "learning_rate": 1e-06, + "loss": 0.0486, + "num_tokens": 16091746.0, + "reward": 6.4002685546875, + "reward_std": 7.458786964416504, + "rewards/rm_reward_func/mean": 6.4002685546875, + "rewards/rm_reward_func/std": 13.408005714416504, + "step": 1225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 341.875, + "completions/mean_terminated_length": 294.239990234375, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.9808, + "grad_norm": 10.492647171020508, + "kl": 2.84521484375, + "learning_rate": 1e-06, + "loss": 0.0935, + "num_tokens": 16105958.0, + "reward": -4.466461181640625, + "reward_std": 5.502716064453125, + "rewards/rm_reward_func/mean": -4.466461181640625, + "rewards/rm_reward_func/std": 6.831468105316162, + "step": 1226 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 480.0, + "completions/mean_length": 310.53125, + "completions/mean_terminated_length": 172.68421936035156, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9816, + "grad_norm": 8.328527450561523, + "kl": 3.4453125, + "learning_rate": 1e-06, + "loss": 0.1165, + "num_tokens": 16120351.0, + "reward": -7.10888671875, + "reward_std": 3.2640366554260254, + "rewards/rm_reward_func/mean": -7.10888671875, + "rewards/rm_reward_func/std": 10.982377052307129, + "step": 1227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 439.0, + "completions/max_terminated_length": 439.0, + "completions/mean_length": 246.4375, + "completions/mean_terminated_length": 246.4375, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9824, + "grad_norm": 10.678609848022461, + "kl": 3.5791015625, + "learning_rate": 1e-06, + "loss": 0.1131, + "num_tokens": 16133981.0, + "reward": 5.48870849609375, + "reward_std": 6.148322105407715, + "rewards/rm_reward_func/mean": 5.48870849609375, + "rewards/rm_reward_func/std": 13.595304489135742, + "step": 1228 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 470.0, + "completions/max_terminated_length": 470.0, + "completions/mean_length": 254.1875, + "completions/mean_terminated_length": 254.1875, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.9832, + "grad_norm": 7.430377006530762, + "kl": 1.12890625, + "learning_rate": 1e-06, + "loss": -0.0441, + "num_tokens": 16145083.0, + "reward": 8.41162109375, + "reward_std": 8.556034088134766, + "rewards/rm_reward_func/mean": 8.41162109375, + "rewards/rm_reward_func/std": 13.60086727142334, + "step": 1229 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 477.0, + "completions/mean_length": 232.53125, + "completions/mean_terminated_length": 213.90000915527344, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.984, + "grad_norm": 5.466613292694092, + "kl": 1.316650390625, + "learning_rate": 1e-06, + "loss": -0.0194, + "num_tokens": 16157052.0, + "reward": 13.6875, + "reward_std": 7.069519996643066, + "rewards/rm_reward_func/mean": 13.6875, + "rewards/rm_reward_func/std": 15.511878967285156, + "step": 1230 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 256.96875, + "completions/mean_terminated_length": 239.9666748046875, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.9848, + "grad_norm": 14.960919380187988, + "kl": 1.33642578125, + "learning_rate": 1e-06, + "loss": 0.048, + "num_tokens": 16167787.0, + "reward": 13.341796875, + "reward_std": 7.339491844177246, + "rewards/rm_reward_func/mean": 13.341796875, + "rewards/rm_reward_func/std": 19.08413314819336, + "step": 1231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 271.5625, + "completions/mean_terminated_length": 263.80645751953125, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.9856, + "grad_norm": 21.033065795898438, + "kl": 3.0126953125, + "learning_rate": 1e-06, + "loss": 0.0807, + "num_tokens": 16178821.0, + "reward": 1.5001983642578125, + "reward_std": 6.436107635498047, + "rewards/rm_reward_func/mean": 1.5001983642578125, + "rewards/rm_reward_func/std": 14.546778678894043, + "step": 1232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 342.78125, + "completions/mean_terminated_length": 286.375, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.9864, + "grad_norm": 11.773747444152832, + "kl": 6.388427734375, + "learning_rate": 1e-06, + "loss": 0.3401, + "num_tokens": 16192638.0, + "reward": -4.092529296875, + "reward_std": 6.334899425506592, + "rewards/rm_reward_func/mean": -4.092529296875, + "rewards/rm_reward_func/std": 12.157096862792969, + "step": 1233 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 448.21875, + "completions/mean_terminated_length": 384.4375, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "epoch": 0.9872, + "grad_norm": 14.910449028015137, + "kl": 4.833984375, + "learning_rate": 1e-06, + "loss": 0.1394, + "num_tokens": 16210957.0, + "reward": -4.65869140625, + "reward_std": 6.343834400177002, + "rewards/rm_reward_func/mean": -4.65869140625, + "rewards/rm_reward_func/std": 21.47957420349121, + "step": 1234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 258.75, + "completions/mean_terminated_length": 187.83999633789062, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.988, + "grad_norm": 31.964218139648438, + "kl": 1.8095703125, + "learning_rate": 1e-06, + "loss": 0.0506, + "num_tokens": 16222165.0, + "reward": -6.220703125, + "reward_std": 4.152710914611816, + "rewards/rm_reward_func/mean": -6.220703125, + "rewards/rm_reward_func/std": 10.58847427368164, + "step": 1235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 472.0, + "completions/max_terminated_length": 472.0, + "completions/mean_length": 198.71875, + "completions/mean_terminated_length": 198.71875, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.9888, + "grad_norm": 5.49161958694458, + "kl": 1.7666015625, + "learning_rate": 1e-06, + "loss": -0.0291, + "num_tokens": 16231932.0, + "reward": -1.51971435546875, + "reward_std": 2.968614101409912, + "rewards/rm_reward_func/mean": -1.51971435546875, + "rewards/rm_reward_func/std": 9.40853500366211, + "step": 1236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 256.84375, + "completions/mean_terminated_length": 239.83334350585938, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9896, + "grad_norm": 7.274431228637695, + "kl": 0.58251953125, + "learning_rate": 1e-06, + "loss": -0.0121, + "num_tokens": 16243055.0, + "reward": 13.570831298828125, + "reward_std": 3.243131637573242, + "rewards/rm_reward_func/mean": 13.570831298828125, + "rewards/rm_reward_func/std": 12.231443405151367, + "step": 1237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 423.1875, + "completions/mean_terminated_length": 369.8999938964844, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "epoch": 0.9904, + "grad_norm": 8.700695037841797, + "kl": 1.9287109375, + "learning_rate": 1e-06, + "loss": 0.0465, + "num_tokens": 16259893.0, + "reward": -7.0559539794921875, + "reward_std": 5.43247652053833, + "rewards/rm_reward_func/mean": -7.0559539794921875, + "rewards/rm_reward_func/std": 7.467658519744873, + "step": 1238 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 320.25, + "completions/mean_terminated_length": 284.7407531738281, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9912, + "grad_norm": 14.817831039428711, + "kl": 1.71875, + "learning_rate": 1e-06, + "loss": 0.0421, + "num_tokens": 16273869.0, + "reward": 7.86328125, + "reward_std": 7.426191806793213, + "rewards/rm_reward_func/mean": 7.86328125, + "rewards/rm_reward_func/std": 9.81981372833252, + "step": 1239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 327.84375, + "completions/mean_terminated_length": 231.38095092773438, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.992, + "grad_norm": 10.064021110534668, + "kl": 1.5625, + "learning_rate": 1e-06, + "loss": 0.0699, + "num_tokens": 16288208.0, + "reward": 5.3447265625, + "reward_std": 4.9022955894470215, + "rewards/rm_reward_func/mean": 5.3447265625, + "rewards/rm_reward_func/std": 9.280166625976562, + "step": 1240 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 480.0, + "completions/max_terminated_length": 480.0, + "completions/mean_length": 348.71875, + "completions/mean_terminated_length": 348.71875, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "epoch": 0.9928, + "grad_norm": 24.340036392211914, + "kl": 0.641357421875, + "learning_rate": 1e-06, + "loss": 0.0044, + "num_tokens": 16301511.0, + "reward": 2.7781982421875, + "reward_std": 5.280330181121826, + "rewards/rm_reward_func/mean": 2.7781982421875, + "rewards/rm_reward_func/std": 7.673682689666748, + "step": 1241 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 320.46875, + "completions/mean_terminated_length": 293.1071472167969, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.9936, + "grad_norm": 5.585651397705078, + "kl": 2.2373046875, + "learning_rate": 1e-06, + "loss": 0.0234, + "num_tokens": 16314094.0, + "reward": 5.1142578125, + "reward_std": 4.644550323486328, + "rewards/rm_reward_func/mean": 5.1142578125, + "rewards/rm_reward_func/std": 6.202043056488037, + "step": 1242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 260.65625, + "completions/mean_terminated_length": 202.6538543701172, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.9944, + "grad_norm": 38.17617416381836, + "kl": 7.8544921875, + "learning_rate": 1e-06, + "loss": 0.3338, + "num_tokens": 16327371.0, + "reward": -8.823577880859375, + "reward_std": 5.635340213775635, + "rewards/rm_reward_func/mean": -8.823577880859375, + "rewards/rm_reward_func/std": 15.1368989944458, + "step": 1243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 302.96875, + "completions/mean_terminated_length": 281.3448181152344, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.9952, + "grad_norm": 13.918490409851074, + "kl": 5.79296875, + "learning_rate": 1e-06, + "loss": 0.1535, + "num_tokens": 16340482.0, + "reward": -7.033935546875, + "reward_std": 8.026933670043945, + "rewards/rm_reward_func/mean": -7.033935546875, + "rewards/rm_reward_func/std": 13.346774101257324, + "step": 1244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 512.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 217.5625, + "completions/mean_terminated_length": 149.61538696289062, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.996, + "grad_norm": 5.955099105834961, + "kl": 1.0595703125, + "learning_rate": 1e-06, + "loss": -0.0099, + "num_tokens": 16350628.0, + "reward": 6.19781494140625, + "reward_std": 5.490054130554199, + "rewards/rm_reward_func/mean": 6.19781494140625, + "rewards/rm_reward_func/std": 7.488785266876221, + "step": 1245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 512.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 313.625, + "completions/mean_terminated_length": 293.10345458984375, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9968, + "grad_norm": 16.472368240356445, + "kl": 6.9775390625, + "learning_rate": 1e-06, + "loss": 0.2136, + "num_tokens": 16364000.0, + "reward": -2.907135009765625, + "reward_std": 5.292792320251465, + "rewards/rm_reward_func/mean": -2.907135009765625, + "rewards/rm_reward_func/std": 9.868986129760742, + "step": 1246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 282.6875, + "completions/mean_terminated_length": 275.2903137207031, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.9976, + "grad_norm": 41.74302673339844, + "kl": 11.40576171875, + "learning_rate": 1e-06, + "loss": 0.2963, + "num_tokens": 16375222.0, + "reward": -6.39111328125, + "reward_std": 3.2877705097198486, + "rewards/rm_reward_func/mean": -6.39111328125, + "rewards/rm_reward_func/std": 19.594057083129883, + "step": 1247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/max_terminated_length": 473.0, + "completions/mean_length": 304.3125, + "completions/mean_terminated_length": 235.08334350585938, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.9984, + "grad_norm": 23.52429962158203, + "kl": 5.68408203125, + "learning_rate": 1e-06, + "loss": -0.0276, + "num_tokens": 16387192.0, + "reward": -7.197265625, + "reward_std": 10.437725067138672, + "rewards/rm_reward_func/mean": -7.197265625, + "rewards/rm_reward_func/std": 14.508234024047852, + "step": 1248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 441.0, + "completions/max_terminated_length": 441.0, + "completions/mean_length": 246.1875, + "completions/mean_terminated_length": 246.1875, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9992, + "grad_norm": 11.459190368652344, + "kl": 6.080078125, + "learning_rate": 1e-06, + "loss": 0.1161, + "num_tokens": 16397814.0, + "reward": -1.294921875, + "reward_std": 7.976162433624268, + "rewards/rm_reward_func/mean": -1.294921875, + "rewards/rm_reward_func/std": 13.71561336517334, + "step": 1249 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 485.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 347.75, + "completions/mean_terminated_length": 347.75, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 1.0, + "grad_norm": 22.80115509033203, + "kl": 4.734375, + "learning_rate": 1e-06, + "loss": 0.0654, + "num_tokens": 16412325.0, + "reward": -4.2939453125, + "reward_std": 7.761114120483398, + "rewards/rm_reward_func/mean": -4.2939453125, + "rewards/rm_reward_func/std": 18.258264541625977, + "step": 1250 + } + ], + "logging_steps": 1, + "max_steps": 1250, + "num_input_tokens_seen": 16412325, + "num_train_epochs": 1, + "save_steps": 250, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..d32502c --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83086064ee0b1d1d04a01a560fdf5ef630cff2e808cdb7841467e8d44d0f2ec0 +size 8913 diff --git a/zero_to_fp32.py b/zero_to_fp32.py new file mode 100644 index 0000000..0e75914 --- /dev/null +++ b/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters)