From ff148490b3b1535e2b7bd9560b16bdbb4535834f Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Wed, 20 May 2026 14:34:40 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-5 Source: Original Platform --- .gitattributes | 36 + README.md | 62 + all_results.json | 9 + config.json | 29 + generation_config.json | 9 + model-00001-of-00007.safetensors | 3 + model-00002-of-00007.safetensors | 3 + model-00003-of-00007.safetensors | 3 + model-00004-of-00007.safetensors | 3 + model-00005-of-00007.safetensors | 3 + model-00006-of-00007.safetensors | 3 + model-00007-of-00007.safetensors | 3 + model.safetensors.index.json | 298 + special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2064 ++++ train_results.json | 9 + trainer_state.json | 15706 +++++++++++++++++++++++++++++ 18 files changed, 18269 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00007.safetensors create mode 100644 model-00002-of-00007.safetensors create mode 100644 model-00003-of-00007.safetensors create mode 100644 model-00004-of-00007.safetensors create mode 100644 model-00005-of-00007.safetensors create mode 100644 model-00006-of-00007.safetensors create mode 100644 model-00007-of-00007.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..a3d2b4e --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-helpful-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-5 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-5 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-helpful-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-helpful-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..8e88355 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.3263754091949014, + "train_runtime": 1741.0925, + "train_samples": 43598, + "train_samples_per_second": 25.041, + "train_steps_per_second": 0.391 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..b1bfc50 --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:313469eed36e349a16d519f5c4d1d53fbd70504a59b14b351df3750b44edb1d9 +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..b7bd9ef --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ceca40be7c52a5caaf6599cf887b355dcfd61c9b86c1e13f423546e4869e012 +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..a5c55cd --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57670a58373971c8b499694a29f67e7d89f22aa6da40cfb442052626a881a17f +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..e19974c --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c23a915a27c9519524037a44ca56f2a67134c6c27f75b590d931d54fc9b62bc +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..a6185fd --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fa69667af5d661d81711efc64e90e769621c6fb4cb1d39f5873c54662ea4a83 +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..4e11081 --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b088390e594b66ba4b56116683240550a3666d0fb0eff379aa3890e698f484c +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..6c3ad15 --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6e08b340db745c971e3b98aa78d84096d5abeb37a4e5a9faa125cc0984b767b +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..8e88355 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.3263754091949014, + "train_runtime": 1741.0925, + "train_samples": 43598, + "train_samples_per_second": 25.041, + "train_steps_per_second": 0.391 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..71ea5e9 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 200, + "global_step": 681, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.00527191162109375, + "KL/mean": 0.016706019639968872, + "KL/rejected_KL_mean": 0.028141021728515625, + "KL/std": 0.272699236869812, + "epoch": 0.0014684287812041115, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02287006378173828, + "fcm_dpo/q_t": 0.5005706548690796, + "grad_norm": 83.51212310791016, + "learning_rate": 0.0, + "logits/chosen": -0.4974287748336792, + "logits/rejected": -0.43299180269241333, + "logps/chosen": -50.1435661315918, + "logps/ref_chosen": -50.14883804321289, + "logps/ref_rejected": -74.1280517578125, + "logps/rejected": -74.09991455078125, + "loss": 1.389, + "margin_dpo/margin_mean": -0.02287048101425171, + "margin_dpo/margin_std": 0.41920793056488037, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.03498649597167969, + "KL/mean": -0.00212840735912323, + "KL/rejected_KL_mean": 0.030735015869140625, + "KL/std": 0.24797174334526062, + "epoch": 0.002936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06572261452674866, + "fcm_dpo/q_t": 0.5016425848007202, + "grad_norm": 72.20186614990234, + "learning_rate": 7.246376811594203e-09, + "logits/chosen": -0.49536412954330444, + "logits/rejected": -0.4594460427761078, + "logps/chosen": -52.65568923950195, + "logps/ref_chosen": -52.620704650878906, + "logps/ref_rejected": -75.30413818359375, + "logps/rejected": -75.27340698242188, + "loss": 1.3932, + "margin_dpo/margin_mean": -0.06572240591049194, + "margin_dpo/margin_std": 0.35048407316207886, + "step": 2 + }, + { + "KL/chosen_KL_mean": -0.00826263427734375, + "KL/mean": 0.004124656319618225, + "KL/rejected_KL_mean": 0.016513824462890625, + "KL/std": 0.2530524432659149, + "epoch": 0.004405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02477547526359558, + "fcm_dpo/q_t": 0.5006192922592163, + "grad_norm": 70.89527130126953, + "learning_rate": 1.4492753623188406e-08, + "logits/chosen": -0.4815559387207031, + "logits/rejected": -0.4421369135379791, + "logps/chosen": -60.98986053466797, + "logps/ref_chosen": -60.981597900390625, + "logps/ref_rejected": -68.67259216308594, + "logps/rejected": -68.65608215332031, + "loss": 1.3891, + "margin_dpo/margin_mean": -0.02477523684501648, + "margin_dpo/margin_std": 0.3734402656555176, + "step": 3 + }, + { + "KL/chosen_KL_mean": -0.02429962158203125, + "KL/mean": 0.015571162104606628, + "KL/rejected_KL_mean": 0.05544281005859375, + "KL/std": 0.252570241689682, + "epoch": 0.005873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.07974502444267273, + "fcm_dpo/q_t": 0.50199294090271, + "grad_norm": 72.09413146972656, + "learning_rate": 2.1739130434782606e-08, + "logits/chosen": -0.4686494469642639, + "logits/rejected": -0.44108062982559204, + "logps/chosen": -56.79201126098633, + "logps/ref_chosen": -56.7677116394043, + "logps/ref_rejected": -86.64710998535156, + "logps/rejected": -86.59166717529297, + "loss": 1.3946, + "margin_dpo/margin_mean": -0.079745352268219, + "margin_dpo/margin_std": 0.35485363006591797, + "step": 4 + }, + { + "KL/chosen_KL_mean": 0.0070819854736328125, + "KL/mean": 0.02031552791595459, + "KL/rejected_KL_mean": 0.0335540771484375, + "KL/std": 0.24011048674583435, + "epoch": 0.007342143906020558, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0264662504196167, + "fcm_dpo/q_t": 0.5006614923477173, + "grad_norm": 89.8433609008789, + "learning_rate": 2.898550724637681e-08, + "logits/chosen": -0.5143770575523376, + "logits/rejected": -0.47061315178871155, + "logps/chosen": -53.852294921875, + "logps/ref_chosen": -53.859375, + "logps/ref_rejected": -84.14918518066406, + "logps/rejected": -84.11563110351562, + "loss": 1.3892, + "margin_dpo/margin_mean": -0.026465564966201782, + "margin_dpo/margin_std": 0.3306843340396881, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.03646278381347656, + "KL/mean": -0.02894607186317444, + "KL/rejected_KL_mean": -0.021427154541015625, + "KL/std": 0.2451232373714447, + "epoch": 0.00881057268722467, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.015035033226013184, + "fcm_dpo/q_t": 0.5003755688667297, + "grad_norm": 92.09574127197266, + "learning_rate": 3.6231884057971014e-08, + "logits/chosen": -0.522139310836792, + "logits/rejected": -0.4815322160720825, + "logps/chosen": -63.0439453125, + "logps/ref_chosen": -63.007484436035156, + "logps/ref_rejected": -92.64534759521484, + "logps/rejected": -92.66677856445312, + "loss": 1.3881, + "margin_dpo/margin_mean": -0.015034288167953491, + "margin_dpo/margin_std": 0.3625510036945343, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.0068035125732421875, + "KL/mean": 0.020816445350646973, + "KL/rejected_KL_mean": 0.03482818603515625, + "KL/std": 0.2857266068458557, + "epoch": 0.010279001468428781, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.028023719787597656, + "fcm_dpo/q_t": 0.5007004141807556, + "grad_norm": 82.25199890136719, + "learning_rate": 4.347826086956521e-08, + "logits/chosen": -0.5038982629776001, + "logits/rejected": -0.4701980650424957, + "logps/chosen": -57.76801681518555, + "logps/ref_chosen": -57.774818420410156, + "logps/ref_rejected": -103.92059326171875, + "logps/rejected": -103.8857650756836, + "loss": 1.3896, + "margin_dpo/margin_mean": -0.02802315354347229, + "margin_dpo/margin_std": 0.42090797424316406, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.03360557556152344, + "KL/mean": 0.027956083416938782, + "KL/rejected_KL_mean": 0.022308349609375, + "KL/std": 0.2712041437625885, + "epoch": 0.011747430249632892, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.011296361684799194, + "fcm_dpo/q_t": 0.4997172951698303, + "grad_norm": 78.32466888427734, + "learning_rate": 5.0724637681159424e-08, + "logits/chosen": -0.5075632333755493, + "logits/rejected": -0.4815378189086914, + "logps/chosen": -58.682430267333984, + "logps/ref_chosen": -58.716033935546875, + "logps/ref_rejected": -79.3114242553711, + "logps/rejected": -79.28911590576172, + "loss": 1.3855, + "margin_dpo/margin_mean": 0.01129680871963501, + "margin_dpo/margin_std": 0.3612467646598816, + "step": 8 + }, + { + "KL/chosen_KL_mean": -0.0057525634765625, + "KL/mean": -0.018807604908943176, + "KL/rejected_KL_mean": -0.031864166259765625, + "KL/std": 0.26702290773391724, + "epoch": 0.013215859030837005, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.026109427213668823, + "fcm_dpo/q_t": 0.499347448348999, + "grad_norm": 84.89402770996094, + "learning_rate": 5.797101449275362e-08, + "logits/chosen": -0.5175442099571228, + "logits/rejected": -0.4723869264125824, + "logps/chosen": -69.87259674072266, + "logps/ref_chosen": -69.8668441772461, + "logps/ref_rejected": -99.6026611328125, + "logps/rejected": -99.63452911376953, + "loss": 1.384, + "margin_dpo/margin_mean": 0.02610918879508972, + "margin_dpo/margin_std": 0.359569787979126, + "step": 9 + }, + { + "KL/chosen_KL_mean": 0.0016269683837890625, + "KL/mean": 0.005393713712692261, + "KL/rejected_KL_mean": 0.00916290283203125, + "KL/std": 0.2506978213787079, + "epoch": 0.014684287812041116, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.007536411285400391, + "fcm_dpo/q_t": 0.5001886487007141, + "grad_norm": 70.9793930053711, + "learning_rate": 6.521739130434782e-08, + "logits/chosen": -0.4903263449668884, + "logits/rejected": -0.446901798248291, + "logps/chosen": -48.356056213378906, + "logps/ref_chosen": -48.35768508911133, + "logps/ref_rejected": -80.37206268310547, + "logps/rejected": -80.36289978027344, + "loss": 1.3874, + "margin_dpo/margin_mean": -0.007535874843597412, + "margin_dpo/margin_std": 0.35438239574432373, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.045078277587890625, + "KL/mean": 0.014331206679344177, + "KL/rejected_KL_mean": -0.016414642333984375, + "KL/std": 0.236942321062088, + "epoch": 0.016152716593245228, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06148645281791687, + "fcm_dpo/q_t": 0.49846333265304565, + "grad_norm": 68.05949401855469, + "learning_rate": 7.246376811594203e-08, + "logits/chosen": -0.45714545249938965, + "logits/rejected": -0.43192940950393677, + "logps/chosen": -52.971778869628906, + "logps/ref_chosen": -53.01685333251953, + "logps/ref_rejected": -87.78038024902344, + "logps/rejected": -87.79679107666016, + "loss": 1.3804, + "margin_dpo/margin_mean": 0.06148698925971985, + "margin_dpo/margin_std": 0.33037588000297546, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.022512435913085938, + "KL/mean": -0.03226548433303833, + "KL/rejected_KL_mean": -0.04201507568359375, + "KL/std": 0.2767062187194824, + "epoch": 0.01762114537444934, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.019503533840179443, + "fcm_dpo/q_t": 0.4995127022266388, + "grad_norm": 96.18541717529297, + "learning_rate": 7.971014492753623e-08, + "logits/chosen": -0.5195867419242859, + "logits/rejected": -0.4828483462333679, + "logps/chosen": -61.82794952392578, + "logps/ref_chosen": -61.80543518066406, + "logps/ref_rejected": -104.8582763671875, + "logps/rejected": -104.90028381347656, + "loss": 1.3846, + "margin_dpo/margin_mean": 0.019502699375152588, + "margin_dpo/margin_std": 0.3449036180973053, + "step": 12 + }, + { + "KL/chosen_KL_mean": -0.00475311279296875, + "KL/mean": -0.012506544589996338, + "KL/rejected_KL_mean": -0.020259857177734375, + "KL/std": 0.26460498571395874, + "epoch": 0.01908957415565345, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.015512943267822266, + "fcm_dpo/q_t": 0.49961256980895996, + "grad_norm": 79.2775650024414, + "learning_rate": 8.695652173913042e-08, + "logits/chosen": -0.48537132143974304, + "logits/rejected": -0.45758724212646484, + "logps/chosen": -64.26510620117188, + "logps/ref_chosen": -64.2603530883789, + "logps/ref_rejected": -87.20307922363281, + "logps/rejected": -87.22334289550781, + "loss": 1.3851, + "margin_dpo/margin_mean": 0.015512913465499878, + "margin_dpo/margin_std": 0.339786171913147, + "step": 13 + }, + { + "KL/chosen_KL_mean": -0.0053424835205078125, + "KL/mean": -0.026964426040649414, + "KL/rejected_KL_mean": -0.04859161376953125, + "KL/std": 0.2796820104122162, + "epoch": 0.020558002936857563, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04325270652770996, + "fcm_dpo/q_t": 0.49891969561576843, + "grad_norm": 85.09868621826172, + "learning_rate": 9.420289855072464e-08, + "logits/chosen": -0.49180495738983154, + "logits/rejected": -0.45296943187713623, + "logps/chosen": -58.11555480957031, + "logps/ref_chosen": -58.11021041870117, + "logps/ref_rejected": -104.04708099365234, + "logps/rejected": -104.09567260742188, + "loss": 1.3825, + "margin_dpo/margin_mean": 0.043252259492874146, + "margin_dpo/margin_std": 0.44487419724464417, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.014326095581054688, + "KL/mean": -0.034481316804885864, + "KL/rejected_KL_mean": -0.054637908935546875, + "KL/std": 0.24156004190444946, + "epoch": 0.022026431718061675, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04031383991241455, + "fcm_dpo/q_t": 0.4989916980266571, + "grad_norm": 64.1568374633789, + "learning_rate": 1.0144927536231885e-07, + "logits/chosen": -0.5022023916244507, + "logits/rejected": -0.4839800298213959, + "logps/chosen": -56.98123550415039, + "logps/ref_chosen": -56.96691131591797, + "logps/ref_rejected": -80.80863952636719, + "logps/rejected": -80.86328125, + "loss": 1.3826, + "margin_dpo/margin_mean": 0.04031360149383545, + "margin_dpo/margin_std": 0.3656350076198578, + "step": 15 + }, + { + "KL/chosen_KL_mean": 0.029695510864257812, + "KL/mean": -0.008761048316955566, + "KL/rejected_KL_mean": -0.047214508056640625, + "KL/std": 0.2751133441925049, + "epoch": 0.023494860499265784, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.07690972089767456, + "fcm_dpo/q_t": 0.49807754158973694, + "grad_norm": 83.51917266845703, + "learning_rate": 1.0869565217391303e-07, + "logits/chosen": -0.5214604139328003, + "logits/rejected": -0.47995007038116455, + "logps/chosen": -61.71019744873047, + "logps/ref_chosen": -61.739891052246094, + "logps/ref_rejected": -84.36947631835938, + "logps/rejected": -84.41668701171875, + "loss": 1.3791, + "margin_dpo/margin_mean": 0.07690989971160889, + "margin_dpo/margin_std": 0.42422112822532654, + "step": 16 + }, + { + "KL/chosen_KL_mean": 0.06725311279296875, + "KL/mean": 0.007791772484779358, + "KL/rejected_KL_mean": -0.051666259765625, + "KL/std": 0.29778245091438293, + "epoch": 0.024963289280469897, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1189166009426117, + "fcm_dpo/q_t": 0.4970288872718811, + "grad_norm": 78.13703155517578, + "learning_rate": 1.1594202898550725e-07, + "logits/chosen": -0.5090387463569641, + "logits/rejected": -0.4729403853416443, + "logps/chosen": -67.64308166503906, + "logps/ref_chosen": -67.71033477783203, + "logps/ref_rejected": -85.37865447998047, + "logps/rejected": -85.4303207397461, + "loss": 1.3749, + "margin_dpo/margin_mean": 0.11891645193099976, + "margin_dpo/margin_std": 0.415335476398468, + "step": 17 + }, + { + "KL/chosen_KL_mean": -0.0026035308837890625, + "KL/mean": -0.012814626097679138, + "KL/rejected_KL_mean": -0.0230255126953125, + "KL/std": 0.2600899636745453, + "epoch": 0.02643171806167401, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.020421653985977173, + "fcm_dpo/q_t": 0.4994891285896301, + "grad_norm": 82.34599304199219, + "learning_rate": 1.2318840579710146e-07, + "logits/chosen": -0.48533254861831665, + "logits/rejected": -0.4290851950645447, + "logps/chosen": -47.742095947265625, + "logps/ref_chosen": -47.7394905090332, + "logps/ref_rejected": -75.4722900390625, + "logps/rejected": -75.49531555175781, + "loss": 1.3846, + "margin_dpo/margin_mean": 0.020421594381332397, + "margin_dpo/margin_std": 0.32879841327667236, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.024383544921875, + "KL/mean": 0.0039770156145095825, + "KL/rejected_KL_mean": -0.01642608642578125, + "KL/std": 0.23556864261627197, + "epoch": 0.027900146842878122, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.040806323289871216, + "fcm_dpo/q_t": 0.49897968769073486, + "grad_norm": 74.32623291015625, + "learning_rate": 1.3043478260869563e-07, + "logits/chosen": -0.4715813398361206, + "logits/rejected": -0.4200263023376465, + "logps/chosen": -70.18097686767578, + "logps/ref_chosen": -70.20536041259766, + "logps/ref_rejected": -89.7575912475586, + "logps/rejected": -89.77401733398438, + "loss": 1.3825, + "margin_dpo/margin_mean": 0.0408073365688324, + "margin_dpo/margin_std": 0.30957865715026855, + "step": 19 + }, + { + "KL/chosen_KL_mean": -0.03499412536621094, + "KL/mean": -0.04548324644565582, + "KL/rejected_KL_mean": -0.05596923828125, + "KL/std": 0.2744811177253723, + "epoch": 0.02936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.020983070135116577, + "fcm_dpo/q_t": 0.4994752109050751, + "grad_norm": 74.43611145019531, + "learning_rate": 1.3768115942028986e-07, + "logits/chosen": -0.5649498105049133, + "logits/rejected": -0.5099019408226013, + "logps/chosen": -50.838233947753906, + "logps/ref_chosen": -50.80324172973633, + "logps/ref_rejected": -78.82334899902344, + "logps/rejected": -78.87931823730469, + "loss": 1.3846, + "margin_dpo/margin_mean": 0.020983099937438965, + "margin_dpo/margin_std": 0.3876810669898987, + "step": 20 + }, + { + "KL/chosen_KL_mean": 0.050243377685546875, + "KL/mean": -0.018013283610343933, + "KL/rejected_KL_mean": -0.08626937866210938, + "KL/std": 0.3195314407348633, + "epoch": 0.030837004405286344, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1365136206150055, + "fcm_dpo/q_t": 0.49658873677253723, + "grad_norm": 77.88671875, + "learning_rate": 1.4492753623188405e-07, + "logits/chosen": -0.5110548138618469, + "logits/rejected": -0.48853152990341187, + "logps/chosen": -50.01277160644531, + "logps/ref_chosen": -50.063018798828125, + "logps/ref_rejected": -77.86878967285156, + "logps/rejected": -77.95506286621094, + "loss": 1.3731, + "margin_dpo/margin_mean": 0.13651368021965027, + "margin_dpo/margin_std": 0.4034786820411682, + "step": 21 + }, + { + "KL/chosen_KL_mean": -0.012243270874023438, + "KL/mean": -0.08835446834564209, + "KL/rejected_KL_mean": -0.16446685791015625, + "KL/std": 0.28480368852615356, + "epoch": 0.032305433186490456, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1522234082221985, + "fcm_dpo/q_t": 0.49619609117507935, + "grad_norm": 84.3863525390625, + "learning_rate": 1.5217391304347825e-07, + "logits/chosen": -0.505510151386261, + "logits/rejected": -0.462830126285553, + "logps/chosen": -59.06987762451172, + "logps/ref_chosen": -59.05763626098633, + "logps/ref_rejected": -97.50466918945312, + "logps/rejected": -97.66914367675781, + "loss": 1.3716, + "margin_dpo/margin_mean": 0.1522233486175537, + "margin_dpo/margin_std": 0.4398460388183594, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.016481399536132812, + "KL/mean": -0.06710737943649292, + "KL/rejected_KL_mean": -0.1506977081298828, + "KL/std": 0.340972900390625, + "epoch": 0.033773861967694566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.16717880964279175, + "fcm_dpo/q_t": 0.49582529067993164, + "grad_norm": 80.40861511230469, + "learning_rate": 1.5942028985507245e-07, + "logits/chosen": -0.5021919012069702, + "logits/rejected": -0.48034122586250305, + "logps/chosen": -60.061214447021484, + "logps/ref_chosen": -60.07769775390625, + "logps/ref_rejected": -81.13955688476562, + "logps/rejected": -81.29025268554688, + "loss": 1.3703, + "margin_dpo/margin_mean": 0.16717904806137085, + "margin_dpo/margin_std": 0.5109284520149231, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.0286712646484375, + "KL/mean": -0.07983805984258652, + "KL/rejected_KL_mean": -0.18835067749023438, + "KL/std": 0.29715216159820557, + "epoch": 0.03524229074889868, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2170248180627823, + "fcm_dpo/q_t": 0.49457818269729614, + "grad_norm": 83.2780990600586, + "learning_rate": 1.6666666666666665e-07, + "logits/chosen": -0.5041535496711731, + "logits/rejected": -0.48770707845687866, + "logps/chosen": -44.26236343383789, + "logps/ref_chosen": -44.29103469848633, + "logps/ref_rejected": -99.12521362304688, + "logps/rejected": -99.31356811523438, + "loss": 1.3652, + "margin_dpo/margin_mean": 0.21702483296394348, + "margin_dpo/margin_std": 0.4385562539100647, + "step": 24 + }, + { + "KL/chosen_KL_mean": 0.02964019775390625, + "KL/mean": -0.08741536736488342, + "KL/rejected_KL_mean": -0.2044677734375, + "KL/std": 0.34008604288101196, + "epoch": 0.03671071953010279, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.23411154747009277, + "fcm_dpo/q_t": 0.4941510558128357, + "grad_norm": 74.14248657226562, + "learning_rate": 1.7391304347826085e-07, + "logits/chosen": -0.4980379641056061, + "logits/rejected": -0.4681099057197571, + "logps/chosen": -52.50741195678711, + "logps/ref_chosen": -52.537052154541016, + "logps/ref_rejected": -89.34219360351562, + "logps/rejected": -89.54666137695312, + "loss": 1.3635, + "margin_dpo/margin_mean": 0.234111487865448, + "margin_dpo/margin_std": 0.41896557807922363, + "step": 25 + }, + { + "KL/chosen_KL_mean": 0.05152320861816406, + "KL/mean": -0.12674298882484436, + "KL/rejected_KL_mean": -0.3050079345703125, + "KL/std": 0.4253169298171997, + "epoch": 0.0381791483113069, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3565349578857422, + "fcm_dpo/q_t": 0.49109458923339844, + "grad_norm": 87.5750732421875, + "learning_rate": 1.8115942028985507e-07, + "logits/chosen": -0.5453453063964844, + "logits/rejected": -0.513872504234314, + "logps/chosen": -53.87128448486328, + "logps/ref_chosen": -53.92280578613281, + "logps/ref_rejected": -103.35971069335938, + "logps/rejected": -103.66471862792969, + "loss": 1.3516, + "margin_dpo/margin_mean": 0.3565359115600586, + "margin_dpo/margin_std": 0.4943789839744568, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.11599159240722656, + "KL/mean": -0.13826510310173035, + "KL/rejected_KL_mean": -0.39252471923828125, + "KL/std": 0.476246178150177, + "epoch": 0.039647577092511016, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5085150599479675, + "fcm_dpo/q_t": 0.48730653524398804, + "grad_norm": 93.57472229003906, + "learning_rate": 1.8840579710144927e-07, + "logits/chosen": -0.5514802932739258, + "logits/rejected": -0.5156171321868896, + "logps/chosen": -42.78253936767578, + "logps/ref_chosen": -42.898529052734375, + "logps/ref_rejected": -98.72419738769531, + "logps/rejected": -99.11672973632812, + "loss": 1.3369, + "margin_dpo/margin_mean": 0.5085144639015198, + "margin_dpo/margin_std": 0.5824718475341797, + "step": 27 + }, + { + "KL/chosen_KL_mean": 0.011579513549804688, + "KL/mean": -0.13822337985038757, + "KL/rejected_KL_mean": -0.2880287170410156, + "KL/std": 0.42481285333633423, + "epoch": 0.041116005873715125, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2996072769165039, + "fcm_dpo/q_t": 0.49251770973205566, + "grad_norm": 75.85013580322266, + "learning_rate": 1.9565217391304347e-07, + "logits/chosen": -0.497283399105072, + "logits/rejected": -0.4410708546638489, + "logps/chosen": -60.544921875, + "logps/ref_chosen": -60.55650329589844, + "logps/ref_rejected": -91.40111541748047, + "logps/rejected": -91.68914794921875, + "loss": 1.3574, + "margin_dpo/margin_mean": 0.2996074855327606, + "margin_dpo/margin_std": 0.562934160232544, + "step": 28 + }, + { + "KL/chosen_KL_mean": 0.12349700927734375, + "KL/mean": -0.16272687911987305, + "KL/rejected_KL_mean": -0.4489555358886719, + "KL/std": 0.4961986243724823, + "epoch": 0.042584434654919234, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5724496841430664, + "fcm_dpo/q_t": 0.4857064485549927, + "grad_norm": 90.49112701416016, + "learning_rate": 2.028985507246377e-07, + "logits/chosen": -0.5566602945327759, + "logits/rejected": -0.5104827880859375, + "logps/chosen": -57.684288024902344, + "logps/ref_chosen": -57.80778503417969, + "logps/ref_rejected": -97.39434814453125, + "logps/rejected": -97.84330749511719, + "loss": 1.3307, + "margin_dpo/margin_mean": 0.5724506378173828, + "margin_dpo/margin_std": 0.5618862509727478, + "step": 29 + }, + { + "KL/chosen_KL_mean": 0.11771583557128906, + "KL/mean": -0.2122490406036377, + "KL/rejected_KL_mean": -0.5422172546386719, + "KL/std": 0.5805710554122925, + "epoch": 0.04405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.659930944442749, + "fcm_dpo/q_t": 0.4835302233695984, + "grad_norm": 87.53767395019531, + "learning_rate": 2.1014492753623187e-07, + "logits/chosen": -0.4962325990200043, + "logits/rejected": -0.4656328558921814, + "logps/chosen": -52.45965576171875, + "logps/ref_chosen": -52.577369689941406, + "logps/ref_rejected": -98.48920440673828, + "logps/rejected": -99.03141784667969, + "loss": 1.3225, + "margin_dpo/margin_mean": 0.659930944442749, + "margin_dpo/margin_std": 0.6480692625045776, + "step": 30 + }, + { + "KL/chosen_KL_mean": 0.10221672058105469, + "KL/mean": -0.15314733982086182, + "KL/rejected_KL_mean": -0.4085121154785156, + "KL/std": 0.5654332637786865, + "epoch": 0.04552129221732746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.510732889175415, + "fcm_dpo/q_t": 0.48725682497024536, + "grad_norm": 67.43456268310547, + "learning_rate": 2.1739130434782607e-07, + "logits/chosen": -0.49099212884902954, + "logits/rejected": -0.44460946321487427, + "logps/chosen": -63.70470428466797, + "logps/ref_chosen": -63.806922912597656, + "logps/ref_rejected": -72.89400482177734, + "logps/rejected": -73.30252075195312, + "loss": 1.3371, + "margin_dpo/margin_mean": 0.5107325315475464, + "margin_dpo/margin_std": 0.6857659220695496, + "step": 31 + }, + { + "KL/chosen_KL_mean": 0.2014923095703125, + "KL/mean": -0.19277532398700714, + "KL/rejected_KL_mean": -0.5870437622070312, + "KL/std": 0.6862952709197998, + "epoch": 0.04698972099853157, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7885334491729736, + "fcm_dpo/q_t": 0.4803431034088135, + "grad_norm": 80.95833587646484, + "learning_rate": 2.2463768115942027e-07, + "logits/chosen": -0.5271800756454468, + "logits/rejected": -0.4867057204246521, + "logps/chosen": -62.53803253173828, + "logps/ref_chosen": -62.739524841308594, + "logps/ref_rejected": -89.3175048828125, + "logps/rejected": -89.904541015625, + "loss": 1.3106, + "margin_dpo/margin_mean": 0.7885338664054871, + "margin_dpo/margin_std": 0.8146342635154724, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.13482093811035156, + "KL/mean": -0.1859203577041626, + "KL/rejected_KL_mean": -0.5066604614257812, + "KL/std": 0.5469956994056702, + "epoch": 0.048458149779735685, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6414769291877747, + "fcm_dpo/q_t": 0.4839860796928406, + "grad_norm": 72.35101318359375, + "learning_rate": 2.318840579710145e-07, + "logits/chosen": -0.5072885751724243, + "logits/rejected": -0.481620728969574, + "logps/chosen": -53.12615203857422, + "logps/ref_chosen": -53.26097106933594, + "logps/ref_rejected": -87.8851318359375, + "logps/rejected": -88.39179992675781, + "loss": 1.3242, + "margin_dpo/margin_mean": 0.6414770483970642, + "margin_dpo/margin_std": 0.6191001534461975, + "step": 33 + }, + { + "KL/chosen_KL_mean": 0.08164787292480469, + "KL/mean": -0.32943296432495117, + "KL/rejected_KL_mean": -0.7405166625976562, + "KL/std": 0.7699615359306335, + "epoch": 0.049926578560939794, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8221615552902222, + "fcm_dpo/q_t": 0.47951143980026245, + "grad_norm": 77.51274871826172, + "learning_rate": 2.391304347826087e-07, + "logits/chosen": -0.48901137709617615, + "logits/rejected": -0.4716091752052307, + "logps/chosen": -50.735679626464844, + "logps/ref_chosen": -50.81732940673828, + "logps/ref_rejected": -101.92184448242188, + "logps/rejected": -102.66236114501953, + "loss": 1.3079, + "margin_dpo/margin_mean": 0.8221613764762878, + "margin_dpo/margin_std": 0.9267702102661133, + "step": 34 + }, + { + "KL/chosen_KL_mean": 0.11441421508789062, + "KL/mean": -0.5115079879760742, + "KL/rejected_KL_mean": -1.1374282836914062, + "KL/std": 1.0209949016571045, + "epoch": 0.0513950073421439, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2518417835235596, + "fcm_dpo/q_t": 0.46891528367996216, + "grad_norm": 82.27989196777344, + "learning_rate": 2.463768115942029e-07, + "logits/chosen": -0.5277581214904785, + "logits/rejected": -0.49088025093078613, + "logps/chosen": -50.91007614135742, + "logps/ref_chosen": -51.02449035644531, + "logps/ref_rejected": -106.82443237304688, + "logps/rejected": -107.96185302734375, + "loss": 1.2684, + "margin_dpo/margin_mean": 1.2518424987792969, + "margin_dpo/margin_std": 1.1583997011184692, + "step": 35 + }, + { + "KL/chosen_KL_mean": 0.054615020751953125, + "KL/mean": -0.5534474849700928, + "KL/rejected_KL_mean": -1.1615104675292969, + "KL/std": 1.1053463220596313, + "epoch": 0.05286343612334802, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2161287069320679, + "fcm_dpo/q_t": 0.469775915145874, + "grad_norm": 72.23912048339844, + "learning_rate": 2.536231884057971e-07, + "logits/chosen": -0.5657041072845459, + "logits/rejected": -0.5297294855117798, + "logps/chosen": -51.93687438964844, + "logps/ref_chosen": -51.991493225097656, + "logps/ref_rejected": -86.0406265258789, + "logps/rejected": -87.20213317871094, + "loss": 1.2726, + "margin_dpo/margin_mean": 1.2161275148391724, + "margin_dpo/margin_std": 1.2665972709655762, + "step": 36 + }, + { + "KL/chosen_KL_mean": 0.014173507690429688, + "KL/mean": -0.5154328942298889, + "KL/rejected_KL_mean": -1.0450401306152344, + "KL/std": 1.0978405475616455, + "epoch": 0.05433186490455213, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.059217929840088, + "fcm_dpo/q_t": 0.4737340807914734, + "grad_norm": 61.976402282714844, + "learning_rate": 2.6086956521739126e-07, + "logits/chosen": -0.5021863579750061, + "logits/rejected": -0.4578307271003723, + "logps/chosen": -62.79293441772461, + "logps/ref_chosen": -62.807106018066406, + "logps/ref_rejected": -77.89507293701172, + "logps/rejected": -78.94010925292969, + "loss": 1.288, + "margin_dpo/margin_mean": 1.0592180490493774, + "margin_dpo/margin_std": 1.3723002672195435, + "step": 37 + }, + { + "KL/chosen_KL_mean": 0.17836570739746094, + "KL/mean": -0.5307787656784058, + "KL/rejected_KL_mean": -1.2399253845214844, + "KL/std": 1.3880457878112793, + "epoch": 0.055800293685756244, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.418290615081787, + "fcm_dpo/q_t": 0.4649973213672638, + "grad_norm": 69.40721893310547, + "learning_rate": 2.681159420289855e-07, + "logits/chosen": -0.5123181343078613, + "logits/rejected": -0.47957277297973633, + "logps/chosen": -48.212154388427734, + "logps/ref_chosen": -48.39051818847656, + "logps/ref_rejected": -97.91244506835938, + "logps/rejected": -99.1523666381836, + "loss": 1.2564, + "margin_dpo/margin_mean": 1.4182908535003662, + "margin_dpo/margin_std": 1.6710336208343506, + "step": 38 + }, + { + "KL/chosen_KL_mean": 0.11879920959472656, + "KL/mean": -0.7346929311752319, + "KL/rejected_KL_mean": -1.5881805419921875, + "KL/std": 1.3292229175567627, + "epoch": 0.05726872246696035, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.7069776058197021, + "fcm_dpo/q_t": 0.4576621651649475, + "grad_norm": 74.3014907836914, + "learning_rate": 2.753623188405797e-07, + "logits/chosen": -0.5710403919219971, + "logits/rejected": -0.5320708155632019, + "logps/chosen": -50.631675720214844, + "logps/ref_chosen": -50.75047302246094, + "logps/ref_rejected": -78.56951141357422, + "logps/rejected": -80.1576919555664, + "loss": 1.228, + "margin_dpo/margin_mean": 1.7069778442382812, + "margin_dpo/margin_std": 1.4111273288726807, + "step": 39 + }, + { + "KL/chosen_KL_mean": 0.2220172882080078, + "KL/mean": -0.5618829131126404, + "KL/rejected_KL_mean": -1.3457870483398438, + "KL/std": 1.41432523727417, + "epoch": 0.05873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5678019523620605, + "fcm_dpo/q_t": 0.46130138635635376, + "grad_norm": 60.054290771484375, + "learning_rate": 2.8260869565217386e-07, + "logits/chosen": -0.507063627243042, + "logits/rejected": -0.4761931002140045, + "logps/chosen": -57.7630500793457, + "logps/ref_chosen": -57.985069274902344, + "logps/ref_rejected": -74.3000717163086, + "logps/rejected": -75.64585876464844, + "loss": 1.2432, + "margin_dpo/margin_mean": 1.5678024291992188, + "margin_dpo/margin_std": 1.709043264389038, + "step": 40 + }, + { + "KL/chosen_KL_mean": 0.0051975250244140625, + "KL/mean": -0.9326056838035583, + "KL/rejected_KL_mean": -1.8704071044921875, + "KL/std": 1.854329228401184, + "epoch": 0.06020558002936858, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8756054639816284, + "fcm_dpo/q_t": 0.4538491666316986, + "grad_norm": 68.05853271484375, + "learning_rate": 2.898550724637681e-07, + "logits/chosen": -0.540554940700531, + "logits/rejected": -0.5041079521179199, + "logps/chosen": -62.69062042236328, + "logps/ref_chosen": -62.69581604003906, + "logps/ref_rejected": -97.02352905273438, + "logps/rejected": -98.89393615722656, + "loss": 1.2179, + "margin_dpo/margin_mean": 1.8756061792373657, + "margin_dpo/margin_std": 2.0067355632781982, + "step": 41 + }, + { + "KL/chosen_KL_mean": 0.21923828125, + "KL/mean": -1.0403838157653809, + "KL/rejected_KL_mean": -2.300006866455078, + "KL/std": 2.130916118621826, + "epoch": 0.06167400881057269, + "fcm_dpo/beta": 0.1539303958415985, + "fcm_dpo/delta": -0.01988767832517624, + "fcm_dpo/margin": 2.5192437171936035, + "fcm_dpo/q_t": 0.4016793966293335, + "grad_norm": 98.06798553466797, + "learning_rate": 2.971014492753623e-07, + "logits/chosen": -0.5261090993881226, + "logits/rejected": -0.4792720377445221, + "logps/chosen": -58.747188568115234, + "logps/ref_chosen": -58.966426849365234, + "logps/ref_rejected": -109.90837097167969, + "logps/rejected": -112.20838165283203, + "loss": 1.0939, + "margin_dpo/margin_mean": 2.5192432403564453, + "margin_dpo/margin_std": 2.3735432624816895, + "step": 42 + }, + { + "KL/chosen_KL_mean": 0.5401420593261719, + "KL/mean": -0.6934725046157837, + "KL/rejected_KL_mean": -1.927093505859375, + "KL/std": 1.8467273712158203, + "epoch": 0.0631424375917768, + "fcm_dpo/beta": 0.18102575838565826, + "fcm_dpo/delta": 0.06920016556978226, + "fcm_dpo/margin": 2.4672341346740723, + "fcm_dpo/q_t": 0.4202408194541931, + "grad_norm": 109.3116455078125, + "learning_rate": 3.043478260869565e-07, + "logits/chosen": -0.540740966796875, + "logits/rejected": -0.5158405303955078, + "logps/chosen": -53.61585235595703, + "logps/ref_chosen": -54.15599822998047, + "logps/ref_rejected": -96.48019409179688, + "logps/rejected": -98.40728759765625, + "loss": 1.0181, + "margin_dpo/margin_mean": 2.4672341346740723, + "margin_dpo/margin_std": 1.9857051372528076, + "step": 43 + }, + { + "KL/chosen_KL_mean": 0.1969776153564453, + "KL/mean": -1.1819499731063843, + "KL/rejected_KL_mean": -2.5608787536621094, + "KL/std": 2.232034206390381, + "epoch": 0.06461086637298091, + "fcm_dpo/beta": 0.14869627356529236, + "fcm_dpo/delta": 0.008696533739566803, + "fcm_dpo/margin": 2.7578556537628174, + "fcm_dpo/q_t": 0.40634626150131226, + "grad_norm": 103.40689849853516, + "learning_rate": 3.115942028985507e-07, + "logits/chosen": -0.49857497215270996, + "logits/rejected": -0.47800567746162415, + "logps/chosen": -49.88152313232422, + "logps/ref_chosen": -50.07849884033203, + "logps/ref_rejected": -108.78376007080078, + "logps/rejected": -111.34463500976562, + "loss": 1.0466, + "margin_dpo/margin_mean": 2.7578554153442383, + "margin_dpo/margin_std": 2.197627067565918, + "step": 44 + }, + { + "KL/chosen_KL_mean": 0.1326732635498047, + "KL/mean": -1.014523983001709, + "KL/rejected_KL_mean": -2.1617279052734375, + "KL/std": 2.0959479808807373, + "epoch": 0.06607929515418502, + "fcm_dpo/beta": 0.15769031643867493, + "fcm_dpo/delta": -0.06119377911090851, + "fcm_dpo/margin": 2.2944016456604004, + "fcm_dpo/q_t": 0.39686933159828186, + "grad_norm": 85.24789428710938, + "learning_rate": 3.188405797101449e-07, + "logits/chosen": -0.4741653800010681, + "logits/rejected": -0.46142202615737915, + "logps/chosen": -48.282257080078125, + "logps/ref_chosen": -48.4149284362793, + "logps/ref_rejected": -77.93643188476562, + "logps/rejected": -80.09815979003906, + "loss": 1.1099, + "margin_dpo/margin_mean": 2.2944014072418213, + "margin_dpo/margin_std": 2.6267547607421875, + "step": 45 + }, + { + "KL/chosen_KL_mean": 0.2033252716064453, + "KL/mean": -1.2960678339004517, + "KL/rejected_KL_mean": -2.795459747314453, + "KL/std": 2.6904854774475098, + "epoch": 0.06754772393538913, + "fcm_dpo/beta": 0.1267249435186386, + "fcm_dpo/delta": -0.04247567802667618, + "fcm_dpo/margin": 2.9987845420837402, + "fcm_dpo/q_t": 0.39998096227645874, + "grad_norm": 75.10869598388672, + "learning_rate": 3.260869565217391e-07, + "logits/chosen": -0.5231410264968872, + "logits/rejected": -0.4719325304031372, + "logps/chosen": -55.79610061645508, + "logps/ref_chosen": -55.999427795410156, + "logps/ref_rejected": -95.652587890625, + "logps/rejected": -98.44804382324219, + "loss": 1.1114, + "margin_dpo/margin_mean": 2.9987847805023193, + "margin_dpo/margin_std": 3.3123412132263184, + "step": 46 + }, + { + "KL/chosen_KL_mean": 0.4382801055908203, + "KL/mean": -1.083209753036499, + "KL/rejected_KL_mean": -2.6046981811523438, + "KL/std": 2.6264405250549316, + "epoch": 0.06901615271659324, + "fcm_dpo/beta": 0.10618115961551666, + "fcm_dpo/delta": 0.0855509415268898, + "fcm_dpo/margin": 3.042980670928955, + "fcm_dpo/q_t": 0.452394038438797, + "grad_norm": 67.91606140136719, + "learning_rate": 3.333333333333333e-07, + "logits/chosen": -0.5615835189819336, + "logits/rejected": -0.5081955194473267, + "logps/chosen": -57.48780059814453, + "logps/ref_chosen": -57.92607879638672, + "logps/ref_rejected": -94.67920684814453, + "logps/rejected": -97.28390502929688, + "loss": 1.0978, + "margin_dpo/margin_mean": 3.0429811477661133, + "margin_dpo/margin_std": 2.6995935440063477, + "step": 47 + }, + { + "KL/chosen_KL_mean": 0.11672401428222656, + "KL/mean": -1.4517210721969604, + "KL/rejected_KL_mean": -3.0201644897460938, + "KL/std": 2.6644673347473145, + "epoch": 0.07048458149779736, + "fcm_dpo/beta": 0.07081333547830582, + "fcm_dpo/delta": -0.07440280169248581, + "fcm_dpo/margin": 3.1368894577026367, + "fcm_dpo/q_t": 0.41475388407707214, + "grad_norm": 55.53757095336914, + "learning_rate": 3.4057971014492755e-07, + "logits/chosen": -0.5801494121551514, + "logits/rejected": -0.5212501287460327, + "logps/chosen": -57.07135009765625, + "logps/ref_chosen": -57.188072204589844, + "logps/ref_rejected": -88.0166015625, + "logps/rejected": -91.0367660522461, + "loss": 1.1888, + "margin_dpo/margin_mean": 3.1368894577026367, + "margin_dpo/margin_std": 2.9127197265625, + "step": 48 + }, + { + "KL/chosen_KL_mean": 0.3498554229736328, + "KL/mean": -1.5747132301330566, + "KL/rejected_KL_mean": -3.499279022216797, + "KL/std": 3.408785581588745, + "epoch": 0.07195301027900147, + "fcm_dpo/beta": 0.09813052415847778, + "fcm_dpo/delta": -0.0769207701086998, + "fcm_dpo/margin": 3.8491296768188477, + "fcm_dpo/q_t": 0.3957645893096924, + "grad_norm": 59.666378021240234, + "learning_rate": 3.478260869565217e-07, + "logits/chosen": -0.5373194813728333, + "logits/rejected": -0.47734540700912476, + "logps/chosen": -61.33541488647461, + "logps/ref_chosen": -61.685272216796875, + "logps/ref_rejected": -83.76747131347656, + "logps/rejected": -87.26675415039062, + "loss": 1.1176, + "margin_dpo/margin_mean": 3.8491291999816895, + "margin_dpo/margin_std": 3.9004533290863037, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.15655517578125, + "KL/mean": -2.1865463256835938, + "KL/rejected_KL_mean": -4.216529846191406, + "KL/std": 3.433018445968628, + "epoch": 0.07342143906020558, + "fcm_dpo/beta": 0.03281348943710327, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.059974193572998, + "fcm_dpo/q_t": 0.4669080376625061, + "grad_norm": 26.208717346191406, + "learning_rate": 3.5507246376811595e-07, + "logits/chosen": -0.5582753419876099, + "logits/rejected": -0.522718608379364, + "logps/chosen": -58.88069152832031, + "logps/ref_chosen": -58.72413635253906, + "logps/ref_rejected": -96.35814666748047, + "logps/rejected": -100.57467651367188, + "loss": 1.2617, + "margin_dpo/margin_mean": 4.059974193572998, + "margin_dpo/margin_std": 3.9510397911071777, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.26395416259765625, + "KL/mean": -2.266573429107666, + "KL/rejected_KL_mean": -4.269195556640625, + "KL/std": 4.127612113952637, + "epoch": 0.07488986784140969, + "fcm_dpo/beta": 0.03281348943710327, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.005241870880127, + "fcm_dpo/q_t": 0.46752673387527466, + "grad_norm": 23.76943588256836, + "learning_rate": 3.6231884057971015e-07, + "logits/chosen": -0.5171994566917419, + "logits/rejected": -0.4839722514152527, + "logps/chosen": -61.63762283325195, + "logps/ref_chosen": -61.3736686706543, + "logps/ref_rejected": -76.00199890136719, + "logps/rejected": -80.27119445800781, + "loss": 1.2662, + "margin_dpo/margin_mean": 4.005241394042969, + "margin_dpo/margin_std": 5.0708160400390625, + "step": 51 + }, + { + "KL/chosen_KL_mean": 0.34140777587890625, + "KL/mean": -2.696107864379883, + "KL/rejected_KL_mean": -5.7336273193359375, + "KL/std": 4.7038187980651855, + "epoch": 0.0763582966226138, + "fcm_dpo/beta": 0.03281348943710327, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.075035572052002, + "fcm_dpo/q_t": 0.4507228136062622, + "grad_norm": 28.817222595214844, + "learning_rate": 3.695652173913043e-07, + "logits/chosen": -0.5797228217124939, + "logits/rejected": -0.525948166847229, + "logps/chosen": -51.995948791503906, + "logps/ref_chosen": -52.33735656738281, + "logps/ref_rejected": -79.97391510009766, + "logps/rejected": -85.7075424194336, + "loss": 1.2035, + "margin_dpo/margin_mean": 6.07503604888916, + "margin_dpo/margin_std": 4.994811058044434, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.22272109985351562, + "KL/mean": -3.3271031379699707, + "KL/rejected_KL_mean": -6.431484222412109, + "KL/std": 5.386918544769287, + "epoch": 0.07782672540381791, + "fcm_dpo/beta": 0.05016179382801056, + "fcm_dpo/delta": 0.07214375585317612, + "fcm_dpo/margin": 6.208770275115967, + "fcm_dpo/q_t": 0.4501815438270569, + "grad_norm": 38.668609619140625, + "learning_rate": 3.7681159420289855e-07, + "logits/chosen": -0.6235509514808655, + "logits/rejected": -0.6023184061050415, + "logps/chosen": -53.53737258911133, + "logps/ref_chosen": -53.31465148925781, + "logps/ref_rejected": -91.78359985351562, + "logps/rejected": -98.21508026123047, + "loss": 1.1228, + "margin_dpo/margin_mean": 6.208770275115967, + "margin_dpo/margin_std": 6.1717681884765625, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.36040306091308594, + "KL/mean": -3.0783324241638184, + "KL/rejected_KL_mean": -5.796260833740234, + "KL/std": 5.118019104003906, + "epoch": 0.07929515418502203, + "fcm_dpo/beta": 0.06915527582168579, + "fcm_dpo/delta": -0.035718172788619995, + "fcm_dpo/margin": 5.435856819152832, + "fcm_dpo/q_t": 0.3983124792575836, + "grad_norm": 43.57950210571289, + "learning_rate": 3.8405797101449274e-07, + "logits/chosen": -0.6051186323165894, + "logits/rejected": -0.5521863102912903, + "logps/chosen": -51.04906463623047, + "logps/ref_chosen": -50.68865966796875, + "logps/ref_rejected": -91.71539306640625, + "logps/rejected": -97.51165771484375, + "loss": 1.0854, + "margin_dpo/margin_mean": 5.435855865478516, + "margin_dpo/margin_std": 5.120929718017578, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.9818630218505859, + "KL/mean": -4.163016319274902, + "KL/rejected_KL_mean": -7.344165802001953, + "KL/std": 6.130636215209961, + "epoch": 0.08076358296622614, + "fcm_dpo/beta": 0.05893651396036148, + "fcm_dpo/delta": -0.0688391923904419, + "fcm_dpo/margin": 6.362303733825684, + "fcm_dpo/q_t": 0.3950488865375519, + "grad_norm": 41.72751998901367, + "learning_rate": 3.9130434782608694e-07, + "logits/chosen": -0.6603301167488098, + "logits/rejected": -0.5991765260696411, + "logps/chosen": -63.59709930419922, + "logps/ref_chosen": -62.615234375, + "logps/ref_rejected": -88.99349975585938, + "logps/rejected": -96.3376693725586, + "loss": 1.1394, + "margin_dpo/margin_mean": 6.362303733825684, + "margin_dpo/margin_std": 7.427134037017822, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.7029590606689453, + "KL/mean": -3.7865757942199707, + "KL/rejected_KL_mean": -6.870197296142578, + "KL/std": 6.062340259552002, + "epoch": 0.08223201174743025, + "fcm_dpo/beta": 0.02372913435101509, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.167237758636475, + "fcm_dpo/q_t": 0.46382689476013184, + "grad_norm": 18.926555633544922, + "learning_rate": 3.9855072463768114e-07, + "logits/chosen": -0.6277616024017334, + "logits/rejected": -0.5860629677772522, + "logps/chosen": -58.63568878173828, + "logps/ref_chosen": -57.9327278137207, + "logps/ref_rejected": -94.1744384765625, + "logps/rejected": -101.04463195800781, + "loss": 1.2529, + "margin_dpo/margin_mean": 6.167238235473633, + "margin_dpo/margin_std": 7.341178894042969, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.7325782775878906, + "KL/mean": -4.221752166748047, + "KL/rejected_KL_mean": -7.710926055908203, + "KL/std": 6.029708385467529, + "epoch": 0.08370044052863436, + "fcm_dpo/beta": 0.02372913435101509, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.978342056274414, + "fcm_dpo/q_t": 0.458979070186615, + "grad_norm": 21.024066925048828, + "learning_rate": 4.057971014492754e-07, + "logits/chosen": -0.6242918968200684, + "logits/rejected": -0.5983684062957764, + "logps/chosen": -71.22785949707031, + "logps/ref_chosen": -70.49528503417969, + "logps/ref_rejected": -95.56546020507812, + "logps/rejected": -103.27638244628906, + "loss": 1.2339, + "margin_dpo/margin_mean": 6.978342533111572, + "margin_dpo/margin_std": 6.642031669616699, + "step": 57 + }, + { + "KL/chosen_KL_mean": -1.0038623809814453, + "KL/mean": -4.862548351287842, + "KL/rejected_KL_mean": -8.721233367919922, + "KL/std": 6.818406105041504, + "epoch": 0.08516886930983847, + "fcm_dpo/beta": 0.04264631122350693, + "fcm_dpo/delta": 0.09533653408288956, + "fcm_dpo/margin": 7.717365264892578, + "fcm_dpo/q_t": 0.4548690915107727, + "grad_norm": 33.23288345336914, + "learning_rate": 4.1304347826086954e-07, + "logits/chosen": -0.6018198132514954, + "logits/rejected": -0.5264607071876526, + "logps/chosen": -63.136802673339844, + "logps/ref_chosen": -62.13294219970703, + "logps/ref_rejected": -84.61729431152344, + "logps/rejected": -93.33853149414062, + "loss": 1.108, + "margin_dpo/margin_mean": 7.7173662185668945, + "margin_dpo/margin_std": 8.156654357910156, + "step": 58 + }, + { + "KL/chosen_KL_mean": -1.4942893981933594, + "KL/mean": -5.754292011260986, + "KL/rejected_KL_mean": -10.014289855957031, + "KL/std": 7.55443000793457, + "epoch": 0.08663729809104258, + "fcm_dpo/beta": 0.01908048428595066, + "fcm_dpo/delta": -0.1171402782201767, + "fcm_dpo/margin": 8.52000617980957, + "fcm_dpo/q_t": 0.41208070516586304, + "grad_norm": 19.520845413208008, + "learning_rate": 4.2028985507246374e-07, + "logits/chosen": -0.6208063364028931, + "logits/rejected": -0.580247163772583, + "logps/chosen": -53.426815032958984, + "logps/ref_chosen": -51.932525634765625, + "logps/ref_rejected": -88.88520050048828, + "logps/rejected": -98.89949035644531, + "loss": 1.2374, + "margin_dpo/margin_mean": 8.520007133483887, + "margin_dpo/margin_std": 8.690988540649414, + "step": 59 + }, + { + "KL/chosen_KL_mean": -2.7874374389648438, + "KL/mean": -6.149212837219238, + "KL/rejected_KL_mean": -9.510990142822266, + "KL/std": 6.844639778137207, + "epoch": 0.0881057268722467, + "fcm_dpo/beta": 0.01908048428595066, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.723544597625732, + "fcm_dpo/q_t": 0.4681331217288971, + "grad_norm": 17.15147590637207, + "learning_rate": 4.2753623188405794e-07, + "logits/chosen": -0.6261638402938843, + "logits/rejected": -0.5673079490661621, + "logps/chosen": -63.7296257019043, + "logps/ref_chosen": -60.94218826293945, + "logps/ref_rejected": -85.39340209960938, + "logps/rejected": -94.90438842773438, + "loss": 1.2677, + "margin_dpo/margin_mean": 6.723544597625732, + "margin_dpo/margin_std": 7.779609680175781, + "step": 60 + }, + { + "KL/chosen_KL_mean": -1.5964698791503906, + "KL/mean": -5.815464973449707, + "KL/rejected_KL_mean": -10.034461975097656, + "KL/std": 9.321011543273926, + "epoch": 0.08957415565345081, + "fcm_dpo/beta": 0.01908048428595066, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 8.437992095947266, + "fcm_dpo/q_t": 0.4607029855251312, + "grad_norm": 17.691282272338867, + "learning_rate": 4.3478260869565214e-07, + "logits/chosen": -0.6190700531005859, + "logits/rejected": -0.585500955581665, + "logps/chosen": -62.22999572753906, + "logps/ref_chosen": -60.633522033691406, + "logps/ref_rejected": -89.85249328613281, + "logps/rejected": -99.886962890625, + "loss": 1.244, + "margin_dpo/margin_mean": 8.437992095947266, + "margin_dpo/margin_std": 11.62002182006836, + "step": 61 + }, + { + "KL/chosen_KL_mean": -1.81982421875, + "KL/mean": -5.002796173095703, + "KL/rejected_KL_mean": -8.185768127441406, + "KL/std": 6.878681182861328, + "epoch": 0.09104258443465492, + "fcm_dpo/beta": 0.01908048428595066, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.365945816040039, + "fcm_dpo/q_t": 0.4699386656284332, + "grad_norm": 14.713135719299316, + "learning_rate": 4.420289855072464e-07, + "logits/chosen": -0.609377384185791, + "logits/rejected": -0.5751521587371826, + "logps/chosen": -57.97059631347656, + "logps/ref_chosen": -56.15077209472656, + "logps/ref_rejected": -75.56619262695312, + "logps/rejected": -83.75196838378906, + "loss": 1.2752, + "margin_dpo/margin_mean": 6.365945816040039, + "margin_dpo/margin_std": 8.420263290405273, + "step": 62 + }, + { + "KL/chosen_KL_mean": -3.0295848846435547, + "KL/mean": -7.3545379638671875, + "KL/rejected_KL_mean": -11.679492950439453, + "KL/std": 8.834741592407227, + "epoch": 0.09251101321585903, + "fcm_dpo/beta": 0.01908048428595066, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 8.649913787841797, + "fcm_dpo/q_t": 0.45918020606040955, + "grad_norm": 17.89116668701172, + "learning_rate": 4.4927536231884053e-07, + "logits/chosen": -0.624936044216156, + "logits/rejected": -0.5820035338401794, + "logps/chosen": -76.1769790649414, + "logps/ref_chosen": -73.14739227294922, + "logps/ref_rejected": -97.61006164550781, + "logps/rejected": -109.28955078125, + "loss": 1.236, + "margin_dpo/margin_mean": 8.64991283416748, + "margin_dpo/margin_std": 9.18691635131836, + "step": 63 + }, + { + "KL/chosen_KL_mean": -1.2771644592285156, + "KL/mean": -6.465435981750488, + "KL/rejected_KL_mean": -11.653709411621094, + "KL/std": 9.661531448364258, + "epoch": 0.09397944199706314, + "fcm_dpo/beta": 0.05195356905460358, + "fcm_dpo/delta": 0.12018927931785583, + "fcm_dpo/margin": 10.376541137695312, + "fcm_dpo/q_t": 0.43298691511154175, + "grad_norm": 37.171104431152344, + "learning_rate": 4.5652173913043473e-07, + "logits/chosen": -0.5985631942749023, + "logits/rejected": -0.5670138597488403, + "logps/chosen": -55.27576446533203, + "logps/ref_chosen": -53.998600006103516, + "logps/ref_rejected": -93.53019714355469, + "logps/rejected": -105.18391418457031, + "loss": 1.0256, + "margin_dpo/margin_mean": 10.376540184020996, + "margin_dpo/margin_std": 11.894201278686523, + "step": 64 + }, + { + "KL/chosen_KL_mean": -3.6957435607910156, + "KL/mean": -8.960607528686523, + "KL/rejected_KL_mean": -14.225475311279297, + "KL/std": 10.946447372436523, + "epoch": 0.09544787077826726, + "fcm_dpo/beta": 0.009888127446174622, + "fcm_dpo/delta": -0.18592241406440735, + "fcm_dpo/margin": 10.529730796813965, + "fcm_dpo/q_t": 0.4064898192882538, + "grad_norm": 11.372929573059082, + "learning_rate": 4.63768115942029e-07, + "logits/chosen": -0.6771352887153625, + "logits/rejected": -0.6651996374130249, + "logps/chosen": -68.53174591064453, + "logps/ref_chosen": -64.83599853515625, + "logps/ref_rejected": -109.94645690917969, + "logps/rejected": -124.17193603515625, + "loss": 1.2885, + "margin_dpo/margin_mean": 10.529730796813965, + "margin_dpo/margin_std": 12.0299072265625, + "step": 65 + }, + { + "KL/chosen_KL_mean": -3.5717506408691406, + "KL/mean": -8.375314712524414, + "KL/rejected_KL_mean": -13.178871154785156, + "KL/std": 9.874279975891113, + "epoch": 0.09691629955947137, + "fcm_dpo/beta": 0.009888127446174622, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 9.607120513916016, + "fcm_dpo/q_t": 0.4763941168785095, + "grad_norm": 9.938729286193848, + "learning_rate": 4.7101449275362313e-07, + "logits/chosen": -0.6698400974273682, + "logits/rejected": -0.6391260623931885, + "logps/chosen": -55.01527404785156, + "logps/ref_chosen": -51.44352722167969, + "logps/ref_rejected": -75.63629913330078, + "logps/rejected": -88.81517028808594, + "loss": 1.2972, + "margin_dpo/margin_mean": 9.607121467590332, + "margin_dpo/margin_std": 11.98110580444336, + "step": 66 + }, + { + "KL/chosen_KL_mean": -3.3190994262695312, + "KL/mean": -8.360406875610352, + "KL/rejected_KL_mean": -13.401721954345703, + "KL/std": 10.303428649902344, + "epoch": 0.09838472834067548, + "fcm_dpo/beta": 0.009888127446174622, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 10.082622528076172, + "fcm_dpo/q_t": 0.47523707151412964, + "grad_norm": 10.002849578857422, + "learning_rate": 4.782608695652174e-07, + "logits/chosen": -0.6043215394020081, + "logits/rejected": -0.5609872341156006, + "logps/chosen": -62.65990447998047, + "logps/ref_chosen": -59.34080505371094, + "logps/ref_rejected": -72.78728485107422, + "logps/rejected": -86.18901062011719, + "loss": 1.293, + "margin_dpo/margin_mean": 10.082620620727539, + "margin_dpo/margin_std": 12.531867027282715, + "step": 67 + }, + { + "KL/chosen_KL_mean": -4.522434234619141, + "KL/mean": -9.367167472839355, + "KL/rejected_KL_mean": -14.211898803710938, + "KL/std": 9.465312957763672, + "epoch": 0.09985315712187959, + "fcm_dpo/beta": 0.009888127446174622, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 9.689462661743164, + "fcm_dpo/q_t": 0.47615760564804077, + "grad_norm": 9.184582710266113, + "learning_rate": 4.855072463768116e-07, + "logits/chosen": -0.6481267213821411, + "logits/rejected": -0.5857020020484924, + "logps/chosen": -69.728271484375, + "logps/ref_chosen": -65.2058334350586, + "logps/ref_rejected": -77.20724487304688, + "logps/rejected": -91.41914367675781, + "loss": 1.2957, + "margin_dpo/margin_mean": 9.689462661743164, + "margin_dpo/margin_std": 10.790483474731445, + "step": 68 + }, + { + "KL/chosen_KL_mean": -5.772527694702148, + "KL/mean": -12.4307222366333, + "KL/rejected_KL_mean": -19.088916778564453, + "KL/std": 12.551191329956055, + "epoch": 0.1013215859030837, + "fcm_dpo/beta": 0.009888127446174622, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 13.31639289855957, + "fcm_dpo/q_t": 0.4673622250556946, + "grad_norm": 11.88070297241211, + "learning_rate": 4.927536231884058e-07, + "logits/chosen": -0.633673906326294, + "logits/rejected": -0.6116066575050354, + "logps/chosen": -65.59176635742188, + "logps/ref_chosen": -59.81924057006836, + "logps/ref_rejected": -103.38886260986328, + "logps/rejected": -122.477783203125, + "loss": 1.264, + "margin_dpo/margin_mean": 13.31639289855957, + "margin_dpo/margin_std": 14.175312042236328, + "step": 69 + }, + { + "KL/chosen_KL_mean": -8.516521453857422, + "KL/mean": -15.706306457519531, + "KL/rejected_KL_mean": -22.89609146118164, + "KL/std": 16.06771469116211, + "epoch": 0.1027900146842878, + "fcm_dpo/beta": 0.009888127446174622, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 14.37956428527832, + "fcm_dpo/q_t": 0.46483179926872253, + "grad_norm": 11.995932579040527, + "learning_rate": 5e-07, + "logits/chosen": -0.6291612386703491, + "logits/rejected": -0.5962782502174377, + "logps/chosen": -70.44715881347656, + "logps/ref_chosen": -61.930641174316406, + "logps/ref_rejected": -91.06078338623047, + "logps/rejected": -113.95687866210938, + "loss": 1.2564, + "margin_dpo/margin_mean": 14.37956428527832, + "margin_dpo/margin_std": 17.146224975585938, + "step": 70 + }, + { + "KL/chosen_KL_mean": -9.071426391601562, + "KL/mean": -17.838401794433594, + "KL/rejected_KL_mean": -26.605384826660156, + "KL/std": 17.67050552368164, + "epoch": 0.10425844346549193, + "fcm_dpo/beta": 0.009888127446174622, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 17.53395652770996, + "fcm_dpo/q_t": 0.4574003219604492, + "grad_norm": 13.351109504699707, + "learning_rate": 4.999967061337492e-07, + "logits/chosen": -0.6778059601783752, + "logits/rejected": -0.6437849998474121, + "logps/chosen": -70.82176208496094, + "logps/ref_chosen": -61.750335693359375, + "logps/ref_rejected": -97.33662414550781, + "logps/rejected": -123.94200134277344, + "loss": 1.2301, + "margin_dpo/margin_mean": 17.533954620361328, + "margin_dpo/margin_std": 19.450382232666016, + "step": 71 + }, + { + "KL/chosen_KL_mean": -10.510194778442383, + "KL/mean": -19.48761749267578, + "KL/rejected_KL_mean": -28.465042114257812, + "KL/std": 18.958513259887695, + "epoch": 0.10572687224669604, + "fcm_dpo/beta": 0.009888127446174622, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 17.954853057861328, + "fcm_dpo/q_t": 0.456373393535614, + "grad_norm": 12.942930221557617, + "learning_rate": 4.999868246217933e-07, + "logits/chosen": -0.6576621532440186, + "logits/rejected": -0.6210486888885498, + "logps/chosen": -76.56360626220703, + "logps/ref_chosen": -66.05341339111328, + "logps/ref_rejected": -95.2869873046875, + "logps/rejected": -123.75202941894531, + "loss": 1.2293, + "margin_dpo/margin_mean": 17.954853057861328, + "margin_dpo/margin_std": 23.021011352539062, + "step": 72 + }, + { + "KL/chosen_KL_mean": -12.810209274291992, + "KL/mean": -21.97817611694336, + "KL/rejected_KL_mean": -31.14615249633789, + "KL/std": 24.339290618896484, + "epoch": 0.10719530102790015, + "fcm_dpo/beta": 0.009888127446174622, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 18.335941314697266, + "fcm_dpo/q_t": 0.45616060495376587, + "grad_norm": 13.09295654296875, + "learning_rate": 4.999703557245192e-07, + "logits/chosen": -0.6870651245117188, + "logits/rejected": -0.6481956243515015, + "logps/chosen": -79.06648254394531, + "logps/ref_chosen": -66.25627136230469, + "logps/ref_rejected": -90.45613098144531, + "logps/rejected": -121.60228729248047, + "loss": 1.2364, + "margin_dpo/margin_mean": 18.335941314697266, + "margin_dpo/margin_std": 31.158466339111328, + "step": 73 + }, + { + "KL/chosen_KL_mean": -13.415903091430664, + "KL/mean": -23.937971115112305, + "KL/rejected_KL_mean": -34.46004104614258, + "KL/std": 27.361499786376953, + "epoch": 0.10866372980910426, + "fcm_dpo/beta": 0.019809015095233917, + "fcm_dpo/delta": 0.04002254828810692, + "fcm_dpo/margin": 21.04413414001465, + "fcm_dpo/q_t": 0.42780494689941406, + "grad_norm": 22.95852279663086, + "learning_rate": 4.999472998758977e-07, + "logits/chosen": -0.6472772359848022, + "logits/rejected": -0.6437745690345764, + "logps/chosen": -66.84078979492188, + "logps/ref_chosen": -53.42488098144531, + "logps/ref_rejected": -95.94693756103516, + "logps/rejected": -130.406982421875, + "loss": 1.1006, + "margin_dpo/margin_mean": 21.04413604736328, + "margin_dpo/margin_std": 34.61186218261719, + "step": 74 + }, + { + "KL/chosen_KL_mean": -13.621763229370117, + "KL/mean": -28.440505981445312, + "KL/rejected_KL_mean": -43.259239196777344, + "KL/std": 28.2391357421875, + "epoch": 0.11013215859030837, + "fcm_dpo/beta": 0.011893379501998425, + "fcm_dpo/delta": -0.11567549407482147, + "fcm_dpo/margin": 29.637487411499023, + "fcm_dpo/q_t": 0.3860425055027008, + "grad_norm": 16.972978591918945, + "learning_rate": 4.999176576834721e-07, + "logits/chosen": -0.6723306179046631, + "logits/rejected": -0.6689744591712952, + "logps/chosen": -65.48342895507812, + "logps/ref_chosen": -51.861663818359375, + "logps/ref_rejected": -111.25398254394531, + "logps/rejected": -154.51321411132812, + "loss": 1.1449, + "margin_dpo/margin_mean": 29.63748550415039, + "margin_dpo/margin_std": 32.814430236816406, + "step": 75 + }, + { + "KL/chosen_KL_mean": -16.395811080932617, + "KL/mean": -25.452667236328125, + "KL/rejected_KL_mean": -34.509517669677734, + "KL/std": 23.012611389160156, + "epoch": 0.11160058737151249, + "fcm_dpo/beta": 0.004640421364456415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 18.113712310791016, + "fcm_dpo/q_t": 0.4790630340576172, + "grad_norm": 7.068286895751953, + "learning_rate": 4.998814299283415e-07, + "logits/chosen": -0.7196544408798218, + "logits/rejected": -0.6837272644042969, + "logps/chosen": -69.66184997558594, + "logps/ref_chosen": -53.26603698730469, + "logps/ref_rejected": -78.21662902832031, + "logps/rejected": -112.72615051269531, + "loss": 1.3076, + "margin_dpo/margin_mean": 18.113712310791016, + "margin_dpo/margin_std": 25.670862197875977, + "step": 76 + }, + { + "KL/chosen_KL_mean": -15.274223327636719, + "KL/mean": -30.40639877319336, + "KL/rejected_KL_mean": -45.53857421875, + "KL/std": 31.36872100830078, + "epoch": 0.1130690161527166, + "fcm_dpo/beta": 0.004640421364456415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 30.26435089111328, + "fcm_dpo/q_t": 0.4651873707771301, + "grad_norm": 9.03381633758545, + "learning_rate": 4.998386175651409e-07, + "logits/chosen": -0.6737496256828308, + "logits/rejected": -0.639689564704895, + "logps/chosen": -73.37090301513672, + "logps/ref_chosen": -58.0966796875, + "logps/ref_rejected": -93.77361297607422, + "logps/rejected": -139.31219482421875, + "loss": 1.2578, + "margin_dpo/margin_mean": 30.26434898376465, + "margin_dpo/margin_std": 35.96830749511719, + "step": 77 + }, + { + "KL/chosen_KL_mean": -15.677688598632812, + "KL/mean": -26.735885620117188, + "KL/rejected_KL_mean": -37.79408264160156, + "KL/std": 26.46023941040039, + "epoch": 0.1145374449339207, + "fcm_dpo/beta": 0.004640421364456415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 22.116390228271484, + "fcm_dpo/q_t": 0.4745638370513916, + "grad_norm": 7.281389236450195, + "learning_rate": 4.997892217220159e-07, + "logits/chosen": -0.644652247428894, + "logits/rejected": -0.6242761015892029, + "logps/chosen": -71.29147338867188, + "logps/ref_chosen": -55.61378479003906, + "logps/ref_rejected": -84.93436431884766, + "logps/rejected": -122.72844696044922, + "loss": 1.2917, + "margin_dpo/margin_mean": 22.116392135620117, + "margin_dpo/margin_std": 31.19441795349121, + "step": 78 + }, + { + "KL/chosen_KL_mean": -16.822900772094727, + "KL/mean": -28.855121612548828, + "KL/rejected_KL_mean": -40.8873405456543, + "KL/std": 29.233537673950195, + "epoch": 0.11600587371512482, + "fcm_dpo/beta": 0.004640421364456415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 24.064435958862305, + "fcm_dpo/q_t": 0.4723590612411499, + "grad_norm": 7.421467304229736, + "learning_rate": 4.997332437005931e-07, + "logits/chosen": -0.6622911095619202, + "logits/rejected": -0.6371140480041504, + "logps/chosen": -72.27339172363281, + "logps/ref_chosen": -55.45048522949219, + "logps/ref_rejected": -87.64756774902344, + "logps/rejected": -128.534912109375, + "loss": 1.2846, + "margin_dpo/margin_mean": 24.064434051513672, + "margin_dpo/margin_std": 35.614994049072266, + "step": 79 + }, + { + "KL/chosen_KL_mean": -21.039852142333984, + "KL/mean": -33.44929504394531, + "KL/rejected_KL_mean": -45.858741760253906, + "KL/std": 33.63664245605469, + "epoch": 0.11747430249632893, + "fcm_dpo/beta": 0.004640421364456415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 24.818889617919922, + "fcm_dpo/q_t": 0.4716973304748535, + "grad_norm": 8.430150032043457, + "learning_rate": 4.996706849759452e-07, + "logits/chosen": -0.7373731732368469, + "logits/rejected": -0.7053723335266113, + "logps/chosen": -79.55914306640625, + "logps/ref_chosen": -58.519290924072266, + "logps/ref_rejected": -87.54750061035156, + "logps/rejected": -133.40625, + "loss": 1.2836, + "margin_dpo/margin_mean": 24.818889617919922, + "margin_dpo/margin_std": 39.66154098510742, + "step": 80 + }, + { + "KL/chosen_KL_mean": -21.12224578857422, + "KL/mean": -38.02497863769531, + "KL/rejected_KL_mean": -54.927711486816406, + "KL/std": 39.90003204345703, + "epoch": 0.11894273127753303, + "fcm_dpo/beta": 0.004640421364456415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 33.80546188354492, + "fcm_dpo/q_t": 0.4615294337272644, + "grad_norm": 9.538393020629883, + "learning_rate": 4.996015471965529e-07, + "logits/chosen": -0.6799561977386475, + "logits/rejected": -0.6571604013442993, + "logps/chosen": -87.57111358642578, + "logps/ref_chosen": -66.44886779785156, + "logps/ref_rejected": -129.66270446777344, + "logps/rejected": -184.59042358398438, + "loss": 1.2478, + "margin_dpo/margin_mean": 33.80546188354492, + "margin_dpo/margin_std": 47.89176940917969, + "step": 81 + }, + { + "KL/chosen_KL_mean": -24.305784225463867, + "KL/mean": -37.62807846069336, + "KL/rejected_KL_mean": -50.950374603271484, + "KL/std": 36.823760986328125, + "epoch": 0.12041116005873716, + "fcm_dpo/beta": 0.004640421364456415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 26.64459228515625, + "fcm_dpo/q_t": 0.4697510600090027, + "grad_norm": 8.230402946472168, + "learning_rate": 4.995258321842611e-07, + "logits/chosen": -0.6198301315307617, + "logits/rejected": -0.6155990958213806, + "logps/chosen": -76.53816986083984, + "logps/ref_chosen": -52.232383728027344, + "logps/ref_rejected": -90.74325561523438, + "logps/rejected": -141.69363403320312, + "loss": 1.2812, + "margin_dpo/margin_mean": 26.644596099853516, + "margin_dpo/margin_std": 51.493133544921875, + "step": 82 + }, + { + "KL/chosen_KL_mean": -25.275197982788086, + "KL/mean": -42.2493782043457, + "KL/rejected_KL_mean": -59.22356033325195, + "KL/std": 38.86623001098633, + "epoch": 0.12187958883994127, + "fcm_dpo/beta": 0.004640421364456415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 33.9483642578125, + "fcm_dpo/q_t": 0.4612438678741455, + "grad_norm": 9.907917022705078, + "learning_rate": 4.994435419342304e-07, + "logits/chosen": -0.6601589322090149, + "logits/rejected": -0.6394397616386414, + "logps/chosen": -81.10258483886719, + "logps/ref_chosen": -55.82738494873047, + "logps/ref_rejected": -103.71589660644531, + "logps/rejected": -162.939453125, + "loss": 1.2466, + "margin_dpo/margin_mean": 33.948368072509766, + "margin_dpo/margin_std": 46.71687698364258, + "step": 83 + }, + { + "KL/chosen_KL_mean": -26.676267623901367, + "KL/mean": -39.81716537475586, + "KL/rejected_KL_mean": -52.958065032958984, + "KL/std": 34.16858673095703, + "epoch": 0.12334801762114538, + "fcm_dpo/beta": 0.004640421364456415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 26.28179359436035, + "fcm_dpo/q_t": 0.46975940465927124, + "grad_norm": 8.419811248779297, + "learning_rate": 4.993546786148857e-07, + "logits/chosen": -0.6571387052536011, + "logits/rejected": -0.6230216026306152, + "logps/chosen": -93.85243225097656, + "logps/ref_chosen": -67.1761703491211, + "logps/ref_rejected": -87.29859924316406, + "logps/rejected": -140.2566680908203, + "loss": 1.2748, + "margin_dpo/margin_mean": 26.281795501708984, + "margin_dpo/margin_std": 33.972564697265625, + "step": 84 + }, + { + "KL/chosen_KL_mean": -26.707151412963867, + "KL/mean": -40.551788330078125, + "KL/rejected_KL_mean": -54.396427154541016, + "KL/std": 34.13534927368164, + "epoch": 0.12481644640234948, + "fcm_dpo/beta": 0.004640421364456415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 27.689273834228516, + "fcm_dpo/q_t": 0.4682667851448059, + "grad_norm": 8.280872344970703, + "learning_rate": 4.992592445678582e-07, + "logits/chosen": -0.6289738416671753, + "logits/rejected": -0.5984049439430237, + "logps/chosen": -85.11376953125, + "logps/ref_chosen": -58.4066162109375, + "logps/ref_rejected": -78.63880157470703, + "logps/rejected": -133.0352325439453, + "loss": 1.2717, + "margin_dpo/margin_mean": 27.689273834228516, + "margin_dpo/margin_std": 41.85002136230469, + "step": 85 + }, + { + "KL/chosen_KL_mean": -36.066551208496094, + "KL/mean": -50.90142822265625, + "KL/rejected_KL_mean": -65.7363052368164, + "KL/std": 51.3672981262207, + "epoch": 0.1262848751835536, + "fcm_dpo/beta": 0.004640421364456415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 29.669748306274414, + "fcm_dpo/q_t": 0.46719542145729065, + "grad_norm": 9.010835647583008, + "learning_rate": 4.991572423079235e-07, + "logits/chosen": -0.6406770348548889, + "logits/rejected": -0.6358238458633423, + "logps/chosen": -92.20401000976562, + "logps/ref_chosen": -56.13746643066406, + "logps/ref_rejected": -88.12165069580078, + "logps/rejected": -153.8579559326172, + "loss": 1.2809, + "margin_dpo/margin_mean": 29.669748306274414, + "margin_dpo/margin_std": 72.3070068359375, + "step": 86 + }, + { + "KL/chosen_KL_mean": -35.95440673828125, + "KL/mean": -55.21514129638672, + "KL/rejected_KL_mean": -74.47587585449219, + "KL/std": 52.28978729248047, + "epoch": 0.1277533039647577, + "fcm_dpo/beta": 0.012798674404621124, + "fcm_dpo/delta": 0.10860809683799744, + "fcm_dpo/margin": 38.52145767211914, + "fcm_dpo/q_t": 0.433593213558197, + "grad_norm": 24.902225494384766, + "learning_rate": 4.990486745229364e-07, + "logits/chosen": -0.6695064306259155, + "logits/rejected": -0.6587918996810913, + "logps/chosen": -91.59049987792969, + "logps/ref_chosen": -55.63609313964844, + "logps/ref_rejected": -95.46757507324219, + "logps/rejected": -169.94345092773438, + "loss": 1.1243, + "margin_dpo/margin_mean": 38.52145767211914, + "margin_dpo/margin_std": 65.65547943115234, + "step": 87 + }, + { + "KL/chosen_KL_mean": -46.075992584228516, + "KL/mean": -60.8734245300293, + "KL/rejected_KL_mean": -75.67085266113281, + "KL/std": 57.557559967041016, + "epoch": 0.12922173274596183, + "fcm_dpo/beta": 0.007393650244921446, + "fcm_dpo/delta": -0.062026459723711014, + "fcm_dpo/margin": 29.5948543548584, + "fcm_dpo/q_t": 0.4339485168457031, + "grad_norm": 14.445233345031738, + "learning_rate": 4.989335440737586e-07, + "logits/chosen": -0.6514161825180054, + "logits/rejected": -0.6543829441070557, + "logps/chosen": -119.74714660644531, + "logps/ref_chosen": -73.67115020751953, + "logps/ref_rejected": -106.70849609375, + "logps/rejected": -182.3793487548828, + "loss": 1.2408, + "margin_dpo/margin_mean": 29.594852447509766, + "margin_dpo/margin_std": 64.25787353515625, + "step": 88 + }, + { + "KL/chosen_KL_mean": -32.12773132324219, + "KL/mean": -48.858558654785156, + "KL/rejected_KL_mean": -65.5893783569336, + "KL/std": 51.16180419921875, + "epoch": 0.13069016152716592, + "fcm_dpo/beta": 0.009811213240027428, + "fcm_dpo/delta": 0.05031702667474747, + "fcm_dpo/margin": 33.46165466308594, + "fcm_dpo/q_t": 0.44110527634620667, + "grad_norm": 17.186080932617188, + "learning_rate": 4.988118539941847e-07, + "logits/chosen": -0.6749956607818604, + "logits/rejected": -0.6535379886627197, + "logps/chosen": -92.75264739990234, + "logps/ref_chosen": -60.624916076660156, + "logps/ref_rejected": -82.08354949951172, + "logps/rejected": -147.6729278564453, + "loss": 1.1533, + "margin_dpo/margin_mean": 33.46165466308594, + "margin_dpo/margin_std": 60.91192626953125, + "step": 89 + }, + { + "KL/chosen_KL_mean": -36.43550109863281, + "KL/mean": -62.53802490234375, + "KL/rejected_KL_mean": -88.64056396484375, + "KL/std": 68.16754913330078, + "epoch": 0.13215859030837004, + "fcm_dpo/beta": 0.0063932533375918865, + "fcm_dpo/delta": -0.19979318976402283, + "fcm_dpo/margin": 52.205047607421875, + "fcm_dpo/q_t": 0.38456064462661743, + "grad_norm": 15.716353416442871, + "learning_rate": 4.986836074908615e-07, + "logits/chosen": -0.6015010476112366, + "logits/rejected": -0.6259626150131226, + "logps/chosen": -89.72080993652344, + "logps/ref_chosen": -53.285308837890625, + "logps/ref_rejected": -111.54470825195312, + "logps/rejected": -200.18527221679688, + "loss": 1.2609, + "margin_dpo/margin_mean": 52.20505142211914, + "margin_dpo/margin_std": 86.0552978515625, + "step": 90 + }, + { + "KL/chosen_KL_mean": -40.36619567871094, + "KL/mean": -57.59088897705078, + "KL/rejected_KL_mean": -74.81558227539062, + "KL/std": 51.144317626953125, + "epoch": 0.13362701908957417, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 34.44939041137695, + "fcm_dpo/q_t": 0.4857920706272125, + "grad_norm": 3.791431188583374, + "learning_rate": 4.985488079432037e-07, + "logits/chosen": -0.6380863189697266, + "logits/rejected": -0.6169267892837524, + "logps/chosen": -102.16915130615234, + "logps/ref_chosen": -61.802955627441406, + "logps/ref_rejected": -87.87395477294922, + "logps/rejected": -162.68954467773438, + "loss": 1.3334, + "margin_dpo/margin_mean": 34.44939041137695, + "margin_dpo/margin_std": 70.18351745605469, + "step": 91 + }, + { + "KL/chosen_KL_mean": -35.530731201171875, + "KL/mean": -54.40737533569336, + "KL/rejected_KL_mean": -73.28401184082031, + "KL/std": 54.11292266845703, + "epoch": 0.13509544787077826, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 37.7532844543457, + "fcm_dpo/q_t": 0.48441988229751587, + "grad_norm": 4.081094264984131, + "learning_rate": 4.984074589033043e-07, + "logits/chosen": -0.6290682554244995, + "logits/rejected": -0.6120110750198364, + "logps/chosen": -87.17150115966797, + "logps/ref_chosen": -51.640769958496094, + "logps/ref_rejected": -77.88117980957031, + "logps/rejected": -151.16519165039062, + "loss": 1.3277, + "margin_dpo/margin_mean": 37.7532844543457, + "margin_dpo/margin_std": 65.40171813964844, + "step": 92 + }, + { + "KL/chosen_KL_mean": -39.78628158569336, + "KL/mean": -58.02899169921875, + "KL/rejected_KL_mean": -76.27169799804688, + "KL/std": 48.12828063964844, + "epoch": 0.13656387665198239, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 36.48542404174805, + "fcm_dpo/q_t": 0.48493102192878723, + "grad_norm": 4.032928466796875, + "learning_rate": 4.982595640958425e-07, + "logits/chosen": -0.6539796590805054, + "logits/rejected": -0.6243857145309448, + "logps/chosen": -92.31552124023438, + "logps/ref_chosen": -52.529239654541016, + "logps/ref_rejected": -77.16075134277344, + "logps/rejected": -153.4324493408203, + "loss": 1.329, + "margin_dpo/margin_mean": 36.48542404174805, + "margin_dpo/margin_std": 57.570045471191406, + "step": 93 + }, + { + "KL/chosen_KL_mean": -43.512413024902344, + "KL/mean": -66.17471313476562, + "KL/rejected_KL_mean": -88.83700561523438, + "KL/std": 59.194549560546875, + "epoch": 0.13803230543318648, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 45.32459259033203, + "fcm_dpo/q_t": 0.4812876582145691, + "grad_norm": 4.497649192810059, + "learning_rate": 4.98105127417984e-07, + "logits/chosen": -0.5847138166427612, + "logits/rejected": -0.5839489698410034, + "logps/chosen": -104.73502349853516, + "logps/ref_chosen": -61.22261047363281, + "logps/ref_rejected": -99.59902954101562, + "logps/rejected": -188.43603515625, + "loss": 1.3155, + "margin_dpo/margin_mean": 45.32460021972656, + "margin_dpo/margin_std": 64.99853515625, + "step": 94 + }, + { + "KL/chosen_KL_mean": -43.288063049316406, + "KL/mean": -61.76789855957031, + "KL/rejected_KL_mean": -80.24773406982422, + "KL/std": 63.553306579589844, + "epoch": 0.1395007342143906, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 36.95966720581055, + "fcm_dpo/q_t": 0.48472627997398376, + "grad_norm": 4.214771270751953, + "learning_rate": 4.979441529392784e-07, + "logits/chosen": -0.6216604113578796, + "logits/rejected": -0.6044141054153442, + "logps/chosen": -95.81170654296875, + "logps/ref_chosen": -52.523643493652344, + "logps/ref_rejected": -75.8803482055664, + "logps/rejected": -156.12808227539062, + "loss": 1.3292, + "margin_dpo/margin_mean": 36.95966339111328, + "margin_dpo/margin_std": 67.57984161376953, + "step": 95 + }, + { + "KL/chosen_KL_mean": -46.38825225830078, + "KL/mean": -72.26142883300781, + "KL/rejected_KL_mean": -98.13460540771484, + "KL/std": 66.60676574707031, + "epoch": 0.14096916299559473, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 51.74634552001953, + "fcm_dpo/q_t": 0.4787074327468872, + "grad_norm": 4.915516376495361, + "learning_rate": 4.977766449015534e-07, + "logits/chosen": -0.5953140258789062, + "logits/rejected": -0.5802059173583984, + "logps/chosen": -108.54522705078125, + "logps/ref_chosen": -62.15697479248047, + "logps/ref_rejected": -96.59601593017578, + "logps/rejected": -194.73062133789062, + "loss": 1.3066, + "margin_dpo/margin_mean": 51.74634552001953, + "margin_dpo/margin_std": 77.54536437988281, + "step": 96 + }, + { + "KL/chosen_KL_mean": -47.02070999145508, + "KL/mean": -65.5418701171875, + "KL/rejected_KL_mean": -84.06302642822266, + "KL/std": 60.283546447753906, + "epoch": 0.14243759177679882, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 37.042320251464844, + "fcm_dpo/q_t": 0.4846979081630707, + "grad_norm": 4.432089328765869, + "learning_rate": 4.976026077188012e-07, + "logits/chosen": -0.4999476671218872, + "logits/rejected": -0.4703494906425476, + "logps/chosen": -101.66707611083984, + "logps/ref_chosen": -54.646366119384766, + "logps/ref_rejected": -76.96475219726562, + "logps/rejected": -161.02777099609375, + "loss": 1.3285, + "margin_dpo/margin_mean": 37.042320251464844, + "margin_dpo/margin_std": 60.82837677001953, + "step": 97 + }, + { + "KL/chosen_KL_mean": -63.49620056152344, + "KL/mean": -85.60096740722656, + "KL/rejected_KL_mean": -107.70573425292969, + "KL/std": 73.20948028564453, + "epoch": 0.14390602055800295, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 44.20954132080078, + "fcm_dpo/q_t": 0.4817187786102295, + "grad_norm": 5.483908653259277, + "learning_rate": 4.974220459770639e-07, + "logits/chosen": -0.5312040448188782, + "logits/rejected": -0.5291551947593689, + "logps/chosen": -128.75482177734375, + "logps/ref_chosen": -65.25862884521484, + "logps/ref_rejected": -96.5274887084961, + "logps/rejected": -204.23321533203125, + "loss": 1.3201, + "margin_dpo/margin_mean": 44.20954132080078, + "margin_dpo/margin_std": 89.74884033203125, + "step": 98 + }, + { + "KL/chosen_KL_mean": -47.63540267944336, + "KL/mean": -77.33531188964844, + "KL/rejected_KL_mean": -107.03522491455078, + "KL/std": 73.14799499511719, + "epoch": 0.14537444933920704, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 59.39982604980469, + "fcm_dpo/q_t": 0.47561943531036377, + "grad_norm": 5.031144618988037, + "learning_rate": 4.972349644343108e-07, + "logits/chosen": -0.5268571376800537, + "logits/rejected": -0.5339376926422119, + "logps/chosen": -93.27388763427734, + "logps/ref_chosen": -45.638484954833984, + "logps/ref_rejected": -86.43793487548828, + "logps/rejected": -193.47315979003906, + "loss": 1.2962, + "margin_dpo/margin_mean": 59.39982604980469, + "margin_dpo/margin_std": 93.68307495117188, + "step": 99 + }, + { + "KL/chosen_KL_mean": -58.90423583984375, + "KL/mean": -70.80164337158203, + "KL/rejected_KL_mean": -82.69905090332031, + "KL/std": 66.31571960449219, + "epoch": 0.14684287812041116, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 23.79481315612793, + "fcm_dpo/q_t": 0.49016737937927246, + "grad_norm": 4.631414413452148, + "learning_rate": 4.970413680203148e-07, + "logits/chosen": -0.5172683596611023, + "logits/rejected": -0.4957023859024048, + "logps/chosen": -116.49821472167969, + "logps/ref_chosen": -57.59397888183594, + "logps/ref_rejected": -74.06021118164062, + "logps/rejected": -156.75926208496094, + "loss": 1.3523, + "margin_dpo/margin_mean": 23.794811248779297, + "margin_dpo/margin_std": 82.77680206298828, + "step": 100 + }, + { + "KL/chosen_KL_mean": -70.8927230834961, + "KL/mean": -88.97613525390625, + "KL/rejected_KL_mean": -107.05953979492188, + "KL/std": 81.61466979980469, + "epoch": 0.14831130690161526, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 36.166805267333984, + "fcm_dpo/q_t": 0.48512861132621765, + "grad_norm": 4.663822650909424, + "learning_rate": 4.968412618365215e-07, + "logits/chosen": -0.5123007297515869, + "logits/rejected": -0.4988871216773987, + "logps/chosen": -132.5415802001953, + "logps/ref_chosen": -61.64885330200195, + "logps/ref_rejected": -83.18968200683594, + "logps/rejected": -190.24923706054688, + "loss": 1.3335, + "margin_dpo/margin_mean": 36.16680908203125, + "margin_dpo/margin_std": 94.42303466796875, + "step": 101 + }, + { + "KL/chosen_KL_mean": -74.67913818359375, + "KL/mean": -87.16828918457031, + "KL/rejected_KL_mean": -99.65742492675781, + "KL/std": 79.29058837890625, + "epoch": 0.14977973568281938, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 24.978286743164062, + "fcm_dpo/q_t": 0.489687979221344, + "grad_norm": 5.5214524269104, + "learning_rate": 4.966346511559149e-07, + "logits/chosen": -0.5297523140907288, + "logits/rejected": -0.5086390972137451, + "logps/chosen": -138.7580108642578, + "logps/ref_chosen": -64.0788803100586, + "logps/ref_rejected": -68.18707275390625, + "logps/rejected": -167.84451293945312, + "loss": 1.352, + "margin_dpo/margin_mean": 24.978286743164062, + "margin_dpo/margin_std": 96.62683868408203, + "step": 102 + }, + { + "KL/chosen_KL_mean": -69.34735870361328, + "KL/mean": -98.80657958984375, + "KL/rejected_KL_mean": -128.26580810546875, + "KL/std": 92.38734436035156, + "epoch": 0.1512481644640235, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 58.918426513671875, + "fcm_dpo/q_t": 0.47593387961387634, + "grad_norm": 5.950850963592529, + "learning_rate": 4.964215414228785e-07, + "logits/chosen": -0.46111953258514404, + "logits/rejected": -0.441642165184021, + "logps/chosen": -130.64663696289062, + "logps/ref_chosen": -61.299278259277344, + "logps/ref_rejected": -93.57270812988281, + "logps/rejected": -221.8385009765625, + "loss": 1.3007, + "margin_dpo/margin_mean": 58.918426513671875, + "margin_dpo/margin_std": 119.41735076904297, + "step": 103 + }, + { + "KL/chosen_KL_mean": -67.7513198852539, + "KL/mean": -96.42362976074219, + "KL/rejected_KL_mean": -125.09593200683594, + "KL/std": 93.0230712890625, + "epoch": 0.1527165932452276, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.34462356567383, + "fcm_dpo/q_t": 0.476635605096817, + "grad_norm": 8.443946838378906, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": -0.4595398008823395, + "logits/rejected": -0.4493861794471741, + "logps/chosen": -122.12409210205078, + "logps/ref_chosen": -54.372772216796875, + "logps/ref_rejected": -89.5647201538086, + "logps/rejected": -214.66064453125, + "loss": 1.3031, + "margin_dpo/margin_mean": 57.344627380371094, + "margin_dpo/margin_std": 118.23411560058594, + "step": 104 + }, + { + "KL/chosen_KL_mean": -68.69507598876953, + "KL/mean": -111.2301254272461, + "KL/rejected_KL_mean": -153.76516723632812, + "KL/std": 86.79656219482422, + "epoch": 0.15418502202643172, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 85.07008361816406, + "fcm_dpo/q_t": 0.46504706144332886, + "grad_norm": 6.229670524597168, + "learning_rate": 4.959758474331832e-07, + "logits/chosen": -0.4372522830963135, + "logits/rejected": -0.4284954369068146, + "logps/chosen": -123.33402252197266, + "logps/ref_chosen": -54.638946533203125, + "logps/ref_rejected": -97.97351837158203, + "logps/rejected": -251.7386932373047, + "loss": 1.2576, + "margin_dpo/margin_mean": 85.07008361816406, + "margin_dpo/margin_std": 102.4740219116211, + "step": 105 + }, + { + "KL/chosen_KL_mean": -80.03727722167969, + "KL/mean": -107.10738372802734, + "KL/rejected_KL_mean": -134.177490234375, + "KL/std": 90.39563751220703, + "epoch": 0.15565345080763582, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 54.140220642089844, + "fcm_dpo/q_t": 0.47773557901382446, + "grad_norm": 5.816026210784912, + "learning_rate": 4.957432749209755e-07, + "logits/chosen": -0.38607245683670044, + "logits/rejected": -0.3913835883140564, + "logps/chosen": -134.87017822265625, + "logps/ref_chosen": -54.83289337158203, + "logps/ref_rejected": -85.22461700439453, + "logps/rejected": -219.40211486816406, + "loss": 1.3052, + "margin_dpo/margin_mean": 54.14021682739258, + "margin_dpo/margin_std": 96.50797271728516, + "step": 106 + }, + { + "KL/chosen_KL_mean": -96.32496643066406, + "KL/mean": -126.12309265136719, + "KL/rejected_KL_mean": -155.92123413085938, + "KL/std": 109.03052520751953, + "epoch": 0.15712187958883994, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 59.59624481201172, + "fcm_dpo/q_t": 0.47559940814971924, + "grad_norm": 6.986702919006348, + "learning_rate": 4.955042268449307e-07, + "logits/chosen": -0.42685896158218384, + "logits/rejected": -0.4117262363433838, + "logps/chosen": -166.03277587890625, + "logps/ref_chosen": -69.70780944824219, + "logps/ref_rejected": -94.73950958251953, + "logps/rejected": -250.66073608398438, + "loss": 1.2995, + "margin_dpo/margin_mean": 59.596248626708984, + "margin_dpo/margin_std": 115.8310775756836, + "step": 107 + }, + { + "KL/chosen_KL_mean": -79.45794677734375, + "KL/mean": -121.89816284179688, + "KL/rejected_KL_mean": -164.33836364746094, + "KL/std": 121.62930297851562, + "epoch": 0.15859030837004406, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 84.88043212890625, + "fcm_dpo/q_t": 0.4657972455024719, + "grad_norm": 8.013676643371582, + "learning_rate": 4.952587095041881e-07, + "logits/chosen": -0.36991098523139954, + "logits/rejected": -0.36751043796539307, + "logps/chosen": -135.46783447265625, + "logps/ref_chosen": -56.0098876953125, + "logps/ref_rejected": -95.79601287841797, + "logps/rejected": -260.1343994140625, + "loss": 1.2671, + "margin_dpo/margin_mean": 84.88043212890625, + "margin_dpo/margin_std": 156.81539916992188, + "step": 108 + }, + { + "KL/chosen_KL_mean": -85.31330871582031, + "KL/mean": -126.01399230957031, + "KL/rejected_KL_mean": -166.71469116210938, + "KL/std": 120.20221710205078, + "epoch": 0.16005873715124816, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 81.40138244628906, + "fcm_dpo/q_t": 0.4666248559951782, + "grad_norm": 6.836396217346191, + "learning_rate": 4.95006729368358e-07, + "logits/chosen": -0.36495447158813477, + "logits/rejected": -0.3560246229171753, + "logps/chosen": -148.19879150390625, + "logps/ref_chosen": -62.88549041748047, + "logps/ref_rejected": -98.68573760986328, + "logps/rejected": -265.4004211425781, + "loss": 1.2734, + "margin_dpo/margin_mean": 81.40138244628906, + "margin_dpo/margin_std": 159.874755859375, + "step": 109 + }, + { + "KL/chosen_KL_mean": -91.46940612792969, + "KL/mean": -125.52250671386719, + "KL/rejected_KL_mean": -159.5756072998047, + "KL/std": 117.68600463867188, + "epoch": 0.16152716593245228, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 68.10619354248047, + "fcm_dpo/q_t": 0.4723784029483795, + "grad_norm": 8.115654945373535, + "learning_rate": 4.947482930773511e-07, + "logits/chosen": -0.32437682151794434, + "logits/rejected": -0.3229568600654602, + "logps/chosen": -150.22308349609375, + "logps/ref_chosen": -58.753684997558594, + "logps/ref_rejected": -79.75001525878906, + "logps/rejected": -239.32562255859375, + "loss": 1.2933, + "margin_dpo/margin_mean": 68.10619354248047, + "margin_dpo/margin_std": 149.91207885742188, + "step": 110 + }, + { + "KL/chosen_KL_mean": -101.22291564941406, + "KL/mean": -143.97872924804688, + "KL/rejected_KL_mean": -186.7345733642578, + "KL/std": 137.13291931152344, + "epoch": 0.16299559471365638, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 85.51166534423828, + "fcm_dpo/q_t": 0.46532922983169556, + "grad_norm": 7.223021507263184, + "learning_rate": 4.944834074412042e-07, + "logits/chosen": -0.33777564764022827, + "logits/rejected": -0.33077672123908997, + "logps/chosen": -169.84701538085938, + "logps/ref_chosen": -68.62410736083984, + "logps/ref_rejected": -98.42886352539062, + "logps/rejected": -285.1634521484375, + "loss": 1.2733, + "margin_dpo/margin_mean": 85.51167297363281, + "margin_dpo/margin_std": 186.15402221679688, + "step": 111 + }, + { + "KL/chosen_KL_mean": -90.35836029052734, + "KL/mean": -112.38624572753906, + "KL/rejected_KL_mean": -134.4141387939453, + "KL/std": 86.84256744384766, + "epoch": 0.1644640234948605, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 44.05577087402344, + "fcm_dpo/q_t": 0.4818727374076843, + "grad_norm": 7.099332809448242, + "learning_rate": 4.942120794399002e-07, + "logits/chosen": -0.2616153955459595, + "logits/rejected": -0.2658056318759918, + "logps/chosen": -140.60800170898438, + "logps/ref_chosen": -50.24964141845703, + "logps/ref_rejected": -64.77442932128906, + "logps/rejected": -199.18856811523438, + "loss": 1.3241, + "margin_dpo/margin_mean": 44.05577087402344, + "margin_dpo/margin_std": 115.68550872802734, + "step": 112 + }, + { + "KL/chosen_KL_mean": -115.47476196289062, + "KL/mean": -136.68405151367188, + "KL/rejected_KL_mean": -157.8933563232422, + "KL/std": 115.38925170898438, + "epoch": 0.16593245227606462, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 42.41858673095703, + "fcm_dpo/q_t": 0.4827752709388733, + "grad_norm": 6.795632839202881, + "learning_rate": 4.939343162231841e-07, + "logits/chosen": -0.28555989265441895, + "logits/rejected": -0.27538594603538513, + "logps/chosen": -182.18771362304688, + "logps/ref_chosen": -66.71295166015625, + "logps/ref_rejected": -77.96870422363281, + "logps/rejected": -235.862060546875, + "loss": 1.327, + "margin_dpo/margin_mean": 42.41858673095703, + "margin_dpo/margin_std": 120.11697387695312, + "step": 113 + }, + { + "KL/chosen_KL_mean": -96.08192443847656, + "KL/mean": -136.51783752441406, + "KL/rejected_KL_mean": -176.95375061035156, + "KL/std": 128.7473602294922, + "epoch": 0.16740088105726872, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.87183380126953, + "fcm_dpo/q_t": 0.4677523374557495, + "grad_norm": 7.049834251403809, + "learning_rate": 4.936501251103751e-07, + "logits/chosen": -0.2732029855251312, + "logits/rejected": -0.2635844945907593, + "logps/chosen": -153.86700439453125, + "logps/ref_chosen": -57.78507995605469, + "logps/ref_rejected": -87.10966491699219, + "logps/rejected": -264.06341552734375, + "loss": 1.2745, + "margin_dpo/margin_mean": 80.87183380126953, + "margin_dpo/margin_std": 159.99867248535156, + "step": 114 + }, + { + "KL/chosen_KL_mean": -116.97596740722656, + "KL/mean": -145.7581787109375, + "KL/rejected_KL_mean": -174.54037475585938, + "KL/std": 134.91941833496094, + "epoch": 0.16886930983847284, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.56439971923828, + "fcm_dpo/q_t": 0.4768410921096802, + "grad_norm": 6.489831924438477, + "learning_rate": 4.933595135901732e-07, + "logits/chosen": -0.31962960958480835, + "logits/rejected": -0.32673972845077515, + "logps/chosen": -182.55862426757812, + "logps/ref_chosen": -65.5826416015625, + "logps/ref_rejected": -98.56552124023438, + "logps/rejected": -273.10589599609375, + "loss": 1.3161, + "margin_dpo/margin_mean": 57.56439971923828, + "margin_dpo/margin_std": 182.9209442138672, + "step": 115 + }, + { + "KL/chosen_KL_mean": -81.60614013671875, + "KL/mean": -113.20507049560547, + "KL/rejected_KL_mean": -144.80401611328125, + "KL/std": 99.23644256591797, + "epoch": 0.17033773861967694, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 63.19786834716797, + "fcm_dpo/q_t": 0.47415468096733093, + "grad_norm": 7.067198276519775, + "learning_rate": 4.930624893204624e-07, + "logits/chosen": -0.31514549255371094, + "logits/rejected": -0.32795295119285583, + "logps/chosen": -133.00645446777344, + "logps/ref_chosen": -51.40031433105469, + "logps/ref_rejected": -80.5218505859375, + "logps/rejected": -225.32586669921875, + "loss": 1.2933, + "margin_dpo/margin_mean": 63.19786071777344, + "margin_dpo/margin_std": 112.28499603271484, + "step": 116 + }, + { + "KL/chosen_KL_mean": -113.90751647949219, + "KL/mean": -129.44996643066406, + "KL/rejected_KL_mean": -144.99240112304688, + "KL/std": 108.0283203125, + "epoch": 0.17180616740088106, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 31.08489227294922, + "fcm_dpo/q_t": 0.4872492551803589, + "grad_norm": 7.091765880584717, + "learning_rate": 4.927590601281083e-07, + "logits/chosen": -0.3172186017036438, + "logits/rejected": -0.3100709021091461, + "logps/chosen": -183.2059326171875, + "logps/ref_chosen": -69.29840850830078, + "logps/ref_rejected": -66.583984375, + "logps/rejected": -211.57638549804688, + "loss": 1.3476, + "margin_dpo/margin_mean": 31.084890365600586, + "margin_dpo/margin_std": 132.21383666992188, + "step": 117 + }, + { + "KL/chosen_KL_mean": -81.68537902832031, + "KL/mean": -112.67160034179688, + "KL/rejected_KL_mean": -143.6578369140625, + "KL/std": 100.91627502441406, + "epoch": 0.17327459618208516, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.97246551513672, + "fcm_dpo/q_t": 0.4746900796890259, + "grad_norm": 6.520020008087158, + "learning_rate": 4.924492340087524e-07, + "logits/chosen": -0.3794967532157898, + "logits/rejected": -0.37046414613723755, + "logps/chosen": -137.32635498046875, + "logps/ref_chosen": -55.6409797668457, + "logps/ref_rejected": -75.66905975341797, + "logps/rejected": -219.32688903808594, + "loss": 1.2958, + "margin_dpo/margin_mean": 61.97245788574219, + "margin_dpo/margin_std": 119.08011627197266, + "step": 118 + }, + { + "KL/chosen_KL_mean": -119.08765411376953, + "KL/mean": -146.5293426513672, + "KL/rejected_KL_mean": -173.97103881835938, + "KL/std": 110.69290924072266, + "epoch": 0.17474302496328928, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 54.88337326049805, + "fcm_dpo/q_t": 0.4776170551776886, + "grad_norm": 6.680598735809326, + "learning_rate": 4.92133019126601e-07, + "logits/chosen": -0.3600790500640869, + "logits/rejected": -0.36749690771102905, + "logps/chosen": -192.59783935546875, + "logps/ref_chosen": -73.51019287109375, + "logps/ref_rejected": -102.977294921875, + "logps/rejected": -276.9483337402344, + "loss": 1.3114, + "margin_dpo/margin_mean": 54.88337326049805, + "margin_dpo/margin_std": 140.7128448486328, + "step": 119 + }, + { + "KL/chosen_KL_mean": -117.65304565429688, + "KL/mean": -159.9397735595703, + "KL/rejected_KL_mean": -202.22650146484375, + "KL/std": 132.9193115234375, + "epoch": 0.1762114537444934, + "fcm_dpo/beta": 0.0016584109980612993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 84.57344055175781, + "fcm_dpo/q_t": 0.46578913927078247, + "grad_norm": 8.834454536437988, + "learning_rate": 4.918104238142103e-07, + "logits/chosen": -0.3894355893135071, + "logits/rejected": -0.38755276799201965, + "logps/chosen": -194.4338836669922, + "logps/ref_chosen": -76.78083801269531, + "logps/ref_rejected": -108.02374267578125, + "logps/rejected": -310.250244140625, + "loss": 1.2665, + "margin_dpo/margin_mean": 84.57344055175781, + "margin_dpo/margin_std": 151.76153564453125, + "step": 120 + }, + { + "KL/chosen_KL_mean": -108.66337585449219, + "KL/mean": -158.3170928955078, + "KL/rejected_KL_mean": -207.9708251953125, + "KL/std": 133.39573669433594, + "epoch": 0.1776798825256975, + "fcm_dpo/beta": 0.005431335419416428, + "fcm_dpo/delta": 0.13696809113025665, + "fcm_dpo/margin": 99.30744934082031, + "fcm_dpo/q_t": 0.43542176485061646, + "grad_norm": 21.185283660888672, + "learning_rate": 4.91481456572267e-07, + "logits/chosen": -0.3457489609718323, + "logits/rejected": -0.348524272441864, + "logps/chosen": -170.45327758789062, + "logps/ref_chosen": -61.789894104003906, + "logps/ref_rejected": -109.99456787109375, + "logps/rejected": -317.96539306640625, + "loss": 1.096, + "margin_dpo/margin_mean": 99.30744934082031, + "margin_dpo/margin_std": 157.79302978515625, + "step": 121 + }, + { + "KL/chosen_KL_mean": -86.52001953125, + "KL/mean": -148.9921112060547, + "KL/rejected_KL_mean": -211.46420288085938, + "KL/std": 132.5182342529297, + "epoch": 0.17914831130690162, + "fcm_dpo/beta": 0.0012442920124158263, + "fcm_dpo/delta": -0.16569742560386658, + "fcm_dpo/margin": 124.94419860839844, + "fcm_dpo/q_t": 0.40711045265197754, + "grad_norm": 7.291841983795166, + "learning_rate": 4.911461260693638e-07, + "logits/chosen": -0.30201756954193115, + "logits/rejected": -0.31801754236221313, + "logps/chosen": -133.42222595214844, + "logps/ref_chosen": -46.9022102355957, + "logps/ref_rejected": -106.71418762207031, + "logps/rejected": -318.17840576171875, + "loss": 1.2453, + "margin_dpo/margin_mean": 124.9441909790039, + "margin_dpo/margin_std": 147.9832305908203, + "step": 122 + }, + { + "KL/chosen_KL_mean": -110.82725524902344, + "KL/mean": -149.17156982421875, + "KL/rejected_KL_mean": -187.515869140625, + "KL/std": 125.73814392089844, + "epoch": 0.18061674008810572, + "fcm_dpo/beta": 0.0012442920124158263, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.68860626220703, + "fcm_dpo/q_t": 0.4764896631240845, + "grad_norm": 5.018327236175537, + "learning_rate": 4.908044411417711e-07, + "logits/chosen": -0.2894030809402466, + "logits/rejected": -0.29247063398361206, + "logps/chosen": -172.1658935546875, + "logps/ref_chosen": -61.33863830566406, + "logps/ref_rejected": -87.775390625, + "logps/rejected": -275.291259765625, + "loss": 1.3046, + "margin_dpo/margin_mean": 76.68860626220703, + "margin_dpo/margin_std": 171.46743774414062, + "step": 123 + }, + { + "KL/chosen_KL_mean": -133.59381103515625, + "KL/mean": -191.1050262451172, + "KL/rejected_KL_mean": -248.61622619628906, + "KL/std": 179.1632080078125, + "epoch": 0.18208516886930984, + "fcm_dpo/beta": 0.0012442920124158263, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 115.02241516113281, + "fcm_dpo/q_t": 0.46520543098449707, + "grad_norm": 5.619283676147461, + "learning_rate": 4.904564107932048e-07, + "logits/chosen": -0.25618571043014526, + "logits/rejected": -0.2648266553878784, + "logps/chosen": -205.04214477539062, + "logps/ref_chosen": -71.44833374023438, + "logps/ref_rejected": -117.58056640625, + "logps/rejected": -366.19677734375, + "loss": 1.269, + "margin_dpo/margin_mean": 115.02241516113281, + "margin_dpo/margin_std": 230.38388061523438, + "step": 124 + }, + { + "KL/chosen_KL_mean": -101.74983215332031, + "KL/mean": -149.4581298828125, + "KL/rejected_KL_mean": -197.16644287109375, + "KL/std": 146.47589111328125, + "epoch": 0.18355359765051396, + "fcm_dpo/beta": 0.0012442920124158263, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 95.41659545898438, + "fcm_dpo/q_t": 0.4708002507686615, + "grad_norm": 5.07564640045166, + "learning_rate": 4.90102044194588e-07, + "logits/chosen": -0.18893632292747498, + "logits/rejected": -0.20313698053359985, + "logps/chosen": -151.88677978515625, + "logps/ref_chosen": -50.136940002441406, + "logps/ref_rejected": -83.98861694335938, + "logps/rejected": -281.1550598144531, + "loss": 1.2842, + "margin_dpo/margin_mean": 95.41659545898438, + "margin_dpo/margin_std": 181.84890747070312, + "step": 125 + }, + { + "KL/chosen_KL_mean": -110.52033996582031, + "KL/mean": -162.48785400390625, + "KL/rejected_KL_mean": -214.45535278320312, + "KL/std": 144.8026123046875, + "epoch": 0.18502202643171806, + "fcm_dpo/beta": 0.0012442920124158263, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 103.93502807617188, + "fcm_dpo/q_t": 0.46845173835754395, + "grad_norm": 5.675121307373047, + "learning_rate": 4.897413506838102e-07, + "logits/chosen": -0.2591179609298706, + "logits/rejected": -0.27047020196914673, + "logps/chosen": -166.18740844726562, + "logps/ref_chosen": -55.66706848144531, + "logps/ref_rejected": -98.1297607421875, + "logps/rejected": -312.5851135253906, + "loss": 1.2746, + "margin_dpo/margin_mean": 103.93502807617188, + "margin_dpo/margin_std": 185.01034545898438, + "step": 126 + }, + { + "KL/chosen_KL_mean": -114.12653350830078, + "KL/mean": -142.77590942382812, + "KL/rejected_KL_mean": -171.42526245117188, + "KL/std": 126.23076629638672, + "epoch": 0.18649045521292218, + "fcm_dpo/beta": 0.0012442920124158263, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.298728942871094, + "fcm_dpo/q_t": 0.4821869730949402, + "grad_norm": 5.324838638305664, + "learning_rate": 4.89374339765481e-07, + "logits/chosen": -0.2520139515399933, + "logits/rejected": -0.25741541385650635, + "logps/chosen": -170.68121337890625, + "logps/ref_chosen": -56.55467987060547, + "logps/ref_rejected": -76.7957763671875, + "logps/rejected": -248.22103881835938, + "loss": 1.3258, + "margin_dpo/margin_mean": 57.298728942871094, + "margin_dpo/margin_std": 157.20663452148438, + "step": 127 + }, + { + "KL/chosen_KL_mean": -117.71072387695312, + "KL/mean": -154.01956176757812, + "KL/rejected_KL_mean": -190.32843017578125, + "KL/std": 155.84530639648438, + "epoch": 0.18795888399412627, + "fcm_dpo/beta": 0.0012442920124158263, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 72.61770629882812, + "fcm_dpo/q_t": 0.4778425097465515, + "grad_norm": 6.066002368927002, + "learning_rate": 4.890010211106795e-07, + "logits/chosen": -0.23010198771953583, + "logits/rejected": -0.24306002259254456, + "logps/chosen": -175.83168029785156, + "logps/ref_chosen": -58.12095642089844, + "logps/ref_rejected": -76.43896484375, + "logps/rejected": -266.76739501953125, + "loss": 1.3139, + "margin_dpo/margin_mean": 72.61770629882812, + "margin_dpo/margin_std": 201.6669158935547, + "step": 128 + }, + { + "KL/chosen_KL_mean": -156.19540405273438, + "KL/mean": -194.95504760742188, + "KL/rejected_KL_mean": -233.71466064453125, + "KL/std": 197.4521484375, + "epoch": 0.1894273127753304, + "fcm_dpo/beta": 0.0012442920124158263, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 77.51925659179688, + "fcm_dpo/q_t": 0.47747713327407837, + "grad_norm": 6.777254581451416, + "learning_rate": 4.88621404556699e-07, + "logits/chosen": -0.2056041806936264, + "logits/rejected": -0.2097330093383789, + "logps/chosen": -223.11178588867188, + "logps/ref_chosen": -66.91637420654297, + "logps/ref_rejected": -96.6422119140625, + "logps/rejected": -330.35687255859375, + "loss": 1.3193, + "margin_dpo/margin_mean": 77.51925659179688, + "margin_dpo/margin_std": 269.54656982421875, + "step": 129 + }, + { + "KL/chosen_KL_mean": -108.04925537109375, + "KL/mean": -162.80902099609375, + "KL/rejected_KL_mean": -217.56878662109375, + "KL/std": 142.10467529296875, + "epoch": 0.19089574155653452, + "fcm_dpo/beta": 0.0012442920124158263, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 109.51951599121094, + "fcm_dpo/q_t": 0.4666077494621277, + "grad_norm": 7.042036533355713, + "learning_rate": 4.882355001067891e-07, + "logits/chosen": -0.15569573640823364, + "logits/rejected": -0.15476679801940918, + "logps/chosen": -152.7161102294922, + "logps/ref_chosen": -44.66685104370117, + "logps/ref_rejected": -82.78165435791016, + "logps/rejected": -300.3504333496094, + "loss": 1.2689, + "margin_dpo/margin_mean": 109.51950073242188, + "margin_dpo/margin_std": 188.17529296875, + "step": 130 + }, + { + "KL/chosen_KL_mean": -112.39668273925781, + "KL/mean": -174.34365844726562, + "KL/rejected_KL_mean": -236.29067993164062, + "KL/std": 177.99996948242188, + "epoch": 0.19236417033773862, + "fcm_dpo/beta": 0.0012442920124158263, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 123.89396667480469, + "fcm_dpo/q_t": 0.46196436882019043, + "grad_norm": 7.167151927947998, + "learning_rate": 4.878433179298909e-07, + "logits/chosen": -0.15293475985527039, + "logits/rejected": -0.15630626678466797, + "logps/chosen": -157.32127380371094, + "logps/ref_chosen": -44.924591064453125, + "logps/ref_rejected": -88.44401550292969, + "logps/rejected": -324.73468017578125, + "loss": 1.2519, + "margin_dpo/margin_mean": 123.89398193359375, + "margin_dpo/margin_std": 189.91864013671875, + "step": 131 + }, + { + "KL/chosen_KL_mean": -142.0668487548828, + "KL/mean": -188.7615966796875, + "KL/rejected_KL_mean": -235.45635986328125, + "KL/std": 182.57955932617188, + "epoch": 0.19383259911894274, + "fcm_dpo/beta": 0.0012442920124158263, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 93.3895034790039, + "fcm_dpo/q_t": 0.47222959995269775, + "grad_norm": 6.9561357498168945, + "learning_rate": 4.874448683603694e-07, + "logits/chosen": -0.1588452309370041, + "logits/rejected": -0.1891343593597412, + "logps/chosen": -201.06793212890625, + "logps/ref_chosen": -59.00108337402344, + "logps/ref_rejected": -87.89215087890625, + "logps/rejected": -323.3485107421875, + "loss": 1.295, + "margin_dpo/margin_mean": 93.3895034790039, + "margin_dpo/margin_std": 238.1549530029297, + "step": 132 + }, + { + "KL/chosen_KL_mean": -163.40499877929688, + "KL/mean": -203.98348999023438, + "KL/rejected_KL_mean": -244.56201171875, + "KL/std": 153.83062744140625, + "epoch": 0.19530102790014683, + "fcm_dpo/beta": 0.0012442920124158263, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 81.15701293945312, + "fcm_dpo/q_t": 0.47538745403289795, + "grad_norm": 7.251523494720459, + "learning_rate": 4.870401618977415e-07, + "logits/chosen": -0.15137168765068054, + "logits/rejected": -0.1648947298526764, + "logps/chosen": -230.00949096679688, + "logps/ref_chosen": -66.60449981689453, + "logps/ref_rejected": -96.33355712890625, + "logps/rejected": -340.89556884765625, + "loss": 1.3037, + "margin_dpo/margin_mean": 81.15701293945312, + "margin_dpo/margin_std": 203.13357543945312, + "step": 133 + }, + { + "KL/chosen_KL_mean": -126.16668701171875, + "KL/mean": -169.11993408203125, + "KL/rejected_KL_mean": -212.0731964111328, + "KL/std": 131.4796600341797, + "epoch": 0.19676945668135096, + "fcm_dpo/beta": 0.0012442920124158263, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 85.90650939941406, + "fcm_dpo/q_t": 0.47373300790786743, + "grad_norm": 7.264800071716309, + "learning_rate": 4.866292092063986e-07, + "logits/chosen": -0.1166238859295845, + "logits/rejected": -0.13339810073375702, + "logps/chosen": -178.23594665527344, + "logps/ref_chosen": -52.06925582885742, + "logps/ref_rejected": -87.6545181274414, + "logps/rejected": -299.72772216796875, + "loss": 1.2925, + "margin_dpo/margin_mean": 85.9065170288086, + "margin_dpo/margin_std": 161.20672607421875, + "step": 134 + }, + { + "KL/chosen_KL_mean": -130.97235107421875, + "KL/mean": -212.07064819335938, + "KL/rejected_KL_mean": -293.1689453125, + "KL/std": 209.9551544189453, + "epoch": 0.19823788546255505, + "fcm_dpo/beta": 0.0025665950961411, + "fcm_dpo/delta": 0.04211168363690376, + "fcm_dpo/margin": 162.19659423828125, + "fcm_dpo/q_t": 0.42933958768844604, + "grad_norm": 16.52530288696289, + "learning_rate": 4.862120211153265e-07, + "logits/chosen": -0.09394662082195282, + "logits/rejected": -0.10758954286575317, + "logps/chosen": -181.32620239257812, + "logps/ref_chosen": -50.353858947753906, + "logps/ref_rejected": -115.97975158691406, + "logps/rejected": -409.148681640625, + "loss": 1.1038, + "margin_dpo/margin_mean": 162.1966094970703, + "margin_dpo/margin_std": 266.60357666015625, + "step": 135 + }, + { + "KL/chosen_KL_mean": -160.09901428222656, + "KL/mean": -211.46200561523438, + "KL/rejected_KL_mean": -262.8249816894531, + "KL/std": 210.24295043945312, + "epoch": 0.19970631424375918, + "fcm_dpo/beta": 0.0018958807922899723, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 102.72596740722656, + "fcm_dpo/q_t": 0.45904627442359924, + "grad_norm": 9.262153625488281, + "learning_rate": 4.857886086178193e-07, + "logits/chosen": -0.1536802351474762, + "logits/rejected": -0.16803759336471558, + "logps/chosen": -225.17152404785156, + "logps/ref_chosen": -65.072509765625, + "logps/ref_rejected": -96.32122802734375, + "logps/rejected": -359.1462097167969, + "loss": 1.2667, + "margin_dpo/margin_mean": 102.72596740722656, + "margin_dpo/margin_std": 286.5235595703125, + "step": 136 + }, + { + "KL/chosen_KL_mean": -135.36795043945312, + "KL/mean": -214.21568298339844, + "KL/rejected_KL_mean": -293.06341552734375, + "KL/std": 230.4893798828125, + "epoch": 0.2011747430249633, + "fcm_dpo/beta": 0.002469564788043499, + "fcm_dpo/delta": -0.016328319907188416, + "fcm_dpo/margin": 157.69546508789062, + "fcm_dpo/q_t": 0.4195204973220825, + "grad_norm": 12.261474609375, + "learning_rate": 4.853589828711902e-07, + "logits/chosen": -0.10306895524263382, + "logits/rejected": -0.11605556309223175, + "logps/chosen": -184.1270751953125, + "logps/ref_chosen": -48.759117126464844, + "logps/ref_rejected": -113.86376953125, + "logps/rejected": -406.92718505859375, + "loss": 1.141, + "margin_dpo/margin_mean": 157.69546508789062, + "margin_dpo/margin_std": 293.899658203125, + "step": 137 + }, + { + "KL/chosen_KL_mean": -148.42919921875, + "KL/mean": -201.90768432617188, + "KL/rejected_KL_mean": -255.3861846923828, + "KL/std": 171.12557983398438, + "epoch": 0.2026431718061674, + "fcm_dpo/beta": 0.0016102675581350923, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 106.95697021484375, + "fcm_dpo/q_t": 0.4581053853034973, + "grad_norm": 9.790092468261719, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": -0.11189040541648865, + "logits/rejected": -0.11567579954862595, + "logps/chosen": -208.9488525390625, + "logps/ref_chosen": -60.519649505615234, + "logps/ref_rejected": -93.19694519042969, + "logps/rejected": -348.5831298828125, + "loss": 1.2412, + "margin_dpo/margin_mean": 106.95697021484375, + "margin_dpo/margin_std": 175.25364685058594, + "step": 138 + }, + { + "KL/chosen_KL_mean": -127.69628143310547, + "KL/mean": -184.2217559814453, + "KL/rejected_KL_mean": -240.7472381591797, + "KL/std": 150.19712829589844, + "epoch": 0.20411160058737152, + "fcm_dpo/beta": 0.0016102675581350923, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 113.05096435546875, + "fcm_dpo/q_t": 0.4562827944755554, + "grad_norm": 9.610363006591797, + "learning_rate": 4.844811370781446e-07, + "logits/chosen": -0.11106328666210175, + "logits/rejected": -0.11769875884056091, + "logps/chosen": -174.58766174316406, + "logps/ref_chosen": -46.89138412475586, + "logps/ref_rejected": -79.72798156738281, + "logps/rejected": -320.4752197265625, + "loss": 1.2391, + "margin_dpo/margin_mean": 113.05096435546875, + "margin_dpo/margin_std": 206.3931427001953, + "step": 139 + }, + { + "KL/chosen_KL_mean": -158.94659423828125, + "KL/mean": -203.65492248535156, + "KL/rejected_KL_mean": -248.36328125, + "KL/std": 166.0042724609375, + "epoch": 0.2055800293685756, + "fcm_dpo/beta": 0.0016102675581350923, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.41668701171875, + "fcm_dpo/q_t": 0.46475258469581604, + "grad_norm": 9.001358032226562, + "learning_rate": 4.840329401637809e-07, + "logits/chosen": -0.11058294773101807, + "logits/rejected": -0.12042254209518433, + "logps/chosen": -217.92129516601562, + "logps/ref_chosen": -58.97471618652344, + "logps/ref_rejected": -83.28410339355469, + "logps/rejected": -331.64739990234375, + "loss": 1.2763, + "margin_dpo/margin_mean": 89.41668701171875, + "margin_dpo/margin_std": 212.3289794921875, + "step": 140 + }, + { + "KL/chosen_KL_mean": -191.18380737304688, + "KL/mean": -233.52825927734375, + "KL/rejected_KL_mean": -275.8727111816406, + "KL/std": 238.5574188232422, + "epoch": 0.20704845814977973, + "fcm_dpo/beta": 0.0016102675581350923, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 84.68890380859375, + "fcm_dpo/q_t": 0.4663596749305725, + "grad_norm": 15.454120635986328, + "learning_rate": 4.83578576263792e-07, + "logits/chosen": -0.10689105093479156, + "logits/rejected": -0.12749376893043518, + "logps/chosen": -266.25946044921875, + "logps/ref_chosen": -75.07566833496094, + "logps/ref_rejected": -98.1922607421875, + "logps/rejected": -374.0649719238281, + "loss": 1.3095, + "margin_dpo/margin_mean": 84.68890380859375, + "margin_dpo/margin_std": 295.63360595703125, + "step": 141 + }, + { + "KL/chosen_KL_mean": -161.26834106445312, + "KL/mean": -219.98394775390625, + "KL/rejected_KL_mean": -278.6995849609375, + "KL/std": 205.99090576171875, + "epoch": 0.20851688693098386, + "fcm_dpo/beta": 0.002624105429276824, + "fcm_dpo/delta": 0.08150181919336319, + "fcm_dpo/margin": 117.43124389648438, + "fcm_dpo/q_t": 0.4544960558414459, + "grad_norm": 13.912692070007324, + "learning_rate": 4.83118057351089e-07, + "logits/chosen": -0.09879723191261292, + "logits/rejected": -0.11309005320072174, + "logps/chosen": -219.2962646484375, + "logps/ref_chosen": -58.027931213378906, + "logps/ref_rejected": -94.58222961425781, + "logps/rejected": -373.28179931640625, + "loss": 1.1618, + "margin_dpo/margin_mean": 117.43125915527344, + "margin_dpo/margin_std": 238.91827392578125, + "step": 142 + }, + { + "KL/chosen_KL_mean": -157.7050323486328, + "KL/mean": -193.36447143554688, + "KL/rejected_KL_mean": -229.02392578125, + "KL/std": 160.72268676757812, + "epoch": 0.20998531571218795, + "fcm_dpo/beta": 0.004260506015270948, + "fcm_dpo/delta": 0.015796946361660957, + "fcm_dpo/margin": 71.31889343261719, + "fcm_dpo/q_t": 0.4452533721923828, + "grad_norm": 27.70894432067871, + "learning_rate": 4.826513955607734e-07, + "logits/chosen": -0.09957142174243927, + "logits/rejected": -0.11266869306564331, + "logps/chosen": -215.30148315429688, + "logps/ref_chosen": -57.59645080566406, + "logps/ref_rejected": -78.99957275390625, + "logps/rejected": -308.02349853515625, + "loss": 1.2612, + "margin_dpo/margin_mean": 71.31889343261719, + "margin_dpo/margin_std": 196.57472229003906, + "step": 143 + }, + { + "KL/chosen_KL_mean": -155.84344482421875, + "KL/mean": -195.27426147460938, + "KL/rejected_KL_mean": -234.705078125, + "KL/std": 159.63229370117188, + "epoch": 0.21145374449339208, + "fcm_dpo/beta": 0.004793780855834484, + "fcm_dpo/delta": -0.03707936033606529, + "fcm_dpo/margin": 78.86161804199219, + "fcm_dpo/q_t": 0.4113781452178955, + "grad_norm": 32.351070404052734, + "learning_rate": 4.821786031898176e-07, + "logits/chosen": -0.12348122149705887, + "logits/rejected": -0.15042133629322052, + "logps/chosen": -215.74981689453125, + "logps/ref_chosen": -59.90636444091797, + "logps/ref_rejected": -82.00025939941406, + "logps/rejected": -316.705322265625, + "loss": 1.2729, + "margin_dpo/margin_mean": 78.86161804199219, + "margin_dpo/margin_std": 176.07415771484375, + "step": 144 + }, + { + "KL/chosen_KL_mean": -133.75552368164062, + "KL/mean": -175.75448608398438, + "KL/rejected_KL_mean": -217.75347900390625, + "KL/std": 131.30337524414062, + "epoch": 0.21292217327459617, + "fcm_dpo/beta": 0.004883944522589445, + "fcm_dpo/delta": 0.020339418202638626, + "fcm_dpo/margin": 83.99797058105469, + "fcm_dpo/q_t": 0.41977953910827637, + "grad_norm": 38.0810661315918, + "learning_rate": 4.816996926967401e-07, + "logits/chosen": -0.21709679067134857, + "logits/rejected": -0.23458248376846313, + "logps/chosen": -190.35618591308594, + "logps/ref_chosen": -56.60066604614258, + "logps/ref_rejected": -77.86631774902344, + "logps/rejected": -295.61981201171875, + "loss": 1.1677, + "margin_dpo/margin_mean": 83.99796295166016, + "margin_dpo/margin_std": 166.22833251953125, + "step": 145 + }, + { + "KL/chosen_KL_mean": -174.54449462890625, + "KL/mean": -203.33346557617188, + "KL/rejected_KL_mean": -232.12242126464844, + "KL/std": 144.7928009033203, + "epoch": 0.2143906020558003, + "fcm_dpo/beta": 0.009946160949766636, + "fcm_dpo/delta": 0.1288416087627411, + "fcm_dpo/margin": 57.57792663574219, + "fcm_dpo/q_t": 0.4425297677516937, + "grad_norm": 173.9798583984375, + "learning_rate": 4.812146767012779e-07, + "logits/chosen": -0.14733245968818665, + "logits/rejected": -0.1549091786146164, + "logps/chosen": -240.54495239257812, + "logps/ref_chosen": -66.00045013427734, + "logps/ref_rejected": -81.70278930664062, + "logps/rejected": -313.8251953125, + "loss": 1.6518, + "margin_dpo/margin_mean": 57.57793426513672, + "margin_dpo/margin_std": 177.08001708984375, + "step": 146 + }, + { + "KL/chosen_KL_mean": -131.95578002929688, + "KL/mean": -173.01547241210938, + "KL/rejected_KL_mean": -214.07516479492188, + "KL/std": 144.98574829101562, + "epoch": 0.21585903083700442, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": -0.4109819829463959, + "fcm_dpo/margin": 82.11940002441406, + "fcm_dpo/q_t": 0.42657575011253357, + "grad_norm": 1.725198745727539, + "learning_rate": 4.807235679840536e-07, + "logits/chosen": -0.20732438564300537, + "logits/rejected": -0.22212302684783936, + "logps/chosen": -185.36126708984375, + "logps/ref_chosen": -53.405487060546875, + "logps/ref_rejected": -71.39060974121094, + "logps/rejected": -285.4657897949219, + "loss": 1.3691, + "margin_dpo/margin_mean": 82.11940002441406, + "margin_dpo/margin_std": 182.35836791992188, + "step": 147 + }, + { + "KL/chosen_KL_mean": -140.5421905517578, + "KL/mean": -184.202392578125, + "KL/rejected_KL_mean": -227.8626251220703, + "KL/std": 168.81472778320312, + "epoch": 0.2173274596182085, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 87.32041931152344, + "fcm_dpo/q_t": 0.4953227639198303, + "grad_norm": 2.0062618255615234, + "learning_rate": 4.802263794862384e-07, + "logits/chosen": -0.23836994171142578, + "logits/rejected": -0.24250321090221405, + "logps/chosen": -205.47927856445312, + "logps/ref_chosen": -64.93708038330078, + "logps/ref_rejected": -103.09384155273438, + "logps/rejected": -330.95648193359375, + "loss": 1.3681, + "margin_dpo/margin_mean": 87.32041931152344, + "margin_dpo/margin_std": 185.8931884765625, + "step": 148 + }, + { + "KL/chosen_KL_mean": -124.4274673461914, + "KL/mean": -176.48355102539062, + "KL/rejected_KL_mean": -228.5396270751953, + "KL/std": 142.7633514404297, + "epoch": 0.21879588839941264, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.11214447021484, + "fcm_dpo/q_t": 0.4944196343421936, + "grad_norm": 1.6070046424865723, + "learning_rate": 4.797231243092118e-07, + "logits/chosen": -0.20803114771842957, + "logits/rejected": -0.21203464269638062, + "logps/chosen": -182.90122985839844, + "logps/ref_chosen": -58.47376251220703, + "logps/ref_rejected": -99.31474304199219, + "logps/rejected": -327.8543701171875, + "loss": 1.3645, + "margin_dpo/margin_mean": 104.11215209960938, + "margin_dpo/margin_std": 176.35842895507812, + "step": 149 + }, + { + "KL/chosen_KL_mean": -107.64643096923828, + "KL/mean": -164.7757568359375, + "KL/rejected_KL_mean": -221.9050750732422, + "KL/std": 159.32635498046875, + "epoch": 0.22026431718061673, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 114.25865173339844, + "fcm_dpo/q_t": 0.49387815594673157, + "grad_norm": 1.4557676315307617, + "learning_rate": 4.792138157142157e-07, + "logits/chosen": -0.2157842516899109, + "logits/rejected": -0.2243994176387787, + "logps/chosen": -153.35223388671875, + "logps/ref_chosen": -45.705810546875, + "logps/ref_rejected": -83.34759521484375, + "logps/rejected": -305.252685546875, + "loss": 1.3624, + "margin_dpo/margin_mean": 114.2586441040039, + "margin_dpo/margin_std": 193.01649475097656, + "step": 150 + }, + { + "KL/chosen_KL_mean": -149.71823120117188, + "KL/mean": -199.18092346191406, + "KL/rejected_KL_mean": -248.64361572265625, + "KL/std": 174.56417846679688, + "epoch": 0.22173274596182085, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 98.92537689208984, + "fcm_dpo/q_t": 0.49469929933547974, + "grad_norm": 1.555479884147644, + "learning_rate": 4.786984671220053e-07, + "logits/chosen": -0.2548075318336487, + "logits/rejected": -0.2567106783390045, + "logps/chosen": -220.2890625, + "logps/ref_chosen": -70.57083129882812, + "logps/ref_rejected": -100.46382141113281, + "logps/rejected": -349.107421875, + "loss": 1.3657, + "margin_dpo/margin_mean": 98.92536926269531, + "margin_dpo/margin_std": 200.89501953125, + "step": 151 + }, + { + "KL/chosen_KL_mean": -130.66839599609375, + "KL/mean": -196.2487335205078, + "KL/rejected_KL_mean": -261.8291015625, + "KL/std": 162.598876953125, + "epoch": 0.22320117474302498, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 131.16065979003906, + "fcm_dpo/q_t": 0.4929710030555725, + "grad_norm": 1.7390542030334473, + "learning_rate": 4.78177092112495e-07, + "logits/chosen": -0.22812870144844055, + "logits/rejected": -0.23715990781784058, + "logps/chosen": -190.83279418945312, + "logps/ref_chosen": -60.16438674926758, + "logps/ref_rejected": -106.14045715332031, + "logps/rejected": -367.96954345703125, + "loss": 1.3588, + "margin_dpo/margin_mean": 131.16067504882812, + "margin_dpo/margin_std": 200.10365295410156, + "step": 152 + }, + { + "KL/chosen_KL_mean": -126.16310119628906, + "KL/mean": -180.94070434570312, + "KL/rejected_KL_mean": -235.71829223632812, + "KL/std": 180.3782501220703, + "epoch": 0.22466960352422907, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 109.55519104003906, + "fcm_dpo/q_t": 0.49413180351257324, + "grad_norm": 1.530712366104126, + "learning_rate": 4.776497044244016e-07, + "logits/chosen": -0.21236395835876465, + "logits/rejected": -0.22474971413612366, + "logps/chosen": -182.47837829589844, + "logps/ref_chosen": -56.315277099609375, + "logps/ref_rejected": -85.65583801269531, + "logps/rejected": -321.3741149902344, + "loss": 1.3636, + "margin_dpo/margin_mean": 109.55519104003906, + "margin_dpo/margin_std": 232.18374633789062, + "step": 153 + }, + { + "KL/chosen_KL_mean": -158.0811767578125, + "KL/mean": -203.92166137695312, + "KL/rejected_KL_mean": -249.7621307373047, + "KL/std": 173.6383514404297, + "epoch": 0.2261380323054332, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.68096923828125, + "fcm_dpo/q_t": 0.4950861930847168, + "grad_norm": 1.4504088163375854, + "learning_rate": 4.771163179548808e-07, + "logits/chosen": -0.21836550533771515, + "logits/rejected": -0.23400922119617462, + "logps/chosen": -220.82374572753906, + "logps/ref_chosen": -62.74256896972656, + "logps/ref_rejected": -104.24420166015625, + "logps/rejected": -354.00634765625, + "loss": 1.3674, + "margin_dpo/margin_mean": 91.68096923828125, + "margin_dpo/margin_std": 240.62171936035156, + "step": 154 + }, + { + "KL/chosen_KL_mean": -142.25665283203125, + "KL/mean": -183.7818145751953, + "KL/rejected_KL_mean": -225.3070068359375, + "KL/std": 151.2476348876953, + "epoch": 0.2276064610866373, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 83.05033874511719, + "fcm_dpo/q_t": 0.49554821848869324, + "grad_norm": 1.3787976503372192, + "learning_rate": 4.7657694675916247e-07, + "logits/chosen": -0.23564405739307404, + "logits/rejected": -0.23982316255569458, + "logps/chosen": -202.9098358154297, + "logps/ref_chosen": -60.65318298339844, + "logps/ref_rejected": -77.49220275878906, + "logps/rejected": -302.7991943359375, + "loss": 1.369, + "margin_dpo/margin_mean": 83.05035400390625, + "margin_dpo/margin_std": 198.95108032226562, + "step": 155 + }, + { + "KL/chosen_KL_mean": -190.4660186767578, + "KL/mean": -208.18328857421875, + "KL/rejected_KL_mean": -225.90054321289062, + "KL/std": 172.94052124023438, + "epoch": 0.2290748898678414, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 35.43452453613281, + "fcm_dpo/q_t": 0.49810007214546204, + "grad_norm": 1.2596914768218994, + "learning_rate": 4.7603160505017893e-07, + "logits/chosen": -0.19964438676834106, + "logits/rejected": -0.22770291566848755, + "logps/chosen": -259.9578857421875, + "logps/ref_chosen": -69.49188232421875, + "logps/ref_rejected": -77.16929626464844, + "logps/rejected": -303.06982421875, + "loss": 1.3793, + "margin_dpo/margin_mean": 35.43452453613281, + "margin_dpo/margin_std": 223.39340209960938, + "step": 156 + }, + { + "KL/chosen_KL_mean": -160.28289794921875, + "KL/mean": -230.226806640625, + "KL/rejected_KL_mean": -300.17071533203125, + "KL/std": 185.30099487304688, + "epoch": 0.2305433186490455, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 139.88784790039062, + "fcm_dpo/q_t": 0.49250328540802, + "grad_norm": 1.719684362411499, + "learning_rate": 4.7548030719819154e-07, + "logits/chosen": -0.09867404401302338, + "logits/rejected": -0.11982227861881256, + "logps/chosen": -221.65133666992188, + "logps/ref_chosen": -61.368438720703125, + "logps/ref_rejected": -107.64636993408203, + "logps/rejected": -407.8171081542969, + "loss": 1.3572, + "margin_dpo/margin_mean": 139.88784790039062, + "margin_dpo/margin_std": 224.976318359375, + "step": 157 + }, + { + "KL/chosen_KL_mean": -170.56492614746094, + "KL/mean": -253.34765625, + "KL/rejected_KL_mean": -336.1304016113281, + "KL/std": 241.12974548339844, + "epoch": 0.23201174743024963, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 165.5655059814453, + "fcm_dpo/q_t": 0.491136372089386, + "grad_norm": 2.193638324737549, + "learning_rate": 4.7492306773041136e-07, + "logits/chosen": -0.13828858733177185, + "logits/rejected": -0.16126753389835358, + "logps/chosen": -228.17784118652344, + "logps/ref_chosen": -57.612918853759766, + "logps/ref_rejected": -113.6946792602539, + "logps/rejected": -449.8250732421875, + "loss": 1.3521, + "margin_dpo/margin_mean": 165.5655059814453, + "margin_dpo/margin_std": 297.131591796875, + "step": 158 + }, + { + "KL/chosen_KL_mean": -226.7890167236328, + "KL/mean": -262.0523681640625, + "KL/rejected_KL_mean": -297.31573486328125, + "KL/std": 231.03848266601562, + "epoch": 0.23348017621145375, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 70.52671813964844, + "fcm_dpo/q_t": 0.49621930718421936, + "grad_norm": 1.778730869293213, + "learning_rate": 4.743599013306165e-07, + "logits/chosen": -0.11859369277954102, + "logits/rejected": -0.12011769413948059, + "logps/chosen": -308.349365234375, + "logps/ref_chosen": -81.56034851074219, + "logps/ref_rejected": -88.89871215820312, + "logps/rejected": -386.21441650390625, + "loss": 1.372, + "margin_dpo/margin_mean": 70.52671813964844, + "margin_dpo/margin_std": 254.50335693359375, + "step": 159 + }, + { + "KL/chosen_KL_mean": -208.1883544921875, + "KL/mean": -273.8866271972656, + "KL/rejected_KL_mean": -339.5848693847656, + "KL/std": 218.83474731445312, + "epoch": 0.23494860499265785, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 131.39654541015625, + "fcm_dpo/q_t": 0.4929630160331726, + "grad_norm": 1.8856486082077026, + "learning_rate": 4.737908228387656e-07, + "logits/chosen": -0.12426799535751343, + "logits/rejected": -0.14046964049339294, + "logps/chosen": -273.91925048828125, + "logps/ref_chosen": -65.73088073730469, + "logps/ref_rejected": -97.21781921386719, + "logps/rejected": -436.80267333984375, + "loss": 1.3592, + "margin_dpo/margin_mean": 131.3965301513672, + "margin_dpo/margin_std": 274.5325927734375, + "step": 160 + }, + { + "KL/chosen_KL_mean": -171.75802612304688, + "KL/mean": -228.2024688720703, + "KL/rejected_KL_mean": -284.64691162109375, + "KL/std": 182.69235229492188, + "epoch": 0.23641703377386197, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 112.88887786865234, + "fcm_dpo/q_t": 0.4939511716365814, + "grad_norm": 1.585429310798645, + "learning_rate": 4.7321584725060594e-07, + "logits/chosen": -0.12636083364486694, + "logits/rejected": -0.1504145711660385, + "logps/chosen": -224.1945037841797, + "logps/ref_chosen": -52.43647003173828, + "logps/ref_rejected": -83.43095397949219, + "logps/rejected": -368.0778503417969, + "loss": 1.3628, + "margin_dpo/margin_mean": 112.88887786865234, + "margin_dpo/margin_std": 216.60546875, + "step": 161 + }, + { + "KL/chosen_KL_mean": -180.044921875, + "KL/mean": -228.87269592285156, + "KL/rejected_KL_mean": -277.700439453125, + "KL/std": 198.00787353515625, + "epoch": 0.23788546255506607, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 97.65554809570312, + "fcm_dpo/q_t": 0.49477049708366394, + "grad_norm": 1.6253687143325806, + "learning_rate": 4.7263498971727905e-07, + "logits/chosen": -0.20148800313472748, + "logits/rejected": -0.2152554988861084, + "logps/chosen": -242.655517578125, + "logps/ref_chosen": -62.6105842590332, + "logps/ref_rejected": -89.39057922363281, + "logps/rejected": -367.091064453125, + "loss": 1.3662, + "margin_dpo/margin_mean": 97.65554809570312, + "margin_dpo/margin_std": 248.9880828857422, + "step": 162 + }, + { + "KL/chosen_KL_mean": -198.3851776123047, + "KL/mean": -241.20689392089844, + "KL/rejected_KL_mean": -284.0285949707031, + "KL/std": 196.6535186767578, + "epoch": 0.2393538913362702, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 85.64341735839844, + "fcm_dpo/q_t": 0.4954114258289337, + "grad_norm": 1.5751463174819946, + "learning_rate": 4.720482655449212e-07, + "logits/chosen": -0.11102645099163055, + "logits/rejected": -0.11732280254364014, + "logps/chosen": -253.4068145751953, + "logps/ref_chosen": -55.021629333496094, + "logps/ref_rejected": -75.418212890625, + "logps/rejected": -359.44683837890625, + "loss": 1.3687, + "margin_dpo/margin_mean": 85.64341735839844, + "margin_dpo/margin_std": 241.90493774414062, + "step": 163 + }, + { + "KL/chosen_KL_mean": -186.4625244140625, + "KL/mean": -250.75245666503906, + "KL/rejected_KL_mean": -315.0423583984375, + "KL/std": 190.93157958984375, + "epoch": 0.24082232011747431, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 128.579833984375, + "fcm_dpo/q_t": 0.49310967326164246, + "grad_norm": 1.862160325050354, + "learning_rate": 4.714556901942599e-07, + "logits/chosen": -0.07840365171432495, + "logits/rejected": -0.09606952965259552, + "logps/chosen": -242.10321044921875, + "logps/ref_chosen": -55.64066696166992, + "logps/ref_rejected": -79.66463470458984, + "logps/rejected": -394.7070007324219, + "loss": 1.3595, + "margin_dpo/margin_mean": 128.579833984375, + "margin_dpo/margin_std": 218.310302734375, + "step": 164 + }, + { + "KL/chosen_KL_mean": -206.51309204101562, + "KL/mean": -245.36529541015625, + "KL/rejected_KL_mean": -284.2174987792969, + "KL/std": 193.79730224609375, + "epoch": 0.2422907488986784, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 77.70442199707031, + "fcm_dpo/q_t": 0.49583813548088074, + "grad_norm": 1.5955755710601807, + "learning_rate": 4.708572792802069e-07, + "logits/chosen": -0.10220310091972351, + "logits/rejected": -0.1038433313369751, + "logps/chosen": -267.82379150390625, + "logps/ref_chosen": -61.310691833496094, + "logps/ref_rejected": -73.67060852050781, + "logps/rejected": -357.88812255859375, + "loss": 1.3706, + "margin_dpo/margin_mean": 77.70442199707031, + "margin_dpo/margin_std": 272.9258117675781, + "step": 165 + }, + { + "KL/chosen_KL_mean": -183.43780517578125, + "KL/mean": -275.9177551269531, + "KL/rejected_KL_mean": -368.397705078125, + "KL/std": 286.80517578125, + "epoch": 0.24375917767988253, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 184.95986938476562, + "fcm_dpo/q_t": 0.4901168942451477, + "grad_norm": 2.0007615089416504, + "learning_rate": 4.702530485714461e-07, + "logits/chosen": -0.11677154153585434, + "logits/rejected": -0.1256922483444214, + "logps/chosen": -234.42141723632812, + "logps/ref_chosen": -50.98360061645508, + "logps/ref_rejected": -98.09512329101562, + "logps/rejected": -466.4927978515625, + "loss": 1.3487, + "margin_dpo/margin_mean": 184.95986938476562, + "margin_dpo/margin_std": 381.46697998046875, + "step": 166 + }, + { + "KL/chosen_KL_mean": -194.1734619140625, + "KL/mean": -280.3365173339844, + "KL/rejected_KL_mean": -366.4996032714844, + "KL/std": 239.65505981445312, + "epoch": 0.24522760646108663, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 172.326171875, + "fcm_dpo/q_t": 0.49077457189559937, + "grad_norm": 2.102647542953491, + "learning_rate": 4.6964301399001877e-07, + "logits/chosen": -0.1024477556347847, + "logits/rejected": -0.11726316809654236, + "logps/chosen": -244.59754943847656, + "logps/ref_chosen": -50.424095153808594, + "logps/ref_rejected": -96.03042602539062, + "logps/rejected": -462.530029296875, + "loss": 1.3508, + "margin_dpo/margin_mean": 172.32615661621094, + "margin_dpo/margin_std": 293.95623779296875, + "step": 167 + }, + { + "KL/chosen_KL_mean": -213.13294982910156, + "KL/mean": -268.35931396484375, + "KL/rejected_KL_mean": -323.58563232421875, + "KL/std": 203.27450561523438, + "epoch": 0.24669603524229075, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 110.45269775390625, + "fcm_dpo/q_t": 0.49408331513404846, + "grad_norm": 1.7750434875488281, + "learning_rate": 4.690271916109034e-07, + "logits/chosen": -0.03718934208154678, + "logits/rejected": -0.04697761684656143, + "logps/chosen": -262.59576416015625, + "logps/ref_chosen": -49.462825775146484, + "logps/ref_rejected": -75.30855560302734, + "logps/rejected": -398.8941650390625, + "loss": 1.3635, + "margin_dpo/margin_mean": 110.45269775390625, + "margin_dpo/margin_std": 252.35952758789062, + "step": 168 + }, + { + "KL/chosen_KL_mean": -244.06619262695312, + "KL/mean": -290.67816162109375, + "KL/rejected_KL_mean": -337.2901306152344, + "KL/std": 225.60400390625, + "epoch": 0.24816446402349487, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 93.22395324707031, + "fcm_dpo/q_t": 0.4950127601623535, + "grad_norm": 2.7092039585113525, + "learning_rate": 4.6840559766159235e-07, + "logits/chosen": -0.061293989419937134, + "logits/rejected": -0.06470146030187607, + "logps/chosen": -303.86962890625, + "logps/ref_chosen": -59.803443908691406, + "logps/ref_rejected": -83.34574890136719, + "logps/rejected": -420.6358947753906, + "loss": 1.3676, + "margin_dpo/margin_mean": 93.22394561767578, + "margin_dpo/margin_std": 315.66082763671875, + "step": 169 + }, + { + "KL/chosen_KL_mean": -223.83151245117188, + "KL/mean": -274.68341064453125, + "KL/rejected_KL_mean": -325.53533935546875, + "KL/std": 236.1355743408203, + "epoch": 0.24963289280469897, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 101.70378112792969, + "fcm_dpo/q_t": 0.49454981088638306, + "grad_norm": 2.4471933841705322, + "learning_rate": 4.6777824852166437e-07, + "logits/chosen": -0.06102032959461212, + "logits/rejected": -0.07576321065425873, + "logps/chosen": -273.30328369140625, + "logps/ref_chosen": -49.471771240234375, + "logps/ref_rejected": -75.91734313964844, + "logps/rejected": -401.45269775390625, + "loss": 1.3656, + "margin_dpo/margin_mean": 101.70378112792969, + "margin_dpo/margin_std": 283.95513916015625, + "step": 170 + }, + { + "KL/chosen_KL_mean": -346.0281066894531, + "KL/mean": -386.1025390625, + "KL/rejected_KL_mean": -426.177001953125, + "KL/std": 327.9516296386719, + "epoch": 0.2511013215859031, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.14892578125, + "fcm_dpo/q_t": 0.4957178831100464, + "grad_norm": 4.012329578399658, + "learning_rate": 4.6714516072235273e-07, + "logits/chosen": -0.04151609167456627, + "logits/rejected": -0.0675613209605217, + "logps/chosen": -430.52740478515625, + "logps/ref_chosen": -84.49931335449219, + "logps/ref_rejected": -109.38209533691406, + "logps/rejected": -535.55908203125, + "loss": 1.3712, + "margin_dpo/margin_mean": 80.14892578125, + "margin_dpo/margin_std": 414.32763671875, + "step": 171 + }, + { + "KL/chosen_KL_mean": -278.4682312011719, + "KL/mean": -318.6019287109375, + "KL/rejected_KL_mean": -358.73565673828125, + "KL/std": 296.55731201171875, + "epoch": 0.2525697503671072, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.26742553710938, + "fcm_dpo/q_t": 0.49569272994995117, + "grad_norm": 2.9481098651885986, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": -0.057665470987558365, + "logits/rejected": -0.08217429369688034, + "logps/chosen": -347.12213134765625, + "logps/ref_chosen": -68.65391540527344, + "logps/ref_rejected": -85.43667602539062, + "logps/rejected": -444.1723327636719, + "loss": 1.3703, + "margin_dpo/margin_mean": 80.26742553710938, + "margin_dpo/margin_std": 307.19091796875, + "step": 172 + }, + { + "KL/chosen_KL_mean": -234.19972229003906, + "KL/mean": -281.9803771972656, + "KL/rejected_KL_mean": -329.7610168457031, + "KL/std": 252.82623291015625, + "epoch": 0.2540381791483113, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 95.56130981445312, + "fcm_dpo/q_t": 0.4948866665363312, + "grad_norm": 1.8302026987075806, + "learning_rate": 4.6586183602616687e-07, + "logits/chosen": -0.0890713781118393, + "logits/rejected": -0.1072927713394165, + "logps/chosen": -297.2506103515625, + "logps/ref_chosen": -63.050880432128906, + "logps/ref_rejected": -78.68392181396484, + "logps/rejected": -408.4449462890625, + "loss": 1.3668, + "margin_dpo/margin_mean": 95.5613021850586, + "margin_dpo/margin_std": 271.940673828125, + "step": 173 + }, + { + "KL/chosen_KL_mean": -216.39028930664062, + "KL/mean": -295.0453796386719, + "KL/rejected_KL_mean": -373.700439453125, + "KL/std": 275.541748046875, + "epoch": 0.2555066079295154, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 157.31015014648438, + "fcm_dpo/q_t": 0.49158352613449097, + "grad_norm": 3.099987268447876, + "learning_rate": 4.652116329460919e-07, + "logits/chosen": -0.02464154362678528, + "logits/rejected": -0.03820271044969559, + "logps/chosen": -269.7532653808594, + "logps/ref_chosen": -53.36296844482422, + "logps/ref_rejected": -101.91120910644531, + "logps/rejected": -475.61163330078125, + "loss": 1.3543, + "margin_dpo/margin_mean": 157.31015014648438, + "margin_dpo/margin_std": 358.8941650390625, + "step": 174 + }, + { + "KL/chosen_KL_mean": -168.59716796875, + "KL/mean": -259.63287353515625, + "KL/rejected_KL_mean": -350.6685485839844, + "KL/std": 224.63267517089844, + "epoch": 0.25697503671071953, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 182.07138061523438, + "fcm_dpo/q_t": 0.4902472496032715, + "grad_norm": 3.9441075325012207, + "learning_rate": 4.645557588393406e-07, + "logits/chosen": -0.058357544243335724, + "logits/rejected": -0.0670979917049408, + "logps/chosen": -214.01492309570312, + "logps/ref_chosen": -45.417762756347656, + "logps/ref_rejected": -89.50579833984375, + "logps/rejected": -440.1743469238281, + "loss": 1.3484, + "margin_dpo/margin_mean": 182.07138061523438, + "margin_dpo/margin_std": 263.89202880859375, + "step": 175 + }, + { + "KL/chosen_KL_mean": -193.49896240234375, + "KL/mean": -283.0251159667969, + "KL/rejected_KL_mean": -372.55126953125, + "KL/std": 272.05682373046875, + "epoch": 0.25844346549192365, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 179.05230712890625, + "fcm_dpo/q_t": 0.49042147397994995, + "grad_norm": 3.722703456878662, + "learning_rate": 4.638942309888058e-07, + "logits/chosen": -0.07522699236869812, + "logits/rejected": -0.08893322944641113, + "logps/chosen": -243.95181274414062, + "logps/ref_chosen": -50.452842712402344, + "logps/ref_rejected": -95.5589599609375, + "logps/rejected": -468.1102294921875, + "loss": 1.3495, + "margin_dpo/margin_mean": 179.05230712890625, + "margin_dpo/margin_std": 324.3958435058594, + "step": 176 + }, + { + "KL/chosen_KL_mean": -248.76412963867188, + "KL/mean": -327.17547607421875, + "KL/rejected_KL_mean": -405.5867919921875, + "KL/std": 315.8409423828125, + "epoch": 0.2599118942731278, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 156.8226318359375, + "fcm_dpo/q_t": 0.4916095733642578, + "grad_norm": 2.4874935150146484, + "learning_rate": 4.6322706682636137e-07, + "logits/chosen": -0.09781675040721893, + "logits/rejected": -0.12420766055583954, + "logps/chosen": -309.9805908203125, + "logps/ref_chosen": -61.216468811035156, + "logps/ref_rejected": -95.89378356933594, + "logps/rejected": -501.4805908203125, + "loss": 1.3544, + "margin_dpo/margin_mean": 156.82264709472656, + "margin_dpo/margin_std": 351.21734619140625, + "step": 177 + }, + { + "KL/chosen_KL_mean": -260.517578125, + "KL/mean": -355.90155029296875, + "KL/rejected_KL_mean": -451.2855224609375, + "KL/std": 332.3189697265625, + "epoch": 0.26138032305433184, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 190.7679443359375, + "fcm_dpo/q_t": 0.48978862166404724, + "grad_norm": 3.6528143882751465, + "learning_rate": 4.6255428393240354e-07, + "logits/chosen": -0.004009534604847431, + "logits/rejected": -0.022813618183135986, + "logps/chosen": -318.7823486328125, + "logps/ref_chosen": -58.26478958129883, + "logps/ref_rejected": -105.3653335571289, + "logps/rejected": -556.65087890625, + "loss": 1.3472, + "margin_dpo/margin_mean": 190.7679443359375, + "margin_dpo/margin_std": 349.21771240234375, + "step": 178 + }, + { + "KL/chosen_KL_mean": -270.78173828125, + "KL/mean": -333.99163818359375, + "KL/rejected_KL_mean": -397.20159912109375, + "KL/std": 320.001708984375, + "epoch": 0.26284875183553597, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 126.41985321044922, + "fcm_dpo/q_t": 0.4932493567466736, + "grad_norm": 2.433837652206421, + "learning_rate": 4.6187590003538724e-07, + "logits/chosen": -0.06824183464050293, + "logits/rejected": -0.09712938964366913, + "logps/chosen": -331.84002685546875, + "logps/ref_chosen": -61.05832290649414, + "logps/ref_rejected": -90.52782440185547, + "logps/rejected": -487.72943115234375, + "loss": 1.3614, + "margin_dpo/margin_mean": 126.41983795166016, + "margin_dpo/margin_std": 422.2001037597656, + "step": 179 + }, + { + "KL/chosen_KL_mean": -251.91395568847656, + "KL/mean": -345.0408935546875, + "KL/rejected_KL_mean": -438.16778564453125, + "KL/std": 260.55181884765625, + "epoch": 0.2643171806167401, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 186.25384521484375, + "fcm_dpo/q_t": 0.4900313913822174, + "grad_norm": 2.2225091457366943, + "learning_rate": 4.611919330113591e-07, + "logits/chosen": -0.046570487320423126, + "logits/rejected": -0.06946319341659546, + "logps/chosen": -306.2566833496094, + "logps/ref_chosen": -54.34272003173828, + "logps/ref_rejected": -98.21183776855469, + "logps/rejected": -536.379638671875, + "loss": 1.3482, + "margin_dpo/margin_mean": 186.25384521484375, + "margin_dpo/margin_std": 353.3677978515625, + "step": 180 + }, + { + "KL/chosen_KL_mean": -218.62420654296875, + "KL/mean": -267.3025207519531, + "KL/rejected_KL_mean": -315.9808654785156, + "KL/std": 229.04489135742188, + "epoch": 0.2657856093979442, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 97.35667419433594, + "fcm_dpo/q_t": 0.49478650093078613, + "grad_norm": 2.206993341445923, + "learning_rate": 4.605024008834863e-07, + "logits/chosen": -0.05855263024568558, + "logits/rejected": -0.06639339029788971, + "logps/chosen": -273.6246643066406, + "logps/ref_chosen": -55.000457763671875, + "logps/ref_rejected": -61.656166076660156, + "logps/rejected": -377.63702392578125, + "loss": 1.3666, + "margin_dpo/margin_mean": 97.35667419433594, + "margin_dpo/margin_std": 312.8726806640625, + "step": 181 + }, + { + "KL/chosen_KL_mean": -229.5802001953125, + "KL/mean": -339.115966796875, + "KL/rejected_KL_mean": -448.6517333984375, + "KL/std": 309.89312744140625, + "epoch": 0.26725403817914833, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 219.07156372070312, + "fcm_dpo/q_t": 0.48828235268592834, + "grad_norm": 2.526320695877075, + "learning_rate": 4.598073218215817e-07, + "logits/chosen": -0.07598879933357239, + "logits/rejected": -0.09682799130678177, + "logps/chosen": -270.68804931640625, + "logps/ref_chosen": -41.107852935791016, + "logps/ref_rejected": -89.5215835571289, + "logps/rejected": -538.17333984375, + "loss": 1.3417, + "margin_dpo/margin_mean": 219.07154846191406, + "margin_dpo/margin_std": 401.08624267578125, + "step": 182 + }, + { + "KL/chosen_KL_mean": -323.77984619140625, + "KL/mean": -366.191162109375, + "KL/rejected_KL_mean": -408.6024475097656, + "KL/std": 279.89532470703125, + "epoch": 0.2687224669603524, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 84.82255554199219, + "fcm_dpo/q_t": 0.49546507000923157, + "grad_norm": 3.5669703483581543, + "learning_rate": 4.5910671414162484e-07, + "logits/chosen": -0.05871109664440155, + "logits/rejected": -0.08482161164283752, + "logps/chosen": -381.304443359375, + "logps/ref_chosen": -57.52456283569336, + "logps/ref_rejected": -75.97572326660156, + "logps/rejected": -484.57818603515625, + "loss": 1.3695, + "margin_dpo/margin_mean": 84.82254791259766, + "margin_dpo/margin_std": 320.90704345703125, + "step": 183 + }, + { + "KL/chosen_KL_mean": -315.3003845214844, + "KL/mean": -351.2978515625, + "KL/rejected_KL_mean": -387.2952575683594, + "KL/std": 260.4176025390625, + "epoch": 0.2701908957415565, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 71.99484252929688, + "fcm_dpo/q_t": 0.4961445927619934, + "grad_norm": 2.970120668411255, + "learning_rate": 4.5840059630527985e-07, + "logits/chosen": -0.059889525175094604, + "logits/rejected": -0.0684148371219635, + "logps/chosen": -373.8453369140625, + "logps/ref_chosen": -58.544952392578125, + "logps/ref_rejected": -76.63406372070312, + "logps/rejected": -463.9293212890625, + "loss": 1.3723, + "margin_dpo/margin_mean": 71.9948501586914, + "margin_dpo/margin_std": 333.66949462890625, + "step": 184 + }, + { + "KL/chosen_KL_mean": -327.66729736328125, + "KL/mean": -346.7569580078125, + "KL/rejected_KL_mean": -365.84661865234375, + "KL/std": 300.96337890625, + "epoch": 0.27165932452276065, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 38.17930221557617, + "fcm_dpo/q_t": 0.49796029925346375, + "grad_norm": 3.1760153770446777, + "learning_rate": 4.5768898691940836e-07, + "logits/chosen": -0.02878168597817421, + "logits/rejected": -0.03691637143492699, + "logps/chosen": -389.69317626953125, + "logps/ref_chosen": -62.025848388671875, + "logps/ref_rejected": -73.7625961303711, + "logps/rejected": -439.6092224121094, + "loss": 1.3798, + "margin_dpo/margin_mean": 38.17930221557617, + "margin_dpo/margin_std": 370.1015625, + "step": 185 + }, + { + "KL/chosen_KL_mean": -320.67669677734375, + "KL/mean": -373.763427734375, + "KL/rejected_KL_mean": -426.8501892089844, + "KL/std": 282.6938171386719, + "epoch": 0.27312775330396477, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 106.17350769042969, + "fcm_dpo/q_t": 0.4943085312843323, + "grad_norm": 2.898653745651245, + "learning_rate": 4.5697190473557947e-07, + "logits/chosen": -0.1143377274274826, + "logits/rejected": -0.15146398544311523, + "logps/chosen": -390.0301513671875, + "logps/ref_chosen": -69.35346984863281, + "logps/ref_rejected": -88.07244873046875, + "logps/rejected": -514.9226684570312, + "loss": 1.3653, + "margin_dpo/margin_mean": 106.17350769042969, + "margin_dpo/margin_std": 373.8057556152344, + "step": 186 + }, + { + "KL/chosen_KL_mean": -235.82952880859375, + "KL/mean": -303.52093505859375, + "KL/rejected_KL_mean": -371.21234130859375, + "KL/std": 257.0897521972656, + "epoch": 0.2745961820851689, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 135.3828125, + "fcm_dpo/q_t": 0.4927484393119812, + "grad_norm": 2.4242630004882812, + "learning_rate": 4.5624936864957555e-07, + "logits/chosen": -0.0784909650683403, + "logits/rejected": -0.09007595479488373, + "logps/chosen": -288.58599853515625, + "logps/ref_chosen": -52.7564582824707, + "logps/ref_rejected": -81.96910095214844, + "logps/rejected": -453.18145751953125, + "loss": 1.3583, + "margin_dpo/margin_mean": 135.38282775878906, + "margin_dpo/margin_std": 272.33575439453125, + "step": 187 + }, + { + "KL/chosen_KL_mean": -211.16748046875, + "KL/mean": -291.66912841796875, + "KL/rejected_KL_mean": -372.1707763671875, + "KL/std": 238.9845733642578, + "epoch": 0.27606461086637296, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 161.00326538085938, + "fcm_dpo/q_t": 0.4913794994354248, + "grad_norm": 2.7519357204437256, + "learning_rate": 4.5552139770089454e-07, + "logits/chosen": -0.10255219042301178, + "logits/rejected": -0.13276180624961853, + "logps/chosen": -260.58294677734375, + "logps/ref_chosen": -49.415489196777344, + "logps/ref_rejected": -89.54043579101562, + "logps/rejected": -461.711181640625, + "loss": 1.353, + "margin_dpo/margin_mean": 161.00326538085938, + "margin_dpo/margin_std": 281.9280090332031, + "step": 188 + }, + { + "KL/chosen_KL_mean": -245.26499938964844, + "KL/mean": -297.78826904296875, + "KL/rejected_KL_mean": -350.3115234375, + "KL/std": 250.982666015625, + "epoch": 0.2775330396475771, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 105.0465087890625, + "fcm_dpo/q_t": 0.49437761306762695, + "grad_norm": 3.0305230617523193, + "learning_rate": 4.5478801107224794e-07, + "logits/chosen": -0.13245689868927002, + "logits/rejected": -0.15994009375572205, + "logps/chosen": -297.6639709472656, + "logps/ref_chosen": -52.39896011352539, + "logps/ref_rejected": -72.16735076904297, + "logps/rejected": -422.4788513183594, + "loss": 1.3651, + "margin_dpo/margin_mean": 105.0465087890625, + "margin_dpo/margin_std": 324.1693115234375, + "step": 189 + }, + { + "KL/chosen_KL_mean": -259.7969970703125, + "KL/mean": -342.51513671875, + "KL/rejected_KL_mean": -425.2333068847656, + "KL/std": 296.94696044921875, + "epoch": 0.2790014684287812, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 165.436279296875, + "fcm_dpo/q_t": 0.4911511242389679, + "grad_norm": 2.5856246948242188, + "learning_rate": 4.5404922808905543e-07, + "logits/chosen": -0.1271246075630188, + "logits/rejected": -0.15945151448249817, + "logps/chosen": -324.4801025390625, + "logps/ref_chosen": -64.68305969238281, + "logps/ref_rejected": -102.55052185058594, + "logps/rejected": -527.7838134765625, + "loss": 1.3527, + "margin_dpo/margin_mean": 165.436279296875, + "margin_dpo/margin_std": 360.4592590332031, + "step": 190 + }, + { + "KL/chosen_KL_mean": -253.1461181640625, + "KL/mean": -364.49249267578125, + "KL/rejected_KL_mean": -475.8388977050781, + "KL/std": 322.4438171386719, + "epoch": 0.28046989720998533, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 222.6928253173828, + "fcm_dpo/q_t": 0.4880927801132202, + "grad_norm": 3.680899143218994, + "learning_rate": 4.5330506821893565e-07, + "logits/chosen": -0.12386652827262878, + "logits/rejected": -0.11776319891214371, + "logps/chosen": -321.80499267578125, + "logps/ref_chosen": -68.65887451171875, + "logps/ref_rejected": -110.1396713256836, + "logps/rejected": -585.9785766601562, + "loss": 1.3408, + "margin_dpo/margin_mean": 222.6928253173828, + "margin_dpo/margin_std": 386.87347412109375, + "step": 191 + }, + { + "KL/chosen_KL_mean": -300.694091796875, + "KL/mean": -368.8135986328125, + "KL/rejected_KL_mean": -436.93310546875, + "KL/std": 301.52886962890625, + "epoch": 0.28193832599118945, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 136.23902893066406, + "fcm_dpo/q_t": 0.49271297454833984, + "grad_norm": 2.103487253189087, + "learning_rate": 4.5255555107119336e-07, + "logits/chosen": -0.09601673483848572, + "logits/rejected": -0.10303403437137604, + "logps/chosen": -370.4210205078125, + "logps/ref_chosen": -69.72691345214844, + "logps/ref_rejected": -103.32135009765625, + "logps/rejected": -540.2544555664062, + "loss": 1.359, + "margin_dpo/margin_mean": 136.23902893066406, + "margin_dpo/margin_std": 378.644287109375, + "step": 192 + }, + { + "KL/chosen_KL_mean": -283.2509460449219, + "KL/mean": -325.3600769042969, + "KL/rejected_KL_mean": -367.46923828125, + "KL/std": 314.67529296875, + "epoch": 0.2834067547723935, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 84.21826171875, + "fcm_dpo/q_t": 0.49550098180770874, + "grad_norm": 2.3068125247955322, + "learning_rate": 4.5180069639630236e-07, + "logits/chosen": -0.07964509725570679, + "logits/rejected": -0.1134941577911377, + "logps/chosen": -343.44146728515625, + "logps/ref_chosen": -60.19049835205078, + "logps/ref_rejected": -76.40755462646484, + "logps/rejected": -443.87677001953125, + "loss": 1.3702, + "margin_dpo/margin_mean": 84.21826934814453, + "margin_dpo/margin_std": 401.8240966796875, + "step": 193 + }, + { + "KL/chosen_KL_mean": -178.2296905517578, + "KL/mean": -237.00930786132812, + "KL/rejected_KL_mean": -295.7889099121094, + "KL/std": 197.4080810546875, + "epoch": 0.28487518355359764, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 117.55923461914062, + "fcm_dpo/q_t": 0.49369940161705017, + "grad_norm": 1.7572847604751587, + "learning_rate": 4.510405240853854e-07, + "logits/chosen": 0.0028687757439911366, + "logits/rejected": -0.012111449614167213, + "logps/chosen": -216.070068359375, + "logps/ref_chosen": -37.84037399291992, + "logps/ref_rejected": -60.684783935546875, + "logps/rejected": -356.47369384765625, + "loss": 1.3618, + "margin_dpo/margin_mean": 117.55921936035156, + "margin_dpo/margin_std": 219.46182250976562, + "step": 194 + }, + { + "KL/chosen_KL_mean": -269.02020263671875, + "KL/mean": -363.59765625, + "KL/rejected_KL_mean": -458.17510986328125, + "KL/std": 306.4468994140625, + "epoch": 0.28634361233480177, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 189.15487670898438, + "fcm_dpo/q_t": 0.4898764491081238, + "grad_norm": 3.175015687942505, + "learning_rate": 4.5027505416968985e-07, + "logits/chosen": -0.10875138640403748, + "logits/rejected": -0.11520682275295258, + "logps/chosen": -323.9117736816406, + "logps/ref_chosen": -54.891571044921875, + "logps/ref_rejected": -96.77095794677734, + "logps/rejected": -554.946044921875, + "loss": 1.3473, + "margin_dpo/margin_mean": 189.15487670898438, + "margin_dpo/margin_std": 321.9908752441406, + "step": 195 + }, + { + "KL/chosen_KL_mean": -243.0733642578125, + "KL/mean": -316.908447265625, + "KL/rejected_KL_mean": -390.7435302734375, + "KL/std": 265.1428527832031, + "epoch": 0.2878120411160059, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 147.670166015625, + "fcm_dpo/q_t": 0.4920935332775116, + "grad_norm": 2.1525228023529053, + "learning_rate": 4.495043068200599e-07, + "logits/chosen": -0.10881956666707993, + "logits/rejected": -0.12770894169807434, + "logps/chosen": -296.318603515625, + "logps/ref_chosen": -53.245243072509766, + "logps/ref_rejected": -76.05294799804688, + "logps/rejected": -466.7964782714844, + "loss": 1.3564, + "margin_dpo/margin_mean": 147.670166015625, + "margin_dpo/margin_std": 344.3466491699219, + "step": 196 + }, + { + "KL/chosen_KL_mean": -274.3104248046875, + "KL/mean": -325.2745361328125, + "KL/rejected_KL_mean": -376.23858642578125, + "KL/std": 251.0189971923828, + "epoch": 0.28928046989721, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 101.92816925048828, + "fcm_dpo/q_t": 0.4945385158061981, + "grad_norm": 2.172877073287964, + "learning_rate": 4.4872830234640493e-07, + "logits/chosen": -0.10666961967945099, + "logits/rejected": -0.11786352097988129, + "logps/chosen": -334.73077392578125, + "logps/ref_chosen": -60.42033386230469, + "logps/ref_rejected": -77.20890808105469, + "logps/rejected": -453.447509765625, + "loss": 1.3655, + "margin_dpo/margin_mean": 101.92816162109375, + "margin_dpo/margin_std": 281.21978759765625, + "step": 197 + }, + { + "KL/chosen_KL_mean": -272.42474365234375, + "KL/mean": -371.7640380859375, + "KL/rejected_KL_mean": -471.1033020019531, + "KL/std": 313.407470703125, + "epoch": 0.2907488986784141, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 198.67855834960938, + "fcm_dpo/q_t": 0.4893642067909241, + "grad_norm": 2.680528402328491, + "learning_rate": 4.479470611971645e-07, + "logits/chosen": -0.13605228066444397, + "logits/rejected": -0.1457493156194687, + "logps/chosen": -327.4609375, + "logps/ref_chosen": -55.03618621826172, + "logps/ref_rejected": -97.24325561523438, + "logps/rejected": -568.3465576171875, + "loss": 1.3453, + "margin_dpo/margin_mean": 198.67855834960938, + "margin_dpo/margin_std": 319.24285888671875, + "step": 198 + }, + { + "KL/chosen_KL_mean": -274.304443359375, + "KL/mean": -358.85919189453125, + "KL/rejected_KL_mean": -443.41387939453125, + "KL/std": 275.95819091796875, + "epoch": 0.2922173274596182, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 169.10943603515625, + "fcm_dpo/q_t": 0.4909524619579315, + "grad_norm": 3.292473077774048, + "learning_rate": 4.471606039587695e-07, + "logits/chosen": -0.14171649515628815, + "logits/rejected": -0.13244497776031494, + "logps/chosen": -331.13330078125, + "logps/ref_chosen": -56.828826904296875, + "logps/ref_rejected": -84.64820861816406, + "logps/rejected": -528.0621337890625, + "loss": 1.352, + "margin_dpo/margin_mean": 169.10943603515625, + "margin_dpo/margin_std": 374.6290283203125, + "step": 199 + }, + { + "KL/chosen_KL_mean": -292.3767395019531, + "KL/mean": -373.58990478515625, + "KL/rejected_KL_mean": -454.8030090332031, + "KL/std": 326.7904052734375, + "epoch": 0.2936857562408223, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 162.42630004882812, + "fcm_dpo/q_t": 0.49131280183792114, + "grad_norm": 3.315680980682373, + "learning_rate": 4.4636895135509966e-07, + "logits/chosen": -0.07947467267513275, + "logits/rejected": -0.10722815245389938, + "logps/chosen": -345.44378662109375, + "logps/ref_chosen": -53.06706237792969, + "logps/ref_rejected": -80.60843658447266, + "logps/rejected": -535.4114379882812, + "loss": 1.3542, + "margin_dpo/margin_mean": 162.42628479003906, + "margin_dpo/margin_std": 453.01910400390625, + "step": 200 + }, + { + "KL/chosen_KL_mean": -343.0718688964844, + "KL/mean": -435.6768493652344, + "KL/rejected_KL_mean": -528.2818603515625, + "KL/std": 393.9222717285156, + "epoch": 0.29515418502202645, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 185.20999145507812, + "fcm_dpo/q_t": 0.49011439085006714, + "grad_norm": 3.324786424636841, + "learning_rate": 4.455721242469372e-07, + "logits/chosen": -0.23055091500282288, + "logits/rejected": -0.24232345819473267, + "logps/chosen": -418.4740905761719, + "logps/ref_chosen": -75.4022216796875, + "logps/ref_rejected": -114.80821990966797, + "logps/rejected": -643.090087890625, + "loss": 1.3498, + "margin_dpo/margin_mean": 185.20999145507812, + "margin_dpo/margin_std": 481.95977783203125, + "step": 201 + }, + { + "KL/chosen_KL_mean": -312.4244689941406, + "KL/mean": -377.79998779296875, + "KL/rejected_KL_mean": -443.1754455566406, + "KL/std": 326.81707763671875, + "epoch": 0.2966226138032305, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 130.75100708007812, + "fcm_dpo/q_t": 0.4930059015750885, + "grad_norm": 3.468919515609741, + "learning_rate": 4.4477014363141755e-07, + "logits/chosen": -0.1302487850189209, + "logits/rejected": -0.14801092445850372, + "logps/chosen": -362.5257873535156, + "logps/ref_chosen": -50.101318359375, + "logps/ref_rejected": -86.98503112792969, + "logps/rejected": -530.1604614257812, + "loss": 1.3601, + "margin_dpo/margin_mean": 130.75100708007812, + "margin_dpo/margin_std": 375.7841796875, + "step": 202 + }, + { + "KL/chosen_KL_mean": -355.20648193359375, + "KL/mean": -438.63665771484375, + "KL/rejected_KL_mean": -522.0668334960938, + "KL/std": 384.030029296875, + "epoch": 0.29809104258443464, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 166.86033630371094, + "fcm_dpo/q_t": 0.491105854511261, + "grad_norm": 3.1627941131591797, + "learning_rate": 4.439630306414758e-07, + "logits/chosen": -0.12041978538036346, + "logits/rejected": -0.12802743911743164, + "logps/chosen": -415.816162109375, + "logps/ref_chosen": -60.60969543457031, + "logps/ref_rejected": -85.89596557617188, + "logps/rejected": -607.9627685546875, + "loss": 1.3532, + "margin_dpo/margin_mean": 166.86032104492188, + "margin_dpo/margin_std": 459.39349365234375, + "step": 203 + }, + { + "KL/chosen_KL_mean": -420.5950622558594, + "KL/mean": -458.009033203125, + "KL/rejected_KL_mean": -495.42303466796875, + "KL/std": 421.4627990722656, + "epoch": 0.29955947136563876, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 74.8279800415039, + "fcm_dpo/q_t": 0.49598783254623413, + "grad_norm": 4.85085391998291, + "learning_rate": 4.431508065452897e-07, + "logits/chosen": -0.22811344265937805, + "logits/rejected": -0.23437625169754028, + "logps/chosen": -500.760009765625, + "logps/ref_chosen": -80.16496276855469, + "logps/ref_rejected": -87.69590759277344, + "logps/rejected": -583.1189575195312, + "loss": 1.3732, + "margin_dpo/margin_mean": 74.8279800415039, + "margin_dpo/margin_std": 487.329833984375, + "step": 204 + }, + { + "KL/chosen_KL_mean": -364.5575256347656, + "KL/mean": -444.526611328125, + "KL/rejected_KL_mean": -524.4957275390625, + "KL/std": 428.46484375, + "epoch": 0.3010279001468429, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 159.93820190429688, + "fcm_dpo/q_t": 0.49142158031463623, + "grad_norm": 4.245858192443848, + "learning_rate": 4.4233349274571974e-07, + "logits/chosen": -0.1302354335784912, + "logits/rejected": -0.14217376708984375, + "logps/chosen": -423.9422607421875, + "logps/ref_chosen": -59.384735107421875, + "logps/ref_rejected": -85.12505340576172, + "logps/rejected": -609.6207885742188, + "loss": 1.3552, + "margin_dpo/margin_mean": 159.93820190429688, + "margin_dpo/margin_std": 488.643310546875, + "step": 205 + }, + { + "KL/chosen_KL_mean": -275.8797607421875, + "KL/mean": -371.9986572265625, + "KL/rejected_KL_mean": -468.1175842285156, + "KL/std": 284.67950439453125, + "epoch": 0.302496328928047, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 192.2378387451172, + "fcm_dpo/q_t": 0.489697128534317, + "grad_norm": 3.0303680896759033, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": -0.15259331464767456, + "logits/rejected": -0.15699225664138794, + "logps/chosen": -322.84423828125, + "logps/ref_chosen": -46.964500427246094, + "logps/ref_rejected": -98.9534912109375, + "logps/rejected": -567.071044921875, + "loss": 1.3468, + "margin_dpo/margin_mean": 192.2378387451172, + "margin_dpo/margin_std": 329.2159423828125, + "step": 206 + }, + { + "KL/chosen_KL_mean": -274.165283203125, + "KL/mean": -373.14715576171875, + "KL/rejected_KL_mean": -472.1290283203125, + "KL/std": 336.2535095214844, + "epoch": 0.3039647577092511, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 197.9637451171875, + "fcm_dpo/q_t": 0.4894152283668518, + "grad_norm": 4.557210445404053, + "learning_rate": 4.4068368231789365e-07, + "logits/chosen": -0.20802870392799377, + "logits/rejected": -0.2097472846508026, + "logps/chosen": -330.2215576171875, + "logps/ref_chosen": -56.05625915527344, + "logps/ref_rejected": -84.44779968261719, + "logps/rejected": -556.5768432617188, + "loss": 1.3464, + "margin_dpo/margin_mean": 197.9637451171875, + "margin_dpo/margin_std": 424.3826904296875, + "step": 207 + }, + { + "KL/chosen_KL_mean": -368.54052734375, + "KL/mean": -451.637451171875, + "KL/rejected_KL_mean": -534.734375, + "KL/std": 326.3774108886719, + "epoch": 0.3054331864904552, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 166.19386291503906, + "fcm_dpo/q_t": 0.49110543727874756, + "grad_norm": 2.7596354484558105, + "learning_rate": 4.398512291636768e-07, + "logits/chosen": -0.20875459909439087, + "logits/rejected": -0.21936002373695374, + "logps/chosen": -435.6081237792969, + "logps/ref_chosen": -67.06761169433594, + "logps/ref_rejected": -94.28689575195312, + "logps/rejected": -629.0213012695312, + "loss": 1.353, + "margin_dpo/margin_mean": 166.19387817382812, + "margin_dpo/margin_std": 422.05780029296875, + "step": 208 + }, + { + "KL/chosen_KL_mean": -301.03485107421875, + "KL/mean": -364.5048828125, + "KL/rejected_KL_mean": -427.9748840332031, + "KL/std": 285.0235900878906, + "epoch": 0.3069016152716593, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 126.94000244140625, + "fcm_dpo/q_t": 0.4932047724723816, + "grad_norm": 2.5512523651123047, + "learning_rate": 4.3901377325300857e-07, + "logits/chosen": -0.1063951924443245, + "logits/rejected": -0.124468132853508, + "logps/chosen": -357.216552734375, + "logps/ref_chosen": -56.18169403076172, + "logps/ref_rejected": -80.94152069091797, + "logps/rejected": -508.9163818359375, + "loss": 1.3607, + "margin_dpo/margin_mean": 126.94000244140625, + "margin_dpo/margin_std": 349.48199462890625, + "step": 209 + }, + { + "KL/chosen_KL_mean": -243.17129516601562, + "KL/mean": -323.7019958496094, + "KL/rejected_KL_mean": -404.2326965332031, + "KL/std": 286.4044189453125, + "epoch": 0.30837004405286345, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 161.06141662597656, + "fcm_dpo/q_t": 0.49137187004089355, + "grad_norm": 3.259232997894287, + "learning_rate": 4.381713366536311e-07, + "logits/chosen": -0.1291487067937851, + "logits/rejected": -0.14884674549102783, + "logps/chosen": -289.5430908203125, + "logps/ref_chosen": -46.371822357177734, + "logps/ref_rejected": -76.68162536621094, + "logps/rejected": -480.914306640625, + "loss": 1.3535, + "margin_dpo/margin_mean": 161.06143188476562, + "margin_dpo/margin_std": 353.78619384765625, + "step": 210 + }, + { + "KL/chosen_KL_mean": -375.2530517578125, + "KL/mean": -432.86187744140625, + "KL/rejected_KL_mean": -490.470703125, + "KL/std": 338.1985778808594, + "epoch": 0.30983847283406757, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 115.21764373779297, + "fcm_dpo/q_t": 0.4938422441482544, + "grad_norm": 2.629580497741699, + "learning_rate": 4.373239415645323e-07, + "logits/chosen": -0.13434600830078125, + "logits/rejected": -0.1492466777563095, + "logps/chosen": -454.1854248046875, + "logps/ref_chosen": -78.93235778808594, + "logps/ref_rejected": -86.82098388671875, + "logps/rejected": -577.2916870117188, + "loss": 1.3638, + "margin_dpo/margin_mean": 115.21763610839844, + "margin_dpo/margin_std": 410.43426513671875, + "step": 211 + }, + { + "KL/chosen_KL_mean": -300.54364013671875, + "KL/mean": -434.3825988769531, + "KL/rejected_KL_mean": -568.2215576171875, + "KL/std": 372.66168212890625, + "epoch": 0.31130690161527164, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 267.6778869628906, + "fcm_dpo/q_t": 0.4856899082660675, + "grad_norm": 5.687210559844971, + "learning_rate": 4.3647161031536086e-07, + "logits/chosen": -0.11962936818599701, + "logits/rejected": -0.1340862214565277, + "logps/chosen": -358.74066162109375, + "logps/ref_chosen": -58.19701385498047, + "logps/ref_rejected": -103.05785369873047, + "logps/rejected": -671.2794189453125, + "loss": 1.332, + "margin_dpo/margin_mean": 267.6778869628906, + "margin_dpo/margin_std": 430.5390625, + "step": 212 + }, + { + "KL/chosen_KL_mean": -320.9078063964844, + "KL/mean": -405.0869140625, + "KL/rejected_KL_mean": -489.2659912109375, + "KL/std": 353.5803527832031, + "epoch": 0.31277533039647576, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 168.35821533203125, + "fcm_dpo/q_t": 0.4909900426864624, + "grad_norm": 3.844770908355713, + "learning_rate": 4.3561436536583774e-07, + "logits/chosen": -0.09740199148654938, + "logits/rejected": -0.12570348381996155, + "logps/chosen": -388.4205322265625, + "logps/ref_chosen": -67.51271057128906, + "logps/ref_rejected": -93.91471862792969, + "logps/rejected": -583.1807250976562, + "loss": 1.3531, + "margin_dpo/margin_mean": 168.35821533203125, + "margin_dpo/margin_std": 470.94073486328125, + "step": 213 + }, + { + "KL/chosen_KL_mean": -238.504638671875, + "KL/mean": -338.5084228515625, + "KL/rejected_KL_mean": -438.51214599609375, + "KL/std": 269.5102233886719, + "epoch": 0.3142437591776799, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 200.00753784179688, + "fcm_dpo/q_t": 0.4892929792404175, + "grad_norm": 3.0564281940460205, + "learning_rate": 4.3475222930516473e-07, + "logits/chosen": -0.06811121106147766, + "logits/rejected": -0.07607707381248474, + "logps/chosen": -280.1095275878906, + "logps/ref_chosen": -41.604888916015625, + "logps/ref_rejected": -77.51741027832031, + "logps/rejected": -516.029541015625, + "loss": 1.3451, + "margin_dpo/margin_mean": 200.0075225830078, + "margin_dpo/margin_std": 330.0015869140625, + "step": 214 + }, + { + "KL/chosen_KL_mean": -308.419677734375, + "KL/mean": -413.72235107421875, + "KL/rejected_KL_mean": -519.0250854492188, + "KL/std": 338.5547790527344, + "epoch": 0.315712187958884, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 210.60540771484375, + "fcm_dpo/q_t": 0.4887344241142273, + "grad_norm": 4.060998916625977, + "learning_rate": 4.3388522485142885e-07, + "logits/chosen": -0.1189073771238327, + "logits/rejected": -0.13333210349082947, + "logps/chosen": -361.69891357421875, + "logps/ref_chosen": -53.279266357421875, + "logps/ref_rejected": -89.96464538574219, + "logps/rejected": -608.98974609375, + "loss": 1.3432, + "margin_dpo/margin_mean": 210.60540771484375, + "margin_dpo/margin_std": 367.251708984375, + "step": 215 + }, + { + "KL/chosen_KL_mean": -300.3120422363281, + "KL/mean": -386.34326171875, + "KL/rejected_KL_mean": -472.37445068359375, + "KL/std": 297.8331298828125, + "epoch": 0.31718061674008813, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 172.0624237060547, + "fcm_dpo/q_t": 0.49078911542892456, + "grad_norm": 3.7745511531829834, + "learning_rate": 4.330133748510036e-07, + "logits/chosen": -0.12054896354675293, + "logits/rejected": -0.12499400973320007, + "logps/chosen": -349.1998291015625, + "logps/ref_chosen": -48.887794494628906, + "logps/ref_rejected": -77.19892883300781, + "logps/rejected": -549.5733642578125, + "loss": 1.3514, + "margin_dpo/margin_mean": 172.0624237060547, + "margin_dpo/margin_std": 384.0893249511719, + "step": 216 + }, + { + "KL/chosen_KL_mean": -347.5061340332031, + "KL/mean": -462.15380859375, + "KL/rejected_KL_mean": -576.801513671875, + "KL/std": 366.660888671875, + "epoch": 0.3186490455212922, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 229.29534912109375, + "fcm_dpo/q_t": 0.48774290084838867, + "grad_norm": 3.4250800609588623, + "learning_rate": 4.3213670227794757e-07, + "logits/chosen": -0.09066063165664673, + "logits/rejected": -0.10337048768997192, + "logps/chosen": -397.3514404296875, + "logps/ref_chosen": -49.845306396484375, + "logps/ref_rejected": -100.07832336425781, + "logps/rejected": -676.8798217773438, + "loss": 1.3402, + "margin_dpo/margin_mean": 229.29534912109375, + "margin_dpo/margin_std": 456.92340087890625, + "step": 217 + }, + { + "KL/chosen_KL_mean": -365.1001892089844, + "KL/mean": -444.54656982421875, + "KL/rejected_KL_mean": -523.9929809570312, + "KL/std": 365.8412170410156, + "epoch": 0.3201174743024963, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 158.892822265625, + "fcm_dpo/q_t": 0.4915262460708618, + "grad_norm": 3.7529687881469727, + "learning_rate": 4.3125523023339815e-07, + "logits/chosen": -0.14683587849140167, + "logits/rejected": -0.16048407554626465, + "logps/chosen": -423.6768798828125, + "logps/ref_chosen": -58.576683044433594, + "logps/ref_rejected": -87.84639739990234, + "logps/rejected": -611.83935546875, + "loss": 1.3551, + "margin_dpo/margin_mean": 158.892822265625, + "margin_dpo/margin_std": 475.37939453125, + "step": 218 + }, + { + "KL/chosen_KL_mean": -386.04595947265625, + "KL/mean": -442.3204650878906, + "KL/rejected_KL_mean": -498.594970703125, + "KL/std": 393.7286376953125, + "epoch": 0.32158590308370044, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 112.54901123046875, + "fcm_dpo/q_t": 0.4940031170845032, + "grad_norm": 4.430355548858643, + "learning_rate": 4.303689819449636e-07, + "logits/chosen": -0.19574600458145142, + "logits/rejected": -0.21589338779449463, + "logps/chosen": -447.12982177734375, + "logps/ref_chosen": -61.083858489990234, + "logps/ref_rejected": -85.83042907714844, + "logps/rejected": -584.4254150390625, + "loss": 1.3655, + "margin_dpo/margin_mean": 112.54901885986328, + "margin_dpo/margin_std": 510.53802490234375, + "step": 219 + }, + { + "KL/chosen_KL_mean": -407.1309814453125, + "KL/mean": -457.27777099609375, + "KL/rejected_KL_mean": -507.4245910644531, + "KL/std": 324.55487060546875, + "epoch": 0.32305433186490456, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 100.29361724853516, + "fcm_dpo/q_t": 0.49463126063346863, + "grad_norm": 3.9129955768585205, + "learning_rate": 4.2947798076611047e-07, + "logits/chosen": -0.22126300632953644, + "logits/rejected": -0.21606583893299103, + "logps/chosen": -477.16229248046875, + "logps/ref_chosen": -70.03128051757812, + "logps/ref_rejected": -87.68551635742188, + "logps/rejected": -595.110107421875, + "loss": 1.3665, + "margin_dpo/margin_mean": 100.29361724853516, + "margin_dpo/margin_std": 378.6761169433594, + "step": 220 + }, + { + "KL/chosen_KL_mean": -290.21429443359375, + "KL/mean": -445.29681396484375, + "KL/rejected_KL_mean": -600.3792724609375, + "KL/std": 374.867919921875, + "epoch": 0.3245227606461087, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 310.1649475097656, + "fcm_dpo/q_t": 0.4834199547767639, + "grad_norm": 7.014521598815918, + "learning_rate": 4.285822501755485e-07, + "logits/chosen": -0.23672357201576233, + "logits/rejected": -0.24645334482192993, + "logps/chosen": -342.3690185546875, + "logps/ref_chosen": -52.15470886230469, + "logps/ref_rejected": -106.46768188476562, + "logps/rejected": -706.846923828125, + "loss": 1.323, + "margin_dpo/margin_mean": 310.1649475097656, + "margin_dpo/margin_std": 433.8343811035156, + "step": 221 + }, + { + "KL/chosen_KL_mean": -317.06378173828125, + "KL/mean": -416.68621826171875, + "KL/rejected_KL_mean": -516.3086547851562, + "KL/std": 329.75885009765625, + "epoch": 0.32599118942731276, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 199.24484252929688, + "fcm_dpo/q_t": 0.48933765292167664, + "grad_norm": 3.5604019165039062, + "learning_rate": 4.276818137766118e-07, + "logits/chosen": -0.24718379974365234, + "logits/rejected": -0.27352237701416016, + "logps/chosen": -378.0348815917969, + "logps/ref_chosen": -60.971099853515625, + "logps/ref_rejected": -100.00115203857422, + "logps/rejected": -616.309814453125, + "loss": 1.3459, + "margin_dpo/margin_mean": 199.24484252929688, + "margin_dpo/margin_std": 405.140869140625, + "step": 222 + }, + { + "KL/chosen_KL_mean": -326.71197509765625, + "KL/mean": -409.00048828125, + "KL/rejected_KL_mean": -491.2890319824219, + "KL/std": 325.3358154296875, + "epoch": 0.3274596182085169, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 164.57704162597656, + "fcm_dpo/q_t": 0.49121803045272827, + "grad_norm": 3.8082237243652344, + "learning_rate": 4.2677669529663686e-07, + "logits/chosen": -0.13181817531585693, + "logits/rejected": -0.14403775334358215, + "logps/chosen": -379.3525390625, + "logps/ref_chosen": -52.64057540893555, + "logps/ref_rejected": -82.82502746582031, + "logps/rejected": -574.1140747070312, + "loss": 1.3537, + "margin_dpo/margin_mean": 164.57704162597656, + "margin_dpo/margin_std": 439.234619140625, + "step": 223 + }, + { + "KL/chosen_KL_mean": -307.269287109375, + "KL/mean": -398.96954345703125, + "KL/rejected_KL_mean": -490.6697998046875, + "KL/std": 385.791748046875, + "epoch": 0.328928046989721, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 183.40045166015625, + "fcm_dpo/q_t": 0.49022960662841797, + "grad_norm": 3.7661983966827393, + "learning_rate": 4.2586691858633747e-07, + "logits/chosen": -0.1584322154521942, + "logits/rejected": -0.1633099913597107, + "logps/chosen": -355.8647155761719, + "logps/ref_chosen": -48.59541320800781, + "logps/ref_rejected": -77.11648559570312, + "logps/rejected": -567.7862548828125, + "loss": 1.3501, + "margin_dpo/margin_mean": 183.40045166015625, + "margin_dpo/margin_std": 472.4094543457031, + "step": 224 + }, + { + "KL/chosen_KL_mean": -362.7328186035156, + "KL/mean": -477.8741760253906, + "KL/rejected_KL_mean": -593.0155639648438, + "KL/std": 375.5802001953125, + "epoch": 0.3303964757709251, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 230.28273010253906, + "fcm_dpo/q_t": 0.4877273440361023, + "grad_norm": 3.3898706436157227, + "learning_rate": 4.249525076191759e-07, + "logits/chosen": -0.14341585338115692, + "logits/rejected": -0.15620696544647217, + "logps/chosen": -420.7332763671875, + "logps/ref_chosen": -58.000465393066406, + "logps/ref_rejected": -99.90291595458984, + "logps/rejected": -692.91845703125, + "loss": 1.3406, + "margin_dpo/margin_mean": 230.28274536132812, + "margin_dpo/margin_std": 511.2721862792969, + "step": 225 + }, + { + "KL/chosen_KL_mean": -322.0030517578125, + "KL/mean": -404.18328857421875, + "KL/rejected_KL_mean": -486.36346435546875, + "KL/std": 324.75518798828125, + "epoch": 0.33186490455212925, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 164.36044311523438, + "fcm_dpo/q_t": 0.49121221899986267, + "grad_norm": 3.274152994155884, + "learning_rate": 4.2403348649073167e-07, + "logits/chosen": -0.19513732194900513, + "logits/rejected": -0.19808658957481384, + "logps/chosen": -380.90185546875, + "logps/ref_chosen": -58.898799896240234, + "logps/ref_rejected": -78.68775939941406, + "logps/rejected": -565.05126953125, + "loss": 1.3535, + "margin_dpo/margin_mean": 164.36044311523438, + "margin_dpo/margin_std": 427.1483154296875, + "step": 226 + }, + { + "KL/chosen_KL_mean": -357.85113525390625, + "KL/mean": -482.2805480957031, + "KL/rejected_KL_mean": -606.7099609375, + "KL/std": 433.9095458984375, + "epoch": 0.3333333333333333, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 248.85882568359375, + "fcm_dpo/q_t": 0.48674476146698, + "grad_norm": 3.5155231952667236, + "learning_rate": 4.2310987941806615e-07, + "logits/chosen": -0.20451249182224274, + "logits/rejected": -0.20552946627140045, + "logps/chosen": -416.92333984375, + "logps/ref_chosen": -59.072181701660156, + "logps/ref_rejected": -99.41236877441406, + "logps/rejected": -706.122314453125, + "loss": 1.3364, + "margin_dpo/margin_mean": 248.85882568359375, + "margin_dpo/margin_std": 477.48724365234375, + "step": 227 + }, + { + "KL/chosen_KL_mean": -384.4470520019531, + "KL/mean": -456.94488525390625, + "KL/rejected_KL_mean": -529.4427490234375, + "KL/std": 377.869140625, + "epoch": 0.33480176211453744, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 144.99569702148438, + "fcm_dpo/q_t": 0.4922485947608948, + "grad_norm": 3.4961659908294678, + "learning_rate": 4.2218171073908463e-07, + "logits/chosen": -0.15710148215293884, + "logits/rejected": -0.1570408046245575, + "logps/chosen": -450.33831787109375, + "logps/ref_chosen": -65.89128875732422, + "logps/ref_rejected": -91.04875183105469, + "logps/rejected": -620.4915161132812, + "loss": 1.3584, + "margin_dpo/margin_mean": 144.99569702148438, + "margin_dpo/margin_std": 505.9360656738281, + "step": 228 + }, + { + "KL/chosen_KL_mean": -427.20635986328125, + "KL/mean": -472.91815185546875, + "KL/rejected_KL_mean": -518.6299438476562, + "KL/std": 411.049560546875, + "epoch": 0.33627019089574156, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.4235610961914, + "fcm_dpo/q_t": 0.49511969089508057, + "grad_norm": 4.707752227783203, + "learning_rate": 4.212490049118951e-07, + "logits/chosen": -0.23124007880687714, + "logits/rejected": -0.23222056031227112, + "logps/chosen": -497.9127502441406, + "logps/ref_chosen": -70.70637512207031, + "logps/ref_rejected": -84.52741241455078, + "logps/rejected": -603.1573486328125, + "loss": 1.3692, + "margin_dpo/margin_mean": 91.4235610961914, + "margin_dpo/margin_std": 452.021240234375, + "step": 229 + }, + { + "KL/chosen_KL_mean": -248.62451171875, + "KL/mean": -390.31243896484375, + "KL/rejected_KL_mean": -532.0003662109375, + "KL/std": 303.40869140625, + "epoch": 0.3377386196769457, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 283.3758544921875, + "fcm_dpo/q_t": 0.48483169078826904, + "grad_norm": 3.5236966609954834, + "learning_rate": 4.203117865141635e-07, + "logits/chosen": -0.10768608748912811, + "logits/rejected": -0.11796237528324127, + "logps/chosen": -287.9065246582031, + "logps/ref_chosen": -39.282005310058594, + "logps/ref_rejected": -85.62191009521484, + "logps/rejected": -617.622314453125, + "loss": 1.3278, + "margin_dpo/margin_mean": 283.3758544921875, + "margin_dpo/margin_std": 349.93524169921875, + "step": 230 + }, + { + "KL/chosen_KL_mean": -329.960693359375, + "KL/mean": -397.3656921386719, + "KL/rejected_KL_mean": -464.77069091796875, + "KL/std": 273.9499816894531, + "epoch": 0.3392070484581498, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 134.8100128173828, + "fcm_dpo/q_t": 0.49278855323791504, + "grad_norm": 3.0640761852264404, + "learning_rate": 4.1937008024246625e-07, + "logits/chosen": -0.19377461075782776, + "logits/rejected": -0.1955992579460144, + "logps/chosen": -393.23712158203125, + "logps/ref_chosen": -63.27644348144531, + "logps/ref_rejected": -74.1239013671875, + "logps/rejected": -538.8945922851562, + "loss": 1.359, + "margin_dpo/margin_mean": 134.81002807617188, + "margin_dpo/margin_std": 346.1912536621094, + "step": 231 + }, + { + "KL/chosen_KL_mean": -400.56787109375, + "KL/mean": -449.58966064453125, + "KL/rejected_KL_mean": -498.61151123046875, + "KL/std": 334.50140380859375, + "epoch": 0.3406754772393539, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 98.04367065429688, + "fcm_dpo/q_t": 0.4947741627693176, + "grad_norm": 5.453951358795166, + "learning_rate": 4.1842391091163933e-07, + "logits/chosen": -0.19259104132652283, + "logits/rejected": -0.20236793160438538, + "logps/chosen": -471.3166198730469, + "logps/ref_chosen": -70.74876403808594, + "logps/ref_rejected": -83.97706604003906, + "logps/rejected": -582.5885620117188, + "loss": 1.3671, + "margin_dpo/margin_mean": 98.04367065429688, + "margin_dpo/margin_std": 388.8453369140625, + "step": 232 + }, + { + "KL/chosen_KL_mean": -313.58184814453125, + "KL/mean": -445.1939697265625, + "KL/rejected_KL_mean": -576.8060302734375, + "KL/std": 369.1995849609375, + "epoch": 0.342143906020558, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 263.2242431640625, + "fcm_dpo/q_t": 0.48596352338790894, + "grad_norm": 4.564657211303711, + "learning_rate": 4.174733034541245e-07, + "logits/chosen": -0.19956615567207336, + "logits/rejected": -0.21853141486644745, + "logps/chosen": -368.46478271484375, + "logps/ref_chosen": -54.8829345703125, + "logps/ref_rejected": -107.4800796508789, + "logps/rejected": -684.2861328125, + "loss": 1.334, + "margin_dpo/margin_mean": 263.2242431640625, + "margin_dpo/margin_std": 516.1271362304688, + "step": 233 + }, + { + "KL/chosen_KL_mean": -285.1370849609375, + "KL/mean": -427.95343017578125, + "KL/rejected_KL_mean": -570.7697143554688, + "KL/std": 362.5662841796875, + "epoch": 0.3436123348017621, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 285.6326904296875, + "fcm_dpo/q_t": 0.4847436547279358, + "grad_norm": 3.6827473640441895, + "learning_rate": 4.165182829193126e-07, + "logits/chosen": -0.20543476939201355, + "logits/rejected": -0.2246108204126358, + "logps/chosen": -329.2315673828125, + "logps/ref_chosen": -44.094520568847656, + "logps/ref_rejected": -100.00663757324219, + "logps/rejected": -670.7763671875, + "loss": 1.3283, + "margin_dpo/margin_mean": 285.6326599121094, + "margin_dpo/margin_std": 430.40399169921875, + "step": 234 + }, + { + "KL/chosen_KL_mean": -379.80419921875, + "KL/mean": -452.5826416015625, + "KL/rejected_KL_mean": -525.361083984375, + "KL/std": 368.2503662109375, + "epoch": 0.34508076358296624, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 145.556884765625, + "fcm_dpo/q_t": 0.4922545552253723, + "grad_norm": 4.7766242027282715, + "learning_rate": 4.1555887447288255e-07, + "logits/chosen": -0.1708293855190277, + "logits/rejected": -0.18167471885681152, + "logps/chosen": -442.0421142578125, + "logps/ref_chosen": -62.237911224365234, + "logps/ref_rejected": -90.39506530761719, + "logps/rejected": -615.756103515625, + "loss": 1.3582, + "margin_dpo/margin_mean": 145.556884765625, + "margin_dpo/margin_std": 496.21075439453125, + "step": 235 + }, + { + "KL/chosen_KL_mean": -263.3704833984375, + "KL/mean": -413.715087890625, + "KL/rejected_KL_mean": -564.0596923828125, + "KL/std": 374.974609375, + "epoch": 0.3465491923641703, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 300.689208984375, + "fcm_dpo/q_t": 0.48393750190734863, + "grad_norm": 10.610316276550293, + "learning_rate": 4.1459510339613946e-07, + "logits/chosen": -0.1809891164302826, + "logits/rejected": -0.1887068748474121, + "logps/chosen": -312.71185302734375, + "logps/ref_chosen": -49.34136199951172, + "logps/ref_rejected": -103.51162719726562, + "logps/rejected": -667.5713500976562, + "loss": 1.325, + "margin_dpo/margin_mean": 300.689208984375, + "margin_dpo/margin_std": 420.67352294921875, + "step": 236 + }, + { + "KL/chosen_KL_mean": -378.86138916015625, + "KL/mean": -480.03680419921875, + "KL/rejected_KL_mean": -581.2122802734375, + "KL/std": 396.18927001953125, + "epoch": 0.34801762114537443, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 202.35086059570312, + "fcm_dpo/q_t": 0.4892232120037079, + "grad_norm": 4.470569610595703, + "learning_rate": 4.136269950853473e-07, + "logits/chosen": -0.180775448679924, + "logits/rejected": -0.19322538375854492, + "logps/chosen": -433.0295104980469, + "logps/ref_chosen": -54.168121337890625, + "logps/ref_rejected": -94.78036499023438, + "logps/rejected": -675.9926147460938, + "loss": 1.3464, + "margin_dpo/margin_mean": 202.35086059570312, + "margin_dpo/margin_std": 515.0560913085938, + "step": 237 + }, + { + "KL/chosen_KL_mean": -354.31298828125, + "KL/mean": -458.6822509765625, + "KL/rejected_KL_mean": -563.051513671875, + "KL/std": 386.575439453125, + "epoch": 0.34948604992657856, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 208.73849487304688, + "fcm_dpo/q_t": 0.48885422945022583, + "grad_norm": 4.142996311187744, + "learning_rate": 4.126545750510605e-07, + "logits/chosen": -0.18518191576004028, + "logits/rejected": -0.19046491384506226, + "logps/chosen": -408.2861328125, + "logps/ref_chosen": -53.973121643066406, + "logps/ref_rejected": -89.41795349121094, + "logps/rejected": -652.469482421875, + "loss": 1.3443, + "margin_dpo/margin_mean": 208.7384796142578, + "margin_dpo/margin_std": 444.2572021484375, + "step": 238 + }, + { + "KL/chosen_KL_mean": -395.993408203125, + "KL/mean": -492.276123046875, + "KL/rejected_KL_mean": -588.558837890625, + "KL/std": 340.7003173828125, + "epoch": 0.3509544787077827, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 192.56541442871094, + "fcm_dpo/q_t": 0.4896959066390991, + "grad_norm": 3.825457811355591, + "learning_rate": 4.116778689174514e-07, + "logits/chosen": -0.16619744896888733, + "logits/rejected": -0.18394985795021057, + "logps/chosen": -454.09124755859375, + "logps/ref_chosen": -58.09782409667969, + "logps/ref_rejected": -93.59294128417969, + "logps/rejected": -682.1517333984375, + "loss": 1.3476, + "margin_dpo/margin_mean": 192.5654296875, + "margin_dpo/margin_std": 428.50225830078125, + "step": 239 + }, + { + "KL/chosen_KL_mean": -454.6864013671875, + "KL/mean": -506.56591796875, + "KL/rejected_KL_mean": -558.4453735351562, + "KL/std": 395.06903076171875, + "epoch": 0.3524229074889868, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 103.75895690917969, + "fcm_dpo/q_t": 0.49443739652633667, + "grad_norm": 3.9248123168945312, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": -0.19379109144210815, + "logits/rejected": -0.19150272011756897, + "logps/chosen": -515.3009033203125, + "logps/ref_chosen": -60.6144905090332, + "logps/ref_rejected": -74.1185302734375, + "logps/rejected": -632.5639038085938, + "loss": 1.367, + "margin_dpo/margin_mean": 103.75895690917969, + "margin_dpo/margin_std": 491.4389953613281, + "step": 240 + }, + { + "KL/chosen_KL_mean": -433.6764221191406, + "KL/mean": -542.6292724609375, + "KL/rejected_KL_mean": -651.5821533203125, + "KL/std": 436.55352783203125, + "epoch": 0.35389133627019087, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 217.90567016601562, + "fcm_dpo/q_t": 0.4883774518966675, + "grad_norm": 4.787429332733154, + "learning_rate": 4.097117014129903e-07, + "logits/chosen": -0.22374919056892395, + "logits/rejected": -0.22779551148414612, + "logps/chosen": -499.76751708984375, + "logps/ref_chosen": -66.091064453125, + "logps/ref_rejected": -88.06088256835938, + "logps/rejected": -739.6430053710938, + "loss": 1.3435, + "margin_dpo/margin_mean": 217.9056854248047, + "margin_dpo/margin_std": 529.632568359375, + "step": 241 + }, + { + "KL/chosen_KL_mean": -466.04278564453125, + "KL/mean": -527.4720458984375, + "KL/rejected_KL_mean": -588.9013671875, + "KL/std": 415.62457275390625, + "epoch": 0.355359765051395, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 122.85851287841797, + "fcm_dpo/q_t": 0.4934060573577881, + "grad_norm": 4.762578010559082, + "learning_rate": 4.087222918524807e-07, + "logits/chosen": -0.2294858694076538, + "logits/rejected": -0.23094022274017334, + "logps/chosen": -533.90673828125, + "logps/ref_chosen": -67.86392974853516, + "logps/ref_rejected": -83.36033630371094, + "logps/rejected": -672.2616577148438, + "loss": 1.364, + "margin_dpo/margin_mean": 122.85851287841797, + "margin_dpo/margin_std": 575.6241455078125, + "step": 242 + }, + { + "KL/chosen_KL_mean": -402.603759765625, + "KL/mean": -486.40032958984375, + "KL/rejected_KL_mean": -570.1968994140625, + "KL/std": 361.3595886230469, + "epoch": 0.3568281938325991, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 167.5931396484375, + "fcm_dpo/q_t": 0.4910268187522888, + "grad_norm": 4.299717903137207, + "learning_rate": 4.07728699811968e-07, + "logits/chosen": -0.22352221608161926, + "logits/rejected": -0.23877659440040588, + "logps/chosen": -465.6880187988281, + "logps/ref_chosen": -63.0842399597168, + "logps/ref_rejected": -76.33563232421875, + "logps/rejected": -646.5325927734375, + "loss": 1.3528, + "margin_dpo/margin_mean": 167.59312438964844, + "margin_dpo/margin_std": 422.52716064453125, + "step": 243 + }, + { + "KL/chosen_KL_mean": -334.23394775390625, + "KL/mean": -451.40582275390625, + "KL/rejected_KL_mean": -568.57763671875, + "KL/std": 372.09405517578125, + "epoch": 0.35829662261380324, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 234.34368896484375, + "fcm_dpo/q_t": 0.48746874928474426, + "grad_norm": 4.2224202156066895, + "learning_rate": 4.067309514735267e-07, + "logits/chosen": -0.2551266849040985, + "logits/rejected": -0.27021169662475586, + "logps/chosen": -395.3746643066406, + "logps/ref_chosen": -61.140689849853516, + "logps/ref_rejected": -94.89193725585938, + "logps/rejected": -663.4696044921875, + "loss": 1.3385, + "margin_dpo/margin_mean": 234.34368896484375, + "margin_dpo/margin_std": 386.5777893066406, + "step": 244 + }, + { + "KL/chosen_KL_mean": -376.46820068359375, + "KL/mean": -438.8470153808594, + "KL/rejected_KL_mean": -501.2258605957031, + "KL/std": 329.3731689453125, + "epoch": 0.35976505139500736, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 124.75768280029297, + "fcm_dpo/q_t": 0.4933217763900757, + "grad_norm": 3.401407480239868, + "learning_rate": 4.057290731287531e-07, + "logits/chosen": -0.2517125606536865, + "logits/rejected": -0.24654418230056763, + "logps/chosen": -443.73046875, + "logps/ref_chosen": -67.26228332519531, + "logps/ref_rejected": -87.64010620117188, + "logps/rejected": -588.865966796875, + "loss": 1.3615, + "margin_dpo/margin_mean": 124.75768280029297, + "margin_dpo/margin_std": 384.70745849609375, + "step": 245 + }, + { + "KL/chosen_KL_mean": -330.4073486328125, + "KL/mean": -413.43792724609375, + "KL/rejected_KL_mean": -496.468505859375, + "KL/std": 368.08837890625, + "epoch": 0.36123348017621143, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 166.0610809326172, + "fcm_dpo/q_t": 0.4911186397075653, + "grad_norm": 4.492984294891357, + "learning_rate": 4.047230911780736e-07, + "logits/chosen": -0.25868698954582214, + "logits/rejected": -0.2564406394958496, + "logps/chosen": -397.10430908203125, + "logps/ref_chosen": -66.69696807861328, + "logps/ref_rejected": -84.34634399414062, + "logps/rejected": -580.8148193359375, + "loss": 1.353, + "margin_dpo/margin_mean": 166.0610809326172, + "margin_dpo/margin_std": 415.17022705078125, + "step": 246 + }, + { + "KL/chosen_KL_mean": -346.08819580078125, + "KL/mean": -501.11810302734375, + "KL/rejected_KL_mean": -656.1480712890625, + "KL/std": 380.09771728515625, + "epoch": 0.36270190895741555, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 310.059814453125, + "fcm_dpo/q_t": 0.48344653844833374, + "grad_norm": 6.050650596618652, + "learning_rate": 4.0371303213004814e-07, + "logits/chosen": -0.23237216472625732, + "logits/rejected": -0.24301677942276, + "logps/chosen": -402.69354248046875, + "logps/ref_chosen": -56.6053466796875, + "logps/ref_rejected": -106.29326629638672, + "logps/rejected": -762.4412841796875, + "loss": 1.3237, + "margin_dpo/margin_mean": 310.0598449707031, + "margin_dpo/margin_std": 467.6119689941406, + "step": 247 + }, + { + "KL/chosen_KL_mean": -282.9842529296875, + "KL/mean": -411.3094482421875, + "KL/rejected_KL_mean": -539.6346435546875, + "KL/std": 325.2694396972656, + "epoch": 0.3641703377386197, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 256.65032958984375, + "fcm_dpo/q_t": 0.4862789809703827, + "grad_norm": 4.713438034057617, + "learning_rate": 4.0269892260067197e-07, + "logits/chosen": -0.18797361850738525, + "logits/rejected": -0.21102368831634521, + "logps/chosen": -327.0274963378906, + "logps/ref_chosen": -44.043216705322266, + "logps/ref_rejected": -91.85687255859375, + "logps/rejected": -631.4915161132812, + "loss": 1.3337, + "margin_dpo/margin_mean": 256.65032958984375, + "margin_dpo/margin_std": 363.392822265625, + "step": 248 + }, + { + "KL/chosen_KL_mean": -423.87060546875, + "KL/mean": -470.3067626953125, + "KL/rejected_KL_mean": -516.742919921875, + "KL/std": 390.5693054199219, + "epoch": 0.3656387665198238, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 92.87237548828125, + "fcm_dpo/q_t": 0.49507373571395874, + "grad_norm": 3.928159236907959, + "learning_rate": 4.0168078931267426e-07, + "logits/chosen": -0.16814130544662476, + "logits/rejected": -0.17811042070388794, + "logps/chosen": -486.31292724609375, + "logps/ref_chosen": -62.442352294921875, + "logps/ref_rejected": -80.46806335449219, + "logps/rejected": -597.2110595703125, + "loss": 1.3692, + "margin_dpo/margin_mean": 92.87238311767578, + "margin_dpo/margin_std": 484.80364990234375, + "step": 249 + }, + { + "KL/chosen_KL_mean": -410.2246398925781, + "KL/mean": -485.90521240234375, + "KL/rejected_KL_mean": -561.5857543945312, + "KL/std": 377.61981201171875, + "epoch": 0.3671071953010279, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 151.361083984375, + "fcm_dpo/q_t": 0.49189114570617676, + "grad_norm": 4.632651329040527, + "learning_rate": 4.006586590948141e-07, + "logits/chosen": -0.1685868501663208, + "logits/rejected": -0.17339754104614258, + "logps/chosen": -475.861328125, + "logps/ref_chosen": -65.63668823242188, + "logps/ref_rejected": -73.87184143066406, + "logps/rejected": -635.4575805664062, + "loss": 1.3563, + "margin_dpo/margin_mean": 151.361083984375, + "margin_dpo/margin_std": 436.9773254394531, + "step": 250 + }, + { + "KL/chosen_KL_mean": -401.18408203125, + "KL/mean": -462.01068115234375, + "KL/rejected_KL_mean": -522.8372192382812, + "KL/std": 380.92681884765625, + "epoch": 0.368575624082232, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 121.65315246582031, + "fcm_dpo/q_t": 0.49349498748779297, + "grad_norm": 4.47353458404541, + "learning_rate": 3.9963255888117325e-07, + "logits/chosen": -0.1774073839187622, + "logits/rejected": -0.18519741296768188, + "logps/chosen": -458.3667907714844, + "logps/ref_chosen": -57.182716369628906, + "logps/ref_rejected": -77.66343688964844, + "logps/rejected": -600.5006713867188, + "loss": 1.3627, + "margin_dpo/margin_mean": 121.65316772460938, + "margin_dpo/margin_std": 439.44793701171875, + "step": 251 + }, + { + "KL/chosen_KL_mean": -429.9615173339844, + "KL/mean": -518.6616821289062, + "KL/rejected_KL_mean": -607.36181640625, + "KL/std": 336.4705505371094, + "epoch": 0.3700440528634361, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 177.4003143310547, + "fcm_dpo/q_t": 0.4905024766921997, + "grad_norm": 4.228276252746582, + "learning_rate": 3.9860251571044666e-07, + "logits/chosen": -0.20860856771469116, + "logits/rejected": -0.19886748492717743, + "logps/chosen": -501.64715576171875, + "logps/ref_chosen": -71.68563842773438, + "logps/ref_rejected": -84.75799560546875, + "logps/rejected": -692.1198120117188, + "loss": 1.3501, + "margin_dpo/margin_mean": 177.4003143310547, + "margin_dpo/margin_std": 356.32275390625, + "step": 252 + }, + { + "KL/chosen_KL_mean": -392.266357421875, + "KL/mean": -508.9656066894531, + "KL/rejected_KL_mean": -625.664794921875, + "KL/std": 355.4100341796875, + "epoch": 0.37151248164464024, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 233.3984832763672, + "fcm_dpo/q_t": 0.4875214695930481, + "grad_norm": 3.7770397663116455, + "learning_rate": 3.9756855672522986e-07, + "logits/chosen": -0.19992950558662415, + "logits/rejected": -0.19830244779586792, + "logps/chosen": -461.4002990722656, + "logps/ref_chosen": -69.1339340209961, + "logps/ref_rejected": -98.70252990722656, + "logps/rejected": -724.3673706054688, + "loss": 1.3391, + "margin_dpo/margin_mean": 233.39849853515625, + "margin_dpo/margin_std": 419.17742919921875, + "step": 253 + }, + { + "KL/chosen_KL_mean": -392.25958251953125, + "KL/mean": -469.58319091796875, + "KL/rejected_KL_mean": -546.90673828125, + "KL/std": 388.672607421875, + "epoch": 0.37298091042584436, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 154.64715576171875, + "fcm_dpo/q_t": 0.4917369484901428, + "grad_norm": 4.755723476409912, + "learning_rate": 3.965307091713037e-07, + "logits/chosen": -0.14065057039260864, + "logits/rejected": -0.1489948332309723, + "logps/chosen": -446.41461181640625, + "logps/ref_chosen": -54.154998779296875, + "logps/ref_rejected": -90.30764770507812, + "logps/rejected": -637.21435546875, + "loss": 1.3558, + "margin_dpo/margin_mean": 154.64715576171875, + "margin_dpo/margin_std": 455.6612854003906, + "step": 254 + }, + { + "KL/chosen_KL_mean": -382.1084899902344, + "KL/mean": -481.79656982421875, + "KL/rejected_KL_mean": -581.484619140625, + "KL/std": 380.31219482421875, + "epoch": 0.3744493392070485, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 199.37615966796875, + "fcm_dpo/q_t": 0.48935091495513916, + "grad_norm": 3.1735172271728516, + "learning_rate": 3.954890003969163e-07, + "logits/chosen": -0.12860265374183655, + "logits/rejected": -0.1277311146259308, + "logps/chosen": -439.2501525878906, + "logps/ref_chosen": -57.14167022705078, + "logps/ref_rejected": -90.2085952758789, + "logps/rejected": -671.6932373046875, + "loss": 1.3468, + "margin_dpo/margin_mean": 199.37615966796875, + "margin_dpo/margin_std": 478.7210388183594, + "step": 255 + }, + { + "KL/chosen_KL_mean": -343.8599548339844, + "KL/mean": -461.6353454589844, + "KL/rejected_KL_mean": -579.4107666015625, + "KL/std": 413.0328674316406, + "epoch": 0.37591776798825255, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 235.55078125, + "fcm_dpo/q_t": 0.4874057471752167, + "grad_norm": 4.488049030303955, + "learning_rate": 3.944434578520628e-07, + "logits/chosen": -0.10436695069074631, + "logits/rejected": -0.12606412172317505, + "logps/chosen": -399.0234375, + "logps/ref_chosen": -55.163490295410156, + "logps/ref_rejected": -92.56291961669922, + "logps/rejected": -671.9736328125, + "loss": 1.3388, + "margin_dpo/margin_mean": 235.55078125, + "margin_dpo/margin_std": 458.3751220703125, + "step": 256 + }, + { + "KL/chosen_KL_mean": -310.64581298828125, + "KL/mean": -431.5376281738281, + "KL/rejected_KL_mean": -552.429443359375, + "KL/std": 358.80474853515625, + "epoch": 0.37738619676945667, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 241.78359985351562, + "fcm_dpo/q_t": 0.4870920777320862, + "grad_norm": 4.349320888519287, + "learning_rate": 3.933941090877615e-07, + "logits/chosen": -0.13144733011722565, + "logits/rejected": -0.13504740595817566, + "logps/chosen": -360.06951904296875, + "logps/ref_chosen": -49.42369842529297, + "logps/ref_rejected": -79.53791809082031, + "logps/rejected": -631.9673461914062, + "loss": 1.3377, + "margin_dpo/margin_mean": 241.78358459472656, + "margin_dpo/margin_std": 457.3230895996094, + "step": 257 + }, + { + "KL/chosen_KL_mean": -410.16473388671875, + "KL/mean": -527.0640869140625, + "KL/rejected_KL_mean": -643.96337890625, + "KL/std": 395.2886047363281, + "epoch": 0.3788546255506608, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 233.79864501953125, + "fcm_dpo/q_t": 0.4875029921531677, + "grad_norm": 3.8991451263427734, + "learning_rate": 3.923409817553284e-07, + "logits/chosen": -0.10106329619884491, + "logits/rejected": -0.10983704030513763, + "logps/chosen": -469.54888916015625, + "logps/ref_chosen": -59.384124755859375, + "logps/ref_rejected": -95.99010467529297, + "logps/rejected": -739.9534912109375, + "loss": 1.3397, + "margin_dpo/margin_mean": 233.79864501953125, + "margin_dpo/margin_std": 500.43988037109375, + "step": 258 + }, + { + "KL/chosen_KL_mean": -380.03558349609375, + "KL/mean": -472.2444152832031, + "KL/rejected_KL_mean": -564.4532470703125, + "KL/std": 359.38470458984375, + "epoch": 0.3803230543318649, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 184.4176788330078, + "fcm_dpo/q_t": 0.49013984203338623, + "grad_norm": 3.6584014892578125, + "learning_rate": 3.9128410360564793e-07, + "logits/chosen": -0.19331489503383636, + "logits/rejected": -0.2091323733329773, + "logps/chosen": -432.8639221191406, + "logps/ref_chosen": -52.828346252441406, + "logps/ref_rejected": -89.191650390625, + "logps/rejected": -653.6448974609375, + "loss": 1.3494, + "margin_dpo/margin_mean": 184.41766357421875, + "margin_dpo/margin_std": 437.661865234375, + "step": 259 + }, + { + "KL/chosen_KL_mean": -365.8872375488281, + "KL/mean": -518.8782958984375, + "KL/rejected_KL_mean": -671.8694458007812, + "KL/std": 458.7395324707031, + "epoch": 0.38179148311306904, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 305.98223876953125, + "fcm_dpo/q_t": 0.48385488986968994, + "grad_norm": 3.9910993576049805, + "learning_rate": 3.9022350248844246e-07, + "logits/chosen": -0.15303289890289307, + "logits/rejected": -0.16400694847106934, + "logps/chosen": -413.3049011230469, + "logps/ref_chosen": -47.41767501831055, + "logps/ref_rejected": -95.08978271484375, + "logps/rejected": -766.959228515625, + "loss": 1.3258, + "margin_dpo/margin_mean": 305.98223876953125, + "margin_dpo/margin_std": 550.6990966796875, + "step": 260 + }, + { + "KL/chosen_KL_mean": -419.76605224609375, + "KL/mean": -529.2583618164062, + "KL/rejected_KL_mean": -638.7506713867188, + "KL/std": 460.34503173828125, + "epoch": 0.3832599118942731, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 218.98464965820312, + "fcm_dpo/q_t": 0.4883573055267334, + "grad_norm": 5.955752372741699, + "learning_rate": 3.891592063515376e-07, + "logits/chosen": -0.08432209491729736, + "logits/rejected": -0.09958672523498535, + "logps/chosen": -472.79742431640625, + "logps/ref_chosen": -53.03137969970703, + "logps/ref_rejected": -88.51494598388672, + "logps/rejected": -727.265625, + "loss": 1.3439, + "margin_dpo/margin_mean": 218.98464965820312, + "margin_dpo/margin_std": 580.8803100585938, + "step": 261 + }, + { + "KL/chosen_KL_mean": -492.68560791015625, + "KL/mean": -573.60986328125, + "KL/rejected_KL_mean": -654.5340576171875, + "KL/std": 429.9180603027344, + "epoch": 0.38472834067547723, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 161.84848022460938, + "fcm_dpo/q_t": 0.49136343598365784, + "grad_norm": 5.33870792388916, + "learning_rate": 3.880912432401264e-07, + "logits/chosen": -0.10938498377799988, + "logits/rejected": -0.10725777596235275, + "logps/chosen": -552.3057861328125, + "logps/ref_chosen": -59.620140075683594, + "logps/ref_rejected": -86.41853332519531, + "logps/rejected": -740.95263671875, + "loss": 1.3542, + "margin_dpo/margin_mean": 161.84848022460938, + "margin_dpo/margin_std": 447.91815185546875, + "step": 262 + }, + { + "KL/chosen_KL_mean": -442.79217529296875, + "KL/mean": -572.5969848632812, + "KL/rejected_KL_mean": -702.40185546875, + "KL/std": 522.2442016601562, + "epoch": 0.38619676945668135, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 259.609619140625, + "fcm_dpo/q_t": 0.4861585795879364, + "grad_norm": 3.8708550930023193, + "learning_rate": 3.870196412960302e-07, + "logits/chosen": -0.12460441887378693, + "logits/rejected": -0.13606280088424683, + "logps/chosen": -502.2131042480469, + "logps/ref_chosen": -59.42094421386719, + "logps/ref_rejected": -96.85720825195312, + "logps/rejected": -799.259033203125, + "loss": 1.3362, + "margin_dpo/margin_mean": 259.609619140625, + "margin_dpo/margin_std": 626.2482299804688, + "step": 263 + }, + { + "KL/chosen_KL_mean": -469.1533508300781, + "KL/mean": -582.8370361328125, + "KL/rejected_KL_mean": -696.5206909179688, + "KL/std": 439.26593017578125, + "epoch": 0.3876651982378855, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 227.36734008789062, + "fcm_dpo/q_t": 0.4878886938095093, + "grad_norm": 3.9945685863494873, + "learning_rate": 3.8594442875695665e-07, + "logits/chosen": -0.15970063209533691, + "logits/rejected": -0.17009925842285156, + "logps/chosen": -531.8754272460938, + "logps/ref_chosen": -62.722084045410156, + "logps/ref_rejected": -93.85620880126953, + "logps/rejected": -790.376953125, + "loss": 1.342, + "margin_dpo/margin_mean": 227.36734008789062, + "margin_dpo/margin_std": 566.97900390625, + "step": 264 + }, + { + "KL/chosen_KL_mean": -494.053466796875, + "KL/mean": -583.5369873046875, + "KL/rejected_KL_mean": -673.0205078125, + "KL/std": 433.59552001953125, + "epoch": 0.3891336270190896, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 178.96697998046875, + "fcm_dpo/q_t": 0.49045416712760925, + "grad_norm": 4.128632068634033, + "learning_rate": 3.848656339557562e-07, + "logits/chosen": -0.09494436532258987, + "logits/rejected": -0.10667555034160614, + "logps/chosen": -556.02490234375, + "logps/ref_chosen": -61.971466064453125, + "logps/ref_rejected": -88.02059936523438, + "logps/rejected": -761.0410766601562, + "loss": 1.3517, + "margin_dpo/margin_mean": 178.96697998046875, + "margin_dpo/margin_std": 544.5645751953125, + "step": 265 + }, + { + "KL/chosen_KL_mean": -539.0609741210938, + "KL/mean": -561.758544921875, + "KL/rejected_KL_mean": -584.4561767578125, + "KL/std": 398.85784912109375, + "epoch": 0.39060205580029367, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 45.395179748535156, + "fcm_dpo/q_t": 0.49756669998168945, + "grad_norm": 6.132102966308594, + "learning_rate": 3.8378328531967507e-07, + "logits/chosen": -0.11285087466239929, + "logits/rejected": -0.12594473361968994, + "logps/chosen": -606.16064453125, + "logps/ref_chosen": -67.09967041015625, + "logps/ref_rejected": -67.97122192382812, + "logps/rejected": -652.4273681640625, + "loss": 1.3796, + "margin_dpo/margin_mean": 45.39517593383789, + "margin_dpo/margin_std": 510.51422119140625, + "step": 266 + }, + { + "KL/chosen_KL_mean": -456.233642578125, + "KL/mean": -543.4124755859375, + "KL/rejected_KL_mean": -630.59130859375, + "KL/std": 447.64996337890625, + "epoch": 0.3920704845814978, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 174.35768127441406, + "fcm_dpo/q_t": 0.49071285128593445, + "grad_norm": 4.379497528076172, + "learning_rate": 3.8269741136960646e-07, + "logits/chosen": -0.13587993383407593, + "logits/rejected": -0.14027976989746094, + "logps/chosen": -525.204345703125, + "logps/ref_chosen": -68.97075653076172, + "logps/ref_rejected": -90.16844940185547, + "logps/rejected": -720.759765625, + "loss": 1.3531, + "margin_dpo/margin_mean": 174.357666015625, + "margin_dpo/margin_std": 578.6551513671875, + "step": 267 + }, + { + "KL/chosen_KL_mean": -435.64410400390625, + "KL/mean": -557.5379638671875, + "KL/rejected_KL_mean": -679.4317626953125, + "KL/std": 507.63592529296875, + "epoch": 0.3935389133627019, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 243.78765869140625, + "fcm_dpo/q_t": 0.48697763681411743, + "grad_norm": 4.641546249389648, + "learning_rate": 3.8160804071933894e-07, + "logits/chosen": -0.17142120003700256, + "logits/rejected": -0.17281493544578552, + "logps/chosen": -491.54443359375, + "logps/ref_chosen": -55.90031051635742, + "logps/ref_rejected": -101.64763641357422, + "logps/rejected": -781.0794067382812, + "loss": 1.3387, + "margin_dpo/margin_mean": 243.78765869140625, + "margin_dpo/margin_std": 590.0145263671875, + "step": 268 + }, + { + "KL/chosen_KL_mean": -501.7329406738281, + "KL/mean": -626.0015869140625, + "KL/rejected_KL_mean": -750.270263671875, + "KL/std": 443.25628662109375, + "epoch": 0.39500734214390604, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 248.5373077392578, + "fcm_dpo/q_t": 0.4867471754550934, + "grad_norm": 4.408675193786621, + "learning_rate": 3.8051520207480204e-07, + "logits/chosen": -0.17563828825950623, + "logits/rejected": -0.192486971616745, + "logps/chosen": -571.7724609375, + "logps/ref_chosen": -70.03955841064453, + "logps/ref_rejected": -107.34937286376953, + "logps/rejected": -857.61962890625, + "loss": 1.3379, + "margin_dpo/margin_mean": 248.53729248046875, + "margin_dpo/margin_std": 601.0906982421875, + "step": 269 + }, + { + "KL/chosen_KL_mean": -464.98211669921875, + "KL/mean": -577.4337158203125, + "KL/rejected_KL_mean": -689.8851928710938, + "KL/std": 445.669189453125, + "epoch": 0.3964757709251101, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 224.90313720703125, + "fcm_dpo/q_t": 0.48800837993621826, + "grad_norm": 4.137222766876221, + "learning_rate": 3.794189242333106e-07, + "logits/chosen": -0.21441274881362915, + "logits/rejected": -0.22988632321357727, + "logps/chosen": -534.515625, + "logps/ref_chosen": -69.53347778320312, + "logps/ref_rejected": -109.92864990234375, + "logps/rejected": -799.8138427734375, + "loss": 1.3429, + "margin_dpo/margin_mean": 224.9031219482422, + "margin_dpo/margin_std": 603.002685546875, + "step": 270 + }, + { + "KL/chosen_KL_mean": -383.33953857421875, + "KL/mean": -498.1918640136719, + "KL/rejected_KL_mean": -613.044189453125, + "KL/std": 389.174560546875, + "epoch": 0.39794419970631423, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 229.7046661376953, + "fcm_dpo/q_t": 0.48772376775741577, + "grad_norm": 3.387704849243164, + "learning_rate": 3.7831923608280514e-07, + "logits/chosen": -0.15242159366607666, + "logits/rejected": -0.17212322354316711, + "logps/chosen": -440.1041259765625, + "logps/ref_chosen": -56.76456832885742, + "logps/ref_rejected": -92.51383972167969, + "logps/rejected": -705.5580444335938, + "loss": 1.3403, + "margin_dpo/margin_mean": 229.70465087890625, + "margin_dpo/margin_std": 480.60894775390625, + "step": 271 + }, + { + "KL/chosen_KL_mean": -352.65478515625, + "KL/mean": -527.4397583007812, + "KL/rejected_KL_mean": -702.2247314453125, + "KL/std": 438.55596923828125, + "epoch": 0.39941262848751835, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 349.5699768066406, + "fcm_dpo/q_t": 0.48131972551345825, + "grad_norm": 8.041435241699219, + "learning_rate": 3.772161666010912e-07, + "logits/chosen": -0.15468257665634155, + "logits/rejected": -0.16906066238880157, + "logps/chosen": -402.15191650390625, + "logps/ref_chosen": -49.497154235839844, + "logps/ref_rejected": -105.54279327392578, + "logps/rejected": -807.767578125, + "loss": 1.3154, + "margin_dpo/margin_mean": 349.5699462890625, + "margin_dpo/margin_std": 480.5194091796875, + "step": 272 + }, + { + "KL/chosen_KL_mean": -454.3601989746094, + "KL/mean": -596.029541015625, + "KL/rejected_KL_mean": -737.698974609375, + "KL/std": 466.94976806640625, + "epoch": 0.4008810572687225, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 283.33868408203125, + "fcm_dpo/q_t": 0.48487192392349243, + "grad_norm": 4.082789421081543, + "learning_rate": 3.761097448550755e-07, + "logits/chosen": -0.16157250106334686, + "logits/rejected": -0.16911356151103973, + "logps/chosen": -517.3355712890625, + "logps/ref_chosen": -62.97539520263672, + "logps/ref_rejected": -92.49858093261719, + "logps/rejected": -830.197509765625, + "loss": 1.3297, + "margin_dpo/margin_mean": 283.33868408203125, + "margin_dpo/margin_std": 531.16064453125, + "step": 273 + }, + { + "KL/chosen_KL_mean": -471.19451904296875, + "KL/mean": -561.68115234375, + "KL/rejected_KL_mean": -652.167724609375, + "KL/std": 369.1112365722656, + "epoch": 0.4023494860499266, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 180.9732208251953, + "fcm_dpo/q_t": 0.49033892154693604, + "grad_norm": 4.708371639251709, + "learning_rate": 3.75e-07, + "logits/chosen": -0.10799916088581085, + "logits/rejected": -0.10551673173904419, + "logps/chosen": -526.8622436523438, + "logps/ref_chosen": -55.66770935058594, + "logps/ref_rejected": -77.33308410644531, + "logps/rejected": -729.5008544921875, + "loss": 1.3505, + "margin_dpo/margin_mean": 180.97323608398438, + "margin_dpo/margin_std": 471.4014587402344, + "step": 274 + }, + { + "KL/chosen_KL_mean": -417.6435546875, + "KL/mean": -547.9447631835938, + "KL/rejected_KL_mean": -678.2459716796875, + "KL/std": 425.20245361328125, + "epoch": 0.40381791483113066, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 260.6024169921875, + "fcm_dpo/q_t": 0.48607248067855835, + "grad_norm": 5.847025394439697, + "learning_rate": 3.738869612786737e-07, + "logits/chosen": -0.15583306550979614, + "logits/rejected": -0.16265209019184113, + "logps/chosen": -466.2382507324219, + "logps/ref_chosen": -48.594703674316406, + "logps/ref_rejected": -93.30369567871094, + "logps/rejected": -771.5496826171875, + "loss": 1.3342, + "margin_dpo/margin_mean": 260.6024169921875, + "margin_dpo/margin_std": 507.73193359375, + "step": 275 + }, + { + "KL/chosen_KL_mean": -472.4503173828125, + "KL/mean": -557.5875244140625, + "KL/rejected_KL_mean": -642.7247314453125, + "KL/std": 441.7587890625, + "epoch": 0.4052863436123348, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 170.2743377685547, + "fcm_dpo/q_t": 0.4909123182296753, + "grad_norm": 5.609163284301758, + "learning_rate": 3.7277065802070204e-07, + "logits/chosen": -0.14788061380386353, + "logits/rejected": -0.1665324866771698, + "logps/chosen": -529.0277099609375, + "logps/ref_chosen": -56.57740783691406, + "logps/ref_rejected": -70.36566925048828, + "logps/rejected": -713.09033203125, + "loss": 1.3544, + "margin_dpo/margin_mean": 170.2743377685547, + "margin_dpo/margin_std": 601.065185546875, + "step": 276 + }, + { + "KL/chosen_KL_mean": -494.017578125, + "KL/mean": -627.9515380859375, + "KL/rejected_KL_mean": -761.8856201171875, + "KL/std": 439.302001953125, + "epoch": 0.4067547723935389, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 267.8680725097656, + "fcm_dpo/q_t": 0.4856921434402466, + "grad_norm": 4.117493629455566, + "learning_rate": 3.71651119641714e-07, + "logits/chosen": -0.14363861083984375, + "logits/rejected": -0.13916437327861786, + "logps/chosen": -550.2891235351562, + "logps/ref_chosen": -56.27156066894531, + "logps/ref_rejected": -92.88127136230469, + "logps/rejected": -854.766845703125, + "loss": 1.333, + "margin_dpo/margin_mean": 267.8680419921875, + "margin_dpo/margin_std": 535.217529296875, + "step": 277 + }, + { + "KL/chosen_KL_mean": -475.8216552734375, + "KL/mean": -630.0145263671875, + "KL/rejected_KL_mean": -784.2073974609375, + "KL/std": 520.323486328125, + "epoch": 0.40822320117474303, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 308.3857116699219, + "fcm_dpo/q_t": 0.4835778772830963, + "grad_norm": 4.502967357635498, + "learning_rate": 3.705283756425872e-07, + "logits/chosen": -0.10104444622993469, + "logits/rejected": -0.11467007547616959, + "logps/chosen": -528.7635498046875, + "logps/ref_chosen": -52.94194030761719, + "logps/ref_rejected": -91.25357818603516, + "logps/rejected": -875.4609375, + "loss": 1.3266, + "margin_dpo/margin_mean": 308.3857116699219, + "margin_dpo/margin_std": 675.7715454101562, + "step": 278 + }, + { + "KL/chosen_KL_mean": -527.8968505859375, + "KL/mean": -687.3350830078125, + "KL/rejected_KL_mean": -846.7733154296875, + "KL/std": 570.1357421875, + "epoch": 0.40969162995594716, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 318.8764953613281, + "fcm_dpo/q_t": 0.4832102060317993, + "grad_norm": 6.868563175201416, + "learning_rate": 3.6940245560867e-07, + "logits/chosen": -0.11306406557559967, + "logits/rejected": -0.1245322972536087, + "logps/chosen": -576.5381469726562, + "logps/ref_chosen": -48.641319274902344, + "logps/ref_rejected": -87.8514404296875, + "logps/rejected": -934.624755859375, + "loss": 1.3266, + "margin_dpo/margin_mean": 318.8764953613281, + "margin_dpo/margin_std": 792.3270874023438, + "step": 279 + }, + { + "KL/chosen_KL_mean": -510.629150390625, + "KL/mean": -667.2784423828125, + "KL/rejected_KL_mean": -823.9278564453125, + "KL/std": 481.0506591796875, + "epoch": 0.4111600587371512, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 313.29864501953125, + "fcm_dpo/q_t": 0.48329728841781616, + "grad_norm": 4.903005123138428, + "learning_rate": 3.6827338920900253e-07, + "logits/chosen": -0.13975031673908234, + "logits/rejected": -0.1519356071949005, + "logps/chosen": -569.42626953125, + "logps/ref_chosen": -58.797122955322266, + "logps/ref_rejected": -98.61885070800781, + "logps/rejected": -922.546630859375, + "loss": 1.3249, + "margin_dpo/margin_mean": 313.29864501953125, + "margin_dpo/margin_std": 634.85693359375, + "step": 280 + }, + { + "KL/chosen_KL_mean": -512.4186401367188, + "KL/mean": -614.4647216796875, + "KL/rejected_KL_mean": -716.5108032226562, + "KL/std": 448.61798095703125, + "epoch": 0.41262848751835535, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 204.09217834472656, + "fcm_dpo/q_t": 0.48906901478767395, + "grad_norm": 6.864736557006836, + "learning_rate": 3.6714120619553435e-07, + "logits/chosen": -0.14065885543823242, + "logits/rejected": -0.15409055352210999, + "logps/chosen": -567.9071655273438, + "logps/ref_chosen": -55.488521575927734, + "logps/ref_rejected": -80.88258361816406, + "logps/rejected": -797.3933715820312, + "loss": 1.3468, + "margin_dpo/margin_mean": 204.09219360351562, + "margin_dpo/margin_std": 557.4766845703125, + "step": 281 + }, + { + "KL/chosen_KL_mean": -550.32373046875, + "KL/mean": -651.2105712890625, + "KL/rejected_KL_mean": -752.0973510742188, + "KL/std": 611.913818359375, + "epoch": 0.41409691629955947, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 201.77359008789062, + "fcm_dpo/q_t": 0.48951423168182373, + "grad_norm": 6.504598140716553, + "learning_rate": 3.660059364023408e-07, + "logits/chosen": -0.18802456557750702, + "logits/rejected": -0.1919810175895691, + "logps/chosen": -623.3939208984375, + "logps/ref_chosen": -73.07014465332031, + "logps/ref_rejected": -95.35098266601562, + "logps/rejected": -847.4483642578125, + "loss": 1.3497, + "margin_dpo/margin_mean": 201.77359008789062, + "margin_dpo/margin_std": 712.3954467773438, + "step": 282 + }, + { + "KL/chosen_KL_mean": -534.77490234375, + "KL/mean": -697.13916015625, + "KL/rejected_KL_mean": -859.5034790039062, + "KL/std": 511.788818359375, + "epoch": 0.4155653450807636, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 324.72857666015625, + "fcm_dpo/q_t": 0.4826442003250122, + "grad_norm": 6.365213871002197, + "learning_rate": 3.6486760974483685e-07, + "logits/chosen": -0.13428352773189545, + "logits/rejected": -0.14583396911621094, + "logps/chosen": -596.67333984375, + "logps/ref_chosen": -61.89844512939453, + "logps/ref_rejected": -96.98655700683594, + "logps/rejected": -956.489990234375, + "loss": 1.3214, + "margin_dpo/margin_mean": 324.72857666015625, + "margin_dpo/margin_std": 536.8545532226562, + "step": 283 + }, + { + "KL/chosen_KL_mean": -439.5884704589844, + "KL/mean": -578.688232421875, + "KL/rejected_KL_mean": -717.7880249023438, + "KL/std": 436.02374267578125, + "epoch": 0.4170337738619677, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 278.19952392578125, + "fcm_dpo/q_t": 0.48516762256622314, + "grad_norm": 5.291563987731934, + "learning_rate": 3.6372625621898863e-07, + "logits/chosen": -0.18925726413726807, + "logits/rejected": -0.18296435475349426, + "logps/chosen": -498.0240173339844, + "logps/ref_chosen": -58.4355354309082, + "logps/ref_rejected": -93.46926879882812, + "logps/rejected": -811.25732421875, + "loss": 1.3307, + "margin_dpo/margin_mean": 278.19952392578125, + "margin_dpo/margin_std": 519.5226440429688, + "step": 284 + }, + { + "KL/chosen_KL_mean": -519.1543579101562, + "KL/mean": -606.4476928710938, + "KL/rejected_KL_mean": -693.740966796875, + "KL/std": 376.2451171875, + "epoch": 0.4185022026431718, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 174.586669921875, + "fcm_dpo/q_t": 0.4906350374221802, + "grad_norm": 4.204360485076904, + "learning_rate": 3.625819059005228e-07, + "logits/chosen": -0.14472293853759766, + "logits/rejected": -0.1445767879486084, + "logps/chosen": -585.3865966796875, + "logps/ref_chosen": -66.23219299316406, + "logps/ref_rejected": -99.1268310546875, + "logps/rejected": -792.8677978515625, + "loss": 1.3521, + "margin_dpo/margin_mean": 174.586669921875, + "margin_dpo/margin_std": 497.9844970703125, + "step": 285 + }, + { + "KL/chosen_KL_mean": -518.2225341796875, + "KL/mean": -618.0490112304688, + "KL/rejected_KL_mean": -717.87548828125, + "KL/std": 502.3042297363281, + "epoch": 0.4199706314243759, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 199.6529541015625, + "fcm_dpo/q_t": 0.48932021856307983, + "grad_norm": 4.919650077819824, + "learning_rate": 3.614345889441346e-07, + "logits/chosen": -0.14968830347061157, + "logits/rejected": -0.1579841524362564, + "logps/chosen": -591.173583984375, + "logps/ref_chosen": -72.95100402832031, + "logps/ref_rejected": -88.58845520019531, + "logps/rejected": -806.4639282226562, + "loss": 1.3487, + "margin_dpo/margin_mean": 199.6529541015625, + "margin_dpo/margin_std": 646.603515625, + "step": 286 + }, + { + "KL/chosen_KL_mean": -417.3997497558594, + "KL/mean": -486.1492919921875, + "KL/rejected_KL_mean": -554.8988647460938, + "KL/std": 410.86279296875, + "epoch": 0.42143906020558003, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 137.49908447265625, + "fcm_dpo/q_t": 0.49265867471694946, + "grad_norm": 3.5626771450042725, + "learning_rate": 3.6028433558269275e-07, + "logits/chosen": -0.14080211520195007, + "logits/rejected": -0.1525568813085556, + "logps/chosen": -478.94091796875, + "logps/ref_chosen": -61.54115295410156, + "logps/ref_rejected": -77.69607543945312, + "logps/rejected": -632.594970703125, + "loss": 1.36, + "margin_dpo/margin_mean": 137.49908447265625, + "margin_dpo/margin_std": 493.0873718261719, + "step": 287 + }, + { + "KL/chosen_KL_mean": -323.7350769042969, + "KL/mean": -437.349365234375, + "KL/rejected_KL_mean": -550.963623046875, + "KL/std": 331.2504577636719, + "epoch": 0.42290748898678415, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 227.22857666015625, + "fcm_dpo/q_t": 0.4878368079662323, + "grad_norm": 3.0430126190185547, + "learning_rate": 3.5913117612644327e-07, + "logits/chosen": -0.1575808823108673, + "logits/rejected": -0.16179470717906952, + "logps/chosen": -380.39630126953125, + "logps/ref_chosen": -56.661224365234375, + "logps/ref_rejected": -87.33570098876953, + "logps/rejected": -638.29931640625, + "loss": 1.3402, + "margin_dpo/margin_mean": 227.22857666015625, + "margin_dpo/margin_std": 409.6346435546875, + "step": 288 + }, + { + "KL/chosen_KL_mean": -281.5063171386719, + "KL/mean": -421.5150451660156, + "KL/rejected_KL_mean": -561.5237426757812, + "KL/std": 394.57177734375, + "epoch": 0.4243759177679883, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 280.01739501953125, + "fcm_dpo/q_t": 0.48507311940193176, + "grad_norm": 6.204428195953369, + "learning_rate": 3.5797514096221024e-07, + "logits/chosen": -0.11954732984304428, + "logits/rejected": -0.1284411996603012, + "logps/chosen": -326.7367248535156, + "logps/ref_chosen": -45.23039245605469, + "logps/ref_rejected": -87.64266967773438, + "logps/rejected": -649.1663818359375, + "loss": 1.3298, + "margin_dpo/margin_mean": 280.01739501953125, + "margin_dpo/margin_std": 468.9491882324219, + "step": 289 + }, + { + "KL/chosen_KL_mean": -315.4320983886719, + "KL/mean": -489.9077453613281, + "KL/rejected_KL_mean": -664.3833618164062, + "KL/std": 450.1668701171875, + "epoch": 0.42584434654919234, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 348.9513244628906, + "fcm_dpo/q_t": 0.48141106963157654, + "grad_norm": 7.722722053527832, + "learning_rate": 3.568162605525952e-07, + "logits/chosen": -0.12268570065498352, + "logits/rejected": -0.14039406180381775, + "logps/chosen": -370.9035949707031, + "logps/ref_chosen": -55.47149658203125, + "logps/ref_rejected": -116.70857238769531, + "logps/rejected": -781.0919189453125, + "loss": 1.3162, + "margin_dpo/margin_mean": 348.9512939453125, + "margin_dpo/margin_std": 530.4649658203125, + "step": 290 + }, + { + "KL/chosen_KL_mean": -322.58355712890625, + "KL/mean": -421.381103515625, + "KL/rejected_KL_mean": -520.1787109375, + "KL/std": 350.70855712890625, + "epoch": 0.42731277533039647, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 197.59518432617188, + "fcm_dpo/q_t": 0.4894236922264099, + "grad_norm": 4.782266139984131, + "learning_rate": 3.5565456543517485e-07, + "logits/chosen": -0.14178414642810822, + "logits/rejected": -0.1538257598876953, + "logps/chosen": -385.8439025878906, + "logps/ref_chosen": -63.26036834716797, + "logps/ref_rejected": -89.29708862304688, + "logps/rejected": -609.475830078125, + "loss": 1.3469, + "margin_dpo/margin_mean": 197.5951690673828, + "margin_dpo/margin_std": 465.94696044921875, + "step": 291 + }, + { + "KL/chosen_KL_mean": -300.87261962890625, + "KL/mean": -426.175537109375, + "KL/rejected_KL_mean": -551.4783935546875, + "KL/std": 354.70635986328125, + "epoch": 0.4287812041116006, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 250.60580444335938, + "fcm_dpo/q_t": 0.48658984899520874, + "grad_norm": 4.581497669219971, + "learning_rate": 3.5449008622169583e-07, + "logits/chosen": -0.1559632122516632, + "logits/rejected": -0.16916170716285706, + "logps/chosen": -354.7911376953125, + "logps/ref_chosen": -53.91852951049805, + "logps/ref_rejected": -89.96138000488281, + "logps/rejected": -641.4398193359375, + "loss": 1.3355, + "margin_dpo/margin_mean": 250.60580444335938, + "margin_dpo/margin_std": 433.79608154296875, + "step": 292 + }, + { + "KL/chosen_KL_mean": -365.57171630859375, + "KL/mean": -445.22955322265625, + "KL/rejected_KL_mean": -524.887451171875, + "KL/std": 410.1599426269531, + "epoch": 0.4302496328928047, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 159.31570434570312, + "fcm_dpo/q_t": 0.4914965033531189, + "grad_norm": 3.991530656814575, + "learning_rate": 3.5332285359726846e-07, + "logits/chosen": -0.15570124983787537, + "logits/rejected": -0.1721823364496231, + "logps/chosen": -425.94775390625, + "logps/ref_chosen": -60.376033782958984, + "logps/ref_rejected": -77.85244750976562, + "logps/rejected": -602.7398681640625, + "loss": 1.3547, + "margin_dpo/margin_mean": 159.315673828125, + "margin_dpo/margin_std": 447.989501953125, + "step": 293 + }, + { + "KL/chosen_KL_mean": -289.04150390625, + "KL/mean": -404.94580078125, + "KL/rejected_KL_mean": -520.8502197265625, + "KL/std": 356.51409912109375, + "epoch": 0.43171806167400884, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 231.8087158203125, + "fcm_dpo/q_t": 0.4876137375831604, + "grad_norm": 5.801051616668701, + "learning_rate": 3.5215289831955786e-07, + "logits/chosen": -0.11081454157829285, + "logits/rejected": -0.12332391738891602, + "logps/chosen": -337.1290283203125, + "logps/ref_chosen": -48.0875358581543, + "logps/ref_rejected": -81.89698791503906, + "logps/rejected": -602.7471923828125, + "loss": 1.3401, + "margin_dpo/margin_mean": 231.80870056152344, + "margin_dpo/margin_std": 499.6016540527344, + "step": 294 + }, + { + "KL/chosen_KL_mean": -377.87554931640625, + "KL/mean": -518.6566162109375, + "KL/rejected_KL_mean": -659.437744140625, + "KL/std": 447.6157531738281, + "epoch": 0.4331864904552129, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 281.56219482421875, + "fcm_dpo/q_t": 0.48502007126808167, + "grad_norm": 3.4757144451141357, + "learning_rate": 3.509802512179737e-07, + "logits/chosen": -0.13640262186527252, + "logits/rejected": -0.13862435519695282, + "logps/chosen": -427.80023193359375, + "logps/ref_chosen": -49.92467498779297, + "logps/ref_rejected": -87.45632934570312, + "logps/rejected": -746.89404296875, + "loss": 1.3308, + "margin_dpo/margin_mean": 281.56219482421875, + "margin_dpo/margin_std": 583.3878173828125, + "step": 295 + }, + { + "KL/chosen_KL_mean": -489.06207275390625, + "KL/mean": -576.8760986328125, + "KL/rejected_KL_mean": -664.6901245117188, + "KL/std": 395.9642333984375, + "epoch": 0.434654919236417, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 175.62806701660156, + "fcm_dpo/q_t": 0.49063578248023987, + "grad_norm": 3.92090106010437, + "learning_rate": 3.498049431928577e-07, + "logits/chosen": -0.12152425199747086, + "logits/rejected": -0.12600529193878174, + "logps/chosen": -554.5533447265625, + "logps/ref_chosen": -65.49124145507812, + "logps/ref_rejected": -93.08908081054688, + "logps/rejected": -757.7791748046875, + "loss": 1.3523, + "margin_dpo/margin_mean": 175.62808227539062, + "margin_dpo/margin_std": 526.7503662109375, + "step": 296 + }, + { + "KL/chosen_KL_mean": -468.5126037597656, + "KL/mean": -587.1865234375, + "KL/rejected_KL_mean": -705.8604736328125, + "KL/std": 483.5227966308594, + "epoch": 0.43612334801762115, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 237.34788513183594, + "fcm_dpo/q_t": 0.48738476634025574, + "grad_norm": 5.903518199920654, + "learning_rate": 3.486270052146694e-07, + "logits/chosen": -0.19832845032215118, + "logits/rejected": -0.213718980550766, + "logps/chosen": -524.9895629882812, + "logps/ref_chosen": -56.476951599121094, + "logps/ref_rejected": -95.1385498046875, + "logps/rejected": -800.9990234375, + "loss": 1.3395, + "margin_dpo/margin_mean": 237.34786987304688, + "margin_dpo/margin_std": 542.2998046875, + "step": 297 + }, + { + "KL/chosen_KL_mean": -583.92919921875, + "KL/mean": -760.1380004882812, + "KL/rejected_KL_mean": -936.3467407226562, + "KL/std": 648.7681884765625, + "epoch": 0.43759177679882527, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 352.41748046875, + "fcm_dpo/q_t": 0.4814052879810333, + "grad_norm": 5.72074556350708, + "learning_rate": 3.474464683231698e-07, + "logits/chosen": -0.15087062120437622, + "logits/rejected": -0.1606747806072235, + "logps/chosen": -651.25439453125, + "logps/ref_chosen": -67.32516479492188, + "logps/ref_rejected": -116.66217041015625, + "logps/rejected": -1053.0089111328125, + "loss": 1.3195, + "margin_dpo/margin_mean": 352.41748046875, + "margin_dpo/margin_std": 788.0234375, + "step": 298 + }, + { + "KL/chosen_KL_mean": -473.8212890625, + "KL/mean": -579.8349609375, + "KL/rejected_KL_mean": -685.8486328125, + "KL/std": 470.7117919921875, + "epoch": 0.4390602055800294, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 212.02735900878906, + "fcm_dpo/q_t": 0.4887089133262634, + "grad_norm": 7.583713054656982, + "learning_rate": 3.462633636266041e-07, + "logits/chosen": -0.23694965243339539, + "logits/rejected": -0.2537599205970764, + "logps/chosen": -522.7833862304688, + "logps/ref_chosen": -48.96209716796875, + "logps/ref_rejected": -84.32823944091797, + "logps/rejected": -770.1768798828125, + "loss": 1.3463, + "margin_dpo/margin_mean": 212.02735900878906, + "margin_dpo/margin_std": 633.6422119140625, + "step": 299 + }, + { + "KL/chosen_KL_mean": -611.6695556640625, + "KL/mean": -765.8885498046875, + "KL/rejected_KL_mean": -920.107666015625, + "KL/std": 540.96484375, + "epoch": 0.44052863436123346, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 308.4380798339844, + "fcm_dpo/q_t": 0.48355600237846375, + "grad_norm": 7.862407207489014, + "learning_rate": 3.4507772230088147e-07, + "logits/chosen": -0.19781388342380524, + "logits/rejected": -0.21643571555614471, + "logps/chosen": -670.7432861328125, + "logps/ref_chosen": -59.07371139526367, + "logps/ref_rejected": -95.9664535522461, + "logps/rejected": -1016.0740966796875, + "loss": 1.327, + "margin_dpo/margin_mean": 308.4381103515625, + "margin_dpo/margin_std": 709.8028564453125, + "step": 300 + }, + { + "KL/chosen_KL_mean": -573.643310546875, + "KL/mean": -723.9158935546875, + "KL/rejected_KL_mean": -874.1885375976562, + "KL/std": 545.066162109375, + "epoch": 0.4419970631424376, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 300.5451965332031, + "fcm_dpo/q_t": 0.48413360118865967, + "grad_norm": 6.303064346313477, + "learning_rate": 3.4388957558875316e-07, + "logits/chosen": -0.19899940490722656, + "logits/rejected": -0.21283400058746338, + "logps/chosen": -630.8927001953125, + "logps/ref_chosen": -57.249366760253906, + "logps/ref_rejected": -92.35354614257812, + "logps/rejected": -966.5421142578125, + "loss": 1.3293, + "margin_dpo/margin_mean": 300.5451965332031, + "margin_dpo/margin_std": 747.690185546875, + "step": 301 + }, + { + "KL/chosen_KL_mean": -525.3902587890625, + "KL/mean": -676.2835693359375, + "KL/rejected_KL_mean": -827.1768798828125, + "KL/std": 550.9168701171875, + "epoch": 0.4434654919236417, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 301.78662109375, + "fcm_dpo/q_t": 0.4839833378791809, + "grad_norm": 6.866415500640869, + "learning_rate": 3.426989547989902e-07, + "logits/chosen": -0.2429056167602539, + "logits/rejected": -0.23922690749168396, + "logps/chosen": -576.5882568359375, + "logps/ref_chosen": -51.197994232177734, + "logps/ref_rejected": -97.22636413574219, + "logps/rejected": -924.4032592773438, + "loss": 1.3291, + "margin_dpo/margin_mean": 301.78662109375, + "margin_dpo/margin_std": 743.7776489257812, + "step": 302 + }, + { + "KL/chosen_KL_mean": -630.103515625, + "KL/mean": -715.7218017578125, + "KL/rejected_KL_mean": -801.3401489257812, + "KL/std": 567.647705078125, + "epoch": 0.44493392070484583, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 171.23655700683594, + "fcm_dpo/q_t": 0.4908214807510376, + "grad_norm": 6.0209126472473145, + "learning_rate": 3.4150589130555773e-07, + "logits/chosen": -0.23265428841114044, + "logits/rejected": -0.24132990837097168, + "logps/chosen": -696.8175048828125, + "logps/ref_chosen": -66.71394348144531, + "logps/ref_rejected": -86.94542694091797, + "logps/rejected": -888.2855834960938, + "loss": 1.3558, + "margin_dpo/margin_mean": 171.23655700683594, + "margin_dpo/margin_std": 712.87548828125, + "step": 303 + }, + { + "KL/chosen_KL_mean": -616.35107421875, + "KL/mean": -709.6608276367188, + "KL/rejected_KL_mean": -802.9705810546875, + "KL/std": 510.59539794921875, + "epoch": 0.44640234948604995, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 186.61947631835938, + "fcm_dpo/q_t": 0.49002233147621155, + "grad_norm": 5.741556167602539, + "learning_rate": 3.403104165467883e-07, + "logits/chosen": -0.2312694489955902, + "logits/rejected": -0.2368827909231186, + "logps/chosen": -688.3017578125, + "logps/ref_chosen": -71.95069885253906, + "logps/ref_rejected": -90.47203063964844, + "logps/rejected": -893.442626953125, + "loss": 1.3511, + "margin_dpo/margin_mean": 186.61947631835938, + "margin_dpo/margin_std": 618.4774169921875, + "step": 304 + }, + { + "KL/chosen_KL_mean": -570.4090576171875, + "KL/mean": -677.1749267578125, + "KL/rejected_KL_mean": -783.9407958984375, + "KL/std": 503.7247009277344, + "epoch": 0.447870778267254, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 213.53167724609375, + "fcm_dpo/q_t": 0.4886142611503601, + "grad_norm": 5.134280681610107, + "learning_rate": 3.391125620245535e-07, + "logits/chosen": -0.27188488841056824, + "logits/rejected": -0.2840760350227356, + "logps/chosen": -637.204345703125, + "logps/ref_chosen": -66.79523468017578, + "logps/ref_rejected": -92.75459289550781, + "logps/rejected": -876.6953735351562, + "loss": 1.3456, + "margin_dpo/margin_mean": 213.53167724609375, + "margin_dpo/margin_std": 633.0828857421875, + "step": 305 + }, + { + "KL/chosen_KL_mean": -537.7818603515625, + "KL/mean": -625.0843505859375, + "KL/rejected_KL_mean": -712.386962890625, + "KL/std": 452.66357421875, + "epoch": 0.44933920704845814, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 174.60516357421875, + "fcm_dpo/q_t": 0.4906894564628601, + "grad_norm": 5.521829128265381, + "learning_rate": 3.3791235930343417e-07, + "logits/chosen": -0.19213925302028656, + "logits/rejected": -0.19932372868061066, + "logps/chosen": -607.4656982421875, + "logps/ref_chosen": -69.68389892578125, + "logps/ref_rejected": -85.15919494628906, + "logps/rejected": -797.546142578125, + "loss": 1.3524, + "margin_dpo/margin_mean": 174.6051483154297, + "margin_dpo/margin_std": 521.4979858398438, + "step": 306 + }, + { + "KL/chosen_KL_mean": -450.71343994140625, + "KL/mean": -559.7984619140625, + "KL/rejected_KL_mean": -668.8834228515625, + "KL/std": 426.73651123046875, + "epoch": 0.45080763582966227, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 218.17001342773438, + "fcm_dpo/q_t": 0.4883463978767395, + "grad_norm": 4.214844226837158, + "learning_rate": 3.367098400098881e-07, + "logits/chosen": -0.22033822536468506, + "logits/rejected": -0.224267840385437, + "logps/chosen": -520.8788452148438, + "logps/ref_chosen": -70.16542053222656, + "logps/ref_rejected": -86.97230529785156, + "logps/rejected": -755.855712890625, + "loss": 1.3434, + "margin_dpo/margin_mean": 218.17001342773438, + "margin_dpo/margin_std": 541.6478271484375, + "step": 307 + }, + { + "KL/chosen_KL_mean": -401.5634765625, + "KL/mean": -537.181396484375, + "KL/rejected_KL_mean": -672.79931640625, + "KL/std": 440.7513122558594, + "epoch": 0.4522760646108664, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 271.23590087890625, + "fcm_dpo/q_t": 0.48551321029663086, + "grad_norm": 4.456334590911865, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": -0.19973334670066833, + "logits/rejected": -0.2119927853345871, + "logps/chosen": -456.8084716796875, + "logps/ref_chosen": -55.2449951171875, + "logps/ref_rejected": -79.37226104736328, + "logps/rejected": -752.171630859375, + "loss": 1.3319, + "margin_dpo/margin_mean": 271.2358703613281, + "margin_dpo/margin_std": 503.54925537109375, + "step": 308 + }, + { + "KL/chosen_KL_mean": -369.8251953125, + "KL/mean": -482.3707275390625, + "KL/rejected_KL_mean": -594.9161987304688, + "KL/std": 423.8313903808594, + "epoch": 0.45374449339207046, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 225.0910186767578, + "fcm_dpo/q_t": 0.48792463541030884, + "grad_norm": 4.421578407287598, + "learning_rate": 3.3429797851573183e-07, + "logits/chosen": -0.16784876585006714, + "logits/rejected": -0.18317045271396637, + "logps/chosen": -418.7843017578125, + "logps/ref_chosen": -48.959083557128906, + "logps/ref_rejected": -82.34072875976562, + "logps/rejected": -677.2569580078125, + "loss": 1.3417, + "margin_dpo/margin_mean": 225.0910186767578, + "margin_dpo/margin_std": 493.40032958984375, + "step": 309 + }, + { + "KL/chosen_KL_mean": -440.13433837890625, + "KL/mean": -548.5415649414062, + "KL/rejected_KL_mean": -656.9487915039062, + "KL/std": 404.796875, + "epoch": 0.4552129221732746, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 216.81442260742188, + "fcm_dpo/q_t": 0.48840105533599854, + "grad_norm": 3.7958574295043945, + "learning_rate": 3.3308869986991487e-07, + "logits/chosen": -0.15935003757476807, + "logits/rejected": -0.17317405343055725, + "logps/chosen": -502.8761291503906, + "logps/ref_chosen": -62.74177932739258, + "logps/ref_rejected": -79.9302978515625, + "logps/rejected": -736.8790893554688, + "loss": 1.3427, + "margin_dpo/margin_mean": 216.81442260742188, + "margin_dpo/margin_std": 443.70294189453125, + "step": 310 + }, + { + "KL/chosen_KL_mean": -415.8055419921875, + "KL/mean": -523.12060546875, + "KL/rejected_KL_mean": -630.4356689453125, + "KL/std": 363.2186279296875, + "epoch": 0.4566813509544787, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 214.63011169433594, + "fcm_dpo/q_t": 0.4885249733924866, + "grad_norm": 4.3125104904174805, + "learning_rate": 3.3187723175958346e-07, + "logits/chosen": -0.21918794512748718, + "logits/rejected": -0.22478339076042175, + "logps/chosen": -468.83349609375, + "logps/ref_chosen": -53.02798080444336, + "logps/ref_rejected": -77.43820190429688, + "logps/rejected": -707.8738403320312, + "loss": 1.3428, + "margin_dpo/margin_mean": 214.63011169433594, + "margin_dpo/margin_std": 417.1317443847656, + "step": 311 + }, + { + "KL/chosen_KL_mean": -427.406005859375, + "KL/mean": -540.7687377929688, + "KL/rejected_KL_mean": -654.1314697265625, + "KL/std": 440.4998779296875, + "epoch": 0.4581497797356828, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 226.72544860839844, + "fcm_dpo/q_t": 0.4879283308982849, + "grad_norm": 4.058272361755371, + "learning_rate": 3.306636061080487e-07, + "logits/chosen": -0.17278644442558289, + "logits/rejected": -0.18266448378562927, + "logps/chosen": -476.7982177734375, + "logps/ref_chosen": -49.39221954345703, + "logps/ref_rejected": -75.79280853271484, + "logps/rejected": -729.92431640625, + "loss": 1.3417, + "margin_dpo/margin_mean": 226.7254638671875, + "margin_dpo/margin_std": 535.8837280273438, + "step": 312 + }, + { + "KL/chosen_KL_mean": -382.3973388671875, + "KL/mean": -525.8050537109375, + "KL/rejected_KL_mean": -669.2127685546875, + "KL/std": 460.6397399902344, + "epoch": 0.45961820851688695, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 286.8153991699219, + "fcm_dpo/q_t": 0.48471421003341675, + "grad_norm": 4.11210298538208, + "learning_rate": 3.2944785489547537e-07, + "logits/chosen": -0.18957430124282837, + "logits/rejected": -0.2057763636112213, + "logps/chosen": -432.55010986328125, + "logps/ref_chosen": -50.152740478515625, + "logps/ref_rejected": -86.40620422363281, + "logps/rejected": -755.6190185546875, + "loss": 1.3291, + "margin_dpo/margin_mean": 286.8153991699219, + "margin_dpo/margin_std": 513.8978271484375, + "step": 313 + }, + { + "KL/chosen_KL_mean": -473.88775634765625, + "KL/mean": -590.21435546875, + "KL/rejected_KL_mean": -706.541015625, + "KL/std": 460.99267578125, + "epoch": 0.461086637298091, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 232.65328979492188, + "fcm_dpo/q_t": 0.4876299202442169, + "grad_norm": 3.9764833450317383, + "learning_rate": 3.2823001015803857e-07, + "logits/chosen": -0.23278236389160156, + "logits/rejected": -0.2444659173488617, + "logps/chosen": -531.1253051757812, + "logps/ref_chosen": -57.237579345703125, + "logps/ref_rejected": -97.5965347290039, + "logps/rejected": -804.1375732421875, + "loss": 1.3414, + "margin_dpo/margin_mean": 232.65330505371094, + "margin_dpo/margin_std": 601.2679443359375, + "step": 314 + }, + { + "KL/chosen_KL_mean": -396.1183166503906, + "KL/mean": -510.1334228515625, + "KL/rejected_KL_mean": -624.1485595703125, + "KL/std": 454.22100830078125, + "epoch": 0.46255506607929514, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 228.03021240234375, + "fcm_dpo/q_t": 0.4878466725349426, + "grad_norm": 4.002105236053467, + "learning_rate": 3.270101039870797e-07, + "logits/chosen": -0.1942490041255951, + "logits/rejected": -0.20377588272094727, + "logps/chosen": -445.18792724609375, + "logps/ref_chosen": -49.06958770751953, + "logps/ref_rejected": -85.68087768554688, + "logps/rejected": -709.8294067382812, + "loss": 1.3415, + "margin_dpo/margin_mean": 228.03021240234375, + "margin_dpo/margin_std": 553.0503540039062, + "step": 315 + }, + { + "KL/chosen_KL_mean": -419.86102294921875, + "KL/mean": -615.5865478515625, + "KL/rejected_KL_mean": -811.3121948242188, + "KL/std": 490.57891845703125, + "epoch": 0.46402349486049926, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 391.451171875, + "fcm_dpo/q_t": 0.4791603088378906, + "grad_norm": 5.238584995269775, + "learning_rate": 3.2578816852826086e-07, + "logits/chosen": -0.2003282904624939, + "logits/rejected": -0.20365044474601746, + "logps/chosen": -474.12176513671875, + "logps/ref_chosen": -54.26074981689453, + "logps/ref_rejected": -101.2814712524414, + "logps/rejected": -912.5936279296875, + "loss": 1.3085, + "margin_dpo/margin_mean": 391.451171875, + "margin_dpo/margin_std": 604.1680908203125, + "step": 316 + }, + { + "KL/chosen_KL_mean": -472.4468994140625, + "KL/mean": -679.812744140625, + "KL/rejected_KL_mean": -887.178466796875, + "KL/std": 529.8192138671875, + "epoch": 0.4654919236417034, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 414.73162841796875, + "fcm_dpo/q_t": 0.47804775834083557, + "grad_norm": 4.577226638793945, + "learning_rate": 3.2456423598071783e-07, + "logits/chosen": -0.2022346556186676, + "logits/rejected": -0.2135677933692932, + "logps/chosen": -528.5411376953125, + "logps/ref_chosen": -56.094207763671875, + "logps/ref_rejected": -100.69905090332031, + "logps/rejected": -987.8775634765625, + "loss": 1.3053, + "margin_dpo/margin_mean": 414.73162841796875, + "margin_dpo/margin_std": 667.3003540039062, + "step": 317 + }, + { + "KL/chosen_KL_mean": -563.6049194335938, + "KL/mean": -657.4984130859375, + "KL/rejected_KL_mean": -751.3919067382812, + "KL/std": 476.6451721191406, + "epoch": 0.4669603524229075, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 187.7869873046875, + "fcm_dpo/q_t": 0.48997563123703003, + "grad_norm": 5.925684452056885, + "learning_rate": 3.233383385962115e-07, + "logits/chosen": -0.2424195110797882, + "logits/rejected": -0.2464376538991928, + "logps/chosen": -628.2506103515625, + "logps/ref_chosen": -64.64569854736328, + "logps/ref_rejected": -82.76425170898438, + "logps/rejected": -834.1561279296875, + "loss": 1.3496, + "margin_dpo/margin_mean": 187.78700256347656, + "margin_dpo/margin_std": 521.0235595703125, + "step": 318 + }, + { + "KL/chosen_KL_mean": -436.5062255859375, + "KL/mean": -647.1476440429688, + "KL/rejected_KL_mean": -857.7891235351562, + "KL/std": 617.1673583984375, + "epoch": 0.4684287812041116, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 421.28289794921875, + "fcm_dpo/q_t": 0.47768670320510864, + "grad_norm": 7.061251163482666, + "learning_rate": 3.2211050867827805e-07, + "logits/chosen": -0.19704844057559967, + "logits/rejected": -0.20887039601802826, + "logps/chosen": -485.88995361328125, + "logps/ref_chosen": -49.383758544921875, + "logps/ref_rejected": -113.90650939941406, + "logps/rejected": -971.6956176757812, + "loss": 1.3037, + "margin_dpo/margin_mean": 421.28289794921875, + "margin_dpo/margin_std": 688.0316162109375, + "step": 319 + }, + { + "KL/chosen_KL_mean": -505.7894592285156, + "KL/mean": -670.9154052734375, + "KL/rejected_KL_mean": -836.041259765625, + "KL/std": 497.88201904296875, + "epoch": 0.4698972099853157, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 330.25177001953125, + "fcm_dpo/q_t": 0.4823932647705078, + "grad_norm": 4.2582244873046875, + "learning_rate": 3.208807785813777e-07, + "logits/chosen": -0.15992069244384766, + "logits/rejected": -0.17337793111801147, + "logps/chosen": -565.2943725585938, + "logps/ref_chosen": -59.50489044189453, + "logps/ref_rejected": -97.66717529296875, + "logps/rejected": -933.70849609375, + "loss": 1.3212, + "margin_dpo/margin_mean": 330.25177001953125, + "margin_dpo/margin_std": 617.7380981445312, + "step": 320 + }, + { + "KL/chosen_KL_mean": -604.0084228515625, + "KL/mean": -743.7211303710938, + "KL/rejected_KL_mean": -883.433837890625, + "KL/std": 562.9217529296875, + "epoch": 0.4713656387665198, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 279.4254455566406, + "fcm_dpo/q_t": 0.4851454198360443, + "grad_norm": 5.155450344085693, + "learning_rate": 3.1964918071004217e-07, + "logits/chosen": -0.14682038128376007, + "logits/rejected": -0.15771764516830444, + "logps/chosen": -665.5570678710938, + "logps/ref_chosen": -61.548683166503906, + "logps/ref_rejected": -91.64103698730469, + "logps/rejected": -975.074951171875, + "loss": 1.3322, + "margin_dpo/margin_mean": 279.42547607421875, + "margin_dpo/margin_std": 654.2274169921875, + "step": 321 + }, + { + "KL/chosen_KL_mean": -511.4791259765625, + "KL/mean": -655.0626831054688, + "KL/rejected_KL_mean": -798.646240234375, + "KL/std": 467.83734130859375, + "epoch": 0.47283406754772395, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 287.16717529296875, + "fcm_dpo/q_t": 0.4846848249435425, + "grad_norm": 5.708774089813232, + "learning_rate": 3.184157475180207e-07, + "logits/chosen": -0.17217618227005005, + "logits/rejected": -0.18240973353385925, + "logps/chosen": -568.7691650390625, + "logps/ref_chosen": -57.29003143310547, + "logps/ref_rejected": -95.74992370605469, + "logps/rejected": -894.396240234375, + "loss": 1.3294, + "margin_dpo/margin_mean": 287.16717529296875, + "margin_dpo/margin_std": 554.1672973632812, + "step": 322 + }, + { + "KL/chosen_KL_mean": -529.456787109375, + "KL/mean": -639.5374755859375, + "KL/rejected_KL_mean": -749.6182861328125, + "KL/std": 478.3594970703125, + "epoch": 0.47430249632892807, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 220.1614990234375, + "fcm_dpo/q_t": 0.4882357716560364, + "grad_norm": 5.594461441040039, + "learning_rate": 3.171805115074251e-07, + "logits/chosen": -0.1582409143447876, + "logits/rejected": -0.17803357541561127, + "logps/chosen": -580.690673828125, + "logps/ref_chosen": -51.23395919799805, + "logps/ref_rejected": -75.06192016601562, + "logps/rejected": -824.68017578125, + "loss": 1.3433, + "margin_dpo/margin_mean": 220.1614990234375, + "margin_dpo/margin_std": 536.7333984375, + "step": 323 + }, + { + "KL/chosen_KL_mean": -634.5806884765625, + "KL/mean": -732.8265380859375, + "KL/rejected_KL_mean": -831.072509765625, + "KL/std": 546.5975952148438, + "epoch": 0.47577092511013214, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 196.49183654785156, + "fcm_dpo/q_t": 0.48958471417427063, + "grad_norm": 6.335326194763184, + "learning_rate": 3.1594350522787295e-07, + "logits/chosen": -0.18952664732933044, + "logits/rejected": -0.20132169127464294, + "logps/chosen": -699.7158203125, + "logps/ref_chosen": -65.13516998291016, + "logps/ref_rejected": -86.47750854492188, + "logps/rejected": -917.5499877929688, + "loss": 1.3508, + "margin_dpo/margin_mean": 196.49185180664062, + "margin_dpo/margin_std": 704.073974609375, + "step": 324 + }, + { + "KL/chosen_KL_mean": -488.4103698730469, + "KL/mean": -557.0770263671875, + "KL/rejected_KL_mean": -625.7437133789062, + "KL/std": 416.86346435546875, + "epoch": 0.47723935389133626, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 137.33335876464844, + "fcm_dpo/q_t": 0.4926797151565552, + "grad_norm": 5.764030456542969, + "learning_rate": 3.147047612756302e-07, + "logits/chosen": -0.23148123919963837, + "logits/rejected": -0.2455836832523346, + "logps/chosen": -544.6259765625, + "logps/ref_chosen": -56.215599060058594, + "logps/ref_rejected": -70.08592987060547, + "logps/rejected": -695.8296508789062, + "loss": 1.3603, + "margin_dpo/margin_mean": 137.3333740234375, + "margin_dpo/margin_std": 519.9088745117188, + "step": 325 + }, + { + "KL/chosen_KL_mean": -546.6886596679688, + "KL/mean": -624.8822021484375, + "KL/rejected_KL_mean": -703.0758666992188, + "KL/std": 415.1629333496094, + "epoch": 0.4787077826725404, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 156.38719177246094, + "fcm_dpo/q_t": 0.4916384220123291, + "grad_norm": 5.459661960601807, + "learning_rate": 3.134643122927519e-07, + "logits/chosen": -0.23426343500614166, + "logits/rejected": -0.24521896243095398, + "logps/chosen": -619.4136352539062, + "logps/ref_chosen": -72.72496032714844, + "logps/ref_rejected": -79.8467788696289, + "logps/rejected": -782.922607421875, + "loss": 1.3553, + "margin_dpo/margin_mean": 156.38717651367188, + "margin_dpo/margin_std": 438.69140625, + "step": 326 + }, + { + "KL/chosen_KL_mean": -510.6233825683594, + "KL/mean": -662.1951904296875, + "KL/rejected_KL_mean": -813.7669677734375, + "KL/std": 460.990478515625, + "epoch": 0.4801762114537445, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 303.1435852050781, + "fcm_dpo/q_t": 0.48379653692245483, + "grad_norm": 6.2359209060668945, + "learning_rate": 3.1222219096622264e-07, + "logits/chosen": -0.19739487767219543, + "logits/rejected": -0.2052072286605835, + "logps/chosen": -579.7578125, + "logps/ref_chosen": -69.13441467285156, + "logps/ref_rejected": -111.93377685546875, + "logps/rejected": -925.7008056640625, + "loss": 1.3262, + "margin_dpo/margin_mean": 303.14361572265625, + "margin_dpo/margin_std": 575.9744873046875, + "step": 327 + }, + { + "KL/chosen_KL_mean": -442.1131591796875, + "KL/mean": -588.37548828125, + "KL/rejected_KL_mean": -734.6378173828125, + "KL/std": 493.975830078125, + "epoch": 0.48164464023494863, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 292.524658203125, + "fcm_dpo/q_t": 0.4843920171260834, + "grad_norm": 5.148623943328857, + "learning_rate": 3.1097843002709427e-07, + "logits/chosen": -0.2125837206840515, + "logits/rejected": -0.2268255650997162, + "logps/chosen": -501.80035400390625, + "logps/ref_chosen": -59.68719482421875, + "logps/ref_rejected": -90.85499572753906, + "logps/rejected": -825.4927978515625, + "loss": 1.3284, + "margin_dpo/margin_mean": 292.524658203125, + "margin_dpo/margin_std": 573.4638671875, + "step": 328 + }, + { + "KL/chosen_KL_mean": -491.8397216796875, + "KL/mean": -647.6664428710938, + "KL/rejected_KL_mean": -803.4930419921875, + "KL/std": 506.99774169921875, + "epoch": 0.4831130690161527, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 311.6534118652344, + "fcm_dpo/q_t": 0.48337361216545105, + "grad_norm": 4.15192985534668, + "learning_rate": 3.0973306224962437e-07, + "logits/chosen": -0.1763824224472046, + "logits/rejected": -0.19332997500896454, + "logps/chosen": -557.0859375, + "logps/ref_chosen": -65.2461929321289, + "logps/ref_rejected": -100.69770812988281, + "logps/rejected": -904.1907958984375, + "loss": 1.325, + "margin_dpo/margin_mean": 311.65338134765625, + "margin_dpo/margin_std": 609.5528564453125, + "step": 329 + }, + { + "KL/chosen_KL_mean": -342.6683349609375, + "KL/mean": -501.531982421875, + "KL/rejected_KL_mean": -660.3956298828125, + "KL/std": 415.15380859375, + "epoch": 0.4845814977973568, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 317.727294921875, + "fcm_dpo/q_t": 0.483031302690506, + "grad_norm": 7.011267185211182, + "learning_rate": 3.084861204504122e-07, + "logits/chosen": -0.14472857117652893, + "logits/rejected": -0.1459788680076599, + "logps/chosen": -389.66668701171875, + "logps/ref_chosen": -46.998348236083984, + "logps/ref_rejected": -86.87684631347656, + "logps/rejected": -747.2724609375, + "loss": 1.3222, + "margin_dpo/margin_mean": 317.7272644042969, + "margin_dpo/margin_std": 499.0316162109375, + "step": 330 + }, + { + "KL/chosen_KL_mean": -391.0220642089844, + "KL/mean": -550.6251220703125, + "KL/rejected_KL_mean": -710.228271484375, + "KL/std": 350.403076171875, + "epoch": 0.48604992657856094, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 319.2061462402344, + "fcm_dpo/q_t": 0.4829198718070984, + "grad_norm": 5.114991664886475, + "learning_rate": 3.072376374875335e-07, + "logits/chosen": -0.16779181361198425, + "logits/rejected": -0.1757754683494568, + "logps/chosen": -441.54632568359375, + "logps/ref_chosen": -50.52424621582031, + "logps/ref_rejected": -89.01544189453125, + "logps/rejected": -799.2437133789062, + "loss": 1.3205, + "margin_dpo/margin_mean": 319.2061462402344, + "margin_dpo/margin_std": 355.31463623046875, + "step": 331 + }, + { + "KL/chosen_KL_mean": -429.01025390625, + "KL/mean": -513.4847412109375, + "KL/rejected_KL_mean": -597.9593505859375, + "KL/std": 391.617919921875, + "epoch": 0.48751835535976507, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 168.9490966796875, + "fcm_dpo/q_t": 0.4909921884536743, + "grad_norm": 4.994297981262207, + "learning_rate": 3.059876462596758e-07, + "logits/chosen": -0.14846037328243256, + "logits/rejected": -0.15179188549518585, + "logps/chosen": -478.1905212402344, + "logps/ref_chosen": -49.18028259277344, + "logps/ref_rejected": -76.48515319824219, + "logps/rejected": -674.4444580078125, + "loss": 1.3528, + "margin_dpo/margin_mean": 168.9490966796875, + "margin_dpo/margin_std": 455.59197998046875, + "step": 332 + }, + { + "KL/chosen_KL_mean": -516.8207397460938, + "KL/mean": -647.05615234375, + "KL/rejected_KL_mean": -777.2916259765625, + "KL/std": 472.027099609375, + "epoch": 0.4889867841409692, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 260.4708251953125, + "fcm_dpo/q_t": 0.48608171939849854, + "grad_norm": 5.020219326019287, + "learning_rate": 3.0473617970527015e-07, + "logits/chosen": -0.22815987467765808, + "logits/rejected": -0.24586617946624756, + "logps/chosen": -580.5764770507812, + "logps/ref_chosen": -63.75574493408203, + "logps/ref_rejected": -95.04411315917969, + "logps/rejected": -872.335693359375, + "loss": 1.3361, + "margin_dpo/margin_mean": 260.4708251953125, + "margin_dpo/margin_std": 628.2245483398438, + "step": 333 + }, + { + "KL/chosen_KL_mean": -535.3387451171875, + "KL/mean": -647.19287109375, + "KL/rejected_KL_mean": -759.047119140625, + "KL/std": 525.2220458984375, + "epoch": 0.49045521292217326, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 223.70834350585938, + "fcm_dpo/q_t": 0.48809105157852173, + "grad_norm": 5.767882823944092, + "learning_rate": 3.034832708016243e-07, + "logits/chosen": -0.2623814344406128, + "logits/rejected": -0.27332639694213867, + "logps/chosen": -602.3184814453125, + "logps/ref_chosen": -66.97975158691406, + "logps/ref_rejected": -95.31692504882812, + "logps/rejected": -854.364013671875, + "loss": 1.3445, + "margin_dpo/margin_mean": 223.70834350585938, + "margin_dpo/margin_std": 687.914794921875, + "step": 334 + }, + { + "KL/chosen_KL_mean": -553.5545654296875, + "KL/mean": -645.1278076171875, + "KL/rejected_KL_mean": -736.701171875, + "KL/std": 531.672119140625, + "epoch": 0.4919236417033774, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 183.14663696289062, + "fcm_dpo/q_t": 0.4902423322200775, + "grad_norm": 5.44934606552124, + "learning_rate": 3.022289525640531e-07, + "logits/chosen": -0.2666965126991272, + "logits/rejected": -0.27081555128097534, + "logps/chosen": -616.0970458984375, + "logps/ref_chosen": -62.54248046875, + "logps/ref_rejected": -87.61770629882812, + "logps/rejected": -824.31884765625, + "loss": 1.351, + "margin_dpo/margin_mean": 183.14662170410156, + "margin_dpo/margin_std": 550.9996948242188, + "step": 335 + }, + { + "KL/chosen_KL_mean": -554.318115234375, + "KL/mean": -750.877685546875, + "KL/rejected_KL_mean": -947.4371948242188, + "KL/std": 650.4267578125, + "epoch": 0.4933920704845815, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 393.1190490722656, + "fcm_dpo/q_t": 0.47911518812179565, + "grad_norm": 7.3909687995910645, + "learning_rate": 3.009732580450086e-07, + "logits/chosen": -0.21140223741531372, + "logits/rejected": -0.21927091479301453, + "logps/chosen": -608.8492431640625, + "logps/ref_chosen": -54.53115463256836, + "logps/ref_rejected": -104.40424346923828, + "logps/rejected": -1051.8414306640625, + "loss": 1.3116, + "margin_dpo/margin_mean": 393.11907958984375, + "margin_dpo/margin_std": 823.8309326171875, + "step": 336 + }, + { + "KL/chosen_KL_mean": -571.74658203125, + "KL/mean": -740.125732421875, + "KL/rejected_KL_mean": -908.5048828125, + "KL/std": 476.75103759765625, + "epoch": 0.4948604992657856, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 336.75836181640625, + "fcm_dpo/q_t": 0.4820534288883209, + "grad_norm": 6.862320899963379, + "learning_rate": 2.9971622033320914e-07, + "logits/chosen": -0.25504040718078613, + "logits/rejected": -0.2622433006763458, + "logps/chosen": -636.875244140625, + "logps/ref_chosen": -65.12869262695312, + "logps/ref_rejected": -101.72701263427734, + "logps/rejected": -1010.23193359375, + "loss": 1.3197, + "margin_dpo/margin_mean": 336.75836181640625, + "margin_dpo/margin_std": 612.5994873046875, + "step": 337 + }, + { + "KL/chosen_KL_mean": -500.8605651855469, + "KL/mean": -660.3416748046875, + "KL/rejected_KL_mean": -819.82275390625, + "KL/std": 447.0543212890625, + "epoch": 0.49632892804698975, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 318.9621887207031, + "fcm_dpo/q_t": 0.48294728994369507, + "grad_norm": 5.339162349700928, + "learning_rate": 2.984578725527675e-07, + "logits/chosen": -0.25964879989624023, + "logits/rejected": -0.27327626943588257, + "logps/chosen": -559.2832641601562, + "logps/ref_chosen": -58.422706604003906, + "logps/ref_rejected": -89.06854248046875, + "logps/rejected": -908.8912963867188, + "loss": 1.3215, + "margin_dpo/margin_mean": 318.9621887207031, + "margin_dpo/margin_std": 463.43841552734375, + "step": 338 + }, + { + "KL/chosen_KL_mean": -594.2242431640625, + "KL/mean": -748.5675048828125, + "KL/rejected_KL_mean": -902.9107666015625, + "KL/std": 501.53179931640625, + "epoch": 0.4977973568281938, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 308.68658447265625, + "fcm_dpo/q_t": 0.48354700207710266, + "grad_norm": 6.215461730957031, + "learning_rate": 2.9719824786231796e-07, + "logits/chosen": -0.2669673264026642, + "logits/rejected": -0.27526944875717163, + "logps/chosen": -654.219482421875, + "logps/ref_chosen": -59.99531555175781, + "logps/ref_rejected": -103.9109115600586, + "logps/rejected": -1006.8216552734375, + "loss": 1.3258, + "margin_dpo/margin_mean": 308.6865539550781, + "margin_dpo/margin_std": 577.9491577148438, + "step": 339 + }, + { + "KL/chosen_KL_mean": -631.038330078125, + "KL/mean": -757.1285400390625, + "KL/rejected_KL_mean": -883.21875, + "KL/std": 515.5421752929688, + "epoch": 0.49926578560939794, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 252.18031311035156, + "fcm_dpo/q_t": 0.48659059405326843, + "grad_norm": 7.41240930557251, + "learning_rate": 2.959373794541426e-07, + "logits/chosen": -0.2925180494785309, + "logits/rejected": -0.2974250912666321, + "logps/chosen": -683.8685913085938, + "logps/ref_chosen": -52.83022689819336, + "logps/ref_rejected": -73.10723114013672, + "logps/rejected": -956.325927734375, + "loss": 1.3383, + "margin_dpo/margin_mean": 252.18031311035156, + "margin_dpo/margin_std": 664.0989990234375, + "step": 340 + }, + { + "KL/chosen_KL_mean": -659.0855712890625, + "KL/mean": -854.7269287109375, + "KL/rejected_KL_mean": -1050.3682861328125, + "KL/std": 603.756591796875, + "epoch": 0.5007342143906021, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 391.2827453613281, + "fcm_dpo/q_t": 0.47924739122390747, + "grad_norm": 8.219803810119629, + "learning_rate": 2.946753005532965e-07, + "logits/chosen": -0.30277353525161743, + "logits/rejected": -0.31048113107681274, + "logps/chosen": -706.9853515625, + "logps/ref_chosen": -47.899803161621094, + "logps/ref_rejected": -101.80987548828125, + "logps/rejected": -1152.17822265625, + "loss": 1.3112, + "margin_dpo/margin_mean": 391.28277587890625, + "margin_dpo/margin_std": 779.0953369140625, + "step": 341 + }, + { + "KL/chosen_KL_mean": -684.173828125, + "KL/mean": -808.3017578125, + "KL/rejected_KL_mean": -932.4296875, + "KL/std": 543.6679077148438, + "epoch": 0.5022026431718062, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 248.255859375, + "fcm_dpo/q_t": 0.4867771863937378, + "grad_norm": 7.662797451019287, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": -0.3431963324546814, + "logits/rejected": -0.34687745571136475, + "logps/chosen": -756.1705322265625, + "logps/ref_chosen": -71.99664306640625, + "logps/ref_rejected": -92.58959197998047, + "logps/rejected": -1025.019287109375, + "loss": 1.3395, + "margin_dpo/margin_mean": 248.255859375, + "margin_dpo/margin_std": 699.264892578125, + "step": 342 + }, + { + "KL/chosen_KL_mean": -627.5256958007812, + "KL/mean": -823.489013671875, + "KL/rejected_KL_mean": -1019.4523315429688, + "KL/std": 555.4841918945312, + "epoch": 0.5036710719530103, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 391.92657470703125, + "fcm_dpo/q_t": 0.4790971279144287, + "grad_norm": 7.085535526275635, + "learning_rate": 2.9214764433242476e-07, + "logits/chosen": -0.31986576318740845, + "logits/rejected": -0.33300432562828064, + "logps/chosen": -681.9312744140625, + "logps/ref_chosen": -54.405616760253906, + "logps/ref_rejected": -111.04142761230469, + "logps/rejected": -1130.4937744140625, + "loss": 1.3076, + "margin_dpo/margin_mean": 391.92657470703125, + "margin_dpo/margin_std": 557.3361206054688, + "step": 343 + }, + { + "KL/chosen_KL_mean": -689.4730224609375, + "KL/mean": -871.2525634765625, + "KL/rejected_KL_mean": -1053.0322265625, + "KL/std": 726.9136962890625, + "epoch": 0.5051395007342144, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 363.5591735839844, + "fcm_dpo/q_t": 0.48091691732406616, + "grad_norm": 9.887747764587402, + "learning_rate": 2.9088213361849126e-07, + "logits/chosen": -0.27034977078437805, + "logits/rejected": -0.2850009799003601, + "logps/chosen": -743.4376220703125, + "logps/ref_chosen": -53.96466827392578, + "logps/ref_rejected": -90.62336730957031, + "logps/rejected": -1143.655517578125, + "loss": 1.3198, + "margin_dpo/margin_mean": 363.55914306640625, + "margin_dpo/margin_std": 895.619140625, + "step": 344 + }, + { + "KL/chosen_KL_mean": -792.3978881835938, + "KL/mean": -966.5239868164062, + "KL/rejected_KL_mean": -1140.650146484375, + "KL/std": 544.9487915039062, + "epoch": 0.5066079295154186, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 348.252197265625, + "fcm_dpo/q_t": 0.48146194219589233, + "grad_norm": 7.0328803062438965, + "learning_rate": 2.896155456223163e-07, + "logits/chosen": -0.279766321182251, + "logits/rejected": -0.2895045280456543, + "logps/chosen": -854.0836181640625, + "logps/ref_chosen": -61.685699462890625, + "logps/ref_rejected": -99.49041748046875, + "logps/rejected": -1240.1405029296875, + "loss": 1.3185, + "margin_dpo/margin_mean": 348.2522277832031, + "margin_dpo/margin_std": 693.2333984375, + "step": 345 + }, + { + "KL/chosen_KL_mean": -717.4666748046875, + "KL/mean": -873.6412353515625, + "KL/rejected_KL_mean": -1029.8157958984375, + "KL/std": 560.0009155273438, + "epoch": 0.5080763582966226, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 312.34906005859375, + "fcm_dpo/q_t": 0.4833087921142578, + "grad_norm": 7.525673866271973, + "learning_rate": 2.883479137196714e-07, + "logits/chosen": -0.24226948618888855, + "logits/rejected": -0.2475300431251526, + "logps/chosen": -772.7229614257812, + "logps/ref_chosen": -55.256263732910156, + "logps/ref_rejected": -77.41532135009766, + "logps/rejected": -1107.2310791015625, + "loss": 1.3241, + "margin_dpo/margin_mean": 312.34906005859375, + "margin_dpo/margin_std": 564.9100952148438, + "step": 346 + }, + { + "KL/chosen_KL_mean": -728.3167114257812, + "KL/mean": -851.8155517578125, + "KL/rejected_KL_mean": -975.3143310546875, + "KL/std": 575.36181640625, + "epoch": 0.5095447870778267, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 246.99755859375, + "fcm_dpo/q_t": 0.4868515729904175, + "grad_norm": 8.351125717163086, + "learning_rate": 2.8707927131383614e-07, + "logits/chosen": -0.20478571951389313, + "logits/rejected": -0.21141314506530762, + "logps/chosen": -785.8829345703125, + "logps/ref_chosen": -57.56623840332031, + "logps/ref_rejected": -92.35509490966797, + "logps/rejected": -1067.66943359375, + "loss": 1.3399, + "margin_dpo/margin_mean": 246.99752807617188, + "margin_dpo/margin_std": 715.6409912109375, + "step": 347 + }, + { + "KL/chosen_KL_mean": -581.2276611328125, + "KL/mean": -719.3048095703125, + "KL/rejected_KL_mean": -857.382080078125, + "KL/std": 477.3961181640625, + "epoch": 0.5110132158590308, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 276.1544494628906, + "fcm_dpo/q_t": 0.485293447971344, + "grad_norm": 5.781659126281738, + "learning_rate": 2.858096518347179e-07, + "logits/chosen": -0.2297358363866806, + "logits/rejected": -0.2374425232410431, + "logps/chosen": -637.5453491210938, + "logps/ref_chosen": -56.31770324707031, + "logps/ref_rejected": -89.13836669921875, + "logps/rejected": -946.5204467773438, + "loss": 1.3329, + "margin_dpo/margin_mean": 276.1544494628906, + "margin_dpo/margin_std": 637.7725219726562, + "step": 348 + }, + { + "KL/chosen_KL_mean": -541.278564453125, + "KL/mean": -737.878662109375, + "KL/rejected_KL_mean": -934.478759765625, + "KL/std": 600.9308471679688, + "epoch": 0.5124816446402349, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 393.2002258300781, + "fcm_dpo/q_t": 0.47914326190948486, + "grad_norm": 16.43769645690918, + "learning_rate": 2.845390887379706e-07, + "logits/chosen": -0.16476523876190186, + "logits/rejected": -0.1687617003917694, + "logps/chosen": -599.3040771484375, + "logps/ref_chosen": -58.025516510009766, + "logps/ref_rejected": -97.50515747070312, + "logps/rejected": -1031.98388671875, + "loss": 1.3102, + "margin_dpo/margin_mean": 393.2002258300781, + "margin_dpo/margin_std": 747.94921875, + "step": 349 + }, + { + "KL/chosen_KL_mean": -617.7612915039062, + "KL/mean": -776.6915283203125, + "KL/rejected_KL_mean": -935.6217041015625, + "KL/std": 595.83447265625, + "epoch": 0.5139500734214391, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 317.8603820800781, + "fcm_dpo/q_t": 0.4831061363220215, + "grad_norm": 10.051895141601562, + "learning_rate": 2.8326761550411346e-07, + "logits/chosen": -0.21435394883155823, + "logits/rejected": -0.23840287327766418, + "logps/chosen": -682.091796875, + "logps/ref_chosen": -64.33049011230469, + "logps/ref_rejected": -89.87164306640625, + "logps/rejected": -1025.4932861328125, + "loss": 1.3262, + "margin_dpo/margin_mean": 317.8603820800781, + "margin_dpo/margin_std": 751.39208984375, + "step": 350 + }, + { + "KL/chosen_KL_mean": -567.7418823242188, + "KL/mean": -760.047119140625, + "KL/rejected_KL_mean": -952.3523559570312, + "KL/std": 605.5358276367188, + "epoch": 0.5154185022026432, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 384.61053466796875, + "fcm_dpo/q_t": 0.4796157479286194, + "grad_norm": 6.818278789520264, + "learning_rate": 2.819952656376487e-07, + "logits/chosen": -0.23846173286437988, + "logits/rejected": -0.25154590606689453, + "logps/chosen": -628.4140014648438, + "logps/ref_chosen": -60.6721305847168, + "logps/ref_rejected": -101.5654296875, + "logps/rejected": -1053.9178466796875, + "loss": 1.3132, + "margin_dpo/margin_mean": 384.61053466796875, + "margin_dpo/margin_std": 796.4315795898438, + "step": 351 + }, + { + "KL/chosen_KL_mean": -745.8148193359375, + "KL/mean": -805.4356689453125, + "KL/rejected_KL_mean": -865.0565185546875, + "KL/std": 641.70947265625, + "epoch": 0.5168869309838473, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 119.24172973632812, + "fcm_dpo/q_t": 0.49365556240081787, + "grad_norm": 10.332298278808594, + "learning_rate": 2.8072207266617854e-07, + "logits/chosen": -0.2152498960494995, + "logits/rejected": -0.20715099573135376, + "logps/chosen": -816.75830078125, + "logps/ref_chosen": -70.9434585571289, + "logps/ref_rejected": -76.6419677734375, + "logps/rejected": -941.698486328125, + "loss": 1.3666, + "margin_dpo/margin_mean": 119.2417221069336, + "margin_dpo/margin_std": 703.8898315429688, + "step": 352 + }, + { + "KL/chosen_KL_mean": -665.606689453125, + "KL/mean": -830.3818359375, + "KL/rejected_KL_mean": -995.156982421875, + "KL/std": 639.630859375, + "epoch": 0.5183553597650514, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 329.55035400390625, + "fcm_dpo/q_t": 0.4826132357120514, + "grad_norm": 10.239563941955566, + "learning_rate": 2.794480701395219e-07, + "logits/chosen": -0.24213480949401855, + "logits/rejected": -0.2454732209444046, + "logps/chosen": -724.001953125, + "logps/ref_chosen": -58.39533996582031, + "logps/ref_rejected": -80.33553314208984, + "logps/rejected": -1075.4925537109375, + "loss": 1.3252, + "margin_dpo/margin_mean": 329.55035400390625, + "margin_dpo/margin_std": 818.1527099609375, + "step": 353 + }, + { + "KL/chosen_KL_mean": -645.8775634765625, + "KL/mean": -835.401123046875, + "KL/rejected_KL_mean": -1024.924560546875, + "KL/std": 729.5835571289062, + "epoch": 0.5198237885462555, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 379.0469665527344, + "fcm_dpo/q_t": 0.48002883791923523, + "grad_norm": 9.386272430419922, + "learning_rate": 2.781732916288303e-07, + "logits/chosen": -0.27470749616622925, + "logits/rejected": -0.28340351581573486, + "logps/chosen": -705.6806030273438, + "logps/ref_chosen": -59.80299377441406, + "logps/ref_rejected": -88.75750732421875, + "logps/rejected": -1113.68212890625, + "loss": 1.3138, + "margin_dpo/margin_mean": 379.0469970703125, + "margin_dpo/margin_std": 765.1248779296875, + "step": 354 + }, + { + "KL/chosen_KL_mean": -676.1866455078125, + "KL/mean": -879.7360229492188, + "KL/rejected_KL_mean": -1083.285400390625, + "KL/std": 590.9983520507812, + "epoch": 0.5212922173274597, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 407.0987243652344, + "fcm_dpo/q_t": 0.47829318046569824, + "grad_norm": 7.246441841125488, + "learning_rate": 2.7689777072570284e-07, + "logits/chosen": -0.3153689503669739, + "logits/rejected": -0.32817888259887695, + "logps/chosen": -730.315185546875, + "logps/ref_chosen": -54.12849807739258, + "logps/ref_rejected": -82.40606689453125, + "logps/rejected": -1165.69140625, + "loss": 1.3055, + "margin_dpo/margin_mean": 407.0987243652344, + "margin_dpo/margin_std": 628.2813720703125, + "step": 355 + }, + { + "KL/chosen_KL_mean": -843.4148559570312, + "KL/mean": -896.946533203125, + "KL/rejected_KL_mean": -950.4781494140625, + "KL/std": 694.555908203125, + "epoch": 0.5227606461086637, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 107.06331634521484, + "fcm_dpo/q_t": 0.4944502115249634, + "grad_norm": 13.43985366821289, + "learning_rate": 2.7562154104130176e-07, + "logits/chosen": -0.3528267443180084, + "logits/rejected": -0.346576452255249, + "logps/chosen": -908.088623046875, + "logps/ref_chosen": -64.6738052368164, + "logps/ref_rejected": -75.89926147460938, + "logps/rejected": -1026.37744140625, + "loss": 1.3731, + "margin_dpo/margin_mean": 107.06331634521484, + "margin_dpo/margin_std": 916.18310546875, + "step": 356 + }, + { + "KL/chosen_KL_mean": -742.629638671875, + "KL/mean": -851.8154907226562, + "KL/rejected_KL_mean": -961.0013427734375, + "KL/std": 635.5723876953125, + "epoch": 0.5242290748898678, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 218.3716583251953, + "fcm_dpo/q_t": 0.48831281065940857, + "grad_norm": 11.403450965881348, + "learning_rate": 2.7434463620546594e-07, + "logits/chosen": -0.34209227561950684, + "logits/rejected": -0.3437783718109131, + "logps/chosen": -795.35546875, + "logps/ref_chosen": -52.725799560546875, + "logps/ref_rejected": -86.84115600585938, + "logps/rejected": -1047.842529296875, + "loss": 1.3475, + "margin_dpo/margin_mean": 218.3716583251953, + "margin_dpo/margin_std": 797.1085205078125, + "step": 357 + }, + { + "KL/chosen_KL_mean": -685.59619140625, + "KL/mean": -857.2205200195312, + "KL/rejected_KL_mean": -1028.844970703125, + "KL/std": 697.257568359375, + "epoch": 0.5256975036710719, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 343.24871826171875, + "fcm_dpo/q_t": 0.4819628894329071, + "grad_norm": 6.759915828704834, + "learning_rate": 2.730670898658255e-07, + "logits/chosen": -0.38823795318603516, + "logits/rejected": -0.39373278617858887, + "logps/chosen": -748.8016357421875, + "logps/ref_chosen": -63.20543670654297, + "logps/ref_rejected": -88.373291015625, + "logps/rejected": -1117.21826171875, + "loss": 1.3229, + "margin_dpo/margin_mean": 343.24871826171875, + "margin_dpo/margin_std": 876.3508911132812, + "step": 358 + }, + { + "KL/chosen_KL_mean": -853.0894775390625, + "KL/mean": -1017.8627319335938, + "KL/rejected_KL_mean": -1182.635986328125, + "KL/std": 709.51904296875, + "epoch": 0.527165932452276, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 329.54638671875, + "fcm_dpo/q_t": 0.4825381338596344, + "grad_norm": 10.495577812194824, + "learning_rate": 2.717889356869146e-07, + "logits/chosen": -0.3527218997478485, + "logits/rejected": -0.3608393669128418, + "logps/chosen": -909.459716796875, + "logps/ref_chosen": -56.370216369628906, + "logps/ref_rejected": -82.17375183105469, + "logps/rejected": -1264.8096923828125, + "loss": 1.3279, + "margin_dpo/margin_mean": 329.54638671875, + "margin_dpo/margin_std": 976.3280639648438, + "step": 359 + }, + { + "KL/chosen_KL_mean": -797.648193359375, + "KL/mean": -908.0338745117188, + "KL/rejected_KL_mean": -1018.4195556640625, + "KL/std": 799.6057739257812, + "epoch": 0.5286343612334802, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 220.7712860107422, + "fcm_dpo/q_t": 0.4882170855998993, + "grad_norm": 9.871207237243652, + "learning_rate": 2.7051020734928443e-07, + "logits/chosen": -0.30490997433662415, + "logits/rejected": -0.2990133762359619, + "logps/chosen": -849.108642578125, + "logps/ref_chosen": -51.460384368896484, + "logps/ref_rejected": -69.83892059326172, + "logps/rejected": -1088.2584228515625, + "loss": 1.3449, + "margin_dpo/margin_mean": 220.77130126953125, + "margin_dpo/margin_std": 689.1402587890625, + "step": 360 + }, + { + "KL/chosen_KL_mean": -762.4368896484375, + "KL/mean": -887.9981689453125, + "KL/rejected_KL_mean": -1013.559326171875, + "KL/std": 713.6858520507812, + "epoch": 0.5301027900146843, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 251.12237548828125, + "fcm_dpo/q_t": 0.4866461157798767, + "grad_norm": 6.593760967254639, + "learning_rate": 2.6923093854861593e-07, + "logits/chosen": -0.2922874689102173, + "logits/rejected": -0.3055328130722046, + "logps/chosen": -816.3064575195312, + "logps/ref_chosen": -53.86951446533203, + "logps/ref_rejected": -90.7692642211914, + "logps/rejected": -1104.32861328125, + "loss": 1.3401, + "margin_dpo/margin_mean": 251.12237548828125, + "margin_dpo/margin_std": 772.4447631835938, + "step": 361 + }, + { + "KL/chosen_KL_mean": -624.3651123046875, + "KL/mean": -883.3717041015625, + "KL/rejected_KL_mean": -1142.37841796875, + "KL/std": 716.97802734375, + "epoch": 0.5315712187958884, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 518.0132446289062, + "fcm_dpo/q_t": 0.47263094782829285, + "grad_norm": 13.076441764831543, + "learning_rate": 2.679511629948319e-07, + "logits/chosen": -0.30570244789123535, + "logits/rejected": -0.3104197382926941, + "logps/chosen": -683.004150390625, + "logps/ref_chosen": -58.639060974121094, + "logps/ref_rejected": -105.58195495605469, + "logps/rejected": -1247.9603271484375, + "loss": 1.2863, + "margin_dpo/margin_mean": 518.01318359375, + "margin_dpo/margin_std": 827.43408203125, + "step": 362 + }, + { + "KL/chosen_KL_mean": -607.5775146484375, + "KL/mean": -829.8275146484375, + "KL/rejected_KL_mean": -1052.0775146484375, + "KL/std": 665.65380859375, + "epoch": 0.5330396475770925, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 444.5, + "fcm_dpo/q_t": 0.4764130711555481, + "grad_norm": 6.851832389831543, + "learning_rate": 2.6667091441120816e-07, + "logits/chosen": -0.29445457458496094, + "logits/rejected": -0.31152263283729553, + "logps/chosen": -652.1359252929688, + "logps/ref_chosen": -44.558380126953125, + "logps/ref_rejected": -74.69496154785156, + "logps/rejected": -1126.7724609375, + "loss": 1.2996, + "margin_dpo/margin_mean": 444.5, + "margin_dpo/margin_std": 740.0186767578125, + "step": 363 + }, + { + "KL/chosen_KL_mean": -626.3096923828125, + "KL/mean": -794.18212890625, + "KL/rejected_KL_mean": -962.0546264648438, + "KL/std": 631.0501708984375, + "epoch": 0.5345080763582967, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 335.74493408203125, + "fcm_dpo/q_t": 0.48214179277420044, + "grad_norm": 7.443741321563721, + "learning_rate": 2.6539022653348575e-07, + "logits/chosen": -0.30771857500076294, + "logits/rejected": -0.32001155614852905, + "logps/chosen": -675.204345703125, + "logps/ref_chosen": -48.894622802734375, + "logps/ref_rejected": -91.395751953125, + "logps/rejected": -1053.4503173828125, + "loss": 1.3219, + "margin_dpo/margin_mean": 335.7449035644531, + "margin_dpo/margin_std": 725.6460571289062, + "step": 364 + }, + { + "KL/chosen_KL_mean": -686.364013671875, + "KL/mean": -917.268798828125, + "KL/rejected_KL_mean": -1148.173583984375, + "KL/std": 698.85107421875, + "epoch": 0.5359765051395007, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 461.8096923828125, + "fcm_dpo/q_t": 0.4755679666996002, + "grad_norm": 10.261777877807617, + "learning_rate": 2.641091331089811e-07, + "logits/chosen": -0.31375348567962646, + "logits/rejected": -0.3348388671875, + "logps/chosen": -737.856689453125, + "logps/ref_chosen": -51.49274444580078, + "logps/ref_rejected": -92.70166778564453, + "logps/rejected": -1240.875244140625, + "loss": 1.2973, + "margin_dpo/margin_mean": 461.8096923828125, + "margin_dpo/margin_std": 814.092041015625, + "step": 365 + }, + { + "KL/chosen_KL_mean": -645.4697265625, + "KL/mean": -870.0846557617188, + "KL/rejected_KL_mean": -1094.69970703125, + "KL/std": 782.6004028320312, + "epoch": 0.5374449339207048, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 449.22991943359375, + "fcm_dpo/q_t": 0.4762273132801056, + "grad_norm": 7.029806613922119, + "learning_rate": 2.6282766789569736e-07, + "logits/chosen": -0.32882604002952576, + "logits/rejected": -0.3432251214981079, + "logps/chosen": -690.1902465820312, + "logps/ref_chosen": -44.7205696105957, + "logps/ref_rejected": -83.31040954589844, + "logps/rejected": -1178.010009765625, + "loss": 1.3052, + "margin_dpo/margin_mean": 449.2299499511719, + "margin_dpo/margin_std": 1045.4915771484375, + "step": 366 + }, + { + "KL/chosen_KL_mean": -684.59326171875, + "KL/mean": -842.3504638671875, + "KL/rejected_KL_mean": -1000.1076049804688, + "KL/std": 633.3516845703125, + "epoch": 0.5389133627019089, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 315.514404296875, + "fcm_dpo/q_t": 0.4832141101360321, + "grad_norm": 6.7098283767700195, + "learning_rate": 2.615458646614349e-07, + "logits/chosen": -0.3017085790634155, + "logits/rejected": -0.3029378056526184, + "logps/chosen": -742.9986572265625, + "logps/ref_chosen": -58.405418395996094, + "logps/ref_rejected": -76.75132751464844, + "logps/rejected": -1076.85888671875, + "loss": 1.326, + "margin_dpo/margin_mean": 315.514404296875, + "margin_dpo/margin_std": 734.252685546875, + "step": 367 + }, + { + "KL/chosen_KL_mean": -611.4556884765625, + "KL/mean": -943.615966796875, + "KL/rejected_KL_mean": -1275.7762451171875, + "KL/std": 775.81884765625, + "epoch": 0.540381791483113, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 664.3204345703125, + "fcm_dpo/q_t": 0.46484458446502686, + "grad_norm": 25.336750030517578, + "learning_rate": 2.6026375718290083e-07, + "logits/chosen": -0.2958596348762512, + "logits/rejected": -0.3062882423400879, + "logps/chosen": -655.908203125, + "logps/ref_chosen": -44.452518463134766, + "logps/ref_rejected": -98.55526733398438, + "logps/rejected": -1374.33154296875, + "loss": 1.2562, + "margin_dpo/margin_mean": 664.3204345703125, + "margin_dpo/margin_std": 804.1547241210938, + "step": 368 + }, + { + "KL/chosen_KL_mean": -814.8972778320312, + "KL/mean": -954.0897216796875, + "KL/rejected_KL_mean": -1093.2823486328125, + "KL/std": 686.5308227539062, + "epoch": 0.5418502202643172, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 278.38507080078125, + "fcm_dpo/q_t": 0.48527395725250244, + "grad_norm": 9.702912330627441, + "learning_rate": 2.589813792448196e-07, + "logits/chosen": -0.3168843388557434, + "logits/rejected": -0.32195502519607544, + "logps/chosen": -886.27880859375, + "logps/ref_chosen": -71.38150024414062, + "logps/ref_rejected": -91.29582214355469, + "logps/rejected": -1184.5782470703125, + "loss": 1.337, + "margin_dpo/margin_mean": 278.38507080078125, + "margin_dpo/margin_std": 897.73291015625, + "step": 369 + }, + { + "KL/chosen_KL_mean": -883.7716064453125, + "KL/mean": -960.92041015625, + "KL/rejected_KL_mean": -1038.0693359375, + "KL/std": 785.7099609375, + "epoch": 0.5433186490455213, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 154.29771423339844, + "fcm_dpo/q_t": 0.491788387298584, + "grad_norm": 19.420272827148438, + "learning_rate": 2.5769876463904263e-07, + "logits/chosen": -0.3010944128036499, + "logits/rejected": -0.301439106464386, + "logps/chosen": -955.3790283203125, + "logps/ref_chosen": -71.60749816894531, + "logps/ref_rejected": -97.25978088378906, + "logps/rejected": -1135.3291015625, + "loss": 1.3638, + "margin_dpo/margin_mean": 154.2977294921875, + "margin_dpo/margin_std": 940.9684448242188, + "step": 370 + }, + { + "KL/chosen_KL_mean": -807.43408203125, + "KL/mean": -988.1475219726562, + "KL/rejected_KL_mean": -1168.861083984375, + "KL/std": 789.1993408203125, + "epoch": 0.5447870778267254, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 361.4268798828125, + "fcm_dpo/q_t": 0.48083311319351196, + "grad_norm": 7.047267913818359, + "learning_rate": 2.5641594716365744e-07, + "logits/chosen": -0.3067021667957306, + "logits/rejected": -0.3089839816093445, + "logps/chosen": -876.8486328125, + "logps/ref_chosen": -69.41448974609375, + "logps/ref_rejected": -99.17217254638672, + "logps/rejected": -1268.033203125, + "loss": 1.3188, + "margin_dpo/margin_mean": 361.4268798828125, + "margin_dpo/margin_std": 860.834228515625, + "step": 371 + }, + { + "KL/chosen_KL_mean": -753.4928588867188, + "KL/mean": -979.7862548828125, + "KL/rejected_KL_mean": -1206.079833984375, + "KL/std": 863.1746215820312, + "epoch": 0.5462555066079295, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 452.5868835449219, + "fcm_dpo/q_t": 0.47651100158691406, + "grad_norm": 18.109085083007812, + "learning_rate": 2.551329606220976e-07, + "logits/chosen": -0.28337526321411133, + "logits/rejected": -0.30206161737442017, + "logps/chosen": -815.3108520507812, + "logps/ref_chosen": -61.8179931640625, + "logps/ref_rejected": -78.53948974609375, + "logps/rejected": -1284.6192626953125, + "loss": 1.3037, + "margin_dpo/margin_mean": 452.5869140625, + "margin_dpo/margin_std": 1032.625732421875, + "step": 372 + }, + { + "KL/chosen_KL_mean": -822.2640380859375, + "KL/mean": -992.8091430664062, + "KL/rejected_KL_mean": -1163.354248046875, + "KL/std": 807.1524047851562, + "epoch": 0.5477239353891337, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 341.0901794433594, + "fcm_dpo/q_t": 0.4819341003894806, + "grad_norm": 9.496335983276367, + "learning_rate": 2.538498388222517e-07, + "logits/chosen": -0.25716885924339294, + "logits/rejected": -0.26763084530830383, + "logps/chosen": -886.481201171875, + "logps/ref_chosen": -64.21713256835938, + "logps/ref_rejected": -85.95960998535156, + "logps/rejected": -1249.31396484375, + "loss": 1.324, + "margin_dpo/margin_mean": 341.0901794433594, + "margin_dpo/margin_std": 889.7625122070312, + "step": 373 + }, + { + "KL/chosen_KL_mean": -697.943115234375, + "KL/mean": -933.2283325195312, + "KL/rejected_KL_mean": -1168.5135498046875, + "KL/std": 870.4952392578125, + "epoch": 0.5491923641703378, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 470.57049560546875, + "fcm_dpo/q_t": 0.47532325983047485, + "grad_norm": 11.988911628723145, + "learning_rate": 2.525666155755725e-07, + "logits/chosen": -0.3318430483341217, + "logits/rejected": -0.33818480372428894, + "logps/chosen": -768.59326171875, + "logps/ref_chosen": -70.65018463134766, + "logps/ref_rejected": -93.64016723632812, + "logps/rejected": -1262.1536865234375, + "loss": 1.3009, + "margin_dpo/margin_mean": 470.57049560546875, + "margin_dpo/margin_std": 1058.4764404296875, + "step": 374 + }, + { + "KL/chosen_KL_mean": -752.4849243164062, + "KL/mean": -971.480712890625, + "KL/rejected_KL_mean": -1190.4764404296875, + "KL/std": 719.1031494140625, + "epoch": 0.5506607929515418, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 437.9915771484375, + "fcm_dpo/q_t": 0.47678330540657043, + "grad_norm": 14.679875373840332, + "learning_rate": 2.512833246961859e-07, + "logits/chosen": -0.33781394362449646, + "logits/rejected": -0.35372868180274963, + "logps/chosen": -812.565185546875, + "logps/ref_chosen": -60.080223083496094, + "logps/ref_rejected": -88.93830871582031, + "logps/rejected": -1279.414794921875, + "loss": 1.3037, + "margin_dpo/margin_mean": 437.9915771484375, + "margin_dpo/margin_std": 881.63671875, + "step": 375 + }, + { + "KL/chosen_KL_mean": -957.6834716796875, + "KL/mean": -1178.202880859375, + "KL/rejected_KL_mean": -1398.72216796875, + "KL/std": 931.96435546875, + "epoch": 0.5521292217327459, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 441.0386962890625, + "fcm_dpo/q_t": 0.4765964150428772, + "grad_norm": 15.095070838928223, + "learning_rate": 2.5e-07, + "logits/chosen": -0.3589029312133789, + "logits/rejected": -0.3605373501777649, + "logps/chosen": -1020.34375, + "logps/ref_chosen": -62.660308837890625, + "logps/ref_rejected": -105.52660369873047, + "logps/rejected": -1504.248779296875, + "loss": 1.3074, + "margin_dpo/margin_mean": 441.0386962890625, + "margin_dpo/margin_std": 1077.4156494140625, + "step": 376 + }, + { + "KL/chosen_KL_mean": -1065.7764892578125, + "KL/mean": -1337.8623046875, + "KL/rejected_KL_mean": -1609.947998046875, + "KL/std": 1192.957763671875, + "epoch": 0.55359765051395, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 544.1714477539062, + "fcm_dpo/q_t": 0.47193536162376404, + "grad_norm": 10.05336856842041, + "learning_rate": 2.487166753038141e-07, + "logits/chosen": -0.3872339129447937, + "logits/rejected": -0.40310975909233093, + "logps/chosen": -1120.2552490234375, + "logps/ref_chosen": -54.478736877441406, + "logps/ref_rejected": -98.70335388183594, + "logps/rejected": -1708.6513671875, + "loss": 1.2974, + "margin_dpo/margin_mean": 544.1714477539062, + "margin_dpo/margin_std": 1465.950439453125, + "step": 377 + }, + { + "KL/chosen_KL_mean": -899.20458984375, + "KL/mean": -1161.714111328125, + "KL/rejected_KL_mean": -1424.2236328125, + "KL/std": 916.3400268554688, + "epoch": 0.5550660792951542, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 525.0191650390625, + "fcm_dpo/q_t": 0.4725556969642639, + "grad_norm": 14.26543140411377, + "learning_rate": 2.4743338442442754e-07, + "logits/chosen": -0.42005640268325806, + "logits/rejected": -0.44007664918899536, + "logps/chosen": -944.2251586914062, + "logps/ref_chosen": -45.02053451538086, + "logps/ref_rejected": -88.0469741821289, + "logps/rejected": -1512.270751953125, + "loss": 1.2936, + "margin_dpo/margin_mean": 525.0191650390625, + "margin_dpo/margin_std": 1212.896728515625, + "step": 378 + }, + { + "KL/chosen_KL_mean": -1210.5340576171875, + "KL/mean": -1521.940673828125, + "KL/rejected_KL_mean": -1833.34716796875, + "KL/std": 1398.197509765625, + "epoch": 0.5565345080763583, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 622.81298828125, + "fcm_dpo/q_t": 0.4689289629459381, + "grad_norm": 16.45003318786621, + "learning_rate": 2.461501611777483e-07, + "logits/chosen": -0.41934919357299805, + "logits/rejected": -0.44911909103393555, + "logps/chosen": -1263.7161865234375, + "logps/ref_chosen": -53.182098388671875, + "logps/ref_rejected": -114.3001708984375, + "logps/rejected": -1947.6473388671875, + "loss": 1.2913, + "margin_dpo/margin_mean": 622.81298828125, + "margin_dpo/margin_std": 1755.5107421875, + "step": 379 + }, + { + "KL/chosen_KL_mean": -1069.46337890625, + "KL/mean": -1406.3856201171875, + "KL/rejected_KL_mean": -1743.3077392578125, + "KL/std": 1310.0059814453125, + "epoch": 0.5580029368575624, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 673.8443603515625, + "fcm_dpo/q_t": 0.4651104509830475, + "grad_norm": 16.879432678222656, + "learning_rate": 2.4486703937790243e-07, + "logits/chosen": -0.43862009048461914, + "logits/rejected": -0.4662778377532959, + "logps/chosen": -1120.8165283203125, + "logps/ref_chosen": -51.3530387878418, + "logps/ref_rejected": -104.19169616699219, + "logps/rejected": -1847.49951171875, + "loss": 1.2767, + "margin_dpo/margin_mean": 673.8442993164062, + "margin_dpo/margin_std": 1641.3406982421875, + "step": 380 + }, + { + "KL/chosen_KL_mean": -1037.765380859375, + "KL/mean": -1180.1309814453125, + "KL/rejected_KL_mean": -1322.49658203125, + "KL/std": 1107.69970703125, + "epoch": 0.5594713656387665, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 284.7312316894531, + "fcm_dpo/q_t": 0.48559147119522095, + "grad_norm": 25.2132625579834, + "learning_rate": 2.435840528363426e-07, + "logits/chosen": -0.4325522482395172, + "logits/rejected": -0.4366803765296936, + "logps/chosen": -1095.568359375, + "logps/ref_chosen": -57.80306625366211, + "logps/ref_rejected": -79.21940612792969, + "logps/rejected": -1401.716064453125, + "loss": 1.3524, + "margin_dpo/margin_mean": 284.73126220703125, + "margin_dpo/margin_std": 1517.94580078125, + "step": 381 + }, + { + "KL/chosen_KL_mean": -853.733642578125, + "KL/mean": -1091.995361328125, + "KL/rejected_KL_mean": -1330.25732421875, + "KL/std": 791.60595703125, + "epoch": 0.5609397944199707, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 476.52362060546875, + "fcm_dpo/q_t": 0.47479626536369324, + "grad_norm": 9.29355525970459, + "learning_rate": 2.4230123536095745e-07, + "logits/chosen": -0.4174577593803406, + "logits/rejected": -0.42780351638793945, + "logps/chosen": -919.75390625, + "logps/ref_chosen": -66.02030181884766, + "logps/ref_rejected": -110.71016693115234, + "logps/rejected": -1440.9674072265625, + "loss": 1.2955, + "margin_dpo/margin_mean": 476.52362060546875, + "margin_dpo/margin_std": 863.3676147460938, + "step": 382 + }, + { + "KL/chosen_KL_mean": -779.2030029296875, + "KL/mean": -1037.3272705078125, + "KL/rejected_KL_mean": -1295.451416015625, + "KL/std": 926.0235595703125, + "epoch": 0.5624082232011748, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 516.2486572265625, + "fcm_dpo/q_t": 0.47278502583503723, + "grad_norm": 11.367318153381348, + "learning_rate": 2.4101862075518037e-07, + "logits/chosen": -0.39227527379989624, + "logits/rejected": -0.41163206100463867, + "logps/chosen": -829.594482421875, + "logps/ref_chosen": -50.39148712158203, + "logps/ref_rejected": -93.71589660644531, + "logps/rejected": -1389.16748046875, + "loss": 1.2977, + "margin_dpo/margin_mean": 516.2485961914062, + "margin_dpo/margin_std": 1285.308837890625, + "step": 383 + }, + { + "KL/chosen_KL_mean": -682.0059814453125, + "KL/mean": -844.9951171875, + "KL/rejected_KL_mean": -1007.984375, + "KL/std": 720.017578125, + "epoch": 0.5638766519823789, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 325.9783935546875, + "fcm_dpo/q_t": 0.48265010118484497, + "grad_norm": 9.724451065063477, + "learning_rate": 2.397362428170992e-07, + "logits/chosen": -0.4297202527523041, + "logits/rejected": -0.4280009865760803, + "logps/chosen": -734.0520629882812, + "logps/ref_chosen": -52.046104431152344, + "logps/ref_rejected": -85.76089477539062, + "logps/rejected": -1093.7452392578125, + "loss": 1.3234, + "margin_dpo/margin_mean": 325.9783630371094, + "margin_dpo/margin_std": 708.7644653320312, + "step": 384 + }, + { + "KL/chosen_KL_mean": -567.719970703125, + "KL/mean": -734.8634033203125, + "KL/rejected_KL_mean": -902.0068359375, + "KL/std": 536.7576904296875, + "epoch": 0.5653450807635829, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 334.28692626953125, + "fcm_dpo/q_t": 0.4821450114250183, + "grad_norm": 15.683626174926758, + "learning_rate": 2.3845413533856514e-07, + "logits/chosen": -0.36829620599746704, + "logits/rejected": -0.37683045864105225, + "logps/chosen": -633.2720947265625, + "logps/ref_chosen": -65.55215454101562, + "logps/ref_rejected": -77.82792663574219, + "logps/rejected": -979.834716796875, + "loss": 1.3211, + "margin_dpo/margin_mean": 334.28692626953125, + "margin_dpo/margin_std": 671.5125732421875, + "step": 385 + }, + { + "KL/chosen_KL_mean": -539.3319091796875, + "KL/mean": -767.2230224609375, + "KL/rejected_KL_mean": -995.1141357421875, + "KL/std": 658.7130126953125, + "epoch": 0.566813509544787, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 455.7822265625, + "fcm_dpo/q_t": 0.4757852554321289, + "grad_norm": 15.568609237670898, + "learning_rate": 2.3717233210430254e-07, + "logits/chosen": -0.35743653774261475, + "logits/rejected": -0.36365118622779846, + "logps/chosen": -597.5537109375, + "logps/ref_chosen": -58.22185516357422, + "logps/ref_rejected": -92.32742309570312, + "logps/rejected": -1087.441650390625, + "loss": 1.2979, + "margin_dpo/margin_mean": 455.7822570800781, + "margin_dpo/margin_std": 782.3114013671875, + "step": 386 + }, + { + "KL/chosen_KL_mean": -720.632080078125, + "KL/mean": -928.3619384765625, + "KL/rejected_KL_mean": -1136.091796875, + "KL/std": 814.0172119140625, + "epoch": 0.5682819383259912, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 415.4595947265625, + "fcm_dpo/q_t": 0.4782324433326721, + "grad_norm": 9.539994239807129, + "learning_rate": 2.3589086689101889e-07, + "logits/chosen": -0.4057348966598511, + "logits/rejected": -0.41669708490371704, + "logps/chosen": -787.051513671875, + "logps/ref_chosen": -66.41944885253906, + "logps/ref_rejected": -92.16915893554688, + "logps/rejected": -1228.2608642578125, + "loss": 1.3127, + "margin_dpo/margin_mean": 415.45965576171875, + "margin_dpo/margin_std": 1032.929443359375, + "step": 387 + }, + { + "KL/chosen_KL_mean": -648.9365234375, + "KL/mean": -877.4620971679688, + "KL/rejected_KL_mean": -1105.9876708984375, + "KL/std": 719.4478759765625, + "epoch": 0.5697503671071953, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 457.05108642578125, + "fcm_dpo/q_t": 0.47609227895736694, + "grad_norm": 7.468353271484375, + "learning_rate": 2.3460977346651428e-07, + "logits/chosen": -0.3469131588935852, + "logits/rejected": -0.3659330904483795, + "logps/chosen": -699.0660400390625, + "logps/ref_chosen": -50.129459381103516, + "logps/ref_rejected": -104.43305969238281, + "logps/rejected": -1210.420654296875, + "loss": 1.3012, + "margin_dpo/margin_mean": 457.05108642578125, + "margin_dpo/margin_std": 926.29736328125, + "step": 388 + }, + { + "KL/chosen_KL_mean": -743.910888671875, + "KL/mean": -964.4112548828125, + "KL/rejected_KL_mean": -1184.91162109375, + "KL/std": 831.6654052734375, + "epoch": 0.5712187958883994, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 441.00067138671875, + "fcm_dpo/q_t": 0.4765745997428894, + "grad_norm": 15.217109680175781, + "learning_rate": 2.3332908558879177e-07, + "logits/chosen": -0.394728422164917, + "logits/rejected": -0.4151400327682495, + "logps/chosen": -801.8175048828125, + "logps/ref_chosen": -57.906593322753906, + "logps/ref_rejected": -77.91454315185547, + "logps/rejected": -1262.826171875, + "loss": 1.3065, + "margin_dpo/margin_mean": 441.00067138671875, + "margin_dpo/margin_std": 1042.0787353515625, + "step": 389 + }, + { + "KL/chosen_KL_mean": -730.2432861328125, + "KL/mean": -956.1160888671875, + "KL/rejected_KL_mean": -1181.9888916015625, + "KL/std": 893.8538208007812, + "epoch": 0.5726872246696035, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 451.74566650390625, + "fcm_dpo/q_t": 0.4772656559944153, + "grad_norm": 14.349326133728027, + "learning_rate": 2.320488370051681e-07, + "logits/chosen": -0.36197686195373535, + "logits/rejected": -0.3790530562400818, + "logps/chosen": -779.4691772460938, + "logps/ref_chosen": -49.22591781616211, + "logps/ref_rejected": -85.5281982421875, + "logps/rejected": -1267.51708984375, + "loss": 1.3101, + "margin_dpo/margin_mean": 451.7456970214844, + "margin_dpo/margin_std": 1213.085693359375, + "step": 390 + }, + { + "KL/chosen_KL_mean": -934.3198852539062, + "KL/mean": -995.6199951171875, + "KL/rejected_KL_mean": -1056.920166015625, + "KL/std": 943.6309814453125, + "epoch": 0.5741556534508077, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 122.60029602050781, + "fcm_dpo/q_t": 0.49323803186416626, + "grad_norm": 19.008642196655273, + "learning_rate": 2.3076906145138405e-07, + "logits/chosen": -0.35622894763946533, + "logits/rejected": -0.3512190580368042, + "logps/chosen": -998.6495361328125, + "logps/ref_chosen": -64.32965087890625, + "logps/ref_rejected": -86.73820495605469, + "logps/rejected": -1143.6583251953125, + "loss": 1.3729, + "margin_dpo/margin_mean": 122.60028076171875, + "margin_dpo/margin_std": 1043.1915283203125, + "step": 391 + }, + { + "KL/chosen_KL_mean": -610.44775390625, + "KL/mean": -939.5457763671875, + "KL/rejected_KL_mean": -1268.643798828125, + "KL/std": 832.3636474609375, + "epoch": 0.5756240822320118, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 658.1959838867188, + "fcm_dpo/q_t": 0.4652509093284607, + "grad_norm": 24.722393035888672, + "learning_rate": 2.294897926507156e-07, + "logits/chosen": -0.356148362159729, + "logits/rejected": -0.3655499219894409, + "logps/chosen": -663.9517822265625, + "logps/ref_chosen": -53.50397872924805, + "logps/ref_rejected": -102.34584045410156, + "logps/rejected": -1370.9896240234375, + "loss": 1.2592, + "margin_dpo/margin_mean": 658.1959228515625, + "margin_dpo/margin_std": 873.6790771484375, + "step": 392 + }, + { + "KL/chosen_KL_mean": -772.148681640625, + "KL/mean": -1021.87158203125, + "KL/rejected_KL_mean": -1271.5946044921875, + "KL/std": 1009.1136474609375, + "epoch": 0.5770925110132159, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 499.4459228515625, + "fcm_dpo/q_t": 0.4741513729095459, + "grad_norm": 10.550104141235352, + "learning_rate": 2.2821106431308543e-07, + "logits/chosen": -0.36827749013900757, + "logits/rejected": -0.38175201416015625, + "logps/chosen": -818.62255859375, + "logps/ref_chosen": -46.473915100097656, + "logps/ref_rejected": -71.96885681152344, + "logps/rejected": -1343.5634765625, + "loss": 1.3002, + "margin_dpo/margin_mean": 499.4459228515625, + "margin_dpo/margin_std": 1268.5267333984375, + "step": 393 + }, + { + "KL/chosen_KL_mean": -866.2552490234375, + "KL/mean": -1105.1195068359375, + "KL/rejected_KL_mean": -1343.98388671875, + "KL/std": 1064.1612548828125, + "epoch": 0.57856093979442, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 477.7286376953125, + "fcm_dpo/q_t": 0.4750671088695526, + "grad_norm": 10.232756614685059, + "learning_rate": 2.2693291013417452e-07, + "logits/chosen": -0.39185428619384766, + "logits/rejected": -0.4062337279319763, + "logps/chosen": -919.1668090820312, + "logps/ref_chosen": -52.91154861450195, + "logps/ref_rejected": -90.8226318359375, + "logps/rejected": -1434.8065185546875, + "loss": 1.2995, + "margin_dpo/margin_mean": 477.7286376953125, + "margin_dpo/margin_std": 1065.57275390625, + "step": 394 + }, + { + "KL/chosen_KL_mean": -986.03759765625, + "KL/mean": -1240.0714111328125, + "KL/rejected_KL_mean": -1494.10498046875, + "KL/std": 1024.42822265625, + "epoch": 0.580029368575624, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 508.0675048828125, + "fcm_dpo/q_t": 0.473642498254776, + "grad_norm": 12.610128402709961, + "learning_rate": 2.2565536379453404e-07, + "logits/chosen": -0.4311944246292114, + "logits/rejected": -0.44871261715888977, + "logps/chosen": -1048.583740234375, + "logps/ref_chosen": -62.546112060546875, + "logps/ref_rejected": -83.78262329101562, + "logps/rejected": -1577.8876953125, + "loss": 1.3004, + "margin_dpo/margin_mean": 508.0675048828125, + "margin_dpo/margin_std": 1298.3829345703125, + "step": 395 + }, + { + "KL/chosen_KL_mean": -1033.08447265625, + "KL/mean": -1220.91015625, + "KL/rejected_KL_mean": -1408.7357177734375, + "KL/std": 1084.0780029296875, + "epoch": 0.5814977973568282, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 375.6512451171875, + "fcm_dpo/q_t": 0.4801104664802551, + "grad_norm": 10.486356735229492, + "learning_rate": 2.2437845895869825e-07, + "logits/chosen": -0.43420612812042236, + "logits/rejected": -0.4390130639076233, + "logps/chosen": -1102.0804443359375, + "logps/ref_chosen": -68.99594116210938, + "logps/ref_rejected": -88.64665985107422, + "logps/rejected": -1497.38232421875, + "loss": 1.3169, + "margin_dpo/margin_mean": 375.6512451171875, + "margin_dpo/margin_std": 911.8556518554688, + "step": 396 + }, + { + "KL/chosen_KL_mean": -1034.108154296875, + "KL/mean": -1459.783447265625, + "KL/rejected_KL_mean": -1885.45849609375, + "KL/std": 1350.3555908203125, + "epoch": 0.5829662261380323, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 851.3504028320312, + "fcm_dpo/q_t": 0.45677071809768677, + "grad_norm": 15.112154006958008, + "learning_rate": 2.2310222927429716e-07, + "logits/chosen": -0.4225345849990845, + "logits/rejected": -0.46220195293426514, + "logps/chosen": -1095.3853759765625, + "logps/ref_chosen": -61.27716827392578, + "logps/ref_rejected": -103.11612701416016, + "logps/rejected": -1988.57470703125, + "loss": 1.2402, + "margin_dpo/margin_mean": 851.3504028320312, + "margin_dpo/margin_std": 1585.188720703125, + "step": 397 + }, + { + "KL/chosen_KL_mean": -1233.375244140625, + "KL/mean": -1574.85107421875, + "KL/rejected_KL_mean": -1916.3270263671875, + "KL/std": 1434.3673095703125, + "epoch": 0.5844346549192364, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 682.951904296875, + "fcm_dpo/q_t": 0.46584078669548035, + "grad_norm": 23.73081398010254, + "learning_rate": 2.2182670837116972e-07, + "logits/chosen": -0.4878053069114685, + "logits/rejected": -0.5119198560714722, + "logps/chosen": -1301.5267333984375, + "logps/ref_chosen": -68.15155029296875, + "logps/ref_rejected": -108.52360534667969, + "logps/rejected": -2024.8505859375, + "loss": 1.2838, + "margin_dpo/margin_mean": 682.951904296875, + "margin_dpo/margin_std": 1882.985595703125, + "step": 398 + }, + { + "KL/chosen_KL_mean": -991.1121826171875, + "KL/mean": -1204.9122314453125, + "KL/rejected_KL_mean": -1418.7122802734375, + "KL/std": 1063.78857421875, + "epoch": 0.5859030837004405, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 427.60003662109375, + "fcm_dpo/q_t": 0.4776034951210022, + "grad_norm": 33.797943115234375, + "learning_rate": 2.2055192986047804e-07, + "logits/chosen": -0.4256382882595062, + "logits/rejected": -0.4321112334728241, + "logps/chosen": -1052.001953125, + "logps/ref_chosen": -60.889801025390625, + "logps/ref_rejected": -77.965576171875, + "logps/rejected": -1496.6778564453125, + "loss": 1.3194, + "margin_dpo/margin_mean": 427.60003662109375, + "margin_dpo/margin_std": 1410.41748046875, + "step": 399 + }, + { + "KL/chosen_KL_mean": -877.5531005859375, + "KL/mean": -1235.20166015625, + "KL/rejected_KL_mean": -1592.8502197265625, + "KL/std": 992.3692016601562, + "epoch": 0.5873715124816447, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 715.2971801757812, + "fcm_dpo/q_t": 0.46237778663635254, + "grad_norm": 10.352943420410156, + "learning_rate": 2.192779273338215e-07, + "logits/chosen": -0.426511287689209, + "logits/rejected": -0.44806522130966187, + "logps/chosen": -941.1966552734375, + "logps/ref_chosen": -63.64359664916992, + "logps/ref_rejected": -105.252685546875, + "logps/rejected": -1698.1029052734375, + "loss": 1.2569, + "margin_dpo/margin_mean": 715.2971801757812, + "margin_dpo/margin_std": 1255.512451171875, + "step": 400 + }, + { + "KL/chosen_KL_mean": -964.7339477539062, + "KL/mean": -1108.6376953125, + "KL/rejected_KL_mean": -1252.54150390625, + "KL/std": 1027.1666259765625, + "epoch": 0.5888399412628488, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 287.8074951171875, + "fcm_dpo/q_t": 0.48464304208755493, + "grad_norm": 16.782976150512695, + "learning_rate": 2.1800473436235136e-07, + "logits/chosen": -0.3969235122203827, + "logits/rejected": -0.39872997999191284, + "logps/chosen": -1021.89697265625, + "logps/ref_chosen": -57.16303253173828, + "logps/ref_rejected": -83.79249572753906, + "logps/rejected": -1336.333984375, + "loss": 1.3476, + "margin_dpo/margin_mean": 287.8074951171875, + "margin_dpo/margin_std": 1398.988037109375, + "step": 401 + }, + { + "KL/chosen_KL_mean": -710.2174072265625, + "KL/mean": -1041.47021484375, + "KL/rejected_KL_mean": -1372.72314453125, + "KL/std": 974.0499877929688, + "epoch": 0.5903083700440529, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 662.5057983398438, + "fcm_dpo/q_t": 0.4654858708381653, + "grad_norm": 8.760465621948242, + "learning_rate": 2.1673238449588665e-07, + "logits/chosen": -0.4055900573730469, + "logits/rejected": -0.4254668951034546, + "logps/chosen": -760.957763671875, + "logps/ref_chosen": -50.74037170410156, + "logps/ref_rejected": -81.0460433959961, + "logps/rejected": -1453.7691650390625, + "loss": 1.264, + "margin_dpo/margin_mean": 662.5057373046875, + "margin_dpo/margin_std": 1141.46630859375, + "step": 402 + }, + { + "KL/chosen_KL_mean": -607.0709228515625, + "KL/mean": -857.822021484375, + "KL/rejected_KL_mean": -1108.5732421875, + "KL/std": 957.948486328125, + "epoch": 0.591776798825257, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 501.5023498535156, + "fcm_dpo/q_t": 0.47387418150901794, + "grad_norm": 21.134559631347656, + "learning_rate": 2.154609112620295e-07, + "logits/chosen": -0.3894062042236328, + "logits/rejected": -0.41035395860671997, + "logps/chosen": -654.2182006835938, + "logps/ref_chosen": -47.14731216430664, + "logps/ref_rejected": -77.2666015625, + "logps/rejected": -1185.83984375, + "loss": 1.2943, + "margin_dpo/margin_mean": 501.5023193359375, + "margin_dpo/margin_std": 1025.334228515625, + "step": 403 + }, + { + "KL/chosen_KL_mean": -573.6932373046875, + "KL/mean": -833.4141845703125, + "KL/rejected_KL_mean": -1093.13525390625, + "KL/std": 773.28173828125, + "epoch": 0.593245227606461, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 519.4420776367188, + "fcm_dpo/q_t": 0.47276201844215393, + "grad_norm": 26.00446128845215, + "learning_rate": 2.1419034816528218e-07, + "logits/chosen": -0.3456383943557739, + "logits/rejected": -0.35928741097450256, + "logps/chosen": -621.5684814453125, + "logps/ref_chosen": -47.875274658203125, + "logps/ref_rejected": -77.15499877929688, + "logps/rejected": -1170.290283203125, + "loss": 1.2903, + "margin_dpo/margin_mean": 519.4420166015625, + "margin_dpo/margin_std": 1035.61083984375, + "step": 404 + }, + { + "KL/chosen_KL_mean": -729.4765625, + "KL/mean": -907.5888671875, + "KL/rejected_KL_mean": -1085.7010498046875, + "KL/std": 880.6702880859375, + "epoch": 0.5947136563876652, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 356.22454833984375, + "fcm_dpo/q_t": 0.48145124316215515, + "grad_norm": 15.456494331359863, + "learning_rate": 2.129207286861638e-07, + "logits/chosen": -0.37179362773895264, + "logits/rejected": -0.37930575013160706, + "logps/chosen": -794.6394653320312, + "logps/ref_chosen": -65.16290283203125, + "logps/ref_rejected": -87.18678283691406, + "logps/rejected": -1172.887939453125, + "loss": 1.3278, + "margin_dpo/margin_mean": 356.22454833984375, + "margin_dpo/margin_std": 1170.162841796875, + "step": 405 + }, + { + "KL/chosen_KL_mean": -589.8623046875, + "KL/mean": -890.648681640625, + "KL/rejected_KL_mean": -1191.43505859375, + "KL/std": 858.3298950195312, + "epoch": 0.5961820851688693, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 601.5728149414062, + "fcm_dpo/q_t": 0.4688172936439514, + "grad_norm": 22.13282585144043, + "learning_rate": 2.1165208628032861e-07, + "logits/chosen": -0.38036268949508667, + "logits/rejected": -0.4102054834365845, + "logps/chosen": -639.6031494140625, + "logps/ref_chosen": -49.740814208984375, + "logps/ref_rejected": -92.07862854003906, + "logps/rejected": -1283.513671875, + "loss": 1.2762, + "margin_dpo/margin_mean": 601.5728759765625, + "margin_dpo/margin_std": 1076.5947265625, + "step": 406 + }, + { + "KL/chosen_KL_mean": -646.1621704101562, + "KL/mean": -756.427978515625, + "KL/rejected_KL_mean": -866.6937255859375, + "KL/std": 696.524169921875, + "epoch": 0.5976505139500734, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 220.5315399169922, + "fcm_dpo/q_t": 0.48832640051841736, + "grad_norm": 9.917305946350098, + "learning_rate": 2.1038445437768375e-07, + "logits/chosen": -0.3698161244392395, + "logits/rejected": -0.3660295903682709, + "logps/chosen": -702.4928588867188, + "logps/ref_chosen": -56.33069610595703, + "logps/ref_rejected": -77.51209259033203, + "logps/rejected": -944.205810546875, + "loss": 1.3508, + "margin_dpo/margin_mean": 220.5315399169922, + "margin_dpo/margin_std": 968.973876953125, + "step": 407 + }, + { + "KL/chosen_KL_mean": -636.0607299804688, + "KL/mean": -787.4437866210938, + "KL/rejected_KL_mean": -938.826904296875, + "KL/std": 689.7813720703125, + "epoch": 0.5991189427312775, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 302.76611328125, + "fcm_dpo/q_t": 0.4842342138290405, + "grad_norm": 18.907068252563477, + "learning_rate": 2.0911786638150872e-07, + "logits/chosen": -0.36819028854370117, + "logits/rejected": -0.37358710169792175, + "logps/chosen": -705.8500366210938, + "logps/ref_chosen": -69.789306640625, + "logps/ref_rejected": -90.09693908691406, + "logps/rejected": -1028.923828125, + "loss": 1.3326, + "margin_dpo/margin_mean": 302.7661437988281, + "margin_dpo/margin_std": 921.2728881835938, + "step": 408 + }, + { + "KL/chosen_KL_mean": -648.5272216796875, + "KL/mean": -830.0584716796875, + "KL/rejected_KL_mean": -1011.5897216796875, + "KL/std": 854.2369384765625, + "epoch": 0.6005873715124816, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 363.0625305175781, + "fcm_dpo/q_t": 0.4812033176422119, + "grad_norm": 13.033660888671875, + "learning_rate": 2.0785235566757517e-07, + "logits/chosen": -0.34932005405426025, + "logits/rejected": -0.3593984544277191, + "logps/chosen": -715.8446044921875, + "logps/ref_chosen": -67.31744384765625, + "logps/ref_rejected": -84.904296875, + "logps/rejected": -1096.4940185546875, + "loss": 1.3264, + "margin_dpo/margin_mean": 363.0625305175781, + "margin_dpo/margin_std": 1165.644775390625, + "step": 409 + }, + { + "KL/chosen_KL_mean": -634.2596435546875, + "KL/mean": -884.68310546875, + "KL/rejected_KL_mean": -1135.1065673828125, + "KL/std": 723.4794311523438, + "epoch": 0.6020558002936858, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 500.8468322753906, + "fcm_dpo/q_t": 0.4734492003917694, + "grad_norm": 18.643348693847656, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": -0.406690776348114, + "logits/rejected": -0.41948169469833374, + "logps/chosen": -685.7250366210938, + "logps/ref_chosen": -51.465354919433594, + "logps/ref_rejected": -83.198974609375, + "logps/rejected": -1218.3055419921875, + "loss": 1.2898, + "margin_dpo/margin_mean": 500.84686279296875, + "margin_dpo/margin_std": 840.258544921875, + "step": 410 + }, + { + "KL/chosen_KL_mean": -756.8233642578125, + "KL/mean": -1003.681640625, + "KL/rejected_KL_mean": -1250.539794921875, + "KL/std": 1031.4246826171875, + "epoch": 0.6035242290748899, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 493.71636962890625, + "fcm_dpo/q_t": 0.4749707281589508, + "grad_norm": 13.850335121154785, + "learning_rate": 2.0532469944670343e-07, + "logits/chosen": -0.41555359959602356, + "logits/rejected": -0.43157142400741577, + "logps/chosen": -809.1306762695312, + "logps/ref_chosen": -52.30727005004883, + "logps/ref_rejected": -80.69495391845703, + "logps/rejected": -1331.2347412109375, + "loss": 1.3047, + "margin_dpo/margin_mean": 493.71636962890625, + "margin_dpo/margin_std": 1287.54248046875, + "step": 411 + }, + { + "KL/chosen_KL_mean": -895.0568237304688, + "KL/mean": -1222.3822021484375, + "KL/rejected_KL_mean": -1549.7076416015625, + "KL/std": 1135.393310546875, + "epoch": 0.604992657856094, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 654.65087890625, + "fcm_dpo/q_t": 0.466141939163208, + "grad_norm": 13.520402908325195, + "learning_rate": 2.0406262054585738e-07, + "logits/chosen": -0.4506192207336426, + "logits/rejected": -0.4855607748031616, + "logps/chosen": -948.200927734375, + "logps/ref_chosen": -53.144126892089844, + "logps/ref_rejected": -100.0608139038086, + "logps/rejected": -1649.7684326171875, + "loss": 1.2784, + "margin_dpo/margin_mean": 654.65087890625, + "margin_dpo/margin_std": 1556.40234375, + "step": 412 + }, + { + "KL/chosen_KL_mean": -1110.75390625, + "KL/mean": -1391.29638671875, + "KL/rejected_KL_mean": -1671.839111328125, + "KL/std": 1254.1436767578125, + "epoch": 0.6064610866372981, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 561.085205078125, + "fcm_dpo/q_t": 0.47032126784324646, + "grad_norm": 16.13272476196289, + "learning_rate": 2.0280175213768205e-07, + "logits/chosen": -0.4394839406013489, + "logits/rejected": -0.455962598323822, + "logps/chosen": -1172.3358154296875, + "logps/ref_chosen": -61.58196258544922, + "logps/ref_rejected": -99.47340393066406, + "logps/rejected": -1771.3125, + "loss": 1.2944, + "margin_dpo/margin_mean": 561.0852661132812, + "margin_dpo/margin_std": 1475.930908203125, + "step": 413 + }, + { + "KL/chosen_KL_mean": -960.60693359375, + "KL/mean": -1207.114013671875, + "KL/rejected_KL_mean": -1453.6209716796875, + "KL/std": 1134.6884765625, + "epoch": 0.6079295154185022, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 493.0139465332031, + "fcm_dpo/q_t": 0.47513312101364136, + "grad_norm": 23.284347534179688, + "learning_rate": 2.0154212744723247e-07, + "logits/chosen": -0.43949374556541443, + "logits/rejected": -0.45314937829971313, + "logps/chosen": -1007.2384643554688, + "logps/ref_chosen": -46.63148498535156, + "logps/ref_rejected": -87.64653015136719, + "logps/rejected": -1541.2674560546875, + "loss": 1.3087, + "margin_dpo/margin_mean": 493.013916015625, + "margin_dpo/margin_std": 1478.425048828125, + "step": 414 + }, + { + "KL/chosen_KL_mean": -1096.2464599609375, + "KL/mean": -1322.900634765625, + "KL/rejected_KL_mean": -1549.5546875, + "KL/std": 1193.9111328125, + "epoch": 0.6093979441997063, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 453.30828857421875, + "fcm_dpo/q_t": 0.476369708776474, + "grad_norm": 14.205986022949219, + "learning_rate": 2.002837796667909e-07, + "logits/chosen": -0.5072834491729736, + "logits/rejected": -0.5145879983901978, + "logps/chosen": -1174.86474609375, + "logps/ref_chosen": -78.6182861328125, + "logps/ref_rejected": -100.47752380371094, + "logps/rejected": -1650.0322265625, + "loss": 1.3179, + "margin_dpo/margin_mean": 453.3082580566406, + "margin_dpo/margin_std": 1521.6240234375, + "step": 415 + }, + { + "KL/chosen_KL_mean": -895.443115234375, + "KL/mean": -1211.381103515625, + "KL/rejected_KL_mean": -1527.31884765625, + "KL/std": 1104.52685546875, + "epoch": 0.6108663729809104, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 631.8758544921875, + "fcm_dpo/q_t": 0.4671364426612854, + "grad_norm": 18.065832138061523, + "learning_rate": 1.990267419549914e-07, + "logits/chosen": -0.42411717772483826, + "logits/rejected": -0.4482443332672119, + "logps/chosen": -953.7222900390625, + "logps/ref_chosen": -58.27912521362305, + "logps/ref_rejected": -90.56871795654297, + "logps/rejected": -1617.8875732421875, + "loss": 1.2725, + "margin_dpo/margin_mean": 631.8757934570312, + "margin_dpo/margin_std": 1231.4012451171875, + "step": 416 + }, + { + "KL/chosen_KL_mean": -706.0615234375, + "KL/mean": -973.61181640625, + "KL/rejected_KL_mean": -1241.162109375, + "KL/std": 916.9801025390625, + "epoch": 0.6123348017621145, + "fcm_dpo/beta": 0.00021450650820042938, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 535.1005859375, + "fcm_dpo/q_t": 0.4720717668533325, + "grad_norm": 10.751476287841797, + "learning_rate": 1.9777104743594686e-07, + "logits/chosen": -0.4009503126144409, + "logits/rejected": -0.4133146107196808, + "logps/chosen": -756.26025390625, + "logps/ref_chosen": -50.1987190246582, + "logps/ref_rejected": -68.15184020996094, + "logps/rejected": -1309.31396484375, + "loss": 1.2872, + "margin_dpo/margin_mean": 535.1005859375, + "margin_dpo/margin_std": 1025.5020751953125, + "step": 417 + }, + { + "KL/chosen_KL_mean": -857.3790893554688, + "KL/mean": -1115.459716796875, + "KL/rejected_KL_mean": -1373.5404052734375, + "KL/std": 974.062255859375, + "epoch": 0.6138032305433186, + "fcm_dpo/beta": 0.00038203209987841547, + "fcm_dpo/delta": 0.09407737106084824, + "fcm_dpo/margin": 516.1611938476562, + "fcm_dpo/q_t": 0.47300809621810913, + "grad_norm": 26.765777587890625, + "learning_rate": 1.965167291983757e-07, + "logits/chosen": -0.45634764432907104, + "logits/rejected": -0.4652099609375, + "logps/chosen": -939.3576049804688, + "logps/ref_chosen": -81.97846984863281, + "logps/ref_rejected": -104.69148254394531, + "logps/rejected": -1478.23193359375, + "loss": 1.2179, + "margin_dpo/margin_mean": 516.1611938476562, + "margin_dpo/margin_std": 1353.4468994140625, + "step": 418 + }, + { + "KL/chosen_KL_mean": -676.5275268554688, + "KL/mean": -1033.2412109375, + "KL/rejected_KL_mean": -1389.954833984375, + "KL/std": 921.36865234375, + "epoch": 0.6152716593245228, + "fcm_dpo/beta": 0.0005777660990133882, + "fcm_dpo/delta": 0.011424558237195015, + "fcm_dpo/margin": 713.4273071289062, + "fcm_dpo/q_t": 0.4135735332965851, + "grad_norm": 23.824268341064453, + "learning_rate": 1.9526382029472988e-07, + "logits/chosen": -0.406105101108551, + "logits/rejected": -0.43193238973617554, + "logps/chosen": -729.4761962890625, + "logps/ref_chosen": -52.948646545410156, + "logps/ref_rejected": -91.58309936523438, + "logps/rejected": -1481.5379638671875, + "loss": 1.0996, + "margin_dpo/margin_mean": 713.4273071289062, + "margin_dpo/margin_std": 1061.15673828125, + "step": 419 + }, + { + "KL/chosen_KL_mean": -1244.012939453125, + "KL/mean": -1279.085693359375, + "KL/rejected_KL_mean": -1314.1585693359375, + "KL/std": 1466.0330810546875, + "epoch": 0.6167400881057269, + "fcm_dpo/beta": 0.000616069242823869, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 70.14578247070312, + "fcm_dpo/q_t": 0.47235944867134094, + "grad_norm": 252.84213256835938, + "learning_rate": 1.9401235374032425e-07, + "logits/chosen": -0.4455627501010895, + "logits/rejected": -0.4392239451408386, + "logps/chosen": -1321.78271484375, + "logps/ref_chosen": -77.7699203491211, + "logps/ref_rejected": -69.31985473632812, + "logps/rejected": -1383.478515625, + "loss": 1.5994, + "margin_dpo/margin_mean": 70.14576721191406, + "margin_dpo/margin_std": 2004.8504638671875, + "step": 420 + }, + { + "KL/chosen_KL_mean": -741.7689208984375, + "KL/mean": -928.7274780273438, + "KL/rejected_KL_mean": -1115.68603515625, + "KL/std": 884.5521240234375, + "epoch": 0.618208516886931, + "fcm_dpo/beta": 0.000668178778141737, + "fcm_dpo/delta": 0.015629228204488754, + "fcm_dpo/margin": 373.91705322265625, + "fcm_dpo/q_t": 0.4435460865497589, + "grad_norm": 44.24656677246094, + "learning_rate": 1.9276236251246653e-07, + "logits/chosen": -0.45810630917549133, + "logits/rejected": -0.4658089280128479, + "logps/chosen": -795.5347900390625, + "logps/ref_chosen": -53.765865325927734, + "logps/ref_rejected": -89.28144836425781, + "logps/rejected": -1204.967529296875, + "loss": 1.2457, + "margin_dpo/margin_mean": 373.91705322265625, + "margin_dpo/margin_std": 1011.509765625, + "step": 421 + }, + { + "KL/chosen_KL_mean": -949.32373046875, + "KL/mean": -1192.359130859375, + "KL/rejected_KL_mean": -1435.394775390625, + "KL/std": 1171.419677734375, + "epoch": 0.6196769456681351, + "fcm_dpo/beta": 0.0010301879374310374, + "fcm_dpo/delta": 0.06974077969789505, + "fcm_dpo/margin": 486.0709533691406, + "fcm_dpo/q_t": 0.4236348569393158, + "grad_norm": 89.19212341308594, + "learning_rate": 1.9151387954958792e-07, + "logits/chosen": -0.4476343095302582, + "logits/rejected": -0.464691162109375, + "logps/chosen": -1017.95751953125, + "logps/ref_chosen": -68.6337661743164, + "logps/ref_rejected": -87.86351013183594, + "logps/rejected": -1523.2581787109375, + "loss": 1.3495, + "margin_dpo/margin_mean": 486.0709228515625, + "margin_dpo/margin_std": 1334.6302490234375, + "step": 422 + }, + { + "KL/chosen_KL_mean": -751.098876953125, + "KL/mean": -995.6851806640625, + "KL/rejected_KL_mean": -1240.2713623046875, + "KL/std": 885.8568115234375, + "epoch": 0.6211453744493393, + "fcm_dpo/beta": 0.00025757343973964453, + "fcm_dpo/delta": -0.1725754588842392, + "fcm_dpo/margin": 489.1725158691406, + "fcm_dpo/q_t": 0.4245114326477051, + "grad_norm": 22.50880241394043, + "learning_rate": 1.902669377503756e-07, + "logits/chosen": -0.45179301500320435, + "logits/rejected": -0.46642887592315674, + "logps/chosen": -806.0891723632812, + "logps/ref_chosen": -54.99030303955078, + "logps/ref_rejected": -86.30654907226562, + "logps/rejected": -1326.577880859375, + "loss": 1.2827, + "margin_dpo/margin_mean": 489.1724853515625, + "margin_dpo/margin_std": 1063.1331787109375, + "step": 423 + }, + { + "KL/chosen_KL_mean": -590.9034423828125, + "KL/mean": -812.5848388671875, + "KL/rejected_KL_mean": -1034.266357421875, + "KL/std": 783.5313720703125, + "epoch": 0.6226138032305433, + "fcm_dpo/beta": 0.00025757343973964453, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 443.36285400390625, + "fcm_dpo/q_t": 0.47206780314445496, + "grad_norm": 10.314723014831543, + "learning_rate": 1.890215699729057e-07, + "logits/chosen": -0.38947632908821106, + "logits/rejected": -0.3932424783706665, + "logps/chosen": -646.9153442382812, + "logps/ref_chosen": -56.01192092895508, + "logps/ref_rejected": -66.47896575927734, + "logps/rejected": -1100.745361328125, + "loss": 1.2904, + "margin_dpo/margin_mean": 443.36285400390625, + "margin_dpo/margin_std": 945.1494750976562, + "step": 424 + }, + { + "KL/chosen_KL_mean": -649.426513671875, + "KL/mean": -852.2593994140625, + "KL/rejected_KL_mean": -1055.0921630859375, + "KL/std": 737.5302124023438, + "epoch": 0.6240822320117474, + "fcm_dpo/beta": 0.00025757343973964453, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 405.6656188964844, + "fcm_dpo/q_t": 0.4741717576980591, + "grad_norm": 16.524267196655273, + "learning_rate": 1.8777780903377732e-07, + "logits/chosen": -0.36731043457984924, + "logits/rejected": -0.38339000940322876, + "logps/chosen": -696.2955322265625, + "logps/ref_chosen": -46.86899948120117, + "logps/ref_rejected": -95.92545318603516, + "logps/rejected": -1151.017578125, + "loss": 1.3006, + "margin_dpo/margin_mean": 405.6656188964844, + "margin_dpo/margin_std": 980.6445922851562, + "step": 425 + }, + { + "KL/chosen_KL_mean": -496.1842041015625, + "KL/mean": -661.1705322265625, + "KL/rejected_KL_mean": -826.15673828125, + "KL/std": 525.2192993164062, + "epoch": 0.6255506607929515, + "fcm_dpo/beta": 0.00025757343973964453, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 329.9725341796875, + "fcm_dpo/q_t": 0.4789263606071472, + "grad_norm": 17.888946533203125, + "learning_rate": 1.8653568770724803e-07, + "logits/chosen": -0.3716368079185486, + "logits/rejected": -0.363339364528656, + "logps/chosen": -572.7677612304688, + "logps/ref_chosen": -76.58354187011719, + "logps/ref_rejected": -81.26658630371094, + "logps/rejected": -907.42333984375, + "loss": 1.3107, + "margin_dpo/margin_mean": 329.9725646972656, + "margin_dpo/margin_std": 637.1539916992188, + "step": 426 + }, + { + "KL/chosen_KL_mean": -430.772216796875, + "KL/mean": -575.3130493164062, + "KL/rejected_KL_mean": -719.8538818359375, + "KL/std": 486.26800537109375, + "epoch": 0.6270190895741556, + "fcm_dpo/beta": 0.00025757343973964453, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 289.08160400390625, + "fcm_dpo/q_t": 0.481527715921402, + "grad_norm": 23.913623809814453, + "learning_rate": 1.8529523872436977e-07, + "logits/chosen": -0.3612695634365082, + "logits/rejected": -0.35891193151474, + "logps/chosen": -495.6260986328125, + "logps/ref_chosen": -64.8538818359375, + "logps/ref_rejected": -78.5660171508789, + "logps/rejected": -798.4198608398438, + "loss": 1.3182, + "margin_dpo/margin_mean": 289.08160400390625, + "margin_dpo/margin_std": 542.7352294921875, + "step": 427 + }, + { + "KL/chosen_KL_mean": -599.3250732421875, + "KL/mean": -861.975341796875, + "KL/rejected_KL_mean": -1124.62548828125, + "KL/std": 748.1722412109375, + "epoch": 0.6284875183553598, + "fcm_dpo/beta": 0.00025757343973964453, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 525.3004760742188, + "fcm_dpo/q_t": 0.46697381138801575, + "grad_norm": 43.41276168823242, + "learning_rate": 1.8405649477212697e-07, + "logits/chosen": -0.41682755947113037, + "logits/rejected": -0.4279855489730835, + "logps/chosen": -661.9617919921875, + "logps/ref_chosen": -62.63666534423828, + "logps/ref_rejected": -103.28181457519531, + "logps/rejected": -1227.9073486328125, + "loss": 1.2697, + "margin_dpo/margin_mean": 525.3004760742188, + "margin_dpo/margin_std": 932.6864013671875, + "step": 428 + }, + { + "KL/chosen_KL_mean": -824.616943359375, + "KL/mean": -973.2047119140625, + "KL/rejected_KL_mean": -1121.79248046875, + "KL/std": 866.7525024414062, + "epoch": 0.6299559471365639, + "fcm_dpo/beta": 0.00025757343973964453, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 297.17559814453125, + "fcm_dpo/q_t": 0.48108816146850586, + "grad_norm": 22.956989288330078, + "learning_rate": 1.828194884925749e-07, + "logits/chosen": -0.42566531896591187, + "logits/rejected": -0.4174090027809143, + "logps/chosen": -905.8509521484375, + "logps/ref_chosen": -81.23401641845703, + "logps/ref_rejected": -91.79493713378906, + "logps/rejected": -1213.58740234375, + "loss": 1.3307, + "margin_dpo/margin_mean": 297.1755676269531, + "margin_dpo/margin_std": 1031.1573486328125, + "step": 429 + }, + { + "KL/chosen_KL_mean": -615.6317138671875, + "KL/mean": -830.307861328125, + "KL/rejected_KL_mean": -1044.98388671875, + "KL/std": 806.1045532226562, + "epoch": 0.631424375917768, + "fcm_dpo/beta": 0.00025757343973964453, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 429.3522033691406, + "fcm_dpo/q_t": 0.473649799823761, + "grad_norm": 22.414127349853516, + "learning_rate": 1.8158425248197928e-07, + "logits/chosen": -0.42373788356781006, + "logits/rejected": -0.4414384365081787, + "logps/chosen": -676.5520629882812, + "logps/ref_chosen": -60.920326232910156, + "logps/ref_rejected": -104.42280578613281, + "logps/rejected": -1149.40673828125, + "loss": 1.2974, + "margin_dpo/margin_mean": 429.35223388671875, + "margin_dpo/margin_std": 1028.1934814453125, + "step": 430 + }, + { + "KL/chosen_KL_mean": -606.7068481445312, + "KL/mean": -882.7003173828125, + "KL/rejected_KL_mean": -1158.69384765625, + "KL/std": 827.6256103515625, + "epoch": 0.6328928046989721, + "fcm_dpo/beta": 0.00025757343973964453, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 551.987060546875, + "fcm_dpo/q_t": 0.46554499864578247, + "grad_norm": 22.636369705200195, + "learning_rate": 1.8035081928995788e-07, + "logits/chosen": -0.41880902647972107, + "logits/rejected": -0.43652844429016113, + "logps/chosen": -664.0556030273438, + "logps/ref_chosen": -57.34874725341797, + "logps/ref_rejected": -92.84022521972656, + "logps/rejected": -1251.5340576171875, + "loss": 1.269, + "margin_dpo/margin_mean": 551.987060546875, + "margin_dpo/margin_std": 1050.994384765625, + "step": 431 + }, + { + "KL/chosen_KL_mean": -738.18017578125, + "KL/mean": -1104.6925048828125, + "KL/rejected_KL_mean": -1471.204833984375, + "KL/std": 1035.17333984375, + "epoch": 0.6343612334801763, + "fcm_dpo/beta": 0.00038527188007719815, + "fcm_dpo/delta": 0.06889130920171738, + "fcm_dpo/margin": 733.0245971679688, + "fcm_dpo/q_t": 0.4547889232635498, + "grad_norm": 28.267343521118164, + "learning_rate": 1.791192214186223e-07, + "logits/chosen": -0.43129995465278625, + "logits/rejected": -0.4455578923225403, + "logps/chosen": -809.2550048828125, + "logps/ref_chosen": -71.07479095458984, + "logps/ref_rejected": -98.57952880859375, + "logps/rejected": -1569.7843017578125, + "loss": 1.1671, + "margin_dpo/margin_mean": 733.0245971679688, + "margin_dpo/margin_std": 1155.4307861328125, + "step": 432 + }, + { + "KL/chosen_KL_mean": -1069.6123046875, + "KL/mean": -1359.947265625, + "KL/rejected_KL_mean": -1650.2822265625, + "KL/std": 1036.75927734375, + "epoch": 0.6358296622613803, + "fcm_dpo/beta": 0.0006475095869973302, + "fcm_dpo/delta": -0.06112983077764511, + "fcm_dpo/margin": 580.669921875, + "fcm_dpo/q_t": 0.4136975407600403, + "grad_norm": 42.71298599243164, + "learning_rate": 1.7788949132172193e-07, + "logits/chosen": -0.5129560232162476, + "logits/rejected": -0.5314432382583618, + "logps/chosen": -1127.885498046875, + "logps/ref_chosen": -58.273193359375, + "logps/ref_rejected": -95.95089721679688, + "logps/rejected": -1746.233154296875, + "loss": 1.2069, + "margin_dpo/margin_mean": 580.669921875, + "margin_dpo/margin_std": 1201.7972412109375, + "step": 433 + }, + { + "KL/chosen_KL_mean": -1070.666748046875, + "KL/mean": -1364.0224609375, + "KL/rejected_KL_mean": -1657.3782958984375, + "KL/std": 1538.32275390625, + "epoch": 0.6372980910425844, + "fcm_dpo/beta": 0.0002783612289931625, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 586.7115478515625, + "fcm_dpo/q_t": 0.4694068133831024, + "grad_norm": 28.23455810546875, + "learning_rate": 1.7666166140378853e-07, + "logits/chosen": -0.5182013511657715, + "logits/rejected": -0.5297871232032776, + "logps/chosen": -1132.640380859375, + "logps/ref_chosen": -61.97370147705078, + "logps/ref_rejected": -78.49861145019531, + "logps/rejected": -1735.876953125, + "loss": 1.2991, + "margin_dpo/margin_mean": 586.7114868164062, + "margin_dpo/margin_std": 2062.91650390625, + "step": 434 + }, + { + "KL/chosen_KL_mean": -1081.22021484375, + "KL/mean": -1402.046875, + "KL/rejected_KL_mean": -1722.873779296875, + "KL/std": 1234.45849609375, + "epoch": 0.6387665198237885, + "fcm_dpo/beta": 0.0002783612289931625, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 641.6536865234375, + "fcm_dpo/q_t": 0.45899978280067444, + "grad_norm": 16.346473693847656, + "learning_rate": 1.7543576401928218e-07, + "logits/chosen": -0.5801471471786499, + "logits/rejected": -0.5891480445861816, + "logps/chosen": -1132.72216796875, + "logps/ref_chosen": -51.502052307128906, + "logps/ref_rejected": -87.56689453125, + "logps/rejected": -1810.440673828125, + "loss": 1.2605, + "margin_dpo/margin_mean": 641.6536865234375, + "margin_dpo/margin_std": 1534.156494140625, + "step": 435 + }, + { + "KL/chosen_KL_mean": -1159.755126953125, + "KL/mean": -1270.0465087890625, + "KL/rejected_KL_mean": -1380.3380126953125, + "KL/std": 917.111328125, + "epoch": 0.6402349486049926, + "fcm_dpo/beta": 0.0002783612289931625, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 220.5829315185547, + "fcm_dpo/q_t": 0.4855788052082062, + "grad_norm": 49.0058708190918, + "learning_rate": 1.742118314717391e-07, + "logits/chosen": -0.5959898233413696, + "logits/rejected": -0.5875068306922913, + "logps/chosen": -1231.15869140625, + "logps/ref_chosen": -71.40371704101562, + "logps/ref_rejected": -82.72775268554688, + "logps/rejected": -1463.065673828125, + "loss": 1.3546, + "margin_dpo/margin_mean": 220.58294677734375, + "margin_dpo/margin_std": 1219.72509765625, + "step": 436 + }, + { + "KL/chosen_KL_mean": -1182.003173828125, + "KL/mean": -1367.249755859375, + "KL/rejected_KL_mean": -1552.496337890625, + "KL/std": 1103.9197998046875, + "epoch": 0.6417033773861968, + "fcm_dpo/beta": 0.0002783612289931625, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 370.4931640625, + "fcm_dpo/q_t": 0.4748278260231018, + "grad_norm": 45.79405212402344, + "learning_rate": 1.7298989601292036e-07, + "logits/chosen": -0.5938007235527039, + "logits/rejected": -0.5952749252319336, + "logps/chosen": -1246.747314453125, + "logps/ref_chosen": -64.7442626953125, + "logps/ref_rejected": -82.04356384277344, + "logps/rejected": -1634.539794921875, + "loss": 1.3213, + "margin_dpo/margin_mean": 370.4931640625, + "margin_dpo/margin_std": 1342.3785400390625, + "step": 437 + }, + { + "KL/chosen_KL_mean": -1072.06591796875, + "KL/mean": -1324.57470703125, + "KL/rejected_KL_mean": -1577.08349609375, + "KL/std": 1039.245849609375, + "epoch": 0.6431718061674009, + "fcm_dpo/beta": 0.0002783612289931625, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 505.0177001953125, + "fcm_dpo/q_t": 0.46574634313583374, + "grad_norm": 17.366430282592773, + "learning_rate": 1.7176998984196144e-07, + "logits/chosen": -0.5239421129226685, + "logits/rejected": -0.5232424736022949, + "logps/chosen": -1131.08447265625, + "logps/ref_chosen": -59.0186653137207, + "logps/ref_rejected": -83.07682800292969, + "logps/rejected": -1660.160400390625, + "loss": 1.2731, + "margin_dpo/margin_mean": 505.0177307128906, + "margin_dpo/margin_std": 1082.669677734375, + "step": 438 + }, + { + "KL/chosen_KL_mean": -940.6976318359375, + "KL/mean": -1246.2998046875, + "KL/rejected_KL_mean": -1551.90185546875, + "KL/std": 1239.5167236328125, + "epoch": 0.644640234948605, + "fcm_dpo/beta": 0.00039391464088112116, + "fcm_dpo/delta": 0.034721456468105316, + "fcm_dpo/margin": 611.2044067382812, + "fcm_dpo/q_t": 0.4621936082839966, + "grad_norm": 33.72039794921875, + "learning_rate": 1.7055214510452458e-07, + "logits/chosen": -0.5249619483947754, + "logits/rejected": -0.5584954023361206, + "logps/chosen": -994.481689453125, + "logps/ref_chosen": -53.78407669067383, + "logps/ref_rejected": -83.98545837402344, + "logps/rejected": -1635.887451171875, + "loss": 1.2525, + "margin_dpo/margin_mean": 611.2044067382812, + "margin_dpo/margin_std": 1597.325439453125, + "step": 439 + }, + { + "KL/chosen_KL_mean": -993.7286376953125, + "KL/mean": -1251.96337890625, + "KL/rejected_KL_mean": -1510.1981201171875, + "KL/std": 1111.6761474609375, + "epoch": 0.6461086637298091, + "fcm_dpo/beta": 0.0006941439351066947, + "fcm_dpo/delta": 0.0925978422164917, + "fcm_dpo/margin": 516.469482421875, + "fcm_dpo/q_t": 0.4560081958770752, + "grad_norm": 44.43145751953125, + "learning_rate": 1.6933639389195134e-07, + "logits/chosen": -0.5849356651306152, + "logits/rejected": -0.5931211709976196, + "logps/chosen": -1072.29541015625, + "logps/ref_chosen": -78.56671905517578, + "logps/ref_rejected": -96.49775695800781, + "logps/rejected": -1606.6959228515625, + "loss": 1.2332, + "margin_dpo/margin_mean": 516.469482421875, + "margin_dpo/margin_std": 1347.8345947265625, + "step": 440 + }, + { + "KL/chosen_KL_mean": -1073.631103515625, + "KL/mean": -1258.8494873046875, + "KL/rejected_KL_mean": -1444.06787109375, + "KL/std": 1060.948486328125, + "epoch": 0.6475770925110133, + "fcm_dpo/beta": 0.0010014603612944484, + "fcm_dpo/delta": -0.04774302989244461, + "fcm_dpo/margin": 370.4366455078125, + "fcm_dpo/q_t": 0.40342986583709717, + "grad_norm": 167.0828399658203, + "learning_rate": 1.681227682404166e-07, + "logits/chosen": -0.5610836744308472, + "logits/rejected": -0.566421389579773, + "logps/chosen": -1134.45556640625, + "logps/ref_chosen": -60.824440002441406, + "logps/ref_rejected": -96.47080993652344, + "logps/rejected": -1540.5386962890625, + "loss": 1.3984, + "margin_dpo/margin_mean": 370.4366455078125, + "margin_dpo/margin_std": 1346.23828125, + "step": 441 + }, + { + "KL/chosen_KL_mean": -677.8002319335938, + "KL/mean": -963.8506469726562, + "KL/rejected_KL_mean": -1249.901123046875, + "KL/std": 910.0428466796875, + "epoch": 0.6490455212922174, + "fcm_dpo/beta": 0.0006988396635279059, + "fcm_dpo/delta": -0.21735386550426483, + "fcm_dpo/margin": 572.1007080078125, + "fcm_dpo/q_t": 0.39990121126174927, + "grad_norm": 58.9281120300293, + "learning_rate": 1.669113001300851e-07, + "logits/chosen": -0.5104144215583801, + "logits/rejected": -0.5223550796508789, + "logps/chosen": -724.8114624023438, + "logps/ref_chosen": -47.01121520996094, + "logps/ref_rejected": -76.53926086425781, + "logps/rejected": -1326.4403076171875, + "loss": 1.3103, + "margin_dpo/margin_mean": 572.1007080078125, + "margin_dpo/margin_std": 1180.846923828125, + "step": 442 + }, + { + "KL/chosen_KL_mean": -682.045654296875, + "KL/mean": -812.193359375, + "KL/rejected_KL_mean": -942.3411254882812, + "KL/std": 811.3766479492188, + "epoch": 0.6505139500734214, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 260.2955627441406, + "fcm_dpo/q_t": 0.4954499900341034, + "grad_norm": 5.396309852600098, + "learning_rate": 1.6570202148426815e-07, + "logits/chosen": -0.477186918258667, + "logits/rejected": -0.4765312373638153, + "logps/chosen": -753.318603515625, + "logps/ref_chosen": -71.27301788330078, + "logps/ref_rejected": -86.679931640625, + "logps/rejected": -1029.0211181640625, + "loss": 1.3694, + "margin_dpo/margin_mean": 260.2955322265625, + "margin_dpo/margin_std": 990.7225341796875, + "step": 443 + }, + { + "KL/chosen_KL_mean": -610.6414184570312, + "KL/mean": -837.4876708984375, + "KL/rejected_KL_mean": -1064.3338623046875, + "KL/std": 821.0623779296875, + "epoch": 0.6519823788546255, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 453.6924743652344, + "fcm_dpo/q_t": 0.4920549988746643, + "grad_norm": 5.598912239074707, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": -0.42270809412002563, + "logits/rejected": -0.42965829372406006, + "logps/chosen": -667.8551025390625, + "logps/ref_chosen": -57.213706970214844, + "logps/ref_rejected": -97.25489807128906, + "logps/rejected": -1161.5887451171875, + "loss": 1.3558, + "margin_dpo/margin_mean": 453.69244384765625, + "margin_dpo/margin_std": 947.6956787109375, + "step": 444 + }, + { + "KL/chosen_KL_mean": -393.3489685058594, + "KL/mean": -593.373291015625, + "KL/rejected_KL_mean": -793.3975830078125, + "KL/std": 562.3740234375, + "epoch": 0.6534508076358296, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 400.0485534667969, + "fcm_dpo/q_t": 0.4929931163787842, + "grad_norm": 7.666253089904785, + "learning_rate": 1.6329015999011182e-07, + "logits/chosen": -0.38615715503692627, + "logits/rejected": -0.38465434312820435, + "logps/chosen": -460.6487731933594, + "logps/ref_chosen": -67.29979705810547, + "logps/ref_rejected": -92.68267059326172, + "logps/rejected": -886.0802001953125, + "loss": 1.3592, + "margin_dpo/margin_mean": 400.048583984375, + "margin_dpo/margin_std": 728.7763061523438, + "step": 445 + }, + { + "KL/chosen_KL_mean": -392.439208984375, + "KL/mean": -607.7997436523438, + "KL/rejected_KL_mean": -823.1602783203125, + "KL/std": 526.53857421875, + "epoch": 0.6549192364170338, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 430.7210693359375, + "fcm_dpo/q_t": 0.4924466013908386, + "grad_norm": 7.6744914054870605, + "learning_rate": 1.6208764069656578e-07, + "logits/chosen": -0.3997993469238281, + "logits/rejected": -0.41034257411956787, + "logps/chosen": -451.5377197265625, + "logps/ref_chosen": -59.098487854003906, + "logps/ref_rejected": -101.26419067382812, + "logps/rejected": -924.4244384765625, + "loss": 1.3568, + "margin_dpo/margin_mean": 430.7210693359375, + "margin_dpo/margin_std": 591.9511108398438, + "step": 446 + }, + { + "KL/chosen_KL_mean": -391.9793701171875, + "KL/mean": -633.8006591796875, + "KL/rejected_KL_mean": -875.6220703125, + "KL/std": 740.5856323242188, + "epoch": 0.6563876651982379, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 483.64263916015625, + "fcm_dpo/q_t": 0.49153846502304077, + "grad_norm": 7.994978427886963, + "learning_rate": 1.608874379754465e-07, + "logits/chosen": -0.434841513633728, + "logits/rejected": -0.4521018862724304, + "logps/chosen": -448.0547180175781, + "logps/ref_chosen": -56.07533264160156, + "logps/ref_rejected": -98.69475555419922, + "logps/rejected": -974.3168334960938, + "loss": 1.3536, + "margin_dpo/margin_mean": 483.64263916015625, + "margin_dpo/margin_std": 850.4795532226562, + "step": 447 + }, + { + "KL/chosen_KL_mean": -371.1406555175781, + "KL/mean": -603.8892822265625, + "KL/rejected_KL_mean": -836.6378784179688, + "KL/std": 571.3126220703125, + "epoch": 0.657856093979442, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 465.4972229003906, + "fcm_dpo/q_t": 0.49184244871139526, + "grad_norm": 7.214312553405762, + "learning_rate": 1.5968958345321177e-07, + "logits/chosen": -0.4187026917934418, + "logits/rejected": -0.4283428192138672, + "logps/chosen": -431.1445007324219, + "logps/ref_chosen": -60.00384521484375, + "logps/ref_rejected": -102.26465606689453, + "logps/rejected": -938.9025268554688, + "loss": 1.3545, + "margin_dpo/margin_mean": 465.4972229003906, + "margin_dpo/margin_std": 671.6864013671875, + "step": 448 + }, + { + "KL/chosen_KL_mean": -494.8532409667969, + "KL/mean": -674.86083984375, + "KL/rejected_KL_mean": -854.8682861328125, + "KL/std": 626.908203125, + "epoch": 0.6593245227606461, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 360.01507568359375, + "fcm_dpo/q_t": 0.4936937391757965, + "grad_norm": 8.744072914123535, + "learning_rate": 1.584941086944423e-07, + "logits/chosen": -0.43162280321121216, + "logits/rejected": -0.4255381226539612, + "logps/chosen": -562.3798828125, + "logps/ref_chosen": -67.52661895751953, + "logps/ref_rejected": -88.59690856933594, + "logps/rejected": -943.4652099609375, + "loss": 1.3619, + "margin_dpo/margin_mean": 360.01507568359375, + "margin_dpo/margin_std": 755.4654541015625, + "step": 449 + }, + { + "KL/chosen_KL_mean": -329.07965087890625, + "KL/mean": -557.316650390625, + "KL/rejected_KL_mean": -785.5537109375, + "KL/std": 504.0699462890625, + "epoch": 0.6607929515418502, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 456.47406005859375, + "fcm_dpo/q_t": 0.49199458956718445, + "grad_norm": 10.857216835021973, + "learning_rate": 1.573010452010098e-07, + "logits/chosen": -0.38689732551574707, + "logits/rejected": -0.40724536776542664, + "logps/chosen": -386.1877746582031, + "logps/ref_chosen": -57.10811996459961, + "logps/ref_rejected": -102.75494384765625, + "logps/rejected": -888.3086547851562, + "loss": 1.3549, + "margin_dpo/margin_mean": 456.47406005859375, + "margin_dpo/margin_std": 548.2633666992188, + "step": 450 + }, + { + "KL/chosen_KL_mean": -549.2373046875, + "KL/mean": -656.2725830078125, + "KL/rejected_KL_mean": -763.307861328125, + "KL/std": 700.226318359375, + "epoch": 0.6622613803230544, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 214.07054138183594, + "fcm_dpo/q_t": 0.4962322413921356, + "grad_norm": 6.819939613342285, + "learning_rate": 1.5611042441124687e-07, + "logits/chosen": -0.44771769642829895, + "logits/rejected": -0.43964436650276184, + "logps/chosen": -607.7061767578125, + "logps/ref_chosen": -58.46883010864258, + "logps/ref_rejected": -72.92941284179688, + "logps/rejected": -836.2373046875, + "loss": 1.3725, + "margin_dpo/margin_mean": 214.07054138183594, + "margin_dpo/margin_std": 986.6764526367188, + "step": 451 + }, + { + "KL/chosen_KL_mean": -501.6789855957031, + "KL/mean": -689.10693359375, + "KL/rejected_KL_mean": -876.5348510742188, + "KL/std": 670.5494384765625, + "epoch": 0.6637298091042585, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 374.8558654785156, + "fcm_dpo/q_t": 0.4934268891811371, + "grad_norm": 3.6413915157318115, + "learning_rate": 1.549222776991186e-07, + "logits/chosen": -0.4632149636745453, + "logits/rejected": -0.47757190465927124, + "logps/chosen": -552.069580078125, + "logps/ref_chosen": -50.39055252075195, + "logps/ref_rejected": -97.77142333984375, + "logps/rejected": -974.3062744140625, + "loss": 1.3608, + "margin_dpo/margin_mean": 374.85589599609375, + "margin_dpo/margin_std": 725.793701171875, + "step": 452 + }, + { + "KL/chosen_KL_mean": -527.853515625, + "KL/mean": -772.795654296875, + "KL/rejected_KL_mean": -1017.73779296875, + "KL/std": 713.8123779296875, + "epoch": 0.6651982378854625, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 489.8843078613281, + "fcm_dpo/q_t": 0.4914173185825348, + "grad_norm": 6.951712608337402, + "learning_rate": 1.5373663637339584e-07, + "logits/chosen": -0.4904746413230896, + "logits/rejected": -0.4950370788574219, + "logps/chosen": -585.568359375, + "logps/ref_chosen": -57.71485137939453, + "logps/ref_rejected": -82.20741271972656, + "logps/rejected": -1099.9451904296875, + "loss": 1.3532, + "margin_dpo/margin_mean": 489.88427734375, + "margin_dpo/margin_std": 879.7352294921875, + "step": 453 + }, + { + "KL/chosen_KL_mean": -756.3292236328125, + "KL/mean": -986.6734619140625, + "KL/rejected_KL_mean": -1217.017578125, + "KL/std": 858.3633422851562, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 460.6884765625, + "fcm_dpo/q_t": 0.4919373393058777, + "grad_norm": 4.276784896850586, + "learning_rate": 1.5255353167683017e-07, + "logits/chosen": -0.48667120933532715, + "logits/rejected": -0.4960416555404663, + "logps/chosen": -817.27490234375, + "logps/ref_chosen": -60.945648193359375, + "logps/ref_rejected": -84.95079040527344, + "logps/rejected": -1301.968505859375, + "loss": 1.3556, + "margin_dpo/margin_mean": 460.6884765625, + "margin_dpo/margin_std": 1064.3143310546875, + "step": 454 + }, + { + "KL/chosen_KL_mean": -746.62548828125, + "KL/mean": -1129.1025390625, + "KL/rejected_KL_mean": -1511.579345703125, + "KL/std": 1107.7918701171875, + "epoch": 0.6681350954478708, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 764.9539794921875, + "fcm_dpo/q_t": 0.4866509437561035, + "grad_norm": 7.09210729598999, + "learning_rate": 1.5137299478533064e-07, + "logits/chosen": -0.5352227687835693, + "logits/rejected": -0.5812809467315674, + "logps/chosen": -791.51220703125, + "logps/ref_chosen": -44.88671112060547, + "logps/ref_rejected": -115.30147552490234, + "logps/rejected": -1626.880859375, + "loss": 1.3358, + "margin_dpo/margin_mean": 764.953857421875, + "margin_dpo/margin_std": 1381.763671875, + "step": 455 + }, + { + "KL/chosen_KL_mean": -1079.772216796875, + "KL/mean": -1386.8994140625, + "KL/rejected_KL_mean": -1694.026611328125, + "KL/std": 1328.23046875, + "epoch": 0.6696035242290749, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 614.2544555664062, + "fcm_dpo/q_t": 0.48927462100982666, + "grad_norm": 4.1105146408081055, + "learning_rate": 1.5019505680714232e-07, + "logits/chosen": -0.5510225296020508, + "logits/rejected": -0.5779255628585815, + "logps/chosen": -1136.8089599609375, + "logps/ref_chosen": -57.036781311035156, + "logps/ref_rejected": -105.21784210205078, + "logps/rejected": -1799.244384765625, + "loss": 1.3461, + "margin_dpo/margin_mean": 614.25439453125, + "margin_dpo/margin_std": 1356.8857421875, + "step": 456 + }, + { + "KL/chosen_KL_mean": -928.1408081054688, + "KL/mean": -1333.076416015625, + "KL/rejected_KL_mean": -1738.0120849609375, + "KL/std": 1123.67529296875, + "epoch": 0.671071953010279, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 809.871337890625, + "fcm_dpo/q_t": 0.4858279526233673, + "grad_norm": 6.433835506439209, + "learning_rate": 1.4901974878202627e-07, + "logits/chosen": -0.6030972003936768, + "logits/rejected": -0.6290802359580994, + "logps/chosen": -982.38330078125, + "logps/ref_chosen": -54.24253845214844, + "logps/ref_rejected": -85.10956573486328, + "logps/rejected": -1823.12158203125, + "loss": 1.3325, + "margin_dpo/margin_mean": 809.871337890625, + "margin_dpo/margin_std": 1329.330810546875, + "step": 457 + }, + { + "KL/chosen_KL_mean": -1218.3466796875, + "KL/mean": -1525.91845703125, + "KL/rejected_KL_mean": -1833.4901123046875, + "KL/std": 1092.089599609375, + "epoch": 0.6725403817914831, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 615.1434326171875, + "fcm_dpo/q_t": 0.4892372488975525, + "grad_norm": 5.473250865936279, + "learning_rate": 1.4784710168044212e-07, + "logits/chosen": -0.6052824854850769, + "logits/rejected": -0.6187411546707153, + "logps/chosen": -1273.755615234375, + "logps/ref_chosen": -55.40888214111328, + "logps/ref_rejected": -97.68325805664062, + "logps/rejected": -1931.17333984375, + "loss": 1.3456, + "margin_dpo/margin_mean": 615.1434326171875, + "margin_dpo/margin_std": 1242.5, + "step": 458 + }, + { + "KL/chosen_KL_mean": -1342.32373046875, + "KL/mean": -1710.5882568359375, + "KL/rejected_KL_mean": -2078.8525390625, + "KL/std": 1349.3956298828125, + "epoch": 0.6740088105726872, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 736.52880859375, + "fcm_dpo/q_t": 0.48712393641471863, + "grad_norm": 6.021612644195557, + "learning_rate": 1.466771464027316e-07, + "logits/chosen": -0.6226590871810913, + "logits/rejected": -0.6531392335891724, + "logps/chosen": -1388.88134765625, + "logps/ref_chosen": -46.55748748779297, + "logps/ref_rejected": -86.16854095458984, + "logps/rejected": -2165.02099609375, + "loss": 1.339, + "margin_dpo/margin_mean": 736.52880859375, + "margin_dpo/margin_std": 1749.3363037109375, + "step": 459 + }, + { + "KL/chosen_KL_mean": -1793.916259765625, + "KL/mean": -2120.57568359375, + "KL/rejected_KL_mean": -2447.23486328125, + "KL/std": 1669.0478515625, + "epoch": 0.6754772393538914, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 653.318603515625, + "fcm_dpo/q_t": 0.4886649549007416, + "grad_norm": 8.884930610656738, + "learning_rate": 1.4550991377830423e-07, + "logits/chosen": -0.6757256984710693, + "logits/rejected": -0.7148680686950684, + "logps/chosen": -1845.55126953125, + "logps/ref_chosen": -51.63489532470703, + "logps/ref_rejected": -104.11935424804688, + "logps/rejected": -2551.3544921875, + "loss": 1.3459, + "margin_dpo/margin_mean": 653.3186645507812, + "margin_dpo/margin_std": 1995.814697265625, + "step": 460 + }, + { + "KL/chosen_KL_mean": -1595.17822265625, + "KL/mean": -1813.5162353515625, + "KL/rejected_KL_mean": -2031.854248046875, + "KL/std": 1255.39697265625, + "epoch": 0.6769456681350955, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 436.6760559082031, + "fcm_dpo/q_t": 0.49233686923980713, + "grad_norm": 9.014615058898926, + "learning_rate": 1.4434543456482518e-07, + "logits/chosen": -0.7007662057876587, + "logits/rejected": -0.7139623761177063, + "logps/chosen": -1650.360107421875, + "logps/ref_chosen": -55.18195724487305, + "logps/ref_rejected": -86.47689819335938, + "logps/rejected": -2118.3310546875, + "loss": 1.359, + "margin_dpo/margin_mean": 436.67608642578125, + "margin_dpo/margin_std": 1588.939453125, + "step": 461 + }, + { + "KL/chosen_KL_mean": -1918.244384765625, + "KL/mean": -2012.81298828125, + "KL/rejected_KL_mean": -2107.3818359375, + "KL/std": 1555.3056640625, + "epoch": 0.6784140969162996, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 189.1370086669922, + "fcm_dpo/q_t": 0.4966925382614136, + "grad_norm": 25.653871536254883, + "learning_rate": 1.4318373944740484e-07, + "logits/chosen": -0.6912171244621277, + "logits/rejected": -0.6754558086395264, + "logps/chosen": -1988.172607421875, + "logps/ref_chosen": -69.92803192138672, + "logps/ref_rejected": -78.84111022949219, + "logps/rejected": -2186.22265625, + "loss": 1.3773, + "margin_dpo/margin_mean": 189.13702392578125, + "margin_dpo/margin_std": 1842.75927734375, + "step": 462 + }, + { + "KL/chosen_KL_mean": -1629.407958984375, + "KL/mean": -1926.924072265625, + "KL/rejected_KL_mean": -2224.440185546875, + "KL/std": 1563.47412109375, + "epoch": 0.6798825256975036, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 595.0322265625, + "fcm_dpo/q_t": 0.48969152569770813, + "grad_norm": 10.694584846496582, + "learning_rate": 1.4202485903778976e-07, + "logits/chosen": -0.6257902383804321, + "logits/rejected": -0.6413998603820801, + "logps/chosen": -1684.682373046875, + "logps/ref_chosen": -55.27437210083008, + "logps/ref_rejected": -89.02497863769531, + "logps/rejected": -2313.46533203125, + "loss": 1.3502, + "margin_dpo/margin_mean": 595.0322875976562, + "margin_dpo/margin_std": 2058.953125, + "step": 463 + }, + { + "KL/chosen_KL_mean": -1547.507080078125, + "KL/mean": -2104.552001953125, + "KL/rejected_KL_mean": -2661.59716796875, + "KL/std": 1696.195068359375, + "epoch": 0.6813509544787077, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1114.0902099609375, + "fcm_dpo/q_t": 0.48058074712753296, + "grad_norm": 6.717080593109131, + "learning_rate": 1.4086882387355658e-07, + "logits/chosen": -0.6039457321166992, + "logits/rejected": -0.6586399078369141, + "logps/chosen": -1598.4193115234375, + "logps/ref_chosen": -50.91230010986328, + "logps/ref_rejected": -102.4893798828125, + "logps/rejected": -2764.08642578125, + "loss": 1.3144, + "margin_dpo/margin_mean": 1114.0902099609375, + "margin_dpo/margin_std": 1911.744140625, + "step": 464 + }, + { + "KL/chosen_KL_mean": -1389.27978515625, + "KL/mean": -1775.259765625, + "KL/rejected_KL_mean": -2161.23974609375, + "KL/std": 1636.043212890625, + "epoch": 0.6828193832599119, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 771.9598388671875, + "fcm_dpo/q_t": 0.4866272807121277, + "grad_norm": 5.775012016296387, + "learning_rate": 1.3971566441730714e-07, + "logits/chosen": -0.5333804488182068, + "logits/rejected": -0.556584358215332, + "logps/chosen": -1449.396728515625, + "logps/ref_chosen": -60.116851806640625, + "logps/ref_rejected": -113.94602966308594, + "logps/rejected": -2275.185791015625, + "loss": 1.3381, + "margin_dpo/margin_mean": 771.9598388671875, + "margin_dpo/margin_std": 1992.7213134765625, + "step": 465 + }, + { + "KL/chosen_KL_mean": -1304.01220703125, + "KL/mean": -1642.657470703125, + "KL/rejected_KL_mean": -1981.302734375, + "KL/std": 1344.09765625, + "epoch": 0.684287812041116, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 677.29052734375, + "fcm_dpo/q_t": 0.4881631135940552, + "grad_norm": 4.701225757598877, + "learning_rate": 1.3856541105586545e-07, + "logits/chosen": -0.5309747457504272, + "logits/rejected": -0.5521206855773926, + "logps/chosen": -1356.9332275390625, + "logps/ref_chosen": -52.920921325683594, + "logps/ref_rejected": -90.3154296875, + "logps/rejected": -2071.6181640625, + "loss": 1.343, + "margin_dpo/margin_mean": 677.29052734375, + "margin_dpo/margin_std": 1733.6058349609375, + "step": 466 + }, + { + "KL/chosen_KL_mean": -2066.13720703125, + "KL/mean": -2241.6923828125, + "KL/rejected_KL_mean": -2417.24755859375, + "KL/std": 1884.09375, + "epoch": 0.6857562408223201, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 351.11004638671875, + "fcm_dpo/q_t": 0.4938202500343323, + "grad_norm": 13.84892463684082, + "learning_rate": 1.3741809409947729e-07, + "logits/chosen": -0.5912429690361023, + "logits/rejected": -0.5880587100982666, + "logps/chosen": -2144.85302734375, + "logps/ref_chosen": -78.7158203125, + "logps/ref_rejected": -102.86019897460938, + "logps/rejected": -2520.10791015625, + "loss": 1.3708, + "margin_dpo/margin_mean": 351.1100158691406, + "margin_dpo/margin_std": 2587.14794921875, + "step": 467 + }, + { + "KL/chosen_KL_mean": -1504.03369140625, + "KL/mean": -1898.8018798828125, + "KL/rejected_KL_mean": -2293.570068359375, + "KL/std": 1620.427001953125, + "epoch": 0.6872246696035242, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 789.5361328125, + "fcm_dpo/q_t": 0.48619183897972107, + "grad_norm": 10.929081916809082, + "learning_rate": 1.362737437810114e-07, + "logits/chosen": -0.5442489385604858, + "logits/rejected": -0.5691794753074646, + "logps/chosen": -1573.9691162109375, + "logps/ref_chosen": -69.93536376953125, + "logps/ref_rejected": -101.02880859375, + "logps/rejected": -2394.598876953125, + "loss": 1.3377, + "margin_dpo/margin_mean": 789.5361938476562, + "margin_dpo/margin_std": 2186.982666015625, + "step": 468 + }, + { + "KL/chosen_KL_mean": -1296.493896484375, + "KL/mean": -1764.9542236328125, + "KL/rejected_KL_mean": -2233.41455078125, + "KL/std": 1456.62353515625, + "epoch": 0.6886930983847284, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 936.920654296875, + "fcm_dpo/q_t": 0.4836467504501343, + "grad_norm": 10.054937362670898, + "learning_rate": 1.351323902551631e-07, + "logits/chosen": -0.5154561996459961, + "logits/rejected": -0.545080840587616, + "logps/chosen": -1364.61865234375, + "logps/ref_chosen": -68.12469482421875, + "logps/ref_rejected": -104.78640747070312, + "logps/rejected": -2338.201171875, + "loss": 1.3254, + "margin_dpo/margin_mean": 936.920654296875, + "margin_dpo/margin_std": 1700.6353759765625, + "step": 469 + }, + { + "KL/chosen_KL_mean": -1085.716552734375, + "KL/mean": -1395.3118896484375, + "KL/rejected_KL_mean": -1704.9072265625, + "KL/std": 1203.486328125, + "epoch": 0.6901615271659325, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 619.190673828125, + "fcm_dpo/q_t": 0.4891727566719055, + "grad_norm": 4.784751892089844, + "learning_rate": 1.339940635976592e-07, + "logits/chosen": -0.4637961983680725, + "logits/rejected": -0.4888746738433838, + "logps/chosen": -1129.508544921875, + "logps/ref_chosen": -43.791927337646484, + "logps/ref_rejected": -82.70285034179688, + "logps/rejected": -1787.610107421875, + "loss": 1.3458, + "margin_dpo/margin_mean": 619.190673828125, + "margin_dpo/margin_std": 1384.6240234375, + "step": 470 + }, + { + "KL/chosen_KL_mean": -1458.683837890625, + "KL/mean": -1615.782958984375, + "KL/rejected_KL_mean": -1772.8822021484375, + "KL/std": 1343.892578125, + "epoch": 0.6916299559471366, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 314.1982727050781, + "fcm_dpo/q_t": 0.4945109486579895, + "grad_norm": 14.473358154296875, + "learning_rate": 1.3285879380446563e-07, + "logits/chosen": -0.5264901518821716, + "logits/rejected": -0.5227891206741333, + "logps/chosen": -1522.0234375, + "logps/ref_chosen": -63.33952331542969, + "logps/ref_rejected": -83.61048126220703, + "logps/rejected": -1856.49267578125, + "loss": 1.3675, + "margin_dpo/margin_mean": 314.19830322265625, + "margin_dpo/margin_std": 1578.263671875, + "step": 471 + }, + { + "KL/chosen_KL_mean": -1396.530029296875, + "KL/mean": -1693.50439453125, + "KL/rejected_KL_mean": -1990.4786376953125, + "KL/std": 1501.013671875, + "epoch": 0.6930983847283406, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 593.9485473632812, + "fcm_dpo/q_t": 0.4896436929702759, + "grad_norm": 5.583034992218018, + "learning_rate": 1.317266107909975e-07, + "logits/chosen": -0.5575324296951294, + "logits/rejected": -0.5637389421463013, + "logps/chosen": -1480.1962890625, + "logps/ref_chosen": -83.66610717773438, + "logps/ref_rejected": -117.20919799804688, + "logps/rejected": -2107.68798828125, + "loss": 1.3496, + "margin_dpo/margin_mean": 593.9485473632812, + "margin_dpo/margin_std": 1845.80224609375, + "step": 472 + }, + { + "KL/chosen_KL_mean": -1678.685791015625, + "KL/mean": -1702.64501953125, + "KL/rejected_KL_mean": -1726.6043701171875, + "KL/std": 1605.0902099609375, + "epoch": 0.6945668135095447, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 47.91863250732422, + "fcm_dpo/q_t": 0.49903565645217896, + "grad_norm": 10.380570411682129, + "learning_rate": 1.3059754439133002e-07, + "logits/chosen": -0.5147888660430908, + "logits/rejected": -0.5062965154647827, + "logps/chosen": -1742.1827392578125, + "logps/ref_chosen": -63.49696731567383, + "logps/ref_rejected": -81.14657592773438, + "logps/rejected": -1807.7509765625, + "loss": 1.389, + "margin_dpo/margin_mean": 47.918617248535156, + "margin_dpo/margin_std": 2152.85302734375, + "step": 473 + }, + { + "KL/chosen_KL_mean": -1045.2838134765625, + "KL/mean": -1396.827880859375, + "KL/rejected_KL_mean": -1748.3720703125, + "KL/std": 1296.06787109375, + "epoch": 0.6960352422907489, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 703.088134765625, + "fcm_dpo/q_t": 0.4877893924713135, + "grad_norm": 12.54921817779541, + "learning_rate": 1.2947162435741277e-07, + "logits/chosen": -0.44941064715385437, + "logits/rejected": -0.465454638004303, + "logps/chosen": -1097.895751953125, + "logps/ref_chosen": -52.6119384765625, + "logps/ref_rejected": -90.08041381835938, + "logps/rejected": -1838.452392578125, + "loss": 1.3413, + "margin_dpo/margin_mean": 703.088134765625, + "margin_dpo/margin_std": 1566.402099609375, + "step": 474 + }, + { + "KL/chosen_KL_mean": -696.1770629882812, + "KL/mean": -1096.912109375, + "KL/rejected_KL_mean": -1497.64697265625, + "KL/std": 1150.5213623046875, + "epoch": 0.697503671071953, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 801.469970703125, + "fcm_dpo/q_t": 0.48600390553474426, + "grad_norm": 16.383455276489258, + "learning_rate": 1.2834888035828596e-07, + "logits/chosen": -0.43204575777053833, + "logits/rejected": -0.46623319387435913, + "logps/chosen": -738.6722412109375, + "logps/ref_chosen": -42.49519348144531, + "logps/ref_rejected": -90.06294250488281, + "logps/rejected": -1587.7099609375, + "loss": 1.3334, + "margin_dpo/margin_mean": 801.469970703125, + "margin_dpo/margin_std": 1452.920654296875, + "step": 475 + }, + { + "KL/chosen_KL_mean": -922.133056640625, + "KL/mean": -1211.9774169921875, + "KL/rejected_KL_mean": -1501.82177734375, + "KL/std": 1047.379150390625, + "epoch": 0.6989720998531571, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 579.6885986328125, + "fcm_dpo/q_t": 0.4898582696914673, + "grad_norm": 7.450405597686768, + "learning_rate": 1.2722934197929802e-07, + "logits/chosen": -0.4594970941543579, + "logits/rejected": -0.4757981300354004, + "logps/chosen": -965.0824584960938, + "logps/ref_chosen": -42.94938278198242, + "logps/ref_rejected": -73.71023559570312, + "logps/rejected": -1575.531982421875, + "loss": 1.3477, + "margin_dpo/margin_mean": 579.6885986328125, + "margin_dpo/margin_std": 1162.8359375, + "step": 476 + }, + { + "KL/chosen_KL_mean": -1014.1900634765625, + "KL/mean": -1202.794189453125, + "KL/rejected_KL_mean": -1391.398193359375, + "KL/std": 987.1451416015625, + "epoch": 0.7004405286343612, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 377.20819091796875, + "fcm_dpo/q_t": 0.4933883547782898, + "grad_norm": 4.9775261878967285, + "learning_rate": 1.2611303872132631e-07, + "logits/chosen": -0.48688119649887085, + "logits/rejected": -0.47269320487976074, + "logps/chosen": -1084.962646484375, + "logps/ref_chosen": -70.77261352539062, + "logps/ref_rejected": -76.13737487792969, + "logps/rejected": -1467.53564453125, + "loss": 1.3615, + "margin_dpo/margin_mean": 377.20819091796875, + "margin_dpo/margin_std": 1102.049560546875, + "step": 477 + }, + { + "KL/chosen_KL_mean": -741.496337890625, + "KL/mean": -1104.615966796875, + "KL/rejected_KL_mean": -1467.7357177734375, + "KL/std": 1056.3428955078125, + "epoch": 0.7019089574155654, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 726.2393798828125, + "fcm_dpo/q_t": 0.48729372024536133, + "grad_norm": 13.8693208694458, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": -0.4414135813713074, + "logits/rejected": -0.4705617129802704, + "logps/chosen": -782.9368286132812, + "logps/ref_chosen": -41.440513610839844, + "logps/ref_rejected": -85.36196899414062, + "logps/rejected": -1553.09765625, + "loss": 1.3379, + "margin_dpo/margin_mean": 726.2393798828125, + "margin_dpo/margin_std": 1235.259033203125, + "step": 478 + }, + { + "KL/chosen_KL_mean": -1152.8778076171875, + "KL/mean": -1485.199951171875, + "KL/rejected_KL_mean": -1817.52197265625, + "KL/std": 1490.449951171875, + "epoch": 0.7033773861967695, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 664.6441650390625, + "fcm_dpo/q_t": 0.4884074628353119, + "grad_norm": 4.815835952758789, + "learning_rate": 1.2389025514492456e-07, + "logits/chosen": -0.4350011944770813, + "logits/rejected": -0.4591215252876282, + "logps/chosen": -1206.7857666015625, + "logps/ref_chosen": -53.907920837402344, + "logps/ref_rejected": -95.1163330078125, + "logps/rejected": -1912.638427734375, + "loss": 1.343, + "margin_dpo/margin_mean": 664.6441650390625, + "margin_dpo/margin_std": 1489.7821044921875, + "step": 479 + }, + { + "KL/chosen_KL_mean": -1410.0196533203125, + "KL/mean": -1525.3935546875, + "KL/rejected_KL_mean": -1640.767578125, + "KL/std": 1323.5804443359375, + "epoch": 0.7048458149779736, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 230.74789428710938, + "fcm_dpo/q_t": 0.4959403872489929, + "grad_norm": 8.32140064239502, + "learning_rate": 1.227838333989088e-07, + "logits/chosen": -0.5214420557022095, + "logits/rejected": -0.5171930193901062, + "logps/chosen": -1468.702392578125, + "logps/ref_chosen": -58.682701110839844, + "logps/ref_rejected": -82.93248748779297, + "logps/rejected": -1723.7000732421875, + "loss": 1.3738, + "margin_dpo/margin_mean": 230.7479248046875, + "margin_dpo/margin_std": 1702.75048828125, + "step": 480 + }, + { + "KL/chosen_KL_mean": -1059.987548828125, + "KL/mean": -1491.16796875, + "KL/rejected_KL_mean": -1922.3486328125, + "KL/std": 1393.0323486328125, + "epoch": 0.7063142437591777, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 862.3609619140625, + "fcm_dpo/q_t": 0.484965980052948, + "grad_norm": 4.985651969909668, + "learning_rate": 1.2168076391719489e-07, + "logits/chosen": -0.5206190347671509, + "logits/rejected": -0.562711238861084, + "logps/chosen": -1114.9517822265625, + "logps/ref_chosen": -54.964271545410156, + "logps/ref_rejected": -92.42044067382812, + "logps/rejected": -2014.76904296875, + "loss": 1.33, + "margin_dpo/margin_mean": 862.3610229492188, + "margin_dpo/margin_std": 1640.7061767578125, + "step": 481 + }, + { + "KL/chosen_KL_mean": -1543.61083984375, + "KL/mean": -1666.1243896484375, + "KL/rejected_KL_mean": -1788.637939453125, + "KL/std": 1566.256591796875, + "epoch": 0.7077826725403817, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 245.027099609375, + "fcm_dpo/q_t": 0.49571603536605835, + "grad_norm": 9.695770263671875, + "learning_rate": 1.2058107576668938e-07, + "logits/chosen": -0.5083905458450317, + "logits/rejected": -0.5077981948852539, + "logps/chosen": -1611.164306640625, + "logps/ref_chosen": -67.553466796875, + "logps/ref_rejected": -87.58953857421875, + "logps/rejected": -1876.2275390625, + "loss": 1.374, + "margin_dpo/margin_mean": 245.02706909179688, + "margin_dpo/margin_std": 1968.4217529296875, + "step": 482 + }, + { + "KL/chosen_KL_mean": -1194.04248046875, + "KL/mean": -1479.513671875, + "KL/rejected_KL_mean": -1764.9847412109375, + "KL/std": 1215.059326171875, + "epoch": 0.7092511013215859, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 570.9423217773438, + "fcm_dpo/q_t": 0.4899991750717163, + "grad_norm": 7.4929304122924805, + "learning_rate": 1.194847979251979e-07, + "logits/chosen": -0.5671132206916809, + "logits/rejected": -0.5771450996398926, + "logps/chosen": -1257.372314453125, + "logps/ref_chosen": -63.32981872558594, + "logps/ref_rejected": -95.78697204589844, + "logps/rejected": -1860.771728515625, + "loss": 1.3497, + "margin_dpo/margin_mean": 570.9423217773438, + "margin_dpo/margin_std": 1546.977783203125, + "step": 483 + }, + { + "KL/chosen_KL_mean": -1078.635498046875, + "KL/mean": -1429.60009765625, + "KL/rejected_KL_mean": -1780.564697265625, + "KL/std": 1515.4169921875, + "epoch": 0.71071953010279, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 701.9291381835938, + "fcm_dpo/q_t": 0.48787161707878113, + "grad_norm": 6.547210216522217, + "learning_rate": 1.1839195928066101e-07, + "logits/chosen": -0.4985977113246918, + "logits/rejected": -0.5153236389160156, + "logps/chosen": -1137.773681640625, + "logps/ref_chosen": -59.13812255859375, + "logps/ref_rejected": -84.37144470214844, + "logps/rejected": -1864.93603515625, + "loss": 1.3425, + "margin_dpo/margin_mean": 701.9291381835938, + "margin_dpo/margin_std": 1918.406494140625, + "step": 484 + }, + { + "KL/chosen_KL_mean": -998.8741455078125, + "KL/mean": -1385.538818359375, + "KL/rejected_KL_mean": -1772.2037353515625, + "KL/std": 1393.5875244140625, + "epoch": 0.7121879588839941, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 773.32958984375, + "fcm_dpo/q_t": 0.4865483045578003, + "grad_norm": 6.182162761688232, + "learning_rate": 1.1730258863039347e-07, + "logits/chosen": -0.5134660005569458, + "logits/rejected": -0.5451745390892029, + "logps/chosen": -1057.7236328125, + "logps/ref_chosen": -58.849571228027344, + "logps/ref_rejected": -103.36408233642578, + "logps/rejected": -1875.56787109375, + "loss": 1.3371, + "margin_dpo/margin_mean": 773.32958984375, + "margin_dpo/margin_std": 1857.324462890625, + "step": 485 + }, + { + "KL/chosen_KL_mean": -1066.927734375, + "KL/mean": -1443.706787109375, + "KL/rejected_KL_mean": -1820.4857177734375, + "KL/std": 1382.1572265625, + "epoch": 0.7136563876651982, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 753.5579833984375, + "fcm_dpo/q_t": 0.48684683442115784, + "grad_norm": 4.5740437507629395, + "learning_rate": 1.1621671468032493e-07, + "logits/chosen": -0.46131497621536255, + "logits/rejected": -0.49075794219970703, + "logps/chosen": -1122.1875, + "logps/ref_chosen": -55.25966262817383, + "logps/ref_rejected": -92.13936614990234, + "logps/rejected": -1912.625, + "loss": 1.3381, + "margin_dpo/margin_mean": 753.5579833984375, + "margin_dpo/margin_std": 1785.576171875, + "step": 486 + }, + { + "KL/chosen_KL_mean": -1072.845458984375, + "KL/mean": -1368.38525390625, + "KL/rejected_KL_mean": -1663.9249267578125, + "KL/std": 1040.571533203125, + "epoch": 0.7151248164464024, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 591.0794677734375, + "fcm_dpo/q_t": 0.4896491765975952, + "grad_norm": 9.592281341552734, + "learning_rate": 1.1513436604424378e-07, + "logits/chosen": -0.47907352447509766, + "logits/rejected": -0.4934772253036499, + "logps/chosen": -1125.90869140625, + "logps/ref_chosen": -53.06330871582031, + "logps/ref_rejected": -92.41883087158203, + "logps/rejected": -1756.34375, + "loss": 1.3469, + "margin_dpo/margin_mean": 591.0795288085938, + "margin_dpo/margin_std": 1158.6181640625, + "step": 487 + }, + { + "KL/chosen_KL_mean": -761.5409545898438, + "KL/mean": -1126.8492431640625, + "KL/rejected_KL_mean": -1492.157470703125, + "KL/std": 1100.26611328125, + "epoch": 0.7165932452276065, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 730.6165161132812, + "fcm_dpo/q_t": 0.487249493598938, + "grad_norm": 11.395562171936035, + "learning_rate": 1.1405557124304335e-07, + "logits/chosen": -0.4758809208869934, + "logits/rejected": -0.49622082710266113, + "logps/chosen": -813.7691040039062, + "logps/ref_chosen": -52.22815704345703, + "logps/ref_rejected": -84.00656127929688, + "logps/rejected": -1576.1640625, + "loss": 1.3382, + "margin_dpo/margin_mean": 730.6165771484375, + "margin_dpo/margin_std": 1362.75, + "step": 488 + }, + { + "KL/chosen_KL_mean": -914.7315673828125, + "KL/mean": -1159.464599609375, + "KL/rejected_KL_mean": -1404.197509765625, + "KL/std": 1258.12744140625, + "epoch": 0.7180616740088106, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 489.4659423828125, + "fcm_dpo/q_t": 0.4914560914039612, + "grad_norm": 5.10060977935791, + "learning_rate": 1.1298035870396985e-07, + "logits/chosen": -0.553819477558136, + "logits/rejected": -0.5615823268890381, + "logps/chosen": -970.72119140625, + "logps/ref_chosen": -55.989627838134766, + "logps/ref_rejected": -79.39812469482422, + "logps/rejected": -1483.595703125, + "loss": 1.3549, + "margin_dpo/margin_mean": 489.4659118652344, + "margin_dpo/margin_std": 1443.1240234375, + "step": 489 + }, + { + "KL/chosen_KL_mean": -992.151123046875, + "KL/mean": -1396.121337890625, + "KL/rejected_KL_mean": -1800.091552734375, + "KL/std": 1577.330078125, + "epoch": 0.7195301027900147, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 807.9404296875, + "fcm_dpo/q_t": 0.4860500693321228, + "grad_norm": 6.243213653564453, + "learning_rate": 1.1190875675987355e-07, + "logits/chosen": -0.5025284290313721, + "logits/rejected": -0.5427592992782593, + "logps/chosen": -1044.517578125, + "logps/ref_chosen": -52.36639404296875, + "logps/ref_rejected": -110.4090576171875, + "logps/rejected": -1910.5006103515625, + "loss": 1.3358, + "margin_dpo/margin_mean": 807.9404296875, + "margin_dpo/margin_std": 1997.775146484375, + "step": 490 + }, + { + "KL/chosen_KL_mean": -961.5281982421875, + "KL/mean": -1136.3218994140625, + "KL/rejected_KL_mean": -1311.1156005859375, + "KL/std": 1014.553955078125, + "epoch": 0.7209985315712188, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 349.58746337890625, + "fcm_dpo/q_t": 0.4938778281211853, + "grad_norm": 3.7116713523864746, + "learning_rate": 1.1084079364846241e-07, + "logits/chosen": -0.5231382846832275, + "logits/rejected": -0.5342763662338257, + "logps/chosen": -1021.6444702148438, + "logps/ref_chosen": -60.11626434326172, + "logps/ref_rejected": -73.27278900146484, + "logps/rejected": -1384.388427734375, + "loss": 1.3634, + "margin_dpo/margin_mean": 349.58746337890625, + "margin_dpo/margin_std": 1089.209228515625, + "step": 491 + }, + { + "KL/chosen_KL_mean": -1040.841796875, + "KL/mean": -1144.0247802734375, + "KL/rejected_KL_mean": -1247.20751953125, + "KL/std": 1073.8876953125, + "epoch": 0.7224669603524229, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 206.36590576171875, + "fcm_dpo/q_t": 0.4963826835155487, + "grad_norm": 5.653712272644043, + "learning_rate": 1.097764975115576e-07, + "logits/chosen": -0.5174509286880493, + "logits/rejected": -0.5069364905357361, + "logps/chosen": -1094.8359375, + "logps/ref_chosen": -53.994178771972656, + "logps/ref_rejected": -72.65962219238281, + "logps/rejected": -1319.8671875, + "loss": 1.3731, + "margin_dpo/margin_mean": 206.36590576171875, + "margin_dpo/margin_std": 1005.8743896484375, + "step": 492 + }, + { + "KL/chosen_KL_mean": -1245.180908203125, + "KL/mean": -1378.21533203125, + "KL/rejected_KL_mean": -1511.2498779296875, + "KL/std": 1198.19384765625, + "epoch": 0.723935389133627, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 266.0688781738281, + "fcm_dpo/q_t": 0.49536770582199097, + "grad_norm": 9.439188003540039, + "learning_rate": 1.0871589639435203e-07, + "logits/chosen": -0.5633972883224487, + "logits/rejected": -0.5487905740737915, + "logps/chosen": -1320.67822265625, + "logps/ref_chosen": -75.49723815917969, + "logps/ref_rejected": -87.32301330566406, + "logps/rejected": -1598.5728759765625, + "loss": 1.3702, + "margin_dpo/margin_mean": 266.06890869140625, + "margin_dpo/margin_std": 1407.188232421875, + "step": 493 + }, + { + "KL/chosen_KL_mean": -743.1343994140625, + "KL/mean": -1110.82080078125, + "KL/rejected_KL_mean": -1478.5072021484375, + "KL/std": 1011.7152709960938, + "epoch": 0.7254038179148311, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 735.372802734375, + "fcm_dpo/q_t": 0.48713040351867676, + "grad_norm": 10.48877239227295, + "learning_rate": 1.0765901824467166e-07, + "logits/chosen": -0.4758908748626709, + "logits/rejected": -0.5146316289901733, + "logps/chosen": -784.49365234375, + "logps/ref_chosen": -41.35926818847656, + "logps/ref_rejected": -86.09136962890625, + "logps/rejected": -1564.5986328125, + "loss": 1.3374, + "margin_dpo/margin_mean": 735.372802734375, + "margin_dpo/margin_std": 1294.782470703125, + "step": 494 + }, + { + "KL/chosen_KL_mean": -1080.8409423828125, + "KL/mean": -1387.46923828125, + "KL/rejected_KL_mean": -1694.097412109375, + "KL/std": 1105.668212890625, + "epoch": 0.7268722466960352, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 613.2564086914062, + "fcm_dpo/q_t": 0.4892851710319519, + "grad_norm": 5.530251502990723, + "learning_rate": 1.0660589091223854e-07, + "logits/chosen": -0.6073266267776489, + "logits/rejected": -0.6195170879364014, + "logps/chosen": -1144.3759765625, + "logps/ref_chosen": -63.53507995605469, + "logps/ref_rejected": -91.42443084716797, + "logps/rejected": -1785.52197265625, + "loss": 1.347, + "margin_dpo/margin_mean": 613.25634765625, + "margin_dpo/margin_std": 1621.5955810546875, + "step": 495 + }, + { + "KL/chosen_KL_mean": -1282.886962890625, + "KL/mean": -1321.6571044921875, + "KL/rejected_KL_mean": -1360.427001953125, + "KL/std": 1002.799560546875, + "epoch": 0.7283406754772394, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 77.54009246826172, + "fcm_dpo/q_t": 0.49863314628601074, + "grad_norm": 8.960994720458984, + "learning_rate": 1.0555654214793722e-07, + "logits/chosen": -0.5475114583969116, + "logits/rejected": -0.5427240133285522, + "logps/chosen": -1355.4788818359375, + "logps/ref_chosen": -72.5919189453125, + "logps/ref_rejected": -84.32933807373047, + "logps/rejected": -1444.75634765625, + "loss": 1.3831, + "margin_dpo/margin_mean": 77.54010009765625, + "margin_dpo/margin_std": 1289.9385986328125, + "step": 496 + }, + { + "KL/chosen_KL_mean": -949.1077270507812, + "KL/mean": -1054.76904296875, + "KL/rejected_KL_mean": -1160.430419921875, + "KL/std": 835.3856201171875, + "epoch": 0.7298091042584435, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 211.32281494140625, + "fcm_dpo/q_t": 0.4962995648384094, + "grad_norm": 3.6305837631225586, + "learning_rate": 1.0451099960308374e-07, + "logits/chosen": -0.5167309045791626, + "logits/rejected": -0.5133849382400513, + "logps/chosen": -1007.70166015625, + "logps/ref_chosen": -58.59397506713867, + "logps/ref_rejected": -76.28836822509766, + "logps/rejected": -1236.71875, + "loss": 1.3724, + "margin_dpo/margin_mean": 211.32281494140625, + "margin_dpo/margin_std": 839.1533813476562, + "step": 497 + }, + { + "KL/chosen_KL_mean": -1179.71044921875, + "KL/mean": -1404.1376953125, + "KL/rejected_KL_mean": -1628.5648193359375, + "KL/std": 1239.938232421875, + "epoch": 0.7312775330396476, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 448.8544006347656, + "fcm_dpo/q_t": 0.4921446144580841, + "grad_norm": 4.796504974365234, + "learning_rate": 1.0346929082869641e-07, + "logits/chosen": -0.5920599102973938, + "logits/rejected": -0.5912704467773438, + "logps/chosen": -1250.916015625, + "logps/ref_chosen": -71.20565795898438, + "logps/ref_rejected": -83.95803833007812, + "logps/rejected": -1712.52294921875, + "loss": 1.3579, + "margin_dpo/margin_mean": 448.8544006347656, + "margin_dpo/margin_std": 1510.208740234375, + "step": 498 + }, + { + "KL/chosen_KL_mean": -847.0122680664062, + "KL/mean": -1311.192138671875, + "KL/rejected_KL_mean": -1775.3720703125, + "KL/std": 1260.3809814453125, + "epoch": 0.7327459618208517, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 928.3598022460938, + "fcm_dpo/q_t": 0.4837894141674042, + "grad_norm": 6.413288593292236, + "learning_rate": 1.0243144327477013e-07, + "logits/chosen": -0.5122471451759338, + "logits/rejected": -0.5531229972839355, + "logps/chosen": -898.2674560546875, + "logps/ref_chosen": -51.25519561767578, + "logps/ref_rejected": -101.07870483398438, + "logps/rejected": -1876.45068359375, + "loss": 1.3257, + "margin_dpo/margin_mean": 928.3598022460938, + "margin_dpo/margin_std": 1673.77978515625, + "step": 499 + }, + { + "KL/chosen_KL_mean": -1049.207763671875, + "KL/mean": -1407.1966552734375, + "KL/rejected_KL_mean": -1765.185546875, + "KL/std": 1214.736328125, + "epoch": 0.7342143906020558, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 715.9777221679688, + "fcm_dpo/q_t": 0.4875142276287079, + "grad_norm": 4.70263671875, + "learning_rate": 1.0139748428955333e-07, + "logits/chosen": -0.5440601706504822, + "logits/rejected": -0.5752770900726318, + "logps/chosen": -1106.2352294921875, + "logps/ref_chosen": -57.027442932128906, + "logps/ref_rejected": -93.93421173095703, + "logps/rejected": -1859.1197509765625, + "loss": 1.34, + "margin_dpo/margin_mean": 715.977783203125, + "margin_dpo/margin_std": 1645.666259765625, + "step": 500 + }, + { + "KL/chosen_KL_mean": -943.0932006835938, + "KL/mean": -1164.473876953125, + "KL/rejected_KL_mean": -1385.8544921875, + "KL/std": 1028.2890625, + "epoch": 0.73568281938326, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 442.7614440917969, + "fcm_dpo/q_t": 0.4922564625740051, + "grad_norm": 5.623405933380127, + "learning_rate": 1.0036744111882672e-07, + "logits/chosen": -0.5240232944488525, + "logits/rejected": -0.5279651880264282, + "logps/chosen": -997.4527587890625, + "logps/ref_chosen": -54.359527587890625, + "logps/ref_rejected": -80.15670013427734, + "logps/rejected": -1466.01123046875, + "loss": 1.3582, + "margin_dpo/margin_mean": 442.76141357421875, + "margin_dpo/margin_std": 1491.604248046875, + "step": 501 + }, + { + "KL/chosen_KL_mean": -986.6317749023438, + "KL/mean": -1168.455078125, + "KL/rejected_KL_mean": -1350.2783203125, + "KL/std": 1039.948486328125, + "epoch": 0.737151248164464, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 363.64642333984375, + "fcm_dpo/q_t": 0.4936283826828003, + "grad_norm": 8.514199256896973, + "learning_rate": 9.934134090518592e-08, + "logits/chosen": -0.5406728982925415, + "logits/rejected": -0.5402701497077942, + "logps/chosen": -1054.2322998046875, + "logps/ref_chosen": -67.60050964355469, + "logps/ref_rejected": -82.94876098632812, + "logps/rejected": -1433.22705078125, + "loss": 1.3624, + "margin_dpo/margin_mean": 363.64642333984375, + "margin_dpo/margin_std": 1048.6695556640625, + "step": 502 + }, + { + "KL/chosen_KL_mean": -824.9329833984375, + "KL/mean": -1109.140869140625, + "KL/rejected_KL_mean": -1393.3486328125, + "KL/std": 1051.9873046875, + "epoch": 0.7386196769456681, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 568.41552734375, + "fcm_dpo/q_t": 0.4900738596916199, + "grad_norm": 6.4620280265808105, + "learning_rate": 9.831921068732571e-08, + "logits/chosen": -0.516451358795166, + "logits/rejected": -0.528926432132721, + "logps/chosen": -880.0114135742188, + "logps/ref_chosen": -55.078407287597656, + "logps/ref_rejected": -82.50544738769531, + "logps/rejected": -1475.85400390625, + "loss": 1.3488, + "margin_dpo/margin_mean": 568.41552734375, + "margin_dpo/margin_std": 1233.030029296875, + "step": 503 + }, + { + "KL/chosen_KL_mean": -967.4698486328125, + "KL/mean": -1384.71337890625, + "KL/rejected_KL_mean": -1801.9569091796875, + "KL/std": 1324.2451171875, + "epoch": 0.7400881057268722, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 834.4871826171875, + "fcm_dpo/q_t": 0.485454797744751, + "grad_norm": 8.587855339050293, + "learning_rate": 9.730107739932805e-08, + "logits/chosen": -0.5549697279930115, + "logits/rejected": -0.5928115844726562, + "logps/chosen": -1027.435546875, + "logps/ref_chosen": -59.96575164794922, + "logps/ref_rejected": -103.76212310791016, + "logps/rejected": -1905.718994140625, + "loss": 1.332, + "margin_dpo/margin_mean": 834.4871826171875, + "margin_dpo/margin_std": 1630.097900390625, + "step": 504 + }, + { + "KL/chosen_KL_mean": -1171.11376953125, + "KL/mean": -1224.62890625, + "KL/rejected_KL_mean": -1278.1439208984375, + "KL/std": 1174.8758544921875, + "epoch": 0.7415565345080763, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 107.03019714355469, + "fcm_dpo/q_t": 0.4981127381324768, + "grad_norm": 7.053234577178955, + "learning_rate": 9.628696786995188e-08, + "logits/chosen": -0.5928906202316284, + "logits/rejected": -0.5843832492828369, + "logps/chosen": -1247.2685546875, + "logps/ref_chosen": -76.1549072265625, + "logps/ref_rejected": -88.58537292480469, + "logps/rejected": -1366.729248046875, + "loss": 1.3813, + "margin_dpo/margin_mean": 107.03020477294922, + "margin_dpo/margin_std": 1291.50732421875, + "step": 505 + }, + { + "KL/chosen_KL_mean": -677.691650390625, + "KL/mean": -1025.074462890625, + "KL/rejected_KL_mean": -1372.457275390625, + "KL/std": 931.436767578125, + "epoch": 0.7430249632892805, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 694.765625, + "fcm_dpo/q_t": 0.48783206939697266, + "grad_norm": 6.811020851135254, + "learning_rate": 9.527690882192635e-08, + "logits/chosen": -0.561811089515686, + "logits/rejected": -0.5848267078399658, + "logps/chosen": -726.6522216796875, + "logps/ref_chosen": -48.96050262451172, + "logps/ref_rejected": -78.41505432128906, + "logps/rejected": -1450.872314453125, + "loss": 1.3395, + "margin_dpo/margin_mean": 694.765625, + "margin_dpo/margin_std": 1041.9476318359375, + "step": 506 + }, + { + "KL/chosen_KL_mean": -971.807373046875, + "KL/mean": -1220.935302734375, + "KL/rejected_KL_mean": -1470.0631103515625, + "KL/std": 1240.0423583984375, + "epoch": 0.7444933920704846, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 498.2557067871094, + "fcm_dpo/q_t": 0.49129223823547363, + "grad_norm": 5.5018086433410645, + "learning_rate": 9.427092687124691e-08, + "logits/chosen": -0.5913007259368896, + "logits/rejected": -0.6078677177429199, + "logps/chosen": -1038.60888671875, + "logps/ref_chosen": -66.80149841308594, + "logps/ref_rejected": -95.37289428710938, + "logps/rejected": -1565.43603515625, + "loss": 1.354, + "margin_dpo/margin_mean": 498.2557373046875, + "margin_dpo/margin_std": 1392.36083984375, + "step": 507 + }, + { + "KL/chosen_KL_mean": -939.2349853515625, + "KL/mean": -1200.439697265625, + "KL/rejected_KL_mean": -1461.644287109375, + "KL/std": 1131.05126953125, + "epoch": 0.7459618208516887, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 522.409423828125, + "fcm_dpo/q_t": 0.4908856451511383, + "grad_norm": 3.9309823513031006, + "learning_rate": 9.326904852647344e-08, + "logits/chosen": -0.5682249665260315, + "logits/rejected": -0.5859044194221497, + "logps/chosen": -1010.5384521484375, + "logps/ref_chosen": -71.303466796875, + "logps/ref_rejected": -95.6275405883789, + "logps/rejected": -1557.2718505859375, + "loss": 1.3528, + "margin_dpo/margin_mean": 522.409423828125, + "margin_dpo/margin_std": 1477.3380126953125, + "step": 508 + }, + { + "KL/chosen_KL_mean": -770.8653564453125, + "KL/mean": -939.4214477539062, + "KL/rejected_KL_mean": -1107.9775390625, + "KL/std": 911.1104736328125, + "epoch": 0.7474302496328928, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 337.11224365234375, + "fcm_dpo/q_t": 0.49410343170166016, + "grad_norm": 5.916812419891357, + "learning_rate": 9.227130018803195e-08, + "logits/chosen": -0.5424858927726746, + "logits/rejected": -0.5421060919761658, + "logps/chosen": -834.684326171875, + "logps/ref_chosen": -63.81895065307617, + "logps/ref_rejected": -83.25643920898438, + "logps/rejected": -1191.2340087890625, + "loss": 1.3649, + "margin_dpo/margin_mean": 337.1122131347656, + "margin_dpo/margin_std": 1224.0833740234375, + "step": 509 + }, + { + "KL/chosen_KL_mean": -931.734130859375, + "KL/mean": -1244.052978515625, + "KL/rejected_KL_mean": -1556.371826171875, + "KL/std": 978.7109375, + "epoch": 0.748898678414097, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 624.6377563476562, + "fcm_dpo/q_t": 0.4890550374984741, + "grad_norm": 4.738297462463379, + "learning_rate": 9.127770814751932e-08, + "logits/chosen": -0.5328891277313232, + "logits/rejected": -0.5605103969573975, + "logps/chosen": -983.612548828125, + "logps/ref_chosen": -51.878448486328125, + "logps/ref_rejected": -102.7651596069336, + "logps/rejected": -1659.1370849609375, + "loss": 1.344, + "margin_dpo/margin_mean": 624.6378173828125, + "margin_dpo/margin_std": 936.5046997070312, + "step": 510 + }, + { + "KL/chosen_KL_mean": -913.4978637695312, + "KL/mean": -1169.09619140625, + "KL/rejected_KL_mean": -1424.6943359375, + "KL/std": 1147.4625244140625, + "epoch": 0.750367107195301, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 511.196533203125, + "fcm_dpo/q_t": 0.4910575747489929, + "grad_norm": 6.8262619972229, + "learning_rate": 9.028829858700973e-08, + "logits/chosen": -0.5740972757339478, + "logits/rejected": -0.5858516693115234, + "logps/chosen": -973.7359619140625, + "logps/ref_chosen": -60.23811721801758, + "logps/ref_rejected": -92.85676574707031, + "logps/rejected": -1517.5511474609375, + "loss": 1.3535, + "margin_dpo/margin_mean": 511.1965026855469, + "margin_dpo/margin_std": 1480.135009765625, + "step": 511 + }, + { + "KL/chosen_KL_mean": -768.2742919921875, + "KL/mean": -1137.4937744140625, + "KL/rejected_KL_mean": -1506.713134765625, + "KL/std": 1220.0850830078125, + "epoch": 0.7518355359765051, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 738.43896484375, + "fcm_dpo/q_t": 0.4870929718017578, + "grad_norm": 6.630341529846191, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": -0.5527737140655518, + "logits/rejected": -0.5937384366989136, + "logps/chosen": -823.1798095703125, + "logps/ref_chosen": -54.905494689941406, + "logps/ref_rejected": -81.87586975097656, + "logps/rejected": -1588.589111328125, + "loss": 1.3376, + "margin_dpo/margin_mean": 738.43896484375, + "margin_dpo/margin_std": 1411.79248046875, + "step": 512 + }, + { + "KL/chosen_KL_mean": -1048.791015625, + "KL/mean": -1195.49755859375, + "KL/rejected_KL_mean": -1342.2042236328125, + "KL/std": 1093.606689453125, + "epoch": 0.7533039647577092, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 293.4132080078125, + "fcm_dpo/q_t": 0.49488991498947144, + "grad_norm": 9.842702865600586, + "learning_rate": 8.832213108254863e-08, + "logits/chosen": -0.567176342010498, + "logits/rejected": -0.552276611328125, + "logps/chosen": -1113.70751953125, + "logps/ref_chosen": -64.91644287109375, + "logps/ref_rejected": -76.06245422363281, + "logps/rejected": -1418.2666015625, + "loss": 1.3686, + "margin_dpo/margin_mean": 293.4131774902344, + "margin_dpo/margin_std": 1416.5491943359375, + "step": 513 + }, + { + "KL/chosen_KL_mean": -1270.703369140625, + "KL/mean": -1452.780517578125, + "KL/rejected_KL_mean": -1634.857666015625, + "KL/std": 1571.225341796875, + "epoch": 0.7547723935389133, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 364.1544189453125, + "fcm_dpo/q_t": 0.4936813712120056, + "grad_norm": 8.145087242126465, + "learning_rate": 8.734542494893954e-08, + "logits/chosen": -0.6005032658576965, + "logits/rejected": -0.6026064157485962, + "logps/chosen": -1344.932861328125, + "logps/ref_chosen": -74.22957611083984, + "logps/ref_rejected": -78.945556640625, + "logps/rejected": -1713.80322265625, + "loss": 1.3656, + "margin_dpo/margin_mean": 364.1544494628906, + "margin_dpo/margin_std": 1899.677001953125, + "step": 514 + }, + { + "KL/chosen_KL_mean": -822.61181640625, + "KL/mean": -1065.546142578125, + "KL/rejected_KL_mean": -1308.4805908203125, + "KL/std": 936.01123046875, + "epoch": 0.7562408223201175, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 485.86871337890625, + "fcm_dpo/q_t": 0.4914933145046234, + "grad_norm": 14.613423347473145, + "learning_rate": 8.637300491465272e-08, + "logits/chosen": -0.5557729601860046, + "logits/rejected": -0.5711244940757751, + "logps/chosen": -873.013427734375, + "logps/ref_chosen": -50.40156555175781, + "logps/ref_rejected": -87.09774780273438, + "logps/rejected": -1395.578369140625, + "loss": 1.3542, + "margin_dpo/margin_mean": 485.8686828613281, + "margin_dpo/margin_std": 1161.409423828125, + "step": 515 + }, + { + "KL/chosen_KL_mean": -1072.080810546875, + "KL/mean": -1407.988037109375, + "KL/rejected_KL_mean": -1743.8955078125, + "KL/std": 1599.38037109375, + "epoch": 0.7577092511013216, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 671.8146362304688, + "fcm_dpo/q_t": 0.4883235692977905, + "grad_norm": 7.461482048034668, + "learning_rate": 8.540489660386064e-08, + "logits/chosen": -0.5883182287216187, + "logits/rejected": -0.621609091758728, + "logps/chosen": -1136.7303466796875, + "logps/ref_chosen": -64.64956665039062, + "logps/ref_rejected": -111.72237396240234, + "logps/rejected": -1855.6177978515625, + "loss": 1.3438, + "margin_dpo/margin_mean": 671.8145751953125, + "margin_dpo/margin_std": 1814.15576171875, + "step": 516 + }, + { + "KL/chosen_KL_mean": -1061.081787109375, + "KL/mean": -1346.005859375, + "KL/rejected_KL_mean": -1630.9300537109375, + "KL/std": 1233.93896484375, + "epoch": 0.7591776798825257, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 569.8483276367188, + "fcm_dpo/q_t": 0.4900466501712799, + "grad_norm": 7.388690948486328, + "learning_rate": 8.444112552711752e-08, + "logits/chosen": -0.5805338621139526, + "logits/rejected": -0.5963911414146423, + "logps/chosen": -1121.995361328125, + "logps/ref_chosen": -60.913551330566406, + "logps/ref_rejected": -89.08308410644531, + "logps/rejected": -1720.01318359375, + "loss": 1.3499, + "margin_dpo/margin_mean": 569.8483276367188, + "margin_dpo/margin_std": 1582.470947265625, + "step": 517 + }, + { + "KL/chosen_KL_mean": -1056.78125, + "KL/mean": -1327.392578125, + "KL/rejected_KL_mean": -1598.0040283203125, + "KL/std": 1047.9013671875, + "epoch": 0.7606461086637298, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 541.22265625, + "fcm_dpo/q_t": 0.49052339792251587, + "grad_norm": 4.970123291015625, + "learning_rate": 8.348171708068747e-08, + "logits/chosen": -0.6027194261550903, + "logits/rejected": -0.6282007694244385, + "logps/chosen": -1114.2371826171875, + "logps/ref_chosen": -57.45589065551758, + "logps/ref_rejected": -85.31269836425781, + "logps/rejected": -1683.316650390625, + "loss": 1.3508, + "margin_dpo/margin_mean": 541.22265625, + "margin_dpo/margin_std": 1296.53173828125, + "step": 518 + }, + { + "KL/chosen_KL_mean": -1016.2708740234375, + "KL/mean": -1329.120849609375, + "KL/rejected_KL_mean": -1641.970947265625, + "KL/std": 1342.2520751953125, + "epoch": 0.762114537444934, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 625.7000732421875, + "fcm_dpo/q_t": 0.4891040325164795, + "grad_norm": 7.316686630249023, + "learning_rate": 8.25266965458755e-08, + "logits/chosen": -0.5834439992904663, + "logits/rejected": -0.5853796005249023, + "logps/chosen": -1090.334228515625, + "logps/ref_chosen": -74.06331634521484, + "logps/ref_rejected": -104.44416809082031, + "logps/rejected": -1746.415283203125, + "loss": 1.347, + "margin_dpo/margin_mean": 625.7000732421875, + "margin_dpo/margin_std": 1800.982421875, + "step": 519 + }, + { + "KL/chosen_KL_mean": -1117.6549072265625, + "KL/mean": -1392.603759765625, + "KL/rejected_KL_mean": -1667.552490234375, + "KL/std": 1104.5682373046875, + "epoch": 0.7635829662261381, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 549.8974609375, + "fcm_dpo/q_t": 0.49038708209991455, + "grad_norm": 6.708014965057373, + "learning_rate": 8.15760890883607e-08, + "logits/chosen": -0.5891440510749817, + "logits/rejected": -0.6028249859809875, + "logps/chosen": -1187.9547119140625, + "logps/ref_chosen": -70.2998275756836, + "logps/ref_rejected": -99.98133850097656, + "logps/rejected": -1767.533935546875, + "loss": 1.3504, + "margin_dpo/margin_mean": 549.8974609375, + "margin_dpo/margin_std": 1362.424560546875, + "step": 520 + }, + { + "KL/chosen_KL_mean": -902.4603271484375, + "KL/mean": -1152.8935546875, + "KL/rejected_KL_mean": -1403.32666015625, + "KL/std": 1028.9302978515625, + "epoch": 0.7650513950073421, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 500.8663330078125, + "fcm_dpo/q_t": 0.4912322163581848, + "grad_norm": 4.412991523742676, + "learning_rate": 8.062991975753378e-08, + "logits/chosen": -0.5892634391784668, + "logits/rejected": -0.5929391980171204, + "logps/chosen": -960.603271484375, + "logps/ref_chosen": -58.14292526245117, + "logps/ref_rejected": -83.28060913085938, + "logps/rejected": -1486.6072998046875, + "loss": 1.3533, + "margin_dpo/margin_mean": 500.8663330078125, + "margin_dpo/margin_std": 1217.1341552734375, + "step": 521 + }, + { + "KL/chosen_KL_mean": -1055.103271484375, + "KL/mean": -1362.3404541015625, + "KL/rejected_KL_mean": -1669.57763671875, + "KL/std": 1621.5078125, + "epoch": 0.7665198237885462, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 614.4742431640625, + "fcm_dpo/q_t": 0.4893651604652405, + "grad_norm": 4.380655288696289, + "learning_rate": 7.968821348583643e-08, + "logits/chosen": -0.5628671646118164, + "logits/rejected": -0.5865719318389893, + "logps/chosen": -1101.6510009765625, + "logps/ref_chosen": -46.54766845703125, + "logps/ref_rejected": -66.01388549804688, + "logps/rejected": -1735.591552734375, + "loss": 1.3489, + "margin_dpo/margin_mean": 614.4742431640625, + "margin_dpo/margin_std": 2025.41748046875, + "step": 522 + }, + { + "KL/chosen_KL_mean": -1028.6043701171875, + "KL/mean": -1341.892578125, + "KL/rejected_KL_mean": -1655.180908203125, + "KL/std": 1353.0596923828125, + "epoch": 0.7679882525697503, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 626.5765380859375, + "fcm_dpo/q_t": 0.48908570408821106, + "grad_norm": 7.474643707275391, + "learning_rate": 7.875099508810484e-08, + "logits/chosen": -0.6015632152557373, + "logits/rejected": -0.6170424222946167, + "logps/chosen": -1090.3740234375, + "logps/ref_chosen": -61.76960372924805, + "logps/ref_rejected": -83.76141357421875, + "logps/rejected": -1738.9423828125, + "loss": 1.346, + "margin_dpo/margin_mean": 626.5764770507812, + "margin_dpo/margin_std": 1597.18505859375, + "step": 523 + }, + { + "KL/chosen_KL_mean": -1175.5543212890625, + "KL/mean": -1509.3448486328125, + "KL/rejected_KL_mean": -1843.1353759765625, + "KL/std": 1454.0643310546875, + "epoch": 0.7694566813509545, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 667.5810546875, + "fcm_dpo/q_t": 0.48826104402542114, + "grad_norm": 5.199390411376953, + "learning_rate": 7.781828926091535e-08, + "logits/chosen": -0.6230287551879883, + "logits/rejected": -0.6203910112380981, + "logps/chosen": -1253.6263427734375, + "logps/ref_chosen": -78.0720443725586, + "logps/ref_rejected": -81.30198669433594, + "logps/rejected": -1924.4373779296875, + "loss": 1.3434, + "margin_dpo/margin_mean": 667.5810546875, + "margin_dpo/margin_std": 1561.3837890625, + "step": 524 + }, + { + "KL/chosen_KL_mean": -979.4048461914062, + "KL/mean": -1361.4544677734375, + "KL/rejected_KL_mean": -1743.50390625, + "KL/std": 1232.875732421875, + "epoch": 0.7709251101321586, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 764.09912109375, + "fcm_dpo/q_t": 0.4866584539413452, + "grad_norm": 7.510035037994385, + "learning_rate": 7.689012058193384e-08, + "logits/chosen": -0.5819898843765259, + "logits/rejected": -0.6207159757614136, + "logps/chosen": -1030.232666015625, + "logps/ref_chosen": -50.827857971191406, + "logps/ref_rejected": -100.05294036865234, + "logps/rejected": -1843.556884765625, + "loss": 1.3363, + "margin_dpo/margin_mean": 764.09912109375, + "margin_dpo/margin_std": 1525.293212890625, + "step": 525 + }, + { + "KL/chosen_KL_mean": -1006.8646240234375, + "KL/mean": -1335.72021484375, + "KL/rejected_KL_mean": -1664.5760498046875, + "KL/std": 1145.358642578125, + "epoch": 0.7723935389133627, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 657.71142578125, + "fcm_dpo/q_t": 0.4884929656982422, + "grad_norm": 4.733247756958008, + "learning_rate": 7.596651350926836e-08, + "logits/chosen": -0.6167298555374146, + "logits/rejected": -0.6341676712036133, + "logps/chosen": -1070.0318603515625, + "logps/ref_chosen": -63.167236328125, + "logps/ref_rejected": -86.30934143066406, + "logps/rejected": -1750.8853759765625, + "loss": 1.3427, + "margin_dpo/margin_mean": 657.71142578125, + "margin_dpo/margin_std": 1287.364990234375, + "step": 526 + }, + { + "KL/chosen_KL_mean": -1146.7838134765625, + "KL/mean": -1294.6519775390625, + "KL/rejected_KL_mean": -1442.520263671875, + "KL/std": 1127.4002685546875, + "epoch": 0.7738619676945668, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 295.73638916015625, + "fcm_dpo/q_t": 0.49481672048568726, + "grad_norm": 8.143030166625977, + "learning_rate": 7.504749238082414e-08, + "logits/chosen": -0.6452538967132568, + "logits/rejected": -0.6307415962219238, + "logps/chosen": -1217.91259765625, + "logps/ref_chosen": -71.12867736816406, + "logps/ref_rejected": -78.3425521850586, + "logps/rejected": -1520.86279296875, + "loss": 1.3671, + "margin_dpo/margin_mean": 295.73638916015625, + "margin_dpo/margin_std": 1076.11376953125, + "step": 527 + }, + { + "KL/chosen_KL_mean": -924.047607421875, + "KL/mean": -1256.195556640625, + "KL/rejected_KL_mean": -1588.343505859375, + "KL/std": 1196.41650390625, + "epoch": 0.775330396475771, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 664.2958374023438, + "fcm_dpo/q_t": 0.4884028434753418, + "grad_norm": 4.170558929443359, + "learning_rate": 7.413308141366254e-08, + "logits/chosen": -0.6017875671386719, + "logits/rejected": -0.6084692478179932, + "logps/chosen": -992.1370849609375, + "logps/ref_chosen": -68.0894546508789, + "logps/ref_rejected": -93.91006469726562, + "logps/rejected": -1682.2535400390625, + "loss": 1.3432, + "margin_dpo/margin_mean": 664.2958374023438, + "margin_dpo/margin_std": 1553.2027587890625, + "step": 528 + }, + { + "KL/chosen_KL_mean": -1159.14599609375, + "KL/mean": -1451.514404296875, + "KL/rejected_KL_mean": -1743.8826904296875, + "KL/std": 1228.73193359375, + "epoch": 0.7767988252569751, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 584.7366333007812, + "fcm_dpo/q_t": 0.48979973793029785, + "grad_norm": 4.807751655578613, + "learning_rate": 7.322330470336313e-08, + "logits/chosen": -0.5786510109901428, + "logits/rejected": -0.5997955799102783, + "logps/chosen": -1214.720947265625, + "logps/ref_chosen": -55.57495880126953, + "logps/ref_rejected": -89.20909118652344, + "logps/rejected": -1833.091796875, + "loss": 1.3491, + "margin_dpo/margin_mean": 584.7366333007812, + "margin_dpo/margin_std": 1670.16748046875, + "step": 529 + }, + { + "KL/chosen_KL_mean": -882.88427734375, + "KL/mean": -1237.404296875, + "KL/rejected_KL_mean": -1591.9244384765625, + "KL/std": 1404.204345703125, + "epoch": 0.7782672540381792, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 709.0401000976562, + "fcm_dpo/q_t": 0.4876710772514343, + "grad_norm": 8.733548164367676, + "learning_rate": 7.231818622338822e-08, + "logits/chosen": -0.5622015595436096, + "logits/rejected": -0.5846884250640869, + "logps/chosen": -930.4857177734375, + "logps/ref_chosen": -47.601417541503906, + "logps/ref_rejected": -87.2845230102539, + "logps/rejected": -1679.208984375, + "loss": 1.3413, + "margin_dpo/margin_mean": 709.0401611328125, + "margin_dpo/margin_std": 1824.845458984375, + "step": 530 + }, + { + "KL/chosen_KL_mean": -859.3704833984375, + "KL/mean": -1154.43994140625, + "KL/rejected_KL_mean": -1449.509521484375, + "KL/std": 1112.5634765625, + "epoch": 0.7797356828193832, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 590.1390380859375, + "fcm_dpo/q_t": 0.48967117071151733, + "grad_norm": 5.344662189483643, + "learning_rate": 7.141774982445147e-08, + "logits/chosen": -0.5867226123809814, + "logits/rejected": -0.6062394380569458, + "logps/chosen": -914.6165161132812, + "logps/ref_chosen": -55.246063232421875, + "logps/ref_rejected": -70.60598754882812, + "logps/rejected": -1520.115478515625, + "loss": 1.3468, + "margin_dpo/margin_mean": 590.1390380859375, + "margin_dpo/margin_std": 1108.8994140625, + "step": 531 + }, + { + "KL/chosen_KL_mean": -1094.8494873046875, + "KL/mean": -1391.7200927734375, + "KL/rejected_KL_mean": -1688.590576171875, + "KL/std": 1260.90576171875, + "epoch": 0.7812041116005873, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 593.7410278320312, + "fcm_dpo/q_t": 0.4896160364151001, + "grad_norm": 7.872839450836182, + "learning_rate": 7.052201923388953e-08, + "logits/chosen": -0.5879281759262085, + "logits/rejected": -0.5886451005935669, + "logps/chosen": -1165.135498046875, + "logps/ref_chosen": -70.28601837158203, + "logps/ref_rejected": -86.5913314819336, + "logps/rejected": -1775.181884765625, + "loss": 1.3489, + "margin_dpo/margin_mean": 593.740966796875, + "margin_dpo/margin_std": 1750.036376953125, + "step": 532 + }, + { + "KL/chosen_KL_mean": -933.3600463867188, + "KL/mean": -1131.0, + "KL/rejected_KL_mean": -1328.639892578125, + "KL/std": 966.4365844726562, + "epoch": 0.7826725403817915, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 395.27978515625, + "fcm_dpo/q_t": 0.4930812120437622, + "grad_norm": 7.988112449645996, + "learning_rate": 6.963101805503646e-08, + "logits/chosen": -0.5576300024986267, + "logits/rejected": -0.5628528594970703, + "logps/chosen": -998.2151489257812, + "logps/ref_chosen": -64.8551025390625, + "logps/ref_rejected": -76.58805847167969, + "logps/rejected": -1405.2279052734375, + "loss": 1.3604, + "margin_dpo/margin_mean": 395.27978515625, + "margin_dpo/margin_std": 1123.464599609375, + "step": 533 + }, + { + "KL/chosen_KL_mean": -992.2314453125, + "KL/mean": -1203.13671875, + "KL/rejected_KL_mean": -1414.042236328125, + "KL/std": 950.9990234375, + "epoch": 0.7841409691629956, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 421.810791015625, + "fcm_dpo/q_t": 0.49261125922203064, + "grad_norm": 3.841683864593506, + "learning_rate": 6.874476976660184e-08, + "logits/chosen": -0.5702451467514038, + "logits/rejected": -0.5771702527999878, + "logps/chosen": -1052.350830078125, + "logps/ref_chosen": -60.119388580322266, + "logps/ref_rejected": -78.54347229003906, + "logps/rejected": -1492.585693359375, + "loss": 1.3584, + "margin_dpo/margin_mean": 421.810791015625, + "margin_dpo/margin_std": 1089.174560546875, + "step": 534 + }, + { + "KL/chosen_KL_mean": -888.7866821289062, + "KL/mean": -1202.163330078125, + "KL/rejected_KL_mean": -1515.5400390625, + "KL/std": 1040.59130859375, + "epoch": 0.7856093979441997, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 626.75341796875, + "fcm_dpo/q_t": 0.48904871940612793, + "grad_norm": 8.149191856384277, + "learning_rate": 6.786329772205246e-08, + "logits/chosen": -0.552294135093689, + "logits/rejected": -0.5709767937660217, + "logps/chosen": -943.1168823242188, + "logps/ref_chosen": -54.330238342285156, + "logps/ref_rejected": -96.30763244628906, + "logps/rejected": -1611.84765625, + "loss": 1.345, + "margin_dpo/margin_mean": 626.75341796875, + "margin_dpo/margin_std": 1275.3328857421875, + "step": 535 + }, + { + "KL/chosen_KL_mean": -841.0042724609375, + "KL/mean": -1279.9410400390625, + "KL/rejected_KL_mean": -1718.8778076171875, + "KL/std": 1255.70849609375, + "epoch": 0.7870778267254038, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 877.8734130859375, + "fcm_dpo/q_t": 0.4847047030925751, + "grad_norm": 5.253050327301025, + "learning_rate": 6.698662514899638e-08, + "logits/chosen": -0.5551550388336182, + "logits/rejected": -0.5932704210281372, + "logps/chosen": -888.0848388671875, + "logps/ref_chosen": -47.08053207397461, + "logps/ref_rejected": -89.09783935546875, + "logps/rejected": -1807.9755859375, + "loss": 1.3289, + "margin_dpo/margin_mean": 877.8734741210938, + "margin_dpo/margin_std": 1561.20703125, + "step": 536 + }, + { + "KL/chosen_KL_mean": -840.8751831054688, + "KL/mean": -1096.5126953125, + "KL/rejected_KL_mean": -1352.150146484375, + "KL/std": 1151.702880859375, + "epoch": 0.788546255506608, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 511.2747802734375, + "fcm_dpo/q_t": 0.49105796217918396, + "grad_norm": 7.912595748901367, + "learning_rate": 6.611477514857114e-08, + "logits/chosen": -0.5522580742835999, + "logits/rejected": -0.5589759945869446, + "logps/chosen": -898.6226806640625, + "logps/ref_chosen": -57.747467041015625, + "logps/ref_rejected": -70.43838500976562, + "logps/rejected": -1422.58837890625, + "loss": 1.3537, + "margin_dpo/margin_mean": 511.2747802734375, + "margin_dpo/margin_std": 1513.6673583984375, + "step": 537 + }, + { + "KL/chosen_KL_mean": -1396.2117919921875, + "KL/mean": -1632.014404296875, + "KL/rejected_KL_mean": -1867.817138671875, + "KL/std": 1259.4403076171875, + "epoch": 0.7900146842878121, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 471.6053466796875, + "fcm_dpo/q_t": 0.49175089597702026, + "grad_norm": 9.711124420166016, + "learning_rate": 6.524777069483525e-08, + "logits/chosen": -0.5915812253952026, + "logits/rejected": -0.5872704982757568, + "logps/chosen": -1462.627685546875, + "logps/ref_chosen": -66.41594696044922, + "logps/ref_rejected": -84.22808837890625, + "logps/rejected": -1952.045166015625, + "loss": 1.3565, + "margin_dpo/margin_mean": 471.6053466796875, + "margin_dpo/margin_std": 1497.396484375, + "step": 538 + }, + { + "KL/chosen_KL_mean": -1097.3935546875, + "KL/mean": -1476.907958984375, + "KL/rejected_KL_mean": -1856.42236328125, + "KL/std": 1343.3634033203125, + "epoch": 0.7914831130690162, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 759.0288696289062, + "fcm_dpo/q_t": 0.48674440383911133, + "grad_norm": 5.35037899017334, + "learning_rate": 6.438563463416221e-08, + "logits/chosen": -0.6155024766921997, + "logits/rejected": -0.6297394633293152, + "logps/chosen": -1155.886474609375, + "logps/ref_chosen": -58.492855072021484, + "logps/ref_rejected": -91.85395050048828, + "logps/rejected": -1948.2763671875, + "loss": 1.3363, + "margin_dpo/margin_mean": 759.0289306640625, + "margin_dpo/margin_std": 1419.5430908203125, + "step": 539 + }, + { + "KL/chosen_KL_mean": -1084.130615234375, + "KL/mean": -1591.833740234375, + "KL/rejected_KL_mean": -2099.53662109375, + "KL/std": 1494.089111328125, + "epoch": 0.7929515418502202, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1015.4059448242188, + "fcm_dpo/q_t": 0.48234492540359497, + "grad_norm": 17.94775390625, + "learning_rate": 6.352838968463919e-08, + "logits/chosen": -0.5842588543891907, + "logits/rejected": -0.6213306188583374, + "logps/chosen": -1147.61328125, + "logps/ref_chosen": -63.482513427734375, + "logps/ref_rejected": -116.42999267578125, + "logps/rejected": -2215.966796875, + "loss": 1.3212, + "margin_dpo/margin_mean": 1015.4058837890625, + "margin_dpo/margin_std": 1876.0716552734375, + "step": 540 + }, + { + "KL/chosen_KL_mean": -1515.512939453125, + "KL/mean": -1609.35546875, + "KL/rejected_KL_mean": -1703.19775390625, + "KL/std": 1322.71044921875, + "epoch": 0.7944199706314243, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 187.684814453125, + "fcm_dpo/q_t": 0.49672752618789673, + "grad_norm": 9.675416946411133, + "learning_rate": 6.267605843546767e-08, + "logits/chosen": -0.6445415019989014, + "logits/rejected": -0.6379104852676392, + "logps/chosen": -1593.7933349609375, + "logps/ref_chosen": -78.28036499023438, + "logps/ref_rejected": -103.273681640625, + "logps/rejected": -1806.471435546875, + "loss": 1.3765, + "margin_dpo/margin_mean": 187.68482971191406, + "margin_dpo/margin_std": 1610.0963134765625, + "step": 541 + }, + { + "KL/chosen_KL_mean": -1154.2373046875, + "KL/mean": -1602.501220703125, + "KL/rejected_KL_mean": -2050.76513671875, + "KL/std": 1615.48046875, + "epoch": 0.7958883994126285, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 896.5280151367188, + "fcm_dpo/q_t": 0.48452144861221313, + "grad_norm": 4.6800994873046875, + "learning_rate": 6.182866334636888e-08, + "logits/chosen": -0.6557344198226929, + "logits/rejected": -0.6881058216094971, + "logps/chosen": -1211.72216796875, + "logps/ref_chosen": -57.48497009277344, + "logps/ref_rejected": -96.47506713867188, + "logps/rejected": -2147.240234375, + "loss": 1.33, + "margin_dpo/margin_mean": 896.5280151367188, + "margin_dpo/margin_std": 2071.083740234375, + "step": 542 + }, + { + "KL/chosen_KL_mean": -1310.2457275390625, + "KL/mean": -1533.172119140625, + "KL/rejected_KL_mean": -1756.0986328125, + "KL/std": 1376.15087890625, + "epoch": 0.7973568281938326, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 445.85296630859375, + "fcm_dpo/q_t": 0.4922771453857422, + "grad_norm": 4.305510520935059, + "learning_rate": 6.098622674699147e-08, + "logits/chosen": -0.664671778678894, + "logits/rejected": -0.6874226331710815, + "logps/chosen": -1370.86328125, + "logps/ref_chosen": -60.61750793457031, + "logps/ref_rejected": -105.59896850585938, + "logps/rejected": -1861.6976318359375, + "loss": 1.36, + "margin_dpo/margin_mean": 445.8529357910156, + "margin_dpo/margin_std": 1953.89453125, + "step": 543 + }, + { + "KL/chosen_KL_mean": -1286.40478515625, + "KL/mean": -1731.963134765625, + "KL/rejected_KL_mean": -2177.521484375, + "KL/std": 1765.681396484375, + "epoch": 0.7988252569750367, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 891.1168212890625, + "fcm_dpo/q_t": 0.48461323976516724, + "grad_norm": 8.807916641235352, + "learning_rate": 6.01487708363232e-08, + "logits/chosen": -0.6092058420181274, + "logits/rejected": -0.6435132026672363, + "logps/chosen": -1346.047119140625, + "logps/ref_chosen": -59.642303466796875, + "logps/ref_rejected": -100.95469665527344, + "logps/rejected": -2278.476318359375, + "loss": 1.3311, + "margin_dpo/margin_mean": 891.1168212890625, + "margin_dpo/margin_std": 2284.972900390625, + "step": 544 + }, + { + "KL/chosen_KL_mean": -1260.89306640625, + "KL/mean": -1646.8466796875, + "KL/rejected_KL_mean": -2032.800537109375, + "KL/std": 1534.07568359375, + "epoch": 0.8002936857562408, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 771.9075927734375, + "fcm_dpo/q_t": 0.48656225204467773, + "grad_norm": 6.247243881225586, + "learning_rate": 5.9316317682106294e-08, + "logits/chosen": -0.5921393632888794, + "logits/rejected": -0.6117866635322571, + "logps/chosen": -1328.54150390625, + "logps/ref_chosen": -67.64859771728516, + "logps/ref_rejected": -95.90800476074219, + "logps/rejected": -2128.70849609375, + "loss": 1.337, + "margin_dpo/margin_mean": 771.9075317382812, + "margin_dpo/margin_std": 1806.5426025390625, + "step": 545 + }, + { + "KL/chosen_KL_mean": -1186.041748046875, + "KL/mean": -1404.183837890625, + "KL/rejected_KL_mean": -1622.325927734375, + "KL/std": 1201.185546875, + "epoch": 0.801762114537445, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 436.28411865234375, + "fcm_dpo/q_t": 0.4923797845840454, + "grad_norm": 12.886591911315918, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": -0.6253443360328674, + "logits/rejected": -0.6243206262588501, + "logps/chosen": -1236.785888671875, + "logps/ref_chosen": -50.744232177734375, + "logps/ref_rejected": -81.86622619628906, + "logps/rejected": -1704.192138671875, + "loss": 1.3586, + "margin_dpo/margin_mean": 436.28411865234375, + "margin_dpo/margin_std": 1439.834716796875, + "step": 546 + }, + { + "KL/chosen_KL_mean": -1329.843505859375, + "KL/mean": -1714.03662109375, + "KL/rejected_KL_mean": -2098.2294921875, + "KL/std": 1930.3846435546875, + "epoch": 0.8032305433186491, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 768.38623046875, + "fcm_dpo/q_t": 0.4867956340312958, + "grad_norm": 9.838577270507812, + "learning_rate": 5.7666507254280265e-08, + "logits/chosen": -0.5816187858581543, + "logits/rejected": -0.6018866300582886, + "logps/chosen": -1403.53125, + "logps/ref_chosen": -73.6877212524414, + "logps/ref_rejected": -90.76136779785156, + "logps/rejected": -2188.990966796875, + "loss": 1.3413, + "margin_dpo/margin_mean": 768.38623046875, + "margin_dpo/margin_std": 2552.30029296875, + "step": 547 + }, + { + "KL/chosen_KL_mean": -1230.2508544921875, + "KL/mean": -1500.3779296875, + "KL/rejected_KL_mean": -1770.5048828125, + "KL/std": 1448.751708984375, + "epoch": 0.8046989720998532, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 540.2540283203125, + "fcm_dpo/q_t": 0.49062401056289673, + "grad_norm": 4.971753120422363, + "learning_rate": 5.684919345471029e-08, + "logits/chosen": -0.6555124521255493, + "logits/rejected": -0.6675156950950623, + "logps/chosen": -1295.4971923828125, + "logps/ref_chosen": -65.24634552001953, + "logps/ref_rejected": -94.11807250976562, + "logps/rejected": -1864.6229248046875, + "loss": 1.3527, + "margin_dpo/margin_mean": 540.2540283203125, + "margin_dpo/margin_std": 1749.078125, + "step": 548 + }, + { + "KL/chosen_KL_mean": -1231.84814453125, + "KL/mean": -1339.867431640625, + "KL/rejected_KL_mean": -1447.8865966796875, + "KL/std": 1256.0994873046875, + "epoch": 0.8061674008810573, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 216.03839111328125, + "fcm_dpo/q_t": 0.4962120056152344, + "grad_norm": 6.037284851074219, + "learning_rate": 5.603696935852426e-08, + "logits/chosen": -0.5892428159713745, + "logits/rejected": -0.5814424753189087, + "logps/chosen": -1281.060546875, + "logps/ref_chosen": -49.21235656738281, + "logps/ref_rejected": -73.91031646728516, + "logps/rejected": -1521.796875, + "loss": 1.3742, + "margin_dpo/margin_mean": 216.0384063720703, + "margin_dpo/margin_std": 1532.143310546875, + "step": 549 + }, + { + "KL/chosen_KL_mean": -1138.1700439453125, + "KL/mean": -1449.984375, + "KL/rejected_KL_mean": -1761.798828125, + "KL/std": 1296.947998046875, + "epoch": 0.8076358296622613, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 623.6287841796875, + "fcm_dpo/q_t": 0.489163339138031, + "grad_norm": 4.7602033615112305, + "learning_rate": 5.5229856368582376e-08, + "logits/chosen": -0.6239445209503174, + "logits/rejected": -0.6451071500778198, + "logps/chosen": -1194.97705078125, + "logps/ref_chosen": -56.80695343017578, + "logps/ref_rejected": -95.12580871582031, + "logps/rejected": -1856.924560546875, + "loss": 1.3463, + "margin_dpo/margin_mean": 623.6287841796875, + "margin_dpo/margin_std": 1583.240478515625, + "step": 550 + }, + { + "KL/chosen_KL_mean": -903.1275634765625, + "KL/mean": -1493.087158203125, + "KL/rejected_KL_mean": -2083.04638671875, + "KL/std": 1501.976806640625, + "epoch": 0.8091042584434655, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1179.9189453125, + "fcm_dpo/q_t": 0.4793858528137207, + "grad_norm": 11.45347785949707, + "learning_rate": 5.4427875753062734e-08, + "logits/chosen": -0.5236402750015259, + "logits/rejected": -0.5718011260032654, + "logps/chosen": -962.23388671875, + "logps/ref_chosen": -59.10633087158203, + "logps/ref_rejected": -111.67280578613281, + "logps/rejected": -2194.71923828125, + "loss": 1.3087, + "margin_dpo/margin_mean": 1179.9189453125, + "margin_dpo/margin_std": 1684.580322265625, + "step": 551 + }, + { + "KL/chosen_KL_mean": -930.559326171875, + "KL/mean": -1511.77001953125, + "KL/rejected_KL_mean": -2092.98046875, + "KL/std": 1509.347412109375, + "epoch": 0.8105726872246696, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1162.4212646484375, + "fcm_dpo/q_t": 0.4797826111316681, + "grad_norm": 5.244424343109131, + "learning_rate": 5.363104864490034e-08, + "logits/chosen": -0.5785896182060242, + "logits/rejected": -0.6230120658874512, + "logps/chosen": -992.9139404296875, + "logps/ref_chosen": -62.35459899902344, + "logps/ref_rejected": -104.56210327148438, + "logps/rejected": -2197.54248046875, + "loss": 1.3114, + "margin_dpo/margin_mean": 1162.421142578125, + "margin_dpo/margin_std": 1976.2950439453125, + "step": 552 + }, + { + "KL/chosen_KL_mean": -1078.3341064453125, + "KL/mean": -1394.103515625, + "KL/rejected_KL_mean": -1709.872802734375, + "KL/std": 1449.83984375, + "epoch": 0.8120411160058737, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 631.5387573242188, + "fcm_dpo/q_t": 0.4890480637550354, + "grad_norm": 3.988219976425171, + "learning_rate": 5.2839396041230415e-08, + "logits/chosen": -0.62043297290802, + "logits/rejected": -0.6310804486274719, + "logps/chosen": -1146.5928955078125, + "logps/ref_chosen": -68.25881958007812, + "logps/ref_rejected": -98.0971450805664, + "logps/rejected": -1807.969970703125, + "loss": 1.3467, + "margin_dpo/margin_mean": 631.5386962890625, + "margin_dpo/margin_std": 1802.6300048828125, + "step": 553 + }, + { + "KL/chosen_KL_mean": -1069.6845703125, + "KL/mean": -1364.756103515625, + "KL/rejected_KL_mean": -1659.8275146484375, + "KL/std": 1380.947998046875, + "epoch": 0.8135095447870778, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 590.1428833007812, + "fcm_dpo/q_t": 0.4896678328514099, + "grad_norm": 8.758122444152832, + "learning_rate": 5.205293880283551e-08, + "logits/chosen": -0.5634171962738037, + "logits/rejected": -0.5632882714271545, + "logps/chosen": -1137.63232421875, + "logps/ref_chosen": -67.94767761230469, + "logps/ref_rejected": -89.78272247314453, + "logps/rejected": -1749.6102294921875, + "loss": 1.35, + "margin_dpo/margin_mean": 590.1428833007812, + "margin_dpo/margin_std": 1906.44287109375, + "step": 554 + }, + { + "KL/chosen_KL_mean": -1021.588623046875, + "KL/mean": -1599.221923828125, + "KL/rejected_KL_mean": -2176.855224609375, + "KL/std": 1767.3104248046875, + "epoch": 0.8149779735682819, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1155.2666015625, + "fcm_dpo/q_t": 0.48013854026794434, + "grad_norm": 10.302849769592285, + "learning_rate": 5.127169765359515e-08, + "logits/chosen": -0.6012399792671204, + "logits/rejected": -0.6592134237289429, + "logps/chosen": -1074.919189453125, + "logps/ref_chosen": -53.33049011230469, + "logps/ref_rejected": -108.47937774658203, + "logps/rejected": -2285.33447265625, + "loss": 1.3141, + "margin_dpo/margin_mean": 1155.2666015625, + "margin_dpo/margin_std": 2393.09033203125, + "step": 555 + }, + { + "KL/chosen_KL_mean": -982.6744384765625, + "KL/mean": -1277.244384765625, + "KL/rejected_KL_mean": -1571.814208984375, + "KL/std": 1051.68212890625, + "epoch": 0.8164464023494861, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 589.1397705078125, + "fcm_dpo/q_t": 0.4896823763847351, + "grad_norm": 13.509299278259277, + "learning_rate": 5.049569317994012e-08, + "logits/chosen": -0.615922212600708, + "logits/rejected": -0.6318573355674744, + "logps/chosen": -1041.31884765625, + "logps/ref_chosen": -58.64447021484375, + "logps/ref_rejected": -101.34040832519531, + "logps/rejected": -1673.1546630859375, + "loss": 1.3468, + "margin_dpo/margin_mean": 589.1397705078125, + "margin_dpo/margin_std": 1059.956787109375, + "step": 556 + }, + { + "KL/chosen_KL_mean": -1247.9884033203125, + "KL/mean": -1629.21337890625, + "KL/rejected_KL_mean": -2010.438232421875, + "KL/std": 1569.9039306640625, + "epoch": 0.8179148311306902, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 762.44970703125, + "fcm_dpo/q_t": 0.4867032766342163, + "grad_norm": 9.238645553588867, + "learning_rate": 4.9724945830310144e-08, + "logits/chosen": -0.6624920964241028, + "logits/rejected": -0.6939408779144287, + "logps/chosen": -1315.8291015625, + "logps/ref_chosen": -67.84066009521484, + "logps/ref_rejected": -109.93965911865234, + "logps/rejected": -2120.3779296875, + "loss": 1.3377, + "margin_dpo/margin_mean": 762.44970703125, + "margin_dpo/margin_std": 1803.6630859375, + "step": 557 + }, + { + "KL/chosen_KL_mean": -1106.59033203125, + "KL/mean": -1585.306396484375, + "KL/rejected_KL_mean": -2064.022216796875, + "KL/std": 1658.90380859375, + "epoch": 0.8193832599118943, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 957.43212890625, + "fcm_dpo/q_t": 0.48335278034210205, + "grad_norm": 7.449563980102539, + "learning_rate": 4.8959475914614554e-08, + "logits/chosen": -0.6731094717979431, + "logits/rejected": -0.7122669219970703, + "logps/chosen": -1168.9586181640625, + "logps/ref_chosen": -62.36824035644531, + "logps/ref_rejected": -102.16102600097656, + "logps/rejected": -2166.183349609375, + "loss": 1.3256, + "margin_dpo/margin_mean": 957.4320678710938, + "margin_dpo/margin_std": 2066.8369140625, + "step": 558 + }, + { + "KL/chosen_KL_mean": -1256.0625, + "KL/mean": -1648.341064453125, + "KL/rejected_KL_mean": -2040.619873046875, + "KL/std": 1527.5263671875, + "epoch": 0.8208516886930984, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 784.557373046875, + "fcm_dpo/q_t": 0.4863905608654022, + "grad_norm": 9.646005630493164, + "learning_rate": 4.8199303603697614e-08, + "logits/chosen": -0.70490962266922, + "logits/rejected": -0.7145035266876221, + "logps/chosen": -1316.814697265625, + "logps/ref_chosen": -60.752323150634766, + "logps/ref_rejected": -93.44229125976562, + "logps/rejected": -2134.06201171875, + "loss": 1.3373, + "margin_dpo/margin_mean": 784.557373046875, + "margin_dpo/margin_std": 2029.985595703125, + "step": 559 + }, + { + "KL/chosen_KL_mean": -1072.104736328125, + "KL/mean": -1330.7318115234375, + "KL/rejected_KL_mean": -1589.35888671875, + "KL/std": 1190.2357177734375, + "epoch": 0.8223201174743024, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 517.25390625, + "fcm_dpo/q_t": 0.4909706115722656, + "grad_norm": 8.734450340270996, + "learning_rate": 4.7444448928806615e-08, + "logits/chosen": -0.5731639862060547, + "logits/rejected": -0.5752190351486206, + "logps/chosen": -1130.2086181640625, + "logps/ref_chosen": -58.10382080078125, + "logps/ref_rejected": -79.99122619628906, + "logps/rejected": -1669.35009765625, + "loss": 1.3529, + "margin_dpo/margin_mean": 517.2539672851562, + "margin_dpo/margin_std": 1448.2685546875, + "step": 560 + }, + { + "KL/chosen_KL_mean": -1208.382568359375, + "KL/mean": -1487.0732421875, + "KL/rejected_KL_mean": -1765.763916015625, + "KL/std": 1353.732421875, + "epoch": 0.8237885462555066, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 557.3814697265625, + "fcm_dpo/q_t": 0.49026691913604736, + "grad_norm": 5.121155738830566, + "learning_rate": 4.669493178106432e-08, + "logits/chosen": -0.6740288734436035, + "logits/rejected": -0.6940807104110718, + "logps/chosen": -1259.29541015625, + "logps/ref_chosen": -50.912879943847656, + "logps/ref_rejected": -99.06856536865234, + "logps/rejected": -1864.83251953125, + "loss": 1.3516, + "margin_dpo/margin_mean": 557.3814697265625, + "margin_dpo/margin_std": 1794.720458984375, + "step": 561 + }, + { + "KL/chosen_KL_mean": -977.68701171875, + "KL/mean": -1403.8623046875, + "KL/rejected_KL_mean": -1830.037841796875, + "KL/std": 1357.09423828125, + "epoch": 0.8252569750367107, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 852.3507080078125, + "fcm_dpo/q_t": 0.48517775535583496, + "grad_norm": 6.560747146606445, + "learning_rate": 4.5950771910944596e-08, + "logits/chosen": -0.6164703965187073, + "logits/rejected": -0.6432167291641235, + "logps/chosen": -1037.1513671875, + "logps/ref_chosen": -59.46440124511719, + "logps/ref_rejected": -96.54266357421875, + "logps/rejected": -1926.580322265625, + "loss": 1.331, + "margin_dpo/margin_mean": 852.3507690429688, + "margin_dpo/margin_std": 1607.605712890625, + "step": 562 + }, + { + "KL/chosen_KL_mean": -1360.0205078125, + "KL/mean": -1654.7742919921875, + "KL/rejected_KL_mean": -1949.5281982421875, + "KL/std": 1659.429931640625, + "epoch": 0.8267254038179148, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 589.5076904296875, + "fcm_dpo/q_t": 0.4897218942642212, + "grad_norm": 8.3922700881958, + "learning_rate": 4.521198892775202e-08, + "logits/chosen": -0.6155869960784912, + "logits/rejected": -0.6347240209579468, + "logps/chosen": -1420.628662109375, + "logps/ref_chosen": -60.60819625854492, + "logps/ref_rejected": -94.56770324707031, + "logps/rejected": -2044.095947265625, + "loss": 1.3502, + "margin_dpo/margin_mean": 589.5076904296875, + "margin_dpo/margin_std": 1928.743408203125, + "step": 563 + }, + { + "KL/chosen_KL_mean": -1113.524658203125, + "KL/mean": -1359.48486328125, + "KL/rejected_KL_mean": -1605.445068359375, + "KL/std": 1332.8668212890625, + "epoch": 0.8281938325991189, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 491.92047119140625, + "fcm_dpo/q_t": 0.49136167764663696, + "grad_norm": 7.608941555023193, + "learning_rate": 4.447860229910544e-08, + "logits/chosen": -0.6324988603591919, + "logits/rejected": -0.6346863508224487, + "logps/chosen": -1187.79296875, + "logps/ref_chosen": -74.26837921142578, + "logps/ref_rejected": -93.23818969726562, + "logps/rejected": -1698.6832275390625, + "loss": 1.3556, + "margin_dpo/margin_mean": 491.9204406738281, + "margin_dpo/margin_std": 1603.78125, + "step": 564 + }, + { + "KL/chosen_KL_mean": -1407.1187744140625, + "KL/mean": -1665.321044921875, + "KL/rejected_KL_mean": -1923.5233154296875, + "KL/std": 1837.506103515625, + "epoch": 0.8296622613803231, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 516.404541015625, + "fcm_dpo/q_t": 0.4908890128135681, + "grad_norm": 13.449864387512207, + "learning_rate": 4.375063135042445e-08, + "logits/chosen": -0.663166880607605, + "logits/rejected": -0.6663168668746948, + "logps/chosen": -1476.138671875, + "logps/ref_chosen": -69.0199203491211, + "logps/ref_rejected": -85.7789306640625, + "logps/rejected": -2009.30224609375, + "loss": 1.3573, + "margin_dpo/margin_mean": 516.404541015625, + "margin_dpo/margin_std": 2329.1201171875, + "step": 565 + }, + { + "KL/chosen_KL_mean": -1349.890869140625, + "KL/mean": -1728.78564453125, + "KL/rejected_KL_mean": -2107.68017578125, + "KL/std": 1578.119873046875, + "epoch": 0.8311306901615272, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 757.7894287109375, + "fcm_dpo/q_t": 0.48675039410591125, + "grad_norm": 5.23419713973999, + "learning_rate": 4.3028095264420525e-08, + "logits/chosen": -0.6609238386154175, + "logits/rejected": -0.6840362548828125, + "logps/chosen": -1416.436279296875, + "logps/ref_chosen": -66.5453109741211, + "logps/ref_rejected": -103.86932373046875, + "logps/rejected": -2211.5498046875, + "loss": 1.3386, + "margin_dpo/margin_mean": 757.7894287109375, + "margin_dpo/margin_std": 1948.74609375, + "step": 566 + }, + { + "KL/chosen_KL_mean": -1095.9215087890625, + "KL/mean": -1309.902099609375, + "KL/rejected_KL_mean": -1523.882568359375, + "KL/std": 1136.513427734375, + "epoch": 0.8325991189427313, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 427.9610595703125, + "fcm_dpo/q_t": 0.4924955368041992, + "grad_norm": 6.573452472686768, + "learning_rate": 4.231101308059165e-08, + "logits/chosen": -0.6667437553405762, + "logits/rejected": -0.676659107208252, + "logps/chosen": -1148.77978515625, + "logps/ref_chosen": -52.85829544067383, + "logps/ref_rejected": -85.37095642089844, + "logps/rejected": -1609.25341796875, + "loss": 1.3586, + "margin_dpo/margin_mean": 427.9610595703125, + "margin_dpo/margin_std": 1303.414306640625, + "step": 567 + }, + { + "KL/chosen_KL_mean": -958.6199951171875, + "KL/mean": -1419.5540771484375, + "KL/rejected_KL_mean": -1880.4881591796875, + "KL/std": 1383.7803955078125, + "epoch": 0.8340675477239354, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 921.8681030273438, + "fcm_dpo/q_t": 0.4839458167552948, + "grad_norm": 5.986705780029297, + "learning_rate": 4.1599403694720145e-08, + "logits/chosen": -0.6187951564788818, + "logits/rejected": -0.6642179489135742, + "logps/chosen": -1003.8123779296875, + "logps/ref_chosen": -45.1923828125, + "logps/ref_rejected": -89.09236907958984, + "logps/rejected": -1969.58056640625, + "loss": 1.3265, + "margin_dpo/margin_mean": 921.8680419921875, + "margin_dpo/margin_std": 1768.663818359375, + "step": 568 + }, + { + "KL/chosen_KL_mean": -1272.758056640625, + "KL/mean": -1551.21484375, + "KL/rejected_KL_mean": -1829.671630859375, + "KL/std": 1590.348876953125, + "epoch": 0.8355359765051396, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 556.9136962890625, + "fcm_dpo/q_t": 0.4902852773666382, + "grad_norm": 6.188879013061523, + "learning_rate": 4.089328585837512e-08, + "logits/chosen": -0.6311490535736084, + "logits/rejected": -0.6400599479675293, + "logps/chosen": -1336.478515625, + "logps/ref_chosen": -63.72056198120117, + "logps/ref_rejected": -79.10325622558594, + "logps/rejected": -1908.77490234375, + "loss": 1.3522, + "margin_dpo/margin_mean": 556.9136962890625, + "margin_dpo/margin_std": 1879.6124267578125, + "step": 569 + }, + { + "KL/chosen_KL_mean": -1346.568603515625, + "KL/mean": -1499.462890625, + "KL/rejected_KL_mean": -1652.357177734375, + "KL/std": 1435.96875, + "epoch": 0.8370044052863436, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 305.7886962890625, + "fcm_dpo/q_t": 0.4945737421512604, + "grad_norm": 8.600349426269531, + "learning_rate": 4.019267817841834e-08, + "logits/chosen": -0.6977522373199463, + "logits/rejected": -0.6893934011459351, + "logps/chosen": -1408.18310546875, + "logps/ref_chosen": -61.61454391479492, + "logps/ref_rejected": -82.14186096191406, + "logps/rejected": -1734.4991455078125, + "loss": 1.3696, + "margin_dpo/margin_mean": 305.7887268066406, + "margin_dpo/margin_std": 1899.91796875, + "step": 570 + }, + { + "KL/chosen_KL_mean": -1334.66357421875, + "KL/mean": -1602.7347412109375, + "KL/rejected_KL_mean": -1870.8057861328125, + "KL/std": 1609.7418212890625, + "epoch": 0.8384728340675477, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 536.1421508789062, + "fcm_dpo/q_t": 0.49062836170196533, + "grad_norm": 9.382889747619629, + "learning_rate": 3.9497599116513705e-08, + "logits/chosen": -0.6136207580566406, + "logits/rejected": -0.6315397024154663, + "logps/chosen": -1387.7177734375, + "logps/ref_chosen": -53.05406188964844, + "logps/ref_rejected": -91.33682250976562, + "logps/rejected": -1962.142578125, + "loss": 1.3535, + "margin_dpo/margin_mean": 536.1421508789062, + "margin_dpo/margin_std": 1897.035400390625, + "step": 571 + }, + { + "KL/chosen_KL_mean": -1041.9559326171875, + "KL/mean": -1387.749267578125, + "KL/rejected_KL_mean": -1733.54248046875, + "KL/std": 1231.0006103515625, + "epoch": 0.8399412628487518, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 691.5866088867188, + "fcm_dpo/q_t": 0.48791706562042236, + "grad_norm": 5.244479179382324, + "learning_rate": 3.880806698864086e-08, + "logits/chosen": -0.6141128540039062, + "logits/rejected": -0.6465499401092529, + "logps/chosen": -1090.415283203125, + "logps/ref_chosen": -48.45928955078125, + "logps/ref_rejected": -83.55703735351562, + "logps/rejected": -1817.099609375, + "loss": 1.3413, + "margin_dpo/margin_mean": 691.5865478515625, + "margin_dpo/margin_std": 1555.69189453125, + "step": 572 + }, + { + "KL/chosen_KL_mean": -1082.886474609375, + "KL/mean": -1360.33203125, + "KL/rejected_KL_mean": -1637.777587890625, + "KL/std": 1212.4599609375, + "epoch": 0.8414096916299559, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 554.8909912109375, + "fcm_dpo/q_t": 0.49030232429504395, + "grad_norm": 9.327940940856934, + "learning_rate": 3.812409996461275e-08, + "logits/chosen": -0.606751561164856, + "logits/rejected": -0.6213085651397705, + "logps/chosen": -1134.5091552734375, + "logps/ref_chosen": -51.62262725830078, + "logps/ref_rejected": -85.32499694824219, + "logps/rejected": -1723.1025390625, + "loss": 1.3498, + "margin_dpo/margin_mean": 554.8909912109375, + "margin_dpo/margin_std": 1286.421142578125, + "step": 573 + }, + { + "KL/chosen_KL_mean": -1081.611328125, + "KL/mean": -1515.553466796875, + "KL/rejected_KL_mean": -1949.4954833984375, + "KL/std": 1357.4376220703125, + "epoch": 0.8428781204111601, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 867.8842163085938, + "fcm_dpo/q_t": 0.4848443865776062, + "grad_norm": 5.942359447479248, + "learning_rate": 3.74457160675965e-08, + "logits/chosen": -0.6815818548202515, + "logits/rejected": -0.7281221151351929, + "logps/chosen": -1132.65576171875, + "logps/ref_chosen": -51.04446029663086, + "logps/ref_rejected": -92.80640411376953, + "logps/rejected": -2042.302001953125, + "loss": 1.3294, + "margin_dpo/margin_mean": 867.8841552734375, + "margin_dpo/margin_std": 1571.0694580078125, + "step": 574 + }, + { + "KL/chosen_KL_mean": -1305.66552734375, + "KL/mean": -1599.9190673828125, + "KL/rejected_KL_mean": -1894.1724853515625, + "KL/std": 1470.6298828125, + "epoch": 0.8443465491923642, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 588.5068359375, + "fcm_dpo/q_t": 0.48974353075027466, + "grad_norm": 7.068485736846924, + "learning_rate": 3.677293317363864e-08, + "logits/chosen": -0.6297034025192261, + "logits/rejected": -0.6442345380783081, + "logps/chosen": -1377.455810546875, + "logps/ref_chosen": -71.7901382446289, + "logps/ref_rejected": -95.38619995117188, + "logps/rejected": -1989.55859375, + "loss": 1.3495, + "margin_dpo/margin_mean": 588.5068359375, + "margin_dpo/margin_std": 1797.72509765625, + "step": 575 + }, + { + "KL/chosen_KL_mean": -1039.58740234375, + "KL/mean": -1412.387451171875, + "KL/rejected_KL_mean": -1785.1873779296875, + "KL/std": 1391.4742431640625, + "epoch": 0.8458149779735683, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 745.5999755859375, + "fcm_dpo/q_t": 0.48703137040138245, + "grad_norm": 12.236254692077637, + "learning_rate": 3.6105769011194224e-08, + "logits/chosen": -0.6339189410209656, + "logits/rejected": -0.6675552129745483, + "logps/chosen": -1093.850341796875, + "logps/ref_chosen": -54.262962341308594, + "logps/ref_rejected": -100.75428009033203, + "logps/rejected": -1885.941650390625, + "loss": 1.3381, + "margin_dpo/margin_mean": 745.5999755859375, + "margin_dpo/margin_std": 1672.704345703125, + "step": 576 + }, + { + "KL/chosen_KL_mean": -1099.926513671875, + "KL/mean": -1301.64794921875, + "KL/rejected_KL_mean": -1503.369384765625, + "KL/std": 1184.22705078125, + "epoch": 0.8472834067547724, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 403.4429626464844, + "fcm_dpo/q_t": 0.49293655157089233, + "grad_norm": 6.366522312164307, + "learning_rate": 3.5444241160659304e-08, + "logits/chosen": -0.6079691648483276, + "logits/rejected": -0.6053318381309509, + "logps/chosen": -1161.836181640625, + "logps/ref_chosen": -61.909706115722656, + "logps/ref_rejected": -84.07069396972656, + "logps/rejected": -1587.440185546875, + "loss": 1.3608, + "margin_dpo/margin_mean": 403.4429626464844, + "margin_dpo/margin_std": 1407.972412109375, + "step": 577 + }, + { + "KL/chosen_KL_mean": -874.846923828125, + "KL/mean": -1265.8436279296875, + "KL/rejected_KL_mean": -1656.84033203125, + "KL/std": 1145.15771484375, + "epoch": 0.8487518355359766, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 781.993408203125, + "fcm_dpo/q_t": 0.48631131649017334, + "grad_norm": 4.333878517150879, + "learning_rate": 3.478836705390808e-08, + "logits/chosen": -0.5466019511222839, + "logits/rejected": -0.5783486366271973, + "logps/chosen": -924.110595703125, + "logps/ref_chosen": -49.26368713378906, + "logps/ref_rejected": -83.4362564086914, + "logps/rejected": -1740.276611328125, + "loss": 1.3339, + "margin_dpo/margin_mean": 781.993408203125, + "margin_dpo/margin_std": 1137.68408203125, + "step": 578 + }, + { + "KL/chosen_KL_mean": -1144.806396484375, + "KL/mean": -1350.88134765625, + "KL/rejected_KL_mean": -1556.956298828125, + "KL/std": 1468.2509765625, + "epoch": 0.8502202643171806, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 412.1499938964844, + "fcm_dpo/q_t": 0.49288052320480347, + "grad_norm": 6.493415355682373, + "learning_rate": 3.41381639738331e-08, + "logits/chosen": -0.609187662601471, + "logits/rejected": -0.6248157024383545, + "logps/chosen": -1203.6922607421875, + "logps/ref_chosen": -58.88581848144531, + "logps/ref_rejected": -94.78762817382812, + "logps/rejected": -1651.743896484375, + "loss": 1.362, + "margin_dpo/margin_mean": 412.14996337890625, + "margin_dpo/margin_std": 1850.548095703125, + "step": 579 + }, + { + "KL/chosen_KL_mean": -775.547119140625, + "KL/mean": -1131.787109375, + "KL/rejected_KL_mean": -1488.02685546875, + "KL/std": 1250.650390625, + "epoch": 0.8516886930983847, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 712.479736328125, + "fcm_dpo/q_t": 0.48757392168045044, + "grad_norm": 5.055511951446533, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": -0.5566902160644531, + "logits/rejected": -0.5791162848472595, + "logps/chosen": -824.2539672851562, + "logps/ref_chosen": -48.70683670043945, + "logps/ref_rejected": -81.7583999633789, + "logps/rejected": -1569.7852783203125, + "loss": 1.3396, + "margin_dpo/margin_mean": 712.479736328125, + "margin_dpo/margin_std": 1466.5977783203125, + "step": 580 + }, + { + "KL/chosen_KL_mean": -1139.18310546875, + "KL/mean": -1376.47412109375, + "KL/rejected_KL_mean": -1613.764892578125, + "KL/std": 1217.5745849609375, + "epoch": 0.8531571218795888, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 474.5818786621094, + "fcm_dpo/q_t": 0.4917064607143402, + "grad_norm": 6.296034812927246, + "learning_rate": 3.285483927764726e-08, + "logits/chosen": -0.6595457196235657, + "logits/rejected": -0.6783765554428101, + "logps/chosen": -1201.4053955078125, + "logps/ref_chosen": -62.22235107421875, + "logps/ref_rejected": -91.73568725585938, + "logps/rejected": -1705.500732421875, + "loss": 1.3559, + "margin_dpo/margin_mean": 474.5818786621094, + "margin_dpo/margin_std": 1438.438232421875, + "step": 581 + }, + { + "KL/chosen_KL_mean": -1091.2376708984375, + "KL/mean": -1487.45654296875, + "KL/rejected_KL_mean": -1883.6754150390625, + "KL/std": 1237.882080078125, + "epoch": 0.8546255506607929, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 792.4378051757812, + "fcm_dpo/q_t": 0.4861498475074768, + "grad_norm": 11.559684753417969, + "learning_rate": 3.222175147833556e-08, + "logits/chosen": -0.6552462577819824, + "logits/rejected": -0.6873466968536377, + "logps/chosen": -1149.46630859375, + "logps/ref_chosen": -58.228660583496094, + "logps/ref_rejected": -110.06959533691406, + "logps/rejected": -1993.7449951171875, + "loss": 1.3342, + "margin_dpo/margin_mean": 792.437744140625, + "margin_dpo/margin_std": 1396.3656005859375, + "step": 582 + }, + { + "KL/chosen_KL_mean": -1086.141357421875, + "KL/mean": -1331.1463623046875, + "KL/rejected_KL_mean": -1576.1513671875, + "KL/std": 1458.034423828125, + "epoch": 0.856093979441997, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 490.01007080078125, + "fcm_dpo/q_t": 0.4915131628513336, + "grad_norm": 12.745737075805664, + "learning_rate": 3.159440233840763e-08, + "logits/chosen": -0.6324259042739868, + "logits/rejected": -0.6466102600097656, + "logps/chosen": -1143.004150390625, + "logps/ref_chosen": -56.86286163330078, + "logps/ref_rejected": -88.4039306640625, + "logps/rejected": -1664.5552978515625, + "loss": 1.3569, + "margin_dpo/margin_mean": 490.0100402832031, + "margin_dpo/margin_std": 1838.203125, + "step": 583 + }, + { + "KL/chosen_KL_mean": -1118.363037109375, + "KL/mean": -1506.4857177734375, + "KL/rejected_KL_mean": -1894.6083984375, + "KL/std": 1478.693359375, + "epoch": 0.8575624082232012, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 776.2452392578125, + "fcm_dpo/q_t": 0.4864642024040222, + "grad_norm": 16.514495849609375, + "learning_rate": 3.0972808389096635e-08, + "logits/chosen": -0.6495020389556885, + "logits/rejected": -0.6715532541275024, + "logps/chosen": -1175.2637939453125, + "logps/ref_chosen": -56.90068054199219, + "logps/ref_rejected": -97.63606262207031, + "logps/rejected": -1992.244384765625, + "loss": 1.3358, + "margin_dpo/margin_mean": 776.2452392578125, + "margin_dpo/margin_std": 1611.7119140625, + "step": 584 + }, + { + "KL/chosen_KL_mean": -1159.185546875, + "KL/mean": -1560.73681640625, + "KL/rejected_KL_mean": -1962.2880859375, + "KL/std": 1544.73486328125, + "epoch": 0.8590308370044053, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 803.1026611328125, + "fcm_dpo/q_t": 0.4860289692878723, + "grad_norm": 4.359808444976807, + "learning_rate": 3.035698600998121e-08, + "logits/chosen": -0.6542025804519653, + "logits/rejected": -0.6867420077323914, + "logps/chosen": -1220.159423828125, + "logps/ref_chosen": -60.973968505859375, + "logps/ref_rejected": -84.16952514648438, + "logps/rejected": -2046.4576416015625, + "loss": 1.3362, + "margin_dpo/margin_mean": 803.1026611328125, + "margin_dpo/margin_std": 2070.01416015625, + "step": 585 + }, + { + "KL/chosen_KL_mean": -1311.316162109375, + "KL/mean": -1604.206787109375, + "KL/rejected_KL_mean": -1897.097412109375, + "KL/std": 1670.98046875, + "epoch": 0.8604992657856094, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 585.7813720703125, + "fcm_dpo/q_t": 0.4897732436656952, + "grad_norm": 6.482140064239502, + "learning_rate": 2.974695142855388e-08, + "logits/chosen": -0.6228867173194885, + "logits/rejected": -0.6541084051132202, + "logps/chosen": -1368.171630859375, + "logps/ref_chosen": -56.85559844970703, + "logps/ref_rejected": -91.80261993408203, + "logps/rejected": -1988.9000244140625, + "loss": 1.3486, + "margin_dpo/margin_mean": 585.7813720703125, + "margin_dpo/margin_std": 1524.811767578125, + "step": 586 + }, + { + "KL/chosen_KL_mean": -844.4599609375, + "KL/mean": -1170.924560546875, + "KL/rejected_KL_mean": -1497.38916015625, + "KL/std": 1240.437255859375, + "epoch": 0.8619676945668135, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 652.92919921875, + "fcm_dpo/q_t": 0.48856592178344727, + "grad_norm": 9.642579078674316, + "learning_rate": 2.9142720719793122e-08, + "logits/chosen": -0.6443250775337219, + "logits/rejected": -0.6658428907394409, + "logps/chosen": -889.151611328125, + "logps/ref_chosen": -44.69159698486328, + "logps/ref_rejected": -82.62385559082031, + "logps/rejected": -1580.0130615234375, + "loss": 1.3437, + "margin_dpo/margin_mean": 652.92919921875, + "margin_dpo/margin_std": 1477.093994140625, + "step": 587 + }, + { + "KL/chosen_KL_mean": -1141.727294921875, + "KL/mean": -1513.7247314453125, + "KL/rejected_KL_mean": -1885.722412109375, + "KL/std": 1343.573486328125, + "epoch": 0.8634361233480177, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 743.9951171875, + "fcm_dpo/q_t": 0.48701196908950806, + "grad_norm": 7.112242221832275, + "learning_rate": 2.8544309805740018e-08, + "logits/chosen": -0.6173974275588989, + "logits/rejected": -0.6490628719329834, + "logps/chosen": -1192.022216796875, + "logps/ref_chosen": -50.29494857788086, + "logps/ref_rejected": -107.36988067626953, + "logps/rejected": -1993.0921630859375, + "loss": 1.3384, + "margin_dpo/margin_mean": 743.9951171875, + "margin_dpo/margin_std": 1684.069091796875, + "step": 588 + }, + { + "KL/chosen_KL_mean": -1334.051513671875, + "KL/mean": -1767.8013916015625, + "KL/rejected_KL_mean": -2201.55126953125, + "KL/std": 1667.269287109375, + "epoch": 0.8649045521292217, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 867.4995727539062, + "fcm_dpo/q_t": 0.48490506410598755, + "grad_norm": 7.0310516357421875, + "learning_rate": 2.7951734455078786e-08, + "logits/chosen": -0.6013877391815186, + "logits/rejected": -0.6371986269950867, + "logps/chosen": -1393.9814453125, + "logps/ref_chosen": -59.929908752441406, + "logps/ref_rejected": -111.65534973144531, + "logps/rejected": -2313.20654296875, + "loss": 1.3308, + "margin_dpo/margin_mean": 867.4996337890625, + "margin_dpo/margin_std": 1884.604736328125, + "step": 589 + }, + { + "KL/chosen_KL_mean": -1027.572998046875, + "KL/mean": -1449.776611328125, + "KL/rejected_KL_mean": -1871.980224609375, + "KL/std": 1585.0914306640625, + "epoch": 0.8663729809104258, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 844.4072875976562, + "fcm_dpo/q_t": 0.4853343367576599, + "grad_norm": 6.130049228668213, + "learning_rate": 2.736501028272095e-08, + "logits/chosen": -0.6168742775917053, + "logits/rejected": -0.6527448892593384, + "logps/chosen": -1083.3828125, + "logps/ref_chosen": -55.80979537963867, + "logps/ref_rejected": -106.06282043457031, + "logps/rejected": -1978.0430908203125, + "loss": 1.3328, + "margin_dpo/margin_mean": 844.4072265625, + "margin_dpo/margin_std": 1984.2763671875, + "step": 590 + }, + { + "KL/chosen_KL_mean": -1019.3551025390625, + "KL/mean": -1367.507568359375, + "KL/rejected_KL_mean": -1715.6600341796875, + "KL/std": 1360.0946044921875, + "epoch": 0.8678414096916299, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 696.304931640625, + "fcm_dpo/q_t": 0.4878562390804291, + "grad_norm": 6.305633068084717, + "learning_rate": 2.678415274939408e-08, + "logits/chosen": -0.6568745970726013, + "logits/rejected": -0.6684634685516357, + "logps/chosen": -1075.595703125, + "logps/ref_chosen": -56.24061965942383, + "logps/ref_rejected": -83.78629302978516, + "logps/rejected": -1799.4462890625, + "loss": 1.342, + "margin_dpo/margin_mean": 696.304931640625, + "margin_dpo/margin_std": 1778.0966796875, + "step": 591 + }, + { + "KL/chosen_KL_mean": -1184.225830078125, + "KL/mean": -1476.70849609375, + "KL/rejected_KL_mean": -1769.1912841796875, + "KL/std": 1288.901123046875, + "epoch": 0.869309838472834, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 584.965576171875, + "fcm_dpo/q_t": 0.4897754192352295, + "grad_norm": 8.049051284790039, + "learning_rate": 2.6209177161234442e-08, + "logits/chosen": -0.6074360609054565, + "logits/rejected": -0.6240702271461487, + "logps/chosen": -1232.166015625, + "logps/ref_chosen": -47.94025421142578, + "logps/ref_rejected": -75.73287963867188, + "logps/rejected": -1844.9241943359375, + "loss": 1.3498, + "margin_dpo/margin_mean": 584.965576171875, + "margin_dpo/margin_std": 1812.1029052734375, + "step": 592 + }, + { + "KL/chosen_KL_mean": -1095.839599609375, + "KL/mean": -1293.16259765625, + "KL/rejected_KL_mean": -1490.4853515625, + "KL/std": 1231.2266845703125, + "epoch": 0.8707782672540382, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 394.645751953125, + "fcm_dpo/q_t": 0.493096262216568, + "grad_norm": 4.078222751617432, + "learning_rate": 2.564009866938349e-08, + "logits/chosen": -0.6004587411880493, + "logits/rejected": -0.6096137166023254, + "logps/chosen": -1144.5303955078125, + "logps/ref_chosen": -48.690757751464844, + "logps/ref_rejected": -60.90800094604492, + "logps/rejected": -1551.3934326171875, + "loss": 1.3614, + "margin_dpo/margin_mean": 394.645751953125, + "margin_dpo/margin_std": 1439.93701171875, + "step": 593 + }, + { + "KL/chosen_KL_mean": -1057.113037109375, + "KL/mean": -1357.0494384765625, + "KL/rejected_KL_mean": -1656.9857177734375, + "KL/std": 1238.75146484375, + "epoch": 0.8722466960352423, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 599.8726806640625, + "fcm_dpo/q_t": 0.4895210862159729, + "grad_norm": 7.381805419921875, + "learning_rate": 2.5076932269588708e-08, + "logits/chosen": -0.6215956211090088, + "logits/rejected": -0.6254661083221436, + "logps/chosen": -1112.0478515625, + "logps/ref_chosen": -54.93488693237305, + "logps/ref_rejected": -86.09967803955078, + "logps/rejected": -1743.08544921875, + "loss": 1.3476, + "margin_dpo/margin_mean": 599.8727416992188, + "margin_dpo/margin_std": 1491.129150390625, + "step": 594 + }, + { + "KL/chosen_KL_mean": -986.426513671875, + "KL/mean": -1438.9229736328125, + "KL/rejected_KL_mean": -1891.41943359375, + "KL/std": 1537.6827392578125, + "epoch": 0.8737151248164464, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 904.9927978515625, + "fcm_dpo/q_t": 0.4843044877052307, + "grad_norm": 8.513747215270996, + "learning_rate": 2.451969280180849e-08, + "logits/chosen": -0.6116786003112793, + "logits/rejected": -0.6459616422653198, + "logps/chosen": -1035.846923828125, + "logps/ref_chosen": -49.4204216003418, + "logps/ref_rejected": -80.62731170654297, + "logps/rejected": -1972.046630859375, + "loss": 1.3286, + "margin_dpo/margin_mean": 904.992919921875, + "margin_dpo/margin_std": 1967.465087890625, + "step": 595 + }, + { + "KL/chosen_KL_mean": -1317.722900390625, + "KL/mean": -1460.748779296875, + "KL/rejected_KL_mean": -1603.77490234375, + "KL/std": 1665.4937744140625, + "epoch": 0.8751835535976505, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 286.0518798828125, + "fcm_dpo/q_t": 0.49503082036972046, + "grad_norm": 5.618963241577148, + "learning_rate": 2.396839494982103e-08, + "logits/chosen": -0.6746952533721924, + "logits/rejected": -0.6740427017211914, + "logps/chosen": -1377.5145263671875, + "logps/ref_chosen": -59.791683197021484, + "logps/ref_rejected": -80.09111785888672, + "logps/rejected": -1683.865966796875, + "loss": 1.3718, + "margin_dpo/margin_mean": 286.0519104003906, + "margin_dpo/margin_std": 2089.884521484375, + "step": 596 + }, + { + "KL/chosen_KL_mean": -1239.4058837890625, + "KL/mean": -1711.820556640625, + "KL/rejected_KL_mean": -2184.235107421875, + "KL/std": 1657.8519287109375, + "epoch": 0.8766519823788547, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 944.8291625976562, + "fcm_dpo/q_t": 0.4835811257362366, + "grad_norm": 6.500996112823486, + "learning_rate": 2.3423053240837514e-08, + "logits/chosen": -0.6523314118385315, + "logits/rejected": -0.6965519785881042, + "logps/chosen": -1296.666748046875, + "logps/ref_chosen": -57.26078796386719, + "logps/ref_rejected": -100.6937255859375, + "logps/rejected": -2284.9287109375, + "loss": 1.3265, + "margin_dpo/margin_mean": 944.8292236328125, + "margin_dpo/margin_std": 1901.935791015625, + "step": 597 + }, + { + "KL/chosen_KL_mean": -1068.144287109375, + "KL/mean": -1389.80712890625, + "KL/rejected_KL_mean": -1711.4698486328125, + "KL/std": 1323.55322265625, + "epoch": 0.8781204111600588, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 643.3255615234375, + "fcm_dpo/q_t": 0.4887608289718628, + "grad_norm": 4.988048076629639, + "learning_rate": 2.2883682045119062e-08, + "logits/chosen": -0.602400541305542, + "logits/rejected": -0.6199566721916199, + "logps/chosen": -1120.662841796875, + "logps/ref_chosen": -52.51850509643555, + "logps/ref_rejected": -89.44385528564453, + "logps/rejected": -1800.913818359375, + "loss": 1.3452, + "margin_dpo/margin_mean": 643.3255615234375, + "margin_dpo/margin_std": 1611.5604248046875, + "step": 598 + }, + { + "KL/chosen_KL_mean": -1212.7427978515625, + "KL/mean": -1437.69384765625, + "KL/rejected_KL_mean": -1662.644775390625, + "KL/std": 1404.40869140625, + "epoch": 0.8795888399412628, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 449.90191650390625, + "fcm_dpo/q_t": 0.4921151101589203, + "grad_norm": 5.333652973175049, + "learning_rate": 2.2350295575598367e-08, + "logits/chosen": -0.6382207274436951, + "logits/rejected": -0.6507253050804138, + "logps/chosen": -1262.5455322265625, + "logps/ref_chosen": -49.802677154541016, + "logps/ref_rejected": -82.978515625, + "logps/rejected": -1745.623291015625, + "loss": 1.3583, + "margin_dpo/margin_mean": 449.90191650390625, + "margin_dpo/margin_std": 1570.1396484375, + "step": 599 + }, + { + "KL/chosen_KL_mean": -1502.628173828125, + "KL/mean": -1721.712646484375, + "KL/rejected_KL_mean": -1940.797119140625, + "KL/std": 1682.96484375, + "epoch": 0.8810572687224669, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 438.1690673828125, + "fcm_dpo/q_t": 0.49244120717048645, + "grad_norm": 6.994654655456543, + "learning_rate": 2.1822907887504932e-08, + "logits/chosen": -0.6980600357055664, + "logits/rejected": -0.7049777507781982, + "logps/chosen": -1569.06298828125, + "logps/ref_chosen": -66.43487548828125, + "logps/ref_rejected": -85.45649719238281, + "logps/rejected": -2026.253662109375, + "loss": 1.3628, + "margin_dpo/margin_mean": 438.1690368652344, + "margin_dpo/margin_std": 2316.57080078125, + "step": 600 + }, + { + "KL/chosen_KL_mean": -1398.1495361328125, + "KL/mean": -1735.676513671875, + "KL/rejected_KL_mean": -2073.203369140625, + "KL/std": 1529.931640625, + "epoch": 0.882525697503671, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 675.0538330078125, + "fcm_dpo/q_t": 0.4882049858570099, + "grad_norm": 6.039072513580322, + "learning_rate": 2.1301532877994742e-08, + "logits/chosen": -0.6777108907699585, + "logits/rejected": -0.6996618509292603, + "logps/chosen": -1457.283203125, + "logps/ref_chosen": -59.13361358642578, + "logps/ref_rejected": -94.69093322753906, + "logps/rejected": -2167.894287109375, + "loss": 1.344, + "margin_dpo/margin_mean": 675.0538330078125, + "margin_dpo/margin_std": 1895.3690185546875, + "step": 601 + }, + { + "KL/chosen_KL_mean": -859.4080810546875, + "KL/mean": -1380.189697265625, + "KL/rejected_KL_mean": -1900.97119140625, + "KL/std": 1388.6561279296875, + "epoch": 0.8839941262848752, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1041.5631103515625, + "fcm_dpo/q_t": 0.4818532466888428, + "grad_norm": 14.612542152404785, + "learning_rate": 2.0786184285784298e-08, + "logits/chosen": -0.6874485015869141, + "logits/rejected": -0.7312799692153931, + "logps/chosen": -908.0015869140625, + "logps/ref_chosen": -48.59352111816406, + "logps/ref_rejected": -87.6685562133789, + "logps/rejected": -1988.6396484375, + "loss": 1.3185, + "margin_dpo/margin_mean": 1041.5631103515625, + "margin_dpo/margin_std": 1663.5296630859375, + "step": 602 + }, + { + "KL/chosen_KL_mean": -1425.88330078125, + "KL/mean": -1779.10498046875, + "KL/rejected_KL_mean": -2132.32666015625, + "KL/std": 1935.75244140625, + "epoch": 0.8854625550660793, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 706.4432983398438, + "fcm_dpo/q_t": 0.48793095350265503, + "grad_norm": 14.034811019897461, + "learning_rate": 2.0276875690788204e-08, + "logits/chosen": -0.7051316499710083, + "logits/rejected": -0.7167890071868896, + "logps/chosen": -1496.2979736328125, + "logps/ref_chosen": -70.41461944580078, + "logps/ref_rejected": -100.32559967041016, + "logps/rejected": -2232.65234375, + "loss": 1.3446, + "margin_dpo/margin_mean": 706.4432983398438, + "margin_dpo/margin_std": 2401.55517578125, + "step": 603 + }, + { + "KL/chosen_KL_mean": -1154.302001953125, + "KL/mean": -1677.122802734375, + "KL/rejected_KL_mean": -2199.943603515625, + "KL/std": 1617.4171142578125, + "epoch": 0.8869309838472834, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1045.6416015625, + "fcm_dpo/q_t": 0.4818729758262634, + "grad_norm": 6.103328704833984, + "learning_rate": 1.977362051376158e-08, + "logits/chosen": -0.6412931680679321, + "logits/rejected": -0.6906075477600098, + "logps/chosen": -1200.760009765625, + "logps/ref_chosen": -46.45808029174805, + "logps/ref_rejected": -91.8544921875, + "logps/rejected": -2291.798095703125, + "loss": 1.3206, + "margin_dpo/margin_mean": 1045.6414794921875, + "margin_dpo/margin_std": 2262.8173828125, + "step": 604 + }, + { + "KL/chosen_KL_mean": -1212.1982421875, + "KL/mean": -1576.411865234375, + "KL/rejected_KL_mean": -1940.62548828125, + "KL/std": 1592.7261962890625, + "epoch": 0.8883994126284875, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 728.4273681640625, + "fcm_dpo/q_t": 0.48738065361976624, + "grad_norm": 7.565073013305664, + "learning_rate": 1.9276432015946446e-08, + "logits/chosen": -0.6622729301452637, + "logits/rejected": -0.6903505325317383, + "logps/chosen": -1278.447509765625, + "logps/ref_chosen": -66.24933624267578, + "logps/ref_rejected": -102.30496978759766, + "logps/rejected": -2042.930419921875, + "loss": 1.3417, + "margin_dpo/margin_mean": 728.4273681640625, + "margin_dpo/margin_std": 2157.499267578125, + "step": 605 + }, + { + "KL/chosen_KL_mean": -1284.903564453125, + "KL/mean": -1668.8912353515625, + "KL/rejected_KL_mean": -2052.87890625, + "KL/std": 2009.388916015625, + "epoch": 0.8898678414096917, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 767.9752807617188, + "fcm_dpo/q_t": 0.48712000250816345, + "grad_norm": 10.69570255279541, + "learning_rate": 1.8785323298722093e-08, + "logits/chosen": -0.6595591306686401, + "logits/rejected": -0.695212721824646, + "logps/chosen": -1339.72265625, + "logps/ref_chosen": -54.819122314453125, + "logps/ref_rejected": -98.37146759033203, + "logps/rejected": -2151.250244140625, + "loss": 1.3417, + "margin_dpo/margin_mean": 767.9752807617188, + "margin_dpo/margin_std": 2621.770263671875, + "step": 606 + }, + { + "KL/chosen_KL_mean": -1284.68359375, + "KL/mean": -1566.37744140625, + "KL/rejected_KL_mean": -1848.0712890625, + "KL/std": 1655.87255859375, + "epoch": 0.8913362701908958, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 563.3876342773438, + "fcm_dpo/q_t": 0.4901534914970398, + "grad_norm": 10.138040542602539, + "learning_rate": 1.8300307303259904e-08, + "logits/chosen": -0.6720389127731323, + "logits/rejected": -0.6777141690254211, + "logps/chosen": -1342.767578125, + "logps/ref_chosen": -58.08403778076172, + "logps/ref_rejected": -79.777099609375, + "logps/rejected": -1927.848388671875, + "loss": 1.3511, + "margin_dpo/margin_mean": 563.3876953125, + "margin_dpo/margin_std": 1770.9501953125, + "step": 607 + }, + { + "KL/chosen_KL_mean": -1165.683837890625, + "KL/mean": -1508.021240234375, + "KL/rejected_KL_mean": -1850.35888671875, + "KL/std": 1325.984130859375, + "epoch": 0.8928046989720999, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 684.6749877929688, + "fcm_dpo/q_t": 0.48802483081817627, + "grad_norm": 5.801738262176514, + "learning_rate": 1.7821396810182437e-08, + "logits/chosen": -0.6812508702278137, + "logits/rejected": -0.7010186910629272, + "logps/chosen": -1223.1346435546875, + "logps/ref_chosen": -57.450836181640625, + "logps/ref_rejected": -94.77339172363281, + "logps/rejected": -1945.1322021484375, + "loss": 1.3419, + "margin_dpo/margin_mean": 684.675048828125, + "margin_dpo/margin_std": 1577.7777099609375, + "step": 608 + }, + { + "KL/chosen_KL_mean": -1125.0963134765625, + "KL/mean": -1520.38818359375, + "KL/rejected_KL_mean": -1915.679931640625, + "KL/std": 1332.13232421875, + "epoch": 0.8942731277533039, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 790.583740234375, + "fcm_dpo/q_t": 0.48619094491004944, + "grad_norm": 10.305415153503418, + "learning_rate": 1.7348604439226617e-08, + "logits/chosen": -0.7046061754226685, + "logits/rejected": -0.7308906316757202, + "logps/chosen": -1183.901611328125, + "logps/ref_chosen": -58.805355072021484, + "logps/ref_rejected": -88.81600952148438, + "logps/rejected": -2004.4959716796875, + "loss": 1.3355, + "margin_dpo/margin_mean": 790.583740234375, + "margin_dpo/margin_std": 1792.01611328125, + "step": 609 + }, + { + "KL/chosen_KL_mean": -1282.007568359375, + "KL/mean": -1432.61962890625, + "KL/rejected_KL_mean": -1583.231689453125, + "KL/std": 1448.6640625, + "epoch": 0.895741556534508, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 301.22406005859375, + "fcm_dpo/q_t": 0.49473780393600464, + "grad_norm": 11.932384490966797, + "learning_rate": 1.6881942648911074e-08, + "logits/chosen": -0.6814596652984619, + "logits/rejected": -0.6760205030441284, + "logps/chosen": -1347.7025146484375, + "logps/ref_chosen": -65.69503784179688, + "logps/ref_rejected": -83.40538787841797, + "logps/rejected": -1666.636962890625, + "loss": 1.3701, + "margin_dpo/margin_mean": 301.22406005859375, + "margin_dpo/margin_std": 1961.03515625, + "step": 610 + }, + { + "KL/chosen_KL_mean": -1148.4498291015625, + "KL/mean": -1598.172607421875, + "KL/rejected_KL_mean": -2047.8955078125, + "KL/std": 1594.612060546875, + "epoch": 0.8972099853157122, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 899.4454956054688, + "fcm_dpo/q_t": 0.48433127999305725, + "grad_norm": 6.665002822875977, + "learning_rate": 1.6421423736208e-08, + "logits/chosen": -0.6560553312301636, + "logits/rejected": -0.6957667469978333, + "logps/chosen": -1201.04931640625, + "logps/ref_chosen": -52.59946823120117, + "logps/ref_rejected": -86.33099365234375, + "logps/rejected": -2134.226318359375, + "loss": 1.3283, + "margin_dpo/margin_mean": 899.4454345703125, + "margin_dpo/margin_std": 1819.7413330078125, + "step": 611 + }, + { + "KL/chosen_KL_mean": -1225.57861328125, + "KL/mean": -1524.23291015625, + "KL/rejected_KL_mean": -1822.88720703125, + "KL/std": 1422.014404296875, + "epoch": 0.8986784140969163, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 597.30859375, + "fcm_dpo/q_t": 0.4895847737789154, + "grad_norm": 8.166829109191895, + "learning_rate": 1.5967059836219042e-08, + "logits/chosen": -0.6424583792686462, + "logits/rejected": -0.671108067035675, + "logps/chosen": -1284.90234375, + "logps/ref_chosen": -59.32372283935547, + "logps/ref_rejected": -88.31239318847656, + "logps/rejected": -1911.19970703125, + "loss": 1.3491, + "margin_dpo/margin_mean": 597.30859375, + "margin_dpo/margin_std": 1860.5745849609375, + "step": 612 + }, + { + "KL/chosen_KL_mean": -1104.078857421875, + "KL/mean": -1501.256591796875, + "KL/rejected_KL_mean": -1898.4345703125, + "KL/std": 1444.134521484375, + "epoch": 0.9001468428781204, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 794.355712890625, + "fcm_dpo/q_t": 0.4861546754837036, + "grad_norm": 6.351512432098389, + "learning_rate": 1.551886292185553e-08, + "logits/chosen": -0.6626429557800293, + "logits/rejected": -0.7097995281219482, + "logps/chosen": -1163.808837890625, + "logps/ref_chosen": -59.72996520996094, + "logps/ref_rejected": -105.10752868652344, + "logps/rejected": -2003.5419921875, + "loss": 1.3355, + "margin_dpo/margin_mean": 794.3555908203125, + "margin_dpo/margin_std": 1837.294189453125, + "step": 613 + }, + { + "KL/chosen_KL_mean": -1072.1090087890625, + "KL/mean": -1519.0906982421875, + "KL/rejected_KL_mean": -1966.072509765625, + "KL/std": 1425.783447265625, + "epoch": 0.9016152716593245, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 893.96337890625, + "fcm_dpo/q_t": 0.4844154119491577, + "grad_norm": 8.47018051147461, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": -0.6103712320327759, + "logits/rejected": -0.6593648195266724, + "logps/chosen": -1125.048095703125, + "logps/ref_chosen": -52.93898010253906, + "logps/ref_rejected": -104.67938232421875, + "logps/rejected": -2070.751953125, + "loss": 1.3294, + "margin_dpo/margin_mean": 893.96337890625, + "margin_dpo/margin_std": 1988.4140625, + "step": 614 + }, + { + "KL/chosen_KL_mean": -1103.2506103515625, + "KL/mean": -1468.0848388671875, + "KL/rejected_KL_mean": -1832.919189453125, + "KL/std": 1459.701171875, + "epoch": 0.9030837004405287, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 729.66845703125, + "fcm_dpo/q_t": 0.4873241186141968, + "grad_norm": 5.5929274559021, + "learning_rate": 1.4641017128809801e-08, + "logits/chosen": -0.6631003022193909, + "logits/rejected": -0.687023401260376, + "logps/chosen": -1169.06787109375, + "logps/ref_chosen": -65.81727600097656, + "logps/ref_rejected": -95.17749786376953, + "logps/rejected": -1928.0966796875, + "loss": 1.3401, + "margin_dpo/margin_mean": 729.66845703125, + "margin_dpo/margin_std": 1842.6663818359375, + "step": 615 + }, + { + "KL/chosen_KL_mean": -1339.16796875, + "KL/mean": -1633.54150390625, + "KL/rejected_KL_mean": -1927.914794921875, + "KL/std": 1515.873779296875, + "epoch": 0.9045521292217328, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 588.7468872070312, + "fcm_dpo/q_t": 0.489709734916687, + "grad_norm": 6.771057605743408, + "learning_rate": 1.4211391382180637e-08, + "logits/chosen": -0.6890289783477783, + "logits/rejected": -0.6939765214920044, + "logps/chosen": -1404.3009033203125, + "logps/ref_chosen": -65.13285827636719, + "logps/ref_rejected": -74.70050048828125, + "logps/rejected": -2002.615234375, + "loss": 1.3497, + "margin_dpo/margin_mean": 588.746826171875, + "margin_dpo/margin_std": 1862.400390625, + "step": 616 + }, + { + "KL/chosen_KL_mean": -1498.676513671875, + "KL/mean": -1554.958251953125, + "KL/rejected_KL_mean": -1611.240234375, + "KL/std": 1531.433837890625, + "epoch": 0.9060205580029369, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 112.56379699707031, + "fcm_dpo/q_t": 0.4979705214500427, + "grad_norm": 20.132686614990234, + "learning_rate": 1.378797888467345e-08, + "logits/chosen": -0.6434112787246704, + "logits/rejected": -0.6232435703277588, + "logps/chosen": -1561.68212890625, + "logps/ref_chosen": -63.005550384521484, + "logps/ref_rejected": -64.234130859375, + "logps/rejected": -1675.474365234375, + "loss": 1.3831, + "margin_dpo/margin_mean": 112.56379699707031, + "margin_dpo/margin_std": 1948.6531982421875, + "step": 617 + }, + { + "KL/chosen_KL_mean": -1434.4833984375, + "KL/mean": -1825.2366943359375, + "KL/rejected_KL_mean": -2215.989990234375, + "KL/std": 1607.75, + "epoch": 0.9074889867841409, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 781.506591796875, + "fcm_dpo/q_t": 0.4863564670085907, + "grad_norm": 9.600302696228027, + "learning_rate": 1.3370790793601371e-08, + "logits/chosen": -0.6751698851585388, + "logits/rejected": -0.7124015092849731, + "logps/chosen": -1501.584716796875, + "logps/ref_chosen": -67.10134887695312, + "logps/ref_rejected": -92.15340423583984, + "logps/rejected": -2308.1435546875, + "loss": 1.3373, + "margin_dpo/margin_mean": 781.506591796875, + "margin_dpo/margin_std": 1978.0693359375, + "step": 618 + }, + { + "KL/chosen_KL_mean": -1300.69775390625, + "KL/mean": -1552.975830078125, + "KL/rejected_KL_mean": -1805.2540283203125, + "KL/std": 1608.731689453125, + "epoch": 0.908957415565345, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 504.5563659667969, + "fcm_dpo/q_t": 0.4911313056945801, + "grad_norm": 7.555780410766602, + "learning_rate": 1.2959838102258535e-08, + "logits/chosen": -0.6084803342819214, + "logits/rejected": -0.6177768707275391, + "logps/chosen": -1356.6759033203125, + "logps/ref_chosen": -55.978233337402344, + "logps/ref_rejected": -93.1854019165039, + "logps/rejected": -1898.439453125, + "loss": 1.3564, + "margin_dpo/margin_mean": 504.55633544921875, + "margin_dpo/margin_std": 2043.724609375, + "step": 619 + }, + { + "KL/chosen_KL_mean": -1157.4849853515625, + "KL/mean": -1404.56494140625, + "KL/rejected_KL_mean": -1651.644775390625, + "KL/std": 1310.7586669921875, + "epoch": 0.9104258443465492, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 494.15985107421875, + "fcm_dpo/q_t": 0.49135249853134155, + "grad_norm": 6.826314449310303, + "learning_rate": 1.2555131639630567e-08, + "logits/chosen": -0.6301568746566772, + "logits/rejected": -0.6424782276153564, + "logps/chosen": -1217.282470703125, + "logps/ref_chosen": -59.79750061035156, + "logps/ref_rejected": -78.41075134277344, + "logps/rejected": -1730.0556640625, + "loss": 1.3558, + "margin_dpo/margin_mean": 494.15985107421875, + "margin_dpo/margin_std": 1771.6112060546875, + "step": 620 + }, + { + "KL/chosen_KL_mean": -1000.7633056640625, + "KL/mean": -1483.132568359375, + "KL/rejected_KL_mean": -1965.501708984375, + "KL/std": 1498.1484375, + "epoch": 0.9118942731277533, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 964.7385864257812, + "fcm_dpo/q_t": 0.48318374156951904, + "grad_norm": 6.330345630645752, + "learning_rate": 1.2156682070109086e-08, + "logits/chosen": -0.6472010612487793, + "logits/rejected": -0.6926671266555786, + "logps/chosen": -1054.697021484375, + "logps/ref_chosen": -53.93375778198242, + "logps/ref_rejected": -88.36951446533203, + "logps/rejected": -2053.871337890625, + "loss": 1.3245, + "margin_dpo/margin_mean": 964.7385864257812, + "margin_dpo/margin_std": 1884.843017578125, + "step": 621 + }, + { + "KL/chosen_KL_mean": -1014.5686645507812, + "KL/mean": -1319.3843994140625, + "KL/rejected_KL_mean": -1624.2001953125, + "KL/std": 1281.564697265625, + "epoch": 0.9133627019089574, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 609.6315307617188, + "fcm_dpo/q_t": 0.48939478397369385, + "grad_norm": 6.59990930557251, + "learning_rate": 1.1764499893210878e-08, + "logits/chosen": -0.6089707612991333, + "logits/rejected": -0.6166027784347534, + "logps/chosen": -1074.8544921875, + "logps/ref_chosen": -60.28582000732422, + "logps/ref_rejected": -85.51873779296875, + "logps/rejected": -1709.718994140625, + "loss": 1.3473, + "margin_dpo/margin_mean": 609.6315307617188, + "margin_dpo/margin_std": 1645.434814453125, + "step": 622 + }, + { + "KL/chosen_KL_mean": -1287.5997314453125, + "KL/mean": -1454.433349609375, + "KL/rejected_KL_mean": -1621.266845703125, + "KL/std": 1537.554443359375, + "epoch": 0.9148311306901615, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 333.66717529296875, + "fcm_dpo/q_t": 0.4941401183605194, + "grad_norm": 15.589156150817871, + "learning_rate": 1.1378595443300998e-08, + "logits/chosen": -0.6384043097496033, + "logits/rejected": -0.6354462504386902, + "logps/chosen": -1351.756591796875, + "logps/ref_chosen": -64.1569595336914, + "logps/ref_rejected": -85.08304595947266, + "logps/rejected": -1706.349853515625, + "loss": 1.3675, + "margin_dpo/margin_mean": 333.6671142578125, + "margin_dpo/margin_std": 1856.07421875, + "step": 623 + }, + { + "KL/chosen_KL_mean": -1173.687744140625, + "KL/mean": -1618.4342041015625, + "KL/rejected_KL_mean": -2063.1806640625, + "KL/std": 1536.404296875, + "epoch": 0.9162995594713657, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 889.4931640625, + "fcm_dpo/q_t": 0.4844810962677002, + "grad_norm": 5.250894546508789, + "learning_rate": 1.0998978889320582e-08, + "logits/chosen": -0.6779031753540039, + "logits/rejected": -0.6975363492965698, + "logps/chosen": -1245.6063232421875, + "logps/ref_chosen": -71.91862487792969, + "logps/ref_rejected": -97.13203430175781, + "logps/rejected": -2160.312744140625, + "loss": 1.3297, + "margin_dpo/margin_mean": 889.4930419921875, + "margin_dpo/margin_std": 1989.96533203125, + "step": 624 + }, + { + "KL/chosen_KL_mean": -1133.847412109375, + "KL/mean": -1514.25537109375, + "KL/rejected_KL_mean": -1894.663330078125, + "KL/std": 1352.28173828125, + "epoch": 0.9177679882525698, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 760.81591796875, + "fcm_dpo/q_t": 0.4867083430290222, + "grad_norm": 6.905755996704102, + "learning_rate": 1.0625660234518913e-08, + "logits/chosen": -0.5888922214508057, + "logits/rejected": -0.605700671672821, + "logps/chosen": -1192.189453125, + "logps/ref_chosen": -58.342071533203125, + "logps/ref_rejected": -86.09038543701172, + "logps/rejected": -1980.753662109375, + "loss": 1.3361, + "margin_dpo/margin_mean": 760.81591796875, + "margin_dpo/margin_std": 1390.97021484375, + "step": 625 + }, + { + "KL/chosen_KL_mean": -1193.1416015625, + "KL/mean": -1446.146728515625, + "KL/rejected_KL_mean": -1699.152099609375, + "KL/std": 1531.291259765625, + "epoch": 0.9192364170337739, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 506.01043701171875, + "fcm_dpo/q_t": 0.491168349981308, + "grad_norm": 6.740545749664307, + "learning_rate": 1.0258649316189721e-08, + "logits/chosen": -0.6106196045875549, + "logits/rejected": -0.6149002909660339, + "logps/chosen": -1268.254150390625, + "logps/ref_chosen": -75.11260986328125, + "logps/ref_rejected": -99.188720703125, + "logps/rejected": -1798.3408203125, + "loss": 1.3553, + "margin_dpo/margin_mean": 506.01043701171875, + "margin_dpo/margin_std": 1811.4095458984375, + "step": 626 + }, + { + "KL/chosen_KL_mean": -773.64111328125, + "KL/mean": -1223.371337890625, + "KL/rejected_KL_mean": -1673.101318359375, + "KL/std": 1513.056884765625, + "epoch": 0.920704845814978, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 899.4602661132812, + "fcm_dpo/q_t": 0.4843396246433258, + "grad_norm": 10.459403991699219, + "learning_rate": 9.897955805412e-09, + "logits/chosen": -0.6010231971740723, + "logits/rejected": -0.6437910795211792, + "logps/chosen": -821.38427734375, + "logps/ref_chosen": -47.74314880371094, + "logps/ref_rejected": -106.75448608398438, + "logps/rejected": -1779.85595703125, + "loss": 1.3293, + "margin_dpo/margin_mean": 899.460205078125, + "margin_dpo/margin_std": 2023.463623046875, + "step": 627 + }, + { + "KL/chosen_KL_mean": -1010.72412109375, + "KL/mean": -1478.8046875, + "KL/rejected_KL_mean": -1946.8851318359375, + "KL/std": 1570.787109375, + "epoch": 0.922173274596182, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 936.1610107421875, + "fcm_dpo/q_t": 0.48373740911483765, + "grad_norm": 8.426224708557129, + "learning_rate": 9.543589206795238e-09, + "logits/chosen": -0.6274340152740479, + "logits/rejected": -0.6652277708053589, + "logps/chosen": -1070.906982421875, + "logps/ref_chosen": -60.182945251464844, + "logps/ref_rejected": -101.55467224121094, + "logps/rejected": -2048.439697265625, + "loss": 1.3262, + "margin_dpo/margin_mean": 936.1610717773438, + "margin_dpo/margin_std": 1925.196044921875, + "step": 628 + }, + { + "KL/chosen_KL_mean": -1270.9169921875, + "KL/mean": -1610.190673828125, + "KL/rejected_KL_mean": -1949.46435546875, + "KL/std": 1889.252197265625, + "epoch": 0.9236417033773862, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 678.5474243164062, + "fcm_dpo/q_t": 0.48802652955055237, + "grad_norm": 6.176880359649658, + "learning_rate": 9.19555885822887e-09, + "logits/chosen": -0.6713994741439819, + "logits/rejected": -0.6894298791885376, + "logps/chosen": -1335.13037109375, + "logps/ref_chosen": -64.21354675292969, + "logps/ref_rejected": -91.65367126464844, + "logps/rejected": -2041.1180419921875, + "loss": 1.3461, + "margin_dpo/margin_mean": 678.5474853515625, + "margin_dpo/margin_std": 2344.418701171875, + "step": 629 + }, + { + "KL/chosen_KL_mean": -1025.02001953125, + "KL/mean": -1086.37158203125, + "KL/rejected_KL_mean": -1147.72314453125, + "KL/std": 1131.6832275390625, + "epoch": 0.9251101321585903, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 122.7031021118164, + "fcm_dpo/q_t": 0.4978800415992737, + "grad_norm": 10.462772369384766, + "learning_rate": 8.85387393063622e-09, + "logits/chosen": -0.6387168169021606, + "logits/rejected": -0.6161863803863525, + "logps/chosen": -1084.31103515625, + "logps/ref_chosen": -59.29100036621094, + "logps/ref_rejected": -83.59829711914062, + "logps/rejected": -1231.3214111328125, + "loss": 1.3809, + "margin_dpo/margin_mean": 122.70311737060547, + "margin_dpo/margin_std": 1551.34130859375, + "step": 630 + }, + { + "KL/chosen_KL_mean": -1031.8355712890625, + "KL/mean": -1342.90185546875, + "KL/rejected_KL_mean": -1653.9681396484375, + "KL/std": 1206.7095947265625, + "epoch": 0.9265785609397944, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 622.132568359375, + "fcm_dpo/q_t": 0.48913437128067017, + "grad_norm": 6.256035327911377, + "learning_rate": 8.518543427732949e-09, + "logits/chosen": -0.643370509147644, + "logits/rejected": -0.6664153337478638, + "logps/chosen": -1091.2891845703125, + "logps/ref_chosen": -59.45360565185547, + "logps/ref_rejected": -80.95156860351562, + "logps/rejected": -1734.919677734375, + "loss": 1.3464, + "margin_dpo/margin_mean": 622.1326904296875, + "margin_dpo/margin_std": 1622.26513671875, + "step": 631 + }, + { + "KL/chosen_KL_mean": -1001.5509643554688, + "KL/mean": -1220.533935546875, + "KL/rejected_KL_mean": -1439.5169677734375, + "KL/std": 1116.1923828125, + "epoch": 0.9280469897209985, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 437.966064453125, + "fcm_dpo/q_t": 0.4923382103443146, + "grad_norm": 4.83076810836792, + "learning_rate": 8.189576185789637e-09, + "logits/chosen": -0.6454243659973145, + "logits/rejected": -0.6561754941940308, + "logps/chosen": -1062.902587890625, + "logps/ref_chosen": -61.35155487060547, + "logps/ref_rejected": -86.16017150878906, + "logps/rejected": -1525.6771240234375, + "loss": 1.359, + "margin_dpo/margin_mean": 437.966064453125, + "margin_dpo/margin_std": 1603.9329833984375, + "step": 632 + }, + { + "KL/chosen_KL_mean": -1161.1319580078125, + "KL/mean": -1383.982666015625, + "KL/rejected_KL_mean": -1606.83349609375, + "KL/std": 1076.3099365234375, + "epoch": 0.9295154185022027, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 445.701416015625, + "fcm_dpo/q_t": 0.4922114312648773, + "grad_norm": 5.985357284545898, + "learning_rate": 7.866980873399015e-09, + "logits/chosen": -0.6461182832717896, + "logits/rejected": -0.6646542549133301, + "logps/chosen": -1218.41015625, + "logps/ref_chosen": -57.27816390991211, + "logps/ref_rejected": -91.58395385742188, + "logps/rejected": -1698.4173583984375, + "loss": 1.3575, + "margin_dpo/margin_mean": 445.701416015625, + "margin_dpo/margin_std": 1343.09716796875, + "step": 633 + }, + { + "KL/chosen_KL_mean": -1291.4398193359375, + "KL/mean": -1477.098876953125, + "KL/rejected_KL_mean": -1662.758056640625, + "KL/std": 1391.55859375, + "epoch": 0.9309838472834068, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 371.3182373046875, + "fcm_dpo/q_t": 0.493512898683548, + "grad_norm": 6.06099796295166, + "learning_rate": 7.550765991247654e-09, + "logits/chosen": -0.6100882291793823, + "logits/rejected": -0.6161783337593079, + "logps/chosen": -1358.058837890625, + "logps/ref_chosen": -66.61896514892578, + "logps/ref_rejected": -107.12564849853516, + "logps/rejected": -1769.8836669921875, + "loss": 1.3639, + "margin_dpo/margin_mean": 371.3182373046875, + "margin_dpo/margin_std": 1650.0980224609375, + "step": 634 + }, + { + "KL/chosen_KL_mean": -1296.58935546875, + "KL/mean": -1485.6541748046875, + "KL/rejected_KL_mean": -1674.71875, + "KL/std": 1795.396728515625, + "epoch": 0.9324522760646109, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 378.1294860839844, + "fcm_dpo/q_t": 0.4934682250022888, + "grad_norm": 10.481958389282227, + "learning_rate": 7.240939871891699e-09, + "logits/chosen": -0.6858574151992798, + "logits/rejected": -0.6715748310089111, + "logps/chosen": -1370.544921875, + "logps/ref_chosen": -73.95551300048828, + "logps/ref_rejected": -82.50045776367188, + "logps/rejected": -1757.21923828125, + "loss": 1.3643, + "margin_dpo/margin_mean": 378.1295166015625, + "margin_dpo/margin_std": 1888.038818359375, + "step": 635 + }, + { + "KL/chosen_KL_mean": -1168.95947265625, + "KL/mean": -1345.31640625, + "KL/rejected_KL_mean": -1521.6732177734375, + "KL/std": 1588.704345703125, + "epoch": 0.933920704845815, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 352.7136535644531, + "fcm_dpo/q_t": 0.49370384216308594, + "grad_norm": 7.528316020965576, + "learning_rate": 6.937510679537628e-09, + "logits/chosen": -0.6131812334060669, + "logits/rejected": -0.6081274747848511, + "logps/chosen": -1228.58837890625, + "logps/ref_chosen": -59.628910064697266, + "logps/ref_rejected": -81.97883605957031, + "logps/rejected": -1603.652099609375, + "loss": 1.3669, + "margin_dpo/margin_mean": 352.713623046875, + "margin_dpo/margin_std": 2045.9951171875, + "step": 636 + }, + { + "KL/chosen_KL_mean": -982.5504150390625, + "KL/mean": -1417.7578125, + "KL/rejected_KL_mean": -1852.9652099609375, + "KL/std": 1381.57568359375, + "epoch": 0.9353891336270191, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 870.4149169921875, + "fcm_dpo/q_t": 0.48480871319770813, + "grad_norm": 6.873878479003906, + "learning_rate": 6.640486409826785e-09, + "logits/chosen": -0.642035961151123, + "logits/rejected": -0.6937611103057861, + "logps/chosen": -1032.203125, + "logps/ref_chosen": -49.652687072753906, + "logps/ref_rejected": -98.40513610839844, + "logps/rejected": -1951.370361328125, + "loss": 1.3292, + "margin_dpo/margin_mean": 870.4149169921875, + "margin_dpo/margin_std": 1557.965087890625, + "step": 637 + }, + { + "KL/chosen_KL_mean": -980.171142578125, + "KL/mean": -1293.06494140625, + "KL/rejected_KL_mean": -1605.9588623046875, + "KL/std": 1329.313720703125, + "epoch": 0.9368575624082232, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 625.7876586914062, + "fcm_dpo/q_t": 0.4890254735946655, + "grad_norm": 4.65677547454834, + "learning_rate": 6.349874889624962e-09, + "logits/chosen": -0.6139806509017944, + "logits/rejected": -0.6274735331535339, + "logps/chosen": -1038.327880859375, + "logps/ref_chosen": -58.156639099121094, + "logps/ref_rejected": -79.3014907836914, + "logps/rejected": -1685.26025390625, + "loss": 1.3471, + "margin_dpo/margin_mean": 625.78759765625, + "margin_dpo/margin_std": 1827.26708984375, + "step": 638 + }, + { + "KL/chosen_KL_mean": -1459.7200927734375, + "KL/mean": -1391.7880859375, + "KL/rejected_KL_mean": -1323.856201171875, + "KL/std": 1219.503662109375, + "epoch": 0.9383259911894273, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -135.86386108398438, + "fcm_dpo/q_t": 0.5023590326309204, + "grad_norm": 26.913433074951172, + "learning_rate": 6.065683776815933e-09, + "logits/chosen": -0.6304311752319336, + "logits/rejected": -0.5896961092948914, + "logps/chosen": -1532.0433349609375, + "logps/ref_chosen": -72.32319641113281, + "logps/ref_rejected": -74.2749252319336, + "logps/rejected": -1398.131103515625, + "loss": 1.3987, + "margin_dpo/margin_mean": -135.86386108398438, + "margin_dpo/margin_std": 1491.50634765625, + "step": 639 + }, + { + "KL/chosen_KL_mean": -954.1146850585938, + "KL/mean": -1476.706298828125, + "KL/rejected_KL_mean": -1999.2978515625, + "KL/std": 1782.7164306640625, + "epoch": 0.9397944199706314, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1045.18310546875, + "fcm_dpo/q_t": 0.4820125997066498, + "grad_norm": 6.035217761993408, + "learning_rate": 5.7879205600998296e-09, + "logits/chosen": -0.591851532459259, + "logits/rejected": -0.6358239650726318, + "logps/chosen": -1010.2490234375, + "logps/ref_chosen": -56.13436508178711, + "logps/ref_rejected": -108.60014343261719, + "logps/rejected": -2107.89794921875, + "loss": 1.3206, + "margin_dpo/margin_mean": 1045.18310546875, + "margin_dpo/margin_std": 2260.031005859375, + "step": 640 + }, + { + "KL/chosen_KL_mean": -1025.90771484375, + "KL/mean": -1481.028076171875, + "KL/rejected_KL_mean": -1936.1484375, + "KL/std": 1427.94873046875, + "epoch": 0.9412628487518355, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 910.2408447265625, + "fcm_dpo/q_t": 0.4841909408569336, + "grad_norm": 10.455007553100586, + "learning_rate": 5.516592558795746e-09, + "logits/chosen": -0.5904731750488281, + "logits/rejected": -0.6200425624847412, + "logps/chosen": -1090.904541015625, + "logps/ref_chosen": -64.99689483642578, + "logps/ref_rejected": -86.99232482910156, + "logps/rejected": -2023.140869140625, + "loss": 1.3278, + "margin_dpo/margin_mean": 910.2408447265625, + "margin_dpo/margin_std": 1894.365966796875, + "step": 641 + }, + { + "KL/chosen_KL_mean": -1221.719970703125, + "KL/mean": -1628.12744140625, + "KL/rejected_KL_mean": -2034.5347900390625, + "KL/std": 1844.57666015625, + "epoch": 0.9427312775330396, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 812.81494140625, + "fcm_dpo/q_t": 0.4858717918395996, + "grad_norm": 5.243577003479004, + "learning_rate": 5.251706922648868e-09, + "logits/chosen": -0.5993083715438843, + "logits/rejected": -0.6353590488433838, + "logps/chosen": -1287.4091796875, + "logps/ref_chosen": -65.68924713134766, + "logps/ref_rejected": -110.24205017089844, + "logps/rejected": -2144.77685546875, + "loss": 1.3379, + "margin_dpo/margin_mean": 812.8150024414062, + "margin_dpo/margin_std": 2420.636474609375, + "step": 642 + }, + { + "KL/chosen_KL_mean": -1205.4664306640625, + "KL/mean": -1394.8968505859375, + "KL/rejected_KL_mean": -1584.3272705078125, + "KL/std": 1325.056640625, + "epoch": 0.9441997063142438, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 378.8608703613281, + "fcm_dpo/q_t": 0.4933786988258362, + "grad_norm": 5.956773281097412, + "learning_rate": 4.993270631642038e-09, + "logits/chosen": -0.6313992738723755, + "logits/rejected": -0.6435078382492065, + "logps/chosen": -1257.41650390625, + "logps/ref_chosen": -51.94999694824219, + "logps/ref_rejected": -87.46833801269531, + "logps/rejected": -1671.795654296875, + "loss": 1.3637, + "margin_dpo/margin_mean": 378.86090087890625, + "margin_dpo/margin_std": 1685.135986328125, + "step": 643 + }, + { + "KL/chosen_KL_mean": -1009.8892822265625, + "KL/mean": -1339.213623046875, + "KL/rejected_KL_mean": -1668.5380859375, + "KL/std": 1681.36669921875, + "epoch": 0.9456681350954479, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 658.6486206054688, + "fcm_dpo/q_t": 0.48869824409484863, + "grad_norm": 5.554941654205322, + "learning_rate": 4.741290495811873e-09, + "logits/chosen": -0.6091523170471191, + "logits/rejected": -0.6272720098495483, + "logps/chosen": -1068.906982421875, + "logps/ref_chosen": -59.017662048339844, + "logps/ref_rejected": -87.13668823242188, + "logps/rejected": -1755.6746826171875, + "loss": 1.3465, + "margin_dpo/margin_mean": 658.6486206054688, + "margin_dpo/margin_std": 1983.645751953125, + "step": 644 + }, + { + "KL/chosen_KL_mean": -1136.2744140625, + "KL/mean": -1262.766845703125, + "KL/rejected_KL_mean": -1389.25927734375, + "KL/std": 1186.4443359375, + "epoch": 0.947136563876652, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 252.98484802246094, + "fcm_dpo/q_t": 0.49558210372924805, + "grad_norm": 8.011646270751953, + "learning_rate": 4.495773155069299e-09, + "logits/chosen": -0.6204506158828735, + "logits/rejected": -0.6305866241455078, + "logps/chosen": -1192.150390625, + "logps/ref_chosen": -55.87602233886719, + "logps/ref_rejected": -97.78080749511719, + "logps/rejected": -1487.0400390625, + "loss": 1.3723, + "margin_dpo/margin_mean": 252.98483276367188, + "margin_dpo/margin_std": 1717.4600830078125, + "step": 645 + }, + { + "KL/chosen_KL_mean": -990.78173828125, + "KL/mean": -1174.6834716796875, + "KL/rejected_KL_mean": -1358.585205078125, + "KL/std": 1137.93994140625, + "epoch": 0.9486049926578561, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 367.80340576171875, + "fcm_dpo/q_t": 0.49355489015579224, + "grad_norm": 5.103117942810059, + "learning_rate": 4.256725079024553e-09, + "logits/chosen": -0.6123974919319153, + "logits/rejected": -0.6034915447235107, + "logps/chosen": -1052.0574951171875, + "logps/ref_chosen": -61.275787353515625, + "logps/ref_rejected": -77.50580596923828, + "logps/rejected": -1436.091064453125, + "loss": 1.3629, + "margin_dpo/margin_mean": 367.80340576171875, + "margin_dpo/margin_std": 1322.474365234375, + "step": 646 + }, + { + "KL/chosen_KL_mean": -822.7291259765625, + "KL/mean": -1107.7655029296875, + "KL/rejected_KL_mean": -1392.802001953125, + "KL/std": 1028.25341796875, + "epoch": 0.9500734214390602, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 570.0728759765625, + "fcm_dpo/q_t": 0.49000051617622375, + "grad_norm": 10.267186164855957, + "learning_rate": 4.024152566816791e-09, + "logits/chosen": -0.5852940678596497, + "logits/rejected": -0.6047427654266357, + "logps/chosen": -877.58154296875, + "logps/ref_chosen": -54.8524169921875, + "logps/ref_rejected": -93.5194091796875, + "logps/rejected": -1486.3214111328125, + "loss": 1.3484, + "margin_dpo/margin_mean": 570.0728759765625, + "margin_dpo/margin_std": 1177.3486328125, + "step": 647 + }, + { + "KL/chosen_KL_mean": -1093.44384765625, + "KL/mean": -1523.8251953125, + "KL/rejected_KL_mean": -1954.206298828125, + "KL/std": 1495.839111328125, + "epoch": 0.9515418502202643, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 860.7623291015625, + "fcm_dpo/q_t": 0.4849778413772583, + "grad_norm": 10.467401504516602, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": -0.6163946390151978, + "logits/rejected": -0.6602287292480469, + "logps/chosen": -1147.6153564453125, + "logps/ref_chosen": -54.17146682739258, + "logps/ref_rejected": -98.7127914428711, + "logps/rejected": -2052.9189453125, + "loss": 1.3305, + "margin_dpo/margin_mean": 860.7623901367188, + "margin_dpo/margin_std": 1694.208984375, + "step": 648 + }, + { + "KL/chosen_KL_mean": -1002.08935546875, + "KL/mean": -1291.5301513671875, + "KL/rejected_KL_mean": -1580.9708251953125, + "KL/std": 1385.653076171875, + "epoch": 0.9530102790014684, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 578.8814697265625, + "fcm_dpo/q_t": 0.4899559020996094, + "grad_norm": 4.749213218688965, + "learning_rate": 3.5784585771215235e-09, + "logits/chosen": -0.6620218753814697, + "logits/rejected": -0.6723358631134033, + "logps/chosen": -1064.5697021484375, + "logps/ref_chosen": -62.480350494384766, + "logps/ref_rejected": -80.07717895507812, + "logps/rejected": -1661.048095703125, + "loss": 1.3503, + "margin_dpo/margin_mean": 578.8814697265625, + "margin_dpo/margin_std": 1804.99658203125, + "step": 649 + }, + { + "KL/chosen_KL_mean": -1217.0234375, + "KL/mean": -1500.784423828125, + "KL/rejected_KL_mean": -1784.54541015625, + "KL/std": 1650.326416015625, + "epoch": 0.9544787077826725, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 567.52197265625, + "fcm_dpo/q_t": 0.4899527430534363, + "grad_norm": 4.821526527404785, + "learning_rate": 3.3653488440851253e-09, + "logits/chosen": -0.6028755903244019, + "logits/rejected": -0.6270517706871033, + "logps/chosen": -1273.1163330078125, + "logps/ref_chosen": -56.09281921386719, + "logps/ref_rejected": -98.26483917236328, + "logps/rejected": -1882.810302734375, + "loss": 1.353, + "margin_dpo/margin_mean": 567.52197265625, + "margin_dpo/margin_std": 2182.583984375, + "step": 650 + }, + { + "KL/chosen_KL_mean": -809.0330200195312, + "KL/mean": -1301.23486328125, + "KL/rejected_KL_mean": -1793.436767578125, + "KL/std": 1311.9166259765625, + "epoch": 0.9559471365638766, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 984.4036865234375, + "fcm_dpo/q_t": 0.48278218507766724, + "grad_norm": 13.06442928314209, + "learning_rate": 3.158738163478475e-09, + "logits/chosen": -0.647139847278595, + "logits/rejected": -0.6998910903930664, + "logps/chosen": -852.45849609375, + "logps/ref_chosen": -43.42544937133789, + "logps/ref_rejected": -99.95791625976562, + "logps/rejected": -1893.3946533203125, + "loss": 1.3208, + "margin_dpo/margin_mean": 984.4036865234375, + "margin_dpo/margin_std": 1392.693359375, + "step": 651 + }, + { + "KL/chosen_KL_mean": -990.5150756835938, + "KL/mean": -1336.54736328125, + "KL/rejected_KL_mean": -1682.579833984375, + "KL/std": 1426.03564453125, + "epoch": 0.9574155653450808, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 692.0648193359375, + "fcm_dpo/q_t": 0.4879456162452698, + "grad_norm": 13.026046752929688, + "learning_rate": 2.9586319796851555e-09, + "logits/chosen": -0.6543087959289551, + "logits/rejected": -0.6866675019264221, + "logps/chosen": -1053.091796875, + "logps/ref_chosen": -62.57680892944336, + "logps/ref_rejected": -111.76779174804688, + "logps/rejected": -1794.34765625, + "loss": 1.3424, + "margin_dpo/margin_mean": 692.0648193359375, + "margin_dpo/margin_std": 1824.699951171875, + "step": 652 + }, + { + "KL/chosen_KL_mean": -1127.3262939453125, + "KL/mean": -1536.869384765625, + "KL/rejected_KL_mean": -1946.41259765625, + "KL/std": 1666.6322021484375, + "epoch": 0.9588839941262849, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 819.0863647460938, + "fcm_dpo/q_t": 0.48590990900993347, + "grad_norm": 12.233861923217773, + "learning_rate": 2.7650355656892166e-09, + "logits/chosen": -0.6801096200942993, + "logits/rejected": -0.7179126143455505, + "logps/chosen": -1188.439208984375, + "logps/ref_chosen": -61.11295700073242, + "logps/ref_rejected": -103.24960327148438, + "logps/rejected": -2049.662109375, + "loss": 1.3351, + "margin_dpo/margin_mean": 819.0863647460938, + "margin_dpo/margin_std": 2013.414794921875, + "step": 653 + }, + { + "KL/chosen_KL_mean": -1065.35546875, + "KL/mean": -1342.395263671875, + "KL/rejected_KL_mean": -1619.43505859375, + "KL/std": 1395.85400390625, + "epoch": 0.960352422907489, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 554.0797119140625, + "fcm_dpo/q_t": 0.49033522605895996, + "grad_norm": 5.433686256408691, + "learning_rate": 2.577954022936174e-09, + "logits/chosen": -0.6489402651786804, + "logits/rejected": -0.6714223623275757, + "logps/chosen": -1127.08349609375, + "logps/ref_chosen": -61.7281379699707, + "logps/ref_rejected": -98.7738037109375, + "logps/rejected": -1718.208984375, + "loss": 1.3513, + "margin_dpo/margin_mean": 554.0797119140625, + "margin_dpo/margin_std": 1696.0687255859375, + "step": 654 + }, + { + "KL/chosen_KL_mean": -971.6968994140625, + "KL/mean": -1341.12060546875, + "KL/rejected_KL_mean": -1710.54443359375, + "KL/std": 1493.365478515625, + "epoch": 0.9618208516886931, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 738.8475341796875, + "fcm_dpo/q_t": 0.4871307611465454, + "grad_norm": 12.034753799438477, + "learning_rate": 2.397392281198729e-09, + "logits/chosen": -0.5706413984298706, + "logits/rejected": -0.6150772571563721, + "logps/chosen": -1021.273681640625, + "logps/ref_chosen": -49.576812744140625, + "logps/ref_rejected": -98.29183197021484, + "logps/rejected": -1808.8363037109375, + "loss": 1.339, + "margin_dpo/margin_mean": 738.8475341796875, + "margin_dpo/margin_std": 1762.6845703125, + "step": 655 + }, + { + "KL/chosen_KL_mean": -1052.604736328125, + "KL/mean": -1667.0531005859375, + "KL/rejected_KL_mean": -2281.50146484375, + "KL/std": 1618.218505859375, + "epoch": 0.9632892804698973, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1228.896484375, + "fcm_dpo/q_t": 0.478542685508728, + "grad_norm": 15.938714027404785, + "learning_rate": 2.223355098446622e-09, + "logits/chosen": -0.5950414538383484, + "logits/rejected": -0.6505473852157593, + "logps/chosen": -1105.154296875, + "logps/ref_chosen": -52.54943084716797, + "logps/ref_rejected": -113.67464447021484, + "logps/rejected": -2395.176025390625, + "loss": 1.3054, + "margin_dpo/margin_mean": 1228.896484375, + "margin_dpo/margin_std": 1683.515625, + "step": 656 + }, + { + "KL/chosen_KL_mean": -1101.297607421875, + "KL/mean": -1607.488037109375, + "KL/rejected_KL_mean": -2113.678466796875, + "KL/std": 1809.890380859375, + "epoch": 0.9647577092511013, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1012.3809814453125, + "fcm_dpo/q_t": 0.4823700189590454, + "grad_norm": 7.817019939422607, + "learning_rate": 2.055847060721566e-09, + "logits/chosen": -0.6291069984436035, + "logits/rejected": -0.6917046308517456, + "logps/chosen": -1147.998046875, + "logps/ref_chosen": -46.700538635253906, + "logps/ref_rejected": -97.91487121582031, + "logps/rejected": -2211.59326171875, + "loss": 1.3215, + "margin_dpo/margin_mean": 1012.3810424804688, + "margin_dpo/margin_std": 2023.4796142578125, + "step": 657 + }, + { + "KL/chosen_KL_mean": -1130.634033203125, + "KL/mean": -1422.20703125, + "KL/rejected_KL_mean": -1713.7802734375, + "KL/std": 1217.5029296875, + "epoch": 0.9662261380323054, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 583.1461791992188, + "fcm_dpo/q_t": 0.4897571802139282, + "grad_norm": 7.5315752029418945, + "learning_rate": 1.8948725820160662e-09, + "logits/chosen": -0.5903108716011047, + "logits/rejected": -0.6097769737243652, + "logps/chosen": -1191.59228515625, + "logps/ref_chosen": -60.95820999145508, + "logps/ref_rejected": -95.93949127197266, + "logps/rejected": -1809.7197265625, + "loss": 1.3492, + "margin_dpo/margin_mean": 583.1461181640625, + "margin_dpo/margin_std": 1597.01708984375, + "step": 658 + }, + { + "KL/chosen_KL_mean": -1060.837158203125, + "KL/mean": -1350.650146484375, + "KL/rejected_KL_mean": -1640.4630126953125, + "KL/std": 1334.296142578125, + "epoch": 0.9676945668135095, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 579.6257934570312, + "fcm_dpo/q_t": 0.48988625407218933, + "grad_norm": 6.343628883361816, + "learning_rate": 1.7404359041573723e-09, + "logits/chosen": -0.6019806861877441, + "logits/rejected": -0.5888450145721436, + "logps/chosen": -1137.580078125, + "logps/ref_chosen": -76.74298095703125, + "logps/ref_rejected": -87.4709701538086, + "logps/rejected": -1727.9339599609375, + "loss": 1.3493, + "margin_dpo/margin_mean": 579.6257934570312, + "margin_dpo/margin_std": 1626.3175048828125, + "step": 659 + }, + { + "KL/chosen_KL_mean": -1123.677490234375, + "KL/mean": -1415.98095703125, + "KL/rejected_KL_mean": -1708.28466796875, + "KL/std": 1241.296630859375, + "epoch": 0.9691629955947136, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 584.6071166992188, + "fcm_dpo/q_t": 0.4897761940956116, + "grad_norm": 15.733861923217773, + "learning_rate": 1.592541096695571e-09, + "logits/chosen": -0.6595280170440674, + "logits/rejected": -0.6711503267288208, + "logps/chosen": -1182.725341796875, + "logps/ref_chosen": -59.04788589477539, + "logps/ref_rejected": -75.96005249023438, + "logps/rejected": -1784.24462890625, + "loss": 1.3485, + "margin_dpo/margin_mean": 584.6071166992188, + "margin_dpo/margin_std": 1485.17626953125, + "step": 660 + }, + { + "KL/chosen_KL_mean": -902.3294067382812, + "KL/mean": -1418.966796875, + "KL/rejected_KL_mean": -1935.60400390625, + "KL/std": 2257.097412109375, + "epoch": 0.9706314243759178, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1033.27490234375, + "fcm_dpo/q_t": 0.48256513476371765, + "grad_norm": 8.566376686096191, + "learning_rate": 1.4511920567963908e-09, + "logits/chosen": -0.6667207479476929, + "logits/rejected": -0.7094805240631104, + "logps/chosen": -953.00341796875, + "logps/ref_chosen": -50.673973083496094, + "logps/ref_rejected": -86.00569152832031, + "logps/rejected": -2021.60986328125, + "loss": 1.3242, + "margin_dpo/margin_mean": 1033.27490234375, + "margin_dpo/margin_std": 2576.364501953125, + "step": 661 + }, + { + "KL/chosen_KL_mean": -1044.2470703125, + "KL/mean": -1371.6861572265625, + "KL/rejected_KL_mean": -1699.1253662109375, + "KL/std": 1256.9990234375, + "epoch": 0.9720998531571219, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 654.878173828125, + "fcm_dpo/q_t": 0.48855116963386536, + "grad_norm": 4.699316501617432, + "learning_rate": 1.3163925091384532e-09, + "logits/chosen": -0.6130234003067017, + "logits/rejected": -0.6202554106712341, + "logps/chosen": -1113.5081787109375, + "logps/ref_chosen": -69.26106262207031, + "logps/ref_rejected": -89.05593872070312, + "logps/rejected": -1788.1812744140625, + "loss": 1.3441, + "margin_dpo/margin_mean": 654.8782348632812, + "margin_dpo/margin_std": 1609.4761962890625, + "step": 662 + }, + { + "KL/chosen_KL_mean": -954.5285034179688, + "KL/mean": -1455.802001953125, + "KL/rejected_KL_mean": -1957.0755615234375, + "KL/std": 1663.22412109375, + "epoch": 0.973568281938326, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1002.547119140625, + "fcm_dpo/q_t": 0.48265382647514343, + "grad_norm": 7.0070013999938965, + "learning_rate": 1.1881460058152382e-09, + "logits/chosen": -0.6588038206100464, + "logits/rejected": -0.7001410126686096, + "logps/chosen": -1019.4073486328125, + "logps/ref_chosen": -64.87890625, + "logps/ref_rejected": -113.92536926269531, + "logps/rejected": -2071.0009765625, + "loss": 1.3231, + "margin_dpo/margin_mean": 1002.5470581054688, + "margin_dpo/margin_std": 2189.273193359375, + "step": 663 + }, + { + "KL/chosen_KL_mean": -1293.694091796875, + "KL/mean": -1853.238525390625, + "KL/rejected_KL_mean": -2412.78271484375, + "KL/std": 2207.93994140625, + "epoch": 0.9750367107195301, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1119.088623046875, + "fcm_dpo/q_t": 0.4808417856693268, + "grad_norm": 5.824970722198486, + "learning_rate": 1.066455926241383e-09, + "logits/chosen": -0.6086447238922119, + "logits/rejected": -0.6509624123573303, + "logps/chosen": -1354.5826416015625, + "logps/ref_chosen": -60.88847351074219, + "logps/ref_rejected": -105.521728515625, + "logps/rejected": -2518.304443359375, + "loss": 1.3188, + "margin_dpo/margin_mean": 1119.088623046875, + "margin_dpo/margin_std": 2782.05810546875, + "step": 664 + }, + { + "KL/chosen_KL_mean": -998.0809326171875, + "KL/mean": -1378.9786376953125, + "KL/rejected_KL_mean": -1759.8763427734375, + "KL/std": 1507.407470703125, + "epoch": 0.9765051395007343, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 761.79541015625, + "fcm_dpo/q_t": 0.4867151081562042, + "grad_norm": 8.16478443145752, + "learning_rate": 9.513254770636137e-10, + "logits/chosen": -0.6453874111175537, + "logits/rejected": -0.682604193687439, + "logps/chosen": -1058.6451416015625, + "logps/ref_chosen": -60.56413269042969, + "logps/ref_rejected": -84.80882263183594, + "logps/rejected": -1844.68505859375, + "loss": 1.3374, + "margin_dpo/margin_mean": 761.79541015625, + "margin_dpo/margin_std": 1707.666748046875, + "step": 665 + }, + { + "KL/chosen_KL_mean": -1018.7044067382812, + "KL/mean": -1426.4305419921875, + "KL/rejected_KL_mean": -1834.1566162109375, + "KL/std": 1435.4952392578125, + "epoch": 0.9779735682819384, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 815.4522705078125, + "fcm_dpo/q_t": 0.48577576875686646, + "grad_norm": 5.858867645263672, + "learning_rate": 8.427576920763956e-10, + "logits/chosen": -0.6097148060798645, + "logits/rejected": -0.6266220808029175, + "logps/chosen": -1083.124267578125, + "logps/ref_chosen": -64.41996002197266, + "logps/ref_rejected": -95.8916244506836, + "logps/rejected": -1930.04833984375, + "loss": 1.3337, + "margin_dpo/margin_mean": 815.4522705078125, + "margin_dpo/margin_std": 1725.0411376953125, + "step": 666 + }, + { + "KL/chosen_KL_mean": -1350.70068359375, + "KL/mean": -1672.00048828125, + "KL/rejected_KL_mean": -1993.3004150390625, + "KL/std": 1688.798583984375, + "epoch": 0.9794419970631424, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 642.599853515625, + "fcm_dpo/q_t": 0.4888229966163635, + "grad_norm": 10.685932159423828, + "learning_rate": 7.407554321417764e-10, + "logits/chosen": -0.5980401039123535, + "logits/rejected": -0.6029761433601379, + "logps/chosen": -1419.9775390625, + "logps/ref_chosen": -69.27702331542969, + "logps/ref_rejected": -87.83549499511719, + "logps/rejected": -2081.1357421875, + "loss": 1.3471, + "margin_dpo/margin_mean": 642.599853515625, + "margin_dpo/margin_std": 2042.0830078125, + "step": 667 + }, + { + "KL/chosen_KL_mean": -1268.402099609375, + "KL/mean": -1488.158935546875, + "KL/rejected_KL_mean": -1707.915771484375, + "KL/std": 1302.4078369140625, + "epoch": 0.9809104258443465, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 439.513671875, + "fcm_dpo/q_t": 0.4923250377178192, + "grad_norm": 15.58211898803711, + "learning_rate": 6.453213851142225e-10, + "logits/chosen": -0.6111631989479065, + "logits/rejected": -0.6139740943908691, + "logps/chosen": -1341.006103515625, + "logps/ref_chosen": -72.60400390625, + "logps/ref_rejected": -103.73905944824219, + "logps/rejected": -1811.65478515625, + "loss": 1.3589, + "margin_dpo/margin_mean": 439.5137023925781, + "margin_dpo/margin_std": 1599.6151123046875, + "step": 668 + }, + { + "KL/chosen_KL_mean": -872.9949951171875, + "KL/mean": -1229.5106201171875, + "KL/rejected_KL_mean": -1586.026123046875, + "KL/std": 1136.226318359375, + "epoch": 0.9823788546255506, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 713.03125, + "fcm_dpo/q_t": 0.48755186796188354, + "grad_norm": 4.998262882232666, + "learning_rate": 5.564580657695939e-10, + "logits/chosen": -0.6233928203582764, + "logits/rejected": -0.6421560049057007, + "logps/chosen": -919.1114501953125, + "logps/ref_chosen": -46.116416931152344, + "logps/ref_rejected": -77.92434692382812, + "logps/rejected": -1663.9505615234375, + "loss": 1.3394, + "margin_dpo/margin_mean": 713.0311889648438, + "margin_dpo/margin_std": 1437.951904296875, + "step": 669 + }, + { + "KL/chosen_KL_mean": -1085.6097412109375, + "KL/mean": -1610.455810546875, + "KL/rejected_KL_mean": -2135.3017578125, + "KL/std": 1911.545166015625, + "epoch": 0.9838472834067548, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1049.692138671875, + "fcm_dpo/q_t": 0.48183974623680115, + "grad_norm": 7.568104267120361, + "learning_rate": 4.741678157389739e-10, + "logits/chosen": -0.6080718040466309, + "logits/rejected": -0.6510467529296875, + "logps/chosen": -1147.9554443359375, + "logps/ref_chosen": -62.34575271606445, + "logps/ref_rejected": -96.9405517578125, + "logps/rejected": -2232.242431640625, + "loss": 1.3207, + "margin_dpo/margin_mean": 1049.692138671875, + "margin_dpo/margin_std": 2298.290283203125, + "step": 670 + }, + { + "KL/chosen_KL_mean": -1065.5369873046875, + "KL/mean": -1437.20361328125, + "KL/rejected_KL_mean": -1808.870361328125, + "KL/std": 1385.409423828125, + "epoch": 0.9853157121879589, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 743.3333129882812, + "fcm_dpo/q_t": 0.4870268702507019, + "grad_norm": 8.674588203430176, + "learning_rate": 3.9845280344705245e-10, + "logits/chosen": -0.6010315418243408, + "logits/rejected": -0.6403799057006836, + "logps/chosen": -1113.537109375, + "logps/ref_chosen": -48.00010681152344, + "logps/ref_rejected": -83.81932067871094, + "logps/rejected": -1892.689697265625, + "loss": 1.3395, + "margin_dpo/margin_mean": 743.3333740234375, + "margin_dpo/margin_std": 1940.243896484375, + "step": 671 + }, + { + "KL/chosen_KL_mean": -1323.4586181640625, + "KL/mean": -1571.2021484375, + "KL/rejected_KL_mean": -1818.945556640625, + "KL/std": 1406.629150390625, + "epoch": 0.986784140969163, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 495.4870910644531, + "fcm_dpo/q_t": 0.4913468062877655, + "grad_norm": 15.005202293395996, + "learning_rate": 3.293150240547549e-10, + "logits/chosen": -0.6775461435317993, + "logits/rejected": -0.6908239126205444, + "logps/chosen": -1382.0418701171875, + "logps/ref_chosen": -58.58328628540039, + "logps/ref_rejected": -93.14015197753906, + "logps/rejected": -1912.085693359375, + "loss": 1.3548, + "margin_dpo/margin_mean": 495.4870300292969, + "margin_dpo/margin_std": 1540.307373046875, + "step": 672 + }, + { + "KL/chosen_KL_mean": -1022.36083984375, + "KL/mean": -1316.231689453125, + "KL/rejected_KL_mean": -1610.1025390625, + "KL/std": 1145.38427734375, + "epoch": 0.9882525697503671, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 587.74169921875, + "fcm_dpo/q_t": 0.4897216856479645, + "grad_norm": 6.545974254608154, + "learning_rate": 2.6675629940689504e-10, + "logits/chosen": -0.6184306740760803, + "logits/rejected": -0.6311053037643433, + "logps/chosen": -1069.083984375, + "logps/ref_chosen": -46.72320556640625, + "logps/ref_rejected": -85.29623413085938, + "logps/rejected": -1695.3988037109375, + "loss": 1.3476, + "margin_dpo/margin_mean": 587.74169921875, + "margin_dpo/margin_std": 1306.4228515625, + "step": 673 + }, + { + "KL/chosen_KL_mean": -847.9764404296875, + "KL/mean": -1195.31689453125, + "KL/rejected_KL_mean": -1542.6575927734375, + "KL/std": 1038.550048828125, + "epoch": 0.9897209985315712, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 694.6810302734375, + "fcm_dpo/q_t": 0.4878394603729248, + "grad_norm": 13.266809463500977, + "learning_rate": 2.1077827798404725e-10, + "logits/chosen": -0.6065397262573242, + "logits/rejected": -0.6238285303115845, + "logps/chosen": -893.4219970703125, + "logps/ref_chosen": -45.445526123046875, + "logps/ref_rejected": -70.04593658447266, + "logps/rejected": -1612.7034912109375, + "loss": 1.3403, + "margin_dpo/margin_mean": 694.6810302734375, + "margin_dpo/margin_std": 1318.276611328125, + "step": 674 + }, + { + "KL/chosen_KL_mean": -1095.146484375, + "KL/mean": -1370.689697265625, + "KL/rejected_KL_mean": -1646.23291015625, + "KL/std": 1590.983154296875, + "epoch": 0.9911894273127754, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 551.0866088867188, + "fcm_dpo/q_t": 0.4904383718967438, + "grad_norm": 9.25955867767334, + "learning_rate": 1.6138243485910863e-10, + "logits/chosen": -0.6318482756614685, + "logits/rejected": -0.6467227935791016, + "logps/chosen": -1139.32275390625, + "logps/ref_chosen": -44.17628479003906, + "logps/ref_rejected": -74.09197998046875, + "logps/rejected": -1720.324951171875, + "loss": 1.3534, + "margin_dpo/margin_mean": 551.0866088867188, + "margin_dpo/margin_std": 2041.986083984375, + "step": 675 + }, + { + "KL/chosen_KL_mean": -1177.5093994140625, + "KL/mean": -1528.5595703125, + "KL/rejected_KL_mean": -1879.60986328125, + "KL/std": 1375.15185546875, + "epoch": 0.9926578560939795, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 702.1004638671875, + "fcm_dpo/q_t": 0.4877719581127167, + "grad_norm": 5.452136516571045, + "learning_rate": 1.1857007165852472e-10, + "logits/chosen": -0.6294345259666443, + "logits/rejected": -0.6576676964759827, + "logps/chosen": -1248.907958984375, + "logps/ref_chosen": -71.39852905273438, + "logps/ref_rejected": -88.3587646484375, + "logps/rejected": -1967.968505859375, + "loss": 1.3415, + "margin_dpo/margin_mean": 702.1004028320312, + "margin_dpo/margin_std": 1760.55029296875, + "step": 676 + }, + { + "KL/chosen_KL_mean": -1092.815185546875, + "KL/mean": -1425.9197998046875, + "KL/rejected_KL_mean": -1759.0244140625, + "KL/std": 1277.3245849609375, + "epoch": 0.9941262848751835, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 666.209228515625, + "fcm_dpo/q_t": 0.4884171485900879, + "grad_norm": 10.614191055297852, + "learning_rate": 8.23423165278725e-11, + "logits/chosen": -0.6302313804626465, + "logits/rejected": -0.6473731994628906, + "logps/chosen": -1149.342529296875, + "logps/ref_chosen": -56.527435302734375, + "logps/ref_rejected": -78.22654724121094, + "logps/rejected": -1837.2509765625, + "loss": 1.3443, + "margin_dpo/margin_mean": 666.209228515625, + "margin_dpo/margin_std": 1806.208251953125, + "step": 677 + }, + { + "KL/chosen_KL_mean": -861.671630859375, + "KL/mean": -1364.0057373046875, + "KL/rejected_KL_mean": -1866.33984375, + "KL/std": 1516.4210205078125, + "epoch": 0.9955947136563876, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1004.6680908203125, + "fcm_dpo/q_t": 0.48245441913604736, + "grad_norm": 14.39037036895752, + "learning_rate": 5.270012410216185e-11, + "logits/chosen": -0.6077277064323425, + "logits/rejected": -0.6513392925262451, + "logps/chosen": -907.80615234375, + "logps/ref_chosen": -46.13447570800781, + "logps/ref_rejected": -80.60462951660156, + "logps/rejected": -1946.9443359375, + "loss": 1.3211, + "margin_dpo/margin_mean": 1004.6680908203125, + "margin_dpo/margin_std": 1805.224609375, + "step": 678 + }, + { + "KL/chosen_KL_mean": -1052.888671875, + "KL/mean": -1298.56982421875, + "KL/rejected_KL_mean": -1544.251220703125, + "KL/std": 1319.9560546875, + "epoch": 0.9970631424375918, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 491.36260986328125, + "fcm_dpo/q_t": 0.4915056824684143, + "grad_norm": 5.045799732208252, + "learning_rate": 2.9644275480772416e-11, + "logits/chosen": -0.6154893636703491, + "logits/rejected": -0.624566912651062, + "logps/chosen": -1103.18359375, + "logps/ref_chosen": -50.294921875, + "logps/ref_rejected": -76.59813690185547, + "logps/rejected": -1620.849365234375, + "loss": 1.356, + "margin_dpo/margin_mean": 491.36260986328125, + "margin_dpo/margin_std": 1768.9346923828125, + "step": 679 + }, + { + "KL/chosen_KL_mean": -1185.60595703125, + "KL/mean": -1536.31689453125, + "KL/rejected_KL_mean": -1887.0277099609375, + "KL/std": 1552.267822265625, + "epoch": 0.9985315712187959, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 701.421630859375, + "fcm_dpo/q_t": 0.4878092408180237, + "grad_norm": 5.114809989929199, + "learning_rate": 1.31753782067201e-11, + "logits/chosen": -0.6257789731025696, + "logits/rejected": -0.6503081321716309, + "logps/chosen": -1262.521728515625, + "logps/ref_chosen": -76.91569519042969, + "logps/ref_rejected": -112.384765625, + "logps/rejected": -1999.4124755859375, + "loss": 1.3422, + "margin_dpo/margin_mean": 701.421630859375, + "margin_dpo/margin_std": 1905.763427734375, + "step": 680 + }, + { + "KL/chosen_KL_mean": -1252.8399658203125, + "KL/mean": -1509.438720703125, + "KL/rejected_KL_mean": -1766.037353515625, + "KL/std": 1613.639404296875, + "epoch": 1.0, + "fcm_dpo/beta": 7.018564065219834e-05, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 513.1973876953125, + "fcm_dpo/q_t": 0.49122941493988037, + "grad_norm": 13.061714172363281, + "learning_rate": 3.2938662507808745e-12, + "logits/chosen": -0.6694597005844116, + "logits/rejected": -0.6893618702888489, + "logps/chosen": -1313.7972412109375, + "logps/ref_chosen": -60.957279205322266, + "logps/ref_rejected": -88.55797576904297, + "logps/rejected": -1854.5953369140625, + "loss": 1.3573, + "margin_dpo/margin_mean": 513.1973876953125, + "margin_dpo/margin_std": 2175.52685546875, + "step": 681 + }, + { + "epoch": 1.0, + "step": 681, + "total_flos": 0.0, + "train_loss": 1.3263754091949014, + "train_runtime": 1741.0925, + "train_samples_per_second": 25.041, + "train_steps_per_second": 0.391 + } + ], + "logging_steps": 1, + "max_steps": 681, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}