From 25ce01a846b6bddbc3fae29606dc2ce5ad9e3f98 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Wed, 13 May 2026 18:55:34 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.5 Source: Original Platform --- .gitattributes | 36 + README.md | 62 + all_results.json | 9 + config.json | 29 + generation_config.json | 9 + model-00001-of-00007.safetensors | 3 + model-00002-of-00007.safetensors | 3 + model-00003-of-00007.safetensors | 3 + model-00004-of-00007.safetensors | 3 + model-00005-of-00007.safetensors | 3 + model-00006-of-00007.safetensors | 3 + model-00007-of-00007.safetensors | 3 + model.safetensors.index.json | 298 + special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2064 ++++ train_results.json | 9 + trainer_state.json | 15706 +++++++++++++++++++++++++++++ 18 files changed, 18269 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00007.safetensors create mode 100644 model-00002-of-00007.safetensors create mode 100644 model-00003-of-00007.safetensors create mode 100644 model-00004-of-00007.safetensors create mode 100644 model-00005-of-00007.safetensors create mode 100644 model-00006-of-00007.safetensors create mode 100644 model-00007-of-00007.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..227b5ec --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-helpful-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.5 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.5 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-helpful-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-helpful-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..d48c058 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.1259927408285604, + "train_runtime": 1736.5167, + "train_samples": 43598, + "train_samples_per_second": 25.107, + "train_steps_per_second": 0.392 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..ee8aa0d --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7fe3338e9e5fc4ae93c5e0fdbe36287033c74f756faa0adbd171f92e5d32f01 +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..1edd0f9 --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e0c434ea0d466ce6f16560dd8b19a7de7360dbe9980dd21ec0240f175b501f5 +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..edeed88 --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab3a06d7cf86f967be34241415848d4248e4e4e7d3e25a81ca47336df3e41217 +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..3ebf432 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9626f1c17916a643a4e6b58bc9f275393ae92c2349395f2b313c6c7901c61792 +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..e8ab34d --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5b473c08de2ec8f366f0448ecf0e3f8bbcd3ce7b308e39c897519da5a2f39cd +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..362f8aa --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9b95759bd26513b83c5435daa18f963a56c4797f5a377db26bb86c4adfc3092 +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..6d0da9f --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd5f97fc111f5c8e47315982fb0d26ac56bc38c81eca0342adb2978e7220dd75 +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..d48c058 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.1259927408285604, + "train_runtime": 1736.5167, + "train_samples": 43598, + "train_samples_per_second": 25.107, + "train_steps_per_second": 0.392 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..35ae0c1 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 200, + "global_step": 681, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.00527191162109375, + "KL/mean": 0.016706019639968872, + "KL/rejected_KL_mean": 0.028141021728515625, + "KL/std": 0.272699236869812, + "epoch": 0.0014684287812041115, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02287006378173828, + "fcm_dpo/q_t": 0.5005706548690796, + "grad_norm": 83.51216888427734, + "learning_rate": 0.0, + "logits/chosen": -0.4974287748336792, + "logits/rejected": -0.43299180269241333, + "logps/chosen": -50.1435661315918, + "logps/ref_chosen": -50.14883804321289, + "logps/ref_rejected": -74.1280517578125, + "logps/rejected": -74.09991455078125, + "loss": 1.389, + "margin_dpo/margin_mean": -0.02287048101425171, + "margin_dpo/margin_std": 0.41920793056488037, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.03498649597167969, + "KL/mean": -0.00212840735912323, + "KL/rejected_KL_mean": 0.030735015869140625, + "KL/std": 0.24797174334526062, + "epoch": 0.002936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06572261452674866, + "fcm_dpo/q_t": 0.5016425848007202, + "grad_norm": 72.19176483154297, + "learning_rate": 7.246376811594203e-09, + "logits/chosen": -0.49536412954330444, + "logits/rejected": -0.4594460427761078, + "logps/chosen": -52.65568923950195, + "logps/ref_chosen": -52.620704650878906, + "logps/ref_rejected": -75.30413818359375, + "logps/rejected": -75.27340698242188, + "loss": 1.3932, + "margin_dpo/margin_mean": -0.06572240591049194, + "margin_dpo/margin_std": 0.35048407316207886, + "step": 2 + }, + { + "KL/chosen_KL_mean": 0.011138916015625, + "KL/mean": 0.01624371111392975, + "KL/rejected_KL_mean": 0.021350860595703125, + "KL/std": 0.2411436140537262, + "epoch": 0.004405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.010211020708084106, + "fcm_dpo/q_t": 0.5002557039260864, + "grad_norm": 71.00299072265625, + "learning_rate": 1.4492753623188406e-08, + "logits/chosen": -0.48165208101272583, + "logits/rejected": -0.44220495223999023, + "logps/chosen": -60.970458984375, + "logps/ref_chosen": -60.981597900390625, + "logps/ref_rejected": -68.67259216308594, + "logps/rejected": -68.6512451171875, + "loss": 1.3877, + "margin_dpo/margin_mean": -0.010210752487182617, + "margin_dpo/margin_std": 0.36640018224716187, + "step": 3 + }, + { + "KL/chosen_KL_mean": 0.000141143798828125, + "KL/mean": 0.032934680581092834, + "KL/rejected_KL_mean": 0.06572723388671875, + "KL/std": 0.26030904054641724, + "epoch": 0.005873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06558361649513245, + "fcm_dpo/q_t": 0.501638650894165, + "grad_norm": 72.05967712402344, + "learning_rate": 2.1739130434782606e-08, + "logits/chosen": -0.46887367963790894, + "logits/rejected": -0.4411189556121826, + "logps/chosen": -56.76757049560547, + "logps/ref_chosen": -56.7677116394043, + "logps/ref_rejected": -86.64710998535156, + "logps/rejected": -86.58138275146484, + "loss": 1.3932, + "margin_dpo/margin_mean": -0.06558331847190857, + "margin_dpo/margin_std": 0.38221287727355957, + "step": 4 + }, + { + "KL/chosen_KL_mean": -0.008298873901367188, + "KL/mean": 0.015575245022773743, + "KL/rejected_KL_mean": 0.03945159912109375, + "KL/std": 0.26084285974502563, + "epoch": 0.007342143906020558, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.04775431752204895, + "fcm_dpo/q_t": 0.5011929273605347, + "grad_norm": 89.86566925048828, + "learning_rate": 2.898550724637681e-08, + "logits/chosen": -0.5145574808120728, + "logits/rejected": -0.47081369161605835, + "logps/chosen": -53.86767578125, + "logps/ref_chosen": -53.859375, + "logps/ref_rejected": -84.14918518066406, + "logps/rejected": -84.10973358154297, + "loss": 1.3914, + "margin_dpo/margin_mean": -0.04775422811508179, + "margin_dpo/margin_std": 0.37420204281806946, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.02581787109375, + "KL/mean": 0.0013223886489868164, + "KL/rejected_KL_mean": 0.02846527099609375, + "KL/std": 0.254623144865036, + "epoch": 0.00881057268722467, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.05428045988082886, + "fcm_dpo/q_t": 0.5013556480407715, + "grad_norm": 91.9102554321289, + "learning_rate": 3.6231884057971014e-08, + "logits/chosen": -0.5099787712097168, + "logits/rejected": -0.4680352807044983, + "logps/chosen": -63.033302307128906, + "logps/ref_chosen": -63.007484436035156, + "logps/ref_rejected": -92.64534759521484, + "logps/rejected": -92.61688232421875, + "loss": 1.3921, + "margin_dpo/margin_mean": -0.05428001284599304, + "margin_dpo/margin_std": 0.3631099462509155, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.014619827270507812, + "KL/mean": 0.007828325033187866, + "KL/rejected_KL_mean": 0.00103759765625, + "KL/std": 0.28590360283851624, + "epoch": 0.010279001468428781, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.013581156730651855, + "fcm_dpo/q_t": 0.4996616840362549, + "grad_norm": 82.45801544189453, + "learning_rate": 4.347826086956521e-08, + "logits/chosen": -0.5054452419281006, + "logits/rejected": -0.47154390811920166, + "logps/chosen": -57.76020050048828, + "logps/ref_chosen": -57.774818420410156, + "logps/ref_rejected": -103.92059326171875, + "logps/rejected": -103.9195556640625, + "loss": 1.3854, + "margin_dpo/margin_mean": 0.01358148455619812, + "margin_dpo/margin_std": 0.4422546327114105, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.01812744140625, + "KL/mean": 0.019948050379753113, + "KL/rejected_KL_mean": 0.021778106689453125, + "KL/std": 0.27578145265579224, + "epoch": 0.011747430249632892, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.003651231527328491, + "fcm_dpo/q_t": 0.5000911951065063, + "grad_norm": 79.03565216064453, + "learning_rate": 5.0724637681159424e-08, + "logits/chosen": -0.49302494525909424, + "logits/rejected": -0.46536964178085327, + "logps/chosen": -58.697906494140625, + "logps/ref_chosen": -58.716033935546875, + "logps/ref_rejected": -79.3114242553711, + "logps/rejected": -79.28964233398438, + "loss": 1.387, + "margin_dpo/margin_mean": -0.0036510825157165527, + "margin_dpo/margin_std": 0.3407641053199768, + "step": 8 + }, + { + "KL/chosen_KL_mean": 0.00136566162109375, + "KL/mean": 0.0017981678247451782, + "KL/rejected_KL_mean": 0.002231597900390625, + "KL/std": 0.2728290259838104, + "epoch": 0.013215859030837005, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.000867694616317749, + "fcm_dpo/q_t": 0.5000215768814087, + "grad_norm": 85.38078308105469, + "learning_rate": 5.797101449275362e-08, + "logits/chosen": -0.48713457584381104, + "logits/rejected": -0.4398614168167114, + "logps/chosen": -69.865478515625, + "logps/ref_chosen": -69.8668441772461, + "logps/ref_rejected": -99.6026611328125, + "logps/rejected": -99.60043334960938, + "loss": 1.3868, + "margin_dpo/margin_mean": -0.0008689463138580322, + "margin_dpo/margin_std": 0.41773098707199097, + "step": 9 + }, + { + "KL/chosen_KL_mean": -0.0075225830078125, + "KL/mean": -0.013112656772136688, + "KL/rejected_KL_mean": -0.018695831298828125, + "KL/std": 0.253761887550354, + "epoch": 0.014684287812041116, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.01117675006389618, + "fcm_dpo/q_t": 0.4997211694717407, + "grad_norm": 70.45758819580078, + "learning_rate": 6.521739130434782e-08, + "logits/chosen": -0.498451828956604, + "logits/rejected": -0.4553406834602356, + "logps/chosen": -48.36520767211914, + "logps/ref_chosen": -48.35768508911133, + "logps/ref_rejected": -80.37206268310547, + "logps/rejected": -80.39076232910156, + "loss": 1.3855, + "margin_dpo/margin_mean": 0.011177659034729004, + "margin_dpo/margin_std": 0.3267936706542969, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.0040302276611328125, + "KL/mean": -0.007809385657310486, + "KL/rejected_KL_mean": -0.019649505615234375, + "KL/std": 0.23007726669311523, + "epoch": 0.016152716593245228, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.023678749799728394, + "fcm_dpo/q_t": 0.49940839409828186, + "grad_norm": 68.75594329833984, + "learning_rate": 7.246376811594203e-08, + "logits/chosen": -0.45761120319366455, + "logits/rejected": -0.4324725270271301, + "logps/chosen": -53.01282501220703, + "logps/ref_chosen": -53.01685333251953, + "logps/ref_rejected": -87.78038024902344, + "logps/rejected": -87.8000259399414, + "loss": 1.3842, + "margin_dpo/margin_mean": 0.02367931604385376, + "margin_dpo/margin_std": 0.3247877359390259, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.022077560424804688, + "KL/mean": -0.030066579580307007, + "KL/rejected_KL_mean": -0.038055419921875, + "KL/std": 0.2843082547187805, + "epoch": 0.01762114537444934, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.015976309776306152, + "fcm_dpo/q_t": 0.4996001422405243, + "grad_norm": 89.8901596069336, + "learning_rate": 7.971014492753623e-08, + "logits/chosen": -0.5207371711730957, + "logits/rejected": -0.48408442735671997, + "logps/chosen": -61.8275146484375, + "logps/ref_chosen": -61.80543518066406, + "logps/ref_rejected": -104.8582763671875, + "logps/rejected": -104.89633178710938, + "loss": 1.3851, + "margin_dpo/margin_mean": 0.015975654125213623, + "margin_dpo/margin_std": 0.39000314474105835, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.018079757690429688, + "KL/mean": 0.03311806917190552, + "KL/rejected_KL_mean": 0.04815673828125, + "KL/std": 0.299828439950943, + "epoch": 0.01908957415565345, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.030076265335083008, + "fcm_dpo/q_t": 0.5007518529891968, + "grad_norm": 79.48653411865234, + "learning_rate": 8.695652173913042e-08, + "logits/chosen": -0.46800822019577026, + "logits/rejected": -0.4389190673828125, + "logps/chosen": -64.24227905273438, + "logps/ref_chosen": -64.2603530883789, + "logps/ref_rejected": -87.20307922363281, + "logps/rejected": -87.15492248535156, + "loss": 1.3898, + "margin_dpo/margin_mean": -0.030076295137405396, + "margin_dpo/margin_std": 0.40763989090919495, + "step": 13 + }, + { + "KL/chosen_KL_mean": -0.027566909790039062, + "KL/mean": -0.022734537720680237, + "KL/rejected_KL_mean": -0.01790618896484375, + "KL/std": 0.27561646699905396, + "epoch": 0.020558002936857563, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.009662479162216187, + "fcm_dpo/q_t": 0.5002409219741821, + "grad_norm": 85.24372863769531, + "learning_rate": 9.420289855072464e-08, + "logits/chosen": -0.46077555418014526, + "logits/rejected": -0.420840322971344, + "logps/chosen": -58.137779235839844, + "logps/ref_chosen": -58.11021041870117, + "logps/ref_rejected": -104.04708099365234, + "logps/rejected": -104.06498718261719, + "loss": 1.3878, + "margin_dpo/margin_mean": -0.009661853313446045, + "margin_dpo/margin_std": 0.4318341016769409, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.0386199951171875, + "KL/mean": -0.045700281858444214, + "KL/rejected_KL_mean": -0.052783966064453125, + "KL/std": 0.23048102855682373, + "epoch": 0.022026431718061675, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.014169454574584961, + "fcm_dpo/q_t": 0.49964454770088196, + "grad_norm": 64.31354522705078, + "learning_rate": 1.0144927536231885e-07, + "logits/chosen": -0.5125108361244202, + "logits/rejected": -0.49486109614372253, + "logps/chosen": -57.005531311035156, + "logps/ref_chosen": -56.96691131591797, + "logps/ref_rejected": -80.80863952636719, + "logps/rejected": -80.8614273071289, + "loss": 1.3852, + "margin_dpo/margin_mean": 0.014169484376907349, + "margin_dpo/margin_std": 0.3576519191265106, + "step": 15 + }, + { + "KL/chosen_KL_mean": 0.019407272338867188, + "KL/mean": -0.03246238827705383, + "KL/rejected_KL_mean": -0.08432769775390625, + "KL/std": 0.27561452984809875, + "epoch": 0.023494860499265784, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.10373848676681519, + "fcm_dpo/q_t": 0.49740785360336304, + "grad_norm": 83.85110473632812, + "learning_rate": 1.0869565217391303e-07, + "logits/chosen": -0.4948147237300873, + "logits/rejected": -0.45085063576698303, + "logps/chosen": -61.72048568725586, + "logps/ref_chosen": -61.739891052246094, + "logps/ref_rejected": -84.36947631835938, + "logps/rejected": -84.45379638671875, + "loss": 1.3763, + "margin_dpo/margin_mean": 0.10373827815055847, + "margin_dpo/margin_std": 0.38368722796440125, + "step": 16 + }, + { + "KL/chosen_KL_mean": 0.032100677490234375, + "KL/mean": -0.02624189853668213, + "KL/rejected_KL_mean": -0.08458328247070312, + "KL/std": 0.2859901189804077, + "epoch": 0.024963289280469897, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.11668473482131958, + "fcm_dpo/q_t": 0.497084379196167, + "grad_norm": 78.15997314453125, + "learning_rate": 1.1594202898550725e-07, + "logits/chosen": -0.511766791343689, + "logits/rejected": -0.475982666015625, + "logps/chosen": -67.67823791503906, + "logps/ref_chosen": -67.71033477783203, + "logps/ref_rejected": -85.37865447998047, + "logps/rejected": -85.46324157714844, + "loss": 1.3751, + "margin_dpo/margin_mean": 0.11668440699577332, + "margin_dpo/margin_std": 0.39858338236808777, + "step": 17 + }, + { + "KL/chosen_KL_mean": 0.030401229858398438, + "KL/mean": -0.01864929497241974, + "KL/rejected_KL_mean": -0.06769943237304688, + "KL/std": 0.24296830594539642, + "epoch": 0.02643171806167401, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.09810277819633484, + "fcm_dpo/q_t": 0.4975478947162628, + "grad_norm": 81.9465560913086, + "learning_rate": 1.2318840579710146e-07, + "logits/chosen": -0.5076760053634644, + "logits/rejected": -0.4538722038269043, + "logps/chosen": -47.70909118652344, + "logps/ref_chosen": -47.7394905090332, + "logps/ref_rejected": -75.4722900390625, + "logps/rejected": -75.53998565673828, + "loss": 1.3768, + "margin_dpo/margin_mean": 0.09810280799865723, + "margin_dpo/margin_std": 0.3186327815055847, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.016094207763671875, + "KL/mean": -0.009688735008239746, + "KL/rejected_KL_mean": -0.035465240478515625, + "KL/std": 0.24112674593925476, + "epoch": 0.027900146842878122, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05155247449874878, + "fcm_dpo/q_t": 0.49871158599853516, + "grad_norm": 73.13494110107422, + "learning_rate": 1.3043478260869563e-07, + "logits/chosen": -0.5065875053405762, + "logits/rejected": -0.4579053223133087, + "logps/chosen": -70.18927001953125, + "logps/ref_chosen": -70.20536041259766, + "logps/ref_rejected": -89.7575912475586, + "logps/rejected": -89.79306030273438, + "loss": 1.3814, + "margin_dpo/margin_mean": 0.05155327916145325, + "margin_dpo/margin_std": 0.3446183204650879, + "step": 19 + }, + { + "KL/chosen_KL_mean": -0.03119659423828125, + "KL/mean": -0.04489029943943024, + "KL/rejected_KL_mean": -0.0585784912109375, + "KL/std": 0.27683231234550476, + "epoch": 0.02936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.027387112379074097, + "fcm_dpo/q_t": 0.4993155002593994, + "grad_norm": 73.76658630371094, + "learning_rate": 1.3768115942028986e-07, + "logits/chosen": -0.5554596185684204, + "logits/rejected": -0.4994776248931885, + "logps/chosen": -50.83443832397461, + "logps/ref_chosen": -50.80324172973633, + "logps/ref_rejected": -78.82334899902344, + "logps/rejected": -78.88192749023438, + "loss": 1.3839, + "margin_dpo/margin_mean": 0.027386903762817383, + "margin_dpo/margin_std": 0.36115431785583496, + "step": 20 + }, + { + "KL/chosen_KL_mean": 0.008577346801757812, + "KL/mean": -0.03955095261335373, + "KL/rejected_KL_mean": -0.087677001953125, + "KL/std": 0.25458982586860657, + "epoch": 0.030837004405286344, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0962568074464798, + "fcm_dpo/q_t": 0.49759435653686523, + "grad_norm": 77.66507720947266, + "learning_rate": 1.4492753623188405e-07, + "logits/chosen": -0.4872417449951172, + "logits/rejected": -0.46275877952575684, + "logps/chosen": -50.054439544677734, + "logps/ref_chosen": -50.063018798828125, + "logps/ref_rejected": -77.86878967285156, + "logps/rejected": -77.95646667480469, + "loss": 1.3769, + "margin_dpo/margin_mean": 0.09625642001628876, + "margin_dpo/margin_std": 0.3011889159679413, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.05830574035644531, + "KL/mean": -0.05371597409248352, + "KL/rejected_KL_mean": -0.16573715209960938, + "KL/std": 0.32429108023643494, + "epoch": 0.032305433186490456, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.22404277324676514, + "fcm_dpo/q_t": 0.494402676820755, + "grad_norm": 84.32189178466797, + "learning_rate": 1.5217391304347825e-07, + "logits/chosen": -0.502210259437561, + "logits/rejected": -0.459368497133255, + "logps/chosen": -58.99932861328125, + "logps/ref_chosen": -59.05763626098633, + "logps/ref_rejected": -97.50466918945312, + "logps/rejected": -97.67041015625, + "loss": 1.3646, + "margin_dpo/margin_mean": 0.22404247522354126, + "margin_dpo/margin_std": 0.4611518979072571, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.014989852905273438, + "KL/mean": -0.061570554971694946, + "KL/rejected_KL_mean": -0.13812637329101562, + "KL/std": 0.32376909255981445, + "epoch": 0.033773861967694566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1531180739402771, + "fcm_dpo/q_t": 0.49617335200309753, + "grad_norm": 80.89447021484375, + "learning_rate": 1.5942028985507245e-07, + "logits/chosen": -0.4944395422935486, + "logits/rejected": -0.4721882939338684, + "logps/chosen": -60.062705993652344, + "logps/ref_chosen": -60.07769775390625, + "logps/ref_rejected": -81.13955688476562, + "logps/rejected": -81.27767944335938, + "loss": 1.3716, + "margin_dpo/margin_mean": 0.1531185805797577, + "margin_dpo/margin_std": 0.4450337886810303, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.009403228759765625, + "KL/mean": -0.08805283904075623, + "KL/rejected_KL_mean": -0.18550872802734375, + "KL/std": 0.26540276408195496, + "epoch": 0.03524229074889868, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.19491511583328247, + "fcm_dpo/q_t": 0.4951290786266327, + "grad_norm": 82.0745849609375, + "learning_rate": 1.6666666666666665e-07, + "logits/chosen": -0.49009591341018677, + "logits/rejected": -0.47387370467185974, + "logps/chosen": -44.28163146972656, + "logps/ref_chosen": -44.29103469848633, + "logps/ref_rejected": -99.12521362304688, + "logps/rejected": -99.31072235107422, + "loss": 1.3672, + "margin_dpo/margin_mean": 0.19491589069366455, + "margin_dpo/margin_std": 0.3483674228191376, + "step": 24 + }, + { + "KL/chosen_KL_mean": 0.0049686431884765625, + "KL/mean": -0.09683172404766083, + "KL/rejected_KL_mean": -0.19862747192382812, + "KL/std": 0.31956106424331665, + "epoch": 0.03671071953010279, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.20359697937965393, + "fcm_dpo/q_t": 0.4949125647544861, + "grad_norm": 73.71135711669922, + "learning_rate": 1.7391304347826085e-07, + "logits/chosen": -0.48128461837768555, + "logits/rejected": -0.45070409774780273, + "logps/chosen": -52.532081604003906, + "logps/ref_chosen": -52.537052154541016, + "logps/ref_rejected": -89.34219360351562, + "logps/rejected": -89.54081726074219, + "loss": 1.3664, + "margin_dpo/margin_mean": 0.20359739661216736, + "margin_dpo/margin_std": 0.36989957094192505, + "step": 25 + }, + { + "KL/chosen_KL_mean": 0.07361602783203125, + "KL/mean": -0.10151402652263641, + "KL/rejected_KL_mean": -0.2766456604003906, + "KL/std": 0.41175055503845215, + "epoch": 0.0381791483113069, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3502626121044159, + "fcm_dpo/q_t": 0.49125543236732483, + "grad_norm": 87.7682876586914, + "learning_rate": 1.8115942028985507e-07, + "logits/chosen": -0.5210403800010681, + "logits/rejected": -0.48897308111190796, + "logps/chosen": -53.84918975830078, + "logps/ref_chosen": -53.92280578613281, + "logps/ref_rejected": -103.35971069335938, + "logps/rejected": -103.63636016845703, + "loss": 1.3523, + "margin_dpo/margin_mean": 0.3502630591392517, + "margin_dpo/margin_std": 0.549583911895752, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.12030792236328125, + "KL/mean": -0.14729268848896027, + "KL/rejected_KL_mean": -0.4149017333984375, + "KL/std": 0.4725148677825928, + "epoch": 0.039647577092511016, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5352023839950562, + "fcm_dpo/q_t": 0.4866352677345276, + "grad_norm": 93.7070541381836, + "learning_rate": 1.8840579710144927e-07, + "logits/chosen": -0.4835624098777771, + "logits/rejected": -0.4467238783836365, + "logps/chosen": -42.778221130371094, + "logps/ref_chosen": -42.898529052734375, + "logps/ref_rejected": -98.72419738769531, + "logps/rejected": -99.13909912109375, + "loss": 1.3343, + "margin_dpo/margin_mean": 0.535202145576477, + "margin_dpo/margin_std": 0.5412513017654419, + "step": 27 + }, + { + "KL/chosen_KL_mean": -0.009973526000976562, + "KL/mean": -0.14437878131866455, + "KL/rejected_KL_mean": -0.27878570556640625, + "KL/std": 0.4356634020805359, + "epoch": 0.041116005873715125, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.26881152391433716, + "fcm_dpo/q_t": 0.4932839870452881, + "grad_norm": 75.4567642211914, + "learning_rate": 1.9565217391304347e-07, + "logits/chosen": -0.5053049921989441, + "logits/rejected": -0.4499465525150299, + "logps/chosen": -60.56647491455078, + "logps/ref_chosen": -60.55650329589844, + "logps/ref_rejected": -91.40111541748047, + "logps/rejected": -91.67990112304688, + "loss": 1.3606, + "margin_dpo/margin_mean": 0.26881229877471924, + "margin_dpo/margin_std": 0.6206564903259277, + "step": 28 + }, + { + "KL/chosen_KL_mean": 0.102294921875, + "KL/mean": -0.17882275581359863, + "KL/rejected_KL_mean": -0.4599418640136719, + "KL/std": 0.5050041079521179, + "epoch": 0.042584434654919234, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5622385144233704, + "fcm_dpo/q_t": 0.4859647750854492, + "grad_norm": 90.32099151611328, + "learning_rate": 2.028985507246377e-07, + "logits/chosen": -0.5919088125228882, + "logits/rejected": -0.5483555793762207, + "logps/chosen": -57.70549011230469, + "logps/ref_chosen": -57.80778503417969, + "logps/ref_rejected": -97.39434814453125, + "logps/rejected": -97.85429382324219, + "loss": 1.3318, + "margin_dpo/margin_mean": 0.562239408493042, + "margin_dpo/margin_std": 0.5976601839065552, + "step": 29 + }, + { + "KL/chosen_KL_mean": 0.1593170166015625, + "KL/mean": -0.19309324026107788, + "KL/rejected_KL_mean": -0.5455055236816406, + "KL/std": 0.6001869440078735, + "epoch": 0.04405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7048233151435852, + "fcm_dpo/q_t": 0.48240745067596436, + "grad_norm": 87.2791519165039, + "learning_rate": 2.1014492753623187e-07, + "logits/chosen": -0.4931980073451996, + "logits/rejected": -0.4624643921852112, + "logps/chosen": -52.418052673339844, + "logps/ref_chosen": -52.577369689941406, + "logps/ref_rejected": -98.48920440673828, + "logps/rejected": -99.03471374511719, + "loss": 1.3181, + "margin_dpo/margin_mean": 0.7048235535621643, + "margin_dpo/margin_std": 0.6384134292602539, + "step": 30 + }, + { + "KL/chosen_KL_mean": 0.028833389282226562, + "KL/mean": -0.1849493831396103, + "KL/rejected_KL_mean": -0.39873504638671875, + "KL/std": 0.5692216157913208, + "epoch": 0.04552129221732746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.42756929993629456, + "fcm_dpo/q_t": 0.489329993724823, + "grad_norm": 67.7244873046875, + "learning_rate": 2.1739130434782607e-07, + "logits/chosen": -0.5044924020767212, + "logits/rejected": -0.4595579504966736, + "logps/chosen": -63.7780876159668, + "logps/ref_chosen": -63.806922912597656, + "logps/ref_rejected": -72.89400482177734, + "logps/rejected": -73.29273986816406, + "loss": 1.3454, + "margin_dpo/margin_mean": 0.42756909132003784, + "margin_dpo/margin_std": 0.738195538520813, + "step": 31 + }, + { + "KL/chosen_KL_mean": 0.25072669982910156, + "KL/mean": -0.14465753734111786, + "KL/rejected_KL_mean": -0.5400428771972656, + "KL/std": 0.7525659799575806, + "epoch": 0.04698972099853157, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7907640933990479, + "fcm_dpo/q_t": 0.4803106188774109, + "grad_norm": 83.765625, + "learning_rate": 2.2463768115942027e-07, + "logits/chosen": -0.5086926221847534, + "logits/rejected": -0.46712368726730347, + "logps/chosen": -62.488800048828125, + "logps/ref_chosen": -62.739524841308594, + "logps/ref_rejected": -89.3175048828125, + "logps/rejected": -89.8575439453125, + "loss": 1.311, + "margin_dpo/margin_mean": 0.790764331817627, + "margin_dpo/margin_std": 0.9426593780517578, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.07011795043945312, + "KL/mean": -0.21620376408100128, + "KL/rejected_KL_mean": -0.5025215148925781, + "KL/std": 0.5353639125823975, + "epoch": 0.048458149779735685, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.572637677192688, + "fcm_dpo/q_t": 0.4857015609741211, + "grad_norm": 72.43000030517578, + "learning_rate": 2.318840579710145e-07, + "logits/chosen": -0.46579599380493164, + "logits/rejected": -0.4388483166694641, + "logps/chosen": -53.19085693359375, + "logps/ref_chosen": -53.26097106933594, + "logps/ref_rejected": -87.8851318359375, + "logps/rejected": -88.38765716552734, + "loss": 1.3308, + "margin_dpo/margin_mean": 0.5726377964019775, + "margin_dpo/margin_std": 0.5959737300872803, + "step": 33 + }, + { + "KL/chosen_KL_mean": 0.08471107482910156, + "KL/mean": -0.3118361830711365, + "KL/rejected_KL_mean": -0.7083892822265625, + "KL/std": 0.7339533567428589, + "epoch": 0.049926578560939794, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7930940389633179, + "fcm_dpo/q_t": 0.48022860288619995, + "grad_norm": 77.76030731201172, + "learning_rate": 2.391304347826087e-07, + "logits/chosen": -0.45824792981147766, + "logits/rejected": -0.4408394694328308, + "logps/chosen": -50.73261642456055, + "logps/ref_chosen": -50.81732940673828, + "logps/ref_rejected": -101.92184448242188, + "logps/rejected": -102.63023376464844, + "loss": 1.3103, + "margin_dpo/margin_mean": 0.7930936813354492, + "margin_dpo/margin_std": 0.8268696069717407, + "step": 34 + }, + { + "KL/chosen_KL_mean": 0.15523910522460938, + "KL/mean": -0.4550577700138092, + "KL/rejected_KL_mean": -1.0653495788574219, + "KL/std": 1.036551594734192, + "epoch": 0.0513950073421439, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2205886840820312, + "fcm_dpo/q_t": 0.46972745656967163, + "grad_norm": 82.48926544189453, + "learning_rate": 2.463768115942029e-07, + "logits/chosen": -0.5093280076980591, + "logits/rejected": -0.47247546911239624, + "logps/chosen": -50.8692512512207, + "logps/ref_chosen": -51.02449035644531, + "logps/ref_rejected": -106.82443237304688, + "logps/rejected": -107.88977813720703, + "loss": 1.2718, + "margin_dpo/margin_mean": 1.2205885648727417, + "margin_dpo/margin_std": 1.2264792919158936, + "step": 35 + }, + { + "KL/chosen_KL_mean": 0.03851318359375, + "KL/mean": -0.5172837972640991, + "KL/rejected_KL_mean": -1.0730781555175781, + "KL/std": 1.0524473190307617, + "epoch": 0.05286343612334802, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1115931272506714, + "fcm_dpo/q_t": 0.47235026955604553, + "grad_norm": 73.26398468017578, + "learning_rate": 2.536231884057971e-07, + "logits/chosen": -0.5541160106658936, + "logits/rejected": -0.5175681114196777, + "logps/chosen": -51.952980041503906, + "logps/ref_chosen": -51.991493225097656, + "logps/ref_rejected": -86.0406265258789, + "logps/rejected": -87.11370849609375, + "loss": 1.282, + "margin_dpo/margin_mean": 1.1115922927856445, + "margin_dpo/margin_std": 1.172994613647461, + "step": 36 + }, + { + "KL/chosen_KL_mean": 0.0073757171630859375, + "KL/mean": -0.5105187892913818, + "KL/rejected_KL_mean": -1.0284156799316406, + "KL/std": 1.0426149368286133, + "epoch": 0.05433186490455213, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0357892513275146, + "fcm_dpo/q_t": 0.4743063151836395, + "grad_norm": 61.77984619140625, + "learning_rate": 2.6086956521739126e-07, + "logits/chosen": -0.511062502861023, + "logits/rejected": -0.4673815965652466, + "logps/chosen": -62.79973220825195, + "logps/ref_chosen": -62.807106018066406, + "logps/ref_rejected": -77.89507293701172, + "logps/rejected": -78.92349243164062, + "loss": 1.2901, + "margin_dpo/margin_mean": 1.0357893705368042, + "margin_dpo/margin_std": 1.3421692848205566, + "step": 37 + }, + { + "KL/chosen_KL_mean": 0.18421363830566406, + "KL/mean": -0.5241457223892212, + "KL/rejected_KL_mean": -1.2325019836425781, + "KL/std": 1.3683629035949707, + "epoch": 0.055800293685756244, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.4167163372039795, + "fcm_dpo/q_t": 0.4650591313838959, + "grad_norm": 69.80412292480469, + "learning_rate": 2.681159420289855e-07, + "logits/chosen": -0.5317132472991943, + "logits/rejected": -0.4989948868751526, + "logps/chosen": -48.20630645751953, + "logps/ref_chosen": -48.39051818847656, + "logps/ref_rejected": -97.91244506835938, + "logps/rejected": -99.14494323730469, + "loss": 1.2566, + "margin_dpo/margin_mean": 1.4167169332504272, + "margin_dpo/margin_std": 1.6653265953063965, + "step": 38 + }, + { + "KL/chosen_KL_mean": 0.10546493530273438, + "KL/mean": -0.7446496486663818, + "KL/rejected_KL_mean": -1.5947608947753906, + "KL/std": 1.3329132795333862, + "epoch": 0.05726872246696035, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.7002233266830444, + "fcm_dpo/q_t": 0.45783674716949463, + "grad_norm": 76.12843322753906, + "learning_rate": 2.753623188405797e-07, + "logits/chosen": -0.5526484847068787, + "logits/rejected": -0.5124742984771729, + "logps/chosen": -50.6450080871582, + "logps/ref_chosen": -50.75047302246094, + "logps/ref_rejected": -78.56951141357422, + "logps/rejected": -80.16427612304688, + "loss": 1.2289, + "margin_dpo/margin_mean": 1.7002232074737549, + "margin_dpo/margin_std": 1.4578522443771362, + "step": 39 + }, + { + "KL/chosen_KL_mean": 0.2315082550048828, + "KL/mean": -0.5488898754119873, + "KL/rejected_KL_mean": -1.3292922973632812, + "KL/std": 1.390199065208435, + "epoch": 0.05873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5607997179031372, + "fcm_dpo/q_t": 0.4614887237548828, + "grad_norm": 59.96656036376953, + "learning_rate": 2.8260869565217386e-07, + "logits/chosen": -0.5169125199317932, + "logits/rejected": -0.48675721883773804, + "logps/chosen": -57.75355911254883, + "logps/ref_chosen": -57.985069274902344, + "logps/ref_rejected": -74.3000717163086, + "logps/rejected": -75.62936401367188, + "loss": 1.2437, + "margin_dpo/margin_mean": 1.560800313949585, + "margin_dpo/margin_std": 1.693495750427246, + "step": 40 + }, + { + "KL/chosen_KL_mean": 0.028055191040039062, + "KL/mean": -0.925218403339386, + "KL/rejected_KL_mean": -1.8784866333007812, + "KL/std": 1.867018699645996, + "epoch": 0.06020558002936858, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.9065409898757935, + "fcm_dpo/q_t": 0.4531286358833313, + "grad_norm": 67.948974609375, + "learning_rate": 2.898550724637681e-07, + "logits/chosen": -0.5261883735656738, + "logits/rejected": -0.4887946844100952, + "logps/chosen": -62.667762756347656, + "logps/ref_chosen": -62.69581604003906, + "logps/ref_rejected": -97.02352905273438, + "logps/rejected": -98.90201568603516, + "loss": 1.2148, + "margin_dpo/margin_mean": 1.9065414667129517, + "margin_dpo/margin_std": 1.9881553649902344, + "step": 41 + }, + { + "KL/chosen_KL_mean": 0.22861289978027344, + "KL/mean": -1.08327054977417, + "KL/rejected_KL_mean": -2.3951492309570312, + "KL/std": 2.1879308223724365, + "epoch": 0.06167400881057269, + "fcm_dpo/beta": 0.11069603264331818, + "fcm_dpo/delta": 0.12672916054725647, + "fcm_dpo/margin": 2.623763084411621, + "fcm_dpo/q_t": 0.43353042006492615, + "grad_norm": 85.34500122070312, + "learning_rate": 2.971014492753623e-07, + "logits/chosen": -0.5527105331420898, + "logits/rejected": -0.5064513683319092, + "logps/chosen": -58.737815856933594, + "logps/ref_chosen": -58.966426849365234, + "logps/ref_rejected": -109.90837097167969, + "logps/rejected": -112.30352783203125, + "loss": 1.134, + "margin_dpo/margin_mean": 2.623763084411621, + "margin_dpo/margin_std": 2.4292526245117188, + "step": 42 + }, + { + "KL/chosen_KL_mean": 0.5411643981933594, + "KL/mean": -0.7341057062149048, + "KL/rejected_KL_mean": -2.0093841552734375, + "KL/std": 1.8598041534423828, + "epoch": 0.0631424375917768, + "fcm_dpo/beta": 0.12222427129745483, + "fcm_dpo/delta": 0.10482510924339294, + "fcm_dpo/margin": 2.550539255142212, + "fcm_dpo/q_t": 0.4277779459953308, + "grad_norm": 83.47020721435547, + "learning_rate": 3.043478260869565e-07, + "logits/chosen": -0.5464547276496887, + "logits/rejected": -0.5218390226364136, + "logps/chosen": -53.614830017089844, + "logps/ref_chosen": -54.15599822998047, + "logps/ref_rejected": -96.48019409179688, + "logps/rejected": -98.48957824707031, + "loss": 1.1131, + "margin_dpo/margin_mean": 2.550539255142212, + "margin_dpo/margin_std": 1.9405533075332642, + "step": 43 + }, + { + "KL/chosen_KL_mean": 0.24042892456054688, + "KL/mean": -1.1693687438964844, + "KL/rejected_KL_mean": -2.5791664123535156, + "KL/std": 2.276883602142334, + "epoch": 0.06461086637298091, + "fcm_dpo/beta": 0.12953037023544312, + "fcm_dpo/delta": 0.042644016444683075, + "fcm_dpo/margin": 2.819594383239746, + "fcm_dpo/q_t": 0.41355597972869873, + "grad_norm": 95.25187683105469, + "learning_rate": 3.115942028985507e-07, + "logits/chosen": -0.4547986388206482, + "logits/rejected": -0.4352615475654602, + "logps/chosen": -49.838069915771484, + "logps/ref_chosen": -50.07849884033203, + "logps/ref_rejected": -108.78376007080078, + "logps/rejected": -111.36293029785156, + "loss": 1.0737, + "margin_dpo/margin_mean": 2.819593906402588, + "margin_dpo/margin_std": 2.199711322784424, + "step": 44 + }, + { + "KL/chosen_KL_mean": 0.1290454864501953, + "KL/mean": -1.0186073780059814, + "KL/rejected_KL_mean": -2.166259765625, + "KL/std": 2.0731301307678223, + "epoch": 0.06607929515418502, + "fcm_dpo/beta": 0.1417458951473236, + "fcm_dpo/delta": 0.08769898861646652, + "fcm_dpo/margin": 2.295309066772461, + "fcm_dpo/q_t": 0.42574048042297363, + "grad_norm": 79.44340515136719, + "learning_rate": 3.188405797101449e-07, + "logits/chosen": -0.4766472280025482, + "logits/rejected": -0.4639527201652527, + "logps/chosen": -48.28588104248047, + "logps/ref_chosen": -48.4149284362793, + "logps/ref_rejected": -77.93643188476562, + "logps/rejected": -80.10269165039062, + "loss": 1.1186, + "margin_dpo/margin_mean": 2.2953085899353027, + "margin_dpo/margin_std": 2.5507259368896484, + "step": 45 + }, + { + "KL/chosen_KL_mean": 0.21740150451660156, + "KL/mean": -1.253394365310669, + "KL/rejected_KL_mean": -2.724193572998047, + "KL/std": 2.6467161178588867, + "epoch": 0.06754772393538913, + "fcm_dpo/beta": 0.14241422712802887, + "fcm_dpo/delta": -0.024083826690912247, + "fcm_dpo/margin": 2.941596031188965, + "fcm_dpo/q_t": 0.4018367528915405, + "grad_norm": 87.27326965332031, + "learning_rate": 3.260869565217391e-07, + "logits/chosen": -0.5110729932785034, + "logits/rejected": -0.4591823220252991, + "logps/chosen": -55.78202438354492, + "logps/ref_chosen": -55.999427795410156, + "logps/ref_rejected": -95.652587890625, + "logps/rejected": -98.37678527832031, + "loss": 1.0582, + "margin_dpo/margin_mean": 2.9415955543518066, + "margin_dpo/margin_std": 3.2013213634490967, + "step": 46 + }, + { + "KL/chosen_KL_mean": 0.37536048889160156, + "KL/mean": -1.1281816959381104, + "KL/rejected_KL_mean": -2.6317214965820312, + "KL/std": 2.644698143005371, + "epoch": 0.06901615271659324, + "fcm_dpo/beta": 0.14004012942314148, + "fcm_dpo/delta": -0.02779523655772209, + "fcm_dpo/margin": 3.007082462310791, + "fcm_dpo/q_t": 0.39907705783843994, + "grad_norm": 82.21805572509766, + "learning_rate": 3.333333333333333e-07, + "logits/chosen": -0.5815363526344299, + "logits/rejected": -0.5293323397636414, + "logps/chosen": -57.55072021484375, + "logps/ref_chosen": -57.92607879638672, + "logps/ref_rejected": -94.67920684814453, + "logps/rejected": -97.31092834472656, + "loss": 1.046, + "margin_dpo/margin_mean": 3.007082462310791, + "margin_dpo/margin_std": 2.746891975402832, + "step": 47 + }, + { + "KL/chosen_KL_mean": 0.05398750305175781, + "KL/mean": -1.5238816738128662, + "KL/rejected_KL_mean": -3.1017494201660156, + "KL/std": 2.673588514328003, + "epoch": 0.07048458149779736, + "fcm_dpo/beta": 0.1313745081424713, + "fcm_dpo/delta": -0.02106369659304619, + "fcm_dpo/margin": 3.1557374000549316, + "fcm_dpo/q_t": 0.4008999466896057, + "grad_norm": 87.41127014160156, + "learning_rate": 3.4057971014492755e-07, + "logits/chosen": -0.5923665761947632, + "logits/rejected": -0.5343451499938965, + "logps/chosen": -57.13408660888672, + "logps/ref_chosen": -57.188072204589844, + "logps/ref_rejected": -88.0166015625, + "logps/rejected": -91.11834716796875, + "loss": 1.0545, + "margin_dpo/margin_mean": 3.15573787689209, + "margin_dpo/margin_std": 2.916724681854248, + "step": 48 + }, + { + "KL/chosen_KL_mean": 0.3004131317138672, + "KL/mean": -1.5658769607543945, + "KL/rejected_KL_mean": -3.4321632385253906, + "KL/std": 3.3690695762634277, + "epoch": 0.07195301027900147, + "fcm_dpo/beta": 0.12684862315654755, + "fcm_dpo/delta": -0.09592577815055847, + "fcm_dpo/margin": 3.7325761318206787, + "fcm_dpo/q_t": 0.38731497526168823, + "grad_norm": 71.87007141113281, + "learning_rate": 3.478260869565217e-07, + "logits/chosen": -0.5684317350387573, + "logits/rejected": -0.5121915936470032, + "logps/chosen": -61.384857177734375, + "logps/ref_chosen": -61.685272216796875, + "logps/ref_rejected": -83.76747131347656, + "logps/rejected": -87.19963836669922, + "loss": 1.025, + "margin_dpo/margin_mean": 3.7325758934020996, + "margin_dpo/margin_std": 3.874316930770874, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.1456451416015625, + "KL/mean": -2.1883692741394043, + "KL/rejected_KL_mean": -4.231086730957031, + "KL/std": 3.4617319107055664, + "epoch": 0.07342143906020558, + "fcm_dpo/beta": 0.11314891278743744, + "fcm_dpo/delta": -0.08234786987304688, + "fcm_dpo/margin": 4.085442066192627, + "fcm_dpo/q_t": 0.3882601261138916, + "grad_norm": 68.02591705322266, + "learning_rate": 3.5507246376811595e-07, + "logits/chosen": -0.5450817346572876, + "logits/rejected": -0.5092841982841492, + "logps/chosen": -58.869781494140625, + "logps/ref_chosen": -58.72413635253906, + "logps/ref_rejected": -96.35814666748047, + "logps/rejected": -100.5892333984375, + "loss": 1.0227, + "margin_dpo/margin_mean": 4.085441589355469, + "margin_dpo/margin_std": 3.960975408554077, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.2403240203857422, + "KL/mean": -2.285604953765869, + "KL/rejected_KL_mean": -4.330890655517578, + "KL/std": 4.1982645988464355, + "epoch": 0.07488986784140969, + "fcm_dpo/beta": 0.10526888072490692, + "fcm_dpo/delta": -0.04377248138189316, + "fcm_dpo/margin": 4.090564250946045, + "fcm_dpo/q_t": 0.4009581208229065, + "grad_norm": 53.3188591003418, + "learning_rate": 3.6231884057971015e-07, + "logits/chosen": -0.5137035250663757, + "logits/rejected": -0.4802321493625641, + "logps/chosen": -61.613990783691406, + "logps/ref_chosen": -61.3736686706543, + "logps/ref_rejected": -76.00199890136719, + "logps/rejected": -80.33289337158203, + "loss": 1.0685, + "margin_dpo/margin_mean": 4.090563774108887, + "margin_dpo/margin_std": 5.139671325683594, + "step": 51 + }, + { + "KL/chosen_KL_mean": 0.37200355529785156, + "KL/mean": -2.7163591384887695, + "KL/rejected_KL_mean": -5.804725646972656, + "KL/std": 4.783724784851074, + "epoch": 0.0763582966226138, + "fcm_dpo/beta": 0.0902351513504982, + "fcm_dpo/delta": -0.22192168235778809, + "fcm_dpo/margin": 6.176729202270508, + "fcm_dpo/q_t": 0.35875505208969116, + "grad_norm": 56.78388595581055, + "learning_rate": 3.695652173913043e-07, + "logits/chosen": -0.5432096719741821, + "logits/rejected": -0.48703598976135254, + "logps/chosen": -51.965354919433594, + "logps/ref_chosen": -52.33735656738281, + "logps/ref_rejected": -79.97391510009766, + "logps/rejected": -85.77864074707031, + "loss": 0.9533, + "margin_dpo/margin_mean": 6.176729202270508, + "margin_dpo/margin_std": 5.141578197479248, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.20578384399414062, + "KL/mean": -3.3196845054626465, + "KL/rejected_KL_mean": -6.433582305908203, + "KL/std": 5.414214611053467, + "epoch": 0.07782672540381791, + "fcm_dpo/beta": 0.08034509420394897, + "fcm_dpo/delta": -0.13391053676605225, + "fcm_dpo/margin": 6.227805137634277, + "fcm_dpo/q_t": 0.38188230991363525, + "grad_norm": 52.141883850097656, + "learning_rate": 3.7681159420289855e-07, + "logits/chosen": -0.6204248666763306, + "logits/rejected": -0.5989758968353271, + "logps/chosen": -53.52043533325195, + "logps/ref_chosen": -53.31465148925781, + "logps/ref_rejected": -91.78359985351562, + "logps/rejected": -98.21717834472656, + "loss": 1.0126, + "margin_dpo/margin_mean": 6.227805137634277, + "margin_dpo/margin_std": 6.215023517608643, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.3681602478027344, + "KL/mean": -3.0784969329833984, + "KL/rejected_KL_mean": -5.788829803466797, + "KL/std": 5.0799665451049805, + "epoch": 0.07929515418502203, + "fcm_dpo/beta": 0.07495428621768951, + "fcm_dpo/delta": -0.008042382076382637, + "fcm_dpo/margin": 5.420670032501221, + "fcm_dpo/q_t": 0.4037601351737976, + "grad_norm": 46.64132308959961, + "learning_rate": 3.8405797101449274e-07, + "logits/chosen": -0.5916078090667725, + "logits/rejected": -0.5381488800048828, + "logps/chosen": -51.05682373046875, + "logps/ref_chosen": -50.68865966796875, + "logps/ref_rejected": -91.71539306640625, + "logps/rejected": -97.50422668457031, + "loss": 1.0548, + "margin_dpo/margin_mean": 5.420670032501221, + "margin_dpo/margin_std": 5.109761714935303, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.9354953765869141, + "KL/mean": -4.157511234283447, + "KL/rejected_KL_mean": -7.379520416259766, + "KL/std": 6.1912150382995605, + "epoch": 0.08076358296622614, + "fcm_dpo/beta": 0.07125985622406006, + "fcm_dpo/delta": -0.07659684866666794, + "fcm_dpo/margin": 6.444026470184326, + "fcm_dpo/q_t": 0.39212772250175476, + "grad_norm": 44.093040466308594, + "learning_rate": 3.9130434782608694e-07, + "logits/chosen": -0.6305919885635376, + "logits/rejected": -0.5667720437049866, + "logps/chosen": -63.55072784423828, + "logps/ref_chosen": -62.615234375, + "logps/ref_rejected": -88.99349975585938, + "logps/rejected": -96.3730239868164, + "loss": 1.0454, + "margin_dpo/margin_mean": 6.44402551651001, + "margin_dpo/margin_std": 7.479229927062988, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.7067909240722656, + "KL/mean": -3.802842140197754, + "KL/rejected_KL_mean": -6.898895263671875, + "KL/std": 6.042604446411133, + "epoch": 0.08223201174743025, + "fcm_dpo/beta": 0.0678362250328064, + "fcm_dpo/delta": -0.02534756436944008, + "fcm_dpo/margin": 6.192107677459717, + "fcm_dpo/q_t": 0.40231359004974365, + "grad_norm": 39.32019805908203, + "learning_rate": 3.9855072463768114e-07, + "logits/chosen": -0.6268018484115601, + "logits/rejected": -0.5851395130157471, + "logps/chosen": -58.63951873779297, + "logps/ref_chosen": -57.9327278137207, + "logps/ref_rejected": -94.1744384765625, + "logps/rejected": -101.07333374023438, + "loss": 1.0686, + "margin_dpo/margin_mean": 6.192108154296875, + "margin_dpo/margin_std": 7.3723554611206055, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.7017936706542969, + "KL/mean": -4.175287246704102, + "KL/rejected_KL_mean": -7.648784637451172, + "KL/std": 5.941527366638184, + "epoch": 0.08370044052863436, + "fcm_dpo/beta": 0.06503595411777496, + "fcm_dpo/delta": -0.06703046709299088, + "fcm_dpo/margin": 6.946985244750977, + "fcm_dpo/q_t": 0.3904920220375061, + "grad_norm": 43.75485610961914, + "learning_rate": 4.057971014492754e-07, + "logits/chosen": -0.588129997253418, + "logits/rejected": -0.5603782534599304, + "logps/chosen": -71.19708251953125, + "logps/ref_chosen": -70.49528503417969, + "logps/ref_rejected": -95.56546020507812, + "logps/rejected": -103.21424102783203, + "loss": 1.0297, + "margin_dpo/margin_mean": 6.946985721588135, + "margin_dpo/margin_std": 6.547435283660889, + "step": 57 + }, + { + "KL/chosen_KL_mean": -1.027395248413086, + "KL/mean": -4.887474060058594, + "KL/rejected_KL_mean": -8.747550964355469, + "KL/std": 6.817049980163574, + "epoch": 0.08516886930983847, + "fcm_dpo/beta": 0.06007317081093788, + "fcm_dpo/delta": -0.08280885219573975, + "fcm_dpo/margin": 7.720156192779541, + "fcm_dpo/q_t": 0.3894719183444977, + "grad_norm": 42.88956069946289, + "learning_rate": 4.1304347826086954e-07, + "logits/chosen": -0.5993902683258057, + "logits/rejected": -0.5237987041473389, + "logps/chosen": -63.160335540771484, + "logps/ref_chosen": -62.13294219970703, + "logps/ref_rejected": -84.61729431152344, + "logps/rejected": -93.3648452758789, + "loss": 1.0325, + "margin_dpo/margin_mean": 7.720156669616699, + "margin_dpo/margin_std": 8.126496315002441, + "step": 58 + }, + { + "KL/chosen_KL_mean": -1.5261516571044922, + "KL/mean": -5.766140460968018, + "KL/rejected_KL_mean": -10.00613021850586, + "KL/std": 7.495608806610107, + "epoch": 0.08663729809104258, + "fcm_dpo/beta": 0.053062424063682556, + "fcm_dpo/delta": -0.07194283604621887, + "fcm_dpo/margin": 8.47997760772705, + "fcm_dpo/q_t": 0.39142686128616333, + "grad_norm": 40.191532135009766, + "learning_rate": 4.2028985507246374e-07, + "logits/chosen": -0.6348840594291687, + "logits/rejected": -0.595291793346405, + "logps/chosen": -53.45867919921875, + "logps/ref_chosen": -51.932525634765625, + "logps/ref_rejected": -88.88520050048828, + "logps/rejected": -98.89132690429688, + "loss": 1.0373, + "margin_dpo/margin_mean": 8.479978561401367, + "margin_dpo/margin_std": 8.641672134399414, + "step": 59 + }, + { + "KL/chosen_KL_mean": -2.7480087280273438, + "KL/mean": -6.107456684112549, + "KL/rejected_KL_mean": -9.466899871826172, + "KL/std": 6.831831455230713, + "epoch": 0.0881057268722467, + "fcm_dpo/beta": 0.054739855229854584, + "fcm_dpo/delta": 0.039379529654979706, + "fcm_dpo/margin": 6.718890190124512, + "fcm_dpo/q_t": 0.4139224886894226, + "grad_norm": 41.76387023925781, + "learning_rate": 4.2753623188405794e-07, + "logits/chosen": -0.6037277579307556, + "logits/rejected": -0.5432413816452026, + "logps/chosen": -63.6901969909668, + "logps/ref_chosen": -60.94218826293945, + "logps/ref_rejected": -85.39340209960938, + "logps/rejected": -94.86029815673828, + "loss": 1.096, + "margin_dpo/margin_mean": 6.718890190124512, + "margin_dpo/margin_std": 7.675610065460205, + "step": 60 + }, + { + "KL/chosen_KL_mean": -1.6316070556640625, + "KL/mean": -5.852381706237793, + "KL/rejected_KL_mean": -10.073162078857422, + "KL/std": 9.248079299926758, + "epoch": 0.08957415565345081, + "fcm_dpo/beta": 0.05262649431824684, + "fcm_dpo/delta": -0.059413738548755646, + "fcm_dpo/margin": 8.441549301147461, + "fcm_dpo/q_t": 0.3997190594673157, + "grad_norm": 35.92430877685547, + "learning_rate": 4.3478260869565214e-07, + "logits/chosen": -0.597582995891571, + "logits/rejected": -0.5633736848831177, + "logps/chosen": -62.26512908935547, + "logps/ref_chosen": -60.633522033691406, + "logps/ref_rejected": -89.85249328613281, + "logps/rejected": -99.9256591796875, + "loss": 1.0687, + "margin_dpo/margin_mean": 8.441547393798828, + "margin_dpo/margin_std": 11.492226600646973, + "step": 61 + }, + { + "KL/chosen_KL_mean": -1.7598438262939453, + "KL/mean": -4.921261787414551, + "KL/rejected_KL_mean": -8.082679748535156, + "KL/std": 6.796797275543213, + "epoch": 0.09104258443465492, + "fcm_dpo/beta": 0.05579136312007904, + "fcm_dpo/delta": 0.055100131779909134, + "fcm_dpo/margin": 6.3228349685668945, + "fcm_dpo/q_t": 0.4199411869049072, + "grad_norm": 35.19557189941406, + "learning_rate": 4.420289855072464e-07, + "logits/chosen": -0.611149251461029, + "logits/rejected": -0.5770503282546997, + "logps/chosen": -57.910614013671875, + "logps/ref_chosen": -56.15077209472656, + "logps/ref_rejected": -75.56619262695312, + "logps/rejected": -83.64888000488281, + "loss": 1.1166, + "margin_dpo/margin_mean": 6.322835922241211, + "margin_dpo/margin_std": 8.347599029541016, + "step": 62 + }, + { + "KL/chosen_KL_mean": -2.888803482055664, + "KL/mean": -7.189858436584473, + "KL/rejected_KL_mean": -11.490913391113281, + "KL/std": 8.740306854248047, + "epoch": 0.09251101321585903, + "fcm_dpo/beta": 0.051797665655612946, + "fcm_dpo/delta": -0.06353449821472168, + "fcm_dpo/margin": 8.602106094360352, + "fcm_dpo/q_t": 0.3924814760684967, + "grad_norm": 37.61563491821289, + "learning_rate": 4.4927536231884053e-07, + "logits/chosen": -0.6047888398170471, + "logits/rejected": -0.5606403350830078, + "logps/chosen": -76.03619384765625, + "logps/ref_chosen": -73.14739227294922, + "logps/ref_rejected": -97.61006164550781, + "logps/rejected": -109.1009750366211, + "loss": 1.0442, + "margin_dpo/margin_mean": 8.602106094360352, + "margin_dpo/margin_std": 9.156990051269531, + "step": 63 + }, + { + "KL/chosen_KL_mean": -1.1866321563720703, + "KL/mean": -6.277315139770508, + "KL/rejected_KL_mean": -11.368000030517578, + "KL/std": 9.430252075195312, + "epoch": 0.09397944199706314, + "fcm_dpo/beta": 0.0463373102247715, + "fcm_dpo/delta": -0.10717649757862091, + "fcm_dpo/margin": 10.181361198425293, + "fcm_dpo/q_t": 0.38632842898368835, + "grad_norm": 33.78566360473633, + "learning_rate": 4.5652173913043473e-07, + "logits/chosen": -0.5959369540214539, + "logits/rejected": -0.5642282366752625, + "logps/chosen": -55.18523406982422, + "logps/ref_chosen": -53.998600006103516, + "logps/ref_rejected": -93.53019714355469, + "logps/rejected": -104.89820098876953, + "loss": 1.0364, + "margin_dpo/margin_mean": 10.18136215209961, + "margin_dpo/margin_std": 11.540030479431152, + "step": 64 + }, + { + "KL/chosen_KL_mean": -3.5648746490478516, + "KL/mean": -8.770621299743652, + "KL/rejected_KL_mean": -13.976367950439453, + "KL/std": 10.70554256439209, + "epoch": 0.09544787077826726, + "fcm_dpo/beta": 0.043465256690979004, + "fcm_dpo/delta": -0.07250865548849106, + "fcm_dpo/margin": 10.411500930786133, + "fcm_dpo/q_t": 0.39109233021736145, + "grad_norm": 33.85374450683594, + "learning_rate": 4.63768115942029e-07, + "logits/chosen": -0.6580019593238831, + "logits/rejected": -0.6455116271972656, + "logps/chosen": -68.40087890625, + "logps/ref_chosen": -64.83599853515625, + "logps/ref_rejected": -109.94645690917969, + "logps/rejected": -123.9228286743164, + "loss": 1.0464, + "margin_dpo/margin_mean": 10.411500930786133, + "margin_dpo/margin_std": 11.7676362991333, + "step": 65 + }, + { + "KL/chosen_KL_mean": -3.3290576934814453, + "KL/mean": -8.054281234741211, + "KL/rejected_KL_mean": -12.77950668334961, + "KL/std": 9.668466567993164, + "epoch": 0.09691629955947137, + "fcm_dpo/beta": 0.042388349771499634, + "fcm_dpo/delta": -0.004084587097167969, + "fcm_dpo/margin": 9.450448989868164, + "fcm_dpo/q_t": 0.4077484607696533, + "grad_norm": 29.53615951538086, + "learning_rate": 4.7101449275362313e-07, + "logits/chosen": -0.6404643058776855, + "logits/rejected": -0.6079626083374023, + "logps/chosen": -54.7725830078125, + "logps/ref_chosen": -51.44352722167969, + "logps/ref_rejected": -75.63629913330078, + "logps/rejected": -88.41580200195312, + "loss": 1.0842, + "margin_dpo/margin_mean": 9.450448989868164, + "margin_dpo/margin_std": 11.738363265991211, + "step": 66 + }, + { + "KL/chosen_KL_mean": -3.056062698364258, + "KL/mean": -7.95568323135376, + "KL/rejected_KL_mean": -12.855304718017578, + "KL/std": 9.883098602294922, + "epoch": 0.09838472834067548, + "fcm_dpo/beta": 0.043049998581409454, + "fcm_dpo/delta": -0.02787696197628975, + "fcm_dpo/margin": 9.79924201965332, + "fcm_dpo/q_t": 0.4025185704231262, + "grad_norm": 30.74718475341797, + "learning_rate": 4.782608695652174e-07, + "logits/chosen": -0.5991535186767578, + "logits/rejected": -0.5557255744934082, + "logps/chosen": -62.39686584472656, + "logps/ref_chosen": -59.34080505371094, + "logps/ref_rejected": -72.78728485107422, + "logps/rejected": -85.64259338378906, + "loss": 1.0678, + "margin_dpo/margin_mean": 9.799242973327637, + "margin_dpo/margin_std": 11.892528533935547, + "step": 67 + }, + { + "KL/chosen_KL_mean": -3.8343505859375, + "KL/mean": -8.516582489013672, + "KL/rejected_KL_mean": -13.198814392089844, + "KL/std": 8.895482063293457, + "epoch": 0.09985315712187959, + "fcm_dpo/beta": 0.04273660480976105, + "fcm_dpo/delta": -0.000813731923699379, + "fcm_dpo/margin": 9.364463806152344, + "fcm_dpo/q_t": 0.40617847442626953, + "grad_norm": 30.00288963317871, + "learning_rate": 4.855072463768116e-07, + "logits/chosen": -0.6549866199493408, + "logits/rejected": -0.5948320627212524, + "logps/chosen": -69.0401840209961, + "logps/ref_chosen": -65.2058334350586, + "logps/ref_rejected": -77.20724487304688, + "logps/rejected": -90.40606689453125, + "loss": 1.0692, + "margin_dpo/margin_mean": 9.364463806152344, + "margin_dpo/margin_std": 10.025825500488281, + "step": 68 + }, + { + "KL/chosen_KL_mean": -4.661190032958984, + "KL/mean": -10.855348587036133, + "KL/rejected_KL_mean": -17.04950714111328, + "KL/std": 11.315301895141602, + "epoch": 0.1013215859030837, + "fcm_dpo/beta": 0.03835180401802063, + "fcm_dpo/delta": -0.10325458645820618, + "fcm_dpo/margin": 12.388309478759766, + "fcm_dpo/q_t": 0.3859746754169464, + "grad_norm": 31.52351188659668, + "learning_rate": 4.927536231884058e-07, + "logits/chosen": -0.6151754856109619, + "logits/rejected": -0.5926914215087891, + "logps/chosen": -64.48043060302734, + "logps/ref_chosen": -59.81924057006836, + "logps/ref_rejected": -103.38886260986328, + "logps/rejected": -120.43836975097656, + "loss": 1.0203, + "margin_dpo/margin_mean": 12.388309478759766, + "margin_dpo/margin_std": 12.643640518188477, + "step": 69 + }, + { + "KL/chosen_KL_mean": -7.054567337036133, + "KL/mean": -13.696563720703125, + "KL/rejected_KL_mean": -20.338558197021484, + "KL/std": 14.29193115234375, + "epoch": 0.1027900146842878, + "fcm_dpo/beta": 0.03588715195655823, + "fcm_dpo/delta": -0.10018091648817062, + "fcm_dpo/margin": 13.283990859985352, + "fcm_dpo/q_t": 0.38754719495773315, + "grad_norm": 31.2943115234375, + "learning_rate": 5e-07, + "logits/chosen": -0.614654004573822, + "logits/rejected": -0.5801568627357483, + "logps/chosen": -68.9852066040039, + "logps/ref_chosen": -61.930641174316406, + "logps/ref_rejected": -91.06078338623047, + "logps/rejected": -111.39933776855469, + "loss": 1.0364, + "margin_dpo/margin_mean": 13.283990859985352, + "margin_dpo/margin_std": 15.29912281036377, + "step": 70 + }, + { + "KL/chosen_KL_mean": -7.13560676574707, + "KL/mean": -15.221653938293457, + "KL/rejected_KL_mean": -23.307708740234375, + "KL/std": 15.503862380981445, + "epoch": 0.10425844346549193, + "fcm_dpo/beta": 0.030674291774630547, + "fcm_dpo/delta": -0.13285793364048004, + "fcm_dpo/margin": 16.172096252441406, + "fcm_dpo/q_t": 0.3823116719722748, + "grad_norm": 28.419921875, + "learning_rate": 4.999967061337492e-07, + "logits/chosen": -0.649775505065918, + "logits/rejected": -0.6120765805244446, + "logps/chosen": -68.88594055175781, + "logps/ref_chosen": -61.750335693359375, + "logps/ref_rejected": -97.33662414550781, + "logps/rejected": -120.64433288574219, + "loss": 1.0137, + "margin_dpo/margin_mean": 16.172096252441406, + "margin_dpo/margin_std": 17.033647537231445, + "step": 71 + }, + { + "KL/chosen_KL_mean": -8.378385543823242, + "KL/mean": -16.67862892150879, + "KL/rejected_KL_mean": -24.978866577148438, + "KL/std": 16.624849319458008, + "epoch": 0.10572687224669604, + "fcm_dpo/beta": 0.028072141110897064, + "fcm_dpo/delta": -0.08738085627555847, + "fcm_dpo/margin": 16.600479125976562, + "fcm_dpo/q_t": 0.38960447907447815, + "grad_norm": 30.47901153564453, + "learning_rate": 4.999868246217933e-07, + "logits/chosen": -0.6570615172386169, + "logits/rejected": -0.6202565431594849, + "logps/chosen": -74.43179321289062, + "logps/ref_chosen": -66.05341339111328, + "logps/ref_rejected": -95.2869873046875, + "logps/rejected": -120.26585388183594, + "loss": 1.0429, + "margin_dpo/margin_mean": 16.600479125976562, + "margin_dpo/margin_std": 19.797344207763672, + "step": 72 + }, + { + "KL/chosen_KL_mean": -10.47215461730957, + "KL/mean": -18.878131866455078, + "KL/rejected_KL_mean": -27.284114837646484, + "KL/std": 21.38900375366211, + "epoch": 0.10719530102790015, + "fcm_dpo/beta": 0.026148542761802673, + "fcm_dpo/delta": -0.0531650111079216, + "fcm_dpo/margin": 16.811960220336914, + "fcm_dpo/q_t": 0.40286535024642944, + "grad_norm": 26.106237411499023, + "learning_rate": 4.999703557245192e-07, + "logits/chosen": -0.7131662368774414, + "logits/rejected": -0.6752426624298096, + "logps/chosen": -76.72842407226562, + "logps/ref_chosen": -66.25627136230469, + "logps/ref_rejected": -90.45613098144531, + "logps/rejected": -117.74024963378906, + "loss": 1.1056, + "margin_dpo/margin_mean": 16.811960220336914, + "margin_dpo/margin_std": 27.132373809814453, + "step": 73 + }, + { + "KL/chosen_KL_mean": -10.981948852539062, + "KL/mean": -20.437519073486328, + "KL/rejected_KL_mean": -29.893081665039062, + "KL/std": 23.818477630615234, + "epoch": 0.10866372980910426, + "fcm_dpo/beta": 0.024446729570627213, + "fcm_dpo/delta": -0.08252619206905365, + "fcm_dpo/margin": 18.911136627197266, + "fcm_dpo/q_t": 0.39837193489074707, + "grad_norm": 26.160118103027344, + "learning_rate": 4.999472998758977e-07, + "logits/chosen": -0.6098172664642334, + "logits/rejected": -0.6038111448287964, + "logps/chosen": -64.40682983398438, + "logps/ref_chosen": -53.42488098144531, + "logps/ref_rejected": -95.94693756103516, + "logps/rejected": -125.84001922607422, + "loss": 1.0783, + "margin_dpo/margin_mean": 18.911136627197266, + "margin_dpo/margin_std": 29.8692569732666, + "step": 74 + }, + { + "KL/chosen_KL_mean": -10.932317733764648, + "KL/mean": -24.136571884155273, + "KL/rejected_KL_mean": -37.3408203125, + "KL/std": 24.335262298583984, + "epoch": 0.11013215859030837, + "fcm_dpo/beta": 0.021133817732334137, + "fcm_dpo/delta": -0.21684792637825012, + "fcm_dpo/margin": 26.408512115478516, + "fcm_dpo/q_t": 0.3661307096481323, + "grad_norm": 24.379812240600586, + "learning_rate": 4.999176576834721e-07, + "logits/chosen": -0.6909558773040771, + "logits/rejected": -0.6851760149002075, + "logps/chosen": -62.79397964477539, + "logps/ref_chosen": -51.861663818359375, + "logps/ref_rejected": -111.25398254394531, + "logps/rejected": -148.5948028564453, + "loss": 0.9864, + "margin_dpo/margin_mean": 26.408510208129883, + "margin_dpo/margin_std": 27.935894012451172, + "step": 75 + }, + { + "KL/chosen_KL_mean": -13.196050643920898, + "KL/mean": -21.37349510192871, + "KL/rejected_KL_mean": -29.55093765258789, + "KL/std": 19.751428604125977, + "epoch": 0.11160058737151249, + "fcm_dpo/beta": 0.02066803351044655, + "fcm_dpo/delta": 0.07377178966999054, + "fcm_dpo/margin": 16.354894638061523, + "fcm_dpo/q_t": 0.4222240149974823, + "grad_norm": 23.001667022705078, + "learning_rate": 4.998814299283415e-07, + "logits/chosen": -0.6944067478179932, + "logits/rejected": -0.6538985967636108, + "logps/chosen": -66.46208953857422, + "logps/ref_chosen": -53.26603698730469, + "logps/ref_rejected": -78.21662902832031, + "logps/rejected": -107.76756286621094, + "loss": 1.1248, + "margin_dpo/margin_mean": 16.354894638061523, + "margin_dpo/margin_std": 21.762468338012695, + "step": 76 + }, + { + "KL/chosen_KL_mean": -12.05851936340332, + "KL/mean": -25.64178466796875, + "KL/rejected_KL_mean": -39.22505187988281, + "KL/std": 27.021289825439453, + "epoch": 0.1130690161527166, + "fcm_dpo/beta": 0.018379777669906616, + "fcm_dpo/delta": -0.14052104949951172, + "fcm_dpo/margin": 27.166534423828125, + "fcm_dpo/q_t": 0.3777894973754883, + "grad_norm": 23.42577362060547, + "learning_rate": 4.998386175651409e-07, + "logits/chosen": -0.7054200172424316, + "logits/rejected": -0.6716604232788086, + "logps/chosen": -70.15519714355469, + "logps/ref_chosen": -58.0966796875, + "logps/ref_rejected": -93.77361297607422, + "logps/rejected": -132.9986572265625, + "loss": 1.0196, + "margin_dpo/margin_mean": 27.166534423828125, + "margin_dpo/margin_std": 30.322921752929688, + "step": 77 + }, + { + "KL/chosen_KL_mean": -12.313678741455078, + "KL/mean": -22.357601165771484, + "KL/rejected_KL_mean": -32.401519775390625, + "KL/std": 22.272029876708984, + "epoch": 0.1145374449339207, + "fcm_dpo/beta": 0.01876748539507389, + "fcm_dpo/delta": 0.02741411328315735, + "fcm_dpo/margin": 20.087839126586914, + "fcm_dpo/q_t": 0.41380858421325684, + "grad_norm": 21.392480850219727, + "learning_rate": 4.997892217220159e-07, + "logits/chosen": -0.6116062998771667, + "logits/rejected": -0.587343156337738, + "logps/chosen": -67.92745971679688, + "logps/ref_chosen": -55.61378479003906, + "logps/ref_rejected": -84.93436431884766, + "logps/rejected": -117.33588409423828, + "loss": 1.1009, + "margin_dpo/margin_mean": 20.087839126586914, + "margin_dpo/margin_std": 25.943618774414062, + "step": 78 + }, + { + "KL/chosen_KL_mean": -12.862058639526367, + "KL/mean": -23.823688507080078, + "KL/rejected_KL_mean": -34.78531265258789, + "KL/std": 24.85503578186035, + "epoch": 0.11600587371512482, + "fcm_dpo/beta": 0.018557265400886536, + "fcm_dpo/delta": -0.008593140169978142, + "fcm_dpo/margin": 21.92325210571289, + "fcm_dpo/q_t": 0.40805089473724365, + "grad_norm": 20.17791748046875, + "learning_rate": 4.997332437005931e-07, + "logits/chosen": -0.6797605752944946, + "logits/rejected": -0.6534685492515564, + "logps/chosen": -68.31254577636719, + "logps/ref_chosen": -55.45048522949219, + "logps/ref_rejected": -87.64756774902344, + "logps/rejected": -122.43287658691406, + "loss": 1.0936, + "margin_dpo/margin_mean": 21.92325210571289, + "margin_dpo/margin_std": 30.41646385192871, + "step": 79 + }, + { + "KL/chosen_KL_mean": -16.53986358642578, + "KL/mean": -27.252384185791016, + "KL/rejected_KL_mean": -37.964908599853516, + "KL/std": 27.36221694946289, + "epoch": 0.11747430249632893, + "fcm_dpo/beta": 0.01815933734178543, + "fcm_dpo/delta": 0.010311007499694824, + "fcm_dpo/margin": 21.425039291381836, + "fcm_dpo/q_t": 0.4146670699119568, + "grad_norm": 22.17724609375, + "learning_rate": 4.996706849759452e-07, + "logits/chosen": -0.7165290117263794, + "logits/rejected": -0.6780725717544556, + "logps/chosen": -75.05915832519531, + "logps/ref_chosen": -58.519290924072266, + "logps/ref_rejected": -87.54750061035156, + "logps/rejected": -125.51240539550781, + "loss": 1.1159, + "margin_dpo/margin_mean": 21.42504119873047, + "margin_dpo/margin_std": 31.943340301513672, + "step": 80 + }, + { + "KL/chosen_KL_mean": -15.941177368164062, + "KL/mean": -30.589588165283203, + "KL/rejected_KL_mean": -45.237998962402344, + "KL/std": 32.461692810058594, + "epoch": 0.11894273127753303, + "fcm_dpo/beta": 0.016676858067512512, + "fcm_dpo/delta": -0.1239388957619667, + "fcm_dpo/margin": 29.296829223632812, + "fcm_dpo/q_t": 0.38681286573410034, + "grad_norm": 21.989290237426758, + "learning_rate": 4.996015471965529e-07, + "logits/chosen": -0.7118068337440491, + "logits/rejected": -0.68656325340271, + "logps/chosen": -82.39004516601562, + "logps/ref_chosen": -66.44886779785156, + "logps/ref_rejected": -129.66270446777344, + "logps/rejected": -174.90069580078125, + "loss": 1.0461, + "margin_dpo/margin_mean": 29.296829223632812, + "margin_dpo/margin_std": 38.70458984375, + "step": 81 + }, + { + "KL/chosen_KL_mean": -18.573457717895508, + "KL/mean": -30.226879119873047, + "KL/rejected_KL_mean": -41.88030242919922, + "KL/std": 29.723350524902344, + "epoch": 0.12041116005873716, + "fcm_dpo/beta": 0.017191056162118912, + "fcm_dpo/delta": -0.009201280772686005, + "fcm_dpo/margin": 23.30685043334961, + "fcm_dpo/q_t": 0.41180986166000366, + "grad_norm": 23.152570724487305, + "learning_rate": 4.995258321842611e-07, + "logits/chosen": -0.6191596984863281, + "logits/rejected": -0.6112022995948792, + "logps/chosen": -70.80584716796875, + "logps/ref_chosen": -52.232383728027344, + "logps/ref_rejected": -90.74325561523438, + "logps/rejected": -132.62355041503906, + "loss": 1.1455, + "margin_dpo/margin_mean": 23.30685043334961, + "margin_dpo/margin_std": 41.297733306884766, + "step": 82 + }, + { + "KL/chosen_KL_mean": -19.169687271118164, + "KL/mean": -33.314022064208984, + "KL/rejected_KL_mean": -47.4583625793457, + "KL/std": 30.860076904296875, + "epoch": 0.12187958883994127, + "fcm_dpo/beta": 0.01552935317158699, + "fcm_dpo/delta": -0.05240233987569809, + "fcm_dpo/margin": 28.28866958618164, + "fcm_dpo/q_t": 0.3983423411846161, + "grad_norm": 21.990673065185547, + "learning_rate": 4.994435419342304e-07, + "logits/chosen": -0.6577206254005432, + "logits/rejected": -0.6294866800308228, + "logps/chosen": -74.9970703125, + "logps/ref_chosen": -55.82738494873047, + "logps/ref_rejected": -103.71589660644531, + "logps/rejected": -151.17425537109375, + "loss": 1.0706, + "margin_dpo/margin_mean": 28.288667678833008, + "margin_dpo/margin_std": 36.98453903198242, + "step": 83 + }, + { + "KL/chosen_KL_mean": -18.952524185180664, + "KL/mean": -30.40399169921875, + "KL/rejected_KL_mean": -41.855464935302734, + "KL/std": 26.464885711669922, + "epoch": 0.12334801762114538, + "fcm_dpo/beta": 0.01563824899494648, + "fcm_dpo/delta": 0.048310477286577225, + "fcm_dpo/margin": 22.902938842773438, + "fcm_dpo/q_t": 0.4166381359100342, + "grad_norm": 20.835840225219727, + "learning_rate": 4.993546786148857e-07, + "logits/chosen": -0.6432810425758362, + "logits/rejected": -0.6044536828994751, + "logps/chosen": -86.12869262695312, + "logps/ref_chosen": -67.1761703491211, + "logps/ref_rejected": -87.29859924316406, + "logps/rejected": -129.15406799316406, + "loss": 1.1085, + "margin_dpo/margin_mean": 22.902938842773438, + "margin_dpo/margin_std": 26.787670135498047, + "step": 84 + }, + { + "KL/chosen_KL_mean": -19.004091262817383, + "KL/mean": -31.023822784423828, + "KL/rejected_KL_mean": -43.043556213378906, + "KL/std": 27.054758071899414, + "epoch": 0.12481644640234948, + "fcm_dpo/beta": 0.016782496124505997, + "fcm_dpo/delta": -0.009143978357315063, + "fcm_dpo/margin": 24.039470672607422, + "fcm_dpo/q_t": 0.40713611245155334, + "grad_norm": 21.613723754882812, + "learning_rate": 4.992592445678582e-07, + "logits/chosen": -0.6534522175788879, + "logits/rejected": -0.622204065322876, + "logps/chosen": -77.41070556640625, + "logps/ref_chosen": -58.4066162109375, + "logps/ref_rejected": -78.63880157470703, + "logps/rejected": -121.68235778808594, + "loss": 1.0991, + "margin_dpo/margin_mean": 24.039470672607422, + "margin_dpo/margin_std": 32.703369140625, + "step": 85 + }, + { + "KL/chosen_KL_mean": -24.901784896850586, + "KL/mean": -37.69927978515625, + "KL/rejected_KL_mean": -50.49677276611328, + "KL/std": 38.232906341552734, + "epoch": 0.1262848751835536, + "fcm_dpo/beta": 0.01626906730234623, + "fcm_dpo/delta": -0.02235548198223114, + "fcm_dpo/margin": 25.594993591308594, + "fcm_dpo/q_t": 0.41634446382522583, + "grad_norm": 25.438549041748047, + "learning_rate": 4.991572423079235e-07, + "logits/chosen": -0.6633529663085938, + "logits/rejected": -0.6544677019119263, + "logps/chosen": -81.03924560546875, + "logps/ref_chosen": -56.13746643066406, + "logps/ref_rejected": -88.12165069580078, + "logps/rejected": -138.61842346191406, + "loss": 1.1718, + "margin_dpo/margin_mean": 25.594993591308594, + "margin_dpo/margin_std": 53.408973693847656, + "step": 86 + }, + { + "KL/chosen_KL_mean": -24.417339324951172, + "KL/mean": -40.079750061035156, + "KL/rejected_KL_mean": -55.74216079711914, + "KL/std": 38.12931823730469, + "epoch": 0.1277533039647577, + "fcm_dpo/beta": 0.014551367610692978, + "fcm_dpo/delta": -0.07786225527524948, + "fcm_dpo/margin": 31.324817657470703, + "fcm_dpo/q_t": 0.39422881603240967, + "grad_norm": 21.019412994384766, + "learning_rate": 4.990486745229364e-07, + "logits/chosen": -0.6794058084487915, + "logits/rejected": -0.6601496934890747, + "logps/chosen": -80.05343627929688, + "logps/ref_chosen": -55.63609313964844, + "logps/ref_rejected": -95.46757507324219, + "logps/rejected": -151.20974731445312, + "loss": 1.0915, + "margin_dpo/margin_mean": 31.324819564819336, + "margin_dpo/margin_std": 47.0085334777832, + "step": 87 + }, + { + "KL/chosen_KL_mean": -30.980731964111328, + "KL/mean": -42.74275207519531, + "KL/rejected_KL_mean": -54.50476837158203, + "KL/std": 40.58386993408203, + "epoch": 0.12922173274596183, + "fcm_dpo/beta": 0.014943553134799004, + "fcm_dpo/delta": 0.057977911084890366, + "fcm_dpo/margin": 23.52403450012207, + "fcm_dpo/q_t": 0.42546147108078003, + "grad_norm": 21.812320709228516, + "learning_rate": 4.989335440737586e-07, + "logits/chosen": -0.6842135787010193, + "logits/rejected": -0.6828039288520813, + "logps/chosen": -104.65188598632812, + "logps/ref_chosen": -73.67115020751953, + "logps/ref_rejected": -106.70849609375, + "logps/rejected": -161.21327209472656, + "loss": 1.1737, + "margin_dpo/margin_mean": 23.52403450012207, + "margin_dpo/margin_std": 43.78386306762695, + "step": 88 + }, + { + "KL/chosen_KL_mean": -19.886613845825195, + "KL/mean": -33.26313781738281, + "KL/rejected_KL_mean": -46.63965606689453, + "KL/std": 34.684234619140625, + "epoch": 0.13069016152716592, + "fcm_dpo/beta": 0.015584435313940048, + "fcm_dpo/delta": -0.02295827493071556, + "fcm_dpo/margin": 26.753049850463867, + "fcm_dpo/q_t": 0.40534842014312744, + "grad_norm": 21.010114669799805, + "learning_rate": 4.988118539941847e-07, + "logits/chosen": -0.7116603851318359, + "logits/rejected": -0.6837852001190186, + "logps/chosen": -80.51152801513672, + "logps/ref_chosen": -60.624916076660156, + "logps/ref_rejected": -82.08354949951172, + "logps/rejected": -128.72320556640625, + "loss": 1.1014, + "margin_dpo/margin_mean": 26.753049850463867, + "margin_dpo/margin_std": 40.44371032714844, + "step": 89 + }, + { + "KL/chosen_KL_mean": -24.155946731567383, + "KL/mean": -43.333587646484375, + "KL/rejected_KL_mean": -62.511234283447266, + "KL/std": 47.100379943847656, + "epoch": 0.13215859030837004, + "fcm_dpo/beta": 0.013927513733506203, + "fcm_dpo/delta": -0.18348340690135956, + "fcm_dpo/margin": 38.355281829833984, + "fcm_dpo/q_t": 0.38429707288742065, + "grad_norm": 21.59499740600586, + "learning_rate": 4.986836074908615e-07, + "logits/chosen": -0.6261708736419678, + "logits/rejected": -0.646406888961792, + "logps/chosen": -77.44125366210938, + "logps/ref_chosen": -53.285308837890625, + "logps/ref_rejected": -111.54470825195312, + "logps/rejected": -174.05593872070312, + "loss": 1.0798, + "margin_dpo/margin_mean": 38.35528564453125, + "margin_dpo/margin_std": 59.089813232421875, + "step": 90 + }, + { + "KL/chosen_KL_mean": -25.931039810180664, + "KL/mean": -40.36906814575195, + "KL/rejected_KL_mean": -54.807098388671875, + "KL/std": 36.47618103027344, + "epoch": 0.13362701908957417, + "fcm_dpo/beta": 0.013058323413133621, + "fcm_dpo/delta": 0.026791641488671303, + "fcm_dpo/margin": 28.876054763793945, + "fcm_dpo/q_t": 0.4173550009727478, + "grad_norm": 19.14020347595215, + "learning_rate": 4.985488079432037e-07, + "logits/chosen": -0.6626990437507629, + "logits/rejected": -0.6330728530883789, + "logps/chosen": -87.73399353027344, + "logps/ref_chosen": -61.802955627441406, + "logps/ref_rejected": -87.87395477294922, + "logps/rejected": -142.68106079101562, + "loss": 1.1368, + "margin_dpo/margin_mean": 28.876056671142578, + "margin_dpo/margin_std": 48.72987365722656, + "step": 91 + }, + { + "KL/chosen_KL_mean": -22.967710494995117, + "KL/mean": -38.062904357910156, + "KL/rejected_KL_mean": -53.15810012817383, + "KL/std": 38.03881072998047, + "epoch": 0.13509544787077826, + "fcm_dpo/beta": 0.013261953368782997, + "fcm_dpo/delta": -0.0033196136355400085, + "fcm_dpo/margin": 30.190397262573242, + "fcm_dpo/q_t": 0.4104510247707367, + "grad_norm": 18.81719398498535, + "learning_rate": 4.984074589033043e-07, + "logits/chosen": -0.6928216218948364, + "logits/rejected": -0.6727030277252197, + "logps/chosen": -74.60847473144531, + "logps/ref_chosen": -51.640769958496094, + "logps/ref_rejected": -77.88117980957031, + "logps/rejected": -131.03927612304688, + "loss": 1.1136, + "margin_dpo/margin_mean": 30.190399169921875, + "margin_dpo/margin_std": 46.062583923339844, + "step": 92 + }, + { + "KL/chosen_KL_mean": -26.070533752441406, + "KL/mean": -40.96013641357422, + "KL/rejected_KL_mean": -55.8497314453125, + "KL/std": 34.986263275146484, + "epoch": 0.13656387665198239, + "fcm_dpo/beta": 0.013152128085494041, + "fcm_dpo/delta": 0.009675238281488419, + "fcm_dpo/margin": 29.77920150756836, + "fcm_dpo/q_t": 0.4116442799568176, + "grad_norm": 19.744487762451172, + "learning_rate": 4.982595640958425e-07, + "logits/chosen": -0.7134292125701904, + "logits/rejected": -0.6725969910621643, + "logps/chosen": -78.59977722167969, + "logps/ref_chosen": -52.529239654541016, + "logps/ref_rejected": -77.16075134277344, + "logps/rejected": -133.01048278808594, + "loss": 1.0986, + "margin_dpo/margin_mean": 29.77920150756836, + "margin_dpo/margin_std": 41.22077941894531, + "step": 93 + }, + { + "KL/chosen_KL_mean": -27.902204513549805, + "KL/mean": -45.52301025390625, + "KL/rejected_KL_mean": -63.14381790161133, + "KL/std": 41.25383758544922, + "epoch": 0.13803230543318648, + "fcm_dpo/beta": 0.012278085574507713, + "fcm_dpo/delta": -0.0462297722697258, + "fcm_dpo/margin": 35.24161148071289, + "fcm_dpo/q_t": 0.39923009276390076, + "grad_norm": 18.810970306396484, + "learning_rate": 4.98105127417984e-07, + "logits/chosen": -0.645513653755188, + "logits/rejected": -0.637039303779602, + "logps/chosen": -89.12481689453125, + "logps/ref_chosen": -61.22261047363281, + "logps/ref_rejected": -99.59902954101562, + "logps/rejected": -162.74285888671875, + "loss": 1.0683, + "margin_dpo/margin_mean": 35.24161148071289, + "margin_dpo/margin_std": 43.92166519165039, + "step": 94 + }, + { + "KL/chosen_KL_mean": -27.263856887817383, + "KL/mean": -41.50673294067383, + "KL/rejected_KL_mean": -55.749610900878906, + "KL/std": 41.475608825683594, + "epoch": 0.1395007342143906, + "fcm_dpo/beta": 0.012666614726185799, + "fcm_dpo/delta": 0.047285325825214386, + "fcm_dpo/margin": 28.485750198364258, + "fcm_dpo/q_t": 0.41799068450927734, + "grad_norm": 18.525175094604492, + "learning_rate": 4.979441529392784e-07, + "logits/chosen": -0.6651968359947205, + "logits/rejected": -0.6414941549301147, + "logps/chosen": -79.78750610351562, + "logps/ref_chosen": -52.523643493652344, + "logps/ref_rejected": -75.8803482055664, + "logps/rejected": -131.6299591064453, + "loss": 1.131, + "margin_dpo/margin_mean": 28.48575210571289, + "margin_dpo/margin_std": 43.52128601074219, + "step": 95 + }, + { + "KL/chosen_KL_mean": -27.1774845123291, + "KL/mean": -46.510169982910156, + "KL/rejected_KL_mean": -65.84285736083984, + "KL/std": 44.56120300292969, + "epoch": 0.14096916299559473, + "fcm_dpo/beta": 0.011798870749771595, + "fcm_dpo/delta": -0.08383983373641968, + "fcm_dpo/margin": 38.665374755859375, + "fcm_dpo/q_t": 0.39412879943847656, + "grad_norm": 18.564252853393555, + "learning_rate": 4.977766449015534e-07, + "logits/chosen": -0.6842184066772461, + "logits/rejected": -0.662356972694397, + "logps/chosen": -89.33445739746094, + "logps/ref_chosen": -62.15697479248047, + "logps/ref_rejected": -96.59601593017578, + "logps/rejected": -162.43887329101562, + "loss": 1.0612, + "margin_dpo/margin_mean": 38.665374755859375, + "margin_dpo/margin_std": 50.07075500488281, + "step": 96 + }, + { + "KL/chosen_KL_mean": -28.180774688720703, + "KL/mean": -42.7745361328125, + "KL/rejected_KL_mean": -57.36829376220703, + "KL/std": 38.20796203613281, + "epoch": 0.14243759177679882, + "fcm_dpo/beta": 0.011734258383512497, + "fcm_dpo/delta": -0.04553954675793648, + "fcm_dpo/margin": 29.187524795532227, + "fcm_dpo/q_t": 0.4178887903690338, + "grad_norm": 19.07831573486328, + "learning_rate": 4.976026077188012e-07, + "logits/chosen": -0.6238841414451599, + "logits/rejected": -0.588768720626831, + "logps/chosen": -82.82714080810547, + "logps/ref_chosen": -54.646366119384766, + "logps/ref_rejected": -76.96475219726562, + "logps/rejected": -134.33303833007812, + "loss": 1.1271, + "margin_dpo/margin_mean": 29.187524795532227, + "margin_dpo/margin_std": 38.123573303222656, + "step": 97 + }, + { + "KL/chosen_KL_mean": -35.863746643066406, + "KL/mean": -52.72106170654297, + "KL/rejected_KL_mean": -69.57838439941406, + "KL/std": 44.223392486572266, + "epoch": 0.14390602055800295, + "fcm_dpo/beta": 0.011502566747367382, + "fcm_dpo/delta": 0.014971842989325523, + "fcm_dpo/margin": 33.714637756347656, + "fcm_dpo/q_t": 0.41005343198776245, + "grad_norm": 20.55078887939453, + "learning_rate": 4.974220459770639e-07, + "logits/chosen": -0.6732558012008667, + "logits/rejected": -0.6637331247329712, + "logps/chosen": -101.12237548828125, + "logps/ref_chosen": -65.25862884521484, + "logps/ref_rejected": -96.5274887084961, + "logps/rejected": -166.10586547851562, + "loss": 1.1266, + "margin_dpo/margin_mean": 33.714637756347656, + "margin_dpo/margin_std": 53.72822570800781, + "step": 98 + }, + { + "KL/chosen_KL_mean": -27.312252044677734, + "KL/mean": -48.77324676513672, + "KL/rejected_KL_mean": -70.2342300415039, + "KL/std": 46.53744125366211, + "epoch": 0.14537444933920704, + "fcm_dpo/beta": 0.010750599205493927, + "fcm_dpo/delta": -0.08328957110643387, + "fcm_dpo/margin": 42.921993255615234, + "fcm_dpo/q_t": 0.39549580216407776, + "grad_norm": 17.073204040527344, + "learning_rate": 4.972349644343108e-07, + "logits/chosen": -0.6471402049064636, + "logits/rejected": -0.651877224445343, + "logps/chosen": -72.95073699951172, + "logps/ref_chosen": -45.638484954833984, + "logps/ref_rejected": -86.43793487548828, + "logps/rejected": -156.6721649169922, + "loss": 1.0633, + "margin_dpo/margin_mean": 42.921993255615234, + "margin_dpo/margin_std": 58.77742004394531, + "step": 99 + }, + { + "KL/chosen_KL_mean": -31.377222061157227, + "KL/mean": -42.55781555175781, + "KL/rejected_KL_mean": -53.738407135009766, + "KL/std": 38.19342803955078, + "epoch": 0.14684287812041116, + "fcm_dpo/beta": 0.010938813909888268, + "fcm_dpo/delta": 0.041864871978759766, + "fcm_dpo/margin": 22.361183166503906, + "fcm_dpo/q_t": 0.4438517093658447, + "grad_norm": 18.95781898498535, + "learning_rate": 4.970413680203148e-07, + "logits/chosen": -0.6145943403244019, + "logits/rejected": -0.5761110782623291, + "logps/chosen": -88.97120666503906, + "logps/ref_chosen": -57.59397888183594, + "logps/ref_rejected": -74.06021118164062, + "logps/rejected": -127.79861450195312, + "loss": 1.2277, + "margin_dpo/margin_mean": 22.361183166503906, + "margin_dpo/margin_std": 49.1837158203125, + "step": 100 + }, + { + "KL/chosen_KL_mean": -38.442962646484375, + "KL/mean": -52.74586486816406, + "KL/rejected_KL_mean": -67.04876708984375, + "KL/std": 47.44648742675781, + "epoch": 0.14831130690161526, + "fcm_dpo/beta": 0.011103025637567043, + "fcm_dpo/delta": -0.0058130607940256596, + "fcm_dpo/margin": 28.60578727722168, + "fcm_dpo/q_t": 0.4297153949737549, + "grad_norm": 19.153141021728516, + "learning_rate": 4.968412618365215e-07, + "logits/chosen": -0.6148165464401245, + "logits/rejected": -0.5886383056640625, + "logps/chosen": -100.09181213378906, + "logps/ref_chosen": -61.64885330200195, + "logps/ref_rejected": -83.18968200683594, + "logps/rejected": -150.2384490966797, + "loss": 1.1865, + "margin_dpo/margin_mean": 28.605789184570312, + "margin_dpo/margin_std": 56.565982818603516, + "step": 101 + }, + { + "KL/chosen_KL_mean": -40.61670684814453, + "KL/mean": -50.855247497558594, + "KL/rejected_KL_mean": -61.093788146972656, + "KL/std": 44.045799255371094, + "epoch": 0.14977973568281938, + "fcm_dpo/beta": 0.011240271851420403, + "fcm_dpo/delta": 0.024421557784080505, + "fcm_dpo/margin": 20.477081298828125, + "fcm_dpo/q_t": 0.44768211245536804, + "grad_norm": 22.304677963256836, + "learning_rate": 4.966346511559149e-07, + "logits/chosen": -0.673102080821991, + "logits/rejected": -0.6373000144958496, + "logps/chosen": -104.69558715820312, + "logps/ref_chosen": -64.0788803100586, + "logps/ref_rejected": -68.18707275390625, + "logps/rejected": -129.28086853027344, + "loss": 1.2568, + "margin_dpo/margin_mean": 20.477081298828125, + "margin_dpo/margin_std": 53.06024169921875, + "step": 102 + }, + { + "KL/chosen_KL_mean": -34.13248825073242, + "KL/mean": -55.830631256103516, + "KL/rejected_KL_mean": -77.52876281738281, + "KL/std": 49.93252182006836, + "epoch": 0.1512481644640235, + "fcm_dpo/beta": 0.010603921487927437, + "fcm_dpo/delta": -0.07997814565896988, + "fcm_dpo/margin": 43.39628601074219, + "fcm_dpo/q_t": 0.3960493803024292, + "grad_norm": 18.789276123046875, + "learning_rate": 4.964215414228785e-07, + "logits/chosen": -0.6219948530197144, + "logits/rejected": -0.59108966588974, + "logps/chosen": -95.4317626953125, + "logps/ref_chosen": -61.299278259277344, + "logps/ref_rejected": -93.57270812988281, + "logps/rejected": -171.10147094726562, + "loss": 1.0721, + "margin_dpo/margin_mean": 43.39628601074219, + "margin_dpo/margin_std": 62.30549621582031, + "step": 103 + }, + { + "KL/chosen_KL_mean": -35.155120849609375, + "KL/mean": -55.93231964111328, + "KL/rejected_KL_mean": -76.70951080322266, + "KL/std": 53.614723205566406, + "epoch": 0.1527165932452276, + "fcm_dpo/beta": 0.010277830064296722, + "fcm_dpo/delta": -0.03438165411353111, + "fcm_dpo/margin": 41.55438995361328, + "fcm_dpo/q_t": 0.40735965967178345, + "grad_norm": 18.074642181396484, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": -0.6393266916275024, + "logits/rejected": -0.6158906817436218, + "logps/chosen": -89.52789306640625, + "logps/ref_chosen": -54.372772216796875, + "logps/ref_rejected": -89.5647201538086, + "logps/rejected": -166.27423095703125, + "loss": 1.1084, + "margin_dpo/margin_mean": 41.55438995361328, + "margin_dpo/margin_std": 66.71173095703125, + "step": 104 + }, + { + "KL/chosen_KL_mean": -33.67083740234375, + "KL/mean": -63.1674919128418, + "KL/rejected_KL_mean": -92.66414642333984, + "KL/std": 50.66936492919922, + "epoch": 0.15418502202643172, + "fcm_dpo/beta": 0.008944441564381123, + "fcm_dpo/delta": -0.17402513325214386, + "fcm_dpo/margin": 58.99330139160156, + "fcm_dpo/q_t": 0.36932459473609924, + "grad_norm": 18.142919540405273, + "learning_rate": 4.959758474331832e-07, + "logits/chosen": -0.5875688791275024, + "logits/rejected": -0.5673741102218628, + "logps/chosen": -88.30978393554688, + "logps/ref_chosen": -54.638946533203125, + "logps/ref_rejected": -97.97351837158203, + "logps/rejected": -190.63766479492188, + "loss": 0.9869, + "margin_dpo/margin_mean": 58.99330139160156, + "margin_dpo/margin_std": 57.22583770751953, + "step": 105 + }, + { + "KL/chosen_KL_mean": -38.18315124511719, + "KL/mean": -56.45439910888672, + "KL/rejected_KL_mean": -74.72565460205078, + "KL/std": 45.731204986572266, + "epoch": 0.15565345080763582, + "fcm_dpo/beta": 0.009024987928569317, + "fcm_dpo/delta": 0.08437706530094147, + "fcm_dpo/margin": 36.542503356933594, + "fcm_dpo/q_t": 0.42528021335601807, + "grad_norm": 16.702926635742188, + "learning_rate": 4.957432749209755e-07, + "logits/chosen": -0.535358190536499, + "logits/rejected": -0.5128501653671265, + "logps/chosen": -93.01604461669922, + "logps/ref_chosen": -54.83289337158203, + "logps/ref_rejected": -85.22461700439453, + "logps/rejected": -159.9502716064453, + "loss": 1.1339, + "margin_dpo/margin_mean": 36.542503356933594, + "margin_dpo/margin_std": 49.62269592285156, + "step": 106 + }, + { + "KL/chosen_KL_mean": -45.14348220825195, + "KL/mean": -65.80903625488281, + "KL/rejected_KL_mean": -86.47457885742188, + "KL/std": 56.287105560302734, + "epoch": 0.15712187958883994, + "fcm_dpo/beta": 0.009254291653633118, + "fcm_dpo/delta": 0.020608151331543922, + "fcm_dpo/margin": 41.33110427856445, + "fcm_dpo/q_t": 0.4126184582710266, + "grad_norm": 17.756500244140625, + "learning_rate": 4.955042268449307e-07, + "logits/chosen": -0.6197066307067871, + "logits/rejected": -0.580697774887085, + "logps/chosen": -114.8512954711914, + "logps/ref_chosen": -69.70780944824219, + "logps/ref_rejected": -94.73950958251953, + "logps/rejected": -181.21408081054688, + "loss": 1.115, + "margin_dpo/margin_mean": 41.33110046386719, + "margin_dpo/margin_std": 59.82461166381836, + "step": 107 + }, + { + "KL/chosen_KL_mean": -39.11261749267578, + "KL/mean": -65.50827026367188, + "KL/rejected_KL_mean": -91.90391540527344, + "KL/std": 62.87705993652344, + "epoch": 0.15859030837004406, + "fcm_dpo/beta": 0.008883420377969742, + "fcm_dpo/delta": -0.09007138013839722, + "fcm_dpo/margin": 52.791290283203125, + "fcm_dpo/q_t": 0.39717066287994385, + "grad_norm": 17.294050216674805, + "learning_rate": 4.952587095041881e-07, + "logits/chosen": -0.5953981280326843, + "logits/rejected": -0.5820811986923218, + "logps/chosen": -95.12250518798828, + "logps/ref_chosen": -56.0098876953125, + "logps/ref_rejected": -95.79601287841797, + "logps/rejected": -187.69992065429688, + "loss": 1.0879, + "margin_dpo/margin_mean": 52.791290283203125, + "margin_dpo/margin_std": 81.55433654785156, + "step": 108 + }, + { + "KL/chosen_KL_mean": -37.30243682861328, + "KL/mean": -64.34707641601562, + "KL/rejected_KL_mean": -91.39170837402344, + "KL/std": 59.183998107910156, + "epoch": 0.16005873715124816, + "fcm_dpo/beta": 0.008149518631398678, + "fcm_dpo/delta": -0.05536172538995743, + "fcm_dpo/margin": 54.08927536010742, + "fcm_dpo/q_t": 0.3969169855117798, + "grad_norm": 18.013111114501953, + "learning_rate": 4.95006729368358e-07, + "logits/chosen": -0.5478425025939941, + "logits/rejected": -0.528139054775238, + "logps/chosen": -100.18792724609375, + "logps/ref_chosen": -62.88549041748047, + "logps/ref_rejected": -98.68573760986328, + "logps/rejected": -190.07745361328125, + "loss": 1.0772, + "margin_dpo/margin_mean": 54.08927536010742, + "margin_dpo/margin_std": 73.88963317871094, + "step": 109 + }, + { + "KL/chosen_KL_mean": -39.2655029296875, + "KL/mean": -63.571868896484375, + "KL/rejected_KL_mean": -87.87823486328125, + "KL/std": 59.547828674316406, + "epoch": 0.16152716593245228, + "fcm_dpo/beta": 0.007406959775835276, + "fcm_dpo/delta": -0.10038627684116364, + "fcm_dpo/margin": 48.61272430419922, + "fcm_dpo/q_t": 0.4130057692527771, + "grad_norm": 13.945056915283203, + "learning_rate": 4.947482930773511e-07, + "logits/chosen": -0.5412209033966064, + "logits/rejected": -0.5109491348266602, + "logps/chosen": -98.0191879272461, + "logps/ref_chosen": -58.753684997558594, + "logps/ref_rejected": -79.75001525878906, + "logps/rejected": -167.6282501220703, + "loss": 1.1332, + "margin_dpo/margin_mean": 48.61272430419922, + "margin_dpo/margin_std": 72.85327911376953, + "step": 110 + }, + { + "KL/chosen_KL_mean": -43.91405487060547, + "KL/mean": -72.29114532470703, + "KL/rejected_KL_mean": -100.66822052001953, + "KL/std": 67.76360321044922, + "epoch": 0.16299559471365638, + "fcm_dpo/beta": 0.0071860793977975845, + "fcm_dpo/delta": -0.012220162898302078, + "fcm_dpo/margin": 56.754173278808594, + "fcm_dpo/q_t": 0.4068235754966736, + "grad_norm": 15.937190055847168, + "learning_rate": 4.944834074412042e-07, + "logits/chosen": -0.5931087732315063, + "logits/rejected": -0.5733714699745178, + "logps/chosen": -112.53816223144531, + "logps/ref_chosen": -68.62410736083984, + "logps/ref_rejected": -98.42886352539062, + "logps/rejected": -199.09707641601562, + "loss": 1.1115, + "margin_dpo/margin_mean": 56.754173278808594, + "margin_dpo/margin_std": 86.54757690429688, + "step": 111 + }, + { + "KL/chosen_KL_mean": -40.607540130615234, + "KL/mean": -57.865150451660156, + "KL/rejected_KL_mean": -75.12275695800781, + "KL/std": 45.083778381347656, + "epoch": 0.1644640234948605, + "fcm_dpo/beta": 0.007618125528097153, + "fcm_dpo/delta": 0.04033050686120987, + "fcm_dpo/margin": 34.51521682739258, + "fcm_dpo/q_t": 0.43914544582366943, + "grad_norm": 15.417643547058105, + "learning_rate": 4.942120794399002e-07, + "logits/chosen": -0.5523747801780701, + "logits/rejected": -0.5217946767807007, + "logps/chosen": -90.857177734375, + "logps/ref_chosen": -50.24964141845703, + "logps/ref_rejected": -64.77442932128906, + "logps/rejected": -139.89718627929688, + "loss": 1.1866, + "margin_dpo/margin_mean": 34.51521682739258, + "margin_dpo/margin_std": 56.524749755859375, + "step": 112 + }, + { + "KL/chosen_KL_mean": -53.0630989074707, + "KL/mean": -70.5762939453125, + "KL/rejected_KL_mean": -88.08948516845703, + "KL/std": 50.8385009765625, + "epoch": 0.16593245227606462, + "fcm_dpo/beta": 0.008391564711928368, + "fcm_dpo/delta": 0.12340584397315979, + "fcm_dpo/margin": 35.0263786315918, + "fcm_dpo/q_t": 0.4351421892642975, + "grad_norm": 17.364025115966797, + "learning_rate": 4.939343162231841e-07, + "logits/chosen": -0.5283902883529663, + "logits/rejected": -0.4908086061477661, + "logps/chosen": -119.77604675292969, + "logps/ref_chosen": -66.71295166015625, + "logps/ref_rejected": -77.96870422363281, + "logps/rejected": -166.05819702148438, + "loss": 1.1661, + "margin_dpo/margin_mean": 35.0263786315918, + "margin_dpo/margin_std": 56.08738327026367, + "step": 113 + }, + { + "KL/chosen_KL_mean": -47.82563018798828, + "KL/mean": -77.25653076171875, + "KL/rejected_KL_mean": -106.68740844726562, + "KL/std": 72.02836608886719, + "epoch": 0.16740088105726872, + "fcm_dpo/beta": 0.007831193506717682, + "fcm_dpo/delta": -0.08718626201152802, + "fcm_dpo/margin": 58.861785888671875, + "fcm_dpo/q_t": 0.39837566018104553, + "grad_norm": 19.08915138244629, + "learning_rate": 4.936501251103751e-07, + "logits/chosen": -0.5314050912857056, + "logits/rejected": -0.5059875845909119, + "logps/chosen": -105.6107177734375, + "logps/ref_chosen": -57.78507995605469, + "logps/ref_rejected": -87.10966491699219, + "logps/rejected": -193.7970733642578, + "loss": 1.0774, + "margin_dpo/margin_mean": 58.861785888671875, + "margin_dpo/margin_std": 88.37593078613281, + "step": 114 + }, + { + "KL/chosen_KL_mean": -60.48192596435547, + "KL/mean": -80.31732177734375, + "KL/rejected_KL_mean": -100.15271759033203, + "KL/std": 69.82859802246094, + "epoch": 0.16886930983847284, + "fcm_dpo/beta": 0.00848847534507513, + "fcm_dpo/delta": 0.0731273964047432, + "fcm_dpo/margin": 39.67078399658203, + "fcm_dpo/q_t": 0.43038254976272583, + "grad_norm": 25.075674057006836, + "learning_rate": 4.933595135901732e-07, + "logits/chosen": -0.5371870398521423, + "logits/rejected": -0.5258318185806274, + "logps/chosen": -126.06456756591797, + "logps/ref_chosen": -65.5826416015625, + "logps/ref_rejected": -98.56552124023438, + "logps/rejected": -198.71823120117188, + "loss": 1.2184, + "margin_dpo/margin_mean": 39.67078399658203, + "margin_dpo/margin_std": 93.87147521972656, + "step": 115 + }, + { + "KL/chosen_KL_mean": -43.80780792236328, + "KL/mean": -66.62762451171875, + "KL/rejected_KL_mean": -89.44744110107422, + "KL/std": 57.25769805908203, + "epoch": 0.17033773861967694, + "fcm_dpo/beta": 0.008733630180358887, + "fcm_dpo/delta": -0.0017695911228656769, + "fcm_dpo/margin": 45.639625549316406, + "fcm_dpo/q_t": 0.4090723991394043, + "grad_norm": 21.11087417602539, + "learning_rate": 4.930624893204624e-07, + "logits/chosen": -0.5265697836875916, + "logits/rejected": -0.5241330862045288, + "logps/chosen": -95.20811462402344, + "logps/ref_chosen": -51.40031433105469, + "logps/ref_rejected": -80.5218505859375, + "logps/rejected": -169.96929931640625, + "loss": 1.1005, + "margin_dpo/margin_mean": 45.639625549316406, + "margin_dpo/margin_std": 63.12974548339844, + "step": 116 + }, + { + "KL/chosen_KL_mean": -56.89840316772461, + "KL/mean": -74.12403869628906, + "KL/rejected_KL_mean": -91.34967041015625, + "KL/std": 59.87590789794922, + "epoch": 0.17180616740088106, + "fcm_dpo/beta": 0.009167262353003025, + "fcm_dpo/delta": 0.09936673939228058, + "fcm_dpo/margin": 34.45126724243164, + "fcm_dpo/q_t": 0.43182051181793213, + "grad_norm": 28.09205436706543, + "learning_rate": 4.927590601281083e-07, + "logits/chosen": -0.5017322301864624, + "logits/rejected": -0.4683611989021301, + "logps/chosen": -126.19680786132812, + "logps/ref_chosen": -69.29840850830078, + "logps/ref_rejected": -66.583984375, + "logps/rejected": -157.93365478515625, + "loss": 1.1893, + "margin_dpo/margin_mean": 34.451271057128906, + "margin_dpo/margin_std": 69.84674072265625, + "step": 117 + }, + { + "KL/chosen_KL_mean": -43.14284133911133, + "KL/mean": -65.05967712402344, + "KL/rejected_KL_mean": -86.97651672363281, + "KL/std": 53.93627166748047, + "epoch": 0.17327459618208516, + "fcm_dpo/beta": 0.009347407147288322, + "fcm_dpo/delta": -0.012348111718893051, + "fcm_dpo/margin": 43.83367919921875, + "fcm_dpo/q_t": 0.4077704846858978, + "grad_norm": 19.32469367980957, + "learning_rate": 4.924492340087524e-07, + "logits/chosen": -0.5494426488876343, + "logits/rejected": -0.5325976014137268, + "logps/chosen": -98.78382110595703, + "logps/ref_chosen": -55.6409797668457, + "logps/ref_rejected": -75.66905975341797, + "logps/rejected": -162.64556884765625, + "loss": 1.0957, + "margin_dpo/margin_mean": 43.83367919921875, + "margin_dpo/margin_std": 62.52387237548828, + "step": 118 + }, + { + "KL/chosen_KL_mean": -59.46000289916992, + "KL/mean": -80.9298095703125, + "KL/rejected_KL_mean": -102.39959716796875, + "KL/std": 60.10303497314453, + "epoch": 0.17474302496328928, + "fcm_dpo/beta": 0.008993204683065414, + "fcm_dpo/delta": 0.011231735348701477, + "fcm_dpo/margin": 42.93959426879883, + "fcm_dpo/q_t": 0.4158446788787842, + "grad_norm": 21.934490203857422, + "learning_rate": 4.92133019126601e-07, + "logits/chosen": -0.5536962151527405, + "logits/rejected": -0.5457164645195007, + "logps/chosen": -132.97019958496094, + "logps/ref_chosen": -73.51019287109375, + "logps/ref_rejected": -102.977294921875, + "logps/rejected": -205.37689208984375, + "loss": 1.1473, + "margin_dpo/margin_mean": 42.939598083496094, + "margin_dpo/margin_std": 75.34599304199219, + "step": 119 + }, + { + "KL/chosen_KL_mean": -58.99882888793945, + "KL/mean": -89.11383056640625, + "KL/rejected_KL_mean": -119.22883605957031, + "KL/std": 73.892822265625, + "epoch": 0.1762114537444934, + "fcm_dpo/beta": 0.008354677818715572, + "fcm_dpo/delta": -0.1406194120645523, + "fcm_dpo/margin": 60.230003356933594, + "fcm_dpo/q_t": 0.3826143443584442, + "grad_norm": 22.082054138183594, + "learning_rate": 4.918104238142103e-07, + "logits/chosen": -0.567842960357666, + "logits/rejected": -0.5426411032676697, + "logps/chosen": -135.7796630859375, + "logps/ref_chosen": -76.78083801269531, + "logps/ref_rejected": -108.02374267578125, + "logps/rejected": -227.25257873535156, + "loss": 1.0416, + "margin_dpo/margin_mean": 60.230003356933594, + "margin_dpo/margin_std": 79.12581634521484, + "step": 120 + }, + { + "KL/chosen_KL_mean": -55.65784454345703, + "KL/mean": -89.1383056640625, + "KL/rejected_KL_mean": -122.61875915527344, + "KL/std": 71.20858764648438, + "epoch": 0.1776798825256975, + "fcm_dpo/beta": 0.007244884967803955, + "fcm_dpo/delta": -0.11996462941169739, + "fcm_dpo/margin": 66.9609375, + "fcm_dpo/q_t": 0.3862529993057251, + "grad_norm": 18.99169158935547, + "learning_rate": 4.91481456572267e-07, + "logits/chosen": -0.5176138877868652, + "logits/rejected": -0.5163988471031189, + "logps/chosen": -117.44773864746094, + "logps/ref_chosen": -61.789894104003906, + "logps/ref_rejected": -109.99456787109375, + "logps/rejected": -232.61334228515625, + "loss": 1.0532, + "margin_dpo/margin_mean": 66.9609375, + "margin_dpo/margin_std": 89.08659362792969, + "step": 121 + }, + { + "KL/chosen_KL_mean": -50.142337799072266, + "KL/mean": -91.1891098022461, + "KL/rejected_KL_mean": -132.2358856201172, + "KL/std": 79.52822875976562, + "epoch": 0.17914831130690162, + "fcm_dpo/beta": 0.0063782669603824615, + "fcm_dpo/delta": -0.16817903518676758, + "fcm_dpo/margin": 82.09354400634766, + "fcm_dpo/q_t": 0.3728944957256317, + "grad_norm": 18.489049911499023, + "learning_rate": 4.911461260693638e-07, + "logits/chosen": -0.46613478660583496, + "logits/rejected": -0.48410728573799133, + "logps/chosen": -97.04454803466797, + "logps/ref_chosen": -46.9022102355957, + "logps/ref_rejected": -106.71418762207031, + "logps/rejected": -238.9500732421875, + "loss": 0.9968, + "margin_dpo/margin_mean": 82.09354400634766, + "margin_dpo/margin_std": 84.7960205078125, + "step": 122 + }, + { + "KL/chosen_KL_mean": -57.29137420654297, + "KL/mean": -83.6397476196289, + "KL/rejected_KL_mean": -109.98812866210938, + "KL/std": 68.29986572265625, + "epoch": 0.18061674008810572, + "fcm_dpo/beta": 0.006330414209514856, + "fcm_dpo/delta": 0.07942930608987808, + "fcm_dpo/margin": 52.696746826171875, + "fcm_dpo/q_t": 0.4271467626094818, + "grad_norm": 17.179462432861328, + "learning_rate": 4.908044411417711e-07, + "logits/chosen": -0.4523668885231018, + "logits/rejected": -0.43804460763931274, + "logps/chosen": -118.63002014160156, + "logps/ref_chosen": -61.33863830566406, + "logps/ref_rejected": -87.775390625, + "logps/rejected": -197.76351928710938, + "loss": 1.1663, + "margin_dpo/margin_mean": 52.69674301147461, + "margin_dpo/margin_std": 93.03097534179688, + "step": 123 + }, + { + "KL/chosen_KL_mean": -66.95594787597656, + "KL/mean": -107.28865051269531, + "KL/rejected_KL_mean": -147.62136840820312, + "KL/std": 95.69377899169922, + "epoch": 0.18208516886930984, + "fcm_dpo/beta": 0.005985944531857967, + "fcm_dpo/delta": -0.11251755058765411, + "fcm_dpo/margin": 80.66542053222656, + "fcm_dpo/q_t": 0.39224308729171753, + "grad_norm": 17.14952278137207, + "learning_rate": 4.904564107932048e-07, + "logits/chosen": -0.46528786420822144, + "logits/rejected": -0.47065889835357666, + "logps/chosen": -138.40428161621094, + "logps/ref_chosen": -71.44833374023438, + "logps/ref_rejected": -117.58056640625, + "logps/rejected": -265.2019348144531, + "loss": 1.0778, + "margin_dpo/margin_mean": 80.66542053222656, + "margin_dpo/margin_std": 120.44160461425781, + "step": 124 + }, + { + "KL/chosen_KL_mean": -54.70975112915039, + "KL/mean": -89.0744857788086, + "KL/rejected_KL_mean": -123.43922424316406, + "KL/std": 80.83454895019531, + "epoch": 0.18355359765051396, + "fcm_dpo/beta": 0.0059068650007247925, + "fcm_dpo/delta": -0.007670434191823006, + "fcm_dpo/margin": 68.72946166992188, + "fcm_dpo/q_t": 0.40873342752456665, + "grad_norm": 14.995677947998047, + "learning_rate": 4.90102044194588e-07, + "logits/chosen": -0.36938369274139404, + "logits/rejected": -0.37332677841186523, + "logps/chosen": -104.84669494628906, + "logps/ref_chosen": -50.136940002441406, + "logps/ref_rejected": -83.98861694335938, + "logps/rejected": -207.42782592773438, + "loss": 1.1061, + "margin_dpo/margin_mean": 68.72946166992188, + "margin_dpo/margin_std": 102.3861083984375, + "step": 125 + }, + { + "KL/chosen_KL_mean": -59.12432861328125, + "KL/mean": -92.44718933105469, + "KL/rejected_KL_mean": -125.77005767822266, + "KL/std": 74.66473388671875, + "epoch": 0.18502202643171806, + "fcm_dpo/beta": 0.005859220400452614, + "fcm_dpo/delta": 0.011660443618893623, + "fcm_dpo/margin": 66.6457290649414, + "fcm_dpo/q_t": 0.4124431610107422, + "grad_norm": 16.17157554626465, + "learning_rate": 4.897413506838102e-07, + "logits/chosen": -0.44136887788772583, + "logits/rejected": -0.441866010427475, + "logps/chosen": -114.79139709472656, + "logps/ref_chosen": -55.66706848144531, + "logps/ref_rejected": -98.1297607421875, + "logps/rejected": -223.89981079101562, + "loss": 1.1071, + "margin_dpo/margin_mean": 66.64572143554688, + "margin_dpo/margin_std": 96.18585968017578, + "step": 126 + }, + { + "KL/chosen_KL_mean": -58.726707458496094, + "KL/mean": -81.67306518554688, + "KL/rejected_KL_mean": -104.61940002441406, + "KL/std": 65.82872009277344, + "epoch": 0.18649045521292218, + "fcm_dpo/beta": 0.006503104232251644, + "fcm_dpo/delta": 0.11829523742198944, + "fcm_dpo/margin": 45.892696380615234, + "fcm_dpo/q_t": 0.43302494287490845, + "grad_norm": 19.481531143188477, + "learning_rate": 4.89374339765481e-07, + "logits/chosen": -0.459053099155426, + "logits/rejected": -0.44523656368255615, + "logps/chosen": -115.28138732910156, + "logps/ref_chosen": -56.55467987060547, + "logps/ref_rejected": -76.7957763671875, + "logps/rejected": -181.41519165039062, + "loss": 1.1793, + "margin_dpo/margin_mean": 45.8927001953125, + "margin_dpo/margin_std": 83.20150756835938, + "step": 127 + }, + { + "KL/chosen_KL_mean": -61.33363342285156, + "KL/mean": -86.33102416992188, + "KL/rejected_KL_mean": -111.32841491699219, + "KL/std": 76.6578598022461, + "epoch": 0.18795888399412627, + "fcm_dpo/beta": 0.007103352341800928, + "fcm_dpo/delta": 0.04982195049524307, + "fcm_dpo/margin": 49.994781494140625, + "fcm_dpo/q_t": 0.42171311378479004, + "grad_norm": 28.149051666259766, + "learning_rate": 4.890010211106795e-07, + "logits/chosen": -0.4325607419013977, + "logits/rejected": -0.42009297013282776, + "logps/chosen": -119.45458984375, + "logps/ref_chosen": -58.12095642089844, + "logps/ref_rejected": -76.43896484375, + "logps/rejected": -187.7673797607422, + "loss": 1.1761, + "margin_dpo/margin_mean": 49.994781494140625, + "margin_dpo/margin_std": 97.5225830078125, + "step": 128 + }, + { + "KL/chosen_KL_mean": -75.2124252319336, + "KL/mean": -101.20868682861328, + "KL/rejected_KL_mean": -127.2049560546875, + "KL/std": 87.85305786132812, + "epoch": 0.1894273127753304, + "fcm_dpo/beta": 0.007200577296316624, + "fcm_dpo/delta": 0.03126451000571251, + "fcm_dpo/margin": 51.99251174926758, + "fcm_dpo/q_t": 0.4244125485420227, + "grad_norm": 21.352294921875, + "learning_rate": 4.88621404556699e-07, + "logits/chosen": -0.4686553478240967, + "logits/rejected": -0.46320652961730957, + "logps/chosen": -142.12879943847656, + "logps/ref_chosen": -66.91637420654297, + "logps/ref_rejected": -96.6422119140625, + "logps/rejected": -223.84716796875, + "loss": 1.1947, + "margin_dpo/margin_mean": 51.992515563964844, + "margin_dpo/margin_std": 116.50946044921875, + "step": 129 + }, + { + "KL/chosen_KL_mean": -54.154781341552734, + "KL/mean": -93.47927856445312, + "KL/rejected_KL_mean": -132.8037567138672, + "KL/std": 83.20738983154297, + "epoch": 0.19089574155653452, + "fcm_dpo/beta": 0.006708543747663498, + "fcm_dpo/delta": -0.17494350671768188, + "fcm_dpo/margin": 78.64897918701172, + "fcm_dpo/q_t": 0.379627525806427, + "grad_norm": 17.628036499023438, + "learning_rate": 4.882355001067891e-07, + "logits/chosen": -0.44320395588874817, + "logits/rejected": -0.4385804831981659, + "logps/chosen": -98.8216323852539, + "logps/ref_chosen": -44.66685104370117, + "logps/ref_rejected": -82.78165435791016, + "logps/rejected": -215.58541870117188, + "loss": 1.0494, + "margin_dpo/margin_mean": 78.64897918701172, + "margin_dpo/margin_std": 104.21408081054688, + "step": 130 + }, + { + "KL/chosen_KL_mean": -50.51225280761719, + "KL/mean": -91.10379791259766, + "KL/rejected_KL_mean": -131.69534301757812, + "KL/std": 87.86686706542969, + "epoch": 0.19236417033773862, + "fcm_dpo/beta": 0.005672046914696693, + "fcm_dpo/delta": -0.07984572649002075, + "fcm_dpo/margin": 81.18309020996094, + "fcm_dpo/q_t": 0.38995444774627686, + "grad_norm": 26.029661178588867, + "learning_rate": 4.878433179298909e-07, + "logits/chosen": -0.4229122996330261, + "logits/rejected": -0.4288443326950073, + "logps/chosen": -95.43684387207031, + "logps/ref_chosen": -44.924591064453125, + "logps/ref_rejected": -88.44401550292969, + "logps/rejected": -220.1393585205078, + "loss": 1.0467, + "margin_dpo/margin_mean": 81.18309020996094, + "margin_dpo/margin_std": 95.4244384765625, + "step": 131 + }, + { + "KL/chosen_KL_mean": -65.28816986083984, + "KL/mean": -97.25325012207031, + "KL/rejected_KL_mean": -129.2183380126953, + "KL/std": 84.85006713867188, + "epoch": 0.19383259911894274, + "fcm_dpo/beta": 0.0057223797775805, + "fcm_dpo/delta": 0.04182172939181328, + "fcm_dpo/margin": 63.93017578125, + "fcm_dpo/q_t": 0.4215155839920044, + "grad_norm": 17.443683624267578, + "learning_rate": 4.874448683603694e-07, + "logits/chosen": -0.4490656852722168, + "logits/rejected": -0.45410823822021484, + "logps/chosen": -124.28925323486328, + "logps/ref_chosen": -59.00108337402344, + "logps/ref_rejected": -87.89215087890625, + "logps/rejected": -217.11048889160156, + "loss": 1.1448, + "margin_dpo/margin_mean": 63.93017578125, + "margin_dpo/margin_std": 110.97628784179688, + "step": 132 + }, + { + "KL/chosen_KL_mean": -76.72837829589844, + "KL/mean": -104.55119323730469, + "KL/rejected_KL_mean": -132.37400817871094, + "KL/std": 75.45304107666016, + "epoch": 0.19530102790014683, + "fcm_dpo/beta": 0.006116401869803667, + "fcm_dpo/delta": 0.07216479629278183, + "fcm_dpo/margin": 55.64562225341797, + "fcm_dpo/q_t": 0.4264053702354431, + "grad_norm": 22.552947998046875, + "learning_rate": 4.870401618977415e-07, + "logits/chosen": -0.42806553840637207, + "logits/rejected": -0.4164883494377136, + "logps/chosen": -143.3328857421875, + "logps/ref_chosen": -66.60449981689453, + "logps/ref_rejected": -96.33355712890625, + "logps/rejected": -228.7075653076172, + "loss": 1.1563, + "margin_dpo/margin_mean": 55.64562225341797, + "margin_dpo/margin_std": 97.3055191040039, + "step": 133 + }, + { + "KL/chosen_KL_mean": -58.90048599243164, + "KL/mean": -88.99266815185547, + "KL/rejected_KL_mean": -119.08485412597656, + "KL/std": 71.10643005371094, + "epoch": 0.19676945668135096, + "fcm_dpo/beta": 0.006456311792135239, + "fcm_dpo/delta": 0.012345774099230766, + "fcm_dpo/margin": 60.18437194824219, + "fcm_dpo/q_t": 0.4125699996948242, + "grad_norm": 18.19769287109375, + "learning_rate": 4.866292092063986e-07, + "logits/chosen": -0.4092817008495331, + "logits/rejected": -0.3999384641647339, + "logps/chosen": -110.96974182128906, + "logps/ref_chosen": -52.06925582885742, + "logps/ref_rejected": -87.6545181274414, + "logps/rejected": -206.7393798828125, + "loss": 1.1018, + "margin_dpo/margin_mean": 60.18437194824219, + "margin_dpo/margin_std": 84.06219482421875, + "step": 134 + }, + { + "KL/chosen_KL_mean": -64.12809753417969, + "KL/mean": -109.45974731445312, + "KL/rejected_KL_mean": -154.7913818359375, + "KL/std": 97.96502685546875, + "epoch": 0.19823788546255505, + "fcm_dpo/beta": 0.005646052770316601, + "fcm_dpo/delta": -0.15241768956184387, + "fcm_dpo/margin": 90.66329193115234, + "fcm_dpo/q_t": 0.38228270411491394, + "grad_norm": 22.45749855041504, + "learning_rate": 4.862120211153265e-07, + "logits/chosen": -0.4356979727745056, + "logits/rejected": -0.46872130036354065, + "logps/chosen": -114.48196411132812, + "logps/ref_chosen": -50.353858947753906, + "logps/ref_rejected": -115.97975158691406, + "logps/rejected": -270.7711181640625, + "loss": 1.0405, + "margin_dpo/margin_mean": 90.66329193115234, + "margin_dpo/margin_std": 120.07051086425781, + "step": 135 + }, + { + "KL/chosen_KL_mean": -76.92674255371094, + "KL/mean": -106.96867370605469, + "KL/rejected_KL_mean": -137.01058959960938, + "KL/std": 91.17539978027344, + "epoch": 0.19970631424375918, + "fcm_dpo/beta": 0.005636686459183693, + "fcm_dpo/delta": 0.07331253588199615, + "fcm_dpo/margin": 60.0838623046875, + "fcm_dpo/q_t": 0.4317668080329895, + "grad_norm": 19.86505699157715, + "learning_rate": 4.857886086178193e-07, + "logits/chosen": -0.42861732840538025, + "logits/rejected": -0.42438289523124695, + "logps/chosen": -141.99925231933594, + "logps/ref_chosen": -65.072509765625, + "logps/ref_rejected": -96.32122802734375, + "logps/rejected": -233.33181762695312, + "loss": 1.1772, + "margin_dpo/margin_mean": 60.083858489990234, + "margin_dpo/margin_std": 119.13693237304688, + "step": 136 + }, + { + "KL/chosen_KL_mean": -70.731201171875, + "KL/mean": -117.82597351074219, + "KL/rejected_KL_mean": -164.92074584960938, + "KL/std": 116.82910919189453, + "epoch": 0.2011747430249633, + "fcm_dpo/beta": 0.005281469784677029, + "fcm_dpo/delta": -0.1319933831691742, + "fcm_dpo/margin": 94.18954467773438, + "fcm_dpo/q_t": 0.3907054662704468, + "grad_norm": 16.0644588470459, + "learning_rate": 4.853589828711902e-07, + "logits/chosen": -0.38837531208992004, + "logits/rejected": -0.4134828746318817, + "logps/chosen": -119.49031829833984, + "logps/ref_chosen": -48.759117126464844, + "logps/ref_rejected": -113.86376953125, + "logps/rejected": -278.7845153808594, + "loss": 1.0696, + "margin_dpo/margin_mean": 94.1895523071289, + "margin_dpo/margin_std": 142.3530731201172, + "step": 137 + }, + { + "KL/chosen_KL_mean": -75.57490539550781, + "KL/mean": -110.52081298828125, + "KL/rejected_KL_mean": -145.46673583984375, + "KL/std": 88.41027069091797, + "epoch": 0.2026431718061674, + "fcm_dpo/beta": 0.005329763051122427, + "fcm_dpo/delta": 0.032860539853572845, + "fcm_dpo/margin": 69.89181518554688, + "fcm_dpo/q_t": 0.41438084840774536, + "grad_norm": 17.347095489501953, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": -0.4172437787055969, + "logits/rejected": -0.40940582752227783, + "logps/chosen": -136.09454345703125, + "logps/ref_chosen": -60.519649505615234, + "logps/ref_rejected": -93.19694519042969, + "logps/rejected": -238.66368103027344, + "loss": 1.1021, + "margin_dpo/margin_mean": 69.89181518554688, + "margin_dpo/margin_std": 89.81456756591797, + "step": 138 + }, + { + "KL/chosen_KL_mean": -63.819305419921875, + "KL/mean": -104.35844421386719, + "KL/rejected_KL_mean": -144.8975830078125, + "KL/std": 82.37762451171875, + "epoch": 0.20411160058737152, + "fcm_dpo/beta": 0.005249645560979843, + "fcm_dpo/delta": -0.032708242535591125, + "fcm_dpo/margin": 81.07829284667969, + "fcm_dpo/q_t": 0.402879536151886, + "grad_norm": 16.59463882446289, + "learning_rate": 4.844811370781446e-07, + "logits/chosen": -0.35705071687698364, + "logits/rejected": -0.3492419719696045, + "logps/chosen": -110.710693359375, + "logps/ref_chosen": -46.89138412475586, + "logps/ref_rejected": -79.72798156738281, + "logps/rejected": -224.62557983398438, + "loss": 1.0796, + "margin_dpo/margin_mean": 81.07829284667969, + "margin_dpo/margin_std": 109.54811096191406, + "step": 139 + }, + { + "KL/chosen_KL_mean": -76.06951904296875, + "KL/mean": -112.28123474121094, + "KL/rejected_KL_mean": -148.49295043945312, + "KL/std": 90.1976318359375, + "epoch": 0.2055800293685756, + "fcm_dpo/beta": 0.0052453503012657166, + "fcm_dpo/delta": 0.02480418235063553, + "fcm_dpo/margin": 72.42343139648438, + "fcm_dpo/q_t": 0.41421765089035034, + "grad_norm": 17.3253116607666, + "learning_rate": 4.840329401637809e-07, + "logits/chosen": -0.35899558663368225, + "logits/rejected": -0.34745436906814575, + "logps/chosen": -135.0442352294922, + "logps/ref_chosen": -58.97471618652344, + "logps/ref_rejected": -83.28410339355469, + "logps/rejected": -231.77706909179688, + "loss": 1.1193, + "margin_dpo/margin_mean": 72.42343139648438, + "margin_dpo/margin_std": 110.29766845703125, + "step": 140 + }, + { + "KL/chosen_KL_mean": -86.08993530273438, + "KL/mean": -120.1361312866211, + "KL/rejected_KL_mean": -154.1823272705078, + "KL/std": 103.17254638671875, + "epoch": 0.20704845814977973, + "fcm_dpo/beta": 0.0054466212168335915, + "fcm_dpo/delta": 0.03537699952721596, + "fcm_dpo/margin": 68.09239196777344, + "fcm_dpo/q_t": 0.4169376492500305, + "grad_norm": 22.393936157226562, + "learning_rate": 4.83578576263792e-07, + "logits/chosen": -0.36837026476860046, + "logits/rejected": -0.35952502489089966, + "logps/chosen": -161.16558837890625, + "logps/ref_chosen": -75.07566833496094, + "logps/ref_rejected": -98.1922607421875, + "logps/rejected": -252.3745880126953, + "loss": 1.148, + "margin_dpo/margin_mean": 68.09239196777344, + "margin_dpo/margin_std": 120.2028579711914, + "step": 141 + }, + { + "KL/chosen_KL_mean": -84.57060241699219, + "KL/mean": -123.71681213378906, + "KL/rejected_KL_mean": -162.86300659179688, + "KL/std": 108.09979248046875, + "epoch": 0.20851688693098386, + "fcm_dpo/beta": 0.005447980482131243, + "fcm_dpo/delta": -0.03504451364278793, + "fcm_dpo/margin": 78.29240417480469, + "fcm_dpo/q_t": 0.40442773699760437, + "grad_norm": 23.142438888549805, + "learning_rate": 4.83118057351089e-07, + "logits/chosen": -0.34042251110076904, + "logits/rejected": -0.341153621673584, + "logps/chosen": -142.59854125976562, + "logps/ref_chosen": -58.027931213378906, + "logps/ref_rejected": -94.58222961425781, + "logps/rejected": -257.44525146484375, + "loss": 1.1208, + "margin_dpo/margin_mean": 78.29240417480469, + "margin_dpo/margin_std": 126.79087829589844, + "step": 142 + }, + { + "KL/chosen_KL_mean": -86.95822143554688, + "KL/mean": -112.62377166748047, + "KL/rejected_KL_mean": -138.28932189941406, + "KL/std": 93.756591796875, + "epoch": 0.20998531571218795, + "fcm_dpo/beta": 0.005426798947155476, + "fcm_dpo/delta": 0.027737241238355637, + "fcm_dpo/margin": 51.33110046386719, + "fcm_dpo/q_t": 0.4396059513092041, + "grad_norm": 20.347646713256836, + "learning_rate": 4.826513955607734e-07, + "logits/chosen": -0.3267577290534973, + "logits/rejected": -0.3233267068862915, + "logps/chosen": -144.55467224121094, + "logps/ref_chosen": -57.59645080566406, + "logps/ref_rejected": -78.99957275390625, + "logps/rejected": -217.2888946533203, + "loss": 1.2111, + "margin_dpo/margin_mean": 51.33110046386719, + "margin_dpo/margin_std": 109.97335815429688, + "step": 143 + }, + { + "KL/chosen_KL_mean": -79.65751647949219, + "KL/mean": -109.87149810791016, + "KL/rejected_KL_mean": -140.08547973632812, + "KL/std": 80.81317138671875, + "epoch": 0.21145374449339208, + "fcm_dpo/beta": 0.005699482746422291, + "fcm_dpo/delta": 0.06714989244937897, + "fcm_dpo/margin": 60.42796325683594, + "fcm_dpo/q_t": 0.42339879274368286, + "grad_norm": 19.846458435058594, + "learning_rate": 4.821786031898176e-07, + "logits/chosen": -0.3399592638015747, + "logits/rejected": -0.3357587456703186, + "logps/chosen": -139.56387329101562, + "logps/ref_chosen": -59.90636444091797, + "logps/ref_rejected": -82.00025939941406, + "logps/rejected": -222.0857391357422, + "loss": 1.1396, + "margin_dpo/margin_mean": 60.427955627441406, + "margin_dpo/margin_std": 94.39094543457031, + "step": 144 + }, + { + "KL/chosen_KL_mean": -76.95576477050781, + "KL/mean": -109.23151397705078, + "KL/rejected_KL_mean": -141.50726318359375, + "KL/std": 79.54987335205078, + "epoch": 0.21292217327459617, + "fcm_dpo/beta": 0.0058981748297810555, + "fcm_dpo/delta": 0.02381444163620472, + "fcm_dpo/margin": 64.55150604248047, + "fcm_dpo/q_t": 0.4141959547996521, + "grad_norm": 23.242799758911133, + "learning_rate": 4.816996926967401e-07, + "logits/chosen": -0.37550991773605347, + "logits/rejected": -0.3661358058452606, + "logps/chosen": -133.55642700195312, + "logps/ref_chosen": -56.60066604614258, + "logps/ref_rejected": -77.86631774902344, + "logps/rejected": -219.3735809326172, + "loss": 1.1192, + "margin_dpo/margin_mean": 64.55149841308594, + "margin_dpo/margin_std": 98.04264831542969, + "step": 145 + }, + { + "KL/chosen_KL_mean": -103.22034454345703, + "KL/mean": -128.1830596923828, + "KL/rejected_KL_mean": -153.14578247070312, + "KL/std": 87.43064880371094, + "epoch": 0.2143906020558003, + "fcm_dpo/beta": 0.006305292714387178, + "fcm_dpo/delta": 0.10162100195884705, + "fcm_dpo/margin": 49.925445556640625, + "fcm_dpo/q_t": 0.43169236183166504, + "grad_norm": 39.6645622253418, + "learning_rate": 4.812146767012779e-07, + "logits/chosen": -0.32967615127563477, + "logits/rejected": -0.30988651514053345, + "logps/chosen": -169.22079467773438, + "logps/ref_chosen": -66.00045013427734, + "logps/ref_rejected": -81.70278930664062, + "logps/rejected": -234.84857177734375, + "loss": 1.206, + "margin_dpo/margin_mean": 49.925445556640625, + "margin_dpo/margin_std": 104.19798278808594, + "step": 146 + }, + { + "KL/chosen_KL_mean": -75.02632904052734, + "KL/mean": -108.63874816894531, + "KL/rejected_KL_mean": -142.25115966796875, + "KL/std": 86.88672637939453, + "epoch": 0.21585903083700442, + "fcm_dpo/beta": 0.0063508497551083565, + "fcm_dpo/delta": -0.035061098635196686, + "fcm_dpo/margin": 67.22482299804688, + "fcm_dpo/q_t": 0.4050912857055664, + "grad_norm": 20.40033721923828, + "learning_rate": 4.807235679840536e-07, + "logits/chosen": -0.3828004002571106, + "logits/rejected": -0.3709457516670227, + "logps/chosen": -128.43182373046875, + "logps/ref_chosen": -53.405487060546875, + "logps/ref_rejected": -71.39060974121094, + "logps/rejected": -213.6417694091797, + "loss": 1.1127, + "margin_dpo/margin_mean": 67.22482299804688, + "margin_dpo/margin_std": 109.10481262207031, + "step": 147 + }, + { + "KL/chosen_KL_mean": -73.86956787109375, + "KL/mean": -103.73741149902344, + "KL/rejected_KL_mean": -133.60525512695312, + "KL/std": 89.116455078125, + "epoch": 0.2173274596182085, + "fcm_dpo/beta": 0.0060739642940461636, + "fcm_dpo/delta": -0.0918317511677742, + "fcm_dpo/margin": 59.735694885253906, + "fcm_dpo/q_t": 0.41868463158607483, + "grad_norm": 18.132606506347656, + "learning_rate": 4.802263794862384e-07, + "logits/chosen": -0.41148072481155396, + "logits/rejected": -0.4058622121810913, + "logps/chosen": -138.806640625, + "logps/ref_chosen": -64.93708038330078, + "logps/ref_rejected": -103.09384155273438, + "logps/rejected": -236.6990966796875, + "loss": 1.1443, + "margin_dpo/margin_mean": 59.735694885253906, + "margin_dpo/margin_std": 94.41209411621094, + "step": 148 + }, + { + "KL/chosen_KL_mean": -69.35594940185547, + "KL/mean": -106.29493713378906, + "KL/rejected_KL_mean": -143.23391723632812, + "KL/std": 77.87938690185547, + "epoch": 0.21879588839941264, + "fcm_dpo/beta": 0.005501337815076113, + "fcm_dpo/delta": -0.0162881538271904, + "fcm_dpo/margin": 73.87796020507812, + "fcm_dpo/q_t": 0.40519750118255615, + "grad_norm": 16.045988082885742, + "learning_rate": 4.797231243092118e-07, + "logits/chosen": -0.428414911031723, + "logits/rejected": -0.4166084825992584, + "logps/chosen": -127.8297119140625, + "logps/ref_chosen": -58.47376251220703, + "logps/ref_rejected": -99.31474304199219, + "logps/rejected": -242.5486602783203, + "loss": 1.0887, + "margin_dpo/margin_mean": 73.87796020507812, + "margin_dpo/margin_std": 93.21217346191406, + "step": 149 + }, + { + "KL/chosen_KL_mean": -60.34211349487305, + "KL/mean": -97.03437042236328, + "KL/rejected_KL_mean": -133.72662353515625, + "KL/std": 91.03880310058594, + "epoch": 0.22026431718061673, + "fcm_dpo/beta": 0.005468531046062708, + "fcm_dpo/delta": -0.007947839796543121, + "fcm_dpo/margin": 73.38451385498047, + "fcm_dpo/q_t": 0.411196231842041, + "grad_norm": 18.763927459716797, + "learning_rate": 4.792138157142157e-07, + "logits/chosen": -0.4295479655265808, + "logits/rejected": -0.4361698627471924, + "logps/chosen": -106.04792785644531, + "logps/ref_chosen": -45.705810546875, + "logps/ref_rejected": -83.34759521484375, + "logps/rejected": -217.07421875, + "loss": 1.1058, + "margin_dpo/margin_mean": 73.38451385498047, + "margin_dpo/margin_std": 108.1720199584961, + "step": 150 + }, + { + "KL/chosen_KL_mean": -77.08334350585938, + "KL/mean": -112.62428283691406, + "KL/rejected_KL_mean": -148.16522216796875, + "KL/std": 86.95265197753906, + "epoch": 0.22173274596182085, + "fcm_dpo/beta": 0.005633922293782234, + "fcm_dpo/delta": -0.0006460570730268955, + "fcm_dpo/margin": 71.08187866210938, + "fcm_dpo/q_t": 0.40812182426452637, + "grad_norm": 22.76422882080078, + "learning_rate": 4.786984671220053e-07, + "logits/chosen": -0.47929656505584717, + "logits/rejected": -0.4552541971206665, + "logps/chosen": -147.6541748046875, + "logps/ref_chosen": -70.57083129882812, + "logps/ref_rejected": -100.46382141113281, + "logps/rejected": -248.62904357910156, + "loss": 1.0926, + "margin_dpo/margin_mean": 71.0818862915039, + "margin_dpo/margin_std": 95.64910888671875, + "step": 151 + }, + { + "KL/chosen_KL_mean": -66.67381286621094, + "KL/mean": -110.26457214355469, + "KL/rejected_KL_mean": -153.8553466796875, + "KL/std": 87.53227233886719, + "epoch": 0.22320117474302498, + "fcm_dpo/beta": 0.0053495545871555805, + "fcm_dpo/delta": -0.08620619773864746, + "fcm_dpo/margin": 87.18153381347656, + "fcm_dpo/q_t": 0.39188152551651, + "grad_norm": 20.685396194458008, + "learning_rate": 4.78177092112495e-07, + "logits/chosen": -0.44135403633117676, + "logits/rejected": -0.4416983723640442, + "logps/chosen": -126.83819580078125, + "logps/ref_chosen": -60.16438674926758, + "logps/ref_rejected": -106.14045715332031, + "logps/rejected": -259.99578857421875, + "loss": 1.0522, + "margin_dpo/margin_mean": 87.18153381347656, + "margin_dpo/margin_std": 108.27259826660156, + "step": 152 + }, + { + "KL/chosen_KL_mean": -65.79019927978516, + "KL/mean": -103.00405883789062, + "KL/rejected_KL_mean": -140.2178955078125, + "KL/std": 96.10725402832031, + "epoch": 0.22466960352422907, + "fcm_dpo/beta": 0.005271708592772484, + "fcm_dpo/delta": 0.008975501172244549, + "fcm_dpo/margin": 74.42771911621094, + "fcm_dpo/q_t": 0.4141613245010376, + "grad_norm": 15.699677467346191, + "learning_rate": 4.776497044244016e-07, + "logits/chosen": -0.4330843687057495, + "logits/rejected": -0.4293479323387146, + "logps/chosen": -122.10546875, + "logps/ref_chosen": -56.315277099609375, + "logps/ref_rejected": -85.65583801269531, + "logps/rejected": -225.87374877929688, + "loss": 1.1194, + "margin_dpo/margin_mean": 74.42771911621094, + "margin_dpo/margin_std": 118.5946273803711, + "step": 153 + }, + { + "KL/chosen_KL_mean": -79.26509094238281, + "KL/mean": -114.19869995117188, + "KL/rejected_KL_mean": -149.13230895996094, + "KL/std": 93.6673812866211, + "epoch": 0.2261380323054332, + "fcm_dpo/beta": 0.005427438300102949, + "fcm_dpo/delta": 0.023521043360233307, + "fcm_dpo/margin": 69.86721801757812, + "fcm_dpo/q_t": 0.41615161299705505, + "grad_norm": 17.67850112915039, + "learning_rate": 4.771163179548808e-07, + "logits/chosen": -0.43425455689430237, + "logits/rejected": -0.4390127658843994, + "logps/chosen": -142.00765991210938, + "logps/ref_chosen": -62.74256896972656, + "logps/ref_rejected": -104.24420166015625, + "logps/rejected": -253.3765106201172, + "loss": 1.1462, + "margin_dpo/margin_mean": 69.86721801757812, + "margin_dpo/margin_std": 122.60856628417969, + "step": 154 + }, + { + "KL/chosen_KL_mean": -72.60445404052734, + "KL/mean": -106.42167663574219, + "KL/rejected_KL_mean": -140.2388916015625, + "KL/std": 85.55455017089844, + "epoch": 0.2276064610866373, + "fcm_dpo/beta": 0.005552366841584444, + "fcm_dpo/delta": 0.028694026172161102, + "fcm_dpo/margin": 67.63446044921875, + "fcm_dpo/q_t": 0.41562047600746155, + "grad_norm": 19.240365982055664, + "learning_rate": 4.7657694675916247e-07, + "logits/chosen": -0.44728028774261475, + "logits/rejected": -0.4302216172218323, + "logps/chosen": -133.25762939453125, + "logps/ref_chosen": -60.65318298339844, + "logps/ref_rejected": -77.49220275878906, + "logps/rejected": -217.73110961914062, + "loss": 1.13, + "margin_dpo/margin_mean": 67.63445281982422, + "margin_dpo/margin_std": 108.91281127929688, + "step": 155 + }, + { + "KL/chosen_KL_mean": -99.16265869140625, + "KL/mean": -118.9976806640625, + "KL/rejected_KL_mean": -138.83270263671875, + "KL/std": 91.43487548828125, + "epoch": 0.2290748898678414, + "fcm_dpo/beta": 0.005871580448001623, + "fcm_dpo/delta": 0.06297972053289413, + "fcm_dpo/margin": 39.67005157470703, + "fcm_dpo/q_t": 0.44856178760528564, + "grad_norm": 27.07191276550293, + "learning_rate": 4.7603160505017893e-07, + "logits/chosen": -0.3714250922203064, + "logits/rejected": -0.3661670684814453, + "logps/chosen": -168.654541015625, + "logps/ref_chosen": -69.49188232421875, + "logps/ref_rejected": -77.16929626464844, + "logps/rejected": -216.0019989013672, + "loss": 1.2847, + "margin_dpo/margin_mean": 39.67005157470703, + "margin_dpo/margin_std": 120.68568420410156, + "step": 156 + }, + { + "KL/chosen_KL_mean": -87.81330871582031, + "KL/mean": -132.32769775390625, + "KL/rejected_KL_mean": -176.84207153320312, + "KL/std": 99.45307159423828, + "epoch": 0.2305433186490455, + "fcm_dpo/beta": 0.005287863779813051, + "fcm_dpo/delta": -0.10068926960229874, + "fcm_dpo/margin": 89.02876281738281, + "fcm_dpo/q_t": 0.3861895203590393, + "grad_norm": 22.06880760192871, + "learning_rate": 4.7548030719819154e-07, + "logits/chosen": -0.36405149102211, + "logits/rejected": -0.3723771572113037, + "logps/chosen": -149.18174743652344, + "logps/ref_chosen": -61.368438720703125, + "logps/ref_rejected": -107.64636993408203, + "logps/rejected": -284.4884338378906, + "loss": 1.0573, + "margin_dpo/margin_mean": 89.02876281738281, + "margin_dpo/margin_std": 111.94908142089844, + "step": 157 + }, + { + "KL/chosen_KL_mean": -89.31040954589844, + "KL/mean": -137.76190185546875, + "KL/rejected_KL_mean": -186.21340942382812, + "KL/std": 123.51625061035156, + "epoch": 0.23201174743024963, + "fcm_dpo/beta": 0.004927870351821184, + "fcm_dpo/delta": -0.10203598439693451, + "fcm_dpo/margin": 96.90299224853516, + "fcm_dpo/q_t": 0.3954607844352722, + "grad_norm": 16.532785415649414, + "learning_rate": 4.7492306773041136e-07, + "logits/chosen": -0.3246377408504486, + "logits/rejected": -0.34248343110084534, + "logps/chosen": -146.92332458496094, + "logps/ref_chosen": -57.612918853759766, + "logps/ref_rejected": -113.6946792602539, + "logps/rejected": -299.9080810546875, + "loss": 1.0826, + "margin_dpo/margin_mean": 96.90299224853516, + "margin_dpo/margin_std": 150.00479125976562, + "step": 158 + }, + { + "KL/chosen_KL_mean": -102.0267333984375, + "KL/mean": -133.82208251953125, + "KL/rejected_KL_mean": -165.6173858642578, + "KL/std": 108.39896392822266, + "epoch": 0.23348017621145375, + "fcm_dpo/beta": 0.005198903381824493, + "fcm_dpo/delta": 0.07878997921943665, + "fcm_dpo/margin": 63.59064483642578, + "fcm_dpo/q_t": 0.42666539549827576, + "grad_norm": 25.30782127380371, + "learning_rate": 4.743599013306165e-07, + "logits/chosen": -0.35725462436676025, + "logits/rejected": -0.32486408948898315, + "logps/chosen": -183.5870819091797, + "logps/ref_chosen": -81.56034851074219, + "logps/ref_rejected": -88.89871215820312, + "logps/rejected": -254.51609802246094, + "loss": 1.1718, + "margin_dpo/margin_mean": 63.59064865112305, + "margin_dpo/margin_std": 117.3126220703125, + "step": 159 + }, + { + "KL/chosen_KL_mean": -103.35143280029297, + "KL/mean": -147.1392059326172, + "KL/rejected_KL_mean": -190.927001953125, + "KL/std": 114.80732727050781, + "epoch": 0.23494860499265785, + "fcm_dpo/beta": 0.0049042413011193275, + "fcm_dpo/delta": -0.04174552857875824, + "fcm_dpo/margin": 87.57557678222656, + "fcm_dpo/q_t": 0.40538012981414795, + "grad_norm": 21.559865951538086, + "learning_rate": 4.737908228387656e-07, + "logits/chosen": -0.3620595932006836, + "logits/rejected": -0.3560444116592407, + "logps/chosen": -169.08230590820312, + "logps/ref_chosen": -65.73088073730469, + "logps/ref_rejected": -97.21781921386719, + "logps/rejected": -288.14483642578125, + "loss": 1.1145, + "margin_dpo/margin_mean": 87.57556915283203, + "margin_dpo/margin_std": 143.64688110351562, + "step": 160 + }, + { + "KL/chosen_KL_mean": -87.88373565673828, + "KL/mean": -126.63963317871094, + "KL/rejected_KL_mean": -165.39553833007812, + "KL/std": 91.78167724609375, + "epoch": 0.23641703377386197, + "fcm_dpo/beta": 0.0050220671109855175, + "fcm_dpo/delta": 0.013326739892363548, + "fcm_dpo/margin": 77.51181030273438, + "fcm_dpo/q_t": 0.4130176603794098, + "grad_norm": 18.22276496887207, + "learning_rate": 4.7321584725060594e-07, + "logits/chosen": -0.362446665763855, + "logits/rejected": -0.3627144396305084, + "logps/chosen": -140.32020568847656, + "logps/ref_chosen": -52.43647003173828, + "logps/ref_rejected": -83.43095397949219, + "logps/rejected": -248.82647705078125, + "loss": 1.1129, + "margin_dpo/margin_mean": 77.51181030273438, + "margin_dpo/margin_std": 115.73544311523438, + "step": 161 + }, + { + "KL/chosen_KL_mean": -86.13906860351562, + "KL/mean": -123.78965759277344, + "KL/rejected_KL_mean": -161.44024658203125, + "KL/std": 101.90530395507812, + "epoch": 0.23788546255506607, + "fcm_dpo/beta": 0.004698555450886488, + "fcm_dpo/delta": -0.07051464915275574, + "fcm_dpo/margin": 75.30118560791016, + "fcm_dpo/q_t": 0.4173450469970703, + "grad_norm": 18.306982040405273, + "learning_rate": 4.7263498971727905e-07, + "logits/chosen": -0.4190049171447754, + "logits/rejected": -0.4046899676322937, + "logps/chosen": -148.74964904785156, + "logps/ref_chosen": -62.6105842590332, + "logps/ref_rejected": -89.39057922363281, + "logps/rejected": -250.83084106445312, + "loss": 1.1434, + "margin_dpo/margin_mean": 75.30118560791016, + "margin_dpo/margin_std": 122.22592163085938, + "step": 162 + }, + { + "KL/chosen_KL_mean": -95.6829833984375, + "KL/mean": -131.91665649414062, + "KL/rejected_KL_mean": -168.15032958984375, + "KL/std": 101.21453857421875, + "epoch": 0.2393538913362702, + "fcm_dpo/beta": 0.004903091117739677, + "fcm_dpo/delta": 0.054114192724227905, + "fcm_dpo/margin": 72.46734619140625, + "fcm_dpo/q_t": 0.4222571849822998, + "grad_norm": 17.79399871826172, + "learning_rate": 4.720482655449212e-07, + "logits/chosen": -0.3502505421638489, + "logits/rejected": -0.3333583474159241, + "logps/chosen": -150.70462036132812, + "logps/ref_chosen": -55.021629333496094, + "logps/ref_rejected": -75.418212890625, + "logps/rejected": -243.5685577392578, + "loss": 1.1443, + "margin_dpo/margin_mean": 72.46734619140625, + "margin_dpo/margin_std": 121.69917297363281, + "step": 163 + }, + { + "KL/chosen_KL_mean": -88.13164520263672, + "KL/mean": -133.67013549804688, + "KL/rejected_KL_mean": -179.20860290527344, + "KL/std": 99.38763427734375, + "epoch": 0.24082232011747431, + "fcm_dpo/beta": 0.00465630367398262, + "fcm_dpo/delta": -0.03786290064454079, + "fcm_dpo/margin": 91.07695770263672, + "fcm_dpo/q_t": 0.4004104733467102, + "grad_norm": 21.530731201171875, + "learning_rate": 4.714556901942599e-07, + "logits/chosen": -0.34161853790283203, + "logits/rejected": -0.3289684057235718, + "logps/chosen": -143.77230834960938, + "logps/ref_chosen": -55.64066696166992, + "logps/ref_rejected": -79.66463470458984, + "logps/rejected": -258.87322998046875, + "loss": 1.0758, + "margin_dpo/margin_mean": 91.07695770263672, + "margin_dpo/margin_std": 114.45172119140625, + "step": 164 + }, + { + "KL/chosen_KL_mean": -95.0910873413086, + "KL/mean": -124.63126373291016, + "KL/rejected_KL_mean": -154.17144775390625, + "KL/std": 86.06841278076172, + "epoch": 0.2422907488986784, + "fcm_dpo/beta": 0.005135393235832453, + "fcm_dpo/delta": 0.1146107017993927, + "fcm_dpo/margin": 59.080352783203125, + "fcm_dpo/q_t": 0.434864342212677, + "grad_norm": 25.962055206298828, + "learning_rate": 4.708572792802069e-07, + "logits/chosen": -0.36488398909568787, + "logits/rejected": -0.34004294872283936, + "logps/chosen": -156.4017791748047, + "logps/ref_chosen": -61.310691833496094, + "logps/ref_rejected": -73.67060852050781, + "logps/rejected": -227.84205627441406, + "loss": 1.192, + "margin_dpo/margin_mean": 59.080360412597656, + "margin_dpo/margin_std": 115.70834350585938, + "step": 165 + }, + { + "KL/chosen_KL_mean": -85.34329223632812, + "KL/mean": -141.81805419921875, + "KL/rejected_KL_mean": -198.2928009033203, + "KL/std": 127.079345703125, + "epoch": 0.24375917767988253, + "fcm_dpo/beta": 0.004592292942106724, + "fcm_dpo/delta": -0.1705796718597412, + "fcm_dpo/margin": 112.94950866699219, + "fcm_dpo/q_t": 0.385653555393219, + "grad_norm": 15.64688777923584, + "learning_rate": 4.702530485714461e-07, + "logits/chosen": -0.29077380895614624, + "logits/rejected": -0.30136969685554504, + "logps/chosen": -136.326904296875, + "logps/ref_chosen": -50.98360061645508, + "logps/ref_rejected": -98.09512329101562, + "logps/rejected": -296.387939453125, + "loss": 1.0467, + "margin_dpo/margin_mean": 112.94950866699219, + "margin_dpo/margin_std": 160.54930114746094, + "step": 166 + }, + { + "KL/chosen_KL_mean": -87.4961166381836, + "KL/mean": -146.18600463867188, + "KL/rejected_KL_mean": -204.87586975097656, + "KL/std": 114.61824035644531, + "epoch": 0.24522760646108663, + "fcm_dpo/beta": 0.004137314390391111, + "fcm_dpo/delta": -0.11398793756961823, + "fcm_dpo/margin": 117.37975311279297, + "fcm_dpo/q_t": 0.3846902847290039, + "grad_norm": 18.566125869750977, + "learning_rate": 4.6964301399001877e-07, + "logits/chosen": -0.31079137325286865, + "logits/rejected": -0.31497058272361755, + "logps/chosen": -137.9202117919922, + "logps/ref_chosen": -50.424095153808594, + "logps/ref_rejected": -96.03042602539062, + "logps/rejected": -300.90631103515625, + "loss": 1.0292, + "margin_dpo/margin_mean": 117.3797607421875, + "margin_dpo/margin_std": 134.6339874267578, + "step": 167 + }, + { + "KL/chosen_KL_mean": -97.19728088378906, + "KL/mean": -139.75576782226562, + "KL/rejected_KL_mean": -182.31427001953125, + "KL/std": 108.02088928222656, + "epoch": 0.24669603524229075, + "fcm_dpo/beta": 0.0041666412726044655, + "fcm_dpo/delta": 0.0552922822535038, + "fcm_dpo/margin": 85.11698913574219, + "fcm_dpo/q_t": 0.42072856426239014, + "grad_norm": 19.070110321044922, + "learning_rate": 4.690271916109034e-07, + "logits/chosen": -0.292841374874115, + "logits/rejected": -0.28525102138519287, + "logps/chosen": -146.6601104736328, + "logps/ref_chosen": -49.462825775146484, + "logps/ref_rejected": -75.30855560302734, + "logps/rejected": -257.6228332519531, + "loss": 1.1239, + "margin_dpo/margin_mean": 85.11698913574219, + "margin_dpo/margin_std": 122.18766021728516, + "step": 168 + }, + { + "KL/chosen_KL_mean": -102.30801391601562, + "KL/mean": -139.7085418701172, + "KL/rejected_KL_mean": -177.10906982421875, + "KL/std": 107.6697998046875, + "epoch": 0.24816446402349487, + "fcm_dpo/beta": 0.004094083793461323, + "fcm_dpo/delta": -0.03929879516363144, + "fcm_dpo/margin": 74.80106353759766, + "fcm_dpo/q_t": 0.43097662925720215, + "grad_norm": 22.41761016845703, + "learning_rate": 4.6840559766159235e-07, + "logits/chosen": -0.3144596219062805, + "logits/rejected": -0.3004973530769348, + "logps/chosen": -162.1114501953125, + "logps/ref_chosen": -59.803443908691406, + "logps/ref_rejected": -83.34574890136719, + "logps/rejected": -260.454833984375, + "loss": 1.1915, + "margin_dpo/margin_mean": 74.80106353759766, + "margin_dpo/margin_std": 148.64874267578125, + "step": 169 + }, + { + "KL/chosen_KL_mean": -92.78736877441406, + "KL/mean": -136.38568115234375, + "KL/rejected_KL_mean": -179.98399353027344, + "KL/std": 105.04856872558594, + "epoch": 0.24963289280469897, + "fcm_dpo/beta": 0.00412098178640008, + "fcm_dpo/delta": 0.04728086292743683, + "fcm_dpo/margin": 87.19662475585938, + "fcm_dpo/q_t": 0.4182136654853821, + "grad_norm": 17.567453384399414, + "learning_rate": 4.6777824852166437e-07, + "logits/chosen": -0.2699648141860962, + "logits/rejected": -0.2612574100494385, + "logps/chosen": -142.25914001464844, + "logps/ref_chosen": -49.471771240234375, + "logps/ref_rejected": -75.91734313964844, + "logps/rejected": -255.90133666992188, + "loss": 1.1277, + "margin_dpo/margin_mean": 87.19662475585938, + "margin_dpo/margin_std": 125.52247619628906, + "step": 170 + }, + { + "KL/chosen_KL_mean": -132.91552734375, + "KL/mean": -168.7076416015625, + "KL/rejected_KL_mean": -204.499755859375, + "KL/std": 122.58496856689453, + "epoch": 0.2511013215859031, + "fcm_dpo/beta": 0.004573632497340441, + "fcm_dpo/delta": 0.08680512011051178, + "fcm_dpo/margin": 71.584228515625, + "fcm_dpo/q_t": 0.4330134391784668, + "grad_norm": 25.66452980041504, + "learning_rate": 4.6714516072235273e-07, + "logits/chosen": -0.2910054922103882, + "logits/rejected": -0.27732402086257935, + "logps/chosen": -217.41482543945312, + "logps/ref_chosen": -84.49931335449219, + "logps/ref_rejected": -109.38209533691406, + "logps/rejected": -313.8818359375, + "loss": 1.2, + "margin_dpo/margin_mean": 71.584228515625, + "margin_dpo/margin_std": 157.88197326660156, + "step": 171 + }, + { + "KL/chosen_KL_mean": -115.34175109863281, + "KL/mean": -152.7630615234375, + "KL/rejected_KL_mean": -190.18435668945312, + "KL/std": 117.2271957397461, + "epoch": 0.2525697503671072, + "fcm_dpo/beta": 0.004836169071495533, + "fcm_dpo/delta": 0.046509332954883575, + "fcm_dpo/margin": 74.84261322021484, + "fcm_dpo/q_t": 0.42060211300849915, + "grad_norm": 21.91267967224121, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": -0.34988853335380554, + "logits/rejected": -0.33963823318481445, + "logps/chosen": -183.99566650390625, + "logps/ref_chosen": -68.65391540527344, + "logps/ref_rejected": -85.43667602539062, + "logps/rejected": -275.62103271484375, + "loss": 1.149, + "margin_dpo/margin_mean": 74.84262084960938, + "margin_dpo/margin_std": 131.61460876464844, + "step": 172 + }, + { + "KL/chosen_KL_mean": -105.72806549072266, + "KL/mean": -143.4074249267578, + "KL/rejected_KL_mean": -181.08677673339844, + "KL/std": 112.28417205810547, + "epoch": 0.2540381791483113, + "fcm_dpo/beta": 0.005077026784420013, + "fcm_dpo/delta": 0.019231334328651428, + "fcm_dpo/margin": 75.35870361328125, + "fcm_dpo/q_t": 0.4149942994117737, + "grad_norm": 22.99903678894043, + "learning_rate": 4.6586183602616687e-07, + "logits/chosen": -0.34687381982803345, + "logits/rejected": -0.32024502754211426, + "logps/chosen": -168.77894592285156, + "logps/ref_chosen": -63.050880432128906, + "logps/ref_rejected": -78.68392181396484, + "logps/rejected": -259.77069091796875, + "loss": 1.1198, + "margin_dpo/margin_mean": 75.35870361328125, + "margin_dpo/margin_std": 116.21147155761719, + "step": 173 + }, + { + "KL/chosen_KL_mean": -97.80389404296875, + "KL/mean": -142.25250244140625, + "KL/rejected_KL_mean": -186.7010955810547, + "KL/std": 110.95504760742188, + "epoch": 0.2555066079295154, + "fcm_dpo/beta": 0.004968566354364157, + "fcm_dpo/delta": -0.05666285380721092, + "fcm_dpo/margin": 88.89718627929688, + "fcm_dpo/q_t": 0.4014216363430023, + "grad_norm": 25.833757400512695, + "learning_rate": 4.652116329460919e-07, + "logits/chosen": -0.28936031460762024, + "logits/rejected": -0.30766457319259644, + "logps/chosen": -151.1668701171875, + "logps/ref_chosen": -53.36296844482422, + "logps/ref_rejected": -101.91120910644531, + "logps/rejected": -288.6123046875, + "loss": 1.0988, + "margin_dpo/margin_mean": 88.8971939086914, + "margin_dpo/margin_std": 134.5579833984375, + "step": 174 + }, + { + "KL/chosen_KL_mean": -90.04649353027344, + "KL/mean": -149.2164306640625, + "KL/rejected_KL_mean": -208.3863525390625, + "KL/std": 116.98501586914062, + "epoch": 0.25697503671071953, + "fcm_dpo/beta": 0.004224025644361973, + "fcm_dpo/delta": -0.13619616627693176, + "fcm_dpo/margin": 118.33984375, + "fcm_dpo/q_t": 0.37867677211761475, + "grad_norm": 29.216033935546875, + "learning_rate": 4.645557588393406e-07, + "logits/chosen": -0.2830924093723297, + "logits/rejected": -0.2700659930706024, + "logps/chosen": -135.46426391601562, + "logps/ref_chosen": -45.417762756347656, + "logps/ref_rejected": -89.50579833984375, + "logps/rejected": -297.89215087890625, + "loss": 1.0097, + "margin_dpo/margin_mean": 118.33984375, + "margin_dpo/margin_std": 123.62623596191406, + "step": 175 + }, + { + "KL/chosen_KL_mean": -93.11599731445312, + "KL/mean": -144.95571899414062, + "KL/rejected_KL_mean": -196.79547119140625, + "KL/std": 118.29825592041016, + "epoch": 0.25844346549192365, + "fcm_dpo/beta": 0.004063988104462624, + "fcm_dpo/delta": -0.027029920369386673, + "fcm_dpo/margin": 103.6794662475586, + "fcm_dpo/q_t": 0.40472695231437683, + "grad_norm": 17.027528762817383, + "learning_rate": 4.638942309888058e-07, + "logits/chosen": -0.287243127822876, + "logits/rejected": -0.30266469717025757, + "logps/chosen": -143.56883239746094, + "logps/ref_chosen": -50.452842712402344, + "logps/ref_rejected": -95.5589599609375, + "logps/rejected": -292.35443115234375, + "loss": 1.0783, + "margin_dpo/margin_mean": 103.67945861816406, + "margin_dpo/margin_std": 136.32940673828125, + "step": 176 + }, + { + "KL/chosen_KL_mean": -108.14009094238281, + "KL/mean": -158.50189208984375, + "KL/rejected_KL_mean": -208.86370849609375, + "KL/std": 128.51873779296875, + "epoch": 0.2599118942731278, + "fcm_dpo/beta": 0.004049934446811676, + "fcm_dpo/delta": -0.011070270091295242, + "fcm_dpo/margin": 100.7236099243164, + "fcm_dpo/q_t": 0.4076334238052368, + "grad_norm": 22.69172477722168, + "learning_rate": 4.6322706682636137e-07, + "logits/chosen": -0.3215820789337158, + "logits/rejected": -0.3144031763076782, + "logps/chosen": -169.35655212402344, + "logps/ref_chosen": -61.216468811035156, + "logps/ref_rejected": -95.89378356933594, + "logps/rejected": -304.75750732421875, + "loss": 1.0863, + "margin_dpo/margin_mean": 100.72361755371094, + "margin_dpo/margin_std": 134.19070434570312, + "step": 177 + }, + { + "KL/chosen_KL_mean": -118.08624267578125, + "KL/mean": -180.44430541992188, + "KL/rejected_KL_mean": -242.80238342285156, + "KL/std": 147.27687072753906, + "epoch": 0.26138032305433184, + "fcm_dpo/beta": 0.003680055495351553, + "fcm_dpo/delta": -0.0795358195900917, + "fcm_dpo/margin": 124.71614074707031, + "fcm_dpo/q_t": 0.3944624662399292, + "grad_norm": 22.354063034057617, + "learning_rate": 4.6255428393240354e-07, + "logits/chosen": -0.2147635966539383, + "logits/rejected": -0.20747539401054382, + "logps/chosen": -176.35104370117188, + "logps/ref_chosen": -58.26478958129883, + "logps/ref_rejected": -105.3653335571289, + "logps/rejected": -348.167724609375, + "loss": 1.0579, + "margin_dpo/margin_mean": 124.71614074707031, + "margin_dpo/margin_std": 161.7968292236328, + "step": 178 + }, + { + "KL/chosen_KL_mean": -123.17328643798828, + "KL/mean": -166.720458984375, + "KL/rejected_KL_mean": -210.26763916015625, + "KL/std": 128.99139404296875, + "epoch": 0.26284875183553597, + "fcm_dpo/beta": 0.0038236351683735847, + "fcm_dpo/delta": 0.08045655488967896, + "fcm_dpo/margin": 87.09436798095703, + "fcm_dpo/q_t": 0.4275627136230469, + "grad_norm": 26.390382766723633, + "learning_rate": 4.6187590003538724e-07, + "logits/chosen": -0.2743037939071655, + "logits/rejected": -0.2854520082473755, + "logps/chosen": -184.23159790039062, + "logps/ref_chosen": -61.05832290649414, + "logps/ref_rejected": -90.52782440185547, + "logps/rejected": -300.79547119140625, + "loss": 1.1747, + "margin_dpo/margin_mean": 87.09436798095703, + "margin_dpo/margin_std": 162.98179626464844, + "step": 179 + }, + { + "KL/chosen_KL_mean": -106.45706176757812, + "KL/mean": -164.29107666015625, + "KL/rejected_KL_mean": -222.1250762939453, + "KL/std": 113.41049194335938, + "epoch": 0.2643171806167401, + "fcm_dpo/beta": 0.003801894374191761, + "fcm_dpo/delta": -0.05152449384331703, + "fcm_dpo/margin": 115.66802978515625, + "fcm_dpo/q_t": 0.39676570892333984, + "grad_norm": 18.645387649536133, + "learning_rate": 4.611919330113591e-07, + "logits/chosen": -0.25623831152915955, + "logits/rejected": -0.25502607226371765, + "logps/chosen": -160.79977416992188, + "logps/ref_chosen": -54.34272003173828, + "logps/ref_rejected": -98.21183776855469, + "logps/rejected": -320.3369140625, + "loss": 1.0622, + "margin_dpo/margin_mean": 115.66802215576172, + "margin_dpo/margin_std": 143.18829345703125, + "step": 180 + }, + { + "KL/chosen_KL_mean": -91.30926513671875, + "KL/mean": -126.50997924804688, + "KL/rejected_KL_mean": -161.710693359375, + "KL/std": 105.37017822265625, + "epoch": 0.2657856093979442, + "fcm_dpo/beta": 0.004171198233962059, + "fcm_dpo/delta": 0.12288638204336166, + "fcm_dpo/margin": 70.40142059326172, + "fcm_dpo/q_t": 0.4360322952270508, + "grad_norm": 18.242246627807617, + "learning_rate": 4.605024008834863e-07, + "logits/chosen": -0.2822926938533783, + "logits/rejected": -0.25778597593307495, + "logps/chosen": -146.30972290039062, + "logps/ref_chosen": -55.000457763671875, + "logps/ref_rejected": -61.656166076660156, + "logps/rejected": -223.36685180664062, + "loss": 1.1884, + "margin_dpo/margin_mean": 70.40142059326172, + "margin_dpo/margin_std": 135.00912475585938, + "step": 181 + }, + { + "KL/chosen_KL_mean": -84.9980239868164, + "KL/mean": -147.97802734375, + "KL/rejected_KL_mean": -210.95803833007812, + "KL/std": 125.26875305175781, + "epoch": 0.26725403817914833, + "fcm_dpo/beta": 0.0038350564427673817, + "fcm_dpo/delta": -0.11499916762113571, + "fcm_dpo/margin": 125.96003723144531, + "fcm_dpo/q_t": 0.3841468095779419, + "grad_norm": 14.498435974121094, + "learning_rate": 4.598073218215817e-07, + "logits/chosen": -0.27837133407592773, + "logits/rejected": -0.28961560130119324, + "logps/chosen": -126.10587310791016, + "logps/ref_chosen": -41.107852935791016, + "logps/ref_rejected": -89.5215835571289, + "logps/rejected": -300.4796142578125, + "loss": 1.0402, + "margin_dpo/margin_mean": 125.96002960205078, + "margin_dpo/margin_std": 152.85032653808594, + "step": 182 + }, + { + "KL/chosen_KL_mean": -127.10066223144531, + "KL/mean": -160.03028869628906, + "KL/rejected_KL_mean": -192.95993041992188, + "KL/std": 105.72233581542969, + "epoch": 0.2687224669603524, + "fcm_dpo/beta": 0.0037473866250365973, + "fcm_dpo/delta": -0.014634971506893635, + "fcm_dpo/margin": 65.85924530029297, + "fcm_dpo/q_t": 0.4414650797843933, + "grad_norm": 16.12941551208496, + "learning_rate": 4.5910671414162484e-07, + "logits/chosen": -0.2845485210418701, + "logits/rejected": -0.27767112851142883, + "logps/chosen": -184.62522888183594, + "logps/ref_chosen": -57.52456283569336, + "logps/ref_rejected": -75.97572326660156, + "logps/rejected": -268.9356689453125, + "loss": 1.2074, + "margin_dpo/margin_mean": 65.85924530029297, + "margin_dpo/margin_std": 116.302734375, + "step": 183 + }, + { + "KL/chosen_KL_mean": -108.40987396240234, + "KL/mean": -143.2193603515625, + "KL/rejected_KL_mean": -178.02880859375, + "KL/std": 99.74555206298828, + "epoch": 0.2701908957415565, + "fcm_dpo/beta": 0.0037929057143628597, + "fcm_dpo/delta": 0.024003401398658752, + "fcm_dpo/margin": 69.61894226074219, + "fcm_dpo/q_t": 0.4396555423736572, + "grad_norm": 15.583113670349121, + "learning_rate": 4.5840059630527985e-07, + "logits/chosen": -0.3264748156070709, + "logits/rejected": -0.31739190220832825, + "logps/chosen": -166.954833984375, + "logps/ref_chosen": -58.544952392578125, + "logps/ref_rejected": -76.63406372070312, + "logps/rejected": -254.66287231445312, + "loss": 1.1946, + "margin_dpo/margin_mean": 69.61894226074219, + "margin_dpo/margin_std": 125.76559448242188, + "step": 184 + }, + { + "KL/chosen_KL_mean": -119.91249084472656, + "KL/mean": -147.1962890625, + "KL/rejected_KL_mean": -174.48007202148438, + "KL/std": 115.40476989746094, + "epoch": 0.27165932452276065, + "fcm_dpo/beta": 0.003838424803689122, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 54.567588806152344, + "fcm_dpo/q_t": 0.45364660024642944, + "grad_norm": 19.94534683227539, + "learning_rate": 4.5768898691940836e-07, + "logits/chosen": -0.2822297513484955, + "logits/rejected": -0.2613981068134308, + "logps/chosen": -181.93833923339844, + "logps/ref_chosen": -62.025848388671875, + "logps/ref_rejected": -73.7625961303711, + "logps/rejected": -248.24266052246094, + "loss": 1.2533, + "margin_dpo/margin_mean": 54.567588806152344, + "margin_dpo/margin_std": 138.52407836914062, + "step": 185 + }, + { + "KL/chosen_KL_mean": -114.25096893310547, + "KL/mean": -166.89956665039062, + "KL/rejected_KL_mean": -219.54815673828125, + "KL/std": 117.91287231445312, + "epoch": 0.27312775330396477, + "fcm_dpo/beta": 0.003811160335317254, + "fcm_dpo/delta": -0.0017963321879506111, + "fcm_dpo/margin": 105.29721069335938, + "fcm_dpo/q_t": 0.40835633873939514, + "grad_norm": 19.27111053466797, + "learning_rate": 4.5697190473557947e-07, + "logits/chosen": -0.31773799657821655, + "logits/rejected": -0.3034238815307617, + "logps/chosen": -183.60443115234375, + "logps/ref_chosen": -69.35346984863281, + "logps/ref_rejected": -88.07244873046875, + "logps/rejected": -307.62060546875, + "loss": 1.0898, + "margin_dpo/margin_mean": 105.29721069335938, + "margin_dpo/margin_std": 139.1805419921875, + "step": 186 + }, + { + "KL/chosen_KL_mean": -102.67378234863281, + "KL/mean": -148.59390258789062, + "KL/rejected_KL_mean": -194.5139923095703, + "KL/std": 112.72882080078125, + "epoch": 0.2745961820851689, + "fcm_dpo/beta": 0.004015129059553146, + "fcm_dpo/delta": 0.03561420738697052, + "fcm_dpo/margin": 91.84022521972656, + "fcm_dpo/q_t": 0.41552722454071045, + "grad_norm": 19.90249252319336, + "learning_rate": 4.5624936864957555e-07, + "logits/chosen": -0.27692389488220215, + "logits/rejected": -0.27118462324142456, + "logps/chosen": -155.43023681640625, + "logps/ref_chosen": -52.7564582824707, + "logps/ref_rejected": -81.96910095214844, + "logps/rejected": -276.48309326171875, + "loss": 1.1093, + "margin_dpo/margin_mean": 91.84022521972656, + "margin_dpo/margin_std": 122.15457916259766, + "step": 187 + }, + { + "KL/chosen_KL_mean": -98.16572570800781, + "KL/mean": -153.54725646972656, + "KL/rejected_KL_mean": -208.92877197265625, + "KL/std": 125.24346923828125, + "epoch": 0.27606461086637296, + "fcm_dpo/beta": 0.003873111680150032, + "fcm_dpo/delta": -0.036867331713438034, + "fcm_dpo/margin": 110.76304626464844, + "fcm_dpo/q_t": 0.4013431966304779, + "grad_norm": 21.790515899658203, + "learning_rate": 4.5552139770089454e-07, + "logits/chosen": -0.2571265697479248, + "logits/rejected": -0.26449519395828247, + "logps/chosen": -147.58120727539062, + "logps/ref_chosen": -49.415489196777344, + "logps/ref_rejected": -89.54043579101562, + "logps/rejected": -298.4692077636719, + "loss": 1.0701, + "margin_dpo/margin_mean": 110.76305389404297, + "margin_dpo/margin_std": 140.52169799804688, + "step": 188 + }, + { + "KL/chosen_KL_mean": -120.33238220214844, + "KL/mean": -165.42770385742188, + "KL/rejected_KL_mean": -210.52301025390625, + "KL/std": 131.658203125, + "epoch": 0.2775330396475771, + "fcm_dpo/beta": 0.003984817303717136, + "fcm_dpo/delta": 0.04916512593626976, + "fcm_dpo/margin": 90.19061279296875, + "fcm_dpo/q_t": 0.42296701669692993, + "grad_norm": 27.90737533569336, + "learning_rate": 4.5478801107224794e-07, + "logits/chosen": -0.2483985424041748, + "logits/rejected": -0.23404812812805176, + "logps/chosen": -172.73133850097656, + "logps/ref_chosen": -52.39896011352539, + "logps/ref_rejected": -72.16735076904297, + "logps/rejected": -282.69036865234375, + "loss": 1.1524, + "margin_dpo/margin_mean": 90.19062042236328, + "margin_dpo/margin_std": 164.16030883789062, + "step": 189 + }, + { + "KL/chosen_KL_mean": -128.05345153808594, + "KL/mean": -182.98655700683594, + "KL/rejected_KL_mean": -237.91966247558594, + "KL/std": 141.72225952148438, + "epoch": 0.2790014684287812, + "fcm_dpo/beta": 0.004035481717437506, + "fcm_dpo/delta": -0.06281507015228271, + "fcm_dpo/margin": 109.86622619628906, + "fcm_dpo/q_t": 0.40009400248527527, + "grad_norm": 21.0288143157959, + "learning_rate": 4.5404922808905543e-07, + "logits/chosen": -0.29694175720214844, + "logits/rejected": -0.2908221483230591, + "logps/chosen": -192.73651123046875, + "logps/ref_chosen": -64.68305969238281, + "logps/ref_rejected": -102.55052185058594, + "logps/rejected": -340.47021484375, + "loss": 1.1029, + "margin_dpo/margin_mean": 109.86621856689453, + "margin_dpo/margin_std": 163.8136444091797, + "step": 190 + }, + { + "KL/chosen_KL_mean": -120.60160827636719, + "KL/mean": -199.1689910888672, + "KL/rejected_KL_mean": -277.73638916015625, + "KL/std": 161.70599365234375, + "epoch": 0.28046989720998533, + "fcm_dpo/beta": 0.0032641002908349037, + "fcm_dpo/delta": -0.16109062731266022, + "fcm_dpo/margin": 157.13475036621094, + "fcm_dpo/q_t": 0.377569317817688, + "grad_norm": 17.576215744018555, + "learning_rate": 4.5330506821893565e-07, + "logits/chosen": -0.2872377634048462, + "logits/rejected": -0.26138466596603394, + "logps/chosen": -189.26048278808594, + "logps/ref_chosen": -68.65887451171875, + "logps/ref_rejected": -110.1396713256836, + "logps/rejected": -387.8760681152344, + "loss": 1.0153, + "margin_dpo/margin_mean": 157.134765625, + "margin_dpo/margin_std": 182.57261657714844, + "step": 191 + }, + { + "KL/chosen_KL_mean": -150.72152709960938, + "KL/mean": -201.56427001953125, + "KL/rejected_KL_mean": -252.40701293945312, + "KL/std": 139.52310180664062, + "epoch": 0.28193832599118945, + "fcm_dpo/beta": 0.0034080445766448975, + "fcm_dpo/delta": 0.06307677179574966, + "fcm_dpo/margin": 101.68548583984375, + "fcm_dpo/q_t": 0.4241996109485626, + "grad_norm": 20.135229110717773, + "learning_rate": 4.5255555107119336e-07, + "logits/chosen": -0.22673380374908447, + "logits/rejected": -0.2247309386730194, + "logps/chosen": -220.44842529296875, + "logps/ref_chosen": -69.72691345214844, + "logps/ref_rejected": -103.32135009765625, + "logps/rejected": -355.7283630371094, + "loss": 1.1507, + "margin_dpo/margin_mean": 101.68548583984375, + "margin_dpo/margin_std": 175.06744384765625, + "step": 192 + }, + { + "KL/chosen_KL_mean": -148.80224609375, + "KL/mean": -178.2123260498047, + "KL/rejected_KL_mean": -207.62240600585938, + "KL/std": 128.90753173828125, + "epoch": 0.2834067547723935, + "fcm_dpo/beta": 0.0035294066183269024, + "fcm_dpo/delta": 0.06064280495047569, + "fcm_dpo/margin": 58.82013702392578, + "fcm_dpo/q_t": 0.45174944400787354, + "grad_norm": 27.31705665588379, + "learning_rate": 4.5180069639630236e-07, + "logits/chosen": -0.24892589449882507, + "logits/rejected": -0.23975345492362976, + "logps/chosen": -208.99273681640625, + "logps/ref_chosen": -60.19049835205078, + "logps/ref_rejected": -76.40755462646484, + "logps/rejected": -284.02996826171875, + "loss": 1.2674, + "margin_dpo/margin_mean": 58.82013702392578, + "margin_dpo/margin_std": 162.4898681640625, + "step": 193 + }, + { + "KL/chosen_KL_mean": -91.62898254394531, + "KL/mean": -139.33377075195312, + "KL/rejected_KL_mean": -187.03854370117188, + "KL/std": 104.66587829589844, + "epoch": 0.28487518355359764, + "fcm_dpo/beta": 0.003725461196154356, + "fcm_dpo/delta": 0.053960170596838, + "fcm_dpo/margin": 95.4095687866211, + "fcm_dpo/q_t": 0.4181906580924988, + "grad_norm": 16.410791397094727, + "learning_rate": 4.510405240853854e-07, + "logits/chosen": -0.19228887557983398, + "logits/rejected": -0.1776125431060791, + "logps/chosen": -129.4693603515625, + "logps/ref_chosen": -37.84037399291992, + "logps/ref_rejected": -60.684783935546875, + "logps/rejected": -247.72332763671875, + "loss": 1.1084, + "margin_dpo/margin_mean": 95.40956115722656, + "margin_dpo/margin_std": 115.09669494628906, + "step": 194 + }, + { + "KL/chosen_KL_mean": -141.21636962890625, + "KL/mean": -192.36294555664062, + "KL/rejected_KL_mean": -243.50949096679688, + "KL/std": 128.15513610839844, + "epoch": 0.28634361233480177, + "fcm_dpo/beta": 0.0038534309715032578, + "fcm_dpo/delta": 0.007251395843923092, + "fcm_dpo/margin": 102.29312133789062, + "fcm_dpo/q_t": 0.40964868664741516, + "grad_norm": 27.93120765686035, + "learning_rate": 4.5027505416968985e-07, + "logits/chosen": -0.26708340644836426, + "logits/rejected": -0.285901814699173, + "logps/chosen": -196.10794067382812, + "logps/ref_chosen": -54.891571044921875, + "logps/ref_rejected": -96.77095794677734, + "logps/rejected": -340.28045654296875, + "loss": 1.0919, + "margin_dpo/margin_mean": 102.29312133789062, + "margin_dpo/margin_std": 133.5623779296875, + "step": 195 + }, + { + "KL/chosen_KL_mean": -111.13381958007812, + "KL/mean": -168.23362731933594, + "KL/rejected_KL_mean": -225.33343505859375, + "KL/std": 128.486083984375, + "epoch": 0.2878120411160059, + "fcm_dpo/beta": 0.0036738794296979904, + "fcm_dpo/delta": -0.02965060994029045, + "fcm_dpo/margin": 114.1996078491211, + "fcm_dpo/q_t": 0.40418341755867004, + "grad_norm": 19.077123641967773, + "learning_rate": 4.495043068200599e-07, + "logits/chosen": -0.2980375289916992, + "logits/rejected": -0.2835182845592499, + "logps/chosen": -164.37905883789062, + "logps/ref_chosen": -53.245243072509766, + "logps/ref_rejected": -76.05294799804688, + "logps/rejected": -301.3863830566406, + "loss": 1.0895, + "margin_dpo/margin_mean": 114.19961547851562, + "margin_dpo/margin_std": 154.56077575683594, + "step": 196 + }, + { + "KL/chosen_KL_mean": -110.27510833740234, + "KL/mean": -154.23858642578125, + "KL/rejected_KL_mean": -198.20204162597656, + "KL/std": 109.70982360839844, + "epoch": 0.28928046989721, + "fcm_dpo/beta": 0.003983784466981888, + "fcm_dpo/delta": 0.05786813423037529, + "fcm_dpo/margin": 87.92694091796875, + "fcm_dpo/q_t": 0.4213848114013672, + "grad_norm": 21.03490447998047, + "learning_rate": 4.4872830234640493e-07, + "logits/chosen": -0.28392934799194336, + "logits/rejected": -0.27791300415992737, + "logps/chosen": -170.6954345703125, + "logps/ref_chosen": -60.42033386230469, + "logps/ref_rejected": -77.20890808105469, + "logps/rejected": -275.41094970703125, + "loss": 1.125, + "margin_dpo/margin_mean": 87.92694091796875, + "margin_dpo/margin_std": 125.18641662597656, + "step": 197 + }, + { + "KL/chosen_KL_mean": -122.45361328125, + "KL/mean": -180.43850708007812, + "KL/rejected_KL_mean": -238.42340087890625, + "KL/std": 138.5358428955078, + "epoch": 0.2907488986784141, + "fcm_dpo/beta": 0.0038145678117871284, + "fcm_dpo/delta": -0.054569028317928314, + "fcm_dpo/margin": 115.96977996826172, + "fcm_dpo/q_t": 0.399750292301178, + "grad_norm": 20.83743667602539, + "learning_rate": 4.479470611971645e-07, + "logits/chosen": -0.32312026619911194, + "logits/rejected": -0.322989821434021, + "logps/chosen": -177.48980712890625, + "logps/ref_chosen": -55.03618621826172, + "logps/ref_rejected": -97.24325561523438, + "logps/rejected": -335.6666564941406, + "loss": 1.0714, + "margin_dpo/margin_mean": 115.96978759765625, + "margin_dpo/margin_std": 157.2657470703125, + "step": 198 + }, + { + "KL/chosen_KL_mean": -116.48646545410156, + "KL/mean": -172.57876586914062, + "KL/rejected_KL_mean": -228.67108154296875, + "KL/std": 122.16108703613281, + "epoch": 0.2922173274596182, + "fcm_dpo/beta": 0.0036303927190601826, + "fcm_dpo/delta": -0.012894738465547562, + "fcm_dpo/margin": 112.18461608886719, + "fcm_dpo/q_t": 0.4059009552001953, + "grad_norm": 20.86189079284668, + "learning_rate": 4.471606039587695e-07, + "logits/chosen": -0.3357727527618408, + "logits/rejected": -0.3183862566947937, + "logps/chosen": -173.31529235839844, + "logps/ref_chosen": -56.828826904296875, + "logps/ref_rejected": -84.64820861816406, + "logps/rejected": -313.31927490234375, + "loss": 1.094, + "margin_dpo/margin_mean": 112.18460845947266, + "margin_dpo/margin_std": 152.09129333496094, + "step": 199 + }, + { + "KL/chosen_KL_mean": -113.27946472167969, + "KL/mean": -171.24513244628906, + "KL/rejected_KL_mean": -229.2107696533203, + "KL/std": 133.42398071289062, + "epoch": 0.2936857562408223, + "fcm_dpo/beta": 0.0035990295000374317, + "fcm_dpo/delta": -0.02324402891099453, + "fcm_dpo/margin": 115.93130493164062, + "fcm_dpo/q_t": 0.40560150146484375, + "grad_norm": 18.164777755737305, + "learning_rate": 4.4636895135509966e-07, + "logits/chosen": -0.2667478621006012, + "logits/rejected": -0.24850307404994965, + "logps/chosen": -166.34652709960938, + "logps/ref_chosen": -53.06706237792969, + "logps/ref_rejected": -80.60843658447266, + "logps/rejected": -309.8192138671875, + "loss": 1.0995, + "margin_dpo/margin_mean": 115.93130493164062, + "margin_dpo/margin_std": 171.93081665039062, + "step": 200 + }, + { + "KL/chosen_KL_mean": -123.16534423828125, + "KL/mean": -179.07032775878906, + "KL/rejected_KL_mean": -234.97531127929688, + "KL/std": 142.31027221679688, + "epoch": 0.29515418502202645, + "fcm_dpo/beta": 0.003661088179796934, + "fcm_dpo/delta": -0.012851493433117867, + "fcm_dpo/margin": 111.80996704101562, + "fcm_dpo/q_t": 0.40664464235305786, + "grad_norm": 20.639188766479492, + "learning_rate": 4.455721242469372e-07, + "logits/chosen": -0.37277939915657043, + "logits/rejected": -0.3691471219062805, + "logps/chosen": -198.56756591796875, + "logps/ref_chosen": -75.4022216796875, + "logps/ref_rejected": -114.80821990966797, + "logps/rejected": -349.78350830078125, + "loss": 1.1012, + "margin_dpo/margin_mean": 111.80996704101562, + "margin_dpo/margin_std": 161.63238525390625, + "step": 201 + }, + { + "KL/chosen_KL_mean": -127.29255676269531, + "KL/mean": -166.71469116210938, + "KL/rejected_KL_mean": -206.13681030273438, + "KL/std": 124.88301849365234, + "epoch": 0.2966226138032305, + "fcm_dpo/beta": 0.003907524980604649, + "fcm_dpo/delta": 0.10762214660644531, + "fcm_dpo/margin": 78.84428405761719, + "fcm_dpo/q_t": 0.4342569410800934, + "grad_norm": 23.870006561279297, + "learning_rate": 4.4477014363141755e-07, + "logits/chosen": -0.28092265129089355, + "logits/rejected": -0.29571932554244995, + "logps/chosen": -177.3938751220703, + "logps/ref_chosen": -50.101318359375, + "logps/ref_rejected": -86.98503112792969, + "logps/rejected": -293.1218566894531, + "loss": 1.1957, + "margin_dpo/margin_mean": 78.84428405761719, + "margin_dpo/margin_std": 163.82876586914062, + "step": 202 + }, + { + "KL/chosen_KL_mean": -127.91600036621094, + "KL/mean": -176.91217041015625, + "KL/rejected_KL_mean": -225.90835571289062, + "KL/std": 129.85752868652344, + "epoch": 0.29809104258443464, + "fcm_dpo/beta": 0.003999835811555386, + "fcm_dpo/delta": 0.009929049760103226, + "fcm_dpo/margin": 97.99237060546875, + "fcm_dpo/q_t": 0.4120560884475708, + "grad_norm": 20.524028778076172, + "learning_rate": 4.439630306414758e-07, + "logits/chosen": -0.31167030334472656, + "logits/rejected": -0.3005950450897217, + "logps/chosen": -188.52569580078125, + "logps/ref_chosen": -60.60969543457031, + "logps/ref_rejected": -85.89596557617188, + "logps/rejected": -311.8043212890625, + "loss": 1.1109, + "margin_dpo/margin_mean": 97.99237060546875, + "margin_dpo/margin_std": 146.369140625, + "step": 203 + }, + { + "KL/chosen_KL_mean": -140.44757080078125, + "KL/mean": -185.04940795898438, + "KL/rejected_KL_mean": -229.6512451171875, + "KL/std": 140.68167114257812, + "epoch": 0.29955947136563876, + "fcm_dpo/beta": 0.0041489386931061745, + "fcm_dpo/delta": 0.03660349175333977, + "fcm_dpo/margin": 89.20367431640625, + "fcm_dpo/q_t": 0.4208757281303406, + "grad_norm": 35.41379928588867, + "learning_rate": 4.431508065452897e-07, + "logits/chosen": -0.3780588209629059, + "logits/rejected": -0.33705025911331177, + "logps/chosen": -220.61251831054688, + "logps/ref_chosen": -80.16496276855469, + "logps/ref_rejected": -87.69590759277344, + "logps/rejected": -317.34716796875, + "loss": 1.1626, + "margin_dpo/margin_mean": 89.20367431640625, + "margin_dpo/margin_std": 168.600341796875, + "step": 204 + }, + { + "KL/chosen_KL_mean": -134.366455078125, + "KL/mean": -193.7669677734375, + "KL/rejected_KL_mean": -253.16749572753906, + "KL/std": 141.5926513671875, + "epoch": 0.3010279001468429, + "fcm_dpo/beta": 0.0037632733583450317, + "fcm_dpo/delta": -0.07647587358951569, + "fcm_dpo/margin": 118.80104064941406, + "fcm_dpo/q_t": 0.3929186463356018, + "grad_norm": 22.13498878479004, + "learning_rate": 4.4233349274571974e-07, + "logits/chosen": -0.3193209767341614, + "logits/rejected": -0.2915324568748474, + "logps/chosen": -193.75119018554688, + "logps/ref_chosen": -59.384735107421875, + "logps/ref_rejected": -85.12505340576172, + "logps/rejected": -338.29254150390625, + "loss": 1.0786, + "margin_dpo/margin_mean": 118.80104064941406, + "margin_dpo/margin_std": 156.5557861328125, + "step": 205 + }, + { + "KL/chosen_KL_mean": -120.28572082519531, + "KL/mean": -186.2382354736328, + "KL/rejected_KL_mean": -252.1907501220703, + "KL/std": 128.577880859375, + "epoch": 0.302496328928047, + "fcm_dpo/beta": 0.003537021577358246, + "fcm_dpo/delta": -0.0920427143573761, + "fcm_dpo/margin": 131.90501403808594, + "fcm_dpo/q_t": 0.3862435817718506, + "grad_norm": 23.741201400756836, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": -0.2517659068107605, + "logits/rejected": -0.2520918548107147, + "logps/chosen": -167.25021362304688, + "logps/ref_chosen": -46.964500427246094, + "logps/ref_rejected": -98.9534912109375, + "logps/rejected": -351.14422607421875, + "loss": 1.0313, + "margin_dpo/margin_mean": 131.90501403808594, + "margin_dpo/margin_std": 139.59078979492188, + "step": 206 + }, + { + "KL/chosen_KL_mean": -114.99893188476562, + "KL/mean": -187.25506591796875, + "KL/rejected_KL_mean": -259.5111999511719, + "KL/std": 148.06365966796875, + "epoch": 0.3039647577092511, + "fcm_dpo/beta": 0.003286686260253191, + "fcm_dpo/delta": -0.09856441617012024, + "fcm_dpo/margin": 144.51226806640625, + "fcm_dpo/q_t": 0.38960766792297363, + "grad_norm": 21.21698760986328, + "learning_rate": 4.4068368231789365e-07, + "logits/chosen": -0.35248827934265137, + "logits/rejected": -0.329440712928772, + "logps/chosen": -171.05519104003906, + "logps/ref_chosen": -56.05625915527344, + "logps/ref_rejected": -84.44779968261719, + "logps/rejected": -343.958984375, + "loss": 1.0401, + "margin_dpo/margin_mean": 144.51226806640625, + "margin_dpo/margin_std": 174.37086486816406, + "step": 207 + }, + { + "KL/chosen_KL_mean": -171.49325561523438, + "KL/mean": -230.3494110107422, + "KL/rejected_KL_mean": -289.20556640625, + "KL/std": 144.81240844726562, + "epoch": 0.3054331864904552, + "fcm_dpo/beta": 0.0032218745909631252, + "fcm_dpo/delta": 0.02522367238998413, + "fcm_dpo/margin": 117.71229553222656, + "fcm_dpo/q_t": 0.41357824206352234, + "grad_norm": 22.431468963623047, + "learning_rate": 4.398512291636768e-07, + "logits/chosen": -0.3535154461860657, + "logits/rejected": -0.3377763032913208, + "logps/chosen": -238.5608673095703, + "logps/ref_chosen": -67.06761169433594, + "logps/ref_rejected": -94.28689575195312, + "logps/rejected": -383.4924621582031, + "loss": 1.1181, + "margin_dpo/margin_mean": 117.71229553222656, + "margin_dpo/margin_std": 175.20639038085938, + "step": 208 + }, + { + "KL/chosen_KL_mean": -144.38653564453125, + "KL/mean": -197.31893920898438, + "KL/rejected_KL_mean": -250.25132751464844, + "KL/std": 131.18588256835938, + "epoch": 0.3069016152716593, + "fcm_dpo/beta": 0.003419704968109727, + "fcm_dpo/delta": 0.04580961540341377, + "fcm_dpo/margin": 105.86478424072266, + "fcm_dpo/q_t": 0.41855117678642273, + "grad_norm": 19.670215606689453, + "learning_rate": 4.3901377325300857e-07, + "logits/chosen": -0.231040820479393, + "logits/rejected": -0.2151014506816864, + "logps/chosen": -200.5682373046875, + "logps/ref_chosen": -56.18169403076172, + "logps/ref_rejected": -80.94152069091797, + "logps/rejected": -331.1928405761719, + "loss": 1.1333, + "margin_dpo/margin_mean": 105.86478424072266, + "margin_dpo/margin_std": 164.6919708251953, + "step": 209 + }, + { + "KL/chosen_KL_mean": -127.3232192993164, + "KL/mean": -187.63894653320312, + "KL/rejected_KL_mean": -247.95468139648438, + "KL/std": 131.5801544189453, + "epoch": 0.30837004405286345, + "fcm_dpo/beta": 0.0034523247741162777, + "fcm_dpo/delta": -0.0221034474670887, + "fcm_dpo/margin": 120.6314697265625, + "fcm_dpo/q_t": 0.4037366807460785, + "grad_norm": 20.2919979095459, + "learning_rate": 4.381713366536311e-07, + "logits/chosen": -0.28856372833251953, + "logits/rejected": -0.2808060348033905, + "logps/chosen": -173.69503784179688, + "logps/ref_chosen": -46.371822357177734, + "logps/ref_rejected": -76.68162536621094, + "logps/rejected": -324.63629150390625, + "loss": 1.0825, + "margin_dpo/margin_mean": 120.6314697265625, + "margin_dpo/margin_std": 156.716552734375, + "step": 210 + }, + { + "KL/chosen_KL_mean": -186.89505004882812, + "KL/mean": -239.9453125, + "KL/rejected_KL_mean": -292.99560546875, + "KL/std": 155.1609649658203, + "epoch": 0.30983847283406757, + "fcm_dpo/beta": 0.003472290001809597, + "fcm_dpo/delta": 0.038492538034915924, + "fcm_dpo/margin": 106.10052490234375, + "fcm_dpo/q_t": 0.42122966051101685, + "grad_norm": 28.37523078918457, + "learning_rate": 4.373239415645323e-07, + "logits/chosen": -0.2967555522918701, + "logits/rejected": -0.2570844888687134, + "logps/chosen": -265.827392578125, + "logps/ref_chosen": -78.93235778808594, + "logps/ref_rejected": -86.82098388671875, + "logps/rejected": -379.81658935546875, + "loss": 1.1436, + "margin_dpo/margin_mean": 106.10053253173828, + "margin_dpo/margin_std": 179.58560180664062, + "step": 211 + }, + { + "KL/chosen_KL_mean": -154.07533264160156, + "KL/mean": -228.19821166992188, + "KL/rejected_KL_mean": -302.32110595703125, + "KL/std": 162.68435668945312, + "epoch": 0.31130690161527164, + "fcm_dpo/beta": 0.0031211567111313343, + "fcm_dpo/delta": -0.09513364732265472, + "fcm_dpo/margin": 148.24575805664062, + "fcm_dpo/q_t": 0.3897473216056824, + "grad_norm": 25.847610473632812, + "learning_rate": 4.3647161031536086e-07, + "logits/chosen": -0.2827294170856476, + "logits/rejected": -0.2756040692329407, + "logps/chosen": -212.2723388671875, + "logps/ref_chosen": -58.19701385498047, + "logps/ref_rejected": -103.05785369873047, + "logps/rejected": -405.3789367675781, + "loss": 1.0549, + "margin_dpo/margin_mean": 148.24575805664062, + "margin_dpo/margin_std": 177.65496826171875, + "step": 212 + }, + { + "KL/chosen_KL_mean": -155.3287811279297, + "KL/mean": -228.24905395507812, + "KL/rejected_KL_mean": -301.16937255859375, + "KL/std": 149.9835205078125, + "epoch": 0.31277533039647576, + "fcm_dpo/beta": 0.0030004819855093956, + "fcm_dpo/delta": -0.05112471804022789, + "fcm_dpo/margin": 145.84056091308594, + "fcm_dpo/q_t": 0.3968704044818878, + "grad_norm": 19.53304672241211, + "learning_rate": 4.3561436536583774e-07, + "logits/chosen": -0.25978952646255493, + "logits/rejected": -0.23081305623054504, + "logps/chosen": -222.84149169921875, + "logps/ref_chosen": -67.51271057128906, + "logps/ref_rejected": -93.91471862792969, + "logps/rejected": -395.08404541015625, + "loss": 1.061, + "margin_dpo/margin_mean": 145.84056091308594, + "margin_dpo/margin_std": 175.0486602783203, + "step": 213 + }, + { + "KL/chosen_KL_mean": -141.36532592773438, + "KL/mean": -206.33975219726562, + "KL/rejected_KL_mean": -271.31414794921875, + "KL/std": 151.19796752929688, + "epoch": 0.3142437591776799, + "fcm_dpo/beta": 0.0030389183666557074, + "fcm_dpo/delta": 0.006323062814772129, + "fcm_dpo/margin": 129.9488525390625, + "fcm_dpo/q_t": 0.41051051020622253, + "grad_norm": 20.851654052734375, + "learning_rate": 4.3475222930516473e-07, + "logits/chosen": -0.2020268589258194, + "logits/rejected": -0.20587500929832458, + "logps/chosen": -182.97021484375, + "logps/ref_chosen": -41.604888916015625, + "logps/ref_rejected": -77.51741027832031, + "logps/rejected": -348.83160400390625, + "loss": 1.0964, + "margin_dpo/margin_mean": 129.94886779785156, + "margin_dpo/margin_std": 176.67694091796875, + "step": 214 + }, + { + "KL/chosen_KL_mean": -167.13980102539062, + "KL/mean": -235.6959228515625, + "KL/rejected_KL_mean": -304.2520446777344, + "KL/std": 154.10137939453125, + "epoch": 0.315712187958884, + "fcm_dpo/beta": 0.0029814334120601416, + "fcm_dpo/delta": -0.011681612581014633, + "fcm_dpo/margin": 137.1122283935547, + "fcm_dpo/q_t": 0.4044470191001892, + "grad_norm": 21.97011947631836, + "learning_rate": 4.3388522485142885e-07, + "logits/chosen": -0.2605951428413391, + "logits/rejected": -0.25532659888267517, + "logps/chosen": -220.4190673828125, + "logps/ref_chosen": -53.279266357421875, + "logps/ref_rejected": -89.96464538574219, + "logps/rejected": -394.2166748046875, + "loss": 1.0713, + "margin_dpo/margin_mean": 137.11224365234375, + "margin_dpo/margin_std": 158.41067504882812, + "step": 215 + }, + { + "KL/chosen_KL_mean": -169.673583984375, + "KL/mean": -237.0709228515625, + "KL/rejected_KL_mean": -304.46826171875, + "KL/std": 161.4061737060547, + "epoch": 0.31718061674008813, + "fcm_dpo/beta": 0.003034801222383976, + "fcm_dpo/delta": -0.012860134243965149, + "fcm_dpo/margin": 134.794677734375, + "fcm_dpo/q_t": 0.4068460166454315, + "grad_norm": 28.930782318115234, + "learning_rate": 4.330133748510036e-07, + "logits/chosen": -0.2749112844467163, + "logits/rejected": -0.260329008102417, + "logps/chosen": -218.56137084960938, + "logps/ref_chosen": -48.887794494628906, + "logps/ref_rejected": -77.19892883300781, + "logps/rejected": -381.66717529296875, + "loss": 1.1026, + "margin_dpo/margin_mean": 134.794677734375, + "margin_dpo/margin_std": 196.2810821533203, + "step": 216 + }, + { + "KL/chosen_KL_mean": -173.36903381347656, + "KL/mean": -254.24615478515625, + "KL/rejected_KL_mean": -335.123291015625, + "KL/std": 167.5054168701172, + "epoch": 0.3186490455212922, + "fcm_dpo/beta": 0.0027936077676713467, + "fcm_dpo/delta": -0.06825002282857895, + "fcm_dpo/margin": 161.75425720214844, + "fcm_dpo/q_t": 0.39398378133773804, + "grad_norm": 19.45440673828125, + "learning_rate": 4.3213670227794757e-07, + "logits/chosen": -0.2623336613178253, + "logits/rejected": -0.2586092948913574, + "logps/chosen": -223.21434020996094, + "logps/ref_chosen": -49.845306396484375, + "logps/ref_rejected": -100.07832336425781, + "logps/rejected": -435.20159912109375, + "loss": 1.0481, + "margin_dpo/margin_mean": 161.75424194335938, + "margin_dpo/margin_std": 188.6632080078125, + "step": 217 + }, + { + "KL/chosen_KL_mean": -177.95050048828125, + "KL/mean": -239.9977264404297, + "KL/rejected_KL_mean": -302.0449523925781, + "KL/std": 164.52496337890625, + "epoch": 0.3201174743024963, + "fcm_dpo/beta": 0.0028708684258162975, + "fcm_dpo/delta": 0.05333707481622696, + "fcm_dpo/margin": 124.09444427490234, + "fcm_dpo/q_t": 0.42141902446746826, + "grad_norm": 22.139909744262695, + "learning_rate": 4.3125523023339815e-07, + "logits/chosen": -0.296860933303833, + "logits/rejected": -0.29232728481292725, + "logps/chosen": -236.52719116210938, + "logps/ref_chosen": -58.576683044433594, + "logps/ref_rejected": -87.84639739990234, + "logps/rejected": -389.891357421875, + "loss": 1.1373, + "margin_dpo/margin_mean": 124.09444427490234, + "margin_dpo/margin_std": 196.5043487548828, + "step": 218 + }, + { + "KL/chosen_KL_mean": -184.52737426757812, + "KL/mean": -234.72857666015625, + "KL/rejected_KL_mean": -284.92974853515625, + "KL/std": 165.06085205078125, + "epoch": 0.32158590308370044, + "fcm_dpo/beta": 0.002932458883151412, + "fcm_dpo/delta": 0.0077366819605231285, + "fcm_dpo/margin": 100.40234375, + "fcm_dpo/q_t": 0.4327548146247864, + "grad_norm": 33.82648468017578, + "learning_rate": 4.303689819449636e-07, + "logits/chosen": -0.30966275930404663, + "logits/rejected": -0.3071323335170746, + "logps/chosen": -245.61123657226562, + "logps/ref_chosen": -61.083858489990234, + "logps/ref_rejected": -85.83042907714844, + "logps/rejected": -370.76019287109375, + "loss": 1.1977, + "margin_dpo/margin_mean": 100.40234375, + "margin_dpo/margin_std": 202.93426513671875, + "step": 219 + }, + { + "KL/chosen_KL_mean": -190.68328857421875, + "KL/mean": -234.95462036132812, + "KL/rejected_KL_mean": -279.2259216308594, + "KL/std": 134.0245361328125, + "epoch": 0.32305433186490456, + "fcm_dpo/beta": 0.00323110050521791, + "fcm_dpo/delta": 0.1334986686706543, + "fcm_dpo/margin": 88.54261779785156, + "fcm_dpo/q_t": 0.4363420009613037, + "grad_norm": 33.53060531616211, + "learning_rate": 4.2947798076611047e-07, + "logits/chosen": -0.3398175835609436, + "logits/rejected": -0.31802213191986084, + "logps/chosen": -260.714599609375, + "logps/ref_chosen": -70.03128051757812, + "logps/ref_rejected": -87.68551635742188, + "logps/rejected": -366.91143798828125, + "loss": 1.1692, + "margin_dpo/margin_mean": 88.54261779785156, + "margin_dpo/margin_std": 137.38961791992188, + "step": 220 + }, + { + "KL/chosen_KL_mean": -137.20208740234375, + "KL/mean": -223.91976928710938, + "KL/rejected_KL_mean": -310.637451171875, + "KL/std": 157.56552124023438, + "epoch": 0.3245227606461087, + "fcm_dpo/beta": 0.003003270598128438, + "fcm_dpo/delta": -0.16315576434135437, + "fcm_dpo/margin": 173.43536376953125, + "fcm_dpo/q_t": 0.37127232551574707, + "grad_norm": 135.19117736816406, + "learning_rate": 4.285822501755485e-07, + "logits/chosen": -0.33175361156463623, + "logits/rejected": -0.3400239944458008, + "logps/chosen": -189.3568115234375, + "logps/ref_chosen": -52.15470886230469, + "logps/ref_rejected": -106.46768188476562, + "logps/rejected": -417.10516357421875, + "loss": 0.987, + "margin_dpo/margin_mean": 173.43536376953125, + "margin_dpo/margin_std": 158.20767211914062, + "step": 221 + }, + { + "KL/chosen_KL_mean": -147.8143768310547, + "KL/mean": -214.55850219726562, + "KL/rejected_KL_mean": -281.3026123046875, + "KL/std": 146.41705322265625, + "epoch": 0.32599118942731276, + "fcm_dpo/beta": 0.0029258220456540585, + "fcm_dpo/delta": 0.01042582280933857, + "fcm_dpo/margin": 133.4882354736328, + "fcm_dpo/q_t": 0.40958648920059204, + "grad_norm": 62.3775520324707, + "learning_rate": 4.276818137766118e-07, + "logits/chosen": -0.38752639293670654, + "logits/rejected": -0.3927309811115265, + "logps/chosen": -208.7854766845703, + "logps/ref_chosen": -60.971099853515625, + "logps/ref_rejected": -100.00115203857422, + "logps/rejected": -381.30377197265625, + "loss": 1.0883, + "margin_dpo/margin_mean": 133.4882354736328, + "margin_dpo/margin_std": 164.23587036132812, + "step": 222 + }, + { + "KL/chosen_KL_mean": -165.07833862304688, + "KL/mean": -222.50390625, + "KL/rejected_KL_mean": -279.92950439453125, + "KL/std": 146.90487670898438, + "epoch": 0.3274596182085169, + "fcm_dpo/beta": 0.0030144588090479374, + "fcm_dpo/delta": 0.06525313854217529, + "fcm_dpo/margin": 114.85115051269531, + "fcm_dpo/q_t": 0.4231482148170471, + "grad_norm": 22.007301330566406, + "learning_rate": 4.2677669529663686e-07, + "logits/chosen": -0.2665255069732666, + "logits/rejected": -0.26349782943725586, + "logps/chosen": -217.71890258789062, + "logps/ref_chosen": -52.64057540893555, + "logps/ref_rejected": -82.82502746582031, + "logps/rejected": -362.7545166015625, + "loss": 1.1441, + "margin_dpo/margin_mean": 114.85115051269531, + "margin_dpo/margin_std": 187.14393615722656, + "step": 223 + }, + { + "KL/chosen_KL_mean": -144.54835510253906, + "KL/mean": -207.91268920898438, + "KL/rejected_KL_mean": -271.27703857421875, + "KL/std": 162.03009033203125, + "epoch": 0.328928046989721, + "fcm_dpo/beta": 0.003018029499799013, + "fcm_dpo/delta": 0.017022643238306046, + "fcm_dpo/margin": 126.72867584228516, + "fcm_dpo/q_t": 0.41384878754615784, + "grad_norm": 22.61432456970215, + "learning_rate": 4.2586691858633747e-07, + "logits/chosen": -0.29570257663726807, + "logits/rejected": -0.28287479281425476, + "logps/chosen": -193.14376831054688, + "logps/ref_chosen": -48.59541320800781, + "logps/ref_rejected": -77.11648559570312, + "logps/rejected": -348.39349365234375, + "loss": 1.1178, + "margin_dpo/margin_mean": 126.72867584228516, + "margin_dpo/margin_std": 185.35171508789062, + "step": 224 + }, + { + "KL/chosen_KL_mean": -163.48281860351562, + "KL/mean": -239.61199951171875, + "KL/rejected_KL_mean": -315.74114990234375, + "KL/std": 156.39309692382812, + "epoch": 0.3303964757709251, + "fcm_dpo/beta": 0.002961981575936079, + "fcm_dpo/delta": -0.066794753074646, + "fcm_dpo/margin": 152.25836181640625, + "fcm_dpo/q_t": 0.3953250050544739, + "grad_norm": 24.43362045288086, + "learning_rate": 4.249525076191759e-07, + "logits/chosen": -0.27601003646850586, + "logits/rejected": -0.26785707473754883, + "logps/chosen": -221.4832763671875, + "logps/ref_chosen": -58.000465393066406, + "logps/ref_rejected": -99.90291595458984, + "logps/rejected": -415.64410400390625, + "loss": 1.0628, + "margin_dpo/margin_mean": 152.25836181640625, + "margin_dpo/margin_std": 197.67453002929688, + "step": 225 + }, + { + "KL/chosen_KL_mean": -141.99343872070312, + "KL/mean": -200.15957641601562, + "KL/rejected_KL_mean": -258.32568359375, + "KL/std": 151.08013916015625, + "epoch": 0.33186490455212925, + "fcm_dpo/beta": 0.002992121037095785, + "fcm_dpo/delta": 0.06153475120663643, + "fcm_dpo/margin": 116.332275390625, + "fcm_dpo/q_t": 0.4230756461620331, + "grad_norm": 27.1488037109375, + "learning_rate": 4.2403348649073167e-07, + "logits/chosen": -0.33648887276649475, + "logits/rejected": -0.3032319247722626, + "logps/chosen": -200.89224243164062, + "logps/ref_chosen": -58.898799896240234, + "logps/ref_rejected": -78.68775939941406, + "logps/rejected": -337.0134582519531, + "loss": 1.1404, + "margin_dpo/margin_mean": 116.332275390625, + "margin_dpo/margin_std": 179.59532165527344, + "step": 226 + }, + { + "KL/chosen_KL_mean": -164.3446807861328, + "KL/mean": -242.88470458984375, + "KL/rejected_KL_mean": -321.42474365234375, + "KL/std": 180.00967407226562, + "epoch": 0.3333333333333333, + "fcm_dpo/beta": 0.002881108084693551, + "fcm_dpo/delta": -0.07316943258047104, + "fcm_dpo/margin": 157.080078125, + "fcm_dpo/q_t": 0.3937509059906006, + "grad_norm": 23.208093643188477, + "learning_rate": 4.2310987941806615e-07, + "logits/chosen": -0.31838715076446533, + "logits/rejected": -0.3055545687675476, + "logps/chosen": -223.41685485839844, + "logps/ref_chosen": -59.072181701660156, + "logps/ref_rejected": -99.41236877441406, + "logps/rejected": -420.8371276855469, + "loss": 1.0542, + "margin_dpo/margin_mean": 157.08009338378906, + "margin_dpo/margin_std": 193.092529296875, + "step": 227 + }, + { + "KL/chosen_KL_mean": -175.99685668945312, + "KL/mean": -229.3104248046875, + "KL/rejected_KL_mean": -282.62396240234375, + "KL/std": 151.4019775390625, + "epoch": 0.33480176211453744, + "fcm_dpo/beta": 0.003110603429377079, + "fcm_dpo/delta": 0.08030124008655548, + "fcm_dpo/margin": 106.62710571289062, + "fcm_dpo/q_t": 0.42634642124176025, + "grad_norm": 26.213809967041016, + "learning_rate": 4.2218171073908463e-07, + "logits/chosen": -0.29573142528533936, + "logits/rejected": -0.28110069036483765, + "logps/chosen": -241.88815307617188, + "logps/ref_chosen": -65.89128875732422, + "logps/ref_rejected": -91.04875183105469, + "logps/rejected": -373.6727294921875, + "loss": 1.1646, + "margin_dpo/margin_mean": 106.62710571289062, + "margin_dpo/margin_std": 190.84280395507812, + "step": 228 + }, + { + "KL/chosen_KL_mean": -185.06878662109375, + "KL/mean": -240.96780395507812, + "KL/rejected_KL_mean": -296.8668212890625, + "KL/std": 172.71731567382812, + "epoch": 0.33627019089574156, + "fcm_dpo/beta": 0.003258654149249196, + "fcm_dpo/delta": 0.043283235281705856, + "fcm_dpo/margin": 111.79805755615234, + "fcm_dpo/q_t": 0.41782456636428833, + "grad_norm": 42.24066925048828, + "learning_rate": 4.212490049118951e-07, + "logits/chosen": -0.40128546953201294, + "logits/rejected": -0.3743479251861572, + "logps/chosen": -255.775146484375, + "logps/ref_chosen": -70.70637512207031, + "logps/ref_rejected": -84.52741241455078, + "logps/rejected": -381.39422607421875, + "loss": 1.1386, + "margin_dpo/margin_mean": 111.79806518554688, + "margin_dpo/margin_std": 184.9072265625, + "step": 229 + }, + { + "KL/chosen_KL_mean": -130.72586059570312, + "KL/mean": -217.06655883789062, + "KL/rejected_KL_mean": -303.4072265625, + "KL/std": 151.964111328125, + "epoch": 0.3377386196769457, + "fcm_dpo/beta": 0.0029333364218473434, + "fcm_dpo/delta": -0.14437267184257507, + "fcm_dpo/margin": 172.68138122558594, + "fcm_dpo/q_t": 0.37497806549072266, + "grad_norm": 67.53182983398438, + "learning_rate": 4.203117865141635e-07, + "logits/chosen": -0.2871912717819214, + "logits/rejected": -0.2930827736854553, + "logps/chosen": -170.0078582763672, + "logps/ref_chosen": -39.282005310058594, + "logps/ref_rejected": -85.62191009521484, + "logps/rejected": -389.0291442871094, + "loss": 0.9974, + "margin_dpo/margin_mean": 172.68138122558594, + "margin_dpo/margin_std": 165.72486877441406, + "step": 230 + }, + { + "KL/chosen_KL_mean": -154.43719482421875, + "KL/mean": -212.76937866210938, + "KL/rejected_KL_mean": -271.1015319824219, + "KL/std": 134.67486572265625, + "epoch": 0.3392070484581498, + "fcm_dpo/beta": 0.002986223204061389, + "fcm_dpo/delta": 0.06204049289226532, + "fcm_dpo/margin": 116.66436767578125, + "fcm_dpo/q_t": 0.42223888635635376, + "grad_norm": 29.774654388427734, + "learning_rate": 4.1937008024246625e-07, + "logits/chosen": -0.36699360609054565, + "logits/rejected": -0.34106624126434326, + "logps/chosen": -217.713623046875, + "logps/ref_chosen": -63.27644348144531, + "logps/ref_rejected": -74.1239013671875, + "logps/rejected": -345.2254638671875, + "loss": 1.1221, + "margin_dpo/margin_mean": 116.66436004638672, + "margin_dpo/margin_std": 160.6752471923828, + "step": 231 + }, + { + "KL/chosen_KL_mean": -198.68173217773438, + "KL/mean": -247.72219848632812, + "KL/rejected_KL_mean": -296.7626953125, + "KL/std": 164.42752075195312, + "epoch": 0.3406754772393539, + "fcm_dpo/beta": 0.003230078611522913, + "fcm_dpo/delta": 0.09927871823310852, + "fcm_dpo/margin": 98.08092498779297, + "fcm_dpo/q_t": 0.43197596073150635, + "grad_norm": 22.91642189025879, + "learning_rate": 4.1842391091163933e-07, + "logits/chosen": -0.3694447875022888, + "logits/rejected": -0.35070955753326416, + "logps/chosen": -269.43048095703125, + "logps/ref_chosen": -70.74876403808594, + "logps/ref_rejected": -83.97706604003906, + "logps/rejected": -380.73974609375, + "loss": 1.1624, + "margin_dpo/margin_mean": 98.08092498779297, + "margin_dpo/margin_std": 167.9994354248047, + "step": 232 + }, + { + "KL/chosen_KL_mean": -184.214599609375, + "KL/mean": -265.7254638671875, + "KL/rejected_KL_mean": -347.236328125, + "KL/std": 183.4271240234375, + "epoch": 0.342143906020558, + "fcm_dpo/beta": 0.0030570742674171925, + "fcm_dpo/delta": -0.13046759366989136, + "fcm_dpo/margin": 163.021728515625, + "fcm_dpo/q_t": 0.38850700855255127, + "grad_norm": 26.11095428466797, + "learning_rate": 4.174733034541245e-07, + "logits/chosen": -0.33819979429244995, + "logits/rejected": -0.3396344780921936, + "logps/chosen": -239.0975341796875, + "logps/ref_chosen": -54.8829345703125, + "logps/ref_rejected": -107.4800796508789, + "logps/rejected": -454.7164001464844, + "loss": 1.0684, + "margin_dpo/margin_mean": 163.021728515625, + "margin_dpo/margin_std": 239.56597900390625, + "step": 233 + }, + { + "KL/chosen_KL_mean": -182.37313842773438, + "KL/mean": -264.9031982421875, + "KL/rejected_KL_mean": -347.43328857421875, + "KL/std": 161.71844482421875, + "epoch": 0.3436123348017621, + "fcm_dpo/beta": 0.0026757342275232077, + "fcm_dpo/delta": -0.06500078737735748, + "fcm_dpo/margin": 165.0601348876953, + "fcm_dpo/q_t": 0.3940809965133667, + "grad_norm": 23.45154571533203, + "learning_rate": 4.165182829193126e-07, + "logits/chosen": -0.3571465015411377, + "logits/rejected": -0.3816158175468445, + "logps/chosen": -226.4676513671875, + "logps/ref_chosen": -44.094520568847656, + "logps/ref_rejected": -100.00663757324219, + "logps/rejected": -447.4399108886719, + "loss": 1.0517, + "margin_dpo/margin_mean": 165.0601348876953, + "margin_dpo/margin_std": 179.3123779296875, + "step": 234 + }, + { + "KL/chosen_KL_mean": -224.09396362304688, + "KL/mean": -279.3703308105469, + "KL/rejected_KL_mean": -334.64666748046875, + "KL/std": 163.57676696777344, + "epoch": 0.34508076358296624, + "fcm_dpo/beta": 0.0029698603320866823, + "fcm_dpo/delta": 0.08258074522018433, + "fcm_dpo/margin": 110.55270385742188, + "fcm_dpo/q_t": 0.42682477831840515, + "grad_norm": 27.62843894958496, + "learning_rate": 4.1555887447288255e-07, + "logits/chosen": -0.34939903020858765, + "logits/rejected": -0.33593958616256714, + "logps/chosen": -286.3318786621094, + "logps/ref_chosen": -62.237911224365234, + "logps/ref_rejected": -90.39506530761719, + "logps/rejected": -425.041748046875, + "loss": 1.1655, + "margin_dpo/margin_mean": 110.55270385742188, + "margin_dpo/margin_std": 197.34738159179688, + "step": 235 + }, + { + "KL/chosen_KL_mean": -168.29141235351562, + "KL/mean": -257.59796142578125, + "KL/rejected_KL_mean": -346.9044494628906, + "KL/std": 175.53897094726562, + "epoch": 0.3465491923641703, + "fcm_dpo/beta": 0.002734276931732893, + "fcm_dpo/delta": -0.11726871132850647, + "fcm_dpo/margin": 178.61306762695312, + "fcm_dpo/q_t": 0.3805997967720032, + "grad_norm": 28.633899688720703, + "learning_rate": 4.1459510339613946e-07, + "logits/chosen": -0.33454635739326477, + "logits/rejected": -0.337698370218277, + "logps/chosen": -217.6327667236328, + "logps/ref_chosen": -49.34136199951172, + "logps/ref_rejected": -103.51162719726562, + "logps/rejected": -450.41607666015625, + "loss": 1.0055, + "margin_dpo/margin_mean": 178.61306762695312, + "margin_dpo/margin_std": 166.25515747070312, + "step": 236 + }, + { + "KL/chosen_KL_mean": -238.2225341796875, + "KL/mean": -303.3404541015625, + "KL/rejected_KL_mean": -368.45843505859375, + "KL/std": 182.74667358398438, + "epoch": 0.34801762114537443, + "fcm_dpo/beta": 0.0027634776197373867, + "fcm_dpo/delta": 0.04837137833237648, + "fcm_dpo/margin": 130.23587036132812, + "fcm_dpo/q_t": 0.4194791316986084, + "grad_norm": 28.578073501586914, + "learning_rate": 4.136269950853473e-07, + "logits/chosen": -0.3175830841064453, + "logits/rejected": -0.31506621837615967, + "logps/chosen": -292.3906555175781, + "logps/ref_chosen": -54.168121337890625, + "logps/ref_rejected": -94.78036499023438, + "logps/rejected": -463.23876953125, + "loss": 1.1321, + "margin_dpo/margin_mean": 130.23587036132812, + "margin_dpo/margin_std": 204.54656982421875, + "step": 237 + }, + { + "KL/chosen_KL_mean": -218.0069580078125, + "KL/mean": -284.595458984375, + "KL/rejected_KL_mean": -351.18402099609375, + "KL/std": 181.19268798828125, + "epoch": 0.34948604992657856, + "fcm_dpo/beta": 0.002782113617286086, + "fcm_dpo/delta": 0.034211453050374985, + "fcm_dpo/margin": 133.17703247070312, + "fcm_dpo/q_t": 0.41864013671875, + "grad_norm": 27.10376739501953, + "learning_rate": 4.126545750510605e-07, + "logits/chosen": -0.2829251289367676, + "logits/rejected": -0.29289430379867554, + "logps/chosen": -271.9801025390625, + "logps/ref_chosen": -53.973121643066406, + "logps/ref_rejected": -89.41795349121094, + "logps/rejected": -440.6019592285156, + "loss": 1.1244, + "margin_dpo/margin_mean": 133.17703247070312, + "margin_dpo/margin_std": 202.1953582763672, + "step": 238 + }, + { + "KL/chosen_KL_mean": -232.07142639160156, + "KL/mean": -307.0456848144531, + "KL/rejected_KL_mean": -382.0199279785156, + "KL/std": 173.7324981689453, + "epoch": 0.3509544787077827, + "fcm_dpo/beta": 0.002740685362368822, + "fcm_dpo/delta": -0.020037677139043808, + "fcm_dpo/margin": 149.948486328125, + "fcm_dpo/q_t": 0.4032841622829437, + "grad_norm": 26.712921142578125, + "learning_rate": 4.116778689174514e-07, + "logits/chosen": -0.308989942073822, + "logits/rejected": -0.3054299056529999, + "logps/chosen": -290.16925048828125, + "logps/ref_chosen": -58.09782409667969, + "logps/ref_rejected": -93.59294128417969, + "logps/rejected": -475.61285400390625, + "loss": 1.0861, + "margin_dpo/margin_mean": 149.948486328125, + "margin_dpo/margin_std": 189.98800659179688, + "step": 239 + }, + { + "KL/chosen_KL_mean": -229.48007202148438, + "KL/mean": -289.2586669921875, + "KL/rejected_KL_mean": -349.0372314453125, + "KL/std": 180.47412109375, + "epoch": 0.3524229074889868, + "fcm_dpo/beta": 0.0029091420583426952, + "fcm_dpo/delta": 0.06307174265384674, + "fcm_dpo/margin": 119.55715942382812, + "fcm_dpo/q_t": 0.4230736494064331, + "grad_norm": 33.61320114135742, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": -0.33556127548217773, + "logits/rejected": -0.3180205523967743, + "logps/chosen": -290.09454345703125, + "logps/ref_chosen": -60.6144905090332, + "logps/ref_rejected": -74.1185302734375, + "logps/rejected": -423.15576171875, + "loss": 1.164, + "margin_dpo/margin_mean": 119.55715942382812, + "margin_dpo/margin_std": 214.35040283203125, + "step": 240 + }, + { + "KL/chosen_KL_mean": -187.94955444335938, + "KL/mean": -281.572998046875, + "KL/rejected_KL_mean": -375.19647216796875, + "KL/std": 189.39794921875, + "epoch": 0.35389133627019087, + "fcm_dpo/beta": 0.0027145131025463343, + "fcm_dpo/delta": -0.1450301706790924, + "fcm_dpo/margin": 187.2469024658203, + "fcm_dpo/q_t": 0.37965553998947144, + "grad_norm": 25.904090881347656, + "learning_rate": 4.097117014129903e-07, + "logits/chosen": -0.3710273504257202, + "logits/rejected": -0.35753804445266724, + "logps/chosen": -254.04063415527344, + "logps/ref_chosen": -66.091064453125, + "logps/ref_rejected": -88.06088256835938, + "logps/rejected": -463.25738525390625, + "loss": 1.0206, + "margin_dpo/margin_mean": 187.24691772460938, + "margin_dpo/margin_std": 215.96273803710938, + "step": 241 + }, + { + "KL/chosen_KL_mean": -197.42364501953125, + "KL/mean": -263.04095458984375, + "KL/rejected_KL_mean": -328.6582336425781, + "KL/std": 158.09832763671875, + "epoch": 0.355359765051395, + "fcm_dpo/beta": 0.002751118503510952, + "fcm_dpo/delta": 0.04367631673812866, + "fcm_dpo/margin": 131.2345428466797, + "fcm_dpo/q_t": 0.418499618768692, + "grad_norm": 23.890216827392578, + "learning_rate": 4.087222918524807e-07, + "logits/chosen": -0.3082933723926544, + "logits/rejected": -0.2922123968601227, + "logps/chosen": -265.2875671386719, + "logps/ref_chosen": -67.86392974853516, + "logps/ref_rejected": -83.36033630371094, + "logps/rejected": -412.0185546875, + "loss": 1.1207, + "margin_dpo/margin_mean": 131.2345428466797, + "margin_dpo/margin_std": 186.8716278076172, + "step": 242 + }, + { + "KL/chosen_KL_mean": -195.4427032470703, + "KL/mean": -275.4404296875, + "KL/rejected_KL_mean": -355.43817138671875, + "KL/std": 174.02914428710938, + "epoch": 0.3568281938325991, + "fcm_dpo/beta": 0.0026427144184708595, + "fcm_dpo/delta": -0.029283707961440086, + "fcm_dpo/margin": 159.99542236328125, + "fcm_dpo/q_t": 0.4009614586830139, + "grad_norm": 22.617460250854492, + "learning_rate": 4.07728699811968e-07, + "logits/chosen": -0.3353385329246521, + "logits/rejected": -0.31745070219039917, + "logps/chosen": -258.5269470214844, + "logps/ref_chosen": -63.0842399597168, + "logps/ref_rejected": -76.33563232421875, + "logps/rejected": -431.7738037109375, + "loss": 1.0667, + "margin_dpo/margin_mean": 159.99542236328125, + "margin_dpo/margin_std": 191.55514526367188, + "step": 243 + }, + { + "KL/chosen_KL_mean": -166.66122436523438, + "KL/mean": -250.8197021484375, + "KL/rejected_KL_mean": -334.9781799316406, + "KL/std": 169.25035095214844, + "epoch": 0.35829662261380324, + "fcm_dpo/beta": 0.002540828660130501, + "fcm_dpo/delta": -0.0361553393304348, + "fcm_dpo/margin": 168.3169403076172, + "fcm_dpo/q_t": 0.3987560272216797, + "grad_norm": 25.22274398803711, + "learning_rate": 4.067309514735267e-07, + "logits/chosen": -0.409039705991745, + "logits/rejected": -0.4093121290206909, + "logps/chosen": -227.80191040039062, + "logps/ref_chosen": -61.140689849853516, + "logps/ref_rejected": -94.89193725585938, + "logps/rejected": -429.8701171875, + "loss": 1.0496, + "margin_dpo/margin_mean": 168.3169403076172, + "margin_dpo/margin_std": 171.68531799316406, + "step": 244 + }, + { + "KL/chosen_KL_mean": -197.50634765625, + "KL/mean": -263.1775817871094, + "KL/rejected_KL_mean": -328.84881591796875, + "KL/std": 168.23464965820312, + "epoch": 0.35976505139500736, + "fcm_dpo/beta": 0.0024484535679221153, + "fcm_dpo/delta": -0.03805915266275406, + "fcm_dpo/margin": 131.34243774414062, + "fcm_dpo/q_t": 0.4226040244102478, + "grad_norm": 26.150102615356445, + "learning_rate": 4.057290731287531e-07, + "logits/chosen": -0.36390721797943115, + "logits/rejected": -0.351279079914093, + "logps/chosen": -264.76861572265625, + "logps/ref_chosen": -67.26228332519531, + "logps/ref_rejected": -87.64010620117188, + "logps/rejected": -416.4888916015625, + "loss": 1.1443, + "margin_dpo/margin_mean": 131.3424530029297, + "margin_dpo/margin_std": 187.38204956054688, + "step": 245 + }, + { + "KL/chosen_KL_mean": -202.00189208984375, + "KL/mean": -270.38726806640625, + "KL/rejected_KL_mean": -338.77264404296875, + "KL/std": 191.9010009765625, + "epoch": 0.36123348017621143, + "fcm_dpo/beta": 0.00253550847992301, + "fcm_dpo/delta": 0.0644979327917099, + "fcm_dpo/margin": 136.77073669433594, + "fcm_dpo/q_t": 0.42329055070877075, + "grad_norm": 24.134305953979492, + "learning_rate": 4.047230911780736e-07, + "logits/chosen": -0.394417405128479, + "logits/rejected": -0.3600098490715027, + "logps/chosen": -268.6988525390625, + "logps/ref_chosen": -66.69696807861328, + "logps/ref_rejected": -84.34634399414062, + "logps/rejected": -423.1189880371094, + "loss": 1.1347, + "margin_dpo/margin_mean": 136.770751953125, + "margin_dpo/margin_std": 207.49807739257812, + "step": 246 + }, + { + "KL/chosen_KL_mean": -252.14999389648438, + "KL/mean": -353.76324462890625, + "KL/rejected_KL_mean": -455.37652587890625, + "KL/std": 205.67657470703125, + "epoch": 0.36270190895741555, + "fcm_dpo/beta": 0.002377366181463003, + "fcm_dpo/delta": -0.11207044124603271, + "fcm_dpo/margin": 203.22650146484375, + "fcm_dpo/q_t": 0.38493090867996216, + "grad_norm": 25.502174377441406, + "learning_rate": 4.0371303213004814e-07, + "logits/chosen": -0.320733904838562, + "logits/rejected": -0.31779003143310547, + "logps/chosen": -308.7553405761719, + "logps/ref_chosen": -56.6053466796875, + "logps/ref_rejected": -106.29326629638672, + "logps/rejected": -561.6697998046875, + "loss": 1.0303, + "margin_dpo/margin_mean": 203.22650146484375, + "margin_dpo/margin_std": 228.47640991210938, + "step": 247 + }, + { + "KL/chosen_KL_mean": -224.57611083984375, + "KL/mean": -309.53057861328125, + "KL/rejected_KL_mean": -394.4850158691406, + "KL/std": 169.4248046875, + "epoch": 0.3641703377386197, + "fcm_dpo/beta": 0.0023004484828561544, + "fcm_dpo/delta": 0.009408872574567795, + "fcm_dpo/margin": 169.908935546875, + "fcm_dpo/q_t": 0.4071027934551239, + "grad_norm": 27.232206344604492, + "learning_rate": 4.0269892260067197e-07, + "logits/chosen": -0.2981037199497223, + "logits/rejected": -0.3177593946456909, + "logps/chosen": -268.61932373046875, + "logps/ref_chosen": -44.043216705322266, + "logps/ref_rejected": -91.85687255859375, + "logps/rejected": -486.3419189453125, + "loss": 1.0721, + "margin_dpo/margin_mean": 169.908935546875, + "margin_dpo/margin_std": 169.20985412597656, + "step": 248 + }, + { + "KL/chosen_KL_mean": -287.9716796875, + "KL/mean": -332.658203125, + "KL/rejected_KL_mean": -377.3447570800781, + "KL/std": 193.0209197998047, + "epoch": 0.3656387665198238, + "fcm_dpo/beta": 0.0025631722528487444, + "fcm_dpo/delta": 0.08395271003246307, + "fcm_dpo/margin": 89.3730697631836, + "fcm_dpo/q_t": 0.45033693313598633, + "grad_norm": 45.21783447265625, + "learning_rate": 4.0168078931267426e-07, + "logits/chosen": -0.35276949405670166, + "logits/rejected": -0.33473479747772217, + "logps/chosen": -350.4140625, + "logps/ref_chosen": -62.442352294921875, + "logps/ref_rejected": -80.46806335449219, + "logps/rejected": -457.81280517578125, + "loss": 1.2555, + "margin_dpo/margin_mean": 89.37306213378906, + "margin_dpo/margin_std": 233.79147338867188, + "step": 249 + }, + { + "KL/chosen_KL_mean": -257.8434143066406, + "KL/mean": -340.539794921875, + "KL/rejected_KL_mean": -423.2361755371094, + "KL/std": 185.04319763183594, + "epoch": 0.3671071953010279, + "fcm_dpo/beta": 0.0025648935697972775, + "fcm_dpo/delta": -0.032477494329214096, + "fcm_dpo/margin": 165.39276123046875, + "fcm_dpo/q_t": 0.40042954683303833, + "grad_norm": 49.331077575683594, + "learning_rate": 4.006586590948141e-07, + "logits/chosen": -0.35356414318084717, + "logits/rejected": -0.31186503171920776, + "logps/chosen": -323.4801025390625, + "logps/ref_chosen": -65.63668823242188, + "logps/ref_rejected": -73.87184143066406, + "logps/rejected": -497.1080322265625, + "loss": 1.0711, + "margin_dpo/margin_mean": 165.39276123046875, + "margin_dpo/margin_std": 199.77915954589844, + "step": 250 + }, + { + "KL/chosen_KL_mean": -263.6090393066406, + "KL/mean": -316.90277099609375, + "KL/rejected_KL_mean": -370.1965026855469, + "KL/std": 189.91702270507812, + "epoch": 0.368575624082232, + "fcm_dpo/beta": 0.002717065392062068, + "fcm_dpo/delta": 0.12958815693855286, + "fcm_dpo/margin": 106.58744812011719, + "fcm_dpo/q_t": 0.43745461106300354, + "grad_norm": 40.08333969116211, + "learning_rate": 3.9963255888117325e-07, + "logits/chosen": -0.32426249980926514, + "logits/rejected": -0.2984544336795807, + "logps/chosen": -320.791748046875, + "logps/ref_chosen": -57.182716369628906, + "logps/ref_rejected": -77.66343688964844, + "logps/rejected": -447.85992431640625, + "loss": 1.1986, + "margin_dpo/margin_mean": 106.58744812011719, + "margin_dpo/margin_std": 211.80274963378906, + "step": 251 + }, + { + "KL/chosen_KL_mean": -247.48324584960938, + "KL/mean": -321.72723388671875, + "KL/rejected_KL_mean": -395.97125244140625, + "KL/std": 152.3659210205078, + "epoch": 0.3700440528634361, + "fcm_dpo/beta": 0.0027652715798467398, + "fcm_dpo/delta": -0.014062562957406044, + "fcm_dpo/margin": 148.48800659179688, + "fcm_dpo/q_t": 0.4011853337287903, + "grad_norm": 32.023529052734375, + "learning_rate": 3.9860251571044666e-07, + "logits/chosen": -0.4131581485271454, + "logits/rejected": -0.37648361921310425, + "logps/chosen": -319.16888427734375, + "logps/ref_chosen": -71.68563842773438, + "logps/ref_rejected": -84.75799560546875, + "logps/rejected": -480.729248046875, + "loss": 1.06, + "margin_dpo/margin_mean": 148.48800659179688, + "margin_dpo/margin_std": 151.195068359375, + "step": 252 + }, + { + "KL/chosen_KL_mean": -203.74603271484375, + "KL/mean": -282.53802490234375, + "KL/rejected_KL_mean": -361.3300476074219, + "KL/std": 168.71078491210938, + "epoch": 0.37151248164464024, + "fcm_dpo/beta": 0.002772426698356867, + "fcm_dpo/delta": -0.049864716827869415, + "fcm_dpo/margin": 157.58399963378906, + "fcm_dpo/q_t": 0.3958263397216797, + "grad_norm": 24.330591201782227, + "learning_rate": 3.9756855672522986e-07, + "logits/chosen": -0.42228442430496216, + "logits/rejected": -0.4147264063358307, + "logps/chosen": -272.8799743652344, + "logps/ref_chosen": -69.1339340209961, + "logps/ref_rejected": -98.70252990722656, + "logps/rejected": -460.0325927734375, + "loss": 1.0635, + "margin_dpo/margin_mean": 157.58401489257812, + "margin_dpo/margin_std": 181.4701690673828, + "step": 253 + }, + { + "KL/chosen_KL_mean": -186.63661193847656, + "KL/mean": -252.297119140625, + "KL/rejected_KL_mean": -317.9576416015625, + "KL/std": 167.77099609375, + "epoch": 0.37298091042584436, + "fcm_dpo/beta": 0.0027375142090022564, + "fcm_dpo/delta": 0.049500077962875366, + "fcm_dpo/margin": 131.32101440429688, + "fcm_dpo/q_t": 0.4207710027694702, + "grad_norm": 25.899538040161133, + "learning_rate": 3.965307091713037e-07, + "logits/chosen": -0.38440778851509094, + "logits/rejected": -0.37307465076446533, + "logps/chosen": -240.79161071777344, + "logps/ref_chosen": -54.154998779296875, + "logps/ref_rejected": -90.30764770507812, + "logps/rejected": -408.2652587890625, + "loss": 1.1333, + "margin_dpo/margin_mean": 131.32101440429688, + "margin_dpo/margin_std": 208.37350463867188, + "step": 254 + }, + { + "KL/chosen_KL_mean": -187.1531982421875, + "KL/mean": -254.55125427246094, + "KL/rejected_KL_mean": -321.9493103027344, + "KL/std": 147.3501739501953, + "epoch": 0.3744493392070485, + "fcm_dpo/beta": 0.002789452439174056, + "fcm_dpo/delta": 0.02846667543053627, + "fcm_dpo/margin": 134.7960968017578, + "fcm_dpo/q_t": 0.41337624192237854, + "grad_norm": 26.342166900634766, + "learning_rate": 3.954890003969163e-07, + "logits/chosen": -0.3332477807998657, + "logits/rejected": -0.3225042521953583, + "logps/chosen": -244.29486083984375, + "logps/ref_chosen": -57.14167022705078, + "logps/ref_rejected": -90.2085952758789, + "logps/rejected": -412.15789794921875, + "loss": 1.1121, + "margin_dpo/margin_mean": 134.7960968017578, + "margin_dpo/margin_std": 186.9753875732422, + "step": 255 + }, + { + "KL/chosen_KL_mean": -162.30020141601562, + "KL/mean": -233.31613159179688, + "KL/rejected_KL_mean": -304.33203125, + "KL/std": 154.13583374023438, + "epoch": 0.37591776798825255, + "fcm_dpo/beta": 0.0028459273744374514, + "fcm_dpo/delta": -0.0053253816440701485, + "fcm_dpo/margin": 142.03182983398438, + "fcm_dpo/q_t": 0.40578973293304443, + "grad_norm": 36.771114349365234, + "learning_rate": 3.944434578520628e-07, + "logits/chosen": -0.30460691452026367, + "logits/rejected": -0.3128143548965454, + "logps/chosen": -217.4636993408203, + "logps/ref_chosen": -55.163490295410156, + "logps/ref_rejected": -92.56291961669922, + "logps/rejected": -396.89495849609375, + "loss": 1.0809, + "margin_dpo/margin_mean": 142.03182983398438, + "margin_dpo/margin_std": 175.64019775390625, + "step": 256 + }, + { + "KL/chosen_KL_mean": -163.07763671875, + "KL/mean": -238.74008178710938, + "KL/rejected_KL_mean": -314.40252685546875, + "KL/std": 163.1609649658203, + "epoch": 0.37738619676945667, + "fcm_dpo/beta": 0.002718387870118022, + "fcm_dpo/delta": -0.019889619201421738, + "fcm_dpo/margin": 151.32489013671875, + "fcm_dpo/q_t": 0.4035347104072571, + "grad_norm": 23.515918731689453, + "learning_rate": 3.933941090877615e-07, + "logits/chosen": -0.35459989309310913, + "logits/rejected": -0.3430092930793762, + "logps/chosen": -212.5013427734375, + "logps/ref_chosen": -49.42369842529297, + "logps/ref_rejected": -79.53791809082031, + "logps/rejected": -393.9404296875, + "loss": 1.0789, + "margin_dpo/margin_mean": 151.32489013671875, + "margin_dpo/margin_std": 181.58273315429688, + "step": 257 + }, + { + "KL/chosen_KL_mean": -215.13340759277344, + "KL/mean": -289.7934875488281, + "KL/rejected_KL_mean": -364.45355224609375, + "KL/std": 176.51971435546875, + "epoch": 0.3788546255506608, + "fcm_dpo/beta": 0.002779233269393444, + "fcm_dpo/delta": -0.019095513969659805, + "fcm_dpo/margin": 149.32015991210938, + "fcm_dpo/q_t": 0.40277743339538574, + "grad_norm": 23.90601348876953, + "learning_rate": 3.923409817553284e-07, + "logits/chosen": -0.3055630028247833, + "logits/rejected": -0.3058106601238251, + "logps/chosen": -274.51751708984375, + "logps/ref_chosen": -59.384124755859375, + "logps/ref_rejected": -95.99010467529297, + "logps/rejected": -460.44366455078125, + "loss": 1.0945, + "margin_dpo/margin_mean": 149.32015991210938, + "margin_dpo/margin_std": 211.73117065429688, + "step": 258 + }, + { + "KL/chosen_KL_mean": -202.0823974609375, + "KL/mean": -263.96234130859375, + "KL/rejected_KL_mean": -325.84234619140625, + "KL/std": 168.348876953125, + "epoch": 0.3803230543318649, + "fcm_dpo/beta": 0.002821682021021843, + "fcm_dpo/delta": 0.06138894706964493, + "fcm_dpo/margin": 123.75993347167969, + "fcm_dpo/q_t": 0.4222329556941986, + "grad_norm": 20.800395965576172, + "learning_rate": 3.9128410360564793e-07, + "logits/chosen": -0.35386550426483154, + "logits/rejected": -0.35250523686408997, + "logps/chosen": -254.91073608398438, + "logps/ref_chosen": -52.828346252441406, + "logps/ref_rejected": -89.191650390625, + "logps/rejected": -415.03399658203125, + "loss": 1.1393, + "margin_dpo/margin_mean": 123.75993347167969, + "margin_dpo/margin_std": 192.9962921142578, + "step": 259 + }, + { + "KL/chosen_KL_mean": -198.77923583984375, + "KL/mean": -283.6119079589844, + "KL/rejected_KL_mean": -368.444580078125, + "KL/std": 172.5734405517578, + "epoch": 0.38179148311306904, + "fcm_dpo/beta": 0.0027726658154278994, + "fcm_dpo/delta": -0.09174390137195587, + "fcm_dpo/margin": 169.66534423828125, + "fcm_dpo/q_t": 0.3891771137714386, + "grad_norm": 29.0617733001709, + "learning_rate": 3.9022350248844246e-07, + "logits/chosen": -0.33367860317230225, + "logits/rejected": -0.3492773771286011, + "logps/chosen": -246.1968994140625, + "logps/ref_chosen": -47.41767501831055, + "logps/ref_rejected": -95.08978271484375, + "logps/rejected": -463.53436279296875, + "loss": 1.0302, + "margin_dpo/margin_mean": 169.66534423828125, + "margin_dpo/margin_std": 185.7799530029297, + "step": 260 + }, + { + "KL/chosen_KL_mean": -203.46487426757812, + "KL/mean": -282.5809326171875, + "KL/rejected_KL_mean": -361.697021484375, + "KL/std": 188.90347290039062, + "epoch": 0.3832599118942731, + "fcm_dpo/beta": 0.0025986204855144024, + "fcm_dpo/delta": -0.014911869540810585, + "fcm_dpo/margin": 158.2321319580078, + "fcm_dpo/q_t": 0.4053801894187927, + "grad_norm": 20.706195831298828, + "learning_rate": 3.891592063515376e-07, + "logits/chosen": -0.30493831634521484, + "logits/rejected": -0.3092959225177765, + "logps/chosen": -256.4962463378906, + "logps/ref_chosen": -53.03137969970703, + "logps/ref_rejected": -88.51494598388672, + "logps/rejected": -450.21197509765625, + "loss": 1.0849, + "margin_dpo/margin_mean": 158.23214721679688, + "margin_dpo/margin_std": 208.61734008789062, + "step": 261 + }, + { + "KL/chosen_KL_mean": -237.848876953125, + "KL/mean": -303.23284912109375, + "KL/rejected_KL_mean": -368.6168212890625, + "KL/std": 171.96548461914062, + "epoch": 0.38472834067547723, + "fcm_dpo/beta": 0.0027585539501160383, + "fcm_dpo/delta": 0.045463643968105316, + "fcm_dpo/margin": 130.7679443359375, + "fcm_dpo/q_t": 0.4169616997241974, + "grad_norm": 26.5394344329834, + "learning_rate": 3.880912432401264e-07, + "logits/chosen": -0.2885762155056, + "logits/rejected": -0.26217561960220337, + "logps/chosen": -297.4690246582031, + "logps/ref_chosen": -59.620140075683594, + "logps/ref_rejected": -86.41853332519531, + "logps/rejected": -455.03533935546875, + "loss": 1.1097, + "margin_dpo/margin_mean": 130.7679443359375, + "margin_dpo/margin_std": 169.8227081298828, + "step": 262 + }, + { + "KL/chosen_KL_mean": -214.3430938720703, + "KL/mean": -308.7781982421875, + "KL/rejected_KL_mean": -403.21331787109375, + "KL/std": 202.47735595703125, + "epoch": 0.38619676945668135, + "fcm_dpo/beta": 0.002491295337677002, + "fcm_dpo/delta": -0.09768804162740707, + "fcm_dpo/margin": 188.87020874023438, + "fcm_dpo/q_t": 0.38718199729919434, + "grad_norm": 18.903581619262695, + "learning_rate": 3.870196412960302e-07, + "logits/chosen": -0.31606876850128174, + "logits/rejected": -0.2949512004852295, + "logps/chosen": -273.7640380859375, + "logps/ref_chosen": -59.42094421386719, + "logps/ref_rejected": -96.85720825195312, + "logps/rejected": -500.0705261230469, + "loss": 1.0366, + "margin_dpo/margin_mean": 188.87022399902344, + "margin_dpo/margin_std": 214.49734497070312, + "step": 263 + }, + { + "KL/chosen_KL_mean": -232.32882690429688, + "KL/mean": -313.197998046875, + "KL/rejected_KL_mean": -394.0671691894531, + "KL/std": 192.44293212890625, + "epoch": 0.3876651982378855, + "fcm_dpo/beta": 0.0024121187161654234, + "fcm_dpo/delta": 0.006785430014133453, + "fcm_dpo/margin": 161.7383270263672, + "fcm_dpo/q_t": 0.41011467576026917, + "grad_norm": 23.93086814880371, + "learning_rate": 3.8594442875695665e-07, + "logits/chosen": -0.35104095935821533, + "logits/rejected": -0.3437178134918213, + "logps/chosen": -295.0509033203125, + "logps/ref_chosen": -62.722084045410156, + "logps/ref_rejected": -93.85620880126953, + "logps/rejected": -487.92340087890625, + "loss": 1.1065, + "margin_dpo/margin_mean": 161.7383270263672, + "margin_dpo/margin_std": 220.1507110595703, + "step": 264 + }, + { + "KL/chosen_KL_mean": -254.60040283203125, + "KL/mean": -330.61474609375, + "KL/rejected_KL_mean": -406.629150390625, + "KL/std": 214.48471069335938, + "epoch": 0.3891336270190896, + "fcm_dpo/beta": 0.0025378770660609007, + "fcm_dpo/delta": 0.017527392134070396, + "fcm_dpo/margin": 152.0287322998047, + "fcm_dpo/q_t": 0.4144975543022156, + "grad_norm": 22.09156036376953, + "learning_rate": 3.848656339557562e-07, + "logits/chosen": -0.2829780578613281, + "logits/rejected": -0.2669457197189331, + "logps/chosen": -316.5718688964844, + "logps/ref_chosen": -61.971466064453125, + "logps/ref_rejected": -88.02059936523438, + "logps/rejected": -494.6497497558594, + "loss": 1.1316, + "margin_dpo/margin_mean": 152.0287322998047, + "margin_dpo/margin_std": 254.72515869140625, + "step": 265 + }, + { + "KL/chosen_KL_mean": -259.6724853515625, + "KL/mean": -320.30584716796875, + "KL/rejected_KL_mean": -380.9391784667969, + "KL/std": 179.23141479492188, + "epoch": 0.39060205580029367, + "fcm_dpo/beta": 0.0027112066745758057, + "fcm_dpo/delta": 0.08521057665348053, + "fcm_dpo/margin": 121.26667022705078, + "fcm_dpo/q_t": 0.4275960922241211, + "grad_norm": 47.17190933227539, + "learning_rate": 3.8378328531967507e-07, + "logits/chosen": -0.31882771849632263, + "logits/rejected": -0.2803471088409424, + "logps/chosen": -326.77215576171875, + "logps/ref_chosen": -67.09967041015625, + "logps/ref_rejected": -67.97122192382812, + "logps/rejected": -448.910400390625, + "loss": 1.1572, + "margin_dpo/margin_mean": 121.26667785644531, + "margin_dpo/margin_std": 205.98829650878906, + "step": 266 + }, + { + "KL/chosen_KL_mean": -229.09750366210938, + "KL/mean": -301.92169189453125, + "KL/rejected_KL_mean": -374.7459411621094, + "KL/std": 188.1318359375, + "epoch": 0.3920704845814978, + "fcm_dpo/beta": 0.002730957930907607, + "fcm_dpo/delta": 0.0015622973442077637, + "fcm_dpo/margin": 145.6484375, + "fcm_dpo/q_t": 0.4101436734199524, + "grad_norm": 41.06880187988281, + "learning_rate": 3.8269741136960646e-07, + "logits/chosen": -0.3704749643802643, + "logits/rejected": -0.3451113700866699, + "logps/chosen": -298.0682373046875, + "logps/ref_chosen": -68.97075653076172, + "logps/ref_rejected": -90.16844940185547, + "logps/rejected": -464.91436767578125, + "loss": 1.1069, + "margin_dpo/margin_mean": 145.64842224121094, + "margin_dpo/margin_std": 215.05789184570312, + "step": 267 + }, + { + "KL/chosen_KL_mean": -242.15602111816406, + "KL/mean": -318.61297607421875, + "KL/rejected_KL_mean": -395.07000732421875, + "KL/std": 188.01754760742188, + "epoch": 0.3935389133627019, + "fcm_dpo/beta": 0.0026982720009982586, + "fcm_dpo/delta": -0.017409201711416245, + "fcm_dpo/margin": 152.9139404296875, + "fcm_dpo/q_t": 0.40618669986724854, + "grad_norm": 27.902847290039062, + "learning_rate": 3.8160804071933894e-07, + "logits/chosen": -0.31142598390579224, + "logits/rejected": -0.31501471996307373, + "logps/chosen": -298.05633544921875, + "logps/ref_chosen": -55.90031051635742, + "logps/ref_rejected": -101.64763641357422, + "logps/rejected": -496.7176208496094, + "loss": 1.1023, + "margin_dpo/margin_mean": 152.9139404296875, + "margin_dpo/margin_std": 228.34695434570312, + "step": 268 + }, + { + "KL/chosen_KL_mean": -274.3188171386719, + "KL/mean": -364.34295654296875, + "KL/rejected_KL_mean": -454.36712646484375, + "KL/std": 197.2436981201172, + "epoch": 0.39500734214390604, + "fcm_dpo/beta": 0.0025738244876265526, + "fcm_dpo/delta": -0.0825297087430954, + "fcm_dpo/margin": 180.04827880859375, + "fcm_dpo/q_t": 0.39260607957839966, + "grad_norm": 29.512990951538086, + "learning_rate": 3.8051520207480204e-07, + "logits/chosen": -0.3739697337150574, + "logits/rejected": -0.3583334684371948, + "logps/chosen": -344.3583984375, + "logps/ref_chosen": -70.03955841064453, + "logps/ref_rejected": -107.34937286376953, + "logps/rejected": -561.7164916992188, + "loss": 1.0732, + "margin_dpo/margin_mean": 180.04827880859375, + "margin_dpo/margin_std": 252.80072021484375, + "step": 269 + }, + { + "KL/chosen_KL_mean": -230.86578369140625, + "KL/mean": -299.2672119140625, + "KL/rejected_KL_mean": -367.66864013671875, + "KL/std": 178.17739868164062, + "epoch": 0.3964757709251101, + "fcm_dpo/beta": 0.0026433742605149746, + "fcm_dpo/delta": 0.0446162149310112, + "fcm_dpo/margin": 136.8028564453125, + "fcm_dpo/q_t": 0.41896626353263855, + "grad_norm": 47.853179931640625, + "learning_rate": 3.794189242333106e-07, + "logits/chosen": -0.4406346380710602, + "logits/rejected": -0.4400424361228943, + "logps/chosen": -300.3992614746094, + "logps/ref_chosen": -69.53347778320312, + "logps/ref_rejected": -109.92864990234375, + "logps/rejected": -477.5972900390625, + "loss": 1.1416, + "margin_dpo/margin_mean": 136.8028564453125, + "margin_dpo/margin_std": 227.7311248779297, + "step": 270 + }, + { + "KL/chosen_KL_mean": -227.30824279785156, + "KL/mean": -316.3374938964844, + "KL/rejected_KL_mean": -405.3667297363281, + "KL/std": 184.22903442382812, + "epoch": 0.39794419970631423, + "fcm_dpo/beta": 0.00247220229357481, + "fcm_dpo/delta": -0.054896239191293716, + "fcm_dpo/margin": 178.0584716796875, + "fcm_dpo/q_t": 0.3969544470310211, + "grad_norm": 26.639240264892578, + "learning_rate": 3.7831923608280514e-07, + "logits/chosen": -0.3195192515850067, + "logits/rejected": -0.3056868612766266, + "logps/chosen": -284.07281494140625, + "logps/ref_chosen": -56.76456832885742, + "logps/ref_rejected": -92.51383972167969, + "logps/rejected": -497.88055419921875, + "loss": 1.0576, + "margin_dpo/margin_mean": 178.0584716796875, + "margin_dpo/margin_std": 211.52894592285156, + "step": 271 + }, + { + "KL/chosen_KL_mean": -217.37164306640625, + "KL/mean": -324.1234130859375, + "KL/rejected_KL_mean": -430.87518310546875, + "KL/std": 205.4072723388672, + "epoch": 0.39941262848751835, + "fcm_dpo/beta": 0.002309414092451334, + "fcm_dpo/delta": -0.12269287556409836, + "fcm_dpo/margin": 213.5035400390625, + "fcm_dpo/q_t": 0.3797604441642761, + "grad_norm": 40.22795486450195, + "learning_rate": 3.772161666010912e-07, + "logits/chosen": -0.2815614342689514, + "logits/rejected": -0.29494649171829224, + "logps/chosen": -266.8688049316406, + "logps/ref_chosen": -49.497154235839844, + "logps/ref_rejected": -105.54279327392578, + "logps/rejected": -536.41796875, + "loss": 1.0089, + "margin_dpo/margin_mean": 213.5035400390625, + "margin_dpo/margin_std": 207.53280639648438, + "step": 272 + }, + { + "KL/chosen_KL_mean": -249.883056640625, + "KL/mean": -348.0146789550781, + "KL/rejected_KL_mean": -446.14630126953125, + "KL/std": 197.6843719482422, + "epoch": 0.4008810572687225, + "fcm_dpo/beta": 0.002137008123099804, + "fcm_dpo/delta": -0.02540270984172821, + "fcm_dpo/margin": 196.2632293701172, + "fcm_dpo/q_t": 0.4012957215309143, + "grad_norm": 24.651897430419922, + "learning_rate": 3.761097448550755e-07, + "logits/chosen": -0.30211585760116577, + "logits/rejected": -0.28442516922950745, + "logps/chosen": -312.85845947265625, + "logps/ref_chosen": -62.97539520263672, + "logps/ref_rejected": -92.49858093261719, + "logps/rejected": -538.6448974609375, + "loss": 1.0672, + "margin_dpo/margin_mean": 196.2632293701172, + "margin_dpo/margin_std": 230.588134765625, + "step": 273 + }, + { + "KL/chosen_KL_mean": -293.39508056640625, + "KL/mean": -369.06951904296875, + "KL/rejected_KL_mean": -444.743896484375, + "KL/std": 191.70803833007812, + "epoch": 0.4023494860499266, + "fcm_dpo/beta": 0.002289203926920891, + "fcm_dpo/delta": 0.061743948608636856, + "fcm_dpo/margin": 151.3488311767578, + "fcm_dpo/q_t": 0.42134177684783936, + "grad_norm": 22.297304153442383, + "learning_rate": 3.75e-07, + "logits/chosen": -0.2628266215324402, + "logits/rejected": -0.24505800008773804, + "logps/chosen": -349.06280517578125, + "logps/ref_chosen": -55.66770935058594, + "logps/ref_rejected": -77.33308410644531, + "logps/rejected": -522.0770263671875, + "loss": 1.1298, + "margin_dpo/margin_mean": 151.3488311767578, + "margin_dpo/margin_std": 218.46170043945312, + "step": 274 + }, + { + "KL/chosen_KL_mean": -226.7274169921875, + "KL/mean": -308.6673889160156, + "KL/rejected_KL_mean": -390.60736083984375, + "KL/std": 187.83544921875, + "epoch": 0.40381791483113066, + "fcm_dpo/beta": 0.0023457312490791082, + "fcm_dpo/delta": 0.01810782589018345, + "fcm_dpo/margin": 163.87994384765625, + "fcm_dpo/q_t": 0.4110303521156311, + "grad_norm": 29.85940933227539, + "learning_rate": 3.738869612786737e-07, + "logits/chosen": -0.31675028800964355, + "logits/rejected": -0.32569241523742676, + "logps/chosen": -275.3221130371094, + "logps/ref_chosen": -48.594703674316406, + "logps/ref_rejected": -93.30369567871094, + "logps/rejected": -483.91107177734375, + "loss": 1.096, + "margin_dpo/margin_mean": 163.87994384765625, + "margin_dpo/margin_std": 209.12115478515625, + "step": 275 + }, + { + "KL/chosen_KL_mean": -254.89431762695312, + "KL/mean": -338.76141357421875, + "KL/rejected_KL_mean": -422.6285400390625, + "KL/std": 204.18536376953125, + "epoch": 0.4052863436123348, + "fcm_dpo/beta": 0.0023221762385219336, + "fcm_dpo/delta": 0.012708023190498352, + "fcm_dpo/margin": 167.73422241210938, + "fcm_dpo/q_t": 0.4127948582172394, + "grad_norm": 25.3048038482666, + "learning_rate": 3.7277065802070204e-07, + "logits/chosen": -0.2963874936103821, + "logits/rejected": -0.2773512601852417, + "logps/chosen": -311.47174072265625, + "logps/ref_chosen": -56.57740783691406, + "logps/ref_rejected": -70.36566925048828, + "logps/rejected": -492.99420166015625, + "loss": 1.1126, + "margin_dpo/margin_mean": 167.73422241210938, + "margin_dpo/margin_std": 250.09458923339844, + "step": 276 + }, + { + "KL/chosen_KL_mean": -274.00732421875, + "KL/mean": -361.88043212890625, + "KL/rejected_KL_mean": -449.7535400390625, + "KL/std": 200.5372772216797, + "epoch": 0.4067547723935389, + "fcm_dpo/beta": 0.0023581404238939285, + "fcm_dpo/delta": -0.018989086151123047, + "fcm_dpo/margin": 175.74618530273438, + "fcm_dpo/q_t": 0.4043876826763153, + "grad_norm": 22.430313110351562, + "learning_rate": 3.71651119641714e-07, + "logits/chosen": -0.31193846464157104, + "logits/rejected": -0.2933298349380493, + "logps/chosen": -330.27886962890625, + "logps/ref_chosen": -56.27156066894531, + "logps/ref_rejected": -92.88127136230469, + "logps/rejected": -542.634765625, + "loss": 1.091, + "margin_dpo/margin_mean": 175.74620056152344, + "margin_dpo/margin_std": 240.4161376953125, + "step": 277 + }, + { + "KL/chosen_KL_mean": -238.85679626464844, + "KL/mean": -339.3966064453125, + "KL/rejected_KL_mean": -439.93634033203125, + "KL/std": 207.98587036132812, + "epoch": 0.40822320117474303, + "fcm_dpo/beta": 0.00218441104516387, + "fcm_dpo/delta": -0.052646711468696594, + "fcm_dpo/margin": 201.0795440673828, + "fcm_dpo/q_t": 0.3967970907688141, + "grad_norm": 22.02371597290039, + "learning_rate": 3.705283756425872e-07, + "logits/chosen": -0.30684125423431396, + "logits/rejected": -0.31390851736068726, + "logps/chosen": -291.7987365722656, + "logps/ref_chosen": -52.94194030761719, + "logps/ref_rejected": -91.25357818603516, + "logps/rejected": -531.18994140625, + "loss": 1.0559, + "margin_dpo/margin_mean": 201.0795440673828, + "margin_dpo/margin_std": 235.0767822265625, + "step": 278 + }, + { + "KL/chosen_KL_mean": -266.6314697265625, + "KL/mean": -369.5718994140625, + "KL/rejected_KL_mean": -472.5123291015625, + "KL/std": 229.59307861328125, + "epoch": 0.40969162995594716, + "fcm_dpo/beta": 0.0020728304516524076, + "fcm_dpo/delta": -0.038570303469896317, + "fcm_dpo/margin": 205.880859375, + "fcm_dpo/q_t": 0.4020066261291504, + "grad_norm": 23.30048179626465, + "learning_rate": 3.6940245560867e-07, + "logits/chosen": -0.25095367431640625, + "logits/rejected": -0.2509291172027588, + "logps/chosen": -315.2727966308594, + "logps/ref_chosen": -48.641319274902344, + "logps/ref_rejected": -87.8514404296875, + "logps/rejected": -560.36376953125, + "loss": 1.0833, + "margin_dpo/margin_mean": 205.880859375, + "margin_dpo/margin_std": 281.98443603515625, + "step": 279 + }, + { + "KL/chosen_KL_mean": -237.64895629882812, + "KL/mean": -337.5757751464844, + "KL/rejected_KL_mean": -437.50262451171875, + "KL/std": 190.99032592773438, + "epoch": 0.4111600587371512, + "fcm_dpo/beta": 0.0020760963670909405, + "fcm_dpo/delta": -0.018862612545490265, + "fcm_dpo/margin": 199.85365295410156, + "fcm_dpo/q_t": 0.40116894245147705, + "grad_norm": 22.5253963470459, + "learning_rate": 3.6827338920900253e-07, + "logits/chosen": -0.3350566625595093, + "logits/rejected": -0.33742278814315796, + "logps/chosen": -296.4460754394531, + "logps/ref_chosen": -58.797122955322266, + "logps/ref_rejected": -98.61885070800781, + "logps/rejected": -536.1214599609375, + "loss": 1.0562, + "margin_dpo/margin_mean": 199.8536376953125, + "margin_dpo/margin_std": 203.86276245117188, + "step": 280 + }, + { + "KL/chosen_KL_mean": -224.87203979492188, + "KL/mean": -309.09222412109375, + "KL/rejected_KL_mean": -393.31243896484375, + "KL/std": 193.19955444335938, + "epoch": 0.41262848751835535, + "fcm_dpo/beta": 0.0021694411989301443, + "fcm_dpo/delta": 0.04002968966960907, + "fcm_dpo/margin": 168.44039916992188, + "fcm_dpo/q_t": 0.4139762818813324, + "grad_norm": 26.64156150817871, + "learning_rate": 3.6714120619553435e-07, + "logits/chosen": -0.3864981532096863, + "logits/rejected": -0.3639860153198242, + "logps/chosen": -280.3605651855469, + "logps/ref_chosen": -55.488521575927734, + "logps/ref_rejected": -80.88258361816406, + "logps/rejected": -474.19500732421875, + "loss": 1.1042, + "margin_dpo/margin_mean": 168.4403839111328, + "margin_dpo/margin_std": 207.57632446289062, + "step": 281 + }, + { + "KL/chosen_KL_mean": -260.92059326171875, + "KL/mean": -328.2933349609375, + "KL/rejected_KL_mean": -395.66607666015625, + "KL/std": 199.9541015625, + "epoch": 0.41409691629955947, + "fcm_dpo/beta": 0.002180553739890456, + "fcm_dpo/delta": 0.025225864723324776, + "fcm_dpo/margin": 134.74557495117188, + "fcm_dpo/q_t": 0.43476957082748413, + "grad_norm": 23.243419647216797, + "learning_rate": 3.660059364023408e-07, + "logits/chosen": -0.39929962158203125, + "logits/rejected": -0.3811987638473511, + "logps/chosen": -333.99072265625, + "logps/ref_chosen": -73.07014465332031, + "logps/ref_rejected": -95.35098266601562, + "logps/rejected": -491.01708984375, + "loss": 1.1715, + "margin_dpo/margin_mean": 134.7455596923828, + "margin_dpo/margin_std": 234.26336669921875, + "step": 282 + }, + { + "KL/chosen_KL_mean": -273.17254638671875, + "KL/mean": -379.52557373046875, + "KL/rejected_KL_mean": -485.87860107421875, + "KL/std": 243.8074951171875, + "epoch": 0.4155653450807636, + "fcm_dpo/beta": 0.0021401268895715475, + "fcm_dpo/delta": -0.07182439416646957, + "fcm_dpo/margin": 212.7060546875, + "fcm_dpo/q_t": 0.3907451927661896, + "grad_norm": 35.38416290283203, + "learning_rate": 3.6486760974483685e-07, + "logits/chosen": -0.3343729078769684, + "logits/rejected": -0.3363478183746338, + "logps/chosen": -335.07098388671875, + "logps/ref_chosen": -61.89844512939453, + "logps/ref_rejected": -96.98655700683594, + "logps/rejected": -582.8651123046875, + "loss": 1.0446, + "margin_dpo/margin_mean": 212.7060546875, + "margin_dpo/margin_std": 232.63815307617188, + "step": 283 + }, + { + "KL/chosen_KL_mean": -261.57415771484375, + "KL/mean": -360.19207763671875, + "KL/rejected_KL_mean": -458.80999755859375, + "KL/std": 222.4651336669922, + "epoch": 0.4170337738619677, + "fcm_dpo/beta": 0.0020390492863953114, + "fcm_dpo/delta": -0.0028852270916104317, + "fcm_dpo/margin": 197.23583984375, + "fcm_dpo/q_t": 0.4071544408798218, + "grad_norm": 24.07853126525879, + "learning_rate": 3.6372625621898863e-07, + "logits/chosen": -0.37924909591674805, + "logits/rejected": -0.3620849549770355, + "logps/chosen": -320.00970458984375, + "logps/ref_chosen": -58.4355354309082, + "logps/ref_rejected": -93.46926879882812, + "logps/rejected": -552.279296875, + "loss": 1.0777, + "margin_dpo/margin_mean": 197.23580932617188, + "margin_dpo/margin_std": 236.99972534179688, + "step": 284 + }, + { + "KL/chosen_KL_mean": -298.85516357421875, + "KL/mean": -381.50390625, + "KL/rejected_KL_mean": -464.1526184082031, + "KL/std": 190.44850158691406, + "epoch": 0.4185022026431718, + "fcm_dpo/beta": 0.0021772286854684353, + "fcm_dpo/delta": 0.044019557535648346, + "fcm_dpo/margin": 165.29745483398438, + "fcm_dpo/q_t": 0.4151372015476227, + "grad_norm": 23.67416763305664, + "learning_rate": 3.625819059005228e-07, + "logits/chosen": -0.3634873628616333, + "logits/rejected": -0.348185658454895, + "logps/chosen": -365.08734130859375, + "logps/ref_chosen": -66.23219299316406, + "logps/ref_rejected": -99.1268310546875, + "logps/rejected": -563.2794189453125, + "loss": 1.1137, + "margin_dpo/margin_mean": 165.29745483398438, + "margin_dpo/margin_std": 213.57778930664062, + "step": 285 + }, + { + "KL/chosen_KL_mean": -292.51617431640625, + "KL/mean": -385.5911560058594, + "KL/rejected_KL_mean": -478.6661682128906, + "KL/std": 223.45005798339844, + "epoch": 0.4199706314243759, + "fcm_dpo/beta": 0.002157814335078001, + "fcm_dpo/delta": -0.0024737920612096786, + "fcm_dpo/margin": 186.14996337890625, + "fcm_dpo/q_t": 0.40657156705856323, + "grad_norm": 23.773475646972656, + "learning_rate": 3.614345889441346e-07, + "logits/chosen": -0.3397786319255829, + "logits/rejected": -0.32064247131347656, + "logps/chosen": -365.4671936035156, + "logps/ref_chosen": -72.95100402832031, + "logps/ref_rejected": -88.58845520019531, + "logps/rejected": -567.254638671875, + "loss": 1.0899, + "margin_dpo/margin_mean": 186.14996337890625, + "margin_dpo/margin_std": 243.01104736328125, + "step": 286 + }, + { + "KL/chosen_KL_mean": -262.6454162597656, + "KL/mean": -341.39129638671875, + "KL/rejected_KL_mean": -420.13714599609375, + "KL/std": 194.3848114013672, + "epoch": 0.42143906020558003, + "fcm_dpo/beta": 0.0022398338187485933, + "fcm_dpo/delta": 0.05684395506978035, + "fcm_dpo/margin": 157.49172973632812, + "fcm_dpo/q_t": 0.4202299118041992, + "grad_norm": 20.517465591430664, + "learning_rate": 3.6028433558269275e-07, + "logits/chosen": -0.30254948139190674, + "logits/rejected": -0.27677813172340393, + "logps/chosen": -324.18658447265625, + "logps/ref_chosen": -61.54115295410156, + "logps/ref_rejected": -77.69607543945312, + "logps/rejected": -497.833251953125, + "loss": 1.1189, + "margin_dpo/margin_mean": 157.49172973632812, + "margin_dpo/margin_std": 216.00836181640625, + "step": 287 + }, + { + "KL/chosen_KL_mean": -242.85244750976562, + "KL/mean": -340.43890380859375, + "KL/rejected_KL_mean": -438.025390625, + "KL/std": 184.95721435546875, + "epoch": 0.42290748898678415, + "fcm_dpo/beta": 0.002138757612556219, + "fcm_dpo/delta": -0.028658173978328705, + "fcm_dpo/margin": 195.1729278564453, + "fcm_dpo/q_t": 0.3991842269897461, + "grad_norm": 20.475711822509766, + "learning_rate": 3.5913117612644327e-07, + "logits/chosen": -0.35998380184173584, + "logits/rejected": -0.34525611996650696, + "logps/chosen": -299.513671875, + "logps/ref_chosen": -56.661224365234375, + "logps/ref_rejected": -87.33570098876953, + "logps/rejected": -525.361083984375, + "loss": 1.0586, + "margin_dpo/margin_mean": 195.17294311523438, + "margin_dpo/margin_std": 196.0745391845703, + "step": 288 + }, + { + "KL/chosen_KL_mean": -236.07003784179688, + "KL/mean": -339.13861083984375, + "KL/rejected_KL_mean": -442.2071533203125, + "KL/std": 204.4755859375, + "epoch": 0.4243759177679883, + "fcm_dpo/beta": 0.002140854485332966, + "fcm_dpo/delta": -0.05301383137702942, + "fcm_dpo/margin": 206.13711547851562, + "fcm_dpo/q_t": 0.3952919840812683, + "grad_norm": 35.90175247192383, + "learning_rate": 3.5797514096221024e-07, + "logits/chosen": -0.28917020559310913, + "logits/rejected": -0.29095658659935, + "logps/chosen": -281.3004455566406, + "logps/ref_chosen": -45.23039245605469, + "logps/ref_rejected": -87.64266967773438, + "logps/rejected": -529.849853515625, + "loss": 1.0471, + "margin_dpo/margin_mean": 206.13711547851562, + "margin_dpo/margin_std": 221.66661071777344, + "step": 289 + }, + { + "KL/chosen_KL_mean": -259.9752502441406, + "KL/mean": -367.92950439453125, + "KL/rejected_KL_mean": -475.8837890625, + "KL/std": 215.7952880859375, + "epoch": 0.42584434654919234, + "fcm_dpo/beta": 0.0020013581961393356, + "fcm_dpo/delta": -0.04185499623417854, + "fcm_dpo/margin": 215.90850830078125, + "fcm_dpo/q_t": 0.39859092235565186, + "grad_norm": 23.12173843383789, + "learning_rate": 3.568162605525952e-07, + "logits/chosen": -0.29490506649017334, + "logits/rejected": -0.3174816370010376, + "logps/chosen": -315.4467468261719, + "logps/ref_chosen": -55.47149658203125, + "logps/ref_rejected": -116.70857238769531, + "logps/rejected": -592.5923461914062, + "loss": 1.0603, + "margin_dpo/margin_mean": 215.9084930419922, + "margin_dpo/margin_std": 255.39891052246094, + "step": 290 + }, + { + "KL/chosen_KL_mean": -236.2718505859375, + "KL/mean": -337.82806396484375, + "KL/rejected_KL_mean": -439.3843078613281, + "KL/std": 192.71876525878906, + "epoch": 0.42731277533039647, + "fcm_dpo/beta": 0.0020202212035655975, + "fcm_dpo/delta": -0.014722079038619995, + "fcm_dpo/margin": 203.11245727539062, + "fcm_dpo/q_t": 0.4027057886123657, + "grad_norm": 20.603652954101562, + "learning_rate": 3.5565456543517485e-07, + "logits/chosen": -0.3010188043117523, + "logits/rejected": -0.29001563787460327, + "logps/chosen": -299.5322265625, + "logps/ref_chosen": -63.26036834716797, + "logps/ref_rejected": -89.29708862304688, + "logps/rejected": -528.681396484375, + "loss": 1.0707, + "margin_dpo/margin_mean": 203.11245727539062, + "margin_dpo/margin_std": 228.27931213378906, + "step": 291 + }, + { + "KL/chosen_KL_mean": -271.1993408203125, + "KL/mean": -373.78936767578125, + "KL/rejected_KL_mean": -476.37933349609375, + "KL/std": 224.70799255371094, + "epoch": 0.4287812041116006, + "fcm_dpo/beta": 0.0019872216507792473, + "fcm_dpo/delta": -0.010810526087880135, + "fcm_dpo/margin": 205.17999267578125, + "fcm_dpo/q_t": 0.40419653058052063, + "grad_norm": 20.72088050842285, + "learning_rate": 3.5449008622169583e-07, + "logits/chosen": -0.3157344460487366, + "logits/rejected": -0.3041796088218689, + "logps/chosen": -325.1178894042969, + "logps/ref_chosen": -53.91852951049805, + "logps/ref_rejected": -89.96138000488281, + "logps/rejected": -566.3406982421875, + "loss": 1.0851, + "margin_dpo/margin_mean": 205.17999267578125, + "margin_dpo/margin_std": 263.08953857421875, + "step": 292 + }, + { + "KL/chosen_KL_mean": -287.68072509765625, + "KL/mean": -370.7479248046875, + "KL/rejected_KL_mean": -453.81512451171875, + "KL/std": 234.54473876953125, + "epoch": 0.4302496328928047, + "fcm_dpo/beta": 0.0020656294655054808, + "fcm_dpo/delta": 0.06754275411367416, + "fcm_dpo/margin": 166.13436889648438, + "fcm_dpo/q_t": 0.42421412467956543, + "grad_norm": 27.3696346282959, + "learning_rate": 3.5332285359726846e-07, + "logits/chosen": -0.332236647605896, + "logits/rejected": -0.3292074203491211, + "logps/chosen": -348.0567626953125, + "logps/ref_chosen": -60.376033782958984, + "logps/ref_rejected": -77.85244750976562, + "logps/rejected": -531.6675415039062, + "loss": 1.144, + "margin_dpo/margin_mean": 166.13436889648438, + "margin_dpo/margin_std": 270.0863037109375, + "step": 293 + }, + { + "KL/chosen_KL_mean": -266.4648742675781, + "KL/mean": -360.0727844238281, + "KL/rejected_KL_mean": -453.68072509765625, + "KL/std": 205.4690704345703, + "epoch": 0.43171806167400884, + "fcm_dpo/beta": 0.002078534569591284, + "fcm_dpo/delta": 0.012987792491912842, + "fcm_dpo/margin": 187.21585083007812, + "fcm_dpo/q_t": 0.41207996010780334, + "grad_norm": 24.852489471435547, + "learning_rate": 3.5215289831955786e-07, + "logits/chosen": -0.3020135164260864, + "logits/rejected": -0.30931025743484497, + "logps/chosen": -314.5523986816406, + "logps/ref_chosen": -48.0875358581543, + "logps/ref_rejected": -81.89698791503906, + "logps/rejected": -535.5776977539062, + "loss": 1.102, + "margin_dpo/margin_mean": 187.21585083007812, + "margin_dpo/margin_std": 258.83837890625, + "step": 294 + }, + { + "KL/chosen_KL_mean": -315.635009765625, + "KL/mean": -417.57843017578125, + "KL/rejected_KL_mean": -519.52197265625, + "KL/std": 232.22276306152344, + "epoch": 0.4331864904552129, + "fcm_dpo/beta": 0.0020754141733050346, + "fcm_dpo/delta": -0.029316715896129608, + "fcm_dpo/margin": 203.88693237304688, + "fcm_dpo/q_t": 0.40314337611198425, + "grad_norm": 19.910776138305664, + "learning_rate": 3.509802512179737e-07, + "logits/chosen": -0.31562861800193787, + "logits/rejected": -0.3209300637245178, + "logps/chosen": -365.5596618652344, + "logps/ref_chosen": -49.92467498779297, + "logps/ref_rejected": -87.45632934570312, + "logps/rejected": -606.978271484375, + "loss": 1.0858, + "margin_dpo/margin_mean": 203.88693237304688, + "margin_dpo/margin_std": 279.4998779296875, + "step": 295 + }, + { + "KL/chosen_KL_mean": -389.09619140625, + "KL/mean": -459.28369140625, + "KL/rejected_KL_mean": -529.4712524414062, + "KL/std": 224.94338989257812, + "epoch": 0.434654919236417, + "fcm_dpo/beta": 0.00205283984541893, + "fcm_dpo/delta": 0.0012772375484928489, + "fcm_dpo/margin": 140.37503051757812, + "fcm_dpo/q_t": 0.4331241846084595, + "grad_norm": 30.800323486328125, + "learning_rate": 3.498049431928577e-07, + "logits/chosen": -0.3613106906414032, + "logits/rejected": -0.3486577272415161, + "logps/chosen": -454.5874328613281, + "logps/ref_chosen": -65.49124145507812, + "logps/ref_rejected": -93.08908081054688, + "logps/rejected": -622.560302734375, + "loss": 1.2004, + "margin_dpo/margin_mean": 140.37501525878906, + "margin_dpo/margin_std": 281.86309814453125, + "step": 296 + }, + { + "KL/chosen_KL_mean": -337.59783935546875, + "KL/mean": -417.08062744140625, + "KL/rejected_KL_mean": -496.5633850097656, + "KL/std": 210.40191650390625, + "epoch": 0.43612334801762115, + "fcm_dpo/beta": 0.002149302512407303, + "fcm_dpo/delta": 0.07061058282852173, + "fcm_dpo/margin": 158.96554565429688, + "fcm_dpo/q_t": 0.4227258563041687, + "grad_norm": 33.55876541137695, + "learning_rate": 3.486270052146694e-07, + "logits/chosen": -0.368080735206604, + "logits/rejected": -0.3742336928844452, + "logps/chosen": -394.0747985839844, + "logps/ref_chosen": -56.476951599121094, + "logps/ref_rejected": -95.1385498046875, + "logps/rejected": -591.701904296875, + "loss": 1.1254, + "margin_dpo/margin_mean": 158.96554565429688, + "margin_dpo/margin_std": 216.52734375, + "step": 297 + }, + { + "KL/chosen_KL_mean": -349.87359619140625, + "KL/mean": -458.6307373046875, + "KL/rejected_KL_mean": -567.387939453125, + "KL/std": 274.2647399902344, + "epoch": 0.43759177679882527, + "fcm_dpo/beta": 0.002113178838044405, + "fcm_dpo/delta": -0.0772165060043335, + "fcm_dpo/margin": 217.51431274414062, + "fcm_dpo/q_t": 0.3965296745300293, + "grad_norm": 24.389272689819336, + "learning_rate": 3.474464683231698e-07, + "logits/chosen": -0.401616632938385, + "logits/rejected": -0.4193766713142395, + "logps/chosen": -417.19873046875, + "logps/ref_chosen": -67.32516479492188, + "logps/ref_rejected": -116.66217041015625, + "logps/rejected": -684.050048828125, + "loss": 1.0813, + "margin_dpo/margin_mean": 217.51431274414062, + "margin_dpo/margin_std": 321.95660400390625, + "step": 298 + }, + { + "KL/chosen_KL_mean": -289.93463134765625, + "KL/mean": -381.6058044433594, + "KL/rejected_KL_mean": -473.2769470214844, + "KL/std": 202.38690185546875, + "epoch": 0.4390602055800294, + "fcm_dpo/beta": 0.002111277775838971, + "fcm_dpo/delta": 0.012855540961027145, + "fcm_dpo/margin": 183.3423614501953, + "fcm_dpo/q_t": 0.4118254780769348, + "grad_norm": 35.68212890625, + "learning_rate": 3.462633636266041e-07, + "logits/chosen": -0.35824018716812134, + "logits/rejected": -0.36311280727386475, + "logps/chosen": -338.896728515625, + "logps/ref_chosen": -48.96209716796875, + "logps/ref_rejected": -84.32823944091797, + "logps/rejected": -557.605224609375, + "loss": 1.104, + "margin_dpo/margin_mean": 183.34234619140625, + "margin_dpo/margin_std": 251.83197021484375, + "step": 299 + }, + { + "KL/chosen_KL_mean": -350.1387939453125, + "KL/mean": -458.59375, + "KL/rejected_KL_mean": -567.0487060546875, + "KL/std": 237.21231079101562, + "epoch": 0.44052863436123346, + "fcm_dpo/beta": 0.0020199865102767944, + "fcm_dpo/delta": -0.048955146223306656, + "fcm_dpo/margin": 216.90985107421875, + "fcm_dpo/q_t": 0.39751458168029785, + "grad_norm": 26.843948364257812, + "learning_rate": 3.4507772230088147e-07, + "logits/chosen": -0.3743543028831482, + "logits/rejected": -0.3817921280860901, + "logps/chosen": -409.2125244140625, + "logps/ref_chosen": -59.07371139526367, + "logps/ref_rejected": -95.9664535522461, + "logps/rejected": -663.01513671875, + "loss": 1.0789, + "margin_dpo/margin_mean": 216.90985107421875, + "margin_dpo/margin_std": 290.77459716796875, + "step": 300 + }, + { + "KL/chosen_KL_mean": -294.7914733886719, + "KL/mean": -403.30487060546875, + "KL/rejected_KL_mean": -511.8182373046875, + "KL/std": 218.48614501953125, + "epoch": 0.4419970631424376, + "fcm_dpo/beta": 0.0019110521534457803, + "fcm_dpo/delta": -0.019859077408909798, + "fcm_dpo/margin": 217.02674865722656, + "fcm_dpo/q_t": 0.40373334288597107, + "grad_norm": 25.57866096496582, + "learning_rate": 3.4388957558875316e-07, + "logits/chosen": -0.3827190399169922, + "logits/rejected": -0.38337594270706177, + "logps/chosen": -352.04083251953125, + "logps/ref_chosen": -57.249366760253906, + "logps/ref_rejected": -92.35354614257812, + "logps/rejected": -604.1717529296875, + "loss": 1.0724, + "margin_dpo/margin_mean": 217.02676391601562, + "margin_dpo/margin_std": 262.17144775390625, + "step": 301 + }, + { + "KL/chosen_KL_mean": -263.9697570800781, + "KL/mean": -353.9146728515625, + "KL/rejected_KL_mean": -443.85955810546875, + "KL/std": 204.00518798828125, + "epoch": 0.4434654919236417, + "fcm_dpo/beta": 0.0019708811305463314, + "fcm_dpo/delta": 0.054747726768255234, + "fcm_dpo/margin": 179.88980102539062, + "fcm_dpo/q_t": 0.4198175072669983, + "grad_norm": 19.74114227294922, + "learning_rate": 3.426989547989902e-07, + "logits/chosen": -0.3587857484817505, + "logits/rejected": -0.36537155508995056, + "logps/chosen": -315.167724609375, + "logps/ref_chosen": -51.197994232177734, + "logps/ref_rejected": -97.22636413574219, + "logps/rejected": -541.0859375, + "loss": 1.1197, + "margin_dpo/margin_mean": 179.88980102539062, + "margin_dpo/margin_std": 243.09542846679688, + "step": 302 + }, + { + "KL/chosen_KL_mean": -275.6668701171875, + "KL/mean": -358.65966796875, + "KL/rejected_KL_mean": -441.6524658203125, + "KL/std": 205.9825439453125, + "epoch": 0.44493392070484583, + "fcm_dpo/beta": 0.002106656786054373, + "fcm_dpo/delta": 0.061119839549064636, + "fcm_dpo/margin": 165.985595703125, + "fcm_dpo/q_t": 0.42138999700546265, + "grad_norm": 28.86655044555664, + "learning_rate": 3.4150589130555773e-07, + "logits/chosen": -0.3849058747291565, + "logits/rejected": -0.3747110366821289, + "logps/chosen": -342.38079833984375, + "logps/ref_chosen": -66.71394348144531, + "logps/ref_rejected": -86.94542694091797, + "logps/rejected": -528.597900390625, + "loss": 1.1338, + "margin_dpo/margin_mean": 165.985595703125, + "margin_dpo/margin_std": 249.7038116455078, + "step": 303 + }, + { + "KL/chosen_KL_mean": -257.2261962890625, + "KL/mean": -351.8838806152344, + "KL/rejected_KL_mean": -446.54156494140625, + "KL/std": 196.91989135742188, + "epoch": 0.44640234948604995, + "fcm_dpo/beta": 0.0021874920930713415, + "fcm_dpo/delta": -0.019974112510681152, + "fcm_dpo/margin": 189.31533813476562, + "fcm_dpo/q_t": 0.40017253160476685, + "grad_norm": 45.90519332885742, + "learning_rate": 3.403104165467883e-07, + "logits/chosen": -0.41976654529571533, + "logits/rejected": -0.4154810905456543, + "logps/chosen": -329.1769104003906, + "logps/ref_chosen": -71.95069885253906, + "logps/ref_rejected": -90.47203063964844, + "logps/rejected": -537.0135498046875, + "loss": 1.0521, + "margin_dpo/margin_mean": 189.31533813476562, + "margin_dpo/margin_std": 171.4813232421875, + "step": 304 + }, + { + "KL/chosen_KL_mean": -267.2148132324219, + "KL/mean": -351.92645263671875, + "KL/rejected_KL_mean": -436.63812255859375, + "KL/std": 221.58193969726562, + "epoch": 0.447870778267254, + "fcm_dpo/beta": 0.0021368793677538633, + "fcm_dpo/delta": 0.04446440190076828, + "fcm_dpo/margin": 169.42330932617188, + "fcm_dpo/q_t": 0.4194003641605377, + "grad_norm": 24.6429500579834, + "learning_rate": 3.391125620245535e-07, + "logits/chosen": -0.431082159280777, + "logits/rejected": -0.417194128036499, + "logps/chosen": -334.0100402832031, + "logps/ref_chosen": -66.79523468017578, + "logps/ref_rejected": -92.75459289550781, + "logps/rejected": -529.3927001953125, + "loss": 1.1257, + "margin_dpo/margin_mean": 169.42330932617188, + "margin_dpo/margin_std": 248.18634033203125, + "step": 305 + }, + { + "KL/chosen_KL_mean": -270.4483947753906, + "KL/mean": -353.85845947265625, + "KL/rejected_KL_mean": -437.2685241699219, + "KL/std": 196.28871154785156, + "epoch": 0.44933920704845814, + "fcm_dpo/beta": 0.002273739315569401, + "fcm_dpo/delta": 0.025212926790118217, + "fcm_dpo/margin": 166.82009887695312, + "fcm_dpo/q_t": 0.41293442249298096, + "grad_norm": 20.455917358398438, + "learning_rate": 3.3791235930343417e-07, + "logits/chosen": -0.4016483426094055, + "logits/rejected": -0.37884002923965454, + "logps/chosen": -340.1322937011719, + "logps/ref_chosen": -69.68389892578125, + "logps/ref_rejected": -85.15919494628906, + "logps/rejected": -522.427734375, + "loss": 1.0949, + "margin_dpo/margin_mean": 166.82009887695312, + "margin_dpo/margin_std": 209.34255981445312, + "step": 306 + }, + { + "KL/chosen_KL_mean": -245.0524139404297, + "KL/mean": -331.04681396484375, + "KL/rejected_KL_mean": -417.041259765625, + "KL/std": 183.830810546875, + "epoch": 0.45080763582966227, + "fcm_dpo/beta": 0.0022681187838315964, + "fcm_dpo/delta": 0.011811528354883194, + "fcm_dpo/margin": 171.98887634277344, + "fcm_dpo/q_t": 0.4100651741027832, + "grad_norm": 28.9613037109375, + "learning_rate": 3.367098400098881e-07, + "logits/chosen": -0.4207409620285034, + "logits/rejected": -0.40590375661849976, + "logps/chosen": -315.21783447265625, + "logps/ref_chosen": -70.16542053222656, + "logps/ref_rejected": -86.97230529785156, + "logps/rejected": -504.0135803222656, + "loss": 1.096, + "margin_dpo/margin_mean": 171.98886108398438, + "margin_dpo/margin_std": 226.1441650390625, + "step": 307 + }, + { + "KL/chosen_KL_mean": -252.00428771972656, + "KL/mean": -349.07568359375, + "KL/rejected_KL_mean": -446.1470642089844, + "KL/std": 207.59048461914062, + "epoch": 0.4522760646108664, + "fcm_dpo/beta": 0.002229331061244011, + "fcm_dpo/delta": -0.04198146238923073, + "fcm_dpo/margin": 194.14276123046875, + "fcm_dpo/q_t": 0.3967881202697754, + "grad_norm": 28.756671905517578, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": -0.4297657012939453, + "logits/rejected": -0.4254001975059509, + "logps/chosen": -307.249267578125, + "logps/ref_chosen": -55.2449951171875, + "logps/ref_rejected": -79.37226104736328, + "logps/rejected": -525.5193481445312, + "loss": 1.0511, + "margin_dpo/margin_mean": 194.14276123046875, + "margin_dpo/margin_std": 207.66583251953125, + "step": 308 + }, + { + "KL/chosen_KL_mean": -257.0223388671875, + "KL/mean": -352.66180419921875, + "KL/rejected_KL_mean": -448.30126953125, + "KL/std": 222.42701721191406, + "epoch": 0.45374449339207046, + "fcm_dpo/beta": 0.0022140173241496086, + "fcm_dpo/delta": -0.03186354413628578, + "fcm_dpo/margin": 191.2789306640625, + "fcm_dpo/q_t": 0.3998178541660309, + "grad_norm": 29.72759437561035, + "learning_rate": 3.3429797851573183e-07, + "logits/chosen": -0.3393629789352417, + "logits/rejected": -0.33330753445625305, + "logps/chosen": -305.98138427734375, + "logps/ref_chosen": -48.959083557128906, + "logps/ref_rejected": -82.34072875976562, + "logps/rejected": -530.6419677734375, + "loss": 1.0754, + "margin_dpo/margin_mean": 191.2789306640625, + "margin_dpo/margin_std": 235.2989959716797, + "step": 309 + }, + { + "KL/chosen_KL_mean": -304.2635498046875, + "KL/mean": -393.31304931640625, + "KL/rejected_KL_mean": -482.362548828125, + "KL/std": 200.93919372558594, + "epoch": 0.4552129221732746, + "fcm_dpo/beta": 0.0021948455832898617, + "fcm_dpo/delta": 0.009528759866952896, + "fcm_dpo/margin": 178.0990447998047, + "fcm_dpo/q_t": 0.4077993631362915, + "grad_norm": 36.597747802734375, + "learning_rate": 3.3308869986991487e-07, + "logits/chosen": -0.4227510690689087, + "logits/rejected": -0.41301751136779785, + "logps/chosen": -367.00531005859375, + "logps/ref_chosen": -62.74177932739258, + "logps/ref_rejected": -79.9302978515625, + "logps/rejected": -562.2928466796875, + "loss": 1.0787, + "margin_dpo/margin_mean": 178.09906005859375, + "margin_dpo/margin_std": 195.08804321289062, + "step": 310 + }, + { + "KL/chosen_KL_mean": -307.46826171875, + "KL/mean": -412.06182861328125, + "KL/rejected_KL_mean": -516.6553955078125, + "KL/std": 252.58099365234375, + "epoch": 0.4566813509544787, + "fcm_dpo/beta": 0.0020815101452171803, + "fcm_dpo/delta": -0.045531198382377625, + "fcm_dpo/margin": 209.18707275390625, + "fcm_dpo/q_t": 0.39924418926239014, + "grad_norm": 25.931556701660156, + "learning_rate": 3.3187723175958346e-07, + "logits/chosen": -0.4020995497703552, + "logits/rejected": -0.3801601529121399, + "logps/chosen": -360.4962463378906, + "logps/ref_chosen": -53.02798080444336, + "logps/ref_rejected": -77.43820190429688, + "logps/rejected": -594.0935668945312, + "loss": 1.0693, + "margin_dpo/margin_mean": 209.18707275390625, + "margin_dpo/margin_std": 267.8011474609375, + "step": 311 + }, + { + "KL/chosen_KL_mean": -301.262451171875, + "KL/mean": -398.7344970703125, + "KL/rejected_KL_mean": -496.20654296875, + "KL/std": 227.84207153320312, + "epoch": 0.4581497797356828, + "fcm_dpo/beta": 0.002090971916913986, + "fcm_dpo/delta": -0.011386025696992874, + "fcm_dpo/margin": 194.94412231445312, + "fcm_dpo/q_t": 0.40784695744514465, + "grad_norm": 25.662092208862305, + "learning_rate": 3.306636061080487e-07, + "logits/chosen": -0.40964388847351074, + "logits/rejected": -0.40318238735198975, + "logps/chosen": -350.6546630859375, + "logps/ref_chosen": -49.39221954345703, + "logps/ref_rejected": -75.79280853271484, + "logps/rejected": -571.9993896484375, + "loss": 1.0933, + "margin_dpo/margin_mean": 194.94412231445312, + "margin_dpo/margin_std": 271.61602783203125, + "step": 312 + }, + { + "KL/chosen_KL_mean": -282.75341796875, + "KL/mean": -380.6422119140625, + "KL/rejected_KL_mean": -478.531005859375, + "KL/std": 236.21067810058594, + "epoch": 0.45961820851688695, + "fcm_dpo/beta": 0.002096590120345354, + "fcm_dpo/delta": -0.019848383963108063, + "fcm_dpo/margin": 195.77755737304688, + "fcm_dpo/q_t": 0.4035242199897766, + "grad_norm": 27.727643966674805, + "learning_rate": 3.2944785489547537e-07, + "logits/chosen": -0.4416351914405823, + "logits/rejected": -0.43936672806739807, + "logps/chosen": -332.90618896484375, + "logps/ref_chosen": -50.152740478515625, + "logps/ref_rejected": -86.40620422363281, + "logps/rejected": -564.9371948242188, + "loss": 1.0933, + "margin_dpo/margin_mean": 195.77755737304688, + "margin_dpo/margin_std": 253.7050018310547, + "step": 313 + }, + { + "KL/chosen_KL_mean": -304.5816345214844, + "KL/mean": -394.99072265625, + "KL/rejected_KL_mean": -485.39984130859375, + "KL/std": 222.5576934814453, + "epoch": 0.461086637298091, + "fcm_dpo/beta": 0.0020326506346464157, + "fcm_dpo/delta": 0.03956056758761406, + "fcm_dpo/margin": 180.8181915283203, + "fcm_dpo/q_t": 0.41847917437553406, + "grad_norm": 20.639629364013672, + "learning_rate": 3.2823001015803857e-07, + "logits/chosen": -0.476603627204895, + "logits/rejected": -0.4820059835910797, + "logps/chosen": -361.8192138671875, + "logps/ref_chosen": -57.237579345703125, + "logps/ref_rejected": -97.5965347290039, + "logps/rejected": -582.996337890625, + "loss": 1.1336, + "margin_dpo/margin_mean": 180.81817626953125, + "margin_dpo/margin_std": 289.74945068359375, + "step": 314 + }, + { + "KL/chosen_KL_mean": -279.29681396484375, + "KL/mean": -364.7213134765625, + "KL/rejected_KL_mean": -450.1457824707031, + "KL/std": 205.07464599609375, + "epoch": 0.46255506607929514, + "fcm_dpo/beta": 0.00213609728962183, + "fcm_dpo/delta": 0.04297637194395065, + "fcm_dpo/margin": 170.8489990234375, + "fcm_dpo/q_t": 0.41723307967185974, + "grad_norm": 21.063323974609375, + "learning_rate": 3.270101039870797e-07, + "logits/chosen": -0.3772663474082947, + "logits/rejected": -0.3812904357910156, + "logps/chosen": -328.36639404296875, + "logps/ref_chosen": -49.06958770751953, + "logps/ref_rejected": -85.68087768554688, + "logps/rejected": -535.82666015625, + "loss": 1.1108, + "margin_dpo/margin_mean": 170.8489990234375, + "margin_dpo/margin_std": 229.18209838867188, + "step": 315 + }, + { + "KL/chosen_KL_mean": -275.72784423828125, + "KL/mean": -397.86114501953125, + "KL/rejected_KL_mean": -519.994384765625, + "KL/std": 247.28073120117188, + "epoch": 0.46402349486049926, + "fcm_dpo/beta": 0.001942518400028348, + "fcm_dpo/delta": -0.10697119683027267, + "fcm_dpo/margin": 244.26651000976562, + "fcm_dpo/q_t": 0.3864254951477051, + "grad_norm": 23.46164321899414, + "learning_rate": 3.2578816852826086e-07, + "logits/chosen": -0.42778435349464417, + "logits/rejected": -0.43224072456359863, + "logps/chosen": -329.9886169433594, + "logps/ref_chosen": -54.26074981689453, + "logps/ref_rejected": -101.2814712524414, + "logps/rejected": -621.27587890625, + "loss": 1.0309, + "margin_dpo/margin_mean": 244.2665252685547, + "margin_dpo/margin_std": 268.9966125488281, + "step": 316 + }, + { + "KL/chosen_KL_mean": -286.0876159667969, + "KL/mean": -423.0874938964844, + "KL/rejected_KL_mean": -560.0873413085938, + "KL/std": 227.30641174316406, + "epoch": 0.4654919236417034, + "fcm_dpo/beta": 0.0018125809729099274, + "fcm_dpo/delta": -0.1276942938566208, + "fcm_dpo/margin": 273.9997253417969, + "fcm_dpo/q_t": 0.3785746097564697, + "grad_norm": 27.784400939941406, + "learning_rate": 3.2456423598071783e-07, + "logits/chosen": -0.4147995114326477, + "logits/rejected": -0.4096861481666565, + "logps/chosen": -342.18182373046875, + "logps/ref_chosen": -56.094207763671875, + "logps/ref_rejected": -100.69905090332031, + "logps/rejected": -660.786376953125, + "loss": 1.0049, + "margin_dpo/margin_mean": 273.99969482421875, + "margin_dpo/margin_std": 257.4659423828125, + "step": 317 + }, + { + "KL/chosen_KL_mean": -317.5200500488281, + "KL/mean": -415.6494140625, + "KL/rejected_KL_mean": -513.7787475585938, + "KL/std": 229.84683227539062, + "epoch": 0.4669603524229075, + "fcm_dpo/beta": 0.0018144804053008556, + "fcm_dpo/delta": 0.05106057599186897, + "fcm_dpo/margin": 196.25872802734375, + "fcm_dpo/q_t": 0.41813182830810547, + "grad_norm": 26.109601974487305, + "learning_rate": 3.233383385962115e-07, + "logits/chosen": -0.4953124225139618, + "logits/rejected": -0.46893250942230225, + "logps/chosen": -382.1657409667969, + "logps/ref_chosen": -64.64569854736328, + "logps/ref_rejected": -82.76425170898438, + "logps/rejected": -596.5430297851562, + "loss": 1.1096, + "margin_dpo/margin_mean": 196.25872802734375, + "margin_dpo/margin_std": 247.5173797607422, + "step": 318 + }, + { + "KL/chosen_KL_mean": -285.78082275390625, + "KL/mean": -403.2270812988281, + "KL/rejected_KL_mean": -520.67333984375, + "KL/std": 257.381591796875, + "epoch": 0.4684287812041116, + "fcm_dpo/beta": 0.0017810399876907468, + "fcm_dpo/delta": -0.023179180920124054, + "fcm_dpo/margin": 234.89251708984375, + "fcm_dpo/q_t": 0.40157684683799744, + "grad_norm": 22.635221481323242, + "learning_rate": 3.2211050867827805e-07, + "logits/chosen": -0.38388240337371826, + "logits/rejected": -0.396739661693573, + "logps/chosen": -335.16455078125, + "logps/ref_chosen": -49.383758544921875, + "logps/ref_rejected": -113.90650939941406, + "logps/rejected": -634.579833984375, + "loss": 1.0691, + "margin_dpo/margin_mean": 234.8925323486328, + "margin_dpo/margin_std": 279.86474609375, + "step": 319 + }, + { + "KL/chosen_KL_mean": -290.6177978515625, + "KL/mean": -416.1560974121094, + "KL/rejected_KL_mean": -541.6944580078125, + "KL/std": 259.626220703125, + "epoch": 0.4698972099853157, + "fcm_dpo/beta": 0.0016947748372331262, + "fcm_dpo/delta": -0.03422039747238159, + "fcm_dpo/margin": 251.07666015625, + "fcm_dpo/q_t": 0.39976027607917786, + "grad_norm": 21.635658264160156, + "learning_rate": 3.208807785813777e-07, + "logits/chosen": -0.44464847445487976, + "logits/rejected": -0.4517730474472046, + "logps/chosen": -350.1226806640625, + "logps/ref_chosen": -59.50489044189453, + "logps/ref_rejected": -97.66717529296875, + "logps/rejected": -639.361572265625, + "loss": 1.0582, + "margin_dpo/margin_mean": 251.07666015625, + "margin_dpo/margin_std": 280.9095458984375, + "step": 320 + }, + { + "KL/chosen_KL_mean": -359.953369140625, + "KL/mean": -468.767333984375, + "KL/rejected_KL_mean": -577.5813598632812, + "KL/std": 275.1668701171875, + "epoch": 0.4713656387665198, + "fcm_dpo/beta": 0.0017093562055379152, + "fcm_dpo/delta": 0.03284335136413574, + "fcm_dpo/margin": 217.62799072265625, + "fcm_dpo/q_t": 0.41511040925979614, + "grad_norm": 19.775318145751953, + "learning_rate": 3.1964918071004217e-07, + "logits/chosen": -0.3967721462249756, + "logits/rejected": -0.38455623388290405, + "logps/chosen": -421.5020446777344, + "logps/ref_chosen": -61.548683166503906, + "logps/ref_rejected": -91.64103698730469, + "logps/rejected": -669.222412109375, + "loss": 1.1103, + "margin_dpo/margin_mean": 217.62796020507812, + "margin_dpo/margin_std": 292.2899169921875, + "step": 321 + }, + { + "KL/chosen_KL_mean": -302.359130859375, + "KL/mean": -423.87506103515625, + "KL/rejected_KL_mean": -545.3909912109375, + "KL/std": 239.97723388671875, + "epoch": 0.47283406754772395, + "fcm_dpo/beta": 0.001710197888314724, + "fcm_dpo/delta": -0.02115067094564438, + "fcm_dpo/margin": 243.0318603515625, + "fcm_dpo/q_t": 0.4011186957359314, + "grad_norm": 20.75707244873047, + "learning_rate": 3.184157475180207e-07, + "logits/chosen": -0.381791889667511, + "logits/rejected": -0.3810097575187683, + "logps/chosen": -359.649169921875, + "logps/ref_chosen": -57.29003143310547, + "logps/ref_rejected": -95.74992370605469, + "logps/rejected": -641.1409912109375, + "loss": 1.0584, + "margin_dpo/margin_mean": 243.0318603515625, + "margin_dpo/margin_std": 252.65171813964844, + "step": 322 + }, + { + "KL/chosen_KL_mean": -329.60552978515625, + "KL/mean": -433.67523193359375, + "KL/rejected_KL_mean": -537.7449340820312, + "KL/std": 240.89913940429688, + "epoch": 0.47430249632892807, + "fcm_dpo/beta": 0.0017628297209739685, + "fcm_dpo/delta": 0.04052872955799103, + "fcm_dpo/margin": 208.13937377929688, + "fcm_dpo/q_t": 0.4153671860694885, + "grad_norm": 26.74566078186035, + "learning_rate": 3.171805115074251e-07, + "logits/chosen": -0.4394986629486084, + "logits/rejected": -0.4391285479068756, + "logps/chosen": -380.8395080566406, + "logps/ref_chosen": -51.23395919799805, + "logps/ref_rejected": -75.06192016601562, + "logps/rejected": -612.806884765625, + "loss": 1.1044, + "margin_dpo/margin_mean": 208.13937377929688, + "margin_dpo/margin_std": 257.1087951660156, + "step": 323 + }, + { + "KL/chosen_KL_mean": -379.00439453125, + "KL/mean": -478.7786865234375, + "KL/rejected_KL_mean": -578.552978515625, + "KL/std": 264.73980712890625, + "epoch": 0.47577092511013214, + "fcm_dpo/beta": 0.0018952081445604563, + "fcm_dpo/delta": 0.019777312874794006, + "fcm_dpo/margin": 199.5486297607422, + "fcm_dpo/q_t": 0.415118932723999, + "grad_norm": 40.34514236450195, + "learning_rate": 3.1594350522787295e-07, + "logits/chosen": -0.446133553981781, + "logits/rejected": -0.43301481008529663, + "logps/chosen": -444.1395568847656, + "logps/ref_chosen": -65.13516998291016, + "logps/ref_rejected": -86.47750854492188, + "logps/rejected": -665.030517578125, + "loss": 1.1304, + "margin_dpo/margin_mean": 199.5486297607422, + "margin_dpo/margin_std": 308.2591247558594, + "step": 324 + }, + { + "KL/chosen_KL_mean": -287.88055419921875, + "KL/mean": -370.6061096191406, + "KL/rejected_KL_mean": -453.3316650390625, + "KL/std": 218.6322021484375, + "epoch": 0.47723935389133626, + "fcm_dpo/beta": 0.001969131175428629, + "fcm_dpo/delta": 0.08780458569526672, + "fcm_dpo/margin": 165.45111083984375, + "fcm_dpo/q_t": 0.4262668192386627, + "grad_norm": 25.783239364624023, + "learning_rate": 3.147047612756302e-07, + "logits/chosen": -0.46015477180480957, + "logits/rejected": -0.4445416331291199, + "logps/chosen": -344.09613037109375, + "logps/ref_chosen": -56.215599060058594, + "logps/ref_rejected": -70.08592987060547, + "logps/rejected": -523.4176025390625, + "loss": 1.1354, + "margin_dpo/margin_mean": 165.45111083984375, + "margin_dpo/margin_std": 224.27264404296875, + "step": 325 + }, + { + "KL/chosen_KL_mean": -307.934814453125, + "KL/mean": -389.87255859375, + "KL/rejected_KL_mean": -471.8103332519531, + "KL/std": 201.8898162841797, + "epoch": 0.4787077826725404, + "fcm_dpo/beta": 0.0020997067913413048, + "fcm_dpo/delta": 0.06763522326946259, + "fcm_dpo/margin": 163.87550354003906, + "fcm_dpo/q_t": 0.4207178056240082, + "grad_norm": 62.081993103027344, + "learning_rate": 3.134643122927519e-07, + "logits/chosen": -0.4982990622520447, + "logits/rejected": -0.4790222644805908, + "logps/chosen": -380.6597595214844, + "logps/ref_chosen": -72.72496032714844, + "logps/ref_rejected": -79.8467788696289, + "logps/rejected": -551.6571044921875, + "loss": 1.1107, + "margin_dpo/margin_mean": 163.87550354003906, + "margin_dpo/margin_std": 188.82302856445312, + "step": 326 + }, + { + "KL/chosen_KL_mean": -277.514404296875, + "KL/mean": -394.7227783203125, + "KL/rejected_KL_mean": -511.93115234375, + "KL/std": 218.5492706298828, + "epoch": 0.4801762114537445, + "fcm_dpo/beta": 0.0019716641400009394, + "fcm_dpo/delta": -0.08498930931091309, + "fcm_dpo/margin": 234.416748046875, + "fcm_dpo/q_t": 0.38751423358917236, + "grad_norm": 51.83660888671875, + "learning_rate": 3.1222219096622264e-07, + "logits/chosen": -0.45873162150382996, + "logits/rejected": -0.4484935998916626, + "logps/chosen": -346.6488037109375, + "logps/ref_chosen": -69.13441467285156, + "logps/ref_rejected": -111.93377685546875, + "logps/rejected": -623.8649291992188, + "loss": 1.0338, + "margin_dpo/margin_mean": 234.41673278808594, + "margin_dpo/margin_std": 248.63563537597656, + "step": 327 + }, + { + "KL/chosen_KL_mean": -291.4725341796875, + "KL/mean": -397.5861511230469, + "KL/rejected_KL_mean": -503.69976806640625, + "KL/std": 236.2239990234375, + "epoch": 0.48164464023494863, + "fcm_dpo/beta": 0.0019511812133714557, + "fcm_dpo/delta": -0.01778026856482029, + "fcm_dpo/margin": 212.22723388671875, + "fcm_dpo/q_t": 0.4037235677242279, + "grad_norm": 34.96001052856445, + "learning_rate": 3.1097843002709427e-07, + "logits/chosen": -0.4395790100097656, + "logits/rejected": -0.4473843276500702, + "logps/chosen": -351.15972900390625, + "logps/ref_chosen": -59.68719482421875, + "logps/ref_rejected": -90.85499572753906, + "logps/rejected": -594.5548095703125, + "loss": 1.0754, + "margin_dpo/margin_mean": 212.22726440429688, + "margin_dpo/margin_std": 262.13189697265625, + "step": 328 + }, + { + "KL/chosen_KL_mean": -334.9793395996094, + "KL/mean": -451.97674560546875, + "KL/rejected_KL_mean": -568.97412109375, + "KL/std": 270.3482666015625, + "epoch": 0.4831130690161527, + "fcm_dpo/beta": 0.0018274824833497405, + "fcm_dpo/delta": -0.03972601890563965, + "fcm_dpo/margin": 233.9947509765625, + "fcm_dpo/q_t": 0.3984408974647522, + "grad_norm": 25.084033966064453, + "learning_rate": 3.0973306224962437e-07, + "logits/chosen": -0.4626386761665344, + "logits/rejected": -0.46101057529449463, + "logps/chosen": -400.22552490234375, + "logps/ref_chosen": -65.2461929321289, + "logps/ref_rejected": -100.69770812988281, + "logps/rejected": -669.671875, + "loss": 1.0719, + "margin_dpo/margin_mean": 233.9947509765625, + "margin_dpo/margin_std": 285.23626708984375, + "step": 329 + }, + { + "KL/chosen_KL_mean": -292.37579345703125, + "KL/mean": -412.8188171386719, + "KL/rejected_KL_mean": -533.2619018554688, + "KL/std": 249.09326171875, + "epoch": 0.4845814977973568, + "fcm_dpo/beta": 0.0017980177653953433, + "fcm_dpo/delta": -0.04239234700798988, + "fcm_dpo/margin": 240.88613891601562, + "fcm_dpo/q_t": 0.3976569175720215, + "grad_norm": 20.89375877380371, + "learning_rate": 3.084861204504122e-07, + "logits/chosen": -0.4702164828777313, + "logits/rejected": -0.46967270970344543, + "logps/chosen": -339.3741149902344, + "logps/ref_chosen": -46.998348236083984, + "logps/ref_rejected": -86.87684631347656, + "logps/rejected": -620.1387329101562, + "loss": 1.0555, + "margin_dpo/margin_mean": 240.88613891601562, + "margin_dpo/margin_std": 273.5047607421875, + "step": 330 + }, + { + "KL/chosen_KL_mean": -330.68896484375, + "KL/mean": -452.95233154296875, + "KL/rejected_KL_mean": -575.2156982421875, + "KL/std": 218.610107421875, + "epoch": 0.48604992657856094, + "fcm_dpo/beta": 0.0017413150053471327, + "fcm_dpo/delta": -0.032744720578193665, + "fcm_dpo/margin": 244.52676391601562, + "fcm_dpo/q_t": 0.3974621891975403, + "grad_norm": 26.476423263549805, + "learning_rate": 3.072376374875335e-07, + "logits/chosen": -0.49178826808929443, + "logits/rejected": -0.4904711842536926, + "logps/chosen": -381.21319580078125, + "logps/ref_chosen": -50.52424621582031, + "logps/ref_rejected": -89.01544189453125, + "logps/rejected": -664.231201171875, + "loss": 1.0385, + "margin_dpo/margin_mean": 244.52676391601562, + "margin_dpo/margin_std": 216.18482971191406, + "step": 331 + }, + { + "KL/chosen_KL_mean": -348.40509033203125, + "KL/mean": -431.3126220703125, + "KL/rejected_KL_mean": -514.2201538085938, + "KL/std": 222.2275390625, + "epoch": 0.48751835535976507, + "fcm_dpo/beta": 0.0018814262002706528, + "fcm_dpo/delta": 0.10219782590866089, + "fcm_dpo/margin": 165.81512451171875, + "fcm_dpo/q_t": 0.43078944087028503, + "grad_norm": 26.434301376342773, + "learning_rate": 3.059876462596758e-07, + "logits/chosen": -0.5475494861602783, + "logits/rejected": -0.5299344062805176, + "logps/chosen": -397.5853576660156, + "logps/ref_chosen": -49.18028259277344, + "logps/ref_rejected": -76.48515319824219, + "logps/rejected": -590.705322265625, + "loss": 1.152, + "margin_dpo/margin_mean": 165.81512451171875, + "margin_dpo/margin_std": 254.50332641601562, + "step": 332 + }, + { + "KL/chosen_KL_mean": -377.22314453125, + "KL/mean": -493.54986572265625, + "KL/rejected_KL_mean": -609.8765869140625, + "KL/std": 277.18450927734375, + "epoch": 0.4889867841409692, + "fcm_dpo/beta": 0.0017979848198592663, + "fcm_dpo/delta": -0.030853308737277985, + "fcm_dpo/margin": 232.65338134765625, + "fcm_dpo/q_t": 0.40362921357154846, + "grad_norm": 32.03265380859375, + "learning_rate": 3.0473617970527015e-07, + "logits/chosen": -0.5083990097045898, + "logits/rejected": -0.5068196654319763, + "logps/chosen": -440.9789123535156, + "logps/ref_chosen": -63.75574493408203, + "logps/ref_rejected": -95.04411315917969, + "logps/rejected": -704.920654296875, + "loss": 1.1067, + "margin_dpo/margin_mean": 232.6533660888672, + "margin_dpo/margin_std": 344.5853271484375, + "step": 333 + }, + { + "KL/chosen_KL_mean": -350.3871765136719, + "KL/mean": -450.59454345703125, + "KL/rejected_KL_mean": -550.802001953125, + "KL/std": 298.42266845703125, + "epoch": 0.49045521292217326, + "fcm_dpo/beta": 0.0018730323063209653, + "fcm_dpo/delta": 0.030153676867485046, + "fcm_dpo/margin": 200.414794921875, + "fcm_dpo/q_t": 0.41732466220855713, + "grad_norm": 27.673667907714844, + "learning_rate": 3.034832708016243e-07, + "logits/chosen": -0.4616066813468933, + "logits/rejected": -0.45988792181015015, + "logps/chosen": -417.366943359375, + "logps/ref_chosen": -66.97975158691406, + "logps/ref_rejected": -95.31692504882812, + "logps/rejected": -646.118896484375, + "loss": 1.1399, + "margin_dpo/margin_mean": 200.41477966308594, + "margin_dpo/margin_std": 338.0478515625, + "step": 334 + }, + { + "KL/chosen_KL_mean": -360.7254333496094, + "KL/mean": -446.093017578125, + "KL/rejected_KL_mean": -531.4605712890625, + "KL/std": 273.3331298828125, + "epoch": 0.4919236417033774, + "fcm_dpo/beta": 0.00204592477530241, + "fcm_dpo/delta": 0.05596485733985901, + "fcm_dpo/margin": 170.73513793945312, + "fcm_dpo/q_t": 0.42263150215148926, + "grad_norm": 35.19724655151367, + "learning_rate": 3.022289525640531e-07, + "logits/chosen": -0.569267749786377, + "logits/rejected": -0.5530425310134888, + "logps/chosen": -423.2679138183594, + "logps/ref_chosen": -62.54248046875, + "logps/ref_rejected": -87.61770629882812, + "logps/rejected": -619.0782470703125, + "loss": 1.15, + "margin_dpo/margin_mean": 170.73513793945312, + "margin_dpo/margin_std": 286.39031982421875, + "step": 335 + }, + { + "KL/chosen_KL_mean": -362.0695495605469, + "KL/mean": -486.9052429199219, + "KL/rejected_KL_mean": -611.740966796875, + "KL/std": 306.3258972167969, + "epoch": 0.4933920704845815, + "fcm_dpo/beta": 0.0019088383996859193, + "fcm_dpo/delta": -0.10008023679256439, + "fcm_dpo/margin": 249.67137145996094, + "fcm_dpo/q_t": 0.3910997807979584, + "grad_norm": 27.048595428466797, + "learning_rate": 3.009732580450086e-07, + "logits/chosen": -0.4783538281917572, + "logits/rejected": -0.4791324734687805, + "logps/chosen": -416.6007080078125, + "logps/ref_chosen": -54.53115463256836, + "logps/ref_rejected": -104.40424346923828, + "logps/rejected": -716.1451416015625, + "loss": 1.078, + "margin_dpo/margin_mean": 249.67135620117188, + "margin_dpo/margin_std": 367.23291015625, + "step": 336 + }, + { + "KL/chosen_KL_mean": -319.1661376953125, + "KL/mean": -445.8056335449219, + "KL/rejected_KL_mean": -572.445068359375, + "KL/std": 243.7745361328125, + "epoch": 0.4948604992657856, + "fcm_dpo/beta": 0.0017321293707937002, + "fcm_dpo/delta": -0.051446449011564255, + "fcm_dpo/margin": 253.27894592285156, + "fcm_dpo/q_t": 0.39632317423820496, + "grad_norm": 19.951826095581055, + "learning_rate": 2.9971622033320914e-07, + "logits/chosen": -0.5001170635223389, + "logits/rejected": -0.4860590696334839, + "logps/chosen": -384.29486083984375, + "logps/ref_chosen": -65.12869262695312, + "logps/ref_rejected": -101.72701263427734, + "logps/rejected": -674.172119140625, + "loss": 1.0528, + "margin_dpo/margin_mean": 253.27896118164062, + "margin_dpo/margin_std": 288.4793701171875, + "step": 337 + }, + { + "KL/chosen_KL_mean": -283.015869140625, + "KL/mean": -405.06072998046875, + "KL/rejected_KL_mean": -527.1055908203125, + "KL/std": 232.23532104492188, + "epoch": 0.49632892804698975, + "fcm_dpo/beta": 0.0016998077044263482, + "fcm_dpo/delta": -0.01908147893846035, + "fcm_dpo/margin": 244.0897216796875, + "fcm_dpo/q_t": 0.40122056007385254, + "grad_norm": 33.17329025268555, + "learning_rate": 2.984578725527675e-07, + "logits/chosen": -0.5003715753555298, + "logits/rejected": -0.49724671244621277, + "logps/chosen": -341.4385986328125, + "logps/ref_chosen": -58.422706604003906, + "logps/ref_rejected": -89.06854248046875, + "logps/rejected": -616.1741333007812, + "loss": 1.0533, + "margin_dpo/margin_mean": 244.0897216796875, + "margin_dpo/margin_std": 240.3944854736328, + "step": 338 + }, + { + "KL/chosen_KL_mean": -306.18310546875, + "KL/mean": -420.02911376953125, + "KL/rejected_KL_mean": -533.8751220703125, + "KL/std": 236.42514038085938, + "epoch": 0.4977973568281938, + "fcm_dpo/beta": 0.0017625135369598866, + "fcm_dpo/delta": -0.008848987519741058, + "fcm_dpo/margin": 227.69198608398438, + "fcm_dpo/q_t": 0.40451204776763916, + "grad_norm": 25.832143783569336, + "learning_rate": 2.9719824786231796e-07, + "logits/chosen": -0.5664355754852295, + "logits/rejected": -0.5560377836227417, + "logps/chosen": -366.17840576171875, + "logps/ref_chosen": -59.99531555175781, + "logps/ref_rejected": -103.9109115600586, + "logps/rejected": -637.7860107421875, + "loss": 1.077, + "margin_dpo/margin_mean": 227.69198608398438, + "margin_dpo/margin_std": 242.7012481689453, + "step": 339 + }, + { + "KL/chosen_KL_mean": -306.2630920410156, + "KL/mean": -406.93267822265625, + "KL/rejected_KL_mean": -507.6022644042969, + "KL/std": 228.15411376953125, + "epoch": 0.49926578560939794, + "fcm_dpo/beta": 0.001739348517730832, + "fcm_dpo/delta": 0.06040128692984581, + "fcm_dpo/margin": 201.33917236328125, + "fcm_dpo/q_t": 0.4204002022743225, + "grad_norm": 19.13730812072754, + "learning_rate": 2.959373794541426e-07, + "logits/chosen": -0.46875977516174316, + "logits/rejected": -0.4506962299346924, + "logps/chosen": -359.09332275390625, + "logps/ref_chosen": -52.83022689819336, + "logps/ref_rejected": -73.10723114013672, + "logps/rejected": -580.70947265625, + "loss": 1.1223, + "margin_dpo/margin_mean": 201.33917236328125, + "margin_dpo/margin_std": 275.57354736328125, + "step": 340 + }, + { + "KL/chosen_KL_mean": -299.728515625, + "KL/mean": -421.93115234375, + "KL/rejected_KL_mean": -544.1337890625, + "KL/std": 255.9670867919922, + "epoch": 0.5007342143906021, + "fcm_dpo/beta": 0.001722757937386632, + "fcm_dpo/delta": -0.028678692877292633, + "fcm_dpo/margin": 244.40525817871094, + "fcm_dpo/q_t": 0.40045589208602905, + "grad_norm": 22.329763412475586, + "learning_rate": 2.946753005532965e-07, + "logits/chosen": -0.4424135684967041, + "logits/rejected": -0.4432716369628906, + "logps/chosen": -347.6282958984375, + "logps/ref_chosen": -47.899803161621094, + "logps/ref_rejected": -101.80987548828125, + "logps/rejected": -645.9436645507812, + "loss": 1.0604, + "margin_dpo/margin_mean": 244.40525817871094, + "margin_dpo/margin_std": 266.0626525878906, + "step": 341 + }, + { + "KL/chosen_KL_mean": -297.13800048828125, + "KL/mean": -400.4307861328125, + "KL/rejected_KL_mean": -503.7236022949219, + "KL/std": 236.86354064941406, + "epoch": 0.5022026431718062, + "fcm_dpo/beta": 0.0018168420065194368, + "fcm_dpo/delta": 0.027337031438946724, + "fcm_dpo/margin": 206.58563232421875, + "fcm_dpo/q_t": 0.41394251585006714, + "grad_norm": 22.068641662597656, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": -0.49012503027915955, + "logits/rejected": -0.4645516276359558, + "logps/chosen": -369.1346435546875, + "logps/ref_chosen": -71.99664306640625, + "logps/ref_rejected": -92.58959197998047, + "logps/rejected": -596.313232421875, + "loss": 1.109, + "margin_dpo/margin_mean": 206.5856170654297, + "margin_dpo/margin_std": 283.00140380859375, + "step": 342 + }, + { + "KL/chosen_KL_mean": -280.635986328125, + "KL/mean": -405.8623046875, + "KL/rejected_KL_mean": -531.088623046875, + "KL/std": 239.86712646484375, + "epoch": 0.5036710719530103, + "fcm_dpo/beta": 0.001725044334307313, + "fcm_dpo/delta": -0.0413195863366127, + "fcm_dpo/margin": 250.45265197753906, + "fcm_dpo/q_t": 0.39542150497436523, + "grad_norm": 20.26138687133789, + "learning_rate": 2.9214764433242476e-07, + "logits/chosen": -0.5200803875923157, + "logits/rejected": -0.5258131623268127, + "logps/chosen": -335.0416259765625, + "logps/ref_chosen": -54.405616760253906, + "logps/ref_rejected": -111.04142761230469, + "logps/rejected": -642.1300048828125, + "loss": 1.0338, + "margin_dpo/margin_mean": 250.45265197753906, + "margin_dpo/margin_std": 214.74595642089844, + "step": 343 + }, + { + "KL/chosen_KL_mean": -282.39642333984375, + "KL/mean": -391.85748291015625, + "KL/rejected_KL_mean": -501.31854248046875, + "KL/std": 262.4002685546875, + "epoch": 0.5051395007342144, + "fcm_dpo/beta": 0.0017953403294086456, + "fcm_dpo/delta": -0.00011947751045227051, + "fcm_dpo/margin": 218.922119140625, + "fcm_dpo/q_t": 0.40945079922676086, + "grad_norm": 18.697933197021484, + "learning_rate": 2.9088213361849126e-07, + "logits/chosen": -0.5233709216117859, + "logits/rejected": -0.5257829427719116, + "logps/chosen": -336.361083984375, + "logps/ref_chosen": -53.96466827392578, + "logps/ref_rejected": -90.62336730957031, + "logps/rejected": -591.94189453125, + "loss": 1.0912, + "margin_dpo/margin_mean": 218.922119140625, + "margin_dpo/margin_std": 262.0262756347656, + "step": 344 + }, + { + "KL/chosen_KL_mean": -341.17315673828125, + "KL/mean": -463.88018798828125, + "KL/rejected_KL_mean": -586.5872802734375, + "KL/std": 246.993408203125, + "epoch": 0.5066079295154186, + "fcm_dpo/beta": 0.0017087084706872702, + "fcm_dpo/delta": -0.024968912824988365, + "fcm_dpo/margin": 245.41409301757812, + "fcm_dpo/q_t": 0.4007132053375244, + "grad_norm": 23.478904724121094, + "learning_rate": 2.896155456223163e-07, + "logits/chosen": -0.5109938383102417, + "logits/rejected": -0.5099775791168213, + "logps/chosen": -402.85882568359375, + "logps/ref_chosen": -61.685699462890625, + "logps/ref_rejected": -99.49041748046875, + "logps/rejected": -686.0776977539062, + "loss": 1.0622, + "margin_dpo/margin_mean": 245.41409301757812, + "margin_dpo/margin_std": 272.0403137207031, + "step": 345 + }, + { + "KL/chosen_KL_mean": -347.7568359375, + "KL/mean": -463.9387512207031, + "KL/rejected_KL_mean": -580.12060546875, + "KL/std": 252.9781494140625, + "epoch": 0.5080763582966226, + "fcm_dpo/beta": 0.001682568108662963, + "fcm_dpo/delta": 0.011223375797271729, + "fcm_dpo/margin": 232.36383056640625, + "fcm_dpo/q_t": 0.40780335664749146, + "grad_norm": 23.162200927734375, + "learning_rate": 2.883479137196714e-07, + "logits/chosen": -0.49171721935272217, + "logits/rejected": -0.4840758442878723, + "logps/chosen": -403.0130920410156, + "logps/ref_chosen": -55.256263732910156, + "logps/ref_rejected": -77.41532135009766, + "logps/rejected": -657.5359497070312, + "loss": 1.0805, + "margin_dpo/margin_mean": 232.36383056640625, + "margin_dpo/margin_std": 264.86688232421875, + "step": 346 + }, + { + "KL/chosen_KL_mean": -359.0545654296875, + "KL/mean": -472.8544616699219, + "KL/rejected_KL_mean": -586.6543579101562, + "KL/std": 266.8034973144531, + "epoch": 0.5095447870778267, + "fcm_dpo/beta": 0.001710890093818307, + "fcm_dpo/delta": 0.013139501214027405, + "fcm_dpo/margin": 227.59982299804688, + "fcm_dpo/q_t": 0.4103218615055084, + "grad_norm": 22.08174705505371, + "learning_rate": 2.8707927131383614e-07, + "logits/chosen": -0.5109871625900269, + "logits/rejected": -0.5056133270263672, + "logps/chosen": -416.62078857421875, + "logps/ref_chosen": -57.56623840332031, + "logps/ref_rejected": -92.35509490966797, + "logps/rejected": -679.0094604492188, + "loss": 1.095, + "margin_dpo/margin_mean": 227.59979248046875, + "margin_dpo/margin_std": 297.329833984375, + "step": 347 + }, + { + "KL/chosen_KL_mean": -317.605712890625, + "KL/mean": -419.5076904296875, + "KL/rejected_KL_mean": -521.40966796875, + "KL/std": 239.0012969970703, + "epoch": 0.5110132158590308, + "fcm_dpo/beta": 0.0017384930979460478, + "fcm_dpo/delta": 0.0538782924413681, + "fcm_dpo/margin": 203.80392456054688, + "fcm_dpo/q_t": 0.41967394948005676, + "grad_norm": 22.87740707397461, + "learning_rate": 2.858096518347179e-07, + "logits/chosen": -0.5410622358322144, + "logits/rejected": -0.5414994359016418, + "logps/chosen": -373.9234313964844, + "logps/ref_chosen": -56.31770324707031, + "logps/ref_rejected": -89.13836669921875, + "logps/rejected": -610.5479736328125, + "loss": 1.122, + "margin_dpo/margin_mean": 203.80392456054688, + "margin_dpo/margin_std": 276.4021301269531, + "step": 348 + }, + { + "KL/chosen_KL_mean": -315.38214111328125, + "KL/mean": -426.52508544921875, + "KL/rejected_KL_mean": -537.6680297851562, + "KL/std": 259.76336669921875, + "epoch": 0.5124816446402349, + "fcm_dpo/beta": 0.0018351702019572258, + "fcm_dpo/delta": -0.011926159262657166, + "fcm_dpo/margin": 222.285888671875, + "fcm_dpo/q_t": 0.407881498336792, + "grad_norm": 21.936599731445312, + "learning_rate": 2.845390887379706e-07, + "logits/chosen": -0.5372369289398193, + "logits/rejected": -0.5374791622161865, + "logps/chosen": -373.40765380859375, + "logps/ref_chosen": -58.025516510009766, + "logps/ref_rejected": -97.50515747070312, + "logps/rejected": -635.1732177734375, + "loss": 1.1019, + "margin_dpo/margin_mean": 222.28587341308594, + "margin_dpo/margin_std": 322.0589904785156, + "step": 349 + }, + { + "KL/chosen_KL_mean": -336.73089599609375, + "KL/mean": -447.26806640625, + "KL/rejected_KL_mean": -557.8052978515625, + "KL/std": 256.64642333984375, + "epoch": 0.5139500734214391, + "fcm_dpo/beta": 0.001765610184520483, + "fcm_dpo/delta": 0.010335829108953476, + "fcm_dpo/margin": 221.074462890625, + "fcm_dpo/q_t": 0.40952080488204956, + "grad_norm": 30.984949111938477, + "learning_rate": 2.8326761550411346e-07, + "logits/chosen": -0.5725632905960083, + "logits/rejected": -0.5793622136116028, + "logps/chosen": -401.0613708496094, + "logps/ref_chosen": -64.33049011230469, + "logps/ref_rejected": -89.87164306640625, + "logps/rejected": -647.677001953125, + "loss": 1.1081, + "margin_dpo/margin_mean": 221.074462890625, + "margin_dpo/margin_std": 314.63995361328125, + "step": 350 + }, + { + "KL/chosen_KL_mean": -301.34967041015625, + "KL/mean": -436.94683837890625, + "KL/rejected_KL_mean": -572.5440673828125, + "KL/std": 292.54278564453125, + "epoch": 0.5154185022026432, + "fcm_dpo/beta": 0.0016822493635118008, + "fcm_dpo/delta": -0.07549858838319778, + "fcm_dpo/margin": 271.1943664550781, + "fcm_dpo/q_t": 0.3937299847602844, + "grad_norm": 28.201416015625, + "learning_rate": 2.819952656376487e-07, + "logits/chosen": -0.5415765047073364, + "logits/rejected": -0.5444436073303223, + "logps/chosen": -362.02178955078125, + "logps/ref_chosen": -60.6721305847168, + "logps/ref_rejected": -101.5654296875, + "logps/rejected": -674.1094970703125, + "loss": 1.0575, + "margin_dpo/margin_mean": 271.19439697265625, + "margin_dpo/margin_std": 342.4002685546875, + "step": 351 + }, + { + "KL/chosen_KL_mean": -353.643798828125, + "KL/mean": -438.07720947265625, + "KL/rejected_KL_mean": -522.5106811523438, + "KL/std": 269.5029602050781, + "epoch": 0.5168869309838473, + "fcm_dpo/beta": 0.0018008626066148281, + "fcm_dpo/delta": 0.11381582170724869, + "fcm_dpo/margin": 168.86691284179688, + "fcm_dpo/q_t": 0.4331480860710144, + "grad_norm": 61.400474548339844, + "learning_rate": 2.8072207266617854e-07, + "logits/chosen": -0.5702152252197266, + "logits/rejected": -0.5352950096130371, + "logps/chosen": -424.58721923828125, + "logps/ref_chosen": -70.9434585571289, + "logps/ref_rejected": -76.6419677734375, + "logps/rejected": -599.1526489257812, + "loss": 1.1724, + "margin_dpo/margin_mean": 168.86691284179688, + "margin_dpo/margin_std": 291.0610656738281, + "step": 352 + }, + { + "KL/chosen_KL_mean": -321.31304931640625, + "KL/mean": -428.07708740234375, + "KL/rejected_KL_mean": -534.8411865234375, + "KL/std": 256.8054504394531, + "epoch": 0.5183553597650514, + "fcm_dpo/beta": 0.0018330267630517483, + "fcm_dpo/delta": 0.006989631801843643, + "fcm_dpo/margin": 213.528076171875, + "fcm_dpo/q_t": 0.4106243848800659, + "grad_norm": 29.105669021606445, + "learning_rate": 2.794480701395219e-07, + "logits/chosen": -0.6194638609886169, + "logits/rejected": -0.604763925075531, + "logps/chosen": -379.7083740234375, + "logps/ref_chosen": -58.39533996582031, + "logps/ref_rejected": -80.33553314208984, + "logps/rejected": -615.1766967773438, + "loss": 1.104, + "margin_dpo/margin_mean": 213.528076171875, + "margin_dpo/margin_std": 291.9771728515625, + "step": 353 + }, + { + "KL/chosen_KL_mean": -276.51214599609375, + "KL/mean": -391.6829833984375, + "KL/rejected_KL_mean": -506.8538513183594, + "KL/std": 234.94125366210938, + "epoch": 0.5198237885462555, + "fcm_dpo/beta": 0.0018385781440883875, + "fcm_dpo/delta": -0.030174316838383675, + "fcm_dpo/margin": 230.3416748046875, + "fcm_dpo/q_t": 0.39935770630836487, + "grad_norm": 32.30131149291992, + "learning_rate": 2.781732916288303e-07, + "logits/chosen": -0.6025009155273438, + "logits/rejected": -0.5897366404533386, + "logps/chosen": -336.3151550292969, + "logps/ref_chosen": -59.80299377441406, + "logps/ref_rejected": -88.75750732421875, + "logps/rejected": -595.611328125, + "loss": 1.0512, + "margin_dpo/margin_mean": 230.3416748046875, + "margin_dpo/margin_std": 236.0069580078125, + "step": 354 + }, + { + "KL/chosen_KL_mean": -267.94769287109375, + "KL/mean": -392.4739685058594, + "KL/rejected_KL_mean": -517.000244140625, + "KL/std": 249.90625, + "epoch": 0.5212922173274597, + "fcm_dpo/beta": 0.0017431924352422357, + "fcm_dpo/delta": -0.045911140739917755, + "fcm_dpo/margin": 249.0526123046875, + "fcm_dpo/q_t": 0.3952483534812927, + "grad_norm": 41.20962905883789, + "learning_rate": 2.7689777072570284e-07, + "logits/chosen": -0.6518046855926514, + "logits/rejected": -0.6433999538421631, + "logps/chosen": -322.076171875, + "logps/ref_chosen": -54.12849807739258, + "logps/ref_rejected": -82.40606689453125, + "logps/rejected": -599.4063110351562, + "loss": 1.0418, + "margin_dpo/margin_mean": 249.05262756347656, + "margin_dpo/margin_std": 242.5153045654297, + "step": 355 + }, + { + "KL/chosen_KL_mean": -348.6526184082031, + "KL/mean": -410.7809753417969, + "KL/rejected_KL_mean": -472.9093322753906, + "KL/std": 255.02890014648438, + "epoch": 0.5227606461086637, + "fcm_dpo/beta": 0.0017826826078817248, + "fcm_dpo/delta": 0.03428832069039345, + "fcm_dpo/margin": 124.2567367553711, + "fcm_dpo/q_t": 0.45051053166389465, + "grad_norm": 29.759109497070312, + "learning_rate": 2.7562154104130176e-07, + "logits/chosen": -0.5975973010063171, + "logits/rejected": -0.57161945104599, + "logps/chosen": -413.326416015625, + "logps/ref_chosen": -64.6738052368164, + "logps/ref_rejected": -75.89926147460938, + "logps/rejected": -548.80859375, + "loss": 1.2561, + "margin_dpo/margin_mean": 124.25672912597656, + "margin_dpo/margin_std": 320.7979736328125, + "step": 356 + }, + { + "KL/chosen_KL_mean": -309.07611083984375, + "KL/mean": -413.7547607421875, + "KL/rejected_KL_mean": -518.4334106445312, + "KL/std": 256.1674499511719, + "epoch": 0.5242290748898678, + "fcm_dpo/beta": 0.0018109192606061697, + "fcm_dpo/delta": 0.02454628050327301, + "fcm_dpo/margin": 209.35728454589844, + "fcm_dpo/q_t": 0.41207998991012573, + "grad_norm": 25.09831428527832, + "learning_rate": 2.7434463620546594e-07, + "logits/chosen": -0.5878846645355225, + "logits/rejected": -0.5737650990486145, + "logps/chosen": -361.80194091796875, + "logps/ref_chosen": -52.725799560546875, + "logps/ref_rejected": -86.84115600585938, + "logps/rejected": -605.2745361328125, + "loss": 1.0991, + "margin_dpo/margin_mean": 209.3572998046875, + "margin_dpo/margin_std": 263.3780517578125, + "step": 357 + }, + { + "KL/chosen_KL_mean": -287.3702697753906, + "KL/mean": -386.8647155761719, + "KL/rejected_KL_mean": -486.3591613769531, + "KL/std": 241.16976928710938, + "epoch": 0.5256975036710719, + "fcm_dpo/beta": 0.0018941482994705439, + "fcm_dpo/delta": 0.0284799225628376, + "fcm_dpo/margin": 198.98886108398438, + "fcm_dpo/q_t": 0.4145725965499878, + "grad_norm": 26.890031814575195, + "learning_rate": 2.730670898658255e-07, + "logits/chosen": -0.5883272886276245, + "logits/rejected": -0.5693084001541138, + "logps/chosen": -350.5757141113281, + "logps/ref_chosen": -63.20543670654297, + "logps/ref_rejected": -88.373291015625, + "logps/rejected": -574.732421875, + "loss": 1.1049, + "margin_dpo/margin_mean": 198.98886108398438, + "margin_dpo/margin_std": 268.44134521484375, + "step": 358 + }, + { + "KL/chosen_KL_mean": -328.99090576171875, + "KL/mean": -444.7538146972656, + "KL/rejected_KL_mean": -560.5167236328125, + "KL/std": 247.1792755126953, + "epoch": 0.527165932452276, + "fcm_dpo/beta": 0.0018442978616803885, + "fcm_dpo/delta": -0.0352255180478096, + "fcm_dpo/margin": 231.5258331298828, + "fcm_dpo/q_t": 0.4013112187385559, + "grad_norm": 33.82530212402344, + "learning_rate": 2.717889356869146e-07, + "logits/chosen": -0.5302337408065796, + "logits/rejected": -0.5221052765846252, + "logps/chosen": -385.3611145019531, + "logps/ref_chosen": -56.370216369628906, + "logps/ref_rejected": -82.17375183105469, + "logps/rejected": -642.6904907226562, + "loss": 1.0792, + "margin_dpo/margin_mean": 231.5258331298828, + "margin_dpo/margin_std": 305.66802978515625, + "step": 359 + }, + { + "KL/chosen_KL_mean": -329.69427490234375, + "KL/mean": -418.23699951171875, + "KL/rejected_KL_mean": -506.77972412109375, + "KL/std": 206.728271484375, + "epoch": 0.5286343612334802, + "fcm_dpo/beta": 0.0019492888823151588, + "fcm_dpo/delta": 0.0655849277973175, + "fcm_dpo/margin": 177.08547973632812, + "fcm_dpo/q_t": 0.420721173286438, + "grad_norm": 44.412681579589844, + "learning_rate": 2.7051020734928443e-07, + "logits/chosen": -0.5013865828514099, + "logits/rejected": -0.48618659377098083, + "logps/chosen": -381.1546630859375, + "logps/ref_chosen": -51.460384368896484, + "logps/ref_rejected": -69.83892059326172, + "logps/rejected": -576.61865234375, + "loss": 1.1116, + "margin_dpo/margin_mean": 177.08547973632812, + "margin_dpo/margin_std": 211.73977661132812, + "step": 360 + }, + { + "KL/chosen_KL_mean": -342.9272766113281, + "KL/mean": -429.4185791015625, + "KL/rejected_KL_mean": -515.9098510742188, + "KL/std": 240.47056579589844, + "epoch": 0.5301027900146843, + "fcm_dpo/beta": 0.0020752213895320892, + "fcm_dpo/delta": 0.047938957810401917, + "fcm_dpo/margin": 172.98260498046875, + "fcm_dpo/q_t": 0.41855740547180176, + "grad_norm": 35.09357833862305, + "learning_rate": 2.6923093854861593e-07, + "logits/chosen": -0.5015807747840881, + "logits/rejected": -0.49445807933807373, + "logps/chosen": -396.79681396484375, + "logps/ref_chosen": -53.86951446533203, + "logps/ref_rejected": -90.7692642211914, + "logps/rejected": -606.6791381835938, + "loss": 1.1306, + "margin_dpo/margin_mean": 172.9825897216797, + "margin_dpo/margin_std": 263.9880676269531, + "step": 361 + }, + { + "KL/chosen_KL_mean": -294.02984619140625, + "KL/mean": -431.71832275390625, + "KL/rejected_KL_mean": -569.40673828125, + "KL/std": 248.303955078125, + "epoch": 0.5315712187958884, + "fcm_dpo/beta": 0.0018198469188064337, + "fcm_dpo/delta": -0.14192384481430054, + "fcm_dpo/margin": 275.376953125, + "fcm_dpo/q_t": 0.37714123725891113, + "grad_norm": 30.883852005004883, + "learning_rate": 2.679511629948319e-07, + "logits/chosen": -0.5016046166419983, + "logits/rejected": -0.5126606225967407, + "logps/chosen": -352.66888427734375, + "logps/ref_chosen": -58.639060974121094, + "logps/ref_rejected": -105.58195495605469, + "logps/rejected": -674.9887084960938, + "loss": 0.9979, + "margin_dpo/margin_mean": 275.376953125, + "margin_dpo/margin_std": 260.5057373046875, + "step": 362 + }, + { + "KL/chosen_KL_mean": -272.70916748046875, + "KL/mean": -416.3315124511719, + "KL/rejected_KL_mean": -559.953857421875, + "KL/std": 247.9210205078125, + "epoch": 0.5330396475770925, + "fcm_dpo/beta": 0.0016754900570958853, + "fcm_dpo/delta": -0.10649609565734863, + "fcm_dpo/margin": 287.24468994140625, + "fcm_dpo/q_t": 0.3827672302722931, + "grad_norm": 27.67137908935547, + "learning_rate": 2.6667091441120816e-07, + "logits/chosen": -0.4572371542453766, + "logits/rejected": -0.4535822868347168, + "logps/chosen": -317.26751708984375, + "logps/ref_chosen": -44.558380126953125, + "logps/ref_rejected": -74.69496154785156, + "logps/rejected": -634.6488037109375, + "loss": 1.0097, + "margin_dpo/margin_mean": 287.24468994140625, + "margin_dpo/margin_std": 268.51934814453125, + "step": 363 + }, + { + "KL/chosen_KL_mean": -304.93585205078125, + "KL/mean": -410.62762451171875, + "KL/rejected_KL_mean": -516.3193969726562, + "KL/std": 243.23239135742188, + "epoch": 0.5345080763582967, + "fcm_dpo/beta": 0.0017072297632694244, + "fcm_dpo/delta": 0.043868713080883026, + "fcm_dpo/margin": 211.38351440429688, + "fcm_dpo/q_t": 0.4163602590560913, + "grad_norm": 26.54780387878418, + "learning_rate": 2.6539022653348575e-07, + "logits/chosen": -0.4880487322807312, + "logits/rejected": -0.49908387660980225, + "logps/chosen": -353.8304443359375, + "logps/ref_chosen": -48.894622802734375, + "logps/ref_rejected": -91.395751953125, + "logps/rejected": -607.7151489257812, + "loss": 1.1142, + "margin_dpo/margin_mean": 211.38351440429688, + "margin_dpo/margin_std": 280.0267028808594, + "step": 364 + }, + { + "KL/chosen_KL_mean": -315.26373291015625, + "KL/mean": -426.0964050292969, + "KL/rejected_KL_mean": -536.9290771484375, + "KL/std": 257.34033203125, + "epoch": 0.5359765051395007, + "fcm_dpo/beta": 0.0017189306672662497, + "fcm_dpo/delta": 0.0233314111828804, + "fcm_dpo/margin": 221.66537475585938, + "fcm_dpo/q_t": 0.41273248195648193, + "grad_norm": 22.404428482055664, + "learning_rate": 2.641091331089811e-07, + "logits/chosen": -0.5078902244567871, + "logits/rejected": -0.5193514227867126, + "logps/chosen": -366.7564697265625, + "logps/ref_chosen": -51.49274444580078, + "logps/ref_rejected": -92.70166778564453, + "logps/rejected": -629.6307373046875, + "loss": 1.0905, + "margin_dpo/margin_mean": 221.66537475585938, + "margin_dpo/margin_std": 264.7396240234375, + "step": 365 + }, + { + "KL/chosen_KL_mean": -298.6568603515625, + "KL/mean": -417.4523010253906, + "KL/rejected_KL_mean": -536.2477416992188, + "KL/std": 257.2602233886719, + "epoch": 0.5374449339207048, + "fcm_dpo/beta": 0.0016834220150485635, + "fcm_dpo/delta": -0.002375733107328415, + "fcm_dpo/margin": 237.59083557128906, + "fcm_dpo/q_t": 0.4071364402770996, + "grad_norm": 26.219554901123047, + "learning_rate": 2.6282766789569736e-07, + "logits/chosen": -0.49354439973831177, + "logits/rejected": -0.507148027420044, + "logps/chosen": -343.37744140625, + "logps/ref_chosen": -44.7205696105957, + "logps/ref_rejected": -83.31040954589844, + "logps/rejected": -619.55810546875, + "loss": 1.0866, + "margin_dpo/margin_mean": 237.59083557128906, + "margin_dpo/margin_std": 297.007080078125, + "step": 366 + }, + { + "KL/chosen_KL_mean": -325.56622314453125, + "KL/mean": -423.55853271484375, + "KL/rejected_KL_mean": -521.5508422851562, + "KL/std": 235.90292358398438, + "epoch": 0.5389133627019089, + "fcm_dpo/beta": 0.001811149064451456, + "fcm_dpo/delta": 0.053424857556819916, + "fcm_dpo/margin": 195.98464965820312, + "fcm_dpo/q_t": 0.41807931661605835, + "grad_norm": 25.911218643188477, + "learning_rate": 2.615458646614349e-07, + "logits/chosen": -0.5182558298110962, + "logits/rejected": -0.5040343999862671, + "logps/chosen": -383.97161865234375, + "logps/ref_chosen": -58.405418395996094, + "logps/ref_rejected": -76.75132751464844, + "logps/rejected": -598.3021850585938, + "loss": 1.1129, + "margin_dpo/margin_mean": 195.98464965820312, + "margin_dpo/margin_std": 253.58770751953125, + "step": 367 + }, + { + "KL/chosen_KL_mean": -320.83441162109375, + "KL/mean": -478.84674072265625, + "KL/rejected_KL_mean": -636.8590087890625, + "KL/std": 274.4027099609375, + "epoch": 0.540381791483113, + "fcm_dpo/beta": 0.001620815135538578, + "fcm_dpo/delta": -0.1518523395061493, + "fcm_dpo/margin": 316.0246276855469, + "fcm_dpo/q_t": 0.3704509735107422, + "grad_norm": 30.158796310424805, + "learning_rate": 2.6026375718290083e-07, + "logits/chosen": -0.5701684951782227, + "logits/rejected": -0.5831023454666138, + "logps/chosen": -365.28692626953125, + "logps/ref_chosen": -44.452518463134766, + "logps/ref_rejected": -98.55526733398438, + "logps/rejected": -735.414306640625, + "loss": 0.9731, + "margin_dpo/margin_mean": 316.02459716796875, + "margin_dpo/margin_std": 235.96087646484375, + "step": 368 + }, + { + "KL/chosen_KL_mean": -417.2987976074219, + "KL/mean": -500.9288330078125, + "KL/rejected_KL_mean": -584.5588989257812, + "KL/std": 269.81658935546875, + "epoch": 0.5418502202643172, + "fcm_dpo/beta": 0.0015922598540782928, + "fcm_dpo/delta": 0.041051216423511505, + "fcm_dpo/margin": 167.26007080078125, + "fcm_dpo/q_t": 0.4384276270866394, + "grad_norm": 24.34749984741211, + "learning_rate": 2.589813792448196e-07, + "logits/chosen": -0.5948071479797363, + "logits/rejected": -0.575736403465271, + "logps/chosen": -488.6802978515625, + "logps/ref_chosen": -71.38150024414062, + "logps/ref_rejected": -91.29582214355469, + "logps/rejected": -675.854736328125, + "loss": 1.1934, + "margin_dpo/margin_mean": 167.26007080078125, + "margin_dpo/margin_std": 302.8238525390625, + "step": 369 + }, + { + "KL/chosen_KL_mean": -449.61785888671875, + "KL/mean": -520.163330078125, + "KL/rejected_KL_mean": -590.7088623046875, + "KL/std": 294.0730895996094, + "epoch": 0.5433186490455213, + "fcm_dpo/beta": 0.0016860844334587455, + "fcm_dpo/delta": 0.07256323844194412, + "fcm_dpo/margin": 141.0910186767578, + "fcm_dpo/q_t": 0.4467887282371521, + "grad_norm": 34.77388000488281, + "learning_rate": 2.5769876463904263e-07, + "logits/chosen": -0.6763471364974976, + "logits/rejected": -0.6677573323249817, + "logps/chosen": -521.225341796875, + "logps/ref_chosen": -71.60749816894531, + "logps/ref_rejected": -97.25978088378906, + "logps/rejected": -687.9686279296875, + "loss": 1.2396, + "margin_dpo/margin_mean": 141.0910186767578, + "margin_dpo/margin_std": 338.9228515625, + "step": 370 + }, + { + "KL/chosen_KL_mean": -435.55035400390625, + "KL/mean": -551.7735595703125, + "KL/rejected_KL_mean": -667.9967041015625, + "KL/std": 304.8518371582031, + "epoch": 0.5447870778267254, + "fcm_dpo/beta": 0.0017521032132208347, + "fcm_dpo/delta": -0.009448290802538395, + "fcm_dpo/margin": 232.4463348388672, + "fcm_dpo/q_t": 0.40719670057296753, + "grad_norm": 27.79642105102539, + "learning_rate": 2.5641594716365744e-07, + "logits/chosen": -0.7335249185562134, + "logits/rejected": -0.720642626285553, + "logps/chosen": -504.96484375, + "logps/ref_chosen": -69.41448974609375, + "logps/ref_rejected": -99.17217254638672, + "logps/rejected": -767.1689453125, + "loss": 1.099, + "margin_dpo/margin_mean": 232.4463348388672, + "margin_dpo/margin_std": 332.423095703125, + "step": 371 + }, + { + "KL/chosen_KL_mean": -432.0025939941406, + "KL/mean": -568.457763671875, + "KL/rejected_KL_mean": -704.9129638671875, + "KL/std": 349.4569396972656, + "epoch": 0.5462555066079295, + "fcm_dpo/beta": 0.001637202687561512, + "fcm_dpo/delta": -0.061564333736896515, + "fcm_dpo/margin": 272.91033935546875, + "fcm_dpo/q_t": 0.3999387323856354, + "grad_norm": 26.669132232666016, + "learning_rate": 2.551329606220976e-07, + "logits/chosen": -0.7301241755485535, + "logits/rejected": -0.7251428365707397, + "logps/chosen": -493.8205871582031, + "logps/ref_chosen": -61.8179931640625, + "logps/ref_rejected": -78.53948974609375, + "logps/rejected": -783.452392578125, + "loss": 1.0802, + "margin_dpo/margin_mean": 272.9103088378906, + "margin_dpo/margin_std": 398.3546142578125, + "step": 372 + }, + { + "KL/chosen_KL_mean": -448.3118896484375, + "KL/mean": -581.019775390625, + "KL/rejected_KL_mean": -713.7276000976562, + "KL/std": 327.57122802734375, + "epoch": 0.5477239353891337, + "fcm_dpo/beta": 0.0016204612329602242, + "fcm_dpo/delta": -0.039967313408851624, + "fcm_dpo/margin": 265.41571044921875, + "fcm_dpo/q_t": 0.39898842573165894, + "grad_norm": 32.85871505737305, + "learning_rate": 2.538498388222517e-07, + "logits/chosen": -0.7037230730056763, + "logits/rejected": -0.6890777349472046, + "logps/chosen": -512.529052734375, + "logps/ref_chosen": -64.21713256835938, + "logps/ref_rejected": -85.95960998535156, + "logps/rejected": -799.687255859375, + "loss": 1.0727, + "margin_dpo/margin_mean": 265.41571044921875, + "margin_dpo/margin_std": 323.6532897949219, + "step": 373 + }, + { + "KL/chosen_KL_mean": -407.01507568359375, + "KL/mean": -524.6700439453125, + "KL/rejected_KL_mean": -642.324951171875, + "KL/std": 347.67022705078125, + "epoch": 0.5491923641703378, + "fcm_dpo/beta": 0.0015636862954124808, + "fcm_dpo/delta": 0.03692319244146347, + "fcm_dpo/margin": 235.3098602294922, + "fcm_dpo/q_t": 0.42001497745513916, + "grad_norm": 27.749187469482422, + "learning_rate": 2.525666155755725e-07, + "logits/chosen": -0.7614662647247314, + "logits/rejected": -0.7477431297302246, + "logps/chosen": -477.6652526855469, + "logps/ref_chosen": -70.65018463134766, + "logps/ref_rejected": -93.64016723632812, + "logps/rejected": -735.965087890625, + "loss": 1.1424, + "margin_dpo/margin_mean": 235.3098602294922, + "margin_dpo/margin_std": 396.4349060058594, + "step": 374 + }, + { + "KL/chosen_KL_mean": -424.40948486328125, + "KL/mean": -535.830078125, + "KL/rejected_KL_mean": -647.2506713867188, + "KL/std": 291.63531494140625, + "epoch": 0.5506607929515418, + "fcm_dpo/beta": 0.0016217189840972424, + "fcm_dpo/delta": 0.04367804527282715, + "fcm_dpo/margin": 222.84117126464844, + "fcm_dpo/q_t": 0.418157160282135, + "grad_norm": 41.795082092285156, + "learning_rate": 2.512833246961859e-07, + "logits/chosen": -0.6727270483970642, + "logits/rejected": -0.6715774536132812, + "logps/chosen": -484.4897155761719, + "logps/ref_chosen": -60.080223083496094, + "logps/ref_rejected": -88.93830871582031, + "logps/rejected": -736.18896484375, + "loss": 1.1357, + "margin_dpo/margin_mean": 222.84115600585938, + "margin_dpo/margin_std": 338.76904296875, + "step": 375 + }, + { + "KL/chosen_KL_mean": -414.8541259765625, + "KL/mean": -552.9400634765625, + "KL/rejected_KL_mean": -691.0260009765625, + "KL/std": 303.83843994140625, + "epoch": 0.5521292217327459, + "fcm_dpo/beta": 0.0016104718670248985, + "fcm_dpo/delta": -0.058481134474277496, + "fcm_dpo/margin": 276.171875, + "fcm_dpo/q_t": 0.39490610361099243, + "grad_norm": 31.177749633789062, + "learning_rate": 2.5e-07, + "logits/chosen": -0.7422023415565491, + "logits/rejected": -0.7344397306442261, + "logps/chosen": -477.5144348144531, + "logps/ref_chosen": -62.660308837890625, + "logps/ref_rejected": -105.52660369873047, + "logps/rejected": -796.5526123046875, + "loss": 1.0616, + "margin_dpo/margin_mean": 276.171875, + "margin_dpo/margin_std": 342.929443359375, + "step": 376 + }, + { + "KL/chosen_KL_mean": -424.4516906738281, + "KL/mean": -556.843994140625, + "KL/rejected_KL_mean": -689.2363891601562, + "KL/std": 344.01544189453125, + "epoch": 0.55359765051395, + "fcm_dpo/beta": 0.0015855635283514857, + "fcm_dpo/delta": -0.02529209665954113, + "fcm_dpo/margin": 264.78466796875, + "fcm_dpo/q_t": 0.40445077419281006, + "grad_norm": 25.540964126586914, + "learning_rate": 2.487166753038141e-07, + "logits/chosen": -0.6894611120223999, + "logits/rejected": -0.6936970949172974, + "logps/chosen": -478.930419921875, + "logps/ref_chosen": -54.478736877441406, + "logps/ref_rejected": -98.70335388183594, + "logps/rejected": -787.939697265625, + "loss": 1.0868, + "margin_dpo/margin_mean": 264.78466796875, + "margin_dpo/margin_std": 363.934326171875, + "step": 377 + }, + { + "KL/chosen_KL_mean": -386.81268310546875, + "KL/mean": -535.3828735351562, + "KL/rejected_KL_mean": -683.9530029296875, + "KL/std": 294.50201416015625, + "epoch": 0.5550660792951542, + "fcm_dpo/beta": 0.001496739685535431, + "fcm_dpo/delta": -0.057502150535583496, + "fcm_dpo/margin": 297.14031982421875, + "fcm_dpo/q_t": 0.3941471576690674, + "grad_norm": 38.976097106933594, + "learning_rate": 2.4743338442442754e-07, + "logits/chosen": -0.7334337830543518, + "logits/rejected": -0.7559584379196167, + "logps/chosen": -431.8332214355469, + "logps/ref_chosen": -45.02053451538086, + "logps/ref_rejected": -88.0469741821289, + "logps/rejected": -772.0, + "loss": 1.0487, + "margin_dpo/margin_mean": 297.14031982421875, + "margin_dpo/margin_std": 334.1382141113281, + "step": 378 + }, + { + "KL/chosen_KL_mean": -417.34716796875, + "KL/mean": -559.3892211914062, + "KL/rejected_KL_mean": -701.4312744140625, + "KL/std": 313.11883544921875, + "epoch": 0.5565345080763583, + "fcm_dpo/beta": 0.001429541502147913, + "fcm_dpo/delta": -0.010058283805847168, + "fcm_dpo/margin": 284.0840759277344, + "fcm_dpo/q_t": 0.4058372974395752, + "grad_norm": 29.72431182861328, + "learning_rate": 2.461501611777483e-07, + "logits/chosen": -0.6593658328056335, + "logits/rejected": -0.6784616112709045, + "logps/chosen": -470.52923583984375, + "logps/ref_chosen": -53.182098388671875, + "logps/ref_rejected": -114.3001708984375, + "logps/rejected": -815.7314453125, + "loss": 1.0836, + "margin_dpo/margin_mean": 284.0840759277344, + "margin_dpo/margin_std": 356.0497131347656, + "step": 379 + }, + { + "KL/chosen_KL_mean": -403.3964538574219, + "KL/mean": -557.42529296875, + "KL/rejected_KL_mean": -711.4541625976562, + "KL/std": 354.645751953125, + "epoch": 0.5580029368575624, + "fcm_dpo/beta": 0.0014169125352054834, + "fcm_dpo/delta": -0.04655863344669342, + "fcm_dpo/margin": 308.05767822265625, + "fcm_dpo/q_t": 0.39721158146858215, + "grad_norm": 28.100101470947266, + "learning_rate": 2.4486703937790243e-07, + "logits/chosen": -0.632209300994873, + "logits/rejected": -0.664146900177002, + "logps/chosen": -454.7494812011719, + "logps/ref_chosen": -51.3530387878418, + "logps/ref_rejected": -104.19169616699219, + "logps/rejected": -815.6458740234375, + "loss": 1.0674, + "margin_dpo/margin_mean": 308.05767822265625, + "margin_dpo/margin_std": 388.6807861328125, + "step": 380 + }, + { + "KL/chosen_KL_mean": -407.2333068847656, + "KL/mean": -519.652587890625, + "KL/rejected_KL_mean": -632.0718994140625, + "KL/std": 297.33251953125, + "epoch": 0.5594713656387665, + "fcm_dpo/beta": 0.0014676698483526707, + "fcm_dpo/delta": 0.08423132449388504, + "fcm_dpo/margin": 224.83859252929688, + "fcm_dpo/q_t": 0.4268224537372589, + "grad_norm": 25.78670883178711, + "learning_rate": 2.435840528363426e-07, + "logits/chosen": -0.6134881973266602, + "logits/rejected": -0.5925121307373047, + "logps/chosen": -465.036376953125, + "logps/ref_chosen": -57.80306625366211, + "logps/ref_rejected": -79.21940612792969, + "logps/rejected": -711.291259765625, + "loss": 1.1633, + "margin_dpo/margin_mean": 224.83859252929688, + "margin_dpo/margin_std": 391.60101318359375, + "step": 381 + }, + { + "KL/chosen_KL_mean": -392.236572265625, + "KL/mean": -526.9730834960938, + "KL/rejected_KL_mean": -661.7095947265625, + "KL/std": 286.292724609375, + "epoch": 0.5609397944199707, + "fcm_dpo/beta": 0.0015065961051732302, + "fcm_dpo/delta": -0.007509663701057434, + "fcm_dpo/margin": 269.4730529785156, + "fcm_dpo/q_t": 0.40533730387687683, + "grad_norm": 28.11603546142578, + "learning_rate": 2.4230123536095745e-07, + "logits/chosen": -0.6188876628875732, + "logits/rejected": -0.6265856027603149, + "logps/chosen": -458.2568359375, + "logps/ref_chosen": -66.02030181884766, + "logps/ref_rejected": -110.71016693115234, + "logps/rejected": -772.4197387695312, + "loss": 1.0722, + "margin_dpo/margin_mean": 269.4730224609375, + "margin_dpo/margin_std": 310.5892333984375, + "step": 382 + }, + { + "KL/chosen_KL_mean": -413.4278869628906, + "KL/mean": -549.6930541992188, + "KL/rejected_KL_mean": -685.958251953125, + "KL/std": 322.25799560546875, + "epoch": 0.5624082232011748, + "fcm_dpo/beta": 0.0014835065230727196, + "fcm_dpo/delta": -0.005721503868699074, + "fcm_dpo/margin": 272.53033447265625, + "fcm_dpo/q_t": 0.4068153500556946, + "grad_norm": 22.861156463623047, + "learning_rate": 2.4101862075518037e-07, + "logits/chosen": -0.5609561204910278, + "logits/rejected": -0.5730615854263306, + "logps/chosen": -463.81939697265625, + "logps/ref_chosen": -50.39148712158203, + "logps/ref_rejected": -93.71589660644531, + "logps/rejected": -779.674072265625, + "loss": 1.1028, + "margin_dpo/margin_mean": 272.53033447265625, + "margin_dpo/margin_std": 395.47784423828125, + "step": 383 + }, + { + "KL/chosen_KL_mean": -410.1112060546875, + "KL/mean": -516.384033203125, + "KL/rejected_KL_mean": -622.6568603515625, + "KL/std": 280.2078857421875, + "epoch": 0.5638766519823789, + "fcm_dpo/beta": 0.001605054596439004, + "fcm_dpo/delta": 0.06686155498027802, + "fcm_dpo/margin": 212.545654296875, + "fcm_dpo/q_t": 0.42119306325912476, + "grad_norm": 24.116987228393555, + "learning_rate": 2.397362428170992e-07, + "logits/chosen": -0.5835964679718018, + "logits/rejected": -0.5749599933624268, + "logps/chosen": -462.15728759765625, + "logps/ref_chosen": -52.046104431152344, + "logps/ref_rejected": -85.76089477539062, + "logps/rejected": -708.417724609375, + "loss": 1.1212, + "margin_dpo/margin_mean": 212.54566955566406, + "margin_dpo/margin_std": 272.0644226074219, + "step": 384 + }, + { + "KL/chosen_KL_mean": -378.491455078125, + "KL/mean": -509.3629150390625, + "KL/rejected_KL_mean": -640.234375, + "KL/std": 248.0293426513672, + "epoch": 0.5653450807635829, + "fcm_dpo/beta": 0.0015787326265126467, + "fcm_dpo/delta": -0.016653146594762802, + "fcm_dpo/margin": 261.7428894042969, + "fcm_dpo/q_t": 0.4017704725265503, + "grad_norm": 38.663368225097656, + "learning_rate": 2.3845413533856514e-07, + "logits/chosen": -0.6341022253036499, + "logits/rejected": -0.6096962690353394, + "logps/chosen": -444.04364013671875, + "logps/ref_chosen": -65.55215454101562, + "logps/ref_rejected": -77.82792663574219, + "logps/rejected": -718.062255859375, + "loss": 1.0631, + "margin_dpo/margin_mean": 261.7428894042969, + "margin_dpo/margin_std": 283.84356689453125, + "step": 385 + }, + { + "KL/chosen_KL_mean": -397.4541015625, + "KL/mean": -538.3778686523438, + "KL/rejected_KL_mean": -679.3016357421875, + "KL/std": 300.32208251953125, + "epoch": 0.566813509544787, + "fcm_dpo/beta": 0.0015297075733542442, + "fcm_dpo/delta": -0.039631377905607224, + "fcm_dpo/margin": 281.8475341796875, + "fcm_dpo/q_t": 0.39953434467315674, + "grad_norm": 25.86907196044922, + "learning_rate": 2.3717233210430254e-07, + "logits/chosen": -0.5929363965988159, + "logits/rejected": -0.5882759094238281, + "logps/chosen": -455.67596435546875, + "logps/ref_chosen": -58.22185516357422, + "logps/ref_rejected": -92.32742309570312, + "logps/rejected": -771.6290283203125, + "loss": 1.0695, + "margin_dpo/margin_mean": 281.8475341796875, + "margin_dpo/margin_std": 355.7461242675781, + "step": 386 + }, + { + "KL/chosen_KL_mean": -450.69427490234375, + "KL/mean": -571.585693359375, + "KL/rejected_KL_mean": -692.4771728515625, + "KL/std": 309.3702392578125, + "epoch": 0.5682819383259912, + "fcm_dpo/beta": 0.0015234483871608973, + "fcm_dpo/delta": 0.03800199180841446, + "fcm_dpo/margin": 241.78292846679688, + "fcm_dpo/q_t": 0.4150587320327759, + "grad_norm": 32.03988265991211, + "learning_rate": 2.3589086689101889e-07, + "logits/chosen": -0.6444403529167175, + "logits/rejected": -0.6319328546524048, + "logps/chosen": -517.1137084960938, + "logps/ref_chosen": -66.41944885253906, + "logps/ref_rejected": -92.16915893554688, + "logps/rejected": -784.6463623046875, + "loss": 1.1086, + "margin_dpo/margin_mean": 241.78292846679688, + "margin_dpo/margin_std": 314.6268310546875, + "step": 387 + }, + { + "KL/chosen_KL_mean": -483.2527770996094, + "KL/mean": -631.25, + "KL/rejected_KL_mean": -779.247314453125, + "KL/std": 369.490234375, + "epoch": 0.5697503671071953, + "fcm_dpo/beta": 0.001472370233386755, + "fcm_dpo/delta": -0.05027584731578827, + "fcm_dpo/margin": 295.9945068359375, + "fcm_dpo/q_t": 0.40026527643203735, + "grad_norm": 68.39986419677734, + "learning_rate": 2.3460977346651428e-07, + "logits/chosen": -0.6584256887435913, + "logits/rejected": -0.6686611175537109, + "logps/chosen": -533.3822021484375, + "logps/ref_chosen": -50.129459381103516, + "logps/ref_rejected": -104.43305969238281, + "logps/rejected": -883.6803588867188, + "loss": 1.0768, + "margin_dpo/margin_mean": 295.9945068359375, + "margin_dpo/margin_std": 401.4984130859375, + "step": 388 + }, + { + "KL/chosen_KL_mean": -505.9647521972656, + "KL/mean": -641.0068359375, + "KL/rejected_KL_mean": -776.0489501953125, + "KL/std": 349.13677978515625, + "epoch": 0.5712187958883994, + "fcm_dpo/beta": 0.001461312174797058, + "fcm_dpo/delta": 0.00432373583316803, + "fcm_dpo/margin": 270.084228515625, + "fcm_dpo/q_t": 0.4113960266113281, + "grad_norm": 51.183631896972656, + "learning_rate": 2.3332908558879177e-07, + "logits/chosen": -0.6953232288360596, + "logits/rejected": -0.696498453617096, + "logps/chosen": -563.871337890625, + "logps/ref_chosen": -57.906593322753906, + "logps/ref_rejected": -77.91454315185547, + "logps/rejected": -853.9635009765625, + "loss": 1.1163, + "margin_dpo/margin_mean": 270.084228515625, + "margin_dpo/margin_std": 410.4221496582031, + "step": 389 + }, + { + "KL/chosen_KL_mean": -545.4517211914062, + "KL/mean": -687.8853149414062, + "KL/rejected_KL_mean": -830.3189086914062, + "KL/std": 405.5350646972656, + "epoch": 0.5726872246696035, + "fcm_dpo/beta": 0.0014416101621463895, + "fcm_dpo/delta": -0.01683669537305832, + "fcm_dpo/margin": 284.8671875, + "fcm_dpo/q_t": 0.4116922616958618, + "grad_norm": 30.454208374023438, + "learning_rate": 2.320488370051681e-07, + "logits/chosen": -0.614460825920105, + "logits/rejected": -0.6173591017723083, + "logps/chosen": -594.6776123046875, + "logps/ref_chosen": -49.22591781616211, + "logps/ref_rejected": -85.5281982421875, + "logps/rejected": -915.8471069335938, + "loss": 1.127, + "margin_dpo/margin_mean": 284.8671875, + "margin_dpo/margin_std": 498.7476806640625, + "step": 390 + }, + { + "KL/chosen_KL_mean": -546.4203491210938, + "KL/mean": -634.4364013671875, + "KL/rejected_KL_mean": -722.452392578125, + "KL/std": 375.4483337402344, + "epoch": 0.5741556534508077, + "fcm_dpo/beta": 0.0016221825499087572, + "fcm_dpo/delta": 0.13364286720752716, + "fcm_dpo/margin": 176.031982421875, + "fcm_dpo/q_t": 0.4394233822822571, + "grad_norm": 55.282936096191406, + "learning_rate": 2.3076906145138405e-07, + "logits/chosen": -0.6284788846969604, + "logits/rejected": -0.6224967241287231, + "logps/chosen": -610.75, + "logps/ref_chosen": -64.32965087890625, + "logps/ref_rejected": -86.73820495605469, + "logps/rejected": -809.1905517578125, + "loss": 1.2094, + "margin_dpo/margin_mean": 176.03196716308594, + "margin_dpo/margin_std": 371.1688537597656, + "step": 391 + }, + { + "KL/chosen_KL_mean": -451.90289306640625, + "KL/mean": -614.0924072265625, + "KL/rejected_KL_mean": -776.2818603515625, + "KL/std": 364.5159912109375, + "epoch": 0.5756240822320118, + "fcm_dpo/beta": 0.001549735781736672, + "fcm_dpo/delta": -0.13615985214710236, + "fcm_dpo/margin": 324.3789367675781, + "fcm_dpo/q_t": 0.3801257610321045, + "grad_norm": 34.18647766113281, + "learning_rate": 2.294897926507156e-07, + "logits/chosen": -0.6204630136489868, + "logits/rejected": -0.6229850053787231, + "logps/chosen": -505.4068908691406, + "logps/ref_chosen": -53.50397872924805, + "logps/ref_rejected": -102.34584045410156, + "logps/rejected": -878.627685546875, + "loss": 1.0167, + "margin_dpo/margin_mean": 324.3789367675781, + "margin_dpo/margin_std": 355.14715576171875, + "step": 392 + }, + { + "KL/chosen_KL_mean": -472.4049072265625, + "KL/mean": -620.7923583984375, + "KL/rejected_KL_mean": -769.1798095703125, + "KL/std": 418.1456604003906, + "epoch": 0.5770925110132159, + "fcm_dpo/beta": 0.0014102270361036062, + "fcm_dpo/delta": -0.026089845225214958, + "fcm_dpo/margin": 296.77490234375, + "fcm_dpo/q_t": 0.4100596308708191, + "grad_norm": 26.125350952148438, + "learning_rate": 2.2821106431308543e-07, + "logits/chosen": -0.6747171878814697, + "logits/rejected": -0.6873846054077148, + "logps/chosen": -518.8788452148438, + "logps/ref_chosen": -46.473915100097656, + "logps/ref_rejected": -71.96885681152344, + "logps/rejected": -841.148681640625, + "loss": 1.1204, + "margin_dpo/margin_mean": 296.77490234375, + "margin_dpo/margin_std": 503.2057189941406, + "step": 393 + }, + { + "KL/chosen_KL_mean": -548.73779296875, + "KL/mean": -690.5399169921875, + "KL/rejected_KL_mean": -832.342041015625, + "KL/std": 433.2088623046875, + "epoch": 0.57856093979442, + "fcm_dpo/beta": 0.0014103709254413843, + "fcm_dpo/delta": -0.0008256174623966217, + "fcm_dpo/margin": 283.604248046875, + "fcm_dpo/q_t": 0.4093853831291199, + "grad_norm": 22.54509162902832, + "learning_rate": 2.2693291013417452e-07, + "logits/chosen": -0.64982008934021, + "logits/rejected": -0.654583215713501, + "logps/chosen": -601.6493530273438, + "logps/ref_chosen": -52.91154861450195, + "logps/ref_rejected": -90.8226318359375, + "logps/rejected": -923.1646728515625, + "loss": 1.1035, + "margin_dpo/margin_mean": 283.604248046875, + "margin_dpo/margin_std": 410.2900695800781, + "step": 394 + }, + { + "KL/chosen_KL_mean": -520.23828125, + "KL/mean": -669.6795654296875, + "KL/rejected_KL_mean": -819.120849609375, + "KL/std": 406.344482421875, + "epoch": 0.580029368575624, + "fcm_dpo/beta": 0.0013870133552700281, + "fcm_dpo/delta": -0.019791439175605774, + "fcm_dpo/margin": 298.882568359375, + "fcm_dpo/q_t": 0.40929269790649414, + "grad_norm": 33.52304458618164, + "learning_rate": 2.2565536379453404e-07, + "logits/chosen": -0.7437783479690552, + "logits/rejected": -0.7509876489639282, + "logps/chosen": -582.784423828125, + "logps/ref_chosen": -62.546112060546875, + "logps/ref_rejected": -83.78262329101562, + "logps/rejected": -902.9034423828125, + "loss": 1.1252, + "margin_dpo/margin_mean": 298.882568359375, + "margin_dpo/margin_std": 504.57586669921875, + "step": 395 + }, + { + "KL/chosen_KL_mean": -525.498779296875, + "KL/mean": -663.1034545898438, + "KL/rejected_KL_mean": -800.7081298828125, + "KL/std": 373.2579650878906, + "epoch": 0.5814977973568282, + "fcm_dpo/beta": 0.0014105092268437147, + "fcm_dpo/delta": 0.014646814204752445, + "fcm_dpo/margin": 275.20941162109375, + "fcm_dpo/q_t": 0.4090290665626526, + "grad_norm": 26.613161087036133, + "learning_rate": 2.2437845895869825e-07, + "logits/chosen": -0.7049363851547241, + "logits/rejected": -0.6887962818145752, + "logps/chosen": -594.4947509765625, + "logps/ref_chosen": -68.99594116210938, + "logps/ref_rejected": -88.64665985107422, + "logps/rejected": -889.3547973632812, + "loss": 1.0943, + "margin_dpo/margin_mean": 275.20941162109375, + "margin_dpo/margin_std": 351.0611572265625, + "step": 396 + }, + { + "KL/chosen_KL_mean": -484.30328369140625, + "KL/mean": -674.1336669921875, + "KL/rejected_KL_mean": -863.964111328125, + "KL/std": 388.2706298828125, + "epoch": 0.5829662261380323, + "fcm_dpo/beta": 0.0012838148977607489, + "fcm_dpo/delta": -0.11897562444210052, + "fcm_dpo/margin": 379.66082763671875, + "fcm_dpo/q_t": 0.3814941942691803, + "grad_norm": 36.56934356689453, + "learning_rate": 2.2310222927429716e-07, + "logits/chosen": -0.7270249128341675, + "logits/rejected": -0.7500828504562378, + "logps/chosen": -545.5804443359375, + "logps/ref_chosen": -61.27716827392578, + "logps/ref_rejected": -103.11612701416016, + "logps/rejected": -967.0802001953125, + "loss": 1.0163, + "margin_dpo/margin_mean": 379.66082763671875, + "margin_dpo/margin_std": 395.0962829589844, + "step": 397 + }, + { + "KL/chosen_KL_mean": -559.9149169921875, + "KL/mean": -713.6265869140625, + "KL/rejected_KL_mean": -867.3382568359375, + "KL/std": 434.58392333984375, + "epoch": 0.5844346549192364, + "fcm_dpo/beta": 0.0012684231624007225, + "fcm_dpo/delta": 0.012464390136301517, + "fcm_dpo/margin": 307.4233093261719, + "fcm_dpo/q_t": 0.41516533493995667, + "grad_norm": 29.09773063659668, + "learning_rate": 2.2182670837116972e-07, + "logits/chosen": -0.7229080200195312, + "logits/rejected": -0.7291736006736755, + "logps/chosen": -628.06640625, + "logps/ref_chosen": -68.15155029296875, + "logps/ref_rejected": -108.52360534667969, + "logps/rejected": -975.86181640625, + "loss": 1.1261, + "margin_dpo/margin_mean": 307.42333984375, + "margin_dpo/margin_std": 505.4172058105469, + "step": 398 + }, + { + "KL/chosen_KL_mean": -482.8316650390625, + "KL/mean": -642.2623901367188, + "KL/rejected_KL_mean": -801.6930541992188, + "KL/std": 400.37786865234375, + "epoch": 0.5859030837004405, + "fcm_dpo/beta": 0.0012626744573935866, + "fcm_dpo/delta": -0.003726685419678688, + "fcm_dpo/margin": 318.8613586425781, + "fcm_dpo/q_t": 0.4093470573425293, + "grad_norm": 36.91798400878906, + "learning_rate": 2.2055192986047804e-07, + "logits/chosen": -0.7428172826766968, + "logits/rejected": -0.7134918570518494, + "logps/chosen": -543.7214965820312, + "logps/ref_chosen": -60.889801025390625, + "logps/ref_rejected": -77.965576171875, + "logps/rejected": -879.6586303710938, + "loss": 1.1144, + "margin_dpo/margin_mean": 318.8613586425781, + "margin_dpo/margin_std": 496.3259582519531, + "step": 399 + }, + { + "KL/chosen_KL_mean": -457.9895324707031, + "KL/mean": -660.6600341796875, + "KL/rejected_KL_mean": -863.33056640625, + "KL/std": 368.4857177734375, + "epoch": 0.5873715124816447, + "fcm_dpo/beta": 0.0011767219984903932, + "fcm_dpo/delta": -0.10189603269100189, + "fcm_dpo/margin": 405.341064453125, + "fcm_dpo/q_t": 0.38432997465133667, + "grad_norm": 36.20450973510742, + "learning_rate": 2.192779273338215e-07, + "logits/chosen": -0.7186334133148193, + "logits/rejected": -0.7250533103942871, + "logps/chosen": -521.6331176757812, + "logps/ref_chosen": -63.64359664916992, + "logps/ref_rejected": -105.252685546875, + "logps/rejected": -968.583251953125, + "loss": 1.0222, + "margin_dpo/margin_mean": 405.341064453125, + "margin_dpo/margin_std": 422.13092041015625, + "step": 400 + }, + { + "KL/chosen_KL_mean": -539.432861328125, + "KL/mean": -647.3896484375, + "KL/rejected_KL_mean": -755.346435546875, + "KL/std": 385.900146484375, + "epoch": 0.5888399412628488, + "fcm_dpo/beta": 0.0012087201466783881, + "fcm_dpo/delta": 0.04972143471240997, + "fcm_dpo/margin": 215.91360473632812, + "fcm_dpo/q_t": 0.44173485040664673, + "grad_norm": 31.359392166137695, + "learning_rate": 2.1800473436235136e-07, + "logits/chosen": -0.7117223739624023, + "logits/rejected": -0.7047185301780701, + "logps/chosen": -596.5958862304688, + "logps/ref_chosen": -57.16303253173828, + "logps/ref_rejected": -83.79249572753906, + "logps/rejected": -839.138916015625, + "loss": 1.2256, + "margin_dpo/margin_mean": 215.9136199951172, + "margin_dpo/margin_std": 492.71820068359375, + "step": 401 + }, + { + "KL/chosen_KL_mean": -420.78167724609375, + "KL/mean": -644.1975708007812, + "KL/rejected_KL_mean": -867.613525390625, + "KL/std": 450.4141845703125, + "epoch": 0.5903083700440529, + "fcm_dpo/beta": 0.0010973564349114895, + "fcm_dpo/delta": -0.12137363106012344, + "fcm_dpo/margin": 446.831787109375, + "fcm_dpo/q_t": 0.3830028772354126, + "grad_norm": 26.827964782714844, + "learning_rate": 2.1673238449588665e-07, + "logits/chosen": -0.7724575996398926, + "logits/rejected": -0.7748699188232422, + "logps/chosen": -471.52203369140625, + "logps/ref_chosen": -50.74037170410156, + "logps/ref_rejected": -81.0460433959961, + "logps/rejected": -948.6595458984375, + "loss": 1.0181, + "margin_dpo/margin_mean": 446.831787109375, + "margin_dpo/margin_std": 491.77197265625, + "step": 402 + }, + { + "KL/chosen_KL_mean": -427.5725402832031, + "KL/mean": -592.4838256835938, + "KL/rejected_KL_mean": -757.39501953125, + "KL/std": 362.35382080078125, + "epoch": 0.591776798825257, + "fcm_dpo/beta": 0.0011073750210925937, + "fcm_dpo/delta": 0.04235755652189255, + "fcm_dpo/margin": 329.82257080078125, + "fcm_dpo/q_t": 0.41572099924087524, + "grad_norm": 28.50166893005371, + "learning_rate": 2.154609112620295e-07, + "logits/chosen": -0.7852243185043335, + "logits/rejected": -0.7927581071853638, + "logps/chosen": -474.7198486328125, + "logps/ref_chosen": -47.14731216430664, + "logps/ref_rejected": -77.2666015625, + "logps/rejected": -834.6616821289062, + "loss": 1.1015, + "margin_dpo/margin_mean": 329.82257080078125, + "margin_dpo/margin_std": 395.7626037597656, + "step": 403 + }, + { + "KL/chosen_KL_mean": -516.7240600585938, + "KL/mean": -690.0592041015625, + "KL/rejected_KL_mean": -863.3944091796875, + "KL/std": 423.0172119140625, + "epoch": 0.593245227606461, + "fcm_dpo/beta": 0.0011222802568227053, + "fcm_dpo/delta": 0.01353571005165577, + "fcm_dpo/margin": 346.67041015625, + "fcm_dpo/q_t": 0.4125150740146637, + "grad_norm": 25.302902221679688, + "learning_rate": 2.1419034816528218e-07, + "logits/chosen": -0.7894694805145264, + "logits/rejected": -0.7903016805648804, + "logps/chosen": -564.5993041992188, + "logps/ref_chosen": -47.875274658203125, + "logps/ref_rejected": -77.15499877929688, + "logps/rejected": -940.5494384765625, + "loss": 1.1134, + "margin_dpo/margin_mean": 346.67041015625, + "margin_dpo/margin_std": 523.8775024414062, + "step": 404 + }, + { + "KL/chosen_KL_mean": -592.7296142578125, + "KL/mean": -713.805419921875, + "KL/rejected_KL_mean": -834.8812255859375, + "KL/std": 422.7801513671875, + "epoch": 0.5947136563876652, + "fcm_dpo/beta": 0.001129148993641138, + "fcm_dpo/delta": -0.005241929553449154, + "fcm_dpo/margin": 242.15170288085938, + "fcm_dpo/q_t": 0.4377569258213043, + "grad_norm": 39.06339645385742, + "learning_rate": 2.129207286861638e-07, + "logits/chosen": -0.783086895942688, + "logits/rejected": -0.7755570411682129, + "logps/chosen": -657.8924560546875, + "logps/ref_chosen": -65.16290283203125, + "logps/ref_rejected": -87.18678283691406, + "logps/rejected": -922.0680541992188, + "loss": 1.2157, + "margin_dpo/margin_mean": 242.15170288085938, + "margin_dpo/margin_std": 514.3509521484375, + "step": 405 + }, + { + "KL/chosen_KL_mean": -550.7774658203125, + "KL/mean": -743.4031982421875, + "KL/rejected_KL_mean": -936.0289306640625, + "KL/std": 466.4752197265625, + "epoch": 0.5961820851688693, + "fcm_dpo/beta": 0.0011199053842574358, + "fcm_dpo/delta": -0.04166682809591293, + "fcm_dpo/margin": 385.25140380859375, + "fcm_dpo/q_t": 0.40454065799713135, + "grad_norm": 29.93971061706543, + "learning_rate": 2.1165208628032861e-07, + "logits/chosen": -0.8605983257293701, + "logits/rejected": -0.8947657346725464, + "logps/chosen": -600.518310546875, + "logps/ref_chosen": -49.740814208984375, + "logps/ref_rejected": -92.07862854003906, + "logps/rejected": -1028.1075439453125, + "loss": 1.0915, + "margin_dpo/margin_mean": 385.25140380859375, + "margin_dpo/margin_std": 562.6111450195312, + "step": 406 + }, + { + "KL/chosen_KL_mean": -557.0032348632812, + "KL/mean": -658.51025390625, + "KL/rejected_KL_mean": -760.0172729492188, + "KL/std": 408.6435852050781, + "epoch": 0.5976505139500734, + "fcm_dpo/beta": 0.0011084176367148757, + "fcm_dpo/delta": 0.05087389424443245, + "fcm_dpo/margin": 203.0139923095703, + "fcm_dpo/q_t": 0.44716814160346985, + "grad_norm": 29.988346099853516, + "learning_rate": 2.1038445437768375e-07, + "logits/chosen": -0.9323893785476685, + "logits/rejected": -0.8998297452926636, + "logps/chosen": -613.333984375, + "logps/ref_chosen": -56.33069610595703, + "logps/ref_rejected": -77.51209259033203, + "logps/rejected": -837.5293579101562, + "loss": 1.2531, + "margin_dpo/margin_mean": 203.01400756835938, + "margin_dpo/margin_std": 529.3558959960938, + "step": 407 + }, + { + "KL/chosen_KL_mean": -562.3118896484375, + "KL/mean": -679.2642822265625, + "KL/rejected_KL_mean": -796.2166748046875, + "KL/std": 361.14996337890625, + "epoch": 0.5991189427312775, + "fcm_dpo/beta": 0.0012600787449628115, + "fcm_dpo/delta": 0.12069714069366455, + "fcm_dpo/margin": 233.90478515625, + "fcm_dpo/q_t": 0.43428733944892883, + "grad_norm": 34.71678161621094, + "learning_rate": 2.0911786638150872e-07, + "logits/chosen": -0.9175188541412354, + "logits/rejected": -0.8953307867050171, + "logps/chosen": -632.1011962890625, + "logps/ref_chosen": -69.789306640625, + "logps/ref_rejected": -90.09693908691406, + "logps/rejected": -886.3135986328125, + "loss": 1.1787, + "margin_dpo/margin_mean": 233.90478515625, + "margin_dpo/margin_std": 410.2579345703125, + "step": 408 + }, + { + "KL/chosen_KL_mean": -531.5665893554688, + "KL/mean": -659.5404663085938, + "KL/rejected_KL_mean": -787.514404296875, + "KL/std": 434.8625183105469, + "epoch": 0.6005873715124816, + "fcm_dpo/beta": 0.0013779137516394258, + "fcm_dpo/delta": 0.05021357536315918, + "fcm_dpo/margin": 255.94775390625, + "fcm_dpo/q_t": 0.41811615228652954, + "grad_norm": 53.07970428466797, + "learning_rate": 2.0785235566757517e-07, + "logits/chosen": -0.801446795463562, + "logits/rejected": -0.78272545337677, + "logps/chosen": -598.884033203125, + "logps/ref_chosen": -67.31744384765625, + "logps/ref_rejected": -84.904296875, + "logps/rejected": -872.4186401367188, + "loss": 1.1736, + "margin_dpo/margin_mean": 255.94773864746094, + "margin_dpo/margin_std": 472.74395751953125, + "step": 409 + }, + { + "KL/chosen_KL_mean": -491.54156494140625, + "KL/mean": -645.4672241210938, + "KL/rejected_KL_mean": -799.392822265625, + "KL/std": 392.8507385253906, + "epoch": 0.6020558002936858, + "fcm_dpo/beta": 0.0013317305129021406, + "fcm_dpo/delta": -0.012640236876904964, + "fcm_dpo/margin": 307.85125732421875, + "fcm_dpo/q_t": 0.40540868043899536, + "grad_norm": 28.094219207763672, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": -0.8844398260116577, + "logits/rejected": -0.8990808725357056, + "logps/chosen": -543.0069580078125, + "logps/ref_chosen": -51.465354919433594, + "logps/ref_rejected": -83.198974609375, + "logps/rejected": -882.591796875, + "loss": 1.0982, + "margin_dpo/margin_mean": 307.85125732421875, + "margin_dpo/margin_std": 440.26861572265625, + "step": 410 + }, + { + "KL/chosen_KL_mean": -492.3099670410156, + "KL/mean": -623.0086059570312, + "KL/rejected_KL_mean": -753.7071533203125, + "KL/std": 405.5526123046875, + "epoch": 0.6035242290748899, + "fcm_dpo/beta": 0.0012510113883763552, + "fcm_dpo/delta": -0.062365103513002396, + "fcm_dpo/margin": 261.3972473144531, + "fcm_dpo/q_t": 0.42639321088790894, + "grad_norm": 39.17562484741211, + "learning_rate": 2.0532469944670343e-07, + "logits/chosen": -0.9218274354934692, + "logits/rejected": -0.9327045679092407, + "logps/chosen": -544.6172485351562, + "logps/ref_chosen": -52.30727005004883, + "logps/ref_rejected": -80.69495391845703, + "logps/rejected": -834.4021606445312, + "loss": 1.1733, + "margin_dpo/margin_mean": 261.397216796875, + "margin_dpo/margin_std": 473.8624267578125, + "step": 411 + }, + { + "KL/chosen_KL_mean": -472.92578125, + "KL/mean": -619.5107421875, + "KL/rejected_KL_mean": -766.095703125, + "KL/std": 371.1781005859375, + "epoch": 0.604992657856094, + "fcm_dpo/beta": 0.0012766936561092734, + "fcm_dpo/delta": 0.031684231013059616, + "fcm_dpo/margin": 293.1698913574219, + "fcm_dpo/q_t": 0.41487789154052734, + "grad_norm": 37.587581634521484, + "learning_rate": 2.0406262054585738e-07, + "logits/chosen": -0.9481087327003479, + "logits/rejected": -0.9869192838668823, + "logps/chosen": -526.0699462890625, + "logps/ref_chosen": -53.144126892089844, + "logps/ref_rejected": -100.0608139038086, + "logps/rejected": -866.156494140625, + "loss": 1.1127, + "margin_dpo/margin_mean": 293.1698913574219, + "margin_dpo/margin_std": 415.78509521484375, + "step": 412 + }, + { + "KL/chosen_KL_mean": -546.9744873046875, + "KL/mean": -695.7308349609375, + "KL/rejected_KL_mean": -844.4873046875, + "KL/std": 423.4039306640625, + "epoch": 0.6064610866372981, + "fcm_dpo/beta": 0.0013015582226216793, + "fcm_dpo/delta": 0.0158353503793478, + "fcm_dpo/margin": 297.5128173828125, + "fcm_dpo/q_t": 0.4121004343032837, + "grad_norm": 36.7993049621582, + "learning_rate": 2.0280175213768205e-07, + "logits/chosen": -0.8297325968742371, + "logits/rejected": -0.8475293517112732, + "logps/chosen": -608.556396484375, + "logps/ref_chosen": -61.58196258544922, + "logps/ref_rejected": -99.47340393066406, + "logps/rejected": -943.960693359375, + "loss": 1.1252, + "margin_dpo/margin_mean": 297.5128173828125, + "margin_dpo/margin_std": 477.887451171875, + "step": 413 + }, + { + "KL/chosen_KL_mean": -513.6051025390625, + "KL/mean": -674.848876953125, + "KL/rejected_KL_mean": -836.0927124023438, + "KL/std": 387.4964294433594, + "epoch": 0.6079295154185022, + "fcm_dpo/beta": 0.0013245465233922005, + "fcm_dpo/delta": -0.03918169438838959, + "fcm_dpo/margin": 322.4875793457031, + "fcm_dpo/q_t": 0.3994929790496826, + "grad_norm": 43.984615325927734, + "learning_rate": 2.0154212744723247e-07, + "logits/chosen": -0.7619544267654419, + "logits/rejected": -0.7554280757904053, + "logps/chosen": -560.236572265625, + "logps/ref_chosen": -46.63148498535156, + "logps/ref_rejected": -87.64653015136719, + "logps/rejected": -923.7392578125, + "loss": 1.096, + "margin_dpo/margin_mean": 322.4875793457031, + "margin_dpo/margin_std": 455.6772766113281, + "step": 414 + }, + { + "KL/chosen_KL_mean": -519.7416381835938, + "KL/mean": -640.4639892578125, + "KL/rejected_KL_mean": -761.186279296875, + "KL/std": 358.64996337890625, + "epoch": 0.6093979441997063, + "fcm_dpo/beta": 0.001341789960861206, + "fcm_dpo/delta": 0.09107168018817902, + "fcm_dpo/margin": 241.4446258544922, + "fcm_dpo/q_t": 0.42783957719802856, + "grad_norm": 31.809572219848633, + "learning_rate": 2.002837796667909e-07, + "logits/chosen": -0.7860684990882874, + "logits/rejected": -0.7803636789321899, + "logps/chosen": -598.3599853515625, + "logps/ref_chosen": -78.6182861328125, + "logps/ref_rejected": -100.47752380371094, + "logps/rejected": -861.663818359375, + "loss": 1.1536, + "margin_dpo/margin_mean": 241.44464111328125, + "margin_dpo/margin_std": 387.5479431152344, + "step": 415 + }, + { + "KL/chosen_KL_mean": -462.289306640625, + "KL/mean": -652.3590087890625, + "KL/rejected_KL_mean": -842.4287109375, + "KL/std": 418.4091796875, + "epoch": 0.6108663729809104, + "fcm_dpo/beta": 0.0012784367427229881, + "fcm_dpo/delta": -0.11329878866672516, + "fcm_dpo/margin": 380.139404296875, + "fcm_dpo/q_t": 0.38521865010261536, + "grad_norm": 30.71705436706543, + "learning_rate": 1.990267419549914e-07, + "logits/chosen": -0.7888989448547363, + "logits/rejected": -0.80275559425354, + "logps/chosen": -520.5684204101562, + "logps/ref_chosen": -58.27912521362305, + "logps/ref_rejected": -90.56871795654297, + "logps/rejected": -932.9974365234375, + "loss": 1.0321, + "margin_dpo/margin_mean": 380.1394348144531, + "margin_dpo/margin_std": 444.5953369140625, + "step": 416 + }, + { + "KL/chosen_KL_mean": -447.36212158203125, + "KL/mean": -610.0339965820312, + "KL/rejected_KL_mean": -772.7059326171875, + "KL/std": 384.17840576171875, + "epoch": 0.6123348017621145, + "fcm_dpo/beta": 0.0012431307695806026, + "fcm_dpo/delta": -0.006137235090136528, + "fcm_dpo/margin": 325.34375, + "fcm_dpo/q_t": 0.4053758382797241, + "grad_norm": 39.485050201416016, + "learning_rate": 1.9777104743594686e-07, + "logits/chosen": -0.7083637714385986, + "logits/rejected": -0.6805850863456726, + "logps/chosen": -497.56085205078125, + "logps/ref_chosen": -50.1987190246582, + "logps/ref_rejected": -68.15184020996094, + "logps/rejected": -840.8577880859375, + "loss": 1.0748, + "margin_dpo/margin_mean": 325.3437805175781, + "margin_dpo/margin_std": 378.35235595703125, + "step": 417 + }, + { + "KL/chosen_KL_mean": -498.91168212890625, + "KL/mean": -664.226318359375, + "KL/rejected_KL_mean": -829.5408935546875, + "KL/std": 416.49688720703125, + "epoch": 0.6138032305433186, + "fcm_dpo/beta": 0.0012598626781255007, + "fcm_dpo/delta": -0.02940238267183304, + "fcm_dpo/margin": 330.62921142578125, + "fcm_dpo/q_t": 0.4053995609283447, + "grad_norm": 24.680252075195312, + "learning_rate": 1.965167291983757e-07, + "logits/chosen": -0.796850323677063, + "logits/rejected": -0.7809560894966125, + "logps/chosen": -580.89013671875, + "logps/ref_chosen": -81.97846984863281, + "logps/ref_rejected": -104.69148254394531, + "logps/rejected": -934.2323608398438, + "loss": 1.1034, + "margin_dpo/margin_mean": 330.62921142578125, + "margin_dpo/margin_std": 468.53973388671875, + "step": 418 + }, + { + "KL/chosen_KL_mean": -468.39617919921875, + "KL/mean": -656.1996459960938, + "KL/rejected_KL_mean": -844.0031127929688, + "KL/std": 375.4820556640625, + "epoch": 0.6152716593245228, + "fcm_dpo/beta": 0.0011481782421469688, + "fcm_dpo/delta": -0.040392979979515076, + "fcm_dpo/margin": 375.6069641113281, + "fcm_dpo/q_t": 0.39766836166381836, + "grad_norm": 35.54838562011719, + "learning_rate": 1.9526382029472988e-07, + "logits/chosen": -0.6798853278160095, + "logits/rejected": -0.6791231632232666, + "logps/chosen": -521.3448486328125, + "logps/ref_chosen": -52.948646545410156, + "logps/ref_rejected": -91.58309936523438, + "logps/rejected": -935.586181640625, + "loss": 1.0594, + "margin_dpo/margin_mean": 375.60693359375, + "margin_dpo/margin_std": 432.56622314453125, + "step": 419 + }, + { + "KL/chosen_KL_mean": -585.70361328125, + "KL/mean": -711.439208984375, + "KL/rejected_KL_mean": -837.1748657226562, + "KL/std": 429.8891296386719, + "epoch": 0.6167400881057269, + "fcm_dpo/beta": 0.0012487138155847788, + "fcm_dpo/delta": 0.09940655529499054, + "fcm_dpo/margin": 251.47128295898438, + "fcm_dpo/q_t": 0.4328416585922241, + "grad_norm": 44.155242919921875, + "learning_rate": 1.9401235374032425e-07, + "logits/chosen": -0.810115396976471, + "logits/rejected": -0.7681174278259277, + "logps/chosen": -663.4735107421875, + "logps/ref_chosen": -77.7699203491211, + "logps/ref_rejected": -69.31985473632812, + "logps/rejected": -906.4947509765625, + "loss": 1.1992, + "margin_dpo/margin_mean": 251.47128295898438, + "margin_dpo/margin_std": 534.525390625, + "step": 420 + }, + { + "KL/chosen_KL_mean": -498.656005859375, + "KL/mean": -636.3494873046875, + "KL/rejected_KL_mean": -774.04296875, + "KL/std": 402.7722473144531, + "epoch": 0.618208516886931, + "fcm_dpo/beta": 0.0013234158977866173, + "fcm_dpo/delta": 0.041565168648958206, + "fcm_dpo/margin": 275.386962890625, + "fcm_dpo/q_t": 0.41690126061439514, + "grad_norm": 27.106782913208008, + "learning_rate": 1.9276236251246653e-07, + "logits/chosen": -0.7847892045974731, + "logits/rejected": -0.7707439661026001, + "logps/chosen": -552.421875, + "logps/ref_chosen": -53.765865325927734, + "logps/ref_rejected": -89.28144836425781, + "logps/rejected": -863.324462890625, + "loss": 1.1328, + "margin_dpo/margin_mean": 275.3869934082031, + "margin_dpo/margin_std": 421.31298828125, + "step": 421 + }, + { + "KL/chosen_KL_mean": -588.316162109375, + "KL/mean": -753.931640625, + "KL/rejected_KL_mean": -919.5469970703125, + "KL/std": 450.30499267578125, + "epoch": 0.6196769456681351, + "fcm_dpo/beta": 0.0012808962492272258, + "fcm_dpo/delta": -0.031088268384337425, + "fcm_dpo/margin": 331.2308349609375, + "fcm_dpo/q_t": 0.4033207297325134, + "grad_norm": 26.31610107421875, + "learning_rate": 1.9151387954958792e-07, + "logits/chosen": -0.8320428133010864, + "logits/rejected": -0.8418641090393066, + "logps/chosen": -656.949951171875, + "logps/ref_chosen": -68.6337661743164, + "logps/ref_rejected": -87.86351013183594, + "logps/rejected": -1007.4105224609375, + "loss": 1.1125, + "margin_dpo/margin_mean": 331.2308349609375, + "margin_dpo/margin_std": 535.4093627929688, + "step": 422 + }, + { + "KL/chosen_KL_mean": -558.1864624023438, + "KL/mean": -731.43212890625, + "KL/rejected_KL_mean": -904.677734375, + "KL/std": 429.63372802734375, + "epoch": 0.6211453744493393, + "fcm_dpo/beta": 0.0012374825309962034, + "fcm_dpo/delta": -0.0368797592818737, + "fcm_dpo/margin": 346.49127197265625, + "fcm_dpo/q_t": 0.40145501494407654, + "grad_norm": 33.875404357910156, + "learning_rate": 1.902669377503756e-07, + "logits/chosen": -0.8104654550552368, + "logits/rejected": -0.827370822429657, + "logps/chosen": -613.1767578125, + "logps/ref_chosen": -54.99030303955078, + "logps/ref_rejected": -86.30654907226562, + "logps/rejected": -990.9842529296875, + "loss": 1.0925, + "margin_dpo/margin_mean": 346.4912414550781, + "margin_dpo/margin_std": 503.84454345703125, + "step": 423 + }, + { + "KL/chosen_KL_mean": -510.28045654296875, + "KL/mean": -682.5518188476562, + "KL/rejected_KL_mean": -854.8231201171875, + "KL/std": 413.15972900390625, + "epoch": 0.6226138032305433, + "fcm_dpo/beta": 0.001186060020700097, + "fcm_dpo/delta": -0.01438332349061966, + "fcm_dpo/margin": 344.54266357421875, + "fcm_dpo/q_t": 0.40733110904693604, + "grad_norm": 37.09081268310547, + "learning_rate": 1.890215699729057e-07, + "logits/chosen": -0.7975935935974121, + "logits/rejected": -0.7788383960723877, + "logps/chosen": -566.2923583984375, + "logps/ref_chosen": -56.01192092895508, + "logps/ref_rejected": -66.47896575927734, + "logps/rejected": -921.3020629882812, + "loss": 1.0954, + "margin_dpo/margin_mean": 344.54266357421875, + "margin_dpo/margin_std": 480.84130859375, + "step": 424 + }, + { + "KL/chosen_KL_mean": -609.7572021484375, + "KL/mean": -721.0950927734375, + "KL/rejected_KL_mean": -832.43310546875, + "KL/std": 407.82891845703125, + "epoch": 0.6240822320117474, + "fcm_dpo/beta": 0.0012294601183384657, + "fcm_dpo/delta": 0.027459844946861267, + "fcm_dpo/margin": 222.6759490966797, + "fcm_dpo/q_t": 0.4368723928928375, + "grad_norm": 42.740379333496094, + "learning_rate": 1.8777780903377732e-07, + "logits/chosen": -0.7805401086807251, + "logits/rejected": -0.7881333827972412, + "logps/chosen": -656.626220703125, + "logps/ref_chosen": -46.86899948120117, + "logps/ref_rejected": -95.92545318603516, + "logps/rejected": -928.3585815429688, + "loss": 1.2167, + "margin_dpo/margin_mean": 222.67593383789062, + "margin_dpo/margin_std": 486.15093994140625, + "step": 425 + }, + { + "KL/chosen_KL_mean": -552.8094482421875, + "KL/mean": -687.5247192382812, + "KL/rejected_KL_mean": -822.239990234375, + "KL/std": 365.3626403808594, + "epoch": 0.6255506607929515, + "fcm_dpo/beta": 0.0013338683638721704, + "fcm_dpo/delta": 0.04180013760924339, + "fcm_dpo/margin": 269.4305419921875, + "fcm_dpo/q_t": 0.4174906015396118, + "grad_norm": 31.232389450073242, + "learning_rate": 1.8653568770724803e-07, + "logits/chosen": -0.7621093988418579, + "logits/rejected": -0.7303955554962158, + "logps/chosen": -629.3930053710938, + "logps/ref_chosen": -76.58354187011719, + "logps/ref_rejected": -81.26658630371094, + "logps/rejected": -903.506591796875, + "loss": 1.1272, + "margin_dpo/margin_mean": 269.4305419921875, + "margin_dpo/margin_std": 378.654052734375, + "step": 426 + }, + { + "KL/chosen_KL_mean": -467.79608154296875, + "KL/mean": -581.63623046875, + "KL/rejected_KL_mean": -695.476318359375, + "KL/std": 333.85369873046875, + "epoch": 0.6270190895741556, + "fcm_dpo/beta": 0.001394979190081358, + "fcm_dpo/delta": 0.09802726656198502, + "fcm_dpo/margin": 227.68026733398438, + "fcm_dpo/q_t": 0.4301951825618744, + "grad_norm": 31.7608642578125, + "learning_rate": 1.8529523872436977e-07, + "logits/chosen": -0.8063141703605652, + "logits/rejected": -0.78669673204422, + "logps/chosen": -532.6499633789062, + "logps/ref_chosen": -64.8538818359375, + "logps/ref_rejected": -78.5660171508789, + "logps/rejected": -774.0423583984375, + "loss": 1.1608, + "margin_dpo/margin_mean": 227.68026733398438, + "margin_dpo/margin_std": 382.787353515625, + "step": 427 + }, + { + "KL/chosen_KL_mean": -608.790771484375, + "KL/mean": -786.853515625, + "KL/rejected_KL_mean": -964.916259765625, + "KL/std": 468.04583740234375, + "epoch": 0.6284875183553598, + "fcm_dpo/beta": 0.0013052403228357434, + "fcm_dpo/delta": -0.090778648853302, + "fcm_dpo/margin": 356.1253967285156, + "fcm_dpo/q_t": 0.3941190838813782, + "grad_norm": 28.149211883544922, + "learning_rate": 1.8405649477212697e-07, + "logits/chosen": -0.7752005457878113, + "logits/rejected": -0.7893061637878418, + "logps/chosen": -671.427490234375, + "logps/ref_chosen": -62.63666534423828, + "logps/ref_rejected": -103.28181457519531, + "logps/rejected": -1068.197998046875, + "loss": 1.088, + "margin_dpo/margin_mean": 356.1253967285156, + "margin_dpo/margin_std": 543.524169921875, + "step": 428 + }, + { + "KL/chosen_KL_mean": -629.3082275390625, + "KL/mean": -756.3204345703125, + "KL/rejected_KL_mean": -883.3326416015625, + "KL/std": 410.94122314453125, + "epoch": 0.6299559471365639, + "fcm_dpo/beta": 0.0012956894934177399, + "fcm_dpo/delta": -0.020538704469799995, + "fcm_dpo/margin": 254.02444458007812, + "fcm_dpo/q_t": 0.4243999719619751, + "grad_norm": 43.50310516357422, + "learning_rate": 1.828194884925749e-07, + "logits/chosen": -0.7827075719833374, + "logits/rejected": -0.7630304098129272, + "logps/chosen": -710.542236328125, + "logps/ref_chosen": -81.23401641845703, + "logps/ref_rejected": -91.79493713378906, + "logps/rejected": -975.1275634765625, + "loss": 1.1856, + "margin_dpo/margin_mean": 254.02444458007812, + "margin_dpo/margin_std": 494.28680419921875, + "step": 429 + }, + { + "KL/chosen_KL_mean": -546.2984619140625, + "KL/mean": -644.2332763671875, + "KL/rejected_KL_mean": -742.1680297851562, + "KL/std": 393.195556640625, + "epoch": 0.631424375917768, + "fcm_dpo/beta": 0.0013126166304573417, + "fcm_dpo/delta": 0.04607279598712921, + "fcm_dpo/margin": 195.86962890625, + "fcm_dpo/q_t": 0.44172602891921997, + "grad_norm": 39.36235427856445, + "learning_rate": 1.8158425248197928e-07, + "logits/chosen": -0.7814037799835205, + "logits/rejected": -0.7843170166015625, + "logps/chosen": -607.21875, + "logps/ref_chosen": -60.920326232910156, + "logps/ref_rejected": -104.42280578613281, + "logps/rejected": -846.5908203125, + "loss": 1.2239, + "margin_dpo/margin_mean": 195.86962890625, + "margin_dpo/margin_std": 441.08013916015625, + "step": 430 + }, + { + "KL/chosen_KL_mean": -474.2396240234375, + "KL/mean": -651.661376953125, + "KL/rejected_KL_mean": -829.0831298828125, + "KL/std": 401.1722412109375, + "epoch": 0.6328928046989721, + "fcm_dpo/beta": 0.0012775419745594263, + "fcm_dpo/delta": -0.06899101287126541, + "fcm_dpo/margin": 354.843505859375, + "fcm_dpo/q_t": 0.39390021562576294, + "grad_norm": 31.863265991210938, + "learning_rate": 1.8035081928995788e-07, + "logits/chosen": -0.7518160343170166, + "logits/rejected": -0.7592271566390991, + "logps/chosen": -531.58837890625, + "logps/ref_chosen": -57.34874725341797, + "logps/ref_rejected": -92.84022521972656, + "logps/rejected": -921.9234008789062, + "loss": 1.067, + "margin_dpo/margin_mean": 354.8435363769531, + "margin_dpo/margin_std": 455.1669921875, + "step": 431 + }, + { + "KL/chosen_KL_mean": -427.3382263183594, + "KL/mean": -617.7256469726562, + "KL/rejected_KL_mean": -808.113037109375, + "KL/std": 403.57916259765625, + "epoch": 0.6343612334801763, + "fcm_dpo/beta": 0.001215646741911769, + "fcm_dpo/delta": -0.08303441107273102, + "fcm_dpo/margin": 380.77484130859375, + "fcm_dpo/q_t": 0.39162373542785645, + "grad_norm": 47.37468719482422, + "learning_rate": 1.791192214186223e-07, + "logits/chosen": -0.6925067901611328, + "logits/rejected": -0.685772180557251, + "logps/chosen": -498.41302490234375, + "logps/ref_chosen": -71.07479095458984, + "logps/ref_rejected": -98.57952880859375, + "logps/rejected": -906.692626953125, + "loss": 1.0385, + "margin_dpo/margin_mean": 380.77484130859375, + "margin_dpo/margin_std": 422.8524169921875, + "step": 432 + }, + { + "KL/chosen_KL_mean": -561.0573120117188, + "KL/mean": -678.9639282226562, + "KL/rejected_KL_mean": -796.8705444335938, + "KL/std": 433.2787780761719, + "epoch": 0.6358296622613803, + "fcm_dpo/beta": 0.0012703599641099572, + "fcm_dpo/delta": 0.11619932949542999, + "fcm_dpo/margin": 235.8131561279297, + "fcm_dpo/q_t": 0.4302235245704651, + "grad_norm": 37.460628509521484, + "learning_rate": 1.7788949132172193e-07, + "logits/chosen": -0.7900607585906982, + "logits/rejected": -0.7896082401275635, + "logps/chosen": -619.3305053710938, + "logps/ref_chosen": -58.273193359375, + "logps/ref_rejected": -95.95089721679688, + "logps/rejected": -892.8214111328125, + "loss": 1.2116, + "margin_dpo/margin_mean": 235.81317138671875, + "margin_dpo/margin_std": 522.4442749023438, + "step": 433 + }, + { + "KL/chosen_KL_mean": -496.65863037109375, + "KL/mean": -640.20361328125, + "KL/rejected_KL_mean": -783.7486572265625, + "KL/std": 404.4720458984375, + "epoch": 0.6372980910425844, + "fcm_dpo/beta": 0.0013023980427533388, + "fcm_dpo/delta": 0.0312434583902359, + "fcm_dpo/margin": 287.09002685546875, + "fcm_dpo/q_t": 0.4213806688785553, + "grad_norm": 25.82297706604004, + "learning_rate": 1.7666166140378853e-07, + "logits/chosen": -0.8235654830932617, + "logits/rejected": -0.8260775208473206, + "logps/chosen": -558.63232421875, + "logps/ref_chosen": -61.97370147705078, + "logps/ref_rejected": -78.49861145019531, + "logps/rejected": -862.2472534179688, + "loss": 1.1396, + "margin_dpo/margin_mean": 287.09002685546875, + "margin_dpo/margin_std": 501.02880859375, + "step": 434 + }, + { + "KL/chosen_KL_mean": -480.15570068359375, + "KL/mean": -635.9765625, + "KL/rejected_KL_mean": -791.79736328125, + "KL/std": 393.2747802734375, + "epoch": 0.6387665198237885, + "fcm_dpo/beta": 0.0013056218158453703, + "fcm_dpo/delta": -0.01002051867544651, + "fcm_dpo/margin": 311.6417236328125, + "fcm_dpo/q_t": 0.4069361090660095, + "grad_norm": 45.785640716552734, + "learning_rate": 1.7543576401928218e-07, + "logits/chosen": -0.8830646276473999, + "logits/rejected": -0.8846416473388672, + "logps/chosen": -531.65771484375, + "logps/ref_chosen": -51.502052307128906, + "logps/ref_rejected": -87.56689453125, + "logps/rejected": -879.3642578125, + "loss": 1.0991, + "margin_dpo/margin_mean": 311.6416931152344, + "margin_dpo/margin_std": 440.7518615722656, + "step": 435 + }, + { + "KL/chosen_KL_mean": -499.36517333984375, + "KL/mean": -633.2975463867188, + "KL/rejected_KL_mean": -767.22998046875, + "KL/std": 352.81036376953125, + "epoch": 0.6402349486049926, + "fcm_dpo/beta": 0.0013422095216810703, + "fcm_dpo/delta": 0.048316676169633865, + "fcm_dpo/margin": 267.86474609375, + "fcm_dpo/q_t": 0.4203526973724365, + "grad_norm": 30.017845153808594, + "learning_rate": 1.742118314717391e-07, + "logits/chosen": -0.732751727104187, + "logits/rejected": -0.6973142623901367, + "logps/chosen": -570.7689208984375, + "logps/ref_chosen": -71.40371704101562, + "logps/ref_rejected": -82.72775268554688, + "logps/rejected": -849.9577026367188, + "loss": 1.1375, + "margin_dpo/margin_mean": 267.86474609375, + "margin_dpo/margin_std": 426.91168212890625, + "step": 436 + }, + { + "KL/chosen_KL_mean": -500.2353515625, + "KL/mean": -638.3201293945312, + "KL/rejected_KL_mean": -776.4049072265625, + "KL/std": 347.3167724609375, + "epoch": 0.6417033773861968, + "fcm_dpo/beta": 0.0014004592085257173, + "fcm_dpo/delta": 0.01640990376472473, + "fcm_dpo/margin": 276.16949462890625, + "fcm_dpo/q_t": 0.4110352396965027, + "grad_norm": 28.889192581176758, + "learning_rate": 1.7298989601292036e-07, + "logits/chosen": -0.8302721977233887, + "logits/rejected": -0.814739465713501, + "logps/chosen": -564.9796142578125, + "logps/ref_chosen": -64.7442626953125, + "logps/ref_rejected": -82.04356384277344, + "logps/rejected": -858.4484252929688, + "loss": 1.1092, + "margin_dpo/margin_mean": 276.16949462890625, + "margin_dpo/margin_std": 396.4515686035156, + "step": 437 + }, + { + "KL/chosen_KL_mean": -522.9442749023438, + "KL/mean": -677.5450439453125, + "KL/rejected_KL_mean": -832.145751953125, + "KL/std": 379.323974609375, + "epoch": 0.6431718061674009, + "fcm_dpo/beta": 0.001349923899397254, + "fcm_dpo/delta": -0.025827843695878983, + "fcm_dpo/margin": 309.201416015625, + "fcm_dpo/q_t": 0.40191444754600525, + "grad_norm": 42.13845443725586, + "learning_rate": 1.7176998984196144e-07, + "logits/chosen": -0.8437032699584961, + "logits/rejected": -0.8374447226524353, + "logps/chosen": -581.9629516601562, + "logps/ref_chosen": -59.0186653137207, + "logps/ref_rejected": -83.07682800292969, + "logps/rejected": -915.2225341796875, + "loss": 1.0822, + "margin_dpo/margin_mean": 309.2014465332031, + "margin_dpo/margin_std": 397.62255859375, + "step": 438 + }, + { + "KL/chosen_KL_mean": -549.39306640625, + "KL/mean": -681.3974609375, + "KL/rejected_KL_mean": -813.4017944335938, + "KL/std": 387.31488037109375, + "epoch": 0.644640234948605, + "fcm_dpo/beta": 0.001252303714863956, + "fcm_dpo/delta": -0.09410447627305984, + "fcm_dpo/margin": 264.0087890625, + "fcm_dpo/q_t": 0.4208075702190399, + "grad_norm": 36.43398666381836, + "learning_rate": 1.7055214510452458e-07, + "logits/chosen": -0.8629584908485413, + "logits/rejected": -0.8729925155639648, + "logps/chosen": -603.1771240234375, + "logps/ref_chosen": -53.78407669067383, + "logps/ref_rejected": -83.98545837402344, + "logps/rejected": -897.3872680664062, + "loss": 1.1705, + "margin_dpo/margin_mean": 264.0087890625, + "margin_dpo/margin_std": 464.46630859375, + "step": 439 + }, + { + "KL/chosen_KL_mean": -529.5724487304688, + "KL/mean": -701.4462890625, + "KL/rejected_KL_mean": -873.320068359375, + "KL/std": 479.51513671875, + "epoch": 0.6461086637298091, + "fcm_dpo/beta": 0.0012315923813730478, + "fcm_dpo/delta": -0.029587693512439728, + "fcm_dpo/margin": 343.74761962890625, + "fcm_dpo/q_t": 0.4066435694694519, + "grad_norm": 53.8433837890625, + "learning_rate": 1.6933639389195134e-07, + "logits/chosen": -0.8866897821426392, + "logits/rejected": -0.8979896903038025, + "logps/chosen": -608.13916015625, + "logps/ref_chosen": -78.56671905517578, + "logps/ref_rejected": -96.49775695800781, + "logps/rejected": -969.81787109375, + "loss": 1.085, + "margin_dpo/margin_mean": 343.74761962890625, + "margin_dpo/margin_std": 490.02239990234375, + "step": 440 + }, + { + "KL/chosen_KL_mean": -619.644775390625, + "KL/mean": -767.532470703125, + "KL/rejected_KL_mean": -915.4201049804688, + "KL/std": 493.6292419433594, + "epoch": 0.6475770925110133, + "fcm_dpo/beta": 0.0012763803824782372, + "fcm_dpo/delta": 0.022389892488718033, + "fcm_dpo/margin": 295.7752685546875, + "fcm_dpo/q_t": 0.4217098355293274, + "grad_norm": 33.3763313293457, + "learning_rate": 1.681227682404166e-07, + "logits/chosen": -0.8680696487426758, + "logits/rejected": -0.8612392544746399, + "logps/chosen": -680.46923828125, + "logps/ref_chosen": -60.824440002441406, + "logps/ref_rejected": -96.47080993652344, + "logps/rejected": -1011.890869140625, + "loss": 1.1805, + "margin_dpo/margin_mean": 295.7752990722656, + "margin_dpo/margin_std": 602.29541015625, + "step": 441 + }, + { + "KL/chosen_KL_mean": -522.046875, + "KL/mean": -706.1878662109375, + "KL/rejected_KL_mean": -890.3289184570312, + "KL/std": 459.5678405761719, + "epoch": 0.6490455212922174, + "fcm_dpo/beta": 0.001231772475875914, + "fcm_dpo/delta": -0.07431311160326004, + "fcm_dpo/margin": 368.2820129394531, + "fcm_dpo/q_t": 0.3975549638271332, + "grad_norm": 34.67390823364258, + "learning_rate": 1.669113001300851e-07, + "logits/chosen": -0.8087929487228394, + "logits/rejected": -0.8081426620483398, + "logps/chosen": -569.05810546875, + "logps/ref_chosen": -47.01121520996094, + "logps/ref_rejected": -76.53926086425781, + "logps/rejected": -966.8681640625, + "loss": 1.089, + "margin_dpo/margin_mean": 368.28204345703125, + "margin_dpo/margin_std": 529.7601928710938, + "step": 442 + }, + { + "KL/chosen_KL_mean": -569.135009765625, + "KL/mean": -666.7360229492188, + "KL/rejected_KL_mean": -764.3370361328125, + "KL/std": 426.0760498046875, + "epoch": 0.6505139500734214, + "fcm_dpo/beta": 0.0012039494467899203, + "fcm_dpo/delta": 0.04213346913456917, + "fcm_dpo/margin": 195.20201110839844, + "fcm_dpo/q_t": 0.44704896211624146, + "grad_norm": 43.334136962890625, + "learning_rate": 1.6570202148426815e-07, + "logits/chosen": -0.8095992803573608, + "logits/rejected": -0.7834180593490601, + "logps/chosen": -640.4080810546875, + "logps/ref_chosen": -71.27301788330078, + "logps/ref_rejected": -86.679931640625, + "logps/rejected": -851.0169677734375, + "loss": 1.2562, + "margin_dpo/margin_mean": 195.20199584960938, + "margin_dpo/margin_std": 502.8472900390625, + "step": 443 + }, + { + "KL/chosen_KL_mean": -557.4791259765625, + "KL/mean": -734.4202270507812, + "KL/rejected_KL_mean": -911.361328125, + "KL/std": 455.6096496582031, + "epoch": 0.6519823788546255, + "fcm_dpo/beta": 0.001166836591437459, + "fcm_dpo/delta": -0.01813267171382904, + "fcm_dpo/margin": 353.8822021484375, + "fcm_dpo/q_t": 0.4072955846786499, + "grad_norm": 32.24861145019531, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": -0.7987985610961914, + "logits/rejected": -0.8144040703773499, + "logps/chosen": -614.6928100585938, + "logps/ref_chosen": -57.213706970214844, + "logps/ref_rejected": -97.25489807128906, + "logps/rejected": -1008.6162109375, + "loss": 1.099, + "margin_dpo/margin_mean": 353.8822021484375, + "margin_dpo/margin_std": 520.219970703125, + "step": 444 + }, + { + "KL/chosen_KL_mean": -508.13671875, + "KL/mean": -659.2096557617188, + "KL/rejected_KL_mean": -810.2826538085938, + "KL/std": 361.33935546875, + "epoch": 0.6534508076358296, + "fcm_dpo/beta": 0.0012313367333263159, + "fcm_dpo/delta": 0.032493121922016144, + "fcm_dpo/margin": 302.14593505859375, + "fcm_dpo/q_t": 0.4164279103279114, + "grad_norm": 38.02067565917969, + "learning_rate": 1.6329015999011182e-07, + "logits/chosen": -0.7978518009185791, + "logits/rejected": -0.7957039475440979, + "logps/chosen": -575.4365234375, + "logps/ref_chosen": -67.29979705810547, + "logps/ref_rejected": -92.68267059326172, + "logps/rejected": -902.96533203125, + "loss": 1.1248, + "margin_dpo/margin_mean": 302.14593505859375, + "margin_dpo/margin_std": 454.81597900390625, + "step": 445 + }, + { + "KL/chosen_KL_mean": -450.0630798339844, + "KL/mean": -626.6373291015625, + "KL/rejected_KL_mean": -803.211669921875, + "KL/std": 392.5597229003906, + "epoch": 0.6549192364170338, + "fcm_dpo/beta": 0.001226876862347126, + "fcm_dpo/delta": -0.046956416219472885, + "fcm_dpo/margin": 353.14862060546875, + "fcm_dpo/q_t": 0.3958050608634949, + "grad_norm": 34.05545425415039, + "learning_rate": 1.6208764069656578e-07, + "logits/chosen": -0.8068540692329407, + "logits/rejected": -0.8256067037582397, + "logps/chosen": -509.16156005859375, + "logps/ref_chosen": -59.098487854003906, + "logps/ref_rejected": -101.26419067382812, + "logps/rejected": -904.475830078125, + "loss": 1.0592, + "margin_dpo/margin_mean": 353.14862060546875, + "margin_dpo/margin_std": 380.9633483886719, + "step": 446 + }, + { + "KL/chosen_KL_mean": -495.2170715332031, + "KL/mean": -681.0345458984375, + "KL/rejected_KL_mean": -866.8519897460938, + "KL/std": 464.1243591308594, + "epoch": 0.6563876651982379, + "fcm_dpo/beta": 0.0011278802994638681, + "fcm_dpo/delta": -0.025529345497488976, + "fcm_dpo/margin": 371.6348876953125, + "fcm_dpo/q_t": 0.4053482413291931, + "grad_norm": 30.737730026245117, + "learning_rate": 1.608874379754465e-07, + "logits/chosen": -0.9250874519348145, + "logits/rejected": -0.9507585763931274, + "logps/chosen": -551.2924194335938, + "logps/ref_chosen": -56.07533264160156, + "logps/ref_rejected": -98.69475555419922, + "logps/rejected": -965.5467529296875, + "loss": 1.086, + "margin_dpo/margin_mean": 371.6348876953125, + "margin_dpo/margin_std": 520.7388916015625, + "step": 447 + }, + { + "KL/chosen_KL_mean": -539.2561645507812, + "KL/mean": -716.0037841796875, + "KL/rejected_KL_mean": -892.7515258789062, + "KL/std": 413.3590087890625, + "epoch": 0.657856093979442, + "fcm_dpo/beta": 0.0011455777566879988, + "fcm_dpo/delta": -0.006920462474226952, + "fcm_dpo/margin": 353.4953308105469, + "fcm_dpo/q_t": 0.4064434766769409, + "grad_norm": 37.22543716430664, + "learning_rate": 1.5968958345321177e-07, + "logits/chosen": -0.8032995462417603, + "logits/rejected": -0.8209420442581177, + "logps/chosen": -599.260009765625, + "logps/ref_chosen": -60.00384521484375, + "logps/ref_rejected": -102.26465606689453, + "logps/rejected": -995.0161743164062, + "loss": 1.083, + "margin_dpo/margin_mean": 353.4953308105469, + "margin_dpo/margin_std": 442.79266357421875, + "step": 448 + }, + { + "KL/chosen_KL_mean": -577.392333984375, + "KL/mean": -767.4705810546875, + "KL/rejected_KL_mean": -957.548828125, + "KL/std": 516.8084716796875, + "epoch": 0.6593245227606461, + "fcm_dpo/beta": 0.001104456139728427, + "fcm_dpo/delta": -0.025262048467993736, + "fcm_dpo/margin": 380.15655517578125, + "fcm_dpo/q_t": 0.40797027945518494, + "grad_norm": 27.65226173400879, + "learning_rate": 1.584941086944423e-07, + "logits/chosen": -0.8589591979980469, + "logits/rejected": -0.8559813499450684, + "logps/chosen": -644.9189453125, + "logps/ref_chosen": -67.52661895751953, + "logps/ref_rejected": -88.59690856933594, + "logps/rejected": -1046.145751953125, + "loss": 1.1076, + "margin_dpo/margin_mean": 380.15655517578125, + "margin_dpo/margin_std": 612.1381225585938, + "step": 449 + }, + { + "KL/chosen_KL_mean": -496.8646240234375, + "KL/mean": -713.1530151367188, + "KL/rejected_KL_mean": -929.4413452148438, + "KL/std": 456.3707275390625, + "epoch": 0.6607929515418502, + "fcm_dpo/beta": 0.00104955630376935, + "fcm_dpo/delta": -0.06971128284931183, + "fcm_dpo/margin": 432.5767822265625, + "fcm_dpo/q_t": 0.3907149136066437, + "grad_norm": 37.34757614135742, + "learning_rate": 1.573010452010098e-07, + "logits/chosen": -0.9782512187957764, + "logits/rejected": -1.0043165683746338, + "logps/chosen": -553.9727172851562, + "logps/ref_chosen": -57.10811996459961, + "logps/ref_rejected": -102.75494384765625, + "logps/rejected": -1032.1962890625, + "loss": 1.0356, + "margin_dpo/margin_mean": 432.5767822265625, + "margin_dpo/margin_std": 448.83770751953125, + "step": 450 + }, + { + "KL/chosen_KL_mean": -742.9453125, + "KL/mean": -891.0797119140625, + "KL/rejected_KL_mean": -1039.214111328125, + "KL/std": 587.5035400390625, + "epoch": 0.6622613803230544, + "fcm_dpo/beta": 0.0011060864198952913, + "fcm_dpo/delta": 0.08447873592376709, + "fcm_dpo/margin": 296.268798828125, + "fcm_dpo/q_t": 0.42436882853507996, + "grad_norm": 39.329833984375, + "learning_rate": 1.5611042441124687e-07, + "logits/chosen": -1.0949430465698242, + "logits/rejected": -1.0765621662139893, + "logps/chosen": -801.4141845703125, + "logps/ref_chosen": -58.46883010864258, + "logps/ref_rejected": -72.92941284179688, + "logps/rejected": -1112.1435546875, + "loss": 1.2346, + "margin_dpo/margin_mean": 296.268798828125, + "margin_dpo/margin_std": 738.4291381835938, + "step": 451 + }, + { + "KL/chosen_KL_mean": -594.1732177734375, + "KL/mean": -763.3720703125, + "KL/rejected_KL_mean": -932.5709228515625, + "KL/std": 460.3730773925781, + "epoch": 0.6637298091042585, + "fcm_dpo/beta": 0.0011289503891021013, + "fcm_dpo/delta": 0.02220895327627659, + "fcm_dpo/margin": 338.39764404296875, + "fcm_dpo/q_t": 0.41150838136672974, + "grad_norm": 33.76683044433594, + "learning_rate": 1.549222776991186e-07, + "logits/chosen": -0.9474809169769287, + "logits/rejected": -0.9722226858139038, + "logps/chosen": -644.5637817382812, + "logps/ref_chosen": -50.39055252075195, + "logps/ref_rejected": -97.77142333984375, + "logps/rejected": -1030.34228515625, + "loss": 1.1039, + "margin_dpo/margin_mean": 338.39764404296875, + "margin_dpo/margin_std": 456.95947265625, + "step": 452 + }, + { + "KL/chosen_KL_mean": -649.19287109375, + "KL/mean": -851.3839721679688, + "KL/rejected_KL_mean": -1053.5750732421875, + "KL/std": 513.1878051757812, + "epoch": 0.6651982378854625, + "fcm_dpo/beta": 0.0010724079329520464, + "fcm_dpo/delta": -0.04788634181022644, + "fcm_dpo/margin": 404.3822021484375, + "fcm_dpo/q_t": 0.40426748991012573, + "grad_norm": 26.855628967285156, + "learning_rate": 1.5373663637339584e-07, + "logits/chosen": -1.0186734199523926, + "logits/rejected": -1.0134930610656738, + "logps/chosen": -706.90771484375, + "logps/ref_chosen": -57.71485137939453, + "logps/ref_rejected": -82.20741271972656, + "logps/rejected": -1135.782470703125, + "loss": 1.1022, + "margin_dpo/margin_mean": 404.3821716308594, + "margin_dpo/margin_std": 631.357421875, + "step": 453 + }, + { + "KL/chosen_KL_mean": -824.8762817382812, + "KL/mean": -1020.529052734375, + "KL/rejected_KL_mean": -1216.181640625, + "KL/std": 584.076171875, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.0010508534032851458, + "fcm_dpo/delta": -0.01683557778596878, + "fcm_dpo/margin": 391.305419921875, + "fcm_dpo/q_t": 0.4103407859802246, + "grad_norm": 36.570125579833984, + "learning_rate": 1.5255353167683017e-07, + "logits/chosen": -1.0660200119018555, + "logits/rejected": -1.0628576278686523, + "logps/chosen": -885.8218994140625, + "logps/ref_chosen": -60.945648193359375, + "logps/ref_rejected": -84.95079040527344, + "logps/rejected": -1301.1324462890625, + "loss": 1.1279, + "margin_dpo/margin_mean": 391.305419921875, + "margin_dpo/margin_std": 684.9031982421875, + "step": 454 + }, + { + "KL/chosen_KL_mean": -688.5186157226562, + "KL/mean": -917.4297485351562, + "KL/rejected_KL_mean": -1146.3408203125, + "KL/std": 531.5499877929688, + "epoch": 0.6681350954478708, + "fcm_dpo/beta": 0.0010322749149054289, + "fcm_dpo/delta": -0.09659485518932343, + "fcm_dpo/margin": 457.82232666015625, + "fcm_dpo/q_t": 0.38912808895111084, + "grad_norm": 33.90567398071289, + "learning_rate": 1.5137299478533064e-07, + "logits/chosen": -0.9941831827163696, + "logits/rejected": -1.0336955785751343, + "logps/chosen": -733.4053344726562, + "logps/ref_chosen": -44.88671112060547, + "logps/ref_rejected": -115.30147552490234, + "logps/rejected": -1261.642333984375, + "loss": 1.063, + "margin_dpo/margin_mean": 457.8222961425781, + "margin_dpo/margin_std": 598.6575927734375, + "step": 455 + }, + { + "KL/chosen_KL_mean": -691.5862426757812, + "KL/mean": -909.7722778320312, + "KL/rejected_KL_mean": -1127.958251953125, + "KL/std": 566.892578125, + "epoch": 0.6696035242290749, + "fcm_dpo/beta": 0.0009644461097195745, + "fcm_dpo/delta": -0.02674005925655365, + "fcm_dpo/margin": 436.37200927734375, + "fcm_dpo/q_t": 0.40306729078292847, + "grad_norm": 33.307159423828125, + "learning_rate": 1.5019505680714232e-07, + "logits/chosen": -0.9692044258117676, + "logits/rejected": -0.9966904520988464, + "logps/chosen": -748.623046875, + "logps/ref_chosen": -57.036781311035156, + "logps/ref_rejected": -105.21784210205078, + "logps/rejected": -1233.1761474609375, + "loss": 1.0658, + "margin_dpo/margin_mean": 436.37200927734375, + "margin_dpo/margin_std": 519.3140258789062, + "step": 456 + }, + { + "KL/chosen_KL_mean": -584.7559204101562, + "KL/mean": -828.9456787109375, + "KL/rejected_KL_mean": -1073.135498046875, + "KL/std": 520.8341064453125, + "epoch": 0.671071953010279, + "fcm_dpo/beta": 0.0009004294406622648, + "fcm_dpo/delta": -0.0521702915430069, + "fcm_dpo/margin": 488.3794860839844, + "fcm_dpo/q_t": 0.39493173360824585, + "grad_norm": 42.788997650146484, + "learning_rate": 1.4901974878202627e-07, + "logits/chosen": -0.982511043548584, + "logits/rejected": -1.0036410093307495, + "logps/chosen": -638.9984130859375, + "logps/ref_chosen": -54.24253845214844, + "logps/ref_rejected": -85.10956573486328, + "logps/rejected": -1158.2449951171875, + "loss": 1.0442, + "margin_dpo/margin_mean": 488.3795166015625, + "margin_dpo/margin_std": 512.117919921875, + "step": 457 + }, + { + "KL/chosen_KL_mean": -680.8214721679688, + "KL/mean": -844.1943359375, + "KL/rejected_KL_mean": -1007.5672607421875, + "KL/std": 468.6507568359375, + "epoch": 0.6725403817914831, + "fcm_dpo/beta": 0.0009023591410368681, + "fcm_dpo/delta": 0.005208446644246578, + "fcm_dpo/margin": 326.74578857421875, + "fcm_dpo/q_t": 0.4328933656215668, + "grad_norm": 29.520099639892578, + "learning_rate": 1.4784710168044212e-07, + "logits/chosen": -0.9825581312179565, + "logits/rejected": -0.9830326437950134, + "logps/chosen": -736.2303466796875, + "logps/ref_chosen": -55.40888214111328, + "logps/ref_rejected": -97.68325805664062, + "logps/rejected": -1105.25048828125, + "loss": 1.1728, + "margin_dpo/margin_mean": 326.74578857421875, + "margin_dpo/margin_std": 542.132080078125, + "step": 458 + }, + { + "KL/chosen_KL_mean": -676.8546752929688, + "KL/mean": -901.5927734375, + "KL/rejected_KL_mean": -1126.330810546875, + "KL/std": 550.5717163085938, + "epoch": 0.6740088105726872, + "fcm_dpo/beta": 0.0008877612417563796, + "fcm_dpo/delta": -1.3709068298339844e-06, + "fcm_dpo/margin": 449.47607421875, + "fcm_dpo/q_t": 0.40948015451431274, + "grad_norm": 22.407264709472656, + "learning_rate": 1.466771464027316e-07, + "logits/chosen": -0.9684402942657471, + "logits/rejected": -1.0020538568496704, + "logps/chosen": -723.4121704101562, + "logps/ref_chosen": -46.55748748779297, + "logps/ref_rejected": -86.16854095458984, + "logps/rejected": -1212.4993896484375, + "loss": 1.1013, + "margin_dpo/margin_mean": 449.47607421875, + "margin_dpo/margin_std": 635.2473754882812, + "step": 459 + }, + { + "KL/chosen_KL_mean": -719.9354248046875, + "KL/mean": -945.6378173828125, + "KL/rejected_KL_mean": -1171.3402099609375, + "KL/std": 503.3649597167969, + "epoch": 0.6754772393538914, + "fcm_dpo/beta": 0.00090052280575037, + "fcm_dpo/delta": -0.008194062858819962, + "fcm_dpo/margin": 451.40478515625, + "fcm_dpo/q_t": 0.40608513355255127, + "grad_norm": 31.070011138916016, + "learning_rate": 1.4550991377830423e-07, + "logits/chosen": -0.9713107943534851, + "logits/rejected": -1.0081372261047363, + "logps/chosen": -771.5703125, + "logps/ref_chosen": -51.63489532470703, + "logps/ref_rejected": -104.11935424804688, + "logps/rejected": -1275.45947265625, + "loss": 1.0757, + "margin_dpo/margin_mean": 451.40478515625, + "margin_dpo/margin_std": 542.3765869140625, + "step": 460 + }, + { + "KL/chosen_KL_mean": -740.08251953125, + "KL/mean": -919.7174072265625, + "KL/rejected_KL_mean": -1099.352294921875, + "KL/std": 511.79473876953125, + "epoch": 0.6769456681350955, + "fcm_dpo/beta": 0.0009528920636512339, + "fcm_dpo/delta": 0.06786449253559113, + "fcm_dpo/margin": 359.2697448730469, + "fcm_dpo/q_t": 0.4245432913303375, + "grad_norm": 25.247751235961914, + "learning_rate": 1.4434543456482518e-07, + "logits/chosen": -1.029995083808899, + "logits/rejected": -1.0514973402023315, + "logps/chosen": -795.2644653320312, + "logps/ref_chosen": -55.18195724487305, + "logps/ref_rejected": -86.47689819335938, + "logps/rejected": -1185.8291015625, + "loss": 1.1423, + "margin_dpo/margin_mean": 359.269775390625, + "margin_dpo/margin_std": 578.6834716796875, + "step": 461 + }, + { + "KL/chosen_KL_mean": -744.9185791015625, + "KL/mean": -907.572265625, + "KL/rejected_KL_mean": -1070.2259521484375, + "KL/std": 544.3043212890625, + "epoch": 0.6784140969162996, + "fcm_dpo/beta": 0.0010181250981986523, + "fcm_dpo/delta": 0.0823717713356018, + "fcm_dpo/margin": 325.307373046875, + "fcm_dpo/q_t": 0.4292236864566803, + "grad_norm": 35.24434280395508, + "learning_rate": 1.4318373944740484e-07, + "logits/chosen": -1.0601680278778076, + "logits/rejected": -1.0431756973266602, + "logps/chosen": -814.8466796875, + "logps/ref_chosen": -69.92803192138672, + "logps/ref_rejected": -78.84111022949219, + "logps/rejected": -1149.067138671875, + "loss": 1.1657, + "margin_dpo/margin_mean": 325.307373046875, + "margin_dpo/margin_std": 593.1353759765625, + "step": 462 + }, + { + "KL/chosen_KL_mean": -733.4375, + "KL/mean": -934.1990966796875, + "KL/rejected_KL_mean": -1134.960693359375, + "KL/std": 516.4095458984375, + "epoch": 0.6798825256975036, + "fcm_dpo/beta": 0.001036065979860723, + "fcm_dpo/delta": -0.020512185990810394, + "fcm_dpo/margin": 401.523193359375, + "fcm_dpo/q_t": 0.4055131673812866, + "grad_norm": 23.178422927856445, + "learning_rate": 1.4202485903778976e-07, + "logits/chosen": -0.9877623319625854, + "logits/rejected": -1.0045299530029297, + "logps/chosen": -788.7119140625, + "logps/ref_chosen": -55.27437210083008, + "logps/ref_rejected": -89.02497863769531, + "logps/rejected": -1223.9857177734375, + "loss": 1.0915, + "margin_dpo/margin_mean": 401.523193359375, + "margin_dpo/margin_std": 563.6845092773438, + "step": 463 + }, + { + "KL/chosen_KL_mean": -697.1048583984375, + "KL/mean": -991.504150390625, + "KL/rejected_KL_mean": -1285.9033203125, + "KL/std": 571.4072265625, + "epoch": 0.6813509544787077, + "fcm_dpo/beta": 0.0008614182588644326, + "fcm_dpo/delta": -0.16394121944904327, + "fcm_dpo/margin": 588.7985229492188, + "fcm_dpo/q_t": 0.37307262420654297, + "grad_norm": 46.0228157043457, + "learning_rate": 1.4086882387355658e-07, + "logits/chosen": -0.9791674017906189, + "logits/rejected": -1.0607080459594727, + "logps/chosen": -748.0171508789062, + "logps/ref_chosen": -50.91230010986328, + "logps/ref_rejected": -102.4893798828125, + "logps/rejected": -1388.3927001953125, + "loss": 1.002, + "margin_dpo/margin_mean": 588.7985229492188, + "margin_dpo/margin_std": 583.0584716796875, + "step": 464 + }, + { + "KL/chosen_KL_mean": -694.8887939453125, + "KL/mean": -952.684814453125, + "KL/rejected_KL_mean": -1210.4808349609375, + "KL/std": 611.1224365234375, + "epoch": 0.6828193832599119, + "fcm_dpo/beta": 0.0008492013439536095, + "fcm_dpo/delta": -0.048867009580135345, + "fcm_dpo/margin": 515.5921020507812, + "fcm_dpo/q_t": 0.39614564180374146, + "grad_norm": 30.500961303710938, + "learning_rate": 1.3971566441730714e-07, + "logits/chosen": -0.9008214473724365, + "logits/rejected": -0.9379734396934509, + "logps/chosen": -755.005615234375, + "logps/ref_chosen": -60.116851806640625, + "logps/ref_rejected": -113.94602966308594, + "logps/rejected": -1324.4268798828125, + "loss": 1.0666, + "margin_dpo/margin_mean": 515.5921020507812, + "margin_dpo/margin_std": 642.5521240234375, + "step": 465 + }, + { + "KL/chosen_KL_mean": -780.0445556640625, + "KL/mean": -987.330322265625, + "KL/rejected_KL_mean": -1194.6162109375, + "KL/std": 571.94091796875, + "epoch": 0.684287812041116, + "fcm_dpo/beta": 0.0008426806889474392, + "fcm_dpo/delta": 0.06057006120681763, + "fcm_dpo/margin": 414.57159423828125, + "fcm_dpo/q_t": 0.4206228256225586, + "grad_norm": 33.371273040771484, + "learning_rate": 1.3856541105586545e-07, + "logits/chosen": -1.0010290145874023, + "logits/rejected": -1.007284164428711, + "logps/chosen": -832.9654541015625, + "logps/ref_chosen": -52.920921325683594, + "logps/ref_rejected": -90.3154296875, + "logps/rejected": -1284.931640625, + "loss": 1.1329, + "margin_dpo/margin_mean": 414.57159423828125, + "margin_dpo/margin_std": 603.785400390625, + "step": 466 + }, + { + "KL/chosen_KL_mean": -1015.46484375, + "KL/mean": -1223.3994140625, + "KL/rejected_KL_mean": -1431.3341064453125, + "KL/std": 710.5194091796875, + "epoch": 0.6857562408223201, + "fcm_dpo/beta": 0.0008342203218489885, + "fcm_dpo/delta": -0.047805603593587875, + "fcm_dpo/margin": 415.86932373046875, + "fcm_dpo/q_t": 0.42023301124572754, + "grad_norm": 45.32796859741211, + "learning_rate": 1.3741809409947729e-07, + "logits/chosen": -1.1425104141235352, + "logits/rejected": -1.14357328414917, + "logps/chosen": -1094.1806640625, + "logps/ref_chosen": -78.7158203125, + "logps/ref_rejected": -102.86019897460938, + "logps/rejected": -1534.1943359375, + "loss": 1.1881, + "margin_dpo/margin_mean": 415.86932373046875, + "margin_dpo/margin_std": 850.1127319335938, + "step": 467 + }, + { + "KL/chosen_KL_mean": -783.6282348632812, + "KL/mean": -1058.208984375, + "KL/rejected_KL_mean": -1332.789794921875, + "KL/std": 650.537109375, + "epoch": 0.6872246696035242, + "fcm_dpo/beta": 0.0008028654847294092, + "fcm_dpo/delta": -0.052505575120449066, + "fcm_dpo/margin": 549.16162109375, + "fcm_dpo/q_t": 0.40023478865623474, + "grad_norm": 24.539749145507812, + "learning_rate": 1.362737437810114e-07, + "logits/chosen": -1.0484925508499146, + "logits/rejected": -1.0707437992095947, + "logps/chosen": -853.5635986328125, + "logps/ref_chosen": -69.93536376953125, + "logps/ref_rejected": -101.02880859375, + "logps/rejected": -1433.818603515625, + "loss": 1.074, + "margin_dpo/margin_mean": 549.16162109375, + "margin_dpo/margin_std": 754.311279296875, + "step": 468 + }, + { + "KL/chosen_KL_mean": -840.949951171875, + "KL/mean": -1125.788330078125, + "KL/rejected_KL_mean": -1410.626953125, + "KL/std": 636.927001953125, + "epoch": 0.6886930983847284, + "fcm_dpo/beta": 0.0007483696099370718, + "fcm_dpo/delta": -0.038286350667476654, + "fcm_dpo/margin": 569.6768798828125, + "fcm_dpo/q_t": 0.3994740843772888, + "grad_norm": 37.96442413330078, + "learning_rate": 1.351323902551631e-07, + "logits/chosen": -1.0542805194854736, + "logits/rejected": -1.0895564556121826, + "logps/chosen": -909.0746459960938, + "logps/ref_chosen": -68.12469482421875, + "logps/ref_rejected": -104.78640747070312, + "logps/rejected": -1515.413330078125, + "loss": 1.0689, + "margin_dpo/margin_mean": 569.6769409179688, + "margin_dpo/margin_std": 686.1154174804688, + "step": 469 + }, + { + "KL/chosen_KL_mean": -708.578125, + "KL/mean": -937.5423583984375, + "KL/rejected_KL_mean": -1166.5064697265625, + "KL/std": 602.3682861328125, + "epoch": 0.6901615271659325, + "fcm_dpo/beta": 0.0007999057415872812, + "fcm_dpo/delta": 0.038524776697158813, + "fcm_dpo/margin": 457.9284362792969, + "fcm_dpo/q_t": 0.41636377573013306, + "grad_norm": 24.361658096313477, + "learning_rate": 1.339940635976592e-07, + "logits/chosen": -0.9929207563400269, + "logits/rejected": -1.0160140991210938, + "logps/chosen": -752.3699951171875, + "logps/ref_chosen": -43.791927337646484, + "logps/ref_rejected": -82.70285034179688, + "logps/rejected": -1249.2093505859375, + "loss": 1.1274, + "margin_dpo/margin_mean": 457.9284362792969, + "margin_dpo/margin_std": 697.048095703125, + "step": 470 + }, + { + "KL/chosen_KL_mean": -843.971923828125, + "KL/mean": -1043.7379150390625, + "KL/rejected_KL_mean": -1243.5037841796875, + "KL/std": 644.9224853515625, + "epoch": 0.6916299559471366, + "fcm_dpo/beta": 0.0008243155316449702, + "fcm_dpo/delta": 0.0843021422624588, + "fcm_dpo/margin": 399.5318603515625, + "fcm_dpo/q_t": 0.42968451976776123, + "grad_norm": 39.733333587646484, + "learning_rate": 1.3285879380446563e-07, + "logits/chosen": -1.1035377979278564, + "logits/rejected": -1.1254429817199707, + "logps/chosen": -907.3114624023438, + "logps/ref_chosen": -63.33952331542969, + "logps/ref_rejected": -83.61048126220703, + "logps/rejected": -1327.1142578125, + "loss": 1.1649, + "margin_dpo/margin_mean": 399.5318603515625, + "margin_dpo/margin_std": 707.281982421875, + "step": 471 + }, + { + "KL/chosen_KL_mean": -862.677734375, + "KL/mean": -1102.48583984375, + "KL/rejected_KL_mean": -1342.2938232421875, + "KL/std": 753.7301025390625, + "epoch": 0.6930983847283406, + "fcm_dpo/beta": 0.0008416681084781885, + "fcm_dpo/delta": -0.006307002156972885, + "fcm_dpo/margin": 479.61614990234375, + "fcm_dpo/q_t": 0.411409854888916, + "grad_norm": 30.461442947387695, + "learning_rate": 1.317266107909975e-07, + "logits/chosen": -1.062950849533081, + "logits/rejected": -1.0488746166229248, + "logps/chosen": -946.3438110351562, + "logps/ref_chosen": -83.66610717773438, + "logps/ref_rejected": -117.20919799804688, + "logps/rejected": -1459.5030517578125, + "loss": 1.1356, + "margin_dpo/margin_mean": 479.61614990234375, + "margin_dpo/margin_std": 826.3499755859375, + "step": 472 + }, + { + "KL/chosen_KL_mean": -1003.8850708007812, + "KL/mean": -1114.1983642578125, + "KL/rejected_KL_mean": -1224.5115966796875, + "KL/std": 738.931640625, + "epoch": 0.6945668135095447, + "fcm_dpo/beta": 0.0008559616981074214, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 220.62660217285156, + "fcm_dpo/q_t": 0.4583190083503723, + "grad_norm": 101.26480102539062, + "learning_rate": 1.3059754439133002e-07, + "logits/chosen": -1.0762083530426025, + "logits/rejected": -1.0582425594329834, + "logps/chosen": -1067.382080078125, + "logps/ref_chosen": -63.49696731567383, + "logps/ref_rejected": -81.14657592773438, + "logps/rejected": -1305.658203125, + "loss": 1.3516, + "margin_dpo/margin_mean": 220.62660217285156, + "margin_dpo/margin_std": 942.107421875, + "step": 473 + }, + { + "KL/chosen_KL_mean": -769.4073486328125, + "KL/mean": -986.923583984375, + "KL/rejected_KL_mean": -1204.43994140625, + "KL/std": 693.4749755859375, + "epoch": 0.6960352422907489, + "fcm_dpo/beta": 0.00081443547969684, + "fcm_dpo/delta": -0.10206399112939835, + "fcm_dpo/margin": 435.03271484375, + "fcm_dpo/q_t": 0.4185516834259033, + "grad_norm": 32.762020111083984, + "learning_rate": 1.2947162435741277e-07, + "logits/chosen": -1.048344612121582, + "logits/rejected": -1.0651113986968994, + "logps/chosen": -822.019287109375, + "logps/ref_chosen": -52.6119384765625, + "logps/ref_rejected": -90.08041381835938, + "logps/rejected": -1294.5203857421875, + "loss": 1.1724, + "margin_dpo/margin_mean": 435.03271484375, + "margin_dpo/margin_std": 791.048583984375, + "step": 474 + }, + { + "KL/chosen_KL_mean": -539.2615356445312, + "KL/mean": -790.7316284179688, + "KL/rejected_KL_mean": -1042.20166015625, + "KL/std": 493.97027587890625, + "epoch": 0.697503671071953, + "fcm_dpo/beta": 0.0007748671341687441, + "fcm_dpo/delta": 0.01266252901405096, + "fcm_dpo/margin": 502.940185546875, + "fcm_dpo/q_t": 0.40863102674484253, + "grad_norm": 52.324066162109375, + "learning_rate": 1.2834888035828596e-07, + "logits/chosen": -1.0229177474975586, + "logits/rejected": -1.0607411861419678, + "logps/chosen": -581.7567138671875, + "logps/ref_chosen": -42.49519348144531, + "logps/ref_rejected": -90.06294250488281, + "logps/rejected": -1132.2646484375, + "loss": 1.0766, + "margin_dpo/margin_mean": 502.940185546875, + "margin_dpo/margin_std": 544.89208984375, + "step": 475 + }, + { + "KL/chosen_KL_mean": -634.4459228515625, + "KL/mean": -844.801025390625, + "KL/rejected_KL_mean": -1055.1561279296875, + "KL/std": 564.7354736328125, + "epoch": 0.6989720998531571, + "fcm_dpo/beta": 0.0008217949070967734, + "fcm_dpo/delta": 0.06553815305233002, + "fcm_dpo/margin": 420.71014404296875, + "fcm_dpo/q_t": 0.4223518669605255, + "grad_norm": 62.65557098388672, + "learning_rate": 1.2722934197929802e-07, + "logits/chosen": -0.9976848363876343, + "logits/rejected": -1.0245535373687744, + "logps/chosen": -677.3953247070312, + "logps/ref_chosen": -42.94938278198242, + "logps/ref_rejected": -73.71023559570312, + "logps/rejected": -1128.866455078125, + "loss": 1.1221, + "margin_dpo/margin_mean": 420.71014404296875, + "margin_dpo/margin_std": 576.537353515625, + "step": 476 + }, + { + "KL/chosen_KL_mean": -674.2581787109375, + "KL/mean": -865.6505126953125, + "KL/rejected_KL_mean": -1057.0428466796875, + "KL/std": 557.9041748046875, + "epoch": 0.7004405286343612, + "fcm_dpo/beta": 0.0008936204831115901, + "fcm_dpo/delta": 0.06722961366176605, + "fcm_dpo/margin": 382.7846984863281, + "fcm_dpo/q_t": 0.42294949293136597, + "grad_norm": 28.50010108947754, + "learning_rate": 1.2611303872132631e-07, + "logits/chosen": -1.0419853925704956, + "logits/rejected": -1.0101861953735352, + "logps/chosen": -745.03076171875, + "logps/ref_chosen": -70.77261352539062, + "logps/ref_rejected": -76.13737487792969, + "logps/rejected": -1133.18017578125, + "loss": 1.1533, + "margin_dpo/margin_mean": 382.78466796875, + "margin_dpo/margin_std": 655.9279174804688, + "step": 477 + }, + { + "KL/chosen_KL_mean": -557.695068359375, + "KL/mean": -778.8765869140625, + "KL/rejected_KL_mean": -1000.0580444335938, + "KL/std": 482.09033203125, + "epoch": 0.7019089574155654, + "fcm_dpo/beta": 0.0009058309369720519, + "fcm_dpo/delta": -0.0017963144928216934, + "fcm_dpo/margin": 442.363037109375, + "fcm_dpo/q_t": 0.407336950302124, + "grad_norm": 43.02243423461914, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": -0.946250319480896, + "logits/rejected": -0.9788520336151123, + "logps/chosen": -599.1356201171875, + "logps/ref_chosen": -41.440513610839844, + "logps/ref_rejected": -85.36196899414062, + "logps/rejected": -1085.4200439453125, + "loss": 1.0866, + "margin_dpo/margin_mean": 442.363037109375, + "margin_dpo/margin_std": 562.7466430664062, + "step": 478 + }, + { + "KL/chosen_KL_mean": -741.406982421875, + "KL/mean": -950.5042114257812, + "KL/rejected_KL_mean": -1159.601318359375, + "KL/std": 631.4393920898438, + "epoch": 0.7033773861967695, + "fcm_dpo/beta": 0.0009333643829450011, + "fcm_dpo/delta": 0.004260525107383728, + "fcm_dpo/margin": 418.19439697265625, + "fcm_dpo/q_t": 0.4147453308105469, + "grad_norm": 30.091028213500977, + "learning_rate": 1.2389025514492456e-07, + "logits/chosen": -0.9998948574066162, + "logits/rejected": -1.0395790338516235, + "logps/chosen": -795.31494140625, + "logps/ref_chosen": -53.907920837402344, + "logps/ref_rejected": -95.1163330078125, + "logps/rejected": -1254.7177734375, + "loss": 1.1436, + "margin_dpo/margin_mean": 418.19439697265625, + "margin_dpo/margin_std": 732.4306640625, + "step": 479 + }, + { + "KL/chosen_KL_mean": -865.813720703125, + "KL/mean": -1064.1876220703125, + "KL/rejected_KL_mean": -1262.5615234375, + "KL/std": 623.685791015625, + "epoch": 0.7048458149779736, + "fcm_dpo/beta": 0.0009484676993452013, + "fcm_dpo/delta": 0.020787112414836884, + "fcm_dpo/margin": 396.74774169921875, + "fcm_dpo/q_t": 0.41742902994155884, + "grad_norm": 50.18581008911133, + "learning_rate": 1.227838333989088e-07, + "logits/chosen": -1.0877306461334229, + "logits/rejected": -1.0981130599975586, + "logps/chosen": -924.4964599609375, + "logps/ref_chosen": -58.682701110839844, + "logps/ref_rejected": -82.93248748779297, + "logps/rejected": -1345.493896484375, + "loss": 1.1513, + "margin_dpo/margin_mean": 396.74774169921875, + "margin_dpo/margin_std": 687.515869140625, + "step": 480 + }, + { + "KL/chosen_KL_mean": -735.0712890625, + "KL/mean": -1004.9774169921875, + "KL/rejected_KL_mean": -1274.883544921875, + "KL/std": 640.9407958984375, + "epoch": 0.7063142437591777, + "fcm_dpo/beta": 0.000862010580021888, + "fcm_dpo/delta": -0.08503904193639755, + "fcm_dpo/margin": 539.812255859375, + "fcm_dpo/q_t": 0.391613632440567, + "grad_norm": 32.593658447265625, + "learning_rate": 1.2168076391719489e-07, + "logits/chosen": -1.1026833057403564, + "logits/rejected": -1.142751693725586, + "logps/chosen": -790.0355224609375, + "logps/ref_chosen": -54.964271545410156, + "logps/ref_rejected": -92.42044067382812, + "logps/rejected": -1367.303955078125, + "loss": 1.0549, + "margin_dpo/margin_mean": 539.812255859375, + "margin_dpo/margin_std": 687.2750854492188, + "step": 481 + }, + { + "KL/chosen_KL_mean": -830.9714965820312, + "KL/mean": -957.885009765625, + "KL/rejected_KL_mean": -1084.7984619140625, + "KL/std": 625.652099609375, + "epoch": 0.7077826725403817, + "fcm_dpo/beta": 0.0008747933316044509, + "fcm_dpo/delta": 0.0788145363330841, + "fcm_dpo/margin": 253.82705688476562, + "fcm_dpo/q_t": 0.4496998190879822, + "grad_norm": 46.61983871459961, + "learning_rate": 1.2058107576668938e-07, + "logits/chosen": -0.9891968369483948, + "logits/rejected": -0.9823375940322876, + "logps/chosen": -898.5249633789062, + "logps/ref_chosen": -67.553466796875, + "logps/ref_rejected": -87.58953857421875, + "logps/rejected": -1172.387939453125, + "loss": 1.2716, + "margin_dpo/margin_mean": 253.82705688476562, + "margin_dpo/margin_std": 734.9395751953125, + "step": 482 + }, + { + "KL/chosen_KL_mean": -761.9283447265625, + "KL/mean": -1030.1768798828125, + "KL/rejected_KL_mean": -1298.425537109375, + "KL/std": 638.6688232421875, + "epoch": 0.7092511013215859, + "fcm_dpo/beta": 0.0008516391972079873, + "fcm_dpo/delta": -0.07522018998861313, + "fcm_dpo/margin": 536.4971923828125, + "fcm_dpo/q_t": 0.3947756588459015, + "grad_norm": 29.71031379699707, + "learning_rate": 1.194847979251979e-07, + "logits/chosen": -1.0698552131652832, + "logits/rejected": -1.0944595336914062, + "logps/chosen": -825.2581787109375, + "logps/ref_chosen": -63.32981872558594, + "logps/ref_rejected": -95.78697204589844, + "logps/rejected": -1394.21240234375, + "loss": 1.0606, + "margin_dpo/margin_mean": 536.4971313476562, + "margin_dpo/margin_std": 694.40087890625, + "step": 483 + }, + { + "KL/chosen_KL_mean": -655.5699462890625, + "KL/mean": -913.00341796875, + "KL/rejected_KL_mean": -1170.43701171875, + "KL/std": 608.8349609375, + "epoch": 0.71071953010279, + "fcm_dpo/beta": 0.000842659268528223, + "fcm_dpo/delta": -0.0462777316570282, + "fcm_dpo/margin": 514.8670043945312, + "fcm_dpo/q_t": 0.3998936414718628, + "grad_norm": 36.89812469482422, + "learning_rate": 1.1839195928066101e-07, + "logits/chosen": -1.0860092639923096, + "logits/rejected": -1.119147539138794, + "logps/chosen": -714.7080688476562, + "logps/ref_chosen": -59.13812255859375, + "logps/ref_rejected": -84.37144470214844, + "logps/rejected": -1254.808349609375, + "loss": 1.0775, + "margin_dpo/margin_mean": 514.8670043945312, + "margin_dpo/margin_std": 665.43310546875, + "step": 484 + }, + { + "KL/chosen_KL_mean": -662.37353515625, + "KL/mean": -922.7737426757812, + "KL/rejected_KL_mean": -1183.174072265625, + "KL/std": 650.9013671875, + "epoch": 0.7121879588839941, + "fcm_dpo/beta": 0.0007992424070835114, + "fcm_dpo/delta": -0.020606661215424538, + "fcm_dpo/margin": 520.8004760742188, + "fcm_dpo/q_t": 0.4069255590438843, + "grad_norm": 32.7896842956543, + "learning_rate": 1.1730258863039347e-07, + "logits/chosen": -0.9968384504318237, + "logits/rejected": -1.0324972867965698, + "logps/chosen": -721.2230834960938, + "logps/ref_chosen": -58.849571228027344, + "logps/ref_rejected": -103.36408233642578, + "logps/rejected": -1286.5380859375, + "loss": 1.0967, + "margin_dpo/margin_mean": 520.8004760742188, + "margin_dpo/margin_std": 764.5574951171875, + "step": 485 + }, + { + "KL/chosen_KL_mean": -828.6250610351562, + "KL/mean": -1101.12109375, + "KL/rejected_KL_mean": -1373.6173095703125, + "KL/std": 724.00244140625, + "epoch": 0.7136563876651982, + "fcm_dpo/beta": 0.0007718969136476517, + "fcm_dpo/delta": -0.02629101648926735, + "fcm_dpo/margin": 544.9921875, + "fcm_dpo/q_t": 0.4044193923473358, + "grad_norm": 39.17950439453125, + "learning_rate": 1.1621671468032493e-07, + "logits/chosen": -1.1160706281661987, + "logits/rejected": -1.1396567821502686, + "logps/chosen": -883.884765625, + "logps/ref_chosen": -55.25966262817383, + "logps/ref_rejected": -92.13936614990234, + "logps/rejected": -1465.756591796875, + "loss": 1.1214, + "margin_dpo/margin_mean": 544.9921875, + "margin_dpo/margin_std": 907.802001953125, + "step": 486 + }, + { + "KL/chosen_KL_mean": -896.8312377929688, + "KL/mean": -1101.290771484375, + "KL/rejected_KL_mean": -1305.750244140625, + "KL/std": 672.8538818359375, + "epoch": 0.7151248164464024, + "fcm_dpo/beta": 0.0008255624561570585, + "fcm_dpo/delta": 0.07200143486261368, + "fcm_dpo/margin": 408.9190673828125, + "fcm_dpo/q_t": 0.42319005727767944, + "grad_norm": 29.267309188842773, + "learning_rate": 1.1513436604424378e-07, + "logits/chosen": -1.1223185062408447, + "logits/rejected": -1.135122537612915, + "logps/chosen": -949.89453125, + "logps/ref_chosen": -53.06330871582031, + "logps/ref_rejected": -92.41883087158203, + "logps/rejected": -1398.1690673828125, + "loss": 1.1616, + "margin_dpo/margin_mean": 408.9190673828125, + "margin_dpo/margin_std": 730.7452392578125, + "step": 487 + }, + { + "KL/chosen_KL_mean": -746.6181640625, + "KL/mean": -975.8692626953125, + "KL/rejected_KL_mean": -1205.120361328125, + "KL/std": 613.75439453125, + "epoch": 0.7165932452276065, + "fcm_dpo/beta": 0.0008516233647242188, + "fcm_dpo/delta": 0.0090005062520504, + "fcm_dpo/margin": 458.5020751953125, + "fcm_dpo/q_t": 0.41139358282089233, + "grad_norm": 33.18574523925781, + "learning_rate": 1.1405557124304335e-07, + "logits/chosen": -1.0628390312194824, + "logits/rejected": -1.0790064334869385, + "logps/chosen": -798.8463745117188, + "logps/ref_chosen": -52.22815704345703, + "logps/ref_rejected": -84.00656127929688, + "logps/rejected": -1289.1268310546875, + "loss": 1.1022, + "margin_dpo/margin_mean": 458.5020751953125, + "margin_dpo/margin_std": 630.7826538085938, + "step": 488 + }, + { + "KL/chosen_KL_mean": -688.994873046875, + "KL/mean": -898.1849365234375, + "KL/rejected_KL_mean": -1107.375, + "KL/std": 607.5086669921875, + "epoch": 0.7180616740088106, + "fcm_dpo/beta": 0.0008741497294977307, + "fcm_dpo/delta": 0.03951010853052139, + "fcm_dpo/margin": 418.3800964355469, + "fcm_dpo/q_t": 0.41885343194007874, + "grad_norm": 23.560508728027344, + "learning_rate": 1.1298035870396985e-07, + "logits/chosen": -1.0819485187530518, + "logits/rejected": -1.084855318069458, + "logps/chosen": -744.9844970703125, + "logps/ref_chosen": -55.989627838134766, + "logps/ref_rejected": -79.39812469482422, + "logps/rejected": -1186.7730712890625, + "loss": 1.1243, + "margin_dpo/margin_mean": 418.380126953125, + "margin_dpo/margin_std": 633.3609619140625, + "step": 489 + }, + { + "KL/chosen_KL_mean": -841.709716796875, + "KL/mean": -1069.041015625, + "KL/rejected_KL_mean": -1296.372314453125, + "KL/std": 748.8676147460938, + "epoch": 0.7195301027900147, + "fcm_dpo/beta": 0.0008761119097471237, + "fcm_dpo/delta": 0.0016551017761230469, + "fcm_dpo/margin": 454.66265869140625, + "fcm_dpo/q_t": 0.41499996185302734, + "grad_norm": 53.128875732421875, + "learning_rate": 1.1190875675987355e-07, + "logits/chosen": -1.144291639328003, + "logits/rejected": -1.196203589439392, + "logps/chosen": -894.0761108398438, + "logps/ref_chosen": -52.36639404296875, + "logps/ref_rejected": -110.4090576171875, + "logps/rejected": -1406.7813720703125, + "loss": 1.1641, + "margin_dpo/margin_mean": 454.6626281738281, + "margin_dpo/margin_std": 890.287353515625, + "step": 490 + }, + { + "KL/chosen_KL_mean": -801.8388061523438, + "KL/mean": -956.7105102539062, + "KL/rejected_KL_mean": -1111.5821533203125, + "KL/std": 636.75390625, + "epoch": 0.7209985315712188, + "fcm_dpo/beta": 0.0009558784076943994, + "fcm_dpo/delta": 0.12077458947896957, + "fcm_dpo/margin": 309.7433776855469, + "fcm_dpo/q_t": 0.4361428916454315, + "grad_norm": 39.206363677978516, + "learning_rate": 1.1084079364846241e-07, + "logits/chosen": -1.1435798406600952, + "logits/rejected": -1.1361366510391235, + "logps/chosen": -861.955078125, + "logps/ref_chosen": -60.11626434326172, + "logps/ref_rejected": -73.27278900146484, + "logps/rejected": -1184.85498046875, + "loss": 1.1942, + "margin_dpo/margin_mean": 309.7433776855469, + "margin_dpo/margin_std": 618.770751953125, + "step": 491 + }, + { + "KL/chosen_KL_mean": -781.2764892578125, + "KL/mean": -925.2930908203125, + "KL/rejected_KL_mean": -1069.3095703125, + "KL/std": 593.3115844726562, + "epoch": 0.7224669603524229, + "fcm_dpo/beta": 0.0010589244775474072, + "fcm_dpo/delta": 0.11252547055482864, + "fcm_dpo/margin": 288.0330810546875, + "fcm_dpo/q_t": 0.43613219261169434, + "grad_norm": 65.94483184814453, + "learning_rate": 1.097764975115576e-07, + "logits/chosen": -1.2232658863067627, + "logits/rejected": -1.1998727321624756, + "logps/chosen": -835.270751953125, + "logps/ref_chosen": -53.994178771972656, + "logps/ref_rejected": -72.65962219238281, + "logps/rejected": -1141.96923828125, + "loss": 1.2344, + "margin_dpo/margin_mean": 288.0330810546875, + "margin_dpo/margin_std": 703.6218872070312, + "step": 492 + }, + { + "KL/chosen_KL_mean": -823.78759765625, + "KL/mean": -995.8330078125, + "KL/rejected_KL_mean": -1167.87841796875, + "KL/std": 663.9137573242188, + "epoch": 0.723935389133627, + "fcm_dpo/beta": 0.001078893314115703, + "fcm_dpo/delta": 0.029794402420520782, + "fcm_dpo/margin": 344.0907897949219, + "fcm_dpo/q_t": 0.41683828830718994, + "grad_norm": 46.201839447021484, + "learning_rate": 1.0871589639435203e-07, + "logits/chosen": -1.2597391605377197, + "logits/rejected": -1.2192950248718262, + "logps/chosen": -899.2848510742188, + "logps/ref_chosen": -75.49723815917969, + "logps/ref_rejected": -87.32301330566406, + "logps/rejected": -1255.201416015625, + "loss": 1.1584, + "margin_dpo/margin_mean": 344.0908203125, + "margin_dpo/margin_std": 617.4366455078125, + "step": 493 + }, + { + "KL/chosen_KL_mean": -648.2301025390625, + "KL/mean": -912.5694580078125, + "KL/rejected_KL_mean": -1176.9088134765625, + "KL/std": 621.4219360351562, + "epoch": 0.7254038179148311, + "fcm_dpo/beta": 0.000986246857792139, + "fcm_dpo/delta": -0.16815879940986633, + "fcm_dpo/margin": 528.6787109375, + "fcm_dpo/q_t": 0.37546783685684204, + "grad_norm": 32.91933822631836, + "learning_rate": 1.0765901824467166e-07, + "logits/chosen": -1.0481430292129517, + "logits/rejected": -1.0907659530639648, + "logps/chosen": -689.58935546875, + "logps/ref_chosen": -41.35926818847656, + "logps/ref_rejected": -86.09136962890625, + "logps/rejected": -1263.000244140625, + "loss": 1.0206, + "margin_dpo/margin_mean": 528.6787109375, + "margin_dpo/margin_std": 635.6686401367188, + "step": 494 + }, + { + "KL/chosen_KL_mean": -706.6998291015625, + "KL/mean": -920.7889404296875, + "KL/rejected_KL_mean": -1134.8779296875, + "KL/std": 593.68115234375, + "epoch": 0.7268722466960352, + "fcm_dpo/beta": 0.0009533504489809275, + "fcm_dpo/delta": -0.010413305833935738, + "fcm_dpo/margin": 428.17816162109375, + "fcm_dpo/q_t": 0.4118821620941162, + "grad_norm": 38.663597106933594, + "learning_rate": 1.0660589091223854e-07, + "logits/chosen": -1.2045929431915283, + "logits/rejected": -1.22074294090271, + "logps/chosen": -770.23486328125, + "logps/ref_chosen": -63.53507995605469, + "logps/ref_rejected": -91.42443084716797, + "logps/rejected": -1226.302490234375, + "loss": 1.1255, + "margin_dpo/margin_mean": 428.17816162109375, + "margin_dpo/margin_std": 734.76904296875, + "step": 495 + }, + { + "KL/chosen_KL_mean": -847.32470703125, + "KL/mean": -956.1488037109375, + "KL/rejected_KL_mean": -1064.972900390625, + "KL/std": 484.2099609375, + "epoch": 0.7283406754772394, + "fcm_dpo/beta": 0.0009861316066235304, + "fcm_dpo/delta": 0.08399142324924469, + "fcm_dpo/margin": 217.6483154296875, + "fcm_dpo/q_t": 0.45106980204582214, + "grad_norm": 73.58963012695312, + "learning_rate": 1.0555654214793722e-07, + "logits/chosen": -1.1229890584945679, + "logits/rejected": -1.0876094102859497, + "logps/chosen": -919.9166259765625, + "logps/ref_chosen": -72.5919189453125, + "logps/ref_rejected": -84.32933807373047, + "logps/rejected": -1149.30224609375, + "loss": 1.2516, + "margin_dpo/margin_mean": 217.6483154296875, + "margin_dpo/margin_std": 544.780517578125, + "step": 496 + }, + { + "KL/chosen_KL_mean": -713.8504028320312, + "KL/mean": -832.6170654296875, + "KL/rejected_KL_mean": -951.3837890625, + "KL/std": 529.62255859375, + "epoch": 0.7298091042584435, + "fcm_dpo/beta": 0.0010455565061420202, + "fcm_dpo/delta": 0.017400572076439857, + "fcm_dpo/margin": 237.53341674804688, + "fcm_dpo/q_t": 0.44449126720428467, + "grad_norm": 23.886857986450195, + "learning_rate": 1.0451099960308374e-07, + "logits/chosen": -1.10499107837677, + "logits/rejected": -1.0940041542053223, + "logps/chosen": -772.4443359375, + "logps/ref_chosen": -58.59397506713867, + "logps/ref_rejected": -76.28836822509766, + "logps/rejected": -1027.672119140625, + "loss": 1.2264, + "margin_dpo/margin_mean": 237.53341674804688, + "margin_dpo/margin_std": 521.065673828125, + "step": 497 + }, + { + "KL/chosen_KL_mean": -697.0528564453125, + "KL/mean": -902.389404296875, + "KL/rejected_KL_mean": -1107.725830078125, + "KL/std": 593.9395751953125, + "epoch": 0.7312775330396476, + "fcm_dpo/beta": 0.0010199977550655603, + "fcm_dpo/delta": -0.024301331490278244, + "fcm_dpo/margin": 410.67303466796875, + "fcm_dpo/q_t": 0.40479618310928345, + "grad_norm": 30.96971321105957, + "learning_rate": 1.0346929082869641e-07, + "logits/chosen": -1.1298916339874268, + "logits/rejected": -1.1229459047317505, + "logps/chosen": -768.258544921875, + "logps/ref_chosen": -71.20565795898438, + "logps/ref_rejected": -83.95803833007812, + "logps/rejected": -1191.6839599609375, + "loss": 1.1151, + "margin_dpo/margin_mean": 410.67303466796875, + "margin_dpo/margin_std": 663.222900390625, + "step": 498 + }, + { + "KL/chosen_KL_mean": -577.2625732421875, + "KL/mean": -813.7861328125, + "KL/rejected_KL_mean": -1050.3096923828125, + "KL/std": 576.3260498046875, + "epoch": 0.7327459618208517, + "fcm_dpo/beta": 0.0009620316559448838, + "fcm_dpo/delta": -0.0720800906419754, + "fcm_dpo/margin": 473.047119140625, + "fcm_dpo/q_t": 0.3953206241130829, + "grad_norm": 36.52189254760742, + "learning_rate": 1.0243144327477013e-07, + "logits/chosen": -1.118801474571228, + "logits/rejected": -1.1649246215820312, + "logps/chosen": -628.5177001953125, + "logps/ref_chosen": -51.25519561767578, + "logps/ref_rejected": -101.07870483398438, + "logps/rejected": -1151.388427734375, + "loss": 1.0697, + "margin_dpo/margin_mean": 473.047119140625, + "margin_dpo/margin_std": 653.6705322265625, + "step": 499 + }, + { + "KL/chosen_KL_mean": -667.104248046875, + "KL/mean": -863.0795288085938, + "KL/rejected_KL_mean": -1059.054931640625, + "KL/std": 504.5117492675781, + "epoch": 0.7342143906020558, + "fcm_dpo/beta": 0.000964233884587884, + "fcm_dpo/delta": 0.027243074029684067, + "fcm_dpo/margin": 391.9505615234375, + "fcm_dpo/q_t": 0.4149992763996124, + "grad_norm": 31.563304901123047, + "learning_rate": 1.0139748428955333e-07, + "logits/chosen": -1.1256649494171143, + "logits/rejected": -1.1642621755599976, + "logps/chosen": -724.1317138671875, + "logps/ref_chosen": -57.027442932128906, + "logps/ref_rejected": -93.93421173095703, + "logps/rejected": -1152.989013671875, + "loss": 1.1294, + "margin_dpo/margin_mean": 391.9505615234375, + "margin_dpo/margin_std": 622.8431396484375, + "step": 500 + }, + { + "KL/chosen_KL_mean": -576.75732421875, + "KL/mean": -784.8577880859375, + "KL/rejected_KL_mean": -992.958251953125, + "KL/std": 533.4993286132812, + "epoch": 0.73568281938326, + "fcm_dpo/beta": 0.0009827818721532822, + "fcm_dpo/delta": -0.012303851544857025, + "fcm_dpo/margin": 416.20098876953125, + "fcm_dpo/q_t": 0.40838170051574707, + "grad_norm": 32.86457061767578, + "learning_rate": 1.0036744111882672e-07, + "logits/chosen": -1.0381156206130981, + "logits/rejected": -1.0312669277191162, + "logps/chosen": -631.1168212890625, + "logps/ref_chosen": -54.359527587890625, + "logps/ref_rejected": -80.15670013427734, + "logps/rejected": -1073.114990234375, + "loss": 1.1167, + "margin_dpo/margin_mean": 416.2009582519531, + "margin_dpo/margin_std": 665.16796875, + "step": 501 + }, + { + "KL/chosen_KL_mean": -536.04345703125, + "KL/mean": -745.6707763671875, + "KL/rejected_KL_mean": -955.2979736328125, + "KL/std": 505.134765625, + "epoch": 0.737151248164464, + "fcm_dpo/beta": 0.0009646775433793664, + "fcm_dpo/delta": -0.00565625075250864, + "fcm_dpo/margin": 419.25445556640625, + "fcm_dpo/q_t": 0.40891292691230774, + "grad_norm": 21.100860595703125, + "learning_rate": 9.934134090518592e-08, + "logits/chosen": -1.0066035985946655, + "logits/rejected": -0.9983298182487488, + "logps/chosen": -603.64404296875, + "logps/ref_chosen": -67.60050964355469, + "logps/ref_rejected": -82.94876098632812, + "logps/rejected": -1038.246826171875, + "loss": 1.0891, + "margin_dpo/margin_mean": 419.25445556640625, + "margin_dpo/margin_std": 562.9284057617188, + "step": 502 + }, + { + "KL/chosen_KL_mean": -536.14404296875, + "KL/mean": -733.31298828125, + "KL/rejected_KL_mean": -930.4820556640625, + "KL/std": 481.45635986328125, + "epoch": 0.7386196769456681, + "fcm_dpo/beta": 0.0009646597900427878, + "fcm_dpo/delta": 0.023929888382554054, + "fcm_dpo/margin": 394.33795166015625, + "fcm_dpo/q_t": 0.4147661328315735, + "grad_norm": 21.984729766845703, + "learning_rate": 9.831921068732571e-08, + "logits/chosen": -0.999343752861023, + "logits/rejected": -0.9944263696670532, + "logps/chosen": -591.222412109375, + "logps/ref_chosen": -55.078407287597656, + "logps/ref_rejected": -82.50544738769531, + "logps/rejected": -1012.9874877929688, + "loss": 1.1043, + "margin_dpo/margin_mean": 394.33795166015625, + "margin_dpo/margin_std": 542.9268798828125, + "step": 503 + }, + { + "KL/chosen_KL_mean": -596.7254028320312, + "KL/mean": -843.4456787109375, + "KL/rejected_KL_mean": -1090.1658935546875, + "KL/std": 561.0550537109375, + "epoch": 0.7400881057268722, + "fcm_dpo/beta": 0.0009342943667434156, + "fcm_dpo/delta": -0.07896675169467926, + "fcm_dpo/margin": 493.44061279296875, + "fcm_dpo/q_t": 0.39245522022247314, + "grad_norm": 32.341758728027344, + "learning_rate": 9.730107739932805e-08, + "logits/chosen": -1.0672714710235596, + "logits/rejected": -1.1046226024627686, + "logps/chosen": -656.691162109375, + "logps/ref_chosen": -59.96575164794922, + "logps/ref_rejected": -103.76212310791016, + "logps/rejected": -1193.927978515625, + "loss": 1.058, + "margin_dpo/margin_mean": 493.4405517578125, + "margin_dpo/margin_std": 614.1143188476562, + "step": 504 + }, + { + "KL/chosen_KL_mean": -736.76806640625, + "KL/mean": -838.422607421875, + "KL/rejected_KL_mean": -940.0771484375, + "KL/std": 535.1500244140625, + "epoch": 0.7415565345080763, + "fcm_dpo/beta": 0.0009070243104360998, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 203.3091583251953, + "fcm_dpo/q_t": 0.45628899335861206, + "grad_norm": 33.11906814575195, + "learning_rate": 9.628696786995188e-08, + "logits/chosen": -1.0850073099136353, + "logits/rejected": -1.0549143552780151, + "logps/chosen": -812.9229736328125, + "logps/ref_chosen": -76.1549072265625, + "logps/ref_rejected": -88.58537292480469, + "logps/rejected": -1028.6624755859375, + "loss": 1.2666, + "margin_dpo/margin_mean": 203.30914306640625, + "margin_dpo/margin_std": 520.629638671875, + "step": 505 + }, + { + "KL/chosen_KL_mean": -579.7349853515625, + "KL/mean": -793.5402221679688, + "KL/rejected_KL_mean": -1007.345458984375, + "KL/std": 549.4849853515625, + "epoch": 0.7430249632892805, + "fcm_dpo/beta": 0.0009010919602587819, + "fcm_dpo/delta": 0.016882240772247314, + "fcm_dpo/margin": 427.61041259765625, + "fcm_dpo/q_t": 0.4115670919418335, + "grad_norm": 39.05805587768555, + "learning_rate": 9.527690882192635e-08, + "logits/chosen": -1.088112235069275, + "logits/rejected": -1.1115524768829346, + "logps/chosen": -628.695556640625, + "logps/ref_chosen": -48.96050262451172, + "logps/ref_rejected": -78.41505432128906, + "logps/rejected": -1085.760498046875, + "loss": 1.1101, + "margin_dpo/margin_mean": 427.61041259765625, + "margin_dpo/margin_std": 606.4409790039062, + "step": 506 + }, + { + "KL/chosen_KL_mean": -689.96875, + "KL/mean": -874.892578125, + "KL/rejected_KL_mean": -1059.81640625, + "KL/std": 653.38037109375, + "epoch": 0.7444933920704846, + "fcm_dpo/beta": 0.0009622873039916158, + "fcm_dpo/delta": 0.053409378975629807, + "fcm_dpo/margin": 369.84759521484375, + "fcm_dpo/q_t": 0.42470353841781616, + "grad_norm": 33.041019439697266, + "learning_rate": 9.427092687124691e-08, + "logits/chosen": -1.1047194004058838, + "logits/rejected": -1.1137211322784424, + "logps/chosen": -756.770263671875, + "logps/ref_chosen": -66.80149841308594, + "logps/ref_rejected": -95.37289428710938, + "logps/rejected": -1155.1893310546875, + "loss": 1.1655, + "margin_dpo/margin_mean": 369.84759521484375, + "margin_dpo/margin_std": 710.9828491210938, + "step": 507 + }, + { + "KL/chosen_KL_mean": -782.4303588867188, + "KL/mean": -958.6895751953125, + "KL/rejected_KL_mean": -1134.94873046875, + "KL/std": 643.3538818359375, + "epoch": 0.7459618208516887, + "fcm_dpo/beta": 0.001016983762383461, + "fcm_dpo/delta": 0.04988085478544235, + "fcm_dpo/margin": 352.5183410644531, + "fcm_dpo/q_t": 0.4246112108230591, + "grad_norm": 35.355621337890625, + "learning_rate": 9.326904852647344e-08, + "logits/chosen": -1.1216602325439453, + "logits/rejected": -1.1300599575042725, + "logps/chosen": -853.7338256835938, + "logps/ref_chosen": -71.303466796875, + "logps/ref_rejected": -95.6275405883789, + "logps/rejected": -1230.576171875, + "loss": 1.1984, + "margin_dpo/margin_mean": 352.51837158203125, + "margin_dpo/margin_std": 768.9783935546875, + "step": 508 + }, + { + "KL/chosen_KL_mean": -592.2100830078125, + "KL/mean": -759.6229248046875, + "KL/rejected_KL_mean": -927.03564453125, + "KL/std": 474.44573974609375, + "epoch": 0.7474302496328928, + "fcm_dpo/beta": 0.0010790289379656315, + "fcm_dpo/delta": 0.04428262263536453, + "fcm_dpo/margin": 334.8254699707031, + "fcm_dpo/q_t": 0.4207102656364441, + "grad_norm": 41.111671447753906, + "learning_rate": 9.227130018803195e-08, + "logits/chosen": -0.9959691166877747, + "logits/rejected": -0.9935662150382996, + "logps/chosen": -656.029052734375, + "logps/ref_chosen": -63.81895065307617, + "logps/ref_rejected": -83.25643920898438, + "logps/rejected": -1010.2921142578125, + "loss": 1.1492, + "margin_dpo/margin_mean": 334.8254699707031, + "margin_dpo/margin_std": 584.1224975585938, + "step": 509 + }, + { + "KL/chosen_KL_mean": -746.1397705078125, + "KL/mean": -961.1923828125, + "KL/rejected_KL_mean": -1176.244873046875, + "KL/std": 542.093017578125, + "epoch": 0.748898678414097, + "fcm_dpo/beta": 0.0010361545719206333, + "fcm_dpo/delta": -0.05862744525074959, + "fcm_dpo/margin": 430.10516357421875, + "fcm_dpo/q_t": 0.394826740026474, + "grad_norm": 30.757688522338867, + "learning_rate": 9.127770814751932e-08, + "logits/chosen": -1.0323694944381714, + "logits/rejected": -1.0630911588668823, + "logps/chosen": -798.0181884765625, + "logps/ref_chosen": -51.878448486328125, + "logps/ref_rejected": -102.7651596069336, + "logps/rejected": -1279.0101318359375, + "loss": 1.0551, + "margin_dpo/margin_mean": 430.10516357421875, + "margin_dpo/margin_std": 506.75775146484375, + "step": 510 + }, + { + "KL/chosen_KL_mean": -670.8756103515625, + "KL/mean": -856.5685424804688, + "KL/rejected_KL_mean": -1042.261474609375, + "KL/std": 570.9805908203125, + "epoch": 0.750367107195301, + "fcm_dpo/beta": 0.001025435863994062, + "fcm_dpo/delta": 0.02368216961622238, + "fcm_dpo/margin": 371.38592529296875, + "fcm_dpo/q_t": 0.415951669216156, + "grad_norm": 40.323726654052734, + "learning_rate": 9.028829858700973e-08, + "logits/chosen": -1.109389305114746, + "logits/rejected": -1.1236944198608398, + "logps/chosen": -731.1136474609375, + "logps/ref_chosen": -60.23811721801758, + "logps/ref_rejected": -92.85676574707031, + "logps/rejected": -1135.1181640625, + "loss": 1.155, + "margin_dpo/margin_mean": 371.38592529296875, + "margin_dpo/margin_std": 696.55078125, + "step": 511 + }, + { + "KL/chosen_KL_mean": -533.606201171875, + "KL/mean": -791.3221435546875, + "KL/rejected_KL_mean": -1049.0382080078125, + "KL/std": 548.933349609375, + "epoch": 0.7518355359765051, + "fcm_dpo/beta": 0.000933335511945188, + "fcm_dpo/delta": -0.11121343821287155, + "fcm_dpo/margin": 515.4320068359375, + "fcm_dpo/q_t": 0.385158896446228, + "grad_norm": 59.88717269897461, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": -0.9930525422096252, + "logits/rejected": -1.025518536567688, + "logps/chosen": -588.51171875, + "logps/ref_chosen": -54.905494689941406, + "logps/ref_rejected": -81.87586975097656, + "logps/rejected": -1130.9140625, + "loss": 1.0262, + "margin_dpo/margin_mean": 515.4320068359375, + "margin_dpo/margin_std": 572.5833740234375, + "step": 512 + }, + { + "KL/chosen_KL_mean": -714.98828125, + "KL/mean": -890.2576904296875, + "KL/rejected_KL_mean": -1065.5272216796875, + "KL/std": 549.4346923828125, + "epoch": 0.7533039647577092, + "fcm_dpo/beta": 0.0008974907686933875, + "fcm_dpo/delta": -0.028862863779067993, + "fcm_dpo/margin": 350.5388488769531, + "fcm_dpo/q_t": 0.4272831082344055, + "grad_norm": 52.60966491699219, + "learning_rate": 8.832213108254863e-08, + "logits/chosen": -1.1262786388397217, + "logits/rejected": -1.1136434078216553, + "logps/chosen": -779.90478515625, + "logps/ref_chosen": -64.91644287109375, + "logps/ref_rejected": -76.06245422363281, + "logps/rejected": -1141.589599609375, + "loss": 1.1783, + "margin_dpo/margin_mean": 350.5388488769531, + "margin_dpo/margin_std": 643.74267578125, + "step": 513 + }, + { + "KL/chosen_KL_mean": -711.4935302734375, + "KL/mean": -886.0343017578125, + "KL/rejected_KL_mean": -1060.5751953125, + "KL/std": 575.5819091796875, + "epoch": 0.7547723935389133, + "fcm_dpo/beta": 0.0009602649370208383, + "fcm_dpo/delta": 0.07626126706600189, + "fcm_dpo/margin": 349.08160400390625, + "fcm_dpo/q_t": 0.4296727478504181, + "grad_norm": 45.58797073364258, + "learning_rate": 8.734542494893954e-08, + "logits/chosen": -1.0165659189224243, + "logits/rejected": -1.0113214254379272, + "logps/chosen": -785.72314453125, + "logps/ref_chosen": -74.22957611083984, + "logps/ref_rejected": -78.945556640625, + "logps/rejected": -1139.520751953125, + "loss": 1.1706, + "margin_dpo/margin_mean": 349.08160400390625, + "margin_dpo/margin_std": 668.0552978515625, + "step": 514 + }, + { + "KL/chosen_KL_mean": -587.929443359375, + "KL/mean": -739.9951171875, + "KL/rejected_KL_mean": -892.060791015625, + "KL/std": 471.86102294921875, + "epoch": 0.7562408223201175, + "fcm_dpo/beta": 0.0010444549843668938, + "fcm_dpo/delta": 0.09740975499153137, + "fcm_dpo/margin": 304.1312255859375, + "fcm_dpo/q_t": 0.42857980728149414, + "grad_norm": 32.755496978759766, + "learning_rate": 8.637300491465272e-08, + "logits/chosen": -0.9877028465270996, + "logits/rejected": -1.01090669631958, + "logps/chosen": -638.3310546875, + "logps/ref_chosen": -50.40156555175781, + "logps/ref_rejected": -87.09774780273438, + "logps/rejected": -979.1585083007812, + "loss": 1.1743, + "margin_dpo/margin_mean": 304.1312561035156, + "margin_dpo/margin_std": 567.49853515625, + "step": 515 + }, + { + "KL/chosen_KL_mean": -636.4197998046875, + "KL/mean": -818.3901977539062, + "KL/rejected_KL_mean": -1000.3605346679688, + "KL/std": 521.9057006835938, + "epoch": 0.7577092511013216, + "fcm_dpo/beta": 0.001094970153644681, + "fcm_dpo/delta": -0.001060720533132553, + "fcm_dpo/margin": 363.9407958984375, + "fcm_dpo/q_t": 0.40709108114242554, + "grad_norm": 48.295448303222656, + "learning_rate": 8.540489660386064e-08, + "logits/chosen": -1.0736280679702759, + "logits/rejected": -1.1098659038543701, + "logps/chosen": -701.0693359375, + "logps/ref_chosen": -64.64956665039062, + "logps/ref_rejected": -111.72237396240234, + "logps/rejected": -1112.0830078125, + "loss": 1.1144, + "margin_dpo/margin_mean": 363.9407958984375, + "margin_dpo/margin_std": 547.8302001953125, + "step": 516 + }, + { + "KL/chosen_KL_mean": -603.1934814453125, + "KL/mean": -841.0816650390625, + "KL/rejected_KL_mean": -1078.9698486328125, + "KL/std": 555.549560546875, + "epoch": 0.7591776798825257, + "fcm_dpo/beta": 0.0009808378526940942, + "fcm_dpo/delta": -0.0903298482298851, + "fcm_dpo/margin": 475.7763671875, + "fcm_dpo/q_t": 0.39226794242858887, + "grad_norm": 38.17343521118164, + "learning_rate": 8.444112552711752e-08, + "logits/chosen": -1.0043656826019287, + "logits/rejected": -1.0225131511688232, + "logps/chosen": -664.1070556640625, + "logps/ref_chosen": -60.913551330566406, + "logps/ref_rejected": -89.08308410644531, + "logps/rejected": -1168.052978515625, + "loss": 1.0563, + "margin_dpo/margin_mean": 475.7763671875, + "margin_dpo/margin_std": 629.1284790039062, + "step": 517 + }, + { + "KL/chosen_KL_mean": -587.3173828125, + "KL/mean": -774.2802734375, + "KL/rejected_KL_mean": -961.2432250976562, + "KL/std": 448.082275390625, + "epoch": 0.7606461086637298, + "fcm_dpo/beta": 0.000986230792477727, + "fcm_dpo/delta": 0.03777404874563217, + "fcm_dpo/margin": 373.92578125, + "fcm_dpo/q_t": 0.4149107336997986, + "grad_norm": 56.80478286743164, + "learning_rate": 8.348171708068747e-08, + "logits/chosen": -1.0099992752075195, + "logits/rejected": -1.0327041149139404, + "logps/chosen": -644.7733154296875, + "logps/ref_chosen": -57.45589065551758, + "logps/ref_rejected": -85.31269836425781, + "logps/rejected": -1046.555908203125, + "loss": 1.111, + "margin_dpo/margin_mean": 373.92578125, + "margin_dpo/margin_std": 499.00146484375, + "step": 518 + }, + { + "KL/chosen_KL_mean": -576.0409545898438, + "KL/mean": -715.2539672851562, + "KL/rejected_KL_mean": -854.4669189453125, + "KL/std": 430.7130432128906, + "epoch": 0.762114537444934, + "fcm_dpo/beta": 0.0011134599335491657, + "fcm_dpo/delta": 0.10244397819042206, + "fcm_dpo/margin": 278.426025390625, + "fcm_dpo/q_t": 0.4303070902824402, + "grad_norm": 32.800045013427734, + "learning_rate": 8.25266965458755e-08, + "logits/chosen": -0.9758340716362, + "logits/rejected": -0.9646449089050293, + "logps/chosen": -650.104248046875, + "logps/ref_chosen": -74.06331634521484, + "logps/ref_rejected": -104.44416809082031, + "logps/rejected": -958.9111328125, + "loss": 1.1866, + "margin_dpo/margin_mean": 278.426025390625, + "margin_dpo/margin_std": 533.133544921875, + "step": 519 + }, + { + "KL/chosen_KL_mean": -643.1387939453125, + "KL/mean": -824.5906372070312, + "KL/rejected_KL_mean": -1006.0426025390625, + "KL/std": 498.93865966796875, + "epoch": 0.7635829662261381, + "fcm_dpo/beta": 0.0011077395174652338, + "fcm_dpo/delta": -0.002978809177875519, + "fcm_dpo/margin": 362.90374755859375, + "fcm_dpo/q_t": 0.4119797348976135, + "grad_norm": 35.526668548583984, + "learning_rate": 8.15760890883607e-08, + "logits/chosen": -0.9478990435600281, + "logits/rejected": -0.9614932537078857, + "logps/chosen": -713.4385986328125, + "logps/ref_chosen": -70.2998275756836, + "logps/ref_rejected": -99.98133850097656, + "logps/rejected": -1106.02392578125, + "loss": 1.1183, + "margin_dpo/margin_mean": 362.90374755859375, + "margin_dpo/margin_std": 579.5814208984375, + "step": 520 + }, + { + "KL/chosen_KL_mean": -554.781005859375, + "KL/mean": -760.6822509765625, + "KL/rejected_KL_mean": -966.5836181640625, + "KL/std": 510.9144287109375, + "epoch": 0.7650513950073421, + "fcm_dpo/beta": 0.001102518173865974, + "fcm_dpo/delta": -0.07349302619695663, + "fcm_dpo/margin": 411.80267333984375, + "fcm_dpo/q_t": 0.394964337348938, + "grad_norm": 34.72807312011719, + "learning_rate": 8.062991975753378e-08, + "logits/chosen": -1.0138747692108154, + "logits/rejected": -1.0302083492279053, + "logps/chosen": -612.9238891601562, + "logps/ref_chosen": -58.14292526245117, + "logps/ref_rejected": -83.28060913085938, + "logps/rejected": -1049.8642578125, + "loss": 1.0735, + "margin_dpo/margin_mean": 411.80267333984375, + "margin_dpo/margin_std": 541.2650146484375, + "step": 521 + }, + { + "KL/chosen_KL_mean": -658.7055053710938, + "KL/mean": -834.223388671875, + "KL/rejected_KL_mean": -1009.7413330078125, + "KL/std": 540.191162109375, + "epoch": 0.7665198237885462, + "fcm_dpo/beta": 0.0010687486501410604, + "fcm_dpo/delta": 0.030154742300510406, + "fcm_dpo/margin": 351.0357666015625, + "fcm_dpo/q_t": 0.4162691831588745, + "grad_norm": 32.86531448364258, + "learning_rate": 7.968821348583643e-08, + "logits/chosen": -1.0368986129760742, + "logits/rejected": -1.046702265739441, + "logps/chosen": -705.253173828125, + "logps/ref_chosen": -46.54766845703125, + "logps/ref_rejected": -66.01388549804688, + "logps/rejected": -1075.755126953125, + "loss": 1.1409, + "margin_dpo/margin_mean": 351.0357666015625, + "margin_dpo/margin_std": 594.4010009765625, + "step": 522 + }, + { + "KL/chosen_KL_mean": -706.0521240234375, + "KL/mean": -906.0491333007812, + "KL/rejected_KL_mean": -1106.046142578125, + "KL/std": 648.95263671875, + "epoch": 0.7679882525697503, + "fcm_dpo/beta": 0.0010503309313207865, + "fcm_dpo/delta": -0.025494202971458435, + "fcm_dpo/margin": 399.9941101074219, + "fcm_dpo/q_t": 0.4063454568386078, + "grad_norm": 37.535797119140625, + "learning_rate": 7.875099508810484e-08, + "logits/chosen": -1.0674784183502197, + "logits/rejected": -1.0807902812957764, + "logps/chosen": -767.8216552734375, + "logps/ref_chosen": -61.76960372924805, + "logps/ref_rejected": -83.76141357421875, + "logps/rejected": -1189.8076171875, + "loss": 1.1229, + "margin_dpo/margin_mean": 399.9941101074219, + "margin_dpo/margin_std": 670.1151733398438, + "step": 523 + }, + { + "KL/chosen_KL_mean": -687.168212890625, + "KL/mean": -907.5570068359375, + "KL/rejected_KL_mean": -1127.94580078125, + "KL/std": 601.67529296875, + "epoch": 0.7694566813509545, + "fcm_dpo/beta": 0.0009792076889425516, + "fcm_dpo/delta": -0.04597056284546852, + "fcm_dpo/margin": 440.77752685546875, + "fcm_dpo/q_t": 0.3970939517021179, + "grad_norm": 35.97694396972656, + "learning_rate": 7.781828926091535e-08, + "logits/chosen": -1.1234776973724365, + "logits/rejected": -1.125186562538147, + "logps/chosen": -765.240234375, + "logps/ref_chosen": -78.0720443725586, + "logps/ref_rejected": -81.30198669433594, + "logps/rejected": -1209.247802734375, + "loss": 1.0894, + "margin_dpo/margin_mean": 440.77752685546875, + "margin_dpo/margin_std": 615.4475708007812, + "step": 524 + }, + { + "KL/chosen_KL_mean": -658.93115234375, + "KL/mean": -915.503173828125, + "KL/rejected_KL_mean": -1172.0750732421875, + "KL/std": 587.5389404296875, + "epoch": 0.7709251101321586, + "fcm_dpo/beta": 0.0009053426329046488, + "fcm_dpo/delta": -0.09118733555078506, + "fcm_dpo/margin": 513.14404296875, + "fcm_dpo/q_t": 0.3911314010620117, + "grad_norm": 35.581817626953125, + "learning_rate": 7.689012058193384e-08, + "logits/chosen": -1.0382120609283447, + "logits/rejected": -1.0887622833251953, + "logps/chosen": -709.7589721679688, + "logps/ref_chosen": -50.827857971191406, + "logps/ref_rejected": -100.05294036865234, + "logps/rejected": -1272.1280517578125, + "loss": 1.0522, + "margin_dpo/margin_mean": 513.1439819335938, + "margin_dpo/margin_std": 641.33203125, + "step": 525 + }, + { + "KL/chosen_KL_mean": -707.2711791992188, + "KL/mean": -966.6031494140625, + "KL/rejected_KL_mean": -1225.93505859375, + "KL/std": 583.5233154296875, + "epoch": 0.7723935389133627, + "fcm_dpo/beta": 0.0008811709121800959, + "fcm_dpo/delta": -0.07432842254638672, + "fcm_dpo/margin": 518.663818359375, + "fcm_dpo/q_t": 0.3920350670814514, + "grad_norm": 27.579317092895508, + "learning_rate": 7.596651350926836e-08, + "logits/chosen": -1.0754790306091309, + "logits/rejected": -1.084367036819458, + "logps/chosen": -770.4384155273438, + "logps/ref_chosen": -63.167236328125, + "logps/ref_rejected": -86.30934143066406, + "logps/rejected": -1312.244384765625, + "loss": 1.0666, + "margin_dpo/margin_mean": 518.663818359375, + "margin_dpo/margin_std": 673.5228271484375, + "step": 526 + }, + { + "KL/chosen_KL_mean": -777.7960815429688, + "KL/mean": -947.6534423828125, + "KL/rejected_KL_mean": -1117.5108642578125, + "KL/std": 600.279296875, + "epoch": 0.7738619676945668, + "fcm_dpo/beta": 0.000911904382519424, + "fcm_dpo/delta": 0.10690906643867493, + "fcm_dpo/margin": 339.7147521972656, + "fcm_dpo/q_t": 0.4307979345321655, + "grad_norm": 33.39142990112305, + "learning_rate": 7.504749238082414e-08, + "logits/chosen": -1.33701753616333, + "logits/rejected": -1.3142709732055664, + "logps/chosen": -848.9248046875, + "logps/ref_chosen": -71.12867736816406, + "logps/ref_rejected": -78.3425521850586, + "logps/rejected": -1195.853515625, + "loss": 1.154, + "margin_dpo/margin_mean": 339.71478271484375, + "margin_dpo/margin_std": 517.4578857421875, + "step": 527 + }, + { + "KL/chosen_KL_mean": -758.856689453125, + "KL/mean": -977.1895751953125, + "KL/rejected_KL_mean": -1195.5224609375, + "KL/std": 594.4635620117188, + "epoch": 0.775330396475771, + "fcm_dpo/beta": 0.0009343413985334337, + "fcm_dpo/delta": -0.010038571432232857, + "fcm_dpo/margin": 436.66583251953125, + "fcm_dpo/q_t": 0.4094654321670532, + "grad_norm": 30.87571144104004, + "learning_rate": 7.413308141366254e-08, + "logits/chosen": -1.206683874130249, + "logits/rejected": -1.1996240615844727, + "logps/chosen": -826.9461059570312, + "logps/ref_chosen": -68.0894546508789, + "logps/ref_rejected": -93.91006469726562, + "logps/rejected": -1289.4324951171875, + "loss": 1.1199, + "margin_dpo/margin_mean": 436.66583251953125, + "margin_dpo/margin_std": 710.944580078125, + "step": 528 + }, + { + "KL/chosen_KL_mean": -954.2504272460938, + "KL/mean": -1106.777099609375, + "KL/rejected_KL_mean": -1259.303955078125, + "KL/std": 612.4547119140625, + "epoch": 0.7767988252569751, + "fcm_dpo/beta": 0.0009363851859234273, + "fcm_dpo/delta": 0.0063527897000312805, + "fcm_dpo/margin": 305.05340576171875, + "fcm_dpo/q_t": 0.43478527665138245, + "grad_norm": 38.029884338378906, + "learning_rate": 7.322330470336313e-08, + "logits/chosen": -1.2603811025619507, + "logits/rejected": -1.278627872467041, + "logps/chosen": -1009.825439453125, + "logps/ref_chosen": -55.57495880126953, + "logps/ref_rejected": -89.20909118652344, + "logps/rejected": -1348.512939453125, + "loss": 1.2423, + "margin_dpo/margin_mean": 305.0534362792969, + "margin_dpo/margin_std": 756.2225341796875, + "step": 529 + }, + { + "KL/chosen_KL_mean": -787.18115234375, + "KL/mean": -1030.659912109375, + "KL/rejected_KL_mean": -1274.138671875, + "KL/std": 670.89990234375, + "epoch": 0.7782672540381792, + "fcm_dpo/beta": 0.00090701476437971, + "fcm_dpo/delta": -0.053333625197410583, + "fcm_dpo/margin": 486.95751953125, + "fcm_dpo/q_t": 0.40362924337387085, + "grad_norm": 35.42485427856445, + "learning_rate": 7.231818622338822e-08, + "logits/chosen": -1.1458381414413452, + "logits/rejected": -1.1499226093292236, + "logps/chosen": -834.7825927734375, + "logps/ref_chosen": -47.601417541503906, + "logps/ref_rejected": -87.2845230102539, + "logps/rejected": -1361.4232177734375, + "loss": 1.1316, + "margin_dpo/margin_mean": 486.95745849609375, + "margin_dpo/margin_std": 884.4109497070312, + "step": 530 + }, + { + "KL/chosen_KL_mean": -894.1881713867188, + "KL/mean": -1081.9078369140625, + "KL/rejected_KL_mean": -1269.6273193359375, + "KL/std": 714.610595703125, + "epoch": 0.7797356828193832, + "fcm_dpo/beta": 0.0009350181790068746, + "fcm_dpo/delta": 0.05843368172645569, + "fcm_dpo/margin": 375.4392395019531, + "fcm_dpo/q_t": 0.42438310384750366, + "grad_norm": 46.751792907714844, + "learning_rate": 7.141774982445147e-08, + "logits/chosen": -1.278576374053955, + "logits/rejected": -1.2574585676193237, + "logps/chosen": -949.4342041015625, + "logps/ref_chosen": -55.246063232421875, + "logps/ref_rejected": -70.60598754882812, + "logps/rejected": -1340.2333984375, + "loss": 1.1589, + "margin_dpo/margin_mean": 375.439208984375, + "margin_dpo/margin_std": 681.22314453125, + "step": 531 + }, + { + "KL/chosen_KL_mean": -914.6450805664062, + "KL/mean": -1124.872314453125, + "KL/rejected_KL_mean": -1335.0997314453125, + "KL/std": 674.3599853515625, + "epoch": 0.7812041116005873, + "fcm_dpo/beta": 0.0009111958206631243, + "fcm_dpo/delta": 0.013179205358028412, + "fcm_dpo/margin": 420.45458984375, + "fcm_dpo/q_t": 0.4144301414489746, + "grad_norm": 74.0493392944336, + "learning_rate": 7.052201923388953e-08, + "logits/chosen": -1.2184380292892456, + "logits/rejected": -1.1921205520629883, + "logps/chosen": -984.93115234375, + "logps/ref_chosen": -70.28601837158203, + "logps/ref_rejected": -86.5913314819336, + "logps/rejected": -1421.6910400390625, + "loss": 1.184, + "margin_dpo/margin_mean": 420.4546203613281, + "margin_dpo/margin_std": 841.721435546875, + "step": 532 + }, + { + "KL/chosen_KL_mean": -811.30615234375, + "KL/mean": -966.7421875, + "KL/rejected_KL_mean": -1122.1781005859375, + "KL/std": 577.7342529296875, + "epoch": 0.7826725403817915, + "fcm_dpo/beta": 0.0009565654909238219, + "fcm_dpo/delta": 0.0060767000541090965, + "fcm_dpo/margin": 310.87200927734375, + "fcm_dpo/q_t": 0.4336027204990387, + "grad_norm": 33.39649200439453, + "learning_rate": 6.963101805503646e-08, + "logits/chosen": -1.299651861190796, + "logits/rejected": -1.2833778858184814, + "logps/chosen": -876.1612548828125, + "logps/ref_chosen": -64.8551025390625, + "logps/ref_rejected": -76.58805847167969, + "logps/rejected": -1198.76611328125, + "loss": 1.2121, + "margin_dpo/margin_mean": 310.87200927734375, + "margin_dpo/margin_std": 690.0610961914062, + "step": 533 + }, + { + "KL/chosen_KL_mean": -830.42724609375, + "KL/mean": -1037.4508056640625, + "KL/rejected_KL_mean": -1244.474365234375, + "KL/std": 619.809814453125, + "epoch": 0.7841409691629956, + "fcm_dpo/beta": 0.0009335308568552136, + "fcm_dpo/delta": 0.01120779663324356, + "fcm_dpo/margin": 414.0472412109375, + "fcm_dpo/q_t": 0.41367307305336, + "grad_norm": 32.04544448852539, + "learning_rate": 6.874476976660184e-08, + "logits/chosen": -1.2206509113311768, + "logits/rejected": -1.2227022647857666, + "logps/chosen": -890.546630859375, + "logps/ref_chosen": -60.119388580322266, + "logps/ref_rejected": -78.54347229003906, + "logps/rejected": -1323.017822265625, + "loss": 1.1307, + "margin_dpo/margin_mean": 414.0472412109375, + "margin_dpo/margin_std": 661.77587890625, + "step": 534 + }, + { + "KL/chosen_KL_mean": -699.33740234375, + "KL/mean": -953.7452392578125, + "KL/rejected_KL_mean": -1208.1529541015625, + "KL/std": 633.1478271484375, + "epoch": 0.7856093979441997, + "fcm_dpo/beta": 0.0009318743832409382, + "fcm_dpo/delta": -0.09808388352394104, + "fcm_dpo/margin": 508.8156433105469, + "fcm_dpo/q_t": 0.390717089176178, + "grad_norm": 24.964298248291016, + "learning_rate": 6.786329772205246e-08, + "logits/chosen": -1.1217422485351562, + "logits/rejected": -1.1360340118408203, + "logps/chosen": -753.6676635742188, + "logps/ref_chosen": -54.330238342285156, + "logps/ref_rejected": -96.30763244628906, + "logps/rejected": -1304.460693359375, + "loss": 1.0588, + "margin_dpo/margin_mean": 508.81561279296875, + "margin_dpo/margin_std": 653.384521484375, + "step": 535 + }, + { + "KL/chosen_KL_mean": -622.41259765625, + "KL/mean": -909.5772705078125, + "KL/rejected_KL_mean": -1196.741943359375, + "KL/std": 661.69384765625, + "epoch": 0.7870778267254038, + "fcm_dpo/beta": 0.0008232737891376019, + "fcm_dpo/delta": -0.09523215144872665, + "fcm_dpo/margin": 574.329345703125, + "fcm_dpo/q_t": 0.3940427899360657, + "grad_norm": 29.166486740112305, + "learning_rate": 6.698662514899638e-08, + "logits/chosen": -1.0844101905822754, + "logits/rejected": -1.1293668746948242, + "logps/chosen": -669.4931030273438, + "logps/ref_chosen": -47.08053207397461, + "logps/ref_rejected": -89.09783935546875, + "logps/rejected": -1285.83984375, + "loss": 1.0587, + "margin_dpo/margin_mean": 574.329345703125, + "margin_dpo/margin_std": 803.4403076171875, + "step": 536 + }, + { + "KL/chosen_KL_mean": -635.1380615234375, + "KL/mean": -826.725830078125, + "KL/rejected_KL_mean": -1018.3135375976562, + "KL/std": 565.7158203125, + "epoch": 0.788546255506608, + "fcm_dpo/beta": 0.0007962306845001876, + "fcm_dpo/delta": -0.008790962398052216, + "fcm_dpo/margin": 383.1754150390625, + "fcm_dpo/q_t": 0.42851394414901733, + "grad_norm": 35.423980712890625, + "learning_rate": 6.611477514857114e-08, + "logits/chosen": -1.1369528770446777, + "logits/rejected": -1.1194478273391724, + "logps/chosen": -692.8856201171875, + "logps/ref_chosen": -57.747467041015625, + "logps/ref_rejected": -70.43838500976562, + "logps/rejected": -1088.751953125, + "loss": 1.1758, + "margin_dpo/margin_mean": 383.1754150390625, + "margin_dpo/margin_std": 672.041015625, + "step": 537 + }, + { + "KL/chosen_KL_mean": -784.5943603515625, + "KL/mean": -1004.52880859375, + "KL/rejected_KL_mean": -1224.46337890625, + "KL/std": 607.7210693359375, + "epoch": 0.7900146842878121, + "fcm_dpo/beta": 0.0008164436439983547, + "fcm_dpo/delta": 0.049948450177907944, + "fcm_dpo/margin": 439.8689270019531, + "fcm_dpo/q_t": 0.4188556671142578, + "grad_norm": 27.3602237701416, + "learning_rate": 6.524777069483525e-08, + "logits/chosen": -1.1543678045272827, + "logits/rejected": -1.1441106796264648, + "logps/chosen": -851.0103149414062, + "logps/ref_chosen": -66.41594696044922, + "logps/ref_rejected": -84.22808837890625, + "logps/rejected": -1308.69140625, + "loss": 1.1229, + "margin_dpo/margin_mean": 439.86895751953125, + "margin_dpo/margin_std": 635.7586669921875, + "step": 538 + }, + { + "KL/chosen_KL_mean": -652.3861083984375, + "KL/mean": -859.1839599609375, + "KL/rejected_KL_mean": -1065.981689453125, + "KL/std": 471.9359130859375, + "epoch": 0.7914831130690162, + "fcm_dpo/beta": 0.0008725939551368356, + "fcm_dpo/delta": 0.04645565152168274, + "fcm_dpo/margin": 413.5956115722656, + "fcm_dpo/q_t": 0.417635977268219, + "grad_norm": 29.32832145690918, + "learning_rate": 6.438563463416221e-08, + "logits/chosen": -1.157194972038269, + "logits/rejected": -1.1582703590393066, + "logps/chosen": -710.8789672851562, + "logps/ref_chosen": -58.492855072021484, + "logps/ref_rejected": -91.85395050048828, + "logps/rejected": -1157.835693359375, + "loss": 1.1091, + "margin_dpo/margin_mean": 413.5955810546875, + "margin_dpo/margin_std": 535.5076904296875, + "step": 539 + }, + { + "KL/chosen_KL_mean": -629.462890625, + "KL/mean": -898.05859375, + "KL/rejected_KL_mean": -1166.654296875, + "KL/std": 595.0738525390625, + "epoch": 0.7929515418502202, + "fcm_dpo/beta": 0.0008491129847243428, + "fcm_dpo/delta": -0.07360121607780457, + "fcm_dpo/margin": 537.1914672851562, + "fcm_dpo/q_t": 0.396462619304657, + "grad_norm": 36.619747161865234, + "learning_rate": 6.352838968463919e-08, + "logits/chosen": -1.031808614730835, + "logits/rejected": -1.0707483291625977, + "logps/chosen": -692.9454345703125, + "logps/ref_chosen": -63.482513427734375, + "logps/ref_rejected": -116.42999267578125, + "logps/rejected": -1283.0843505859375, + "loss": 1.0689, + "margin_dpo/margin_mean": 537.1914672851562, + "margin_dpo/margin_std": 714.2697143554688, + "step": 540 + }, + { + "KL/chosen_KL_mean": -754.865966796875, + "KL/mean": -900.2635498046875, + "KL/rejected_KL_mean": -1045.6612548828125, + "KL/std": 539.58740234375, + "epoch": 0.7944199706314243, + "fcm_dpo/beta": 0.000835648737847805, + "fcm_dpo/delta": 0.02992171235382557, + "fcm_dpo/margin": 290.7952880859375, + "fcm_dpo/q_t": 0.44497525691986084, + "grad_norm": 49.2709846496582, + "learning_rate": 6.267605843546767e-08, + "logits/chosen": -1.191582202911377, + "logits/rejected": -1.1900990009307861, + "logps/chosen": -833.1463012695312, + "logps/ref_chosen": -78.28036499023438, + "logps/ref_rejected": -103.273681640625, + "logps/rejected": -1148.934814453125, + "loss": 1.2274, + "margin_dpo/margin_mean": 290.79522705078125, + "margin_dpo/margin_std": 635.66015625, + "step": 541 + }, + { + "KL/chosen_KL_mean": -641.0167236328125, + "KL/mean": -899.6398315429688, + "KL/rejected_KL_mean": -1158.262939453125, + "KL/std": 611.9912109375, + "epoch": 0.7958883994126285, + "fcm_dpo/beta": 0.0007914283778518438, + "fcm_dpo/delta": -0.019995450973510742, + "fcm_dpo/margin": 517.24609375, + "fcm_dpo/q_t": 0.40515321493148804, + "grad_norm": 45.66024398803711, + "learning_rate": 6.182866334636888e-08, + "logits/chosen": -1.1752982139587402, + "logits/rejected": -1.219357967376709, + "logps/chosen": -698.501708984375, + "logps/ref_chosen": -57.48497009277344, + "logps/ref_rejected": -96.47506713867188, + "logps/rejected": -1254.738037109375, + "loss": 1.0996, + "margin_dpo/margin_mean": 517.24609375, + "margin_dpo/margin_std": 717.5678100585938, + "step": 542 + }, + { + "KL/chosen_KL_mean": -710.8384399414062, + "KL/mean": -893.802001953125, + "KL/rejected_KL_mean": -1076.7655029296875, + "KL/std": 660.750244140625, + "epoch": 0.7973568281938326, + "fcm_dpo/beta": 0.0008738016476854682, + "fcm_dpo/delta": 0.09592510759830475, + "fcm_dpo/margin": 365.9271240234375, + "fcm_dpo/q_t": 0.4370731711387634, + "grad_norm": 29.643529891967773, + "learning_rate": 6.098622674699147e-08, + "logits/chosen": -1.101868748664856, + "logits/rejected": -1.1417419910430908, + "logps/chosen": -771.4559326171875, + "logps/ref_chosen": -60.61750793457031, + "logps/ref_rejected": -105.59896850585938, + "logps/rejected": -1182.364501953125, + "loss": 1.2076, + "margin_dpo/margin_mean": 365.9271545410156, + "margin_dpo/margin_std": 836.8877563476562, + "step": 543 + }, + { + "KL/chosen_KL_mean": -727.2119140625, + "KL/mean": -949.6517333984375, + "KL/rejected_KL_mean": -1172.091552734375, + "KL/std": 569.668701171875, + "epoch": 0.7988252569750367, + "fcm_dpo/beta": 0.0008942891145125031, + "fcm_dpo/delta": 0.0022725816816091537, + "fcm_dpo/margin": 444.879638671875, + "fcm_dpo/q_t": 0.4095289707183838, + "grad_norm": 34.229854583740234, + "learning_rate": 6.01487708363232e-08, + "logits/chosen": -1.1131432056427002, + "logits/rejected": -1.14817476272583, + "logps/chosen": -786.8541870117188, + "logps/ref_chosen": -59.642303466796875, + "logps/ref_rejected": -100.95469665527344, + "logps/rejected": -1273.046142578125, + "loss": 1.1092, + "margin_dpo/margin_mean": 444.879638671875, + "margin_dpo/margin_std": 663.5440673828125, + "step": 544 + }, + { + "KL/chosen_KL_mean": -659.4088745117188, + "KL/mean": -919.2174072265625, + "KL/rejected_KL_mean": -1179.02587890625, + "KL/std": 598.9300537109375, + "epoch": 0.8002936857562408, + "fcm_dpo/beta": 0.000862646906170994, + "fcm_dpo/delta": -0.06230539083480835, + "fcm_dpo/margin": 519.6170043945312, + "fcm_dpo/q_t": 0.3970434069633484, + "grad_norm": 32.03169631958008, + "learning_rate": 5.9316317682106294e-08, + "logits/chosen": -0.9890373349189758, + "logits/rejected": -1.033557653427124, + "logps/chosen": -727.0574951171875, + "logps/ref_chosen": -67.64859771728516, + "logps/ref_rejected": -95.90800476074219, + "logps/rejected": -1274.933837890625, + "loss": 1.0686, + "margin_dpo/margin_mean": 519.6170654296875, + "margin_dpo/margin_std": 691.0440673828125, + "step": 545 + }, + { + "KL/chosen_KL_mean": -640.8040771484375, + "KL/mean": -804.4512939453125, + "KL/rejected_KL_mean": -968.0985717773438, + "KL/std": 497.0710144042969, + "epoch": 0.801762114537445, + "fcm_dpo/beta": 0.0009246976114809513, + "fcm_dpo/delta": 0.11454042792320251, + "fcm_dpo/margin": 327.29449462890625, + "fcm_dpo/q_t": 0.4316098690032959, + "grad_norm": 30.14081573486328, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": -1.0944793224334717, + "logits/rejected": -1.0822169780731201, + "logps/chosen": -691.5482788085938, + "logps/ref_chosen": -50.744232177734375, + "logps/ref_rejected": -81.86622619628906, + "logps/rejected": -1049.96484375, + "loss": 1.1688, + "margin_dpo/margin_mean": 327.2945251464844, + "margin_dpo/margin_std": 540.7611694335938, + "step": 546 + }, + { + "KL/chosen_KL_mean": -660.4383544921875, + "KL/mean": -884.519287109375, + "KL/rejected_KL_mean": -1108.60009765625, + "KL/std": 564.5079345703125, + "epoch": 0.8032305433186491, + "fcm_dpo/beta": 0.0009412175277248025, + "fcm_dpo/delta": -0.02773449756205082, + "fcm_dpo/margin": 448.1617431640625, + "fcm_dpo/q_t": 0.4049096703529358, + "grad_norm": 41.60638427734375, + "learning_rate": 5.7666507254280265e-08, + "logits/chosen": -1.058458685874939, + "logits/rejected": -1.0802452564239502, + "logps/chosen": -734.1260986328125, + "logps/ref_chosen": -73.6877212524414, + "logps/ref_rejected": -90.76136779785156, + "logps/rejected": -1199.3614501953125, + "loss": 1.0946, + "margin_dpo/margin_mean": 448.1617431640625, + "margin_dpo/margin_std": 651.8272094726562, + "step": 547 + }, + { + "KL/chosen_KL_mean": -706.6986694335938, + "KL/mean": -914.0684814453125, + "KL/rejected_KL_mean": -1121.4383544921875, + "KL/std": 620.1270141601562, + "epoch": 0.8046989720998532, + "fcm_dpo/beta": 0.0009347201557829976, + "fcm_dpo/delta": 0.015313264913856983, + "fcm_dpo/margin": 414.7396240234375, + "fcm_dpo/q_t": 0.41696617007255554, + "grad_norm": 34.767852783203125, + "learning_rate": 5.684919345471029e-08, + "logits/chosen": -1.1285653114318848, + "logits/rejected": -1.1398406028747559, + "logps/chosen": -771.9450073242188, + "logps/ref_chosen": -65.24634552001953, + "logps/ref_rejected": -94.11807250976562, + "logps/rejected": -1215.556396484375, + "loss": 1.1259, + "margin_dpo/margin_mean": 414.7396240234375, + "margin_dpo/margin_std": 696.6866455078125, + "step": 548 + }, + { + "KL/chosen_KL_mean": -716.9879760742188, + "KL/mean": -874.5885009765625, + "KL/rejected_KL_mean": -1032.1890869140625, + "KL/std": 489.6221008300781, + "epoch": 0.8061674008810573, + "fcm_dpo/beta": 0.0009415035601705313, + "fcm_dpo/delta": 0.00250604678876698, + "fcm_dpo/margin": 315.2010803222656, + "fcm_dpo/q_t": 0.4317595362663269, + "grad_norm": 44.2504768371582, + "learning_rate": 5.603696935852426e-08, + "logits/chosen": -1.176891803741455, + "logits/rejected": -1.1708965301513672, + "logps/chosen": -766.2003173828125, + "logps/ref_chosen": -49.21235656738281, + "logps/ref_rejected": -73.91031646728516, + "logps/rejected": -1106.099365234375, + "loss": 1.188, + "margin_dpo/margin_mean": 315.2010803222656, + "margin_dpo/margin_std": 595.5227661132812, + "step": 549 + }, + { + "KL/chosen_KL_mean": -721.7799682617188, + "KL/mean": -916.480224609375, + "KL/rejected_KL_mean": -1111.1806640625, + "KL/std": 571.6375732421875, + "epoch": 0.8076358296622613, + "fcm_dpo/beta": 0.000960228149779141, + "fcm_dpo/delta": 0.03213486075401306, + "fcm_dpo/margin": 389.4005126953125, + "fcm_dpo/q_t": 0.4163832664489746, + "grad_norm": 41.49302673339844, + "learning_rate": 5.5229856368582376e-08, + "logits/chosen": -1.0517723560333252, + "logits/rejected": -1.0827746391296387, + "logps/chosen": -778.5869140625, + "logps/ref_chosen": -56.80695343017578, + "logps/ref_rejected": -95.12580871582031, + "logps/rejected": -1206.306396484375, + "loss": 1.1212, + "margin_dpo/margin_mean": 389.4005126953125, + "margin_dpo/margin_std": 595.0445556640625, + "step": 550 + }, + { + "KL/chosen_KL_mean": -613.4744262695312, + "KL/mean": -925.0634765625, + "KL/rejected_KL_mean": -1236.652587890625, + "KL/std": 624.64697265625, + "epoch": 0.8091042584434655, + "fcm_dpo/beta": 0.0008410347509197891, + "fcm_dpo/delta": -0.17439556121826172, + "fcm_dpo/margin": 623.1781005859375, + "fcm_dpo/q_t": 0.3711840510368347, + "grad_norm": 51.76773452758789, + "learning_rate": 5.4427875753062734e-08, + "logits/chosen": -1.0433658361434937, + "logits/rejected": -1.1170668601989746, + "logps/chosen": -672.5807495117188, + "logps/ref_chosen": -59.10633087158203, + "logps/ref_rejected": -111.67280578613281, + "logps/rejected": -1348.3253173828125, + "loss": 0.9923, + "margin_dpo/margin_mean": 623.1781005859375, + "margin_dpo/margin_std": 630.8492431640625, + "step": 551 + }, + { + "KL/chosen_KL_mean": -602.475341796875, + "KL/mean": -958.9727783203125, + "KL/rejected_KL_mean": -1315.47021484375, + "KL/std": 719.78564453125, + "epoch": 0.8105726872246696, + "fcm_dpo/beta": 0.0006975846481509507, + "fcm_dpo/delta": -0.14975111186504364, + "fcm_dpo/margin": 712.994873046875, + "fcm_dpo/q_t": 0.3810081481933594, + "grad_norm": 42.0149040222168, + "learning_rate": 5.363104864490034e-08, + "logits/chosen": -1.1741893291473389, + "logits/rejected": -1.2385377883911133, + "logps/chosen": -664.8298950195312, + "logps/ref_chosen": -62.35459899902344, + "logps/ref_rejected": -104.56210327148438, + "logps/rejected": -1420.0322265625, + "loss": 1.0282, + "margin_dpo/margin_mean": 712.994873046875, + "margin_dpo/margin_std": 833.3380737304688, + "step": 552 + }, + { + "KL/chosen_KL_mean": -699.2514038085938, + "KL/mean": -884.60302734375, + "KL/rejected_KL_mean": -1069.95458984375, + "KL/std": 591.6678466796875, + "epoch": 0.8120411160058737, + "fcm_dpo/beta": 0.0007735902327112854, + "fcm_dpo/delta": 0.1325617879629135, + "fcm_dpo/margin": 370.7031555175781, + "fcm_dpo/q_t": 0.43872201442718506, + "grad_norm": 23.199739456176758, + "learning_rate": 5.2839396041230415e-08, + "logits/chosen": -1.1220920085906982, + "logits/rejected": -1.1197218894958496, + "logps/chosen": -767.51025390625, + "logps/ref_chosen": -68.25881958007812, + "logps/ref_rejected": -98.0971450805664, + "logps/rejected": -1168.0517578125, + "loss": 1.1789, + "margin_dpo/margin_mean": 370.70318603515625, + "margin_dpo/margin_std": 653.1337890625, + "step": 553 + }, + { + "KL/chosen_KL_mean": -751.51806640625, + "KL/mean": -983.1180419921875, + "KL/rejected_KL_mean": -1214.7178955078125, + "KL/std": 659.7369995117188, + "epoch": 0.8135095447870778, + "fcm_dpo/beta": 0.0008373720338568091, + "fcm_dpo/delta": 0.009692121297121048, + "fcm_dpo/margin": 463.19989013671875, + "fcm_dpo/q_t": 0.4117772877216339, + "grad_norm": 118.59552001953125, + "learning_rate": 5.205293880283551e-08, + "logits/chosen": -1.087932825088501, + "logits/rejected": -1.0606003999710083, + "logps/chosen": -819.4658203125, + "logps/ref_chosen": -67.94767761230469, + "logps/ref_rejected": -89.78272247314453, + "logps/rejected": -1304.5006103515625, + "loss": 1.1619, + "margin_dpo/margin_mean": 463.1998596191406, + "margin_dpo/margin_std": 863.3409423828125, + "step": 554 + }, + { + "KL/chosen_KL_mean": -739.2120971679688, + "KL/mean": -1043.806396484375, + "KL/rejected_KL_mean": -1348.400634765625, + "KL/std": 680.260009765625, + "epoch": 0.8149779735682819, + "fcm_dpo/beta": 0.0007673618383705616, + "fcm_dpo/delta": -0.08767017722129822, + "fcm_dpo/margin": 609.1884155273438, + "fcm_dpo/q_t": 0.3946698307991028, + "grad_norm": 30.83856773376465, + "learning_rate": 5.127169765359515e-08, + "logits/chosen": -1.1824922561645508, + "logits/rejected": -1.2582507133483887, + "logps/chosen": -792.5426025390625, + "logps/ref_chosen": -53.33049011230469, + "logps/ref_rejected": -108.47937774658203, + "logps/rejected": -1456.8798828125, + "loss": 1.0729, + "margin_dpo/margin_mean": 609.1884765625, + "margin_dpo/margin_std": 872.76953125, + "step": 555 + }, + { + "KL/chosen_KL_mean": -732.7947998046875, + "KL/mean": -903.1065673828125, + "KL/rejected_KL_mean": -1073.418212890625, + "KL/std": 530.9589233398438, + "epoch": 0.8164464023494861, + "fcm_dpo/beta": 0.0008233854314312339, + "fcm_dpo/delta": 0.13905900716781616, + "fcm_dpo/margin": 340.62347412109375, + "fcm_dpo/q_t": 0.4381856620311737, + "grad_norm": 29.910762786865234, + "learning_rate": 5.049569317994012e-08, + "logits/chosen": -1.185271143913269, + "logits/rejected": -1.1860347986221313, + "logps/chosen": -791.4393310546875, + "logps/ref_chosen": -58.64447021484375, + "logps/ref_rejected": -101.34040832519531, + "logps/rejected": -1174.7587890625, + "loss": 1.1731, + "margin_dpo/margin_mean": 340.6235046386719, + "margin_dpo/margin_std": 539.8701171875, + "step": 556 + }, + { + "KL/chosen_KL_mean": -786.4280395507812, + "KL/mean": -1055.865234375, + "KL/rejected_KL_mean": -1325.302490234375, + "KL/std": 711.47607421875, + "epoch": 0.8179148311306902, + "fcm_dpo/beta": 0.0008084324654191732, + "fcm_dpo/delta": -0.04845455288887024, + "fcm_dpo/margin": 538.87451171875, + "fcm_dpo/q_t": 0.4022066295146942, + "grad_norm": 46.43569564819336, + "learning_rate": 4.9724945830310144e-08, + "logits/chosen": -1.2151281833648682, + "logits/rejected": -1.2664202451705933, + "logps/chosen": -854.2686767578125, + "logps/ref_chosen": -67.84066009521484, + "logps/ref_rejected": -109.93965911865234, + "logps/rejected": -1435.2421875, + "loss": 1.0994, + "margin_dpo/margin_mean": 538.8744506835938, + "margin_dpo/margin_std": 822.3883056640625, + "step": 557 + }, + { + "KL/chosen_KL_mean": -752.0896606445312, + "KL/mean": -1117.599853515625, + "KL/rejected_KL_mean": -1483.10986328125, + "KL/std": 693.58544921875, + "epoch": 0.8193832599118943, + "fcm_dpo/beta": 0.0007167826406657696, + "fcm_dpo/delta": -0.16944512724876404, + "fcm_dpo/margin": 731.0201416015625, + "fcm_dpo/q_t": 0.3706546425819397, + "grad_norm": 24.203054428100586, + "learning_rate": 4.8959475914614554e-08, + "logits/chosen": -1.274876594543457, + "logits/rejected": -1.3060567378997803, + "logps/chosen": -814.4578857421875, + "logps/ref_chosen": -62.36824035644531, + "logps/ref_rejected": -102.16102600097656, + "logps/rejected": -1585.2708740234375, + "loss": 1.0044, + "margin_dpo/margin_mean": 731.0202026367188, + "margin_dpo/margin_std": 783.4100341796875, + "step": 558 + }, + { + "KL/chosen_KL_mean": -840.5665893554688, + "KL/mean": -1157.062744140625, + "KL/rejected_KL_mean": -1473.558837890625, + "KL/std": 744.0479736328125, + "epoch": 0.8208516886930984, + "fcm_dpo/beta": 0.0006758394883945584, + "fcm_dpo/delta": -0.03540560603141785, + "fcm_dpo/margin": 632.9921264648438, + "fcm_dpo/q_t": 0.401652991771698, + "grad_norm": 23.325172424316406, + "learning_rate": 4.8199303603697614e-08, + "logits/chosen": -1.4109092950820923, + "logits/rejected": -1.432593822479248, + "logps/chosen": -901.3189697265625, + "logps/ref_chosen": -60.752323150634766, + "logps/ref_rejected": -93.44229125976562, + "logps/rejected": -1567.0009765625, + "loss": 1.0774, + "margin_dpo/margin_mean": 632.9921875, + "margin_dpo/margin_std": 839.2745361328125, + "step": 559 + }, + { + "KL/chosen_KL_mean": -801.013916015625, + "KL/mean": -1016.2203369140625, + "KL/rejected_KL_mean": -1231.4267578125, + "KL/std": 670.739990234375, + "epoch": 0.8223201174743024, + "fcm_dpo/beta": 0.0006765261059626937, + "fcm_dpo/delta": 0.019914967939257622, + "fcm_dpo/margin": 430.41290283203125, + "fcm_dpo/q_t": 0.43432098627090454, + "grad_norm": 40.84651184082031, + "learning_rate": 4.7444448928806615e-08, + "logits/chosen": -1.0937294960021973, + "logits/rejected": -1.0813250541687012, + "logps/chosen": -859.11767578125, + "logps/ref_chosen": -58.10382080078125, + "logps/ref_rejected": -79.99122619628906, + "logps/rejected": -1311.41796875, + "loss": 1.1813, + "margin_dpo/margin_mean": 430.41290283203125, + "margin_dpo/margin_std": 772.7431640625, + "step": 560 + }, + { + "KL/chosen_KL_mean": -897.7193603515625, + "KL/mean": -1091.67431640625, + "KL/rejected_KL_mean": -1285.6295166015625, + "KL/std": 642.45654296875, + "epoch": 0.8237885462555066, + "fcm_dpo/beta": 0.0007529509020969272, + "fcm_dpo/delta": 0.1230059415102005, + "fcm_dpo/margin": 387.91015625, + "fcm_dpo/q_t": 0.435150682926178, + "grad_norm": 30.059293746948242, + "learning_rate": 4.669493178106432e-08, + "logits/chosen": -1.2934693098068237, + "logits/rejected": -1.3272886276245117, + "logps/chosen": -948.6322021484375, + "logps/ref_chosen": -50.912879943847656, + "logps/ref_rejected": -99.06856536865234, + "logps/rejected": -1384.6981201171875, + "loss": 1.1972, + "margin_dpo/margin_mean": 387.91015625, + "margin_dpo/margin_std": 791.5865478515625, + "step": 561 + }, + { + "KL/chosen_KL_mean": -864.7420654296875, + "KL/mean": -1108.170654296875, + "KL/rejected_KL_mean": -1351.5992431640625, + "KL/std": 697.1679077148438, + "epoch": 0.8252569750367107, + "fcm_dpo/beta": 0.0007649194449186325, + "fcm_dpo/delta": 0.03210524842143059, + "fcm_dpo/margin": 486.857177734375, + "fcm_dpo/q_t": 0.416918009519577, + "grad_norm": 28.984006881713867, + "learning_rate": 4.5950771910944596e-08, + "logits/chosen": -1.2221224308013916, + "logits/rejected": -1.2382316589355469, + "logps/chosen": -924.2064819335938, + "logps/ref_chosen": -59.46440124511719, + "logps/ref_rejected": -96.54266357421875, + "logps/rejected": -1448.141845703125, + "loss": 1.1243, + "margin_dpo/margin_mean": 486.857177734375, + "margin_dpo/margin_std": 740.4078369140625, + "step": 562 + }, + { + "KL/chosen_KL_mean": -973.072021484375, + "KL/mean": -1153.509033203125, + "KL/rejected_KL_mean": -1333.9459228515625, + "KL/std": 739.7679443359375, + "epoch": 0.8267254038179148, + "fcm_dpo/beta": 0.0007791572716087103, + "fcm_dpo/delta": -0.02793397195637226, + "fcm_dpo/margin": 360.8739013671875, + "fcm_dpo/q_t": 0.4309625029563904, + "grad_norm": 40.373382568359375, + "learning_rate": 4.521198892775202e-08, + "logits/chosen": -1.2764613628387451, + "logits/rejected": -1.2854523658752441, + "logps/chosen": -1033.68017578125, + "logps/ref_chosen": -60.60819625854492, + "logps/ref_rejected": -94.56770324707031, + "logps/rejected": -1428.513671875, + "loss": 1.2446, + "margin_dpo/margin_mean": 360.8739013671875, + "margin_dpo/margin_std": 878.1210327148438, + "step": 563 + }, + { + "KL/chosen_KL_mean": -857.3223266601562, + "KL/mean": -1105.55712890625, + "KL/rejected_KL_mean": -1353.7919921875, + "KL/std": 703.2030029296875, + "epoch": 0.8281938325991189, + "fcm_dpo/beta": 0.0007832963019609451, + "fcm_dpo/delta": 0.013243109919130802, + "fcm_dpo/margin": 496.4696960449219, + "fcm_dpo/q_t": 0.4130350947380066, + "grad_norm": 32.131954193115234, + "learning_rate": 4.447860229910544e-08, + "logits/chosen": -1.2859337329864502, + "logits/rejected": -1.2894582748413086, + "logps/chosen": -931.5906982421875, + "logps/ref_chosen": -74.26837921142578, + "logps/ref_rejected": -93.23818969726562, + "logps/rejected": -1447.0301513671875, + "loss": 1.1066, + "margin_dpo/margin_mean": 496.4696960449219, + "margin_dpo/margin_std": 708.5772094726562, + "step": 564 + }, + { + "KL/chosen_KL_mean": -940.7181396484375, + "KL/mean": -1180.5489501953125, + "KL/rejected_KL_mean": -1420.3797607421875, + "KL/std": 811.2318115234375, + "epoch": 0.8296622613803231, + "fcm_dpo/beta": 0.0007922237273305655, + "fcm_dpo/delta": 0.024702370166778564, + "fcm_dpo/margin": 479.6616516113281, + "fcm_dpo/q_t": 0.4200623631477356, + "grad_norm": 52.22495651245117, + "learning_rate": 4.375063135042445e-08, + "logits/chosen": -1.1902871131896973, + "logits/rejected": -1.1874666213989258, + "logps/chosen": -1009.738037109375, + "logps/ref_chosen": -69.0199203491211, + "logps/ref_rejected": -85.7789306640625, + "logps/rejected": -1506.15869140625, + "loss": 1.1803, + "margin_dpo/margin_mean": 479.6616516113281, + "margin_dpo/margin_std": 994.3869018554688, + "step": 565 + }, + { + "KL/chosen_KL_mean": -863.586669921875, + "KL/mean": -1147.6942138671875, + "KL/rejected_KL_mean": -1431.8017578125, + "KL/std": 777.2615966796875, + "epoch": 0.8311306901615272, + "fcm_dpo/beta": 0.0007855127332732081, + "fcm_dpo/delta": -0.061380840837955475, + "fcm_dpo/margin": 568.2152099609375, + "fcm_dpo/q_t": 0.3984350264072418, + "grad_norm": 28.156330108642578, + "learning_rate": 4.3028095264420525e-08, + "logits/chosen": -1.2587931156158447, + "logits/rejected": -1.2928824424743652, + "logps/chosen": -930.1319580078125, + "logps/ref_chosen": -66.5453109741211, + "logps/ref_rejected": -103.86932373046875, + "logps/rejected": -1535.671142578125, + "loss": 1.1124, + "margin_dpo/margin_mean": 568.2152099609375, + "margin_dpo/margin_std": 901.3643188476562, + "step": 566 + }, + { + "KL/chosen_KL_mean": -786.615478515625, + "KL/mean": -1003.1062622070312, + "KL/rejected_KL_mean": -1219.5970458984375, + "KL/std": 541.6770629882812, + "epoch": 0.8325991189427313, + "fcm_dpo/beta": 0.000789005309343338, + "fcm_dpo/delta": 0.07048732787370682, + "fcm_dpo/margin": 432.98162841796875, + "fcm_dpo/q_t": 0.4219757318496704, + "grad_norm": 25.511581420898438, + "learning_rate": 4.231101308059165e-08, + "logits/chosen": -1.3670188188552856, + "logits/rejected": -1.3844826221466064, + "logps/chosen": -839.4737548828125, + "logps/ref_chosen": -52.85829544067383, + "logps/ref_rejected": -85.37095642089844, + "logps/rejected": -1304.968017578125, + "loss": 1.1217, + "margin_dpo/margin_mean": 432.98162841796875, + "margin_dpo/margin_std": 570.1590576171875, + "step": 567 + }, + { + "KL/chosen_KL_mean": -747.6436767578125, + "KL/mean": -1037.877197265625, + "KL/rejected_KL_mean": -1328.11083984375, + "KL/std": 627.2274780273438, + "epoch": 0.8340675477239354, + "fcm_dpo/beta": 0.0007679754635319114, + "fcm_dpo/delta": -0.05931420996785164, + "fcm_dpo/margin": 580.4671630859375, + "fcm_dpo/q_t": 0.39421334862709045, + "grad_norm": 34.71445846557617, + "learning_rate": 4.1599403694720145e-08, + "logits/chosen": -1.1856874227523804, + "logits/rejected": -1.243179202079773, + "logps/chosen": -792.8360595703125, + "logps/ref_chosen": -45.1923828125, + "logps/ref_rejected": -89.09236907958984, + "logps/rejected": -1417.2032470703125, + "loss": 1.0511, + "margin_dpo/margin_mean": 580.4671630859375, + "margin_dpo/margin_std": 675.10009765625, + "step": 568 + }, + { + "KL/chosen_KL_mean": -887.8179321289062, + "KL/mean": -1117.02783203125, + "KL/rejected_KL_mean": -1346.2376708984375, + "KL/std": 795.90283203125, + "epoch": 0.8355359765051396, + "fcm_dpo/beta": 0.0008052626508288085, + "fcm_dpo/delta": 0.030735157430171967, + "fcm_dpo/margin": 458.41961669921875, + "fcm_dpo/q_t": 0.41605180501937866, + "grad_norm": 35.24816131591797, + "learning_rate": 4.089328585837512e-08, + "logits/chosen": -1.2379553318023682, + "logits/rejected": -1.2469182014465332, + "logps/chosen": -951.5385131835938, + "logps/ref_chosen": -63.72056198120117, + "logps/ref_rejected": -79.10325622558594, + "logps/rejected": -1425.3408203125, + "loss": 1.1615, + "margin_dpo/margin_mean": 458.41961669921875, + "margin_dpo/margin_std": 826.8089599609375, + "step": 569 + }, + { + "KL/chosen_KL_mean": -801.3278198242188, + "KL/mean": -1011.711669921875, + "KL/rejected_KL_mean": -1222.095458984375, + "KL/std": 619.7181396484375, + "epoch": 0.8370044052863436, + "fcm_dpo/beta": 0.0008181704906746745, + "fcm_dpo/delta": 0.06755281984806061, + "fcm_dpo/margin": 420.76763916015625, + "fcm_dpo/q_t": 0.42422983050346375, + "grad_norm": 28.688831329345703, + "learning_rate": 4.019267817841834e-08, + "logits/chosen": -1.3244519233703613, + "logits/rejected": -1.3194363117218018, + "logps/chosen": -862.9423828125, + "logps/ref_chosen": -61.61454391479492, + "logps/ref_rejected": -82.14186096191406, + "logps/rejected": -1304.2373046875, + "loss": 1.1451, + "margin_dpo/margin_mean": 420.76763916015625, + "margin_dpo/margin_std": 679.84033203125, + "step": 570 + }, + { + "KL/chosen_KL_mean": -844.9622802734375, + "KL/mean": -1082.0240478515625, + "KL/rejected_KL_mean": -1319.085693359375, + "KL/std": 643.911865234375, + "epoch": 0.8384728340675477, + "fcm_dpo/beta": 0.0008347969851456583, + "fcm_dpo/delta": 0.005073328502476215, + "fcm_dpo/margin": 474.1234130859375, + "fcm_dpo/q_t": 0.412005752325058, + "grad_norm": 48.486839294433594, + "learning_rate": 3.9497599116513705e-08, + "logits/chosen": -1.1971943378448486, + "logits/rejected": -1.2182611227035522, + "logps/chosen": -898.016357421875, + "logps/ref_chosen": -53.05406188964844, + "logps/ref_rejected": -91.33682250976562, + "logps/rejected": -1410.422607421875, + "loss": 1.1393, + "margin_dpo/margin_mean": 474.1234436035156, + "margin_dpo/margin_std": 829.1712646484375, + "step": 571 + }, + { + "KL/chosen_KL_mean": -877.3575439453125, + "KL/mean": -1135.824951171875, + "KL/rejected_KL_mean": -1394.29248046875, + "KL/std": 772.8182373046875, + "epoch": 0.8399412628487518, + "fcm_dpo/beta": 0.000814650789834559, + "fcm_dpo/delta": -0.027804527431726456, + "fcm_dpo/margin": 516.9349365234375, + "fcm_dpo/q_t": 0.40952882170677185, + "grad_norm": 26.61905288696289, + "learning_rate": 3.880806698864086e-08, + "logits/chosen": -1.2630120515823364, + "logits/rejected": -1.3064543008804321, + "logps/chosen": -925.8168334960938, + "logps/ref_chosen": -48.45928955078125, + "logps/ref_rejected": -83.55703735351562, + "logps/rejected": -1477.849609375, + "loss": 1.1352, + "margin_dpo/margin_mean": 516.9349365234375, + "margin_dpo/margin_std": 933.8126220703125, + "step": 572 + }, + { + "KL/chosen_KL_mean": -831.7431640625, + "KL/mean": -1071.42333984375, + "KL/rejected_KL_mean": -1311.103515625, + "KL/std": 665.356201171875, + "epoch": 0.8414096916299559, + "fcm_dpo/beta": 0.0008302472997456789, + "fcm_dpo/delta": 0.0017357263714075089, + "fcm_dpo/margin": 479.3603515625, + "fcm_dpo/q_t": 0.4101864695549011, + "grad_norm": 33.60465621948242, + "learning_rate": 3.812409996461275e-08, + "logits/chosen": -1.2730963230133057, + "logits/rejected": -1.296579360961914, + "logps/chosen": -883.36572265625, + "logps/ref_chosen": -51.62262725830078, + "logps/ref_rejected": -85.32499694824219, + "logps/rejected": -1396.428466796875, + "loss": 1.0996, + "margin_dpo/margin_mean": 479.3603515625, + "margin_dpo/margin_std": 673.96435546875, + "step": 573 + }, + { + "KL/chosen_KL_mean": -761.8274536132812, + "KL/mean": -1024.19091796875, + "KL/rejected_KL_mean": -1286.554443359375, + "KL/std": 611.08935546875, + "epoch": 0.8428781204111601, + "fcm_dpo/beta": 0.0008096909150481224, + "fcm_dpo/delta": -0.03171788156032562, + "fcm_dpo/margin": 524.7269287109375, + "fcm_dpo/q_t": 0.40161073207855225, + "grad_norm": 34.691165924072266, + "learning_rate": 3.74457160675965e-08, + "logits/chosen": -1.2997612953186035, + "logits/rejected": -1.3460713624954224, + "logps/chosen": -812.8719482421875, + "logps/ref_chosen": -51.04446029663086, + "logps/ref_rejected": -92.80640411376953, + "logps/rejected": -1379.36083984375, + "loss": 1.0834, + "margin_dpo/margin_mean": 524.7269287109375, + "margin_dpo/margin_std": 706.32373046875, + "step": 574 + }, + { + "KL/chosen_KL_mean": -815.347412109375, + "KL/mean": -1056.3719482421875, + "KL/rejected_KL_mean": -1297.396484375, + "KL/std": 656.7642211914062, + "epoch": 0.8443465491923642, + "fcm_dpo/beta": 0.0007790890522301197, + "fcm_dpo/delta": 0.024538405239582062, + "fcm_dpo/margin": 482.0490417480469, + "fcm_dpo/q_t": 0.4154299199581146, + "grad_norm": 46.73399353027344, + "learning_rate": 3.677293317363864e-08, + "logits/chosen": -1.071021318435669, + "logits/rejected": -1.0894410610198975, + "logps/chosen": -887.1375732421875, + "logps/ref_chosen": -71.7901382446289, + "logps/ref_rejected": -95.38619995117188, + "logps/rejected": -1392.78271484375, + "loss": 1.1529, + "margin_dpo/margin_mean": 482.049072265625, + "margin_dpo/margin_std": 828.638671875, + "step": 575 + }, + { + "KL/chosen_KL_mean": -783.9774780273438, + "KL/mean": -965.3140869140625, + "KL/rejected_KL_mean": -1146.650634765625, + "KL/std": 556.8798828125, + "epoch": 0.8458149779735683, + "fcm_dpo/beta": 0.000867416150867939, + "fcm_dpo/delta": 0.10194016993045807, + "fcm_dpo/margin": 362.67315673828125, + "fcm_dpo/q_t": 0.4307815134525299, + "grad_norm": 33.000823974609375, + "learning_rate": 3.6105769011194224e-08, + "logits/chosen": -1.2250196933746338, + "logits/rejected": -1.2645316123962402, + "logps/chosen": -838.240478515625, + "logps/ref_chosen": -54.262962341308594, + "logps/ref_rejected": -100.75428009033203, + "logps/rejected": -1247.405029296875, + "loss": 1.1724, + "margin_dpo/margin_mean": 362.67315673828125, + "margin_dpo/margin_std": 646.2176513671875, + "step": 576 + }, + { + "KL/chosen_KL_mean": -681.5286254882812, + "KL/mean": -917.073974609375, + "KL/rejected_KL_mean": -1152.619384765625, + "KL/std": 618.7384033203125, + "epoch": 0.8472834067547724, + "fcm_dpo/beta": 0.0008785222889855504, + "fcm_dpo/delta": -0.01841025985777378, + "fcm_dpo/margin": 471.0908203125, + "fcm_dpo/q_t": 0.4073464274406433, + "grad_norm": 27.403722763061523, + "learning_rate": 3.5444241160659304e-08, + "logits/chosen": -1.1902132034301758, + "logits/rejected": -1.181257724761963, + "logps/chosen": -743.4383544921875, + "logps/ref_chosen": -61.909706115722656, + "logps/ref_rejected": -84.07069396972656, + "logps/rejected": -1236.6900634765625, + "loss": 1.1109, + "margin_dpo/margin_mean": 471.0908203125, + "margin_dpo/margin_std": 701.18603515625, + "step": 577 + }, + { + "KL/chosen_KL_mean": -655.4379272460938, + "KL/mean": -906.4620361328125, + "KL/rejected_KL_mean": -1157.486083984375, + "KL/std": 596.8307495117188, + "epoch": 0.8487518355359766, + "fcm_dpo/beta": 0.000839193002320826, + "fcm_dpo/delta": -0.03209435194730759, + "fcm_dpo/margin": 502.04815673828125, + "fcm_dpo/q_t": 0.4018974304199219, + "grad_norm": 33.629188537597656, + "learning_rate": 3.478836705390808e-08, + "logits/chosen": -1.0385322570800781, + "logits/rejected": -1.082366943359375, + "logps/chosen": -704.7015991210938, + "logps/ref_chosen": -49.26368713378906, + "logps/ref_rejected": -83.4362564086914, + "logps/rejected": -1240.92236328125, + "loss": 1.0797, + "margin_dpo/margin_mean": 502.04815673828125, + "margin_dpo/margin_std": 644.6083984375, + "step": 578 + }, + { + "KL/chosen_KL_mean": -802.5963134765625, + "KL/mean": -953.2222900390625, + "KL/rejected_KL_mean": -1103.8482666015625, + "KL/std": 631.9645385742188, + "epoch": 0.8502202643171806, + "fcm_dpo/beta": 0.0009417862165719271, + "fcm_dpo/delta": 0.13616657257080078, + "fcm_dpo/margin": 301.25201416015625, + "fcm_dpo/q_t": 0.43976956605911255, + "grad_norm": 40.763694763183594, + "learning_rate": 3.41381639738331e-08, + "logits/chosen": -1.1537394523620605, + "logits/rejected": -1.164426326751709, + "logps/chosen": -861.482177734375, + "logps/ref_chosen": -58.88581848144531, + "logps/ref_rejected": -94.78762817382812, + "logps/rejected": -1198.635986328125, + "loss": 1.2241, + "margin_dpo/margin_mean": 301.25201416015625, + "margin_dpo/margin_std": 703.5513916015625, + "step": 579 + }, + { + "KL/chosen_KL_mean": -557.6210327148438, + "KL/mean": -839.1734619140625, + "KL/rejected_KL_mean": -1120.725830078125, + "KL/std": 694.3538208007812, + "epoch": 0.8516886930983847, + "fcm_dpo/beta": 0.0008712954586371779, + "fcm_dpo/delta": -0.1274590790271759, + "fcm_dpo/margin": 563.1048583984375, + "fcm_dpo/q_t": 0.38948333263397217, + "grad_norm": 29.91592788696289, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": -0.9779269695281982, + "logits/rejected": -1.0172150135040283, + "logps/chosen": -606.327880859375, + "logps/ref_chosen": -48.70683670043945, + "logps/ref_rejected": -81.7583999633789, + "logps/rejected": -1202.4842529296875, + "loss": 1.0591, + "margin_dpo/margin_mean": 563.1048583984375, + "margin_dpo/margin_std": 814.8396606445312, + "step": 580 + }, + { + "KL/chosen_KL_mean": -770.3370361328125, + "KL/mean": -978.4288330078125, + "KL/rejected_KL_mean": -1186.520751953125, + "KL/std": 666.4249267578125, + "epoch": 0.8531571218795888, + "fcm_dpo/beta": 0.0008886848227120936, + "fcm_dpo/delta": 0.03695109486579895, + "fcm_dpo/margin": 416.1837158203125, + "fcm_dpo/q_t": 0.41858458518981934, + "grad_norm": 36.696754455566406, + "learning_rate": 3.285483927764726e-08, + "logits/chosen": -1.2336806058883667, + "logits/rejected": -1.262026071548462, + "logps/chosen": -832.559326171875, + "logps/ref_chosen": -62.22235107421875, + "logps/ref_rejected": -91.73568725585938, + "logps/rejected": -1278.25634765625, + "loss": 1.1399, + "margin_dpo/margin_mean": 416.1837158203125, + "margin_dpo/margin_std": 706.4039306640625, + "step": 581 + }, + { + "KL/chosen_KL_mean": -691.3490600585938, + "KL/mean": -915.9049072265625, + "KL/rejected_KL_mean": -1140.460693359375, + "KL/std": 554.6382446289062, + "epoch": 0.8546255506607929, + "fcm_dpo/beta": 0.000924235675483942, + "fcm_dpo/delta": -0.028372012078762054, + "fcm_dpo/margin": 449.111572265625, + "fcm_dpo/q_t": 0.40326303243637085, + "grad_norm": 34.956790924072266, + "learning_rate": 3.222175147833556e-08, + "logits/chosen": -1.1378693580627441, + "logits/rejected": -1.1712710857391357, + "logps/chosen": -749.5777587890625, + "logps/ref_chosen": -58.228660583496094, + "logps/ref_rejected": -110.06959533691406, + "logps/rejected": -1250.5302734375, + "loss": 1.1015, + "margin_dpo/margin_mean": 449.111572265625, + "margin_dpo/margin_std": 610.8257446289062, + "step": 582 + }, + { + "KL/chosen_KL_mean": -759.1519775390625, + "KL/mean": -915.6939697265625, + "KL/rejected_KL_mean": -1072.2359619140625, + "KL/std": 620.579833984375, + "epoch": 0.856093979441997, + "fcm_dpo/beta": 0.0008432507747784257, + "fcm_dpo/delta": -0.03345673531293869, + "fcm_dpo/margin": 313.083984375, + "fcm_dpo/q_t": 0.43994593620300293, + "grad_norm": 49.73270034790039, + "learning_rate": 3.159440233840763e-08, + "logits/chosen": -1.1442230939865112, + "logits/rejected": -1.1517397165298462, + "logps/chosen": -816.0148315429688, + "logps/ref_chosen": -56.86286163330078, + "logps/ref_rejected": -88.4039306640625, + "logps/rejected": -1160.639892578125, + "loss": 1.2397, + "margin_dpo/margin_mean": 313.083984375, + "margin_dpo/margin_std": 745.907470703125, + "step": 583 + }, + { + "KL/chosen_KL_mean": -676.06298828125, + "KL/mean": -953.474853515625, + "KL/rejected_KL_mean": -1230.88671875, + "KL/std": 634.57861328125, + "epoch": 0.8575624082232012, + "fcm_dpo/beta": 0.0008079511462710798, + "fcm_dpo/delta": -0.06212177127599716, + "fcm_dpo/margin": 554.8237915039062, + "fcm_dpo/q_t": 0.3955593705177307, + "grad_norm": 27.940731048583984, + "learning_rate": 3.0972808389096635e-08, + "logits/chosen": -1.0924582481384277, + "logits/rejected": -1.1197701692581177, + "logps/chosen": -732.9636840820312, + "logps/ref_chosen": -56.90068054199219, + "logps/ref_rejected": -97.63606262207031, + "logps/rejected": -1328.5228271484375, + "loss": 1.0545, + "margin_dpo/margin_mean": 554.8238525390625, + "margin_dpo/margin_std": 655.2168579101562, + "step": 584 + }, + { + "KL/chosen_KL_mean": -754.1314697265625, + "KL/mean": -1025.2532958984375, + "KL/rejected_KL_mean": -1296.3751220703125, + "KL/std": 712.186279296875, + "epoch": 0.8590308370044053, + "fcm_dpo/beta": 0.0007676214445382357, + "fcm_dpo/delta": -0.022018061950802803, + "fcm_dpo/margin": 542.24365234375, + "fcm_dpo/q_t": 0.40494245290756226, + "grad_norm": 26.670785903930664, + "learning_rate": 3.035698600998121e-08, + "logits/chosen": -1.1742076873779297, + "logits/rejected": -1.2190337181091309, + "logps/chosen": -815.10546875, + "logps/ref_chosen": -60.973968505859375, + "logps/ref_rejected": -84.16952514648438, + "logps/rejected": -1380.544677734375, + "loss": 1.1091, + "margin_dpo/margin_mean": 542.24365234375, + "margin_dpo/margin_std": 845.0147094726562, + "step": 585 + }, + { + "KL/chosen_KL_mean": -803.51416015625, + "KL/mean": -982.947021484375, + "KL/rejected_KL_mean": -1162.3798828125, + "KL/std": 643.473876953125, + "epoch": 0.8604992657856094, + "fcm_dpo/beta": 0.0008414636831730604, + "fcm_dpo/delta": 0.11576494574546814, + "fcm_dpo/margin": 358.8658447265625, + "fcm_dpo/q_t": 0.43414974212646484, + "grad_norm": 35.62641525268555, + "learning_rate": 2.974695142855388e-08, + "logits/chosen": -1.1895267963409424, + "logits/rejected": -1.2259807586669922, + "logps/chosen": -860.3697509765625, + "logps/ref_chosen": -56.85559844970703, + "logps/ref_rejected": -91.80261993408203, + "logps/rejected": -1254.1826171875, + "loss": 1.1841, + "margin_dpo/margin_mean": 358.8658447265625, + "margin_dpo/margin_std": 677.7850952148438, + "step": 586 + }, + { + "KL/chosen_KL_mean": -560.0857543945312, + "KL/mean": -783.292724609375, + "KL/rejected_KL_mean": -1006.4996337890625, + "KL/std": 627.01611328125, + "epoch": 0.8619676945668135, + "fcm_dpo/beta": 0.0008853634353727102, + "fcm_dpo/delta": 0.005062144249677658, + "fcm_dpo/margin": 446.41387939453125, + "fcm_dpo/q_t": 0.4096928834915161, + "grad_norm": 31.3030948638916, + "learning_rate": 2.9142720719793122e-08, + "logits/chosen": -1.2153189182281494, + "logits/rejected": -1.2506670951843262, + "logps/chosen": -604.77734375, + "logps/ref_chosen": -44.69159698486328, + "logps/ref_rejected": -82.62385559082031, + "logps/rejected": -1089.12353515625, + "loss": 1.1015, + "margin_dpo/margin_mean": 446.41387939453125, + "margin_dpo/margin_std": 625.9844970703125, + "step": 587 + }, + { + "KL/chosen_KL_mean": -758.8787841796875, + "KL/mean": -942.8692626953125, + "KL/rejected_KL_mean": -1126.859619140625, + "KL/std": 569.2108154296875, + "epoch": 0.8634361233480177, + "fcm_dpo/beta": 0.0009099560556933284, + "fcm_dpo/delta": 0.07821156829595566, + "fcm_dpo/margin": 367.9808349609375, + "fcm_dpo/q_t": 0.42506054043769836, + "grad_norm": 27.26927947998047, + "learning_rate": 2.8544309805740018e-08, + "logits/chosen": -1.142134666442871, + "logits/rejected": -1.1726654767990112, + "logps/chosen": -809.1737670898438, + "logps/ref_chosen": -50.29494857788086, + "logps/ref_rejected": -107.36988067626953, + "logps/rejected": -1234.2294921875, + "loss": 1.1514, + "margin_dpo/margin_mean": 367.9808349609375, + "margin_dpo/margin_std": 589.927734375, + "step": 588 + }, + { + "KL/chosen_KL_mean": -769.4779663085938, + "KL/mean": -1022.2039794921875, + "KL/rejected_KL_mean": -1274.929931640625, + "KL/std": 642.8394775390625, + "epoch": 0.8649045521292217, + "fcm_dpo/beta": 0.0008996998658403754, + "fcm_dpo/delta": -0.07083002477884293, + "fcm_dpo/margin": 505.4520263671875, + "fcm_dpo/q_t": 0.39429712295532227, + "grad_norm": 30.192138671875, + "learning_rate": 2.7951734455078786e-08, + "logits/chosen": -1.128014087677002, + "logits/rejected": -1.1533284187316895, + "logps/chosen": -829.4078369140625, + "logps/ref_chosen": -59.929908752441406, + "logps/ref_rejected": -111.65534973144531, + "logps/rejected": -1386.5853271484375, + "loss": 1.0682, + "margin_dpo/margin_mean": 505.4520263671875, + "margin_dpo/margin_std": 678.6458740234375, + "step": 589 + }, + { + "KL/chosen_KL_mean": -634.3226928710938, + "KL/mean": -899.4765625, + "KL/rejected_KL_mean": -1164.63037109375, + "KL/std": 646.88037109375, + "epoch": 0.8663729809104258, + "fcm_dpo/beta": 0.000832444173283875, + "fcm_dpo/delta": -0.05546820908784866, + "fcm_dpo/margin": 530.3076782226562, + "fcm_dpo/q_t": 0.3984396457672119, + "grad_norm": 26.239267349243164, + "learning_rate": 2.736501028272095e-08, + "logits/chosen": -1.1118061542510986, + "logits/rejected": -1.1564977169036865, + "logps/chosen": -690.1324462890625, + "logps/ref_chosen": -55.80979537963867, + "logps/ref_rejected": -106.06282043457031, + "logps/rejected": -1270.693115234375, + "loss": 1.0645, + "margin_dpo/margin_mean": 530.3077392578125, + "margin_dpo/margin_std": 680.8991088867188, + "step": 590 + }, + { + "KL/chosen_KL_mean": -681.802490234375, + "KL/mean": -927.5146484375, + "KL/rejected_KL_mean": -1173.226806640625, + "KL/std": 607.2724609375, + "epoch": 0.8678414096916299, + "fcm_dpo/beta": 0.0008302570786327124, + "fcm_dpo/delta": -0.010073492303490639, + "fcm_dpo/margin": 491.42431640625, + "fcm_dpo/q_t": 0.40566641092300415, + "grad_norm": 36.62338638305664, + "learning_rate": 2.678415274939408e-08, + "logits/chosen": -1.1887190341949463, + "logits/rejected": -1.203287124633789, + "logps/chosen": -738.0430908203125, + "logps/ref_chosen": -56.24061965942383, + "logps/ref_rejected": -83.78629302978516, + "logps/rejected": -1257.012939453125, + "loss": 1.0977, + "margin_dpo/margin_mean": 491.42431640625, + "margin_dpo/margin_std": 697.486572265625, + "step": 591 + }, + { + "KL/chosen_KL_mean": -740.6405639648438, + "KL/mean": -975.1524658203125, + "KL/rejected_KL_mean": -1209.6644287109375, + "KL/std": 639.179931640625, + "epoch": 0.869309838472834, + "fcm_dpo/beta": 0.0008298831526190042, + "fcm_dpo/delta": 0.013343624770641327, + "fcm_dpo/margin": 469.0238037109375, + "fcm_dpo/q_t": 0.413235068321228, + "grad_norm": 32.85460662841797, + "learning_rate": 2.6209177161234442e-08, + "logits/chosen": -1.1930391788482666, + "logits/rejected": -1.2116713523864746, + "logps/chosen": -788.580810546875, + "logps/ref_chosen": -47.94025421142578, + "logps/ref_rejected": -75.73287963867188, + "logps/rejected": -1285.397216796875, + "loss": 1.152, + "margin_dpo/margin_mean": 469.0238342285156, + "margin_dpo/margin_std": 840.769775390625, + "step": 592 + }, + { + "KL/chosen_KL_mean": -722.489013671875, + "KL/mean": -891.135986328125, + "KL/rejected_KL_mean": -1059.782958984375, + "KL/std": 666.1737060546875, + "epoch": 0.8707782672540382, + "fcm_dpo/beta": 0.0008565220050513744, + "fcm_dpo/delta": 0.02374492585659027, + "fcm_dpo/margin": 337.2939147949219, + "fcm_dpo/q_t": 0.4361189305782318, + "grad_norm": 36.292320251464844, + "learning_rate": 2.564009866938349e-08, + "logits/chosen": -1.0026201009750366, + "logits/rejected": -1.000308871269226, + "logps/chosen": -771.1798095703125, + "logps/ref_chosen": -48.690757751464844, + "logps/ref_rejected": -60.90800094604492, + "logps/rejected": -1120.69091796875, + "loss": 1.2122, + "margin_dpo/margin_mean": 337.2939147949219, + "margin_dpo/margin_std": 735.4708862304688, + "step": 593 + }, + { + "KL/chosen_KL_mean": -694.244384765625, + "KL/mean": -903.6416015625, + "KL/rejected_KL_mean": -1113.038818359375, + "KL/std": 638.9234619140625, + "epoch": 0.8722466960352423, + "fcm_dpo/beta": 0.000834466889500618, + "fcm_dpo/delta": -0.05287259444594383, + "fcm_dpo/margin": 418.79449462890625, + "fcm_dpo/q_t": 0.41835910081863403, + "grad_norm": 43.15362548828125, + "learning_rate": 2.5076932269588708e-08, + "logits/chosen": -1.1837817430496216, + "logits/rejected": -1.1828114986419678, + "logps/chosen": -749.1792602539062, + "logps/ref_chosen": -54.93488693237305, + "logps/ref_rejected": -86.09967803955078, + "logps/rejected": -1199.1385498046875, + "loss": 1.1504, + "margin_dpo/margin_mean": 418.7945556640625, + "margin_dpo/margin_std": 674.802001953125, + "step": 594 + }, + { + "KL/chosen_KL_mean": -639.7455444335938, + "KL/mean": -877.6624755859375, + "KL/rejected_KL_mean": -1115.579345703125, + "KL/std": 593.7979736328125, + "epoch": 0.8737151248164464, + "fcm_dpo/beta": 0.0008135140524245799, + "fcm_dpo/delta": 0.015700072050094604, + "fcm_dpo/margin": 475.833740234375, + "fcm_dpo/q_t": 0.4137133061885834, + "grad_norm": 40.35921859741211, + "learning_rate": 2.451969280180849e-08, + "logits/chosen": -1.1239649057388306, + "logits/rejected": -1.1565492153167725, + "logps/chosen": -689.1659545898438, + "logps/ref_chosen": -49.4204216003418, + "logps/ref_rejected": -80.62731170654297, + "logps/rejected": -1196.20654296875, + "loss": 1.1059, + "margin_dpo/margin_mean": 475.83380126953125, + "margin_dpo/margin_std": 678.7783203125, + "step": 595 + }, + { + "KL/chosen_KL_mean": -760.5462646484375, + "KL/mean": -910.779052734375, + "KL/rejected_KL_mean": -1061.011962890625, + "KL/std": 600.0560913085938, + "epoch": 0.8751835535976505, + "fcm_dpo/beta": 0.0009213665034621954, + "fcm_dpo/delta": 0.14197216928005219, + "fcm_dpo/margin": 300.4656982421875, + "fcm_dpo/q_t": 0.4435434341430664, + "grad_norm": 95.65324401855469, + "learning_rate": 2.396839494982103e-08, + "logits/chosen": -1.1321377754211426, + "logits/rejected": -1.1026864051818848, + "logps/chosen": -820.3379516601562, + "logps/ref_chosen": -59.791683197021484, + "logps/ref_rejected": -80.09111785888672, + "logps/rejected": -1141.10302734375, + "loss": 1.2344, + "margin_dpo/margin_mean": 300.4656677246094, + "margin_dpo/margin_std": 739.6461181640625, + "step": 596 + }, + { + "KL/chosen_KL_mean": -720.8272705078125, + "KL/mean": -1009.4996337890625, + "KL/rejected_KL_mean": -1298.1719970703125, + "KL/std": 710.2843017578125, + "epoch": 0.8766519823788547, + "fcm_dpo/beta": 0.0008209669031202793, + "fcm_dpo/delta": -0.12117333710193634, + "fcm_dpo/margin": 577.3447265625, + "fcm_dpo/q_t": 0.389828622341156, + "grad_norm": 26.715089797973633, + "learning_rate": 2.3423053240837514e-08, + "logits/chosen": -1.0798090696334839, + "logits/rejected": -1.139901876449585, + "logps/chosen": -778.0880126953125, + "logps/ref_chosen": -57.26078796386719, + "logps/ref_rejected": -100.6937255859375, + "logps/rejected": -1398.86572265625, + "loss": 1.0726, + "margin_dpo/margin_mean": 577.3447265625, + "margin_dpo/margin_std": 803.3443603515625, + "step": 597 + }, + { + "KL/chosen_KL_mean": -685.9241943359375, + "KL/mean": -916.6578369140625, + "KL/rejected_KL_mean": -1147.391357421875, + "KL/std": 597.9251098632812, + "epoch": 0.8781204111600588, + "fcm_dpo/beta": 0.0008235409623011947, + "fcm_dpo/delta": 0.01904139667749405, + "fcm_dpo/margin": 461.46710205078125, + "fcm_dpo/q_t": 0.4120750427246094, + "grad_norm": 30.65525245666504, + "learning_rate": 2.2883682045119062e-08, + "logits/chosen": -1.1381169557571411, + "logits/rejected": -1.1627564430236816, + "logps/chosen": -738.4427490234375, + "logps/ref_chosen": -52.51850509643555, + "logps/ref_rejected": -89.44385528564453, + "logps/rejected": -1236.835205078125, + "loss": 1.1215, + "margin_dpo/margin_mean": 461.4671630859375, + "margin_dpo/margin_std": 674.5496826171875, + "step": 598 + }, + { + "KL/chosen_KL_mean": -730.5025634765625, + "KL/mean": -930.897216796875, + "KL/rejected_KL_mean": -1131.2918701171875, + "KL/std": 586.238037109375, + "epoch": 0.8795888399412628, + "fcm_dpo/beta": 0.0008123376173898578, + "fcm_dpo/delta": -0.05563164874911308, + "fcm_dpo/margin": 400.7892761230469, + "fcm_dpo/q_t": 0.4195353388786316, + "grad_norm": 32.0465202331543, + "learning_rate": 2.2350295575598367e-08, + "logits/chosen": -1.0951640605926514, + "logits/rejected": -1.108811616897583, + "logps/chosen": -780.3052978515625, + "logps/ref_chosen": -49.802677154541016, + "logps/ref_rejected": -82.978515625, + "logps/rejected": -1214.2703857421875, + "loss": 1.1353, + "margin_dpo/margin_mean": 400.789306640625, + "margin_dpo/margin_std": 522.7140502929688, + "step": 599 + }, + { + "KL/chosen_KL_mean": -804.6961669921875, + "KL/mean": -995.6710205078125, + "KL/rejected_KL_mean": -1186.645751953125, + "KL/std": 597.6373291015625, + "epoch": 0.8810572687224669, + "fcm_dpo/beta": 0.0008582416921854019, + "fcm_dpo/delta": 0.08676525950431824, + "fcm_dpo/margin": 381.94964599609375, + "fcm_dpo/q_t": 0.4284188151359558, + "grad_norm": 34.12265396118164, + "learning_rate": 2.1822907887504932e-08, + "logits/chosen": -1.236138939857483, + "logits/rejected": -1.2366291284561157, + "logps/chosen": -871.131103515625, + "logps/ref_chosen": -66.43487548828125, + "logps/ref_rejected": -85.45649719238281, + "logps/rejected": -1272.102294921875, + "loss": 1.1788, + "margin_dpo/margin_mean": 381.94964599609375, + "margin_dpo/margin_std": 733.8538818359375, + "step": 600 + }, + { + "KL/chosen_KL_mean": -790.10302734375, + "KL/mean": -1049.4549560546875, + "KL/rejected_KL_mean": -1308.806884765625, + "KL/std": 661.8978271484375, + "epoch": 0.882525697503671, + "fcm_dpo/beta": 0.000848105177283287, + "fcm_dpo/delta": -0.05170179903507233, + "fcm_dpo/margin": 518.703857421875, + "fcm_dpo/q_t": 0.39643436670303345, + "grad_norm": 31.908023834228516, + "learning_rate": 2.1301532877994742e-08, + "logits/chosen": -1.1642279624938965, + "logits/rejected": -1.2025851011276245, + "logps/chosen": -849.2366943359375, + "logps/ref_chosen": -59.13361358642578, + "logps/ref_rejected": -94.69093322753906, + "logps/rejected": -1403.497802734375, + "loss": 1.0642, + "margin_dpo/margin_mean": 518.703857421875, + "margin_dpo/margin_std": 650.58935546875, + "step": 601 + }, + { + "KL/chosen_KL_mean": -551.074951171875, + "KL/mean": -840.6611938476562, + "KL/rejected_KL_mean": -1130.24755859375, + "KL/std": 586.427490234375, + "epoch": 0.8839941262848752, + "fcm_dpo/beta": 0.0008106931345537305, + "fcm_dpo/delta": -0.0917593240737915, + "fcm_dpo/margin": 579.1724853515625, + "fcm_dpo/q_t": 0.38940227031707764, + "grad_norm": 74.04801177978516, + "learning_rate": 2.0786184285784298e-08, + "logits/chosen": -1.1794953346252441, + "logits/rejected": -1.2283287048339844, + "logps/chosen": -599.66845703125, + "logps/ref_chosen": -48.59352111816406, + "logps/ref_rejected": -87.6685562133789, + "logps/rejected": -1217.916015625, + "loss": 1.0376, + "margin_dpo/margin_mean": 579.1725463867188, + "margin_dpo/margin_std": 647.2698974609375, + "step": 602 + }, + { + "KL/chosen_KL_mean": -703.7813720703125, + "KL/mean": -975.3844604492188, + "KL/rejected_KL_mean": -1246.987548828125, + "KL/std": 681.866943359375, + "epoch": 0.8854625550660793, + "fcm_dpo/beta": 0.0007574381306767464, + "fcm_dpo/delta": -0.014486655592918396, + "fcm_dpo/margin": 543.206298828125, + "fcm_dpo/q_t": 0.4078108072280884, + "grad_norm": 36.255306243896484, + "learning_rate": 2.0276875690788204e-08, + "logits/chosen": -1.1725047826766968, + "logits/rejected": -1.1789551973342896, + "logps/chosen": -774.1959838867188, + "logps/ref_chosen": -70.41461944580078, + "logps/ref_rejected": -100.32559967041016, + "logps/rejected": -1347.313232421875, + "loss": 1.0949, + "margin_dpo/margin_mean": 543.206298828125, + "margin_dpo/margin_std": 783.991943359375, + "step": 603 + }, + { + "KL/chosen_KL_mean": -677.0938110351562, + "KL/mean": -970.4219360351562, + "KL/rejected_KL_mean": -1263.75, + "KL/std": 666.1785888671875, + "epoch": 0.8869309838472834, + "fcm_dpo/beta": 0.0007347924984060228, + "fcm_dpo/delta": -0.03963564336299896, + "fcm_dpo/margin": 586.65625, + "fcm_dpo/q_t": 0.40320873260498047, + "grad_norm": 34.325931549072266, + "learning_rate": 1.977362051376158e-08, + "logits/chosen": -1.1817028522491455, + "logits/rejected": -1.2359271049499512, + "logps/chosen": -723.5518798828125, + "logps/ref_chosen": -46.45808029174805, + "logps/ref_rejected": -91.8544921875, + "logps/rejected": -1355.6044921875, + "loss": 1.0899, + "margin_dpo/margin_mean": 586.65625, + "margin_dpo/margin_std": 854.323974609375, + "step": 604 + }, + { + "KL/chosen_KL_mean": -748.796875, + "KL/mean": -951.0224609375, + "KL/rejected_KL_mean": -1153.248046875, + "KL/std": 612.2020263671875, + "epoch": 0.8883994126284875, + "fcm_dpo/beta": 0.000774614978581667, + "fcm_dpo/delta": 0.10340875387191772, + "fcm_dpo/margin": 404.4511413574219, + "fcm_dpo/q_t": 0.4324561655521393, + "grad_norm": 47.217437744140625, + "learning_rate": 1.9276432015946446e-08, + "logits/chosen": -1.118881344795227, + "logits/rejected": -1.1395517587661743, + "logps/chosen": -815.0462646484375, + "logps/ref_chosen": -66.24933624267578, + "logps/ref_rejected": -102.30496978759766, + "logps/rejected": -1255.552978515625, + "loss": 1.176, + "margin_dpo/margin_mean": 404.451171875, + "margin_dpo/margin_std": 743.8991088867188, + "step": 605 + }, + { + "KL/chosen_KL_mean": -717.6261596679688, + "KL/mean": -960.13623046875, + "KL/rejected_KL_mean": -1202.646240234375, + "KL/std": 601.002685546875, + "epoch": 0.8898678414096917, + "fcm_dpo/beta": 0.0008218929287977517, + "fcm_dpo/delta": 0.00010516494512557983, + "fcm_dpo/margin": 485.0201721191406, + "fcm_dpo/q_t": 0.40895628929138184, + "grad_norm": 25.46479606628418, + "learning_rate": 1.8785323298722093e-08, + "logits/chosen": -1.141870141029358, + "logits/rejected": -1.163278579711914, + "logps/chosen": -772.4452514648438, + "logps/ref_chosen": -54.819122314453125, + "logps/ref_rejected": -98.37146759033203, + "logps/rejected": -1301.017822265625, + "loss": 1.0914, + "margin_dpo/margin_mean": 485.02020263671875, + "margin_dpo/margin_std": 640.8707275390625, + "step": 606 + }, + { + "KL/chosen_KL_mean": -767.1453247070312, + "KL/mean": -966.7313232421875, + "KL/rejected_KL_mean": -1166.3173828125, + "KL/std": 642.6415405273438, + "epoch": 0.8913362701908958, + "fcm_dpo/beta": 0.0008614829275757074, + "fcm_dpo/delta": 0.06497818231582642, + "fcm_dpo/margin": 399.1719970703125, + "fcm_dpo/q_t": 0.42393672466278076, + "grad_norm": 33.80935287475586, + "learning_rate": 1.8300307303259904e-08, + "logits/chosen": -1.1142241954803467, + "logits/rejected": -1.104911208152771, + "logps/chosen": -825.2293701171875, + "logps/ref_chosen": -58.08403778076172, + "logps/ref_rejected": -79.777099609375, + "logps/rejected": -1246.094482421875, + "loss": 1.159, + "margin_dpo/margin_mean": 399.1720275878906, + "margin_dpo/margin_std": 708.9419555664062, + "step": 607 + }, + { + "KL/chosen_KL_mean": -686.8588256835938, + "KL/mean": -919.4632568359375, + "KL/rejected_KL_mean": -1152.0677490234375, + "KL/std": 572.4481811523438, + "epoch": 0.8928046989720999, + "fcm_dpo/beta": 0.0008695480646565557, + "fcm_dpo/delta": -0.006582053378224373, + "fcm_dpo/margin": 465.2089538574219, + "fcm_dpo/q_t": 0.40590620040893555, + "grad_norm": 34.56772232055664, + "learning_rate": 1.7821396810182437e-08, + "logits/chosen": -1.1690213680267334, + "logits/rejected": -1.1923552751541138, + "logps/chosen": -744.3096923828125, + "logps/ref_chosen": -57.450836181640625, + "logps/ref_rejected": -94.77339172363281, + "logps/rejected": -1246.841064453125, + "loss": 1.0818, + "margin_dpo/margin_mean": 465.208984375, + "margin_dpo/margin_std": 576.3578491210938, + "step": 608 + }, + { + "KL/chosen_KL_mean": -712.329345703125, + "KL/mean": -1004.8988037109375, + "KL/rejected_KL_mean": -1297.46826171875, + "KL/std": 753.2168579101562, + "epoch": 0.8942731277533039, + "fcm_dpo/beta": 0.000799752539023757, + "fcm_dpo/delta": -0.0890444964170456, + "fcm_dpo/margin": 585.1388549804688, + "fcm_dpo/q_t": 0.39761674404144287, + "grad_norm": 28.98295783996582, + "learning_rate": 1.7348604439226617e-08, + "logits/chosen": -1.2440345287322998, + "logits/rejected": -1.281120777130127, + "logps/chosen": -771.134765625, + "logps/ref_chosen": -58.805355072021484, + "logps/ref_rejected": -88.81600952148438, + "logps/rejected": -1386.2841796875, + "loss": 1.082, + "margin_dpo/margin_mean": 585.1388549804688, + "margin_dpo/margin_std": 920.4691162109375, + "step": 609 + }, + { + "KL/chosen_KL_mean": -670.919189453125, + "KL/mean": -864.196044921875, + "KL/rejected_KL_mean": -1057.472900390625, + "KL/std": 553.8067626953125, + "epoch": 0.895741556534508, + "fcm_dpo/beta": 0.0008423338877037168, + "fcm_dpo/delta": 0.08745961636304855, + "fcm_dpo/margin": 386.55364990234375, + "fcm_dpo/q_t": 0.42681533098220825, + "grad_norm": 36.83287811279297, + "learning_rate": 1.6881942648911074e-08, + "logits/chosen": -1.1055896282196045, + "logits/rejected": -1.0923852920532227, + "logps/chosen": -736.6142578125, + "logps/ref_chosen": -65.69503784179688, + "logps/ref_rejected": -83.40538787841797, + "logps/rejected": -1140.878173828125, + "loss": 1.1577, + "margin_dpo/margin_mean": 386.55364990234375, + "margin_dpo/margin_std": 650.5970458984375, + "step": 610 + }, + { + "KL/chosen_KL_mean": -696.1116943359375, + "KL/mean": -999.7044677734375, + "KL/rejected_KL_mean": -1303.29736328125, + "KL/std": 736.2969970703125, + "epoch": 0.8972099853157122, + "fcm_dpo/beta": 0.0007899197516962886, + "fcm_dpo/delta": -0.10505129396915436, + "fcm_dpo/margin": 607.1857299804688, + "fcm_dpo/q_t": 0.39069780707359314, + "grad_norm": 37.02203369140625, + "learning_rate": 1.6421423736208e-08, + "logits/chosen": -1.1951907873153687, + "logits/rejected": -1.256514310836792, + "logps/chosen": -748.7111206054688, + "logps/ref_chosen": -52.59946823120117, + "logps/ref_rejected": -86.33099365234375, + "logps/rejected": -1389.62841796875, + "loss": 1.0611, + "margin_dpo/margin_mean": 607.1856689453125, + "margin_dpo/margin_std": 833.6317138671875, + "step": 611 + }, + { + "KL/chosen_KL_mean": -774.832275390625, + "KL/mean": -1000.640625, + "KL/rejected_KL_mean": -1226.448974609375, + "KL/std": 606.4058837890625, + "epoch": 0.8986784140969163, + "fcm_dpo/beta": 0.0008069322793744504, + "fcm_dpo/delta": 0.04110585153102875, + "fcm_dpo/margin": 451.61676025390625, + "fcm_dpo/q_t": 0.417686402797699, + "grad_norm": 29.89786148071289, + "learning_rate": 1.5967059836219042e-08, + "logits/chosen": -1.1885337829589844, + "logits/rejected": -1.1959373950958252, + "logps/chosen": -834.156005859375, + "logps/ref_chosen": -59.32372283935547, + "logps/ref_rejected": -88.31239318847656, + "logps/rejected": -1314.761474609375, + "loss": 1.1247, + "margin_dpo/margin_mean": 451.61676025390625, + "margin_dpo/margin_std": 676.6371459960938, + "step": 612 + }, + { + "KL/chosen_KL_mean": -677.6046142578125, + "KL/mean": -974.4893188476562, + "KL/rejected_KL_mean": -1271.3740234375, + "KL/std": 672.0018310546875, + "epoch": 0.9001468428781204, + "fcm_dpo/beta": 0.0007531556766480207, + "fcm_dpo/delta": -0.0629609227180481, + "fcm_dpo/margin": 593.7694702148438, + "fcm_dpo/q_t": 0.39546507596969604, + "grad_norm": 26.967117309570312, + "learning_rate": 1.551886292185553e-08, + "logits/chosen": -1.1772477626800537, + "logits/rejected": -1.2423419952392578, + "logps/chosen": -737.3345336914062, + "logps/ref_chosen": -59.72996520996094, + "logps/ref_rejected": -105.10752868652344, + "logps/rejected": -1376.481689453125, + "loss": 1.0554, + "margin_dpo/margin_mean": 593.7694702148438, + "margin_dpo/margin_std": 724.92333984375, + "step": 613 + }, + { + "KL/chosen_KL_mean": -767.2695922851562, + "KL/mean": -1050.36181640625, + "KL/rejected_KL_mean": -1333.453857421875, + "KL/std": 694.145751953125, + "epoch": 0.9016152716593245, + "fcm_dpo/beta": 0.0007397818844765425, + "fcm_dpo/delta": -0.0238279327750206, + "fcm_dpo/margin": 566.184326171875, + "fcm_dpo/q_t": 0.4034821391105652, + "grad_norm": 31.130779266357422, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": -1.1655349731445312, + "logits/rejected": -1.2575234174728394, + "logps/chosen": -820.2085571289062, + "logps/ref_chosen": -52.93898010253906, + "logps/ref_rejected": -104.67938232421875, + "logps/rejected": -1438.13330078125, + "loss": 1.0995, + "margin_dpo/margin_mean": 566.184326171875, + "margin_dpo/margin_std": 844.2413330078125, + "step": 614 + }, + { + "KL/chosen_KL_mean": -710.9827270507812, + "KL/mean": -932.9111328125, + "KL/rejected_KL_mean": -1154.839599609375, + "KL/std": 673.3746337890625, + "epoch": 0.9030837004405287, + "fcm_dpo/beta": 0.0007818863959982991, + "fcm_dpo/delta": 0.06144233047962189, + "fcm_dpo/margin": 443.85687255859375, + "fcm_dpo/q_t": 0.4220801591873169, + "grad_norm": 32.169227600097656, + "learning_rate": 1.4641017128809801e-08, + "logits/chosen": -1.1130198240280151, + "logits/rejected": -1.1419811248779297, + "logps/chosen": -776.800048828125, + "logps/ref_chosen": -65.81727600097656, + "logps/ref_rejected": -95.17749786376953, + "logps/rejected": -1250.01708984375, + "loss": 1.1437, + "margin_dpo/margin_mean": 443.85687255859375, + "margin_dpo/margin_std": 725.8150634765625, + "step": 615 + }, + { + "KL/chosen_KL_mean": -881.0988159179688, + "KL/mean": -1073.1583251953125, + "KL/rejected_KL_mean": -1265.2177734375, + "KL/std": 594.451171875, + "epoch": 0.9045521292217328, + "fcm_dpo/beta": 0.0008393003372475505, + "fcm_dpo/delta": 0.09190573543310165, + "fcm_dpo/margin": 384.11895751953125, + "fcm_dpo/q_t": 0.428872287273407, + "grad_norm": 44.38339614868164, + "learning_rate": 1.4211391382180637e-08, + "logits/chosen": -1.2580994367599487, + "logits/rejected": -1.2420811653137207, + "logps/chosen": -946.231689453125, + "logps/ref_chosen": -65.13285827636719, + "logps/ref_rejected": -74.70050048828125, + "logps/rejected": -1339.9183349609375, + "loss": 1.1633, + "margin_dpo/margin_mean": 384.11895751953125, + "margin_dpo/margin_std": 659.7879638671875, + "step": 616 + }, + { + "KL/chosen_KL_mean": -825.0336303710938, + "KL/mean": -983.9627685546875, + "KL/rejected_KL_mean": -1142.891845703125, + "KL/std": 582.34765625, + "epoch": 0.9060205580029369, + "fcm_dpo/beta": 0.0009453271050006151, + "fcm_dpo/delta": 0.11414283514022827, + "fcm_dpo/margin": 317.8581848144531, + "fcm_dpo/q_t": 0.4370740056037903, + "grad_norm": 73.6228256225586, + "learning_rate": 1.378797888467345e-08, + "logits/chosen": -1.1125876903533936, + "logits/rejected": -1.0867927074432373, + "logps/chosen": -888.0391845703125, + "logps/ref_chosen": -63.005550384521484, + "logps/ref_rejected": -64.234130859375, + "logps/rejected": -1207.1259765625, + "loss": 1.2115, + "margin_dpo/margin_mean": 317.85821533203125, + "margin_dpo/margin_std": 713.1170654296875, + "step": 617 + }, + { + "KL/chosen_KL_mean": -846.281494140625, + "KL/mean": -1157.50244140625, + "KL/rejected_KL_mean": -1468.7235107421875, + "KL/std": 774.007080078125, + "epoch": 0.9074889867841409, + "fcm_dpo/beta": 0.0008609164506196976, + "fcm_dpo/delta": -0.18371938169002533, + "fcm_dpo/margin": 622.4420776367188, + "fcm_dpo/q_t": 0.38035058975219727, + "grad_norm": 32.81733322143555, + "learning_rate": 1.3370790793601371e-08, + "logits/chosen": -1.1643829345703125, + "logits/rejected": -1.221153736114502, + "logps/chosen": -913.3828125, + "logps/ref_chosen": -67.10134887695312, + "logps/ref_rejected": -92.15340423583984, + "logps/rejected": -1560.876953125, + "loss": 1.087, + "margin_dpo/margin_mean": 622.4420776367188, + "margin_dpo/margin_std": 1005.3324584960938, + "step": 618 + }, + { + "KL/chosen_KL_mean": -804.8983154296875, + "KL/mean": -1043.05419921875, + "KL/rejected_KL_mean": -1281.2099609375, + "KL/std": 693.5120239257812, + "epoch": 0.908957415565345, + "fcm_dpo/beta": 0.000811479170806706, + "fcm_dpo/delta": 0.0165516696870327, + "fcm_dpo/margin": 476.311767578125, + "fcm_dpo/q_t": 0.4198111295700073, + "grad_norm": 43.509002685546875, + "learning_rate": 1.2959838102258535e-08, + "logits/chosen": -1.1278049945831299, + "logits/rejected": -1.1508402824401855, + "logps/chosen": -860.87646484375, + "logps/ref_chosen": -55.978233337402344, + "logps/ref_rejected": -93.1854019165039, + "logps/rejected": -1374.3953857421875, + "loss": 1.1626, + "margin_dpo/margin_mean": 476.311767578125, + "margin_dpo/margin_std": 932.0376586914062, + "step": 619 + }, + { + "KL/chosen_KL_mean": -741.0186157226562, + "KL/mean": -960.3197021484375, + "KL/rejected_KL_mean": -1179.6207275390625, + "KL/std": 604.758544921875, + "epoch": 0.9104258443465492, + "fcm_dpo/beta": 0.000822564004920423, + "fcm_dpo/delta": 0.046854112297296524, + "fcm_dpo/margin": 438.6020812988281, + "fcm_dpo/q_t": 0.4194413125514984, + "grad_norm": 36.871070861816406, + "learning_rate": 1.2555131639630567e-08, + "logits/chosen": -1.2204086780548096, + "logits/rejected": -1.2356070280075073, + "logps/chosen": -800.8161010742188, + "logps/ref_chosen": -59.79750061035156, + "logps/ref_rejected": -78.41075134277344, + "logps/rejected": -1258.031494140625, + "loss": 1.131, + "margin_dpo/margin_mean": 438.6020812988281, + "margin_dpo/margin_std": 668.6788940429688, + "step": 620 + }, + { + "KL/chosen_KL_mean": -726.4075317382812, + "KL/mean": -1026.935546875, + "KL/rejected_KL_mean": -1327.463623046875, + "KL/std": 694.7574462890625, + "epoch": 0.9118942731277533, + "fcm_dpo/beta": 0.0008126673055812716, + "fcm_dpo/delta": -0.11893101036548615, + "fcm_dpo/margin": 601.0560302734375, + "fcm_dpo/q_t": 0.38432037830352783, + "grad_norm": 41.782691955566406, + "learning_rate": 1.2156682070109086e-08, + "logits/chosen": -1.2633055448532104, + "logits/rejected": -1.3201026916503906, + "logps/chosen": -780.34130859375, + "logps/ref_chosen": -53.93375778198242, + "logps/ref_rejected": -88.36951446533203, + "logps/rejected": -1415.8330078125, + "loss": 1.0593, + "margin_dpo/margin_mean": 601.0560302734375, + "margin_dpo/margin_std": 780.2889404296875, + "step": 621 + }, + { + "KL/chosen_KL_mean": -730.9711303710938, + "KL/mean": -956.7696533203125, + "KL/rejected_KL_mean": -1182.568115234375, + "KL/std": 568.827392578125, + "epoch": 0.9133627019089574, + "fcm_dpo/beta": 0.0007760204607620835, + "fcm_dpo/delta": 0.0598348043859005, + "fcm_dpo/margin": 451.59698486328125, + "fcm_dpo/q_t": 0.4228161573410034, + "grad_norm": 30.10220718383789, + "learning_rate": 1.1764499893210878e-08, + "logits/chosen": -1.0861146450042725, + "logits/rejected": -1.0753684043884277, + "logps/chosen": -791.2569580078125, + "logps/ref_chosen": -60.28582000732422, + "logps/ref_rejected": -85.51873779296875, + "logps/rejected": -1268.0869140625, + "loss": 1.1376, + "margin_dpo/margin_mean": 451.5970153808594, + "margin_dpo/margin_std": 714.3831787109375, + "step": 622 + }, + { + "KL/chosen_KL_mean": -812.854736328125, + "KL/mean": -974.1890258789062, + "KL/rejected_KL_mean": -1135.523193359375, + "KL/std": 589.9398193359375, + "epoch": 0.9148311306901615, + "fcm_dpo/beta": 0.000887159607373178, + "fcm_dpo/delta": 0.13209515810012817, + "fcm_dpo/margin": 322.6684875488281, + "fcm_dpo/q_t": 0.440715491771698, + "grad_norm": 47.28063201904297, + "learning_rate": 1.1378595443300998e-08, + "logits/chosen": -1.2864371538162231, + "logits/rejected": -1.2883734703063965, + "logps/chosen": -877.01171875, + "logps/ref_chosen": -64.1569595336914, + "logps/ref_rejected": -85.08304595947266, + "logps/rejected": -1220.606201171875, + "loss": 1.2157, + "margin_dpo/margin_mean": 322.6684875488281, + "margin_dpo/margin_std": 727.4967041015625, + "step": 623 + }, + { + "KL/chosen_KL_mean": -803.52783203125, + "KL/mean": -1085.2625732421875, + "KL/rejected_KL_mean": -1366.997314453125, + "KL/std": 636.1299438476562, + "epoch": 0.9162995594713657, + "fcm_dpo/beta": 0.0008517538080923259, + "fcm_dpo/delta": -0.10489484667778015, + "fcm_dpo/margin": 563.469482421875, + "fcm_dpo/q_t": 0.3851096034049988, + "grad_norm": 33.36922836303711, + "learning_rate": 1.0998978889320582e-08, + "logits/chosen": -1.2476253509521484, + "logits/rejected": -1.259102463722229, + "logps/chosen": -875.446533203125, + "logps/ref_chosen": -71.91862487792969, + "logps/ref_rejected": -97.13203430175781, + "logps/rejected": -1464.12939453125, + "loss": 1.0463, + "margin_dpo/margin_mean": 563.469482421875, + "margin_dpo/margin_std": 700.0657958984375, + "step": 624 + }, + { + "KL/chosen_KL_mean": -754.678466796875, + "KL/mean": -1054.7698974609375, + "KL/rejected_KL_mean": -1354.8614501953125, + "KL/std": 678.5252685546875, + "epoch": 0.9177679882525698, + "fcm_dpo/beta": 0.000783138326369226, + "fcm_dpo/delta": -0.09115005284547806, + "fcm_dpo/margin": 600.1829833984375, + "fcm_dpo/q_t": 0.38854146003723145, + "grad_norm": 52.66489791870117, + "learning_rate": 1.0625660234518913e-08, + "logits/chosen": -1.1888365745544434, + "logits/rejected": -1.228683352470398, + "logps/chosen": -813.0205078125, + "logps/ref_chosen": -58.342071533203125, + "logps/ref_rejected": -86.09038543701172, + "logps/rejected": -1440.9517822265625, + "loss": 1.0299, + "margin_dpo/margin_mean": 600.1829833984375, + "margin_dpo/margin_std": 645.32763671875, + "step": 625 + }, + { + "KL/chosen_KL_mean": -938.5303344726562, + "KL/mean": -1111.8682861328125, + "KL/rejected_KL_mean": -1285.206298828125, + "KL/std": 750.4013671875, + "epoch": 0.9192364170337739, + "fcm_dpo/beta": 0.0008260097820311785, + "fcm_dpo/delta": 0.13311442732810974, + "fcm_dpo/margin": 346.6759033203125, + "fcm_dpo/q_t": 0.4367343485355377, + "grad_norm": 28.725139617919922, + "learning_rate": 1.0258649316189721e-08, + "logits/chosen": -1.1261563301086426, + "logits/rejected": -1.1264156103134155, + "logps/chosen": -1013.6429443359375, + "logps/ref_chosen": -75.11260986328125, + "logps/ref_rejected": -99.188720703125, + "logps/rejected": -1384.39501953125, + "loss": 1.2138, + "margin_dpo/margin_mean": 346.6759033203125, + "margin_dpo/margin_std": 732.3070068359375, + "step": 626 + }, + { + "KL/chosen_KL_mean": -613.0308837890625, + "KL/mean": -944.4686279296875, + "KL/rejected_KL_mean": -1275.90625, + "KL/std": 782.295166015625, + "epoch": 0.920704845814978, + "fcm_dpo/beta": 0.000785567332059145, + "fcm_dpo/delta": -0.16194023191928864, + "fcm_dpo/margin": 662.8754272460938, + "fcm_dpo/q_t": 0.38459348678588867, + "grad_norm": 26.760295867919922, + "learning_rate": 9.897955805412e-09, + "logits/chosen": -1.0580464601516724, + "logits/rejected": -1.1465545892715454, + "logps/chosen": -660.7740478515625, + "logps/ref_chosen": -47.74314880371094, + "logps/ref_rejected": -106.75448608398438, + "logps/rejected": -1382.6607666015625, + "loss": 1.0457, + "margin_dpo/margin_mean": 662.8753662109375, + "margin_dpo/margin_std": 917.1488647460938, + "step": 627 + }, + { + "KL/chosen_KL_mean": -848.233642578125, + "KL/mean": -1102.293212890625, + "KL/rejected_KL_mean": -1356.35302734375, + "KL/std": 702.3494873046875, + "epoch": 0.922173274596182, + "fcm_dpo/beta": 0.0007409874233417213, + "fcm_dpo/delta": 0.02891882136464119, + "fcm_dpo/margin": 508.11920166015625, + "fcm_dpo/q_t": 0.41614317893981934, + "grad_norm": 28.3284854888916, + "learning_rate": 9.543589206795238e-09, + "logits/chosen": -1.2321488857269287, + "logits/rejected": -1.258644700050354, + "logps/chosen": -908.4166259765625, + "logps/ref_chosen": -60.182945251464844, + "logps/ref_rejected": -101.55467224121094, + "logps/rejected": -1457.90771484375, + "loss": 1.1272, + "margin_dpo/margin_mean": 508.1192626953125, + "margin_dpo/margin_std": 800.9361572265625, + "step": 628 + }, + { + "KL/chosen_KL_mean": -841.8361206054688, + "KL/mean": -1071.118408203125, + "KL/rejected_KL_mean": -1300.4007568359375, + "KL/std": 656.1351318359375, + "epoch": 0.9236417033773862, + "fcm_dpo/beta": 0.0007858018507249653, + "fcm_dpo/delta": 0.04754742980003357, + "fcm_dpo/margin": 458.56463623046875, + "fcm_dpo/q_t": 0.4175514578819275, + "grad_norm": 37.84564208984375, + "learning_rate": 9.19555885822887e-09, + "logits/chosen": -1.2372636795043945, + "logits/rejected": -1.2568615674972534, + "logps/chosen": -906.0496826171875, + "logps/ref_chosen": -64.21354675292969, + "logps/ref_rejected": -91.65367126464844, + "logps/rejected": -1392.054443359375, + "loss": 1.1255, + "margin_dpo/margin_mean": 458.56463623046875, + "margin_dpo/margin_std": 679.4099731445312, + "step": 629 + }, + { + "KL/chosen_KL_mean": -726.173583984375, + "KL/mean": -865.2925415039062, + "KL/rejected_KL_mean": -1004.4114990234375, + "KL/std": 653.0287475585938, + "epoch": 0.9251101321585903, + "fcm_dpo/beta": 0.0008041888941079378, + "fcm_dpo/delta": 0.038379114121198654, + "fcm_dpo/margin": 278.2379455566406, + "fcm_dpo/q_t": 0.45435744524002075, + "grad_norm": 49.139530181884766, + "learning_rate": 8.85387393063622e-09, + "logits/chosen": -1.1940525770187378, + "logits/rejected": -1.1697330474853516, + "logps/chosen": -785.464599609375, + "logps/ref_chosen": -59.29100036621094, + "logps/ref_rejected": -83.59829711914062, + "logps/rejected": -1088.009765625, + "loss": 1.2713, + "margin_dpo/margin_mean": 278.2379455566406, + "margin_dpo/margin_std": 808.236572265625, + "step": 630 + }, + { + "KL/chosen_KL_mean": -864.2023315429688, + "KL/mean": -1085.380126953125, + "KL/rejected_KL_mean": -1306.557861328125, + "KL/std": 646.6876831054688, + "epoch": 0.9265785609397944, + "fcm_dpo/beta": 0.000846027978695929, + "fcm_dpo/delta": 0.031014172360301018, + "fcm_dpo/margin": 442.35552978515625, + "fcm_dpo/q_t": 0.4147540330886841, + "grad_norm": 34.697113037109375, + "learning_rate": 8.518543427732949e-09, + "logits/chosen": -1.3160864114761353, + "logits/rejected": -1.3359469175338745, + "logps/chosen": -923.6559448242188, + "logps/ref_chosen": -59.45360565185547, + "logps/ref_rejected": -80.95156860351562, + "logps/rejected": -1387.509521484375, + "loss": 1.1439, + "margin_dpo/margin_mean": 442.35552978515625, + "margin_dpo/margin_std": 760.668212890625, + "step": 631 + }, + { + "KL/chosen_KL_mean": -795.5738525390625, + "KL/mean": -1007.8973388671875, + "KL/rejected_KL_mean": -1220.220947265625, + "KL/std": 619.8748779296875, + "epoch": 0.9280469897209985, + "fcm_dpo/beta": 0.0008544891024939716, + "fcm_dpo/delta": 0.044195935130119324, + "fcm_dpo/margin": 424.64697265625, + "fcm_dpo/q_t": 0.4180447459220886, + "grad_norm": 51.36729049682617, + "learning_rate": 8.189576185789637e-09, + "logits/chosen": -1.279064416885376, + "logits/rejected": -1.281667947769165, + "logps/chosen": -856.9254150390625, + "logps/ref_chosen": -61.35155487060547, + "logps/ref_rejected": -86.16017150878906, + "logps/rejected": -1306.381103515625, + "loss": 1.1631, + "margin_dpo/margin_mean": 424.64697265625, + "margin_dpo/margin_std": 784.182373046875, + "step": 632 + }, + { + "KL/chosen_KL_mean": -906.9592895507812, + "KL/mean": -1059.207275390625, + "KL/rejected_KL_mean": -1211.455322265625, + "KL/std": 601.04248046875, + "epoch": 0.9295154185022027, + "fcm_dpo/beta": 0.000900345272384584, + "fcm_dpo/delta": 0.03711273893713951, + "fcm_dpo/margin": 304.4959716796875, + "fcm_dpo/q_t": 0.4383799433708191, + "grad_norm": 50.936614990234375, + "learning_rate": 7.866980873399015e-09, + "logits/chosen": -1.30977463722229, + "logits/rejected": -1.322211503982544, + "logps/chosen": -964.2374267578125, + "logps/ref_chosen": -57.27816390991211, + "logps/ref_rejected": -91.58395385742188, + "logps/rejected": -1303.039306640625, + "loss": 1.2248, + "margin_dpo/margin_mean": 304.4960021972656, + "margin_dpo/margin_std": 702.047119140625, + "step": 633 + }, + { + "KL/chosen_KL_mean": -957.939208984375, + "KL/mean": -1103.0045166015625, + "KL/rejected_KL_mean": -1248.06982421875, + "KL/std": 700.0567626953125, + "epoch": 0.9309838472834068, + "fcm_dpo/beta": 0.0010044092778116465, + "fcm_dpo/delta": 0.12742158770561218, + "fcm_dpo/margin": 290.1304931640625, + "fcm_dpo/q_t": 0.4410281479358673, + "grad_norm": 48.39108657836914, + "learning_rate": 7.550765991247654e-09, + "logits/chosen": -1.1118695735931396, + "logits/rejected": -1.1093440055847168, + "logps/chosen": -1024.5582275390625, + "logps/ref_chosen": -66.61896514892578, + "logps/ref_rejected": -107.12564849853516, + "logps/rejected": -1355.1954345703125, + "loss": 1.2305, + "margin_dpo/margin_mean": 290.1304931640625, + "margin_dpo/margin_std": 708.994873046875, + "step": 634 + }, + { + "KL/chosen_KL_mean": -850.6639404296875, + "KL/mean": -1053.29736328125, + "KL/rejected_KL_mean": -1255.930908203125, + "KL/std": 765.2185668945312, + "epoch": 0.9324522760646109, + "fcm_dpo/beta": 0.001036031637340784, + "fcm_dpo/delta": -0.02556215599179268, + "fcm_dpo/margin": 405.26690673828125, + "fcm_dpo/q_t": 0.41270193457603455, + "grad_norm": 61.66108703613281, + "learning_rate": 7.240939871891699e-09, + "logits/chosen": -1.2386977672576904, + "logits/rejected": -1.2290127277374268, + "logps/chosen": -924.6194458007812, + "logps/ref_chosen": -73.95551300048828, + "logps/ref_rejected": -82.50045776367188, + "logps/rejected": -1338.431396484375, + "loss": 1.1466, + "margin_dpo/margin_mean": 405.2669372558594, + "margin_dpo/margin_std": 769.2952880859375, + "step": 635 + }, + { + "KL/chosen_KL_mean": -771.644775390625, + "KL/mean": -1004.4685668945312, + "KL/rejected_KL_mean": -1237.29248046875, + "KL/std": 704.637451171875, + "epoch": 0.933920704845815, + "fcm_dpo/beta": 0.0009967228397727013, + "fcm_dpo/delta": -0.08795134723186493, + "fcm_dpo/margin": 465.6476135253906, + "fcm_dpo/q_t": 0.39764294028282166, + "grad_norm": 32.073509216308594, + "learning_rate": 6.937510679537628e-09, + "logits/chosen": -1.1573783159255981, + "logits/rejected": -1.1642378568649292, + "logps/chosen": -831.273681640625, + "logps/ref_chosen": -59.628910064697266, + "logps/ref_rejected": -81.97883605957031, + "logps/rejected": -1319.271240234375, + "loss": 1.1023, + "margin_dpo/margin_mean": 465.6475830078125, + "margin_dpo/margin_std": 754.404541015625, + "step": 636 + }, + { + "KL/chosen_KL_mean": -774.3438110351562, + "KL/mean": -1051.230712890625, + "KL/rejected_KL_mean": -1328.117431640625, + "KL/std": 712.5130615234375, + "epoch": 0.9353891336270191, + "fcm_dpo/beta": 0.0008392821764573455, + "fcm_dpo/delta": -0.09347308427095413, + "fcm_dpo/margin": 553.773681640625, + "fcm_dpo/q_t": 0.39158618450164795, + "grad_norm": 31.033771514892578, + "learning_rate": 6.640486409826785e-09, + "logits/chosen": -1.2771704196929932, + "logits/rejected": -1.3423259258270264, + "logps/chosen": -823.9964599609375, + "logps/ref_chosen": -49.652687072753906, + "logps/ref_rejected": -98.40513610839844, + "logps/rejected": -1426.5225830078125, + "loss": 1.0572, + "margin_dpo/margin_mean": 553.773681640625, + "margin_dpo/margin_std": 714.4674072265625, + "step": 637 + }, + { + "KL/chosen_KL_mean": -740.0574951171875, + "KL/mean": -960.1834716796875, + "KL/rejected_KL_mean": -1180.3095703125, + "KL/std": 647.9360961914062, + "epoch": 0.9368575624082232, + "fcm_dpo/beta": 0.0008902228437364101, + "fcm_dpo/delta": -0.0007760748267173767, + "fcm_dpo/margin": 440.2519836425781, + "fcm_dpo/q_t": 0.40659117698669434, + "grad_norm": 46.73960494995117, + "learning_rate": 6.349874889624962e-09, + "logits/chosen": -1.0999634265899658, + "logits/rejected": -1.0819281339645386, + "logps/chosen": -798.2141723632812, + "logps/ref_chosen": -58.156639099121094, + "logps/ref_rejected": -79.3014907836914, + "logps/rejected": -1259.6109619140625, + "loss": 1.1511, + "margin_dpo/margin_mean": 440.2519836425781, + "margin_dpo/margin_std": 770.6236572265625, + "step": 638 + }, + { + "KL/chosen_KL_mean": -1026.832275390625, + "KL/mean": -1113.654052734375, + "KL/rejected_KL_mean": -1200.4757080078125, + "KL/std": 620.957275390625, + "epoch": 0.9383259911894273, + "fcm_dpo/beta": 0.0008462556288577616, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 173.64344787597656, + "fcm_dpo/q_t": 0.46698197722435, + "grad_norm": 103.9713363647461, + "learning_rate": 6.065683776815933e-09, + "logits/chosen": -1.081902265548706, + "logits/rejected": -1.0169289112091064, + "logps/chosen": -1099.155517578125, + "logps/ref_chosen": -72.32319641113281, + "logps/ref_rejected": -74.2749252319336, + "logps/rejected": -1274.7506103515625, + "loss": 1.3501, + "margin_dpo/margin_mean": 173.64346313476562, + "margin_dpo/margin_std": 787.2015380859375, + "step": 639 + }, + { + "KL/chosen_KL_mean": -761.8798828125, + "KL/mean": -1064.907958984375, + "KL/rejected_KL_mean": -1367.93603515625, + "KL/std": 741.346923828125, + "epoch": 0.9397944199706314, + "fcm_dpo/beta": 0.000793866696767509, + "fcm_dpo/delta": -0.10620252788066864, + "fcm_dpo/margin": 606.05615234375, + "fcm_dpo/q_t": 0.3925554156303406, + "grad_norm": 52.245452880859375, + "learning_rate": 5.7879205600998296e-09, + "logits/chosen": -1.0945394039154053, + "logits/rejected": -1.133887767791748, + "logps/chosen": -818.0142822265625, + "logps/ref_chosen": -56.13436508178711, + "logps/ref_rejected": -108.60014343261719, + "logps/rejected": -1476.5361328125, + "loss": 1.0622, + "margin_dpo/margin_mean": 606.05615234375, + "margin_dpo/margin_std": 859.9293212890625, + "step": 640 + }, + { + "KL/chosen_KL_mean": -864.1361083984375, + "KL/mean": -1076.710693359375, + "KL/rejected_KL_mean": -1289.285400390625, + "KL/std": 632.7630615234375, + "epoch": 0.9412628487518355, + "fcm_dpo/beta": 0.0008008034783415496, + "fcm_dpo/delta": 0.0718461126089096, + "fcm_dpo/margin": 425.14935302734375, + "fcm_dpo/q_t": 0.4279997944831848, + "grad_norm": 27.010753631591797, + "learning_rate": 5.516592558795746e-09, + "logits/chosen": -1.1861107349395752, + "logits/rejected": -1.2011582851409912, + "logps/chosen": -929.1329345703125, + "logps/ref_chosen": -64.99689483642578, + "logps/ref_rejected": -86.99232482910156, + "logps/rejected": -1376.2777099609375, + "loss": 1.1776, + "margin_dpo/margin_mean": 425.14935302734375, + "margin_dpo/margin_std": 845.0499877929688, + "step": 641 + }, + { + "KL/chosen_KL_mean": -837.578125, + "KL/mean": -1093.5718994140625, + "KL/rejected_KL_mean": -1349.565673828125, + "KL/std": 812.8326416015625, + "epoch": 0.9427312775330396, + "fcm_dpo/beta": 0.0008224358316510916, + "fcm_dpo/delta": -0.02903025597333908, + "fcm_dpo/margin": 511.987548828125, + "fcm_dpo/q_t": 0.412765234708786, + "grad_norm": 36.307071685791016, + "learning_rate": 5.251706922648868e-09, + "logits/chosen": -1.1126117706298828, + "logits/rejected": -1.1542222499847412, + "logps/chosen": -903.267333984375, + "logps/ref_chosen": -65.68924713134766, + "logps/ref_rejected": -110.24205017089844, + "logps/rejected": -1459.8076171875, + "loss": 1.152, + "margin_dpo/margin_mean": 511.987548828125, + "margin_dpo/margin_std": 1003.8565673828125, + "step": 642 + }, + { + "KL/chosen_KL_mean": -778.371826171875, + "KL/mean": -960.838623046875, + "KL/rejected_KL_mean": -1143.305419921875, + "KL/std": 605.3773193359375, + "epoch": 0.9441997063142438, + "fcm_dpo/beta": 0.0007829120149835944, + "fcm_dpo/delta": -0.014414433389902115, + "fcm_dpo/margin": 364.93359375, + "fcm_dpo/q_t": 0.43285301327705383, + "grad_norm": 33.376853942871094, + "learning_rate": 4.993270631642038e-09, + "logits/chosen": -1.2616610527038574, + "logits/rejected": -1.2592954635620117, + "logps/chosen": -830.32177734375, + "logps/ref_chosen": -51.94999694824219, + "logps/ref_rejected": -87.46833801269531, + "logps/rejected": -1230.7738037109375, + "loss": 1.1815, + "margin_dpo/margin_mean": 364.93359375, + "margin_dpo/margin_std": 620.4666748046875, + "step": 643 + }, + { + "KL/chosen_KL_mean": -716.4739990234375, + "KL/mean": -920.228515625, + "KL/rejected_KL_mean": -1123.983154296875, + "KL/std": 688.679931640625, + "epoch": 0.9456681350954479, + "fcm_dpo/beta": 0.0008282763883471489, + "fcm_dpo/delta": 0.07501886785030365, + "fcm_dpo/margin": 407.50909423828125, + "fcm_dpo/q_t": 0.4266040325164795, + "grad_norm": 32.15571594238281, + "learning_rate": 4.741290495811873e-09, + "logits/chosen": -1.1580393314361572, + "logits/rejected": -1.1756101846694946, + "logps/chosen": -775.49169921875, + "logps/ref_chosen": -59.017662048339844, + "logps/ref_rejected": -87.13668823242188, + "logps/rejected": -1211.1197509765625, + "loss": 1.1794, + "margin_dpo/margin_mean": 407.50909423828125, + "margin_dpo/margin_std": 801.0850830078125, + "step": 644 + }, + { + "KL/chosen_KL_mean": -772.6549072265625, + "KL/mean": -870.29248046875, + "KL/rejected_KL_mean": -967.9300537109375, + "KL/std": 544.7769165039062, + "epoch": 0.947136563876652, + "fcm_dpo/beta": 0.0009023309103213251, + "fcm_dpo/delta": 0.06694209575653076, + "fcm_dpo/margin": 195.2750701904297, + "fcm_dpo/q_t": 0.4632098078727722, + "grad_norm": 85.04663848876953, + "learning_rate": 4.495773155069299e-09, + "logits/chosen": -1.126030683517456, + "logits/rejected": -1.115586280822754, + "logps/chosen": -828.5309448242188, + "logps/ref_chosen": -55.87602233886719, + "logps/ref_rejected": -97.78080749511719, + "logps/rejected": -1065.7108154296875, + "loss": 1.3318, + "margin_dpo/margin_mean": 195.2751007080078, + "margin_dpo/margin_std": 740.9451904296875, + "step": 645 + }, + { + "KL/chosen_KL_mean": -735.0755615234375, + "KL/mean": -908.2972412109375, + "KL/rejected_KL_mean": -1081.5189208984375, + "KL/std": 522.7762451171875, + "epoch": 0.9486049926578561, + "fcm_dpo/beta": 0.0009436524705961347, + "fcm_dpo/delta": 0.08735965192317963, + "fcm_dpo/margin": 346.443359375, + "fcm_dpo/q_t": 0.4290798604488373, + "grad_norm": 62.62091064453125, + "learning_rate": 4.256725079024553e-09, + "logits/chosen": -1.1851344108581543, + "logits/rejected": -1.1642956733703613, + "logps/chosen": -796.351318359375, + "logps/ref_chosen": -61.275787353515625, + "logps/ref_rejected": -77.50580596923828, + "logps/rejected": -1159.024658203125, + "loss": 1.1762, + "margin_dpo/margin_mean": 346.4433288574219, + "margin_dpo/margin_std": 640.25537109375, + "step": 646 + }, + { + "KL/chosen_KL_mean": -630.638671875, + "KL/mean": -835.8479614257812, + "KL/rejected_KL_mean": -1041.0572509765625, + "KL/std": 575.6964111328125, + "epoch": 0.9500734214390602, + "fcm_dpo/beta": 0.0009774916106835008, + "fcm_dpo/delta": -0.0016585299745202065, + "fcm_dpo/margin": 410.41851806640625, + "fcm_dpo/q_t": 0.40817171335220337, + "grad_norm": 27.86429786682129, + "learning_rate": 4.024152566816791e-09, + "logits/chosen": -1.0603611469268799, + "logits/rejected": -1.0982961654663086, + "logps/chosen": -685.4910888671875, + "logps/ref_chosen": -54.8524169921875, + "logps/ref_rejected": -93.5194091796875, + "logps/rejected": -1134.57666015625, + "loss": 1.1114, + "margin_dpo/margin_mean": 410.41851806640625, + "margin_dpo/margin_std": 619.114990234375, + "step": 647 + }, + { + "KL/chosen_KL_mean": -673.4736328125, + "KL/mean": -996.9141845703125, + "KL/rejected_KL_mean": -1320.354736328125, + "KL/std": 748.5928955078125, + "epoch": 0.9515418502202643, + "fcm_dpo/beta": 0.0008425582782365382, + "fcm_dpo/delta": -0.20228338241577148, + "fcm_dpo/margin": 646.8811645507812, + "fcm_dpo/q_t": 0.3778739869594574, + "grad_norm": 32.02235412597656, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": -1.1987550258636475, + "logits/rejected": -1.2712297439575195, + "logps/chosen": -727.6451416015625, + "logps/ref_chosen": -54.17146682739258, + "logps/ref_rejected": -98.7127914428711, + "logps/rejected": -1419.067626953125, + "loss": 1.0347, + "margin_dpo/margin_mean": 646.8811645507812, + "margin_dpo/margin_std": 918.9876708984375, + "step": 648 + }, + { + "KL/chosen_KL_mean": -728.5607299804688, + "KL/mean": -868.1865234375, + "KL/rejected_KL_mean": -1007.812255859375, + "KL/std": 557.9706420898438, + "epoch": 0.9530102790014684, + "fcm_dpo/beta": 0.0008311655838042498, + "fcm_dpo/delta": 0.06771388649940491, + "fcm_dpo/margin": 279.2515869140625, + "fcm_dpo/q_t": 0.4502527713775635, + "grad_norm": 32.22618865966797, + "learning_rate": 3.5784585771215235e-09, + "logits/chosen": -1.224090814590454, + "logits/rejected": -1.2157777547836304, + "logps/chosen": -791.0410766601562, + "logps/ref_chosen": -62.480350494384766, + "logps/ref_rejected": -80.07717895507812, + "logps/rejected": -1087.8895263671875, + "loss": 1.2478, + "margin_dpo/margin_mean": 279.2515869140625, + "margin_dpo/margin_std": 673.363525390625, + "step": 649 + }, + { + "KL/chosen_KL_mean": -760.45458984375, + "KL/mean": -985.4943237304688, + "KL/rejected_KL_mean": -1210.533935546875, + "KL/std": 681.9244995117188, + "epoch": 0.9544787077826725, + "fcm_dpo/beta": 0.0008620332810096443, + "fcm_dpo/delta": 0.014763827435672283, + "fcm_dpo/margin": 450.07940673828125, + "fcm_dpo/q_t": 0.4147789478302002, + "grad_norm": 35.338233947753906, + "learning_rate": 3.3653488440851253e-09, + "logits/chosen": -1.1790560483932495, + "logits/rejected": -1.2069194316864014, + "logps/chosen": -816.5474243164062, + "logps/ref_chosen": -56.09281921386719, + "logps/ref_rejected": -98.26483917236328, + "logps/rejected": -1308.798828125, + "loss": 1.1394, + "margin_dpo/margin_mean": 450.0793762207031, + "margin_dpo/margin_std": 782.4627685546875, + "step": 650 + }, + { + "KL/chosen_KL_mean": -495.2056884765625, + "KL/mean": -796.1655883789062, + "KL/rejected_KL_mean": -1097.12548828125, + "KL/std": 634.3084716796875, + "epoch": 0.9559471365638766, + "fcm_dpo/beta": 0.0007995986961759627, + "fcm_dpo/delta": -0.10846766829490662, + "fcm_dpo/margin": 601.9198608398438, + "fcm_dpo/q_t": 0.3852821886539459, + "grad_norm": 45.97235107421875, + "learning_rate": 3.158738163478475e-09, + "logits/chosen": -1.143014669418335, + "logits/rejected": -1.2076648473739624, + "logps/chosen": -538.631103515625, + "logps/ref_chosen": -43.42544937133789, + "logps/ref_rejected": -99.95791625976562, + "logps/rejected": -1197.08349609375, + "loss": 1.023, + "margin_dpo/margin_mean": 601.9197998046875, + "margin_dpo/margin_std": 651.2605590820312, + "step": 651 + }, + { + "KL/chosen_KL_mean": -655.0028076171875, + "KL/mean": -886.6578979492188, + "KL/rejected_KL_mean": -1118.31298828125, + "KL/std": 625.3679809570312, + "epoch": 0.9574155653450808, + "fcm_dpo/beta": 0.0008092697244137526, + "fcm_dpo/delta": 0.029820626601576805, + "fcm_dpo/margin": 463.31011962890625, + "fcm_dpo/q_t": 0.41610437631607056, + "grad_norm": 35.829437255859375, + "learning_rate": 2.9586319796851555e-09, + "logits/chosen": -1.1463913917541504, + "logits/rejected": -1.1780339479446411, + "logps/chosen": -717.5796508789062, + "logps/ref_chosen": -62.57680892944336, + "logps/ref_rejected": -111.76779174804688, + "logps/rejected": -1230.080810546875, + "loss": 1.1258, + "margin_dpo/margin_mean": 463.31011962890625, + "margin_dpo/margin_std": 727.44580078125, + "step": 652 + }, + { + "KL/chosen_KL_mean": -785.6455078125, + "KL/mean": -1017.7234497070312, + "KL/rejected_KL_mean": -1249.8013916015625, + "KL/std": 702.5745849609375, + "epoch": 0.9588839941262849, + "fcm_dpo/beta": 0.0008226313511840999, + "fcm_dpo/delta": 0.022288989275693893, + "fcm_dpo/margin": 464.1558837890625, + "fcm_dpo/q_t": 0.41595858335494995, + "grad_norm": 36.48124313354492, + "learning_rate": 2.7650355656892166e-09, + "logits/chosen": -1.2419447898864746, + "logits/rejected": -1.2747020721435547, + "logps/chosen": -846.7584838867188, + "logps/ref_chosen": -61.11295700073242, + "logps/ref_rejected": -103.24960327148438, + "logps/rejected": -1353.051025390625, + "loss": 1.123, + "margin_dpo/margin_mean": 464.1558837890625, + "margin_dpo/margin_std": 734.2294311523438, + "step": 653 + }, + { + "KL/chosen_KL_mean": -720.1655883789062, + "KL/mean": -930.6776733398438, + "KL/rejected_KL_mean": -1141.189697265625, + "KL/std": 579.1441650390625, + "epoch": 0.960352422907489, + "fcm_dpo/beta": 0.0008555342210456729, + "fcm_dpo/delta": 0.04842513054609299, + "fcm_dpo/margin": 421.0242004394531, + "fcm_dpo/q_t": 0.4207690358161926, + "grad_norm": 29.795970916748047, + "learning_rate": 2.577954022936174e-09, + "logits/chosen": -1.1985931396484375, + "logits/rejected": -1.2237248420715332, + "logps/chosen": -781.8937377929688, + "logps/ref_chosen": -61.7281379699707, + "logps/ref_rejected": -98.7738037109375, + "logps/rejected": -1239.963623046875, + "loss": 1.1322, + "margin_dpo/margin_mean": 421.0242004394531, + "margin_dpo/margin_std": 659.9177856445312, + "step": 654 + }, + { + "KL/chosen_KL_mean": -680.0551147460938, + "KL/mean": -888.6227416992188, + "KL/rejected_KL_mean": -1097.1903076171875, + "KL/std": 573.6949462890625, + "epoch": 0.9618208516886931, + "fcm_dpo/beta": 0.000887269270606339, + "fcm_dpo/delta": 0.036719754338264465, + "fcm_dpo/margin": 417.13525390625, + "fcm_dpo/q_t": 0.4181985855102539, + "grad_norm": 32.07929229736328, + "learning_rate": 2.397392281198729e-09, + "logits/chosen": -1.17026948928833, + "logits/rejected": -1.2217252254486084, + "logps/chosen": -729.6319580078125, + "logps/ref_chosen": -49.576812744140625, + "logps/ref_rejected": -98.29183197021484, + "logps/rejected": -1195.482177734375, + "loss": 1.1344, + "margin_dpo/margin_mean": 417.1352233886719, + "margin_dpo/margin_std": 680.7271728515625, + "step": 655 + }, + { + "KL/chosen_KL_mean": -690.5496826171875, + "KL/mean": -1050.8857421875, + "KL/rejected_KL_mean": -1411.2216796875, + "KL/std": 771.2440185546875, + "epoch": 0.9632892804698973, + "fcm_dpo/beta": 0.0007734264945611358, + "fcm_dpo/delta": -0.2174438238143921, + "fcm_dpo/margin": 720.6719970703125, + "fcm_dpo/q_t": 0.3665843605995178, + "grad_norm": 77.32756805419922, + "learning_rate": 2.223355098446622e-09, + "logits/chosen": -1.065166711807251, + "logits/rejected": -1.151442289352417, + "logps/chosen": -743.09912109375, + "logps/ref_chosen": -52.54943084716797, + "logps/ref_rejected": -113.67464447021484, + "logps/rejected": -1524.896484375, + "loss": 0.9841, + "margin_dpo/margin_mean": 720.6719970703125, + "margin_dpo/margin_std": 759.9878540039062, + "step": 656 + }, + { + "KL/chosen_KL_mean": -651.868896484375, + "KL/mean": -958.4510498046875, + "KL/rejected_KL_mean": -1265.033203125, + "KL/std": 730.9705810546875, + "epoch": 0.9647577092511013, + "fcm_dpo/beta": 0.0006971274269744754, + "fcm_dpo/delta": -0.03567207232117653, + "fcm_dpo/margin": 613.1644287109375, + "fcm_dpo/q_t": 0.4013122618198395, + "grad_norm": 41.946922302246094, + "learning_rate": 2.055847060721566e-09, + "logits/chosen": -1.2377593517303467, + "logits/rejected": -1.309966802597046, + "logps/chosen": -698.5694580078125, + "logps/ref_chosen": -46.700538635253906, + "logps/ref_rejected": -97.91487121582031, + "logps/rejected": -1362.947998046875, + "loss": 1.0675, + "margin_dpo/margin_mean": 613.1643676757812, + "margin_dpo/margin_std": 760.7774658203125, + "step": 657 + }, + { + "KL/chosen_KL_mean": -706.0740966796875, + "KL/mean": -946.1854248046875, + "KL/rejected_KL_mean": -1186.2967529296875, + "KL/std": 566.0189208984375, + "epoch": 0.9662261380323054, + "fcm_dpo/beta": 0.0007183150737546384, + "fcm_dpo/delta": 0.06623679399490356, + "fcm_dpo/margin": 480.22265625, + "fcm_dpo/q_t": 0.42133376002311707, + "grad_norm": 38.043453216552734, + "learning_rate": 1.8948725820160662e-09, + "logits/chosen": -1.1505177021026611, + "logits/rejected": -1.187692642211914, + "logps/chosen": -767.0323486328125, + "logps/ref_chosen": -60.95820999145508, + "logps/ref_rejected": -95.93949127197266, + "logps/rejected": -1282.2362060546875, + "loss": 1.1244, + "margin_dpo/margin_mean": 480.2226257324219, + "margin_dpo/margin_std": 638.424072265625, + "step": 658 + }, + { + "KL/chosen_KL_mean": -624.73291015625, + "KL/mean": -855.0606689453125, + "KL/rejected_KL_mean": -1085.388427734375, + "KL/std": 586.4114990234375, + "epoch": 0.9676945668135095, + "fcm_dpo/beta": 0.0007668976904824376, + "fcm_dpo/delta": 0.0567876435816288, + "fcm_dpo/margin": 460.6553649902344, + "fcm_dpo/q_t": 0.4216195344924927, + "grad_norm": 27.48058319091797, + "learning_rate": 1.7404359041573723e-09, + "logits/chosen": -1.0954115390777588, + "logits/rejected": -1.0741057395935059, + "logps/chosen": -701.4759521484375, + "logps/ref_chosen": -76.74298095703125, + "logps/ref_rejected": -87.4709701538086, + "logps/rejected": -1172.859375, + "loss": 1.1233, + "margin_dpo/margin_mean": 460.6553955078125, + "margin_dpo/margin_std": 662.3754272460938, + "step": 659 + }, + { + "KL/chosen_KL_mean": -714.209228515625, + "KL/mean": -1003.320068359375, + "KL/rejected_KL_mean": -1292.430908203125, + "KL/std": 690.353515625, + "epoch": 0.9691629955947136, + "fcm_dpo/beta": 0.0007632029009982944, + "fcm_dpo/delta": -0.052839018404483795, + "fcm_dpo/margin": 578.2216796875, + "fcm_dpo/q_t": 0.3979361057281494, + "grad_norm": 33.15445327758789, + "learning_rate": 1.592541096695571e-09, + "logits/chosen": -1.1898115873336792, + "logits/rejected": -1.206876277923584, + "logps/chosen": -773.257080078125, + "logps/ref_chosen": -59.04788589477539, + "logps/ref_rejected": -75.96005249023438, + "logps/rejected": -1368.3909912109375, + "loss": 1.0682, + "margin_dpo/margin_mean": 578.2216796875, + "margin_dpo/margin_std": 753.8338623046875, + "step": 660 + }, + { + "KL/chosen_KL_mean": -610.59130859375, + "KL/mean": -878.7642211914062, + "KL/rejected_KL_mean": -1146.9371337890625, + "KL/std": 757.8616943359375, + "epoch": 0.9706314243759178, + "fcm_dpo/beta": 0.0007565034320577979, + "fcm_dpo/delta": -0.0085027776658535, + "fcm_dpo/margin": 536.3458251953125, + "fcm_dpo/q_t": 0.40905699133872986, + "grad_norm": 47.93645477294922, + "learning_rate": 1.4511920567963908e-09, + "logits/chosen": -1.2252672910690308, + "logits/rejected": -1.252467393875122, + "logps/chosen": -661.2652587890625, + "logps/ref_chosen": -50.673973083496094, + "logps/ref_rejected": -86.00569152832031, + "logps/rejected": -1232.94287109375, + "loss": 1.0877, + "margin_dpo/margin_mean": 536.3458251953125, + "margin_dpo/margin_std": 738.3463134765625, + "step": 661 + }, + { + "KL/chosen_KL_mean": -732.677490234375, + "KL/mean": -931.5089721679688, + "KL/rejected_KL_mean": -1130.3404541015625, + "KL/std": 597.563232421875, + "epoch": 0.9720998531571219, + "fcm_dpo/beta": 0.0007978413486853242, + "fcm_dpo/delta": 0.09812071919441223, + "fcm_dpo/margin": 397.6629333496094, + "fcm_dpo/q_t": 0.4306020140647888, + "grad_norm": 29.834253311157227, + "learning_rate": 1.3163925091384532e-09, + "logits/chosen": -1.1183321475982666, + "logits/rejected": -1.1220099925994873, + "logps/chosen": -801.9385986328125, + "logps/ref_chosen": -69.26106262207031, + "logps/ref_rejected": -89.05593872070312, + "logps/rejected": -1219.3963623046875, + "loss": 1.178, + "margin_dpo/margin_mean": 397.66290283203125, + "margin_dpo/margin_std": 750.21630859375, + "step": 662 + }, + { + "KL/chosen_KL_mean": -667.935546875, + "KL/mean": -934.6746826171875, + "KL/rejected_KL_mean": -1201.4139404296875, + "KL/std": 711.580810546875, + "epoch": 0.973568281938326, + "fcm_dpo/beta": 0.0007925699464976788, + "fcm_dpo/delta": -0.02965104579925537, + "fcm_dpo/margin": 533.4783935546875, + "fcm_dpo/q_t": 0.40630367398262024, + "grad_norm": 32.280399322509766, + "learning_rate": 1.1881460058152382e-09, + "logits/chosen": -1.1441211700439453, + "logits/rejected": -1.1800665855407715, + "logps/chosen": -732.814453125, + "logps/ref_chosen": -64.87890625, + "logps/ref_rejected": -113.92536926269531, + "logps/rejected": -1315.33935546875, + "loss": 1.1028, + "margin_dpo/margin_mean": 533.4784545898438, + "margin_dpo/margin_std": 843.329345703125, + "step": 663 + }, + { + "KL/chosen_KL_mean": -713.7249145507812, + "KL/mean": -985.4025268554688, + "KL/rejected_KL_mean": -1257.0802001953125, + "KL/std": 668.9918823242188, + "epoch": 0.9750367107195301, + "fcm_dpo/beta": 0.0007669959450140595, + "fcm_dpo/delta": -0.022985609248280525, + "fcm_dpo/margin": 543.3553466796875, + "fcm_dpo/q_t": 0.40422388911247253, + "grad_norm": 36.076908111572266, + "learning_rate": 1.066455926241383e-09, + "logits/chosen": -1.1553318500518799, + "logits/rejected": -1.198824167251587, + "logps/chosen": -774.6134033203125, + "logps/ref_chosen": -60.88847351074219, + "logps/ref_rejected": -105.521728515625, + "logps/rejected": -1362.6019287109375, + "loss": 1.0837, + "margin_dpo/margin_mean": 543.3553466796875, + "margin_dpo/margin_std": 723.924072265625, + "step": 664 + }, + { + "KL/chosen_KL_mean": -651.1171875, + "KL/mean": -885.782470703125, + "KL/rejected_KL_mean": -1120.4478759765625, + "KL/std": 580.0274658203125, + "epoch": 0.9765051395007343, + "fcm_dpo/beta": 0.0007947899866849184, + "fcm_dpo/delta": 0.033155716955661774, + "fcm_dpo/margin": 469.3306579589844, + "fcm_dpo/q_t": 0.4146922826766968, + "grad_norm": 52.002296447753906, + "learning_rate": 9.513254770636137e-10, + "logits/chosen": -1.2848122119903564, + "logits/rejected": -1.3209171295166016, + "logps/chosen": -711.6812744140625, + "logps/ref_chosen": -60.56413269042969, + "logps/ref_rejected": -84.80882263183594, + "logps/rejected": -1205.256591796875, + "loss": 1.1028, + "margin_dpo/margin_mean": 469.33062744140625, + "margin_dpo/margin_std": 604.48974609375, + "step": 665 + }, + { + "KL/chosen_KL_mean": -704.703125, + "KL/mean": -937.7362060546875, + "KL/rejected_KL_mean": -1170.769287109375, + "KL/std": 603.8134765625, + "epoch": 0.9779735682819384, + "fcm_dpo/beta": 0.0008114855154417455, + "fcm_dpo/delta": 0.026824505999684334, + "fcm_dpo/margin": 466.06622314453125, + "fcm_dpo/q_t": 0.4145530164241791, + "grad_norm": 36.94413375854492, + "learning_rate": 8.427576920763956e-10, + "logits/chosen": -1.0528377294540405, + "logits/rejected": -1.0693092346191406, + "logps/chosen": -769.123046875, + "logps/ref_chosen": -64.41996002197266, + "logps/ref_rejected": -95.8916244506836, + "logps/rejected": -1266.660888671875, + "loss": 1.1138, + "margin_dpo/margin_mean": 466.06622314453125, + "margin_dpo/margin_std": 662.6096801757812, + "step": 666 + }, + { + "KL/chosen_KL_mean": -802.0358276367188, + "KL/mean": -1084.480712890625, + "KL/rejected_KL_mean": -1366.925537109375, + "KL/std": 679.0506591796875, + "epoch": 0.9794419970631424, + "fcm_dpo/beta": 0.0007717009866610169, + "fcm_dpo/delta": -0.05106900632381439, + "fcm_dpo/margin": 564.8895874023438, + "fcm_dpo/q_t": 0.39843225479125977, + "grad_norm": 29.158369064331055, + "learning_rate": 7.407554321417764e-10, + "logits/chosen": -1.1201040744781494, + "logits/rejected": -1.1252251863479614, + "logps/chosen": -871.3128662109375, + "logps/ref_chosen": -69.27702331542969, + "logps/ref_rejected": -87.83549499511719, + "logps/rejected": -1454.760986328125, + "loss": 1.0729, + "margin_dpo/margin_mean": 564.8895874023438, + "margin_dpo/margin_std": 735.8818359375, + "step": 667 + }, + { + "KL/chosen_KL_mean": -843.2930297851562, + "KL/mean": -1036.6741943359375, + "KL/rejected_KL_mean": -1230.05517578125, + "KL/std": 693.7163696289062, + "epoch": 0.9809104258443465, + "fcm_dpo/beta": 0.0008507584570907056, + "fcm_dpo/delta": 0.08011265844106674, + "fcm_dpo/margin": 386.7622375488281, + "fcm_dpo/q_t": 0.43072831630706787, + "grad_norm": 44.3238410949707, + "learning_rate": 6.453213851142225e-10, + "logits/chosen": -1.2108023166656494, + "logits/rejected": -1.2172951698303223, + "logps/chosen": -915.8970336914062, + "logps/ref_chosen": -72.60400390625, + "logps/ref_rejected": -103.73905944824219, + "logps/rejected": -1333.7943115234375, + "loss": 1.1988, + "margin_dpo/margin_mean": 386.7622375488281, + "margin_dpo/margin_std": 833.273681640625, + "step": 668 + }, + { + "KL/chosen_KL_mean": -618.487060546875, + "KL/mean": -896.7724609375, + "KL/rejected_KL_mean": -1175.057861328125, + "KL/std": 641.3170166015625, + "epoch": 0.9823788546255506, + "fcm_dpo/beta": 0.0008110044291242957, + "fcm_dpo/delta": -0.06620398163795471, + "fcm_dpo/margin": 556.57080078125, + "fcm_dpo/q_t": 0.39469441771507263, + "grad_norm": 24.252483367919922, + "learning_rate": 5.564580657695939e-10, + "logits/chosen": -1.144885778427124, + "logits/rejected": -1.1587302684783936, + "logps/chosen": -664.603515625, + "logps/ref_chosen": -46.116416931152344, + "logps/ref_rejected": -77.92434692382812, + "logps/rejected": -1252.9822998046875, + "loss": 1.0677, + "margin_dpo/margin_mean": 556.57080078125, + "margin_dpo/margin_std": 739.805419921875, + "step": 669 + }, + { + "KL/chosen_KL_mean": -602.510009765625, + "KL/mean": -883.6866455078125, + "KL/rejected_KL_mean": -1164.86328125, + "KL/std": 612.798583984375, + "epoch": 0.9838472834067548, + "fcm_dpo/beta": 0.0007813957636244595, + "fcm_dpo/delta": -0.051431987434625626, + "fcm_dpo/margin": 562.3533935546875, + "fcm_dpo/q_t": 0.3977850079536438, + "grad_norm": 31.681093215942383, + "learning_rate": 4.741678157389739e-10, + "logits/chosen": -1.0778193473815918, + "logits/rejected": -1.1026027202606201, + "logps/chosen": -664.8557739257812, + "logps/ref_chosen": -62.34575271606445, + "logps/ref_rejected": -96.9405517578125, + "logps/rejected": -1261.803955078125, + "loss": 1.0741, + "margin_dpo/margin_mean": 562.3533935546875, + "margin_dpo/margin_std": 736.040771484375, + "step": 670 + }, + { + "KL/chosen_KL_mean": -771.7396240234375, + "KL/mean": -993.390380859375, + "KL/rejected_KL_mean": -1215.041015625, + "KL/std": 630.7474365234375, + "epoch": 0.9853157121879589, + "fcm_dpo/beta": 0.0008008808945305645, + "fcm_dpo/delta": 0.051010433584451675, + "fcm_dpo/margin": 443.301513671875, + "fcm_dpo/q_t": 0.4195772409439087, + "grad_norm": 30.4383602142334, + "learning_rate": 3.9845280344705245e-10, + "logits/chosen": -1.2151209115982056, + "logits/rejected": -1.2516989707946777, + "logps/chosen": -819.73974609375, + "logps/ref_chosen": -48.00010681152344, + "logps/ref_rejected": -83.81932067871094, + "logps/rejected": -1298.8603515625, + "loss": 1.1545, + "margin_dpo/margin_mean": 443.301513671875, + "margin_dpo/margin_std": 778.079833984375, + "step": 671 + }, + { + "KL/chosen_KL_mean": -849.791015625, + "KL/mean": -1072.2750244140625, + "KL/rejected_KL_mean": -1294.759033203125, + "KL/std": 755.86083984375, + "epoch": 0.986784140969163, + "fcm_dpo/beta": 0.0008099371916614473, + "fcm_dpo/delta": 0.04807715862989426, + "fcm_dpo/margin": 444.9681396484375, + "fcm_dpo/q_t": 0.42253392934799194, + "grad_norm": 44.19379425048828, + "learning_rate": 3.293150240547549e-10, + "logits/chosen": -1.274925947189331, + "logits/rejected": -1.2912979125976562, + "logps/chosen": -908.374267578125, + "logps/ref_chosen": -58.58328628540039, + "logps/ref_rejected": -93.14015197753906, + "logps/rejected": -1387.899169921875, + "loss": 1.1696, + "margin_dpo/margin_mean": 444.9681091308594, + "margin_dpo/margin_std": 849.6900634765625, + "step": 672 + }, + { + "KL/chosen_KL_mean": -751.1322631835938, + "KL/mean": -956.3372802734375, + "KL/rejected_KL_mean": -1161.542236328125, + "KL/std": 581.4315795898438, + "epoch": 0.9882525697503671, + "fcm_dpo/beta": 0.0008629739750176668, + "fcm_dpo/delta": 0.05584951490163803, + "fcm_dpo/margin": 410.40997314453125, + "fcm_dpo/q_t": 0.4219851493835449, + "grad_norm": 32.456729888916016, + "learning_rate": 2.6675629940689504e-10, + "logits/chosen": -1.208233118057251, + "logits/rejected": -1.214905023574829, + "logps/chosen": -797.85546875, + "logps/ref_chosen": -46.72320556640625, + "logps/ref_rejected": -85.29623413085938, + "logps/rejected": -1246.83837890625, + "loss": 1.1363, + "margin_dpo/margin_mean": 410.40997314453125, + "margin_dpo/margin_std": 664.936279296875, + "step": 673 + }, + { + "KL/chosen_KL_mean": -606.1369018554688, + "KL/mean": -891.3228759765625, + "KL/rejected_KL_mean": -1176.5089111328125, + "KL/std": 615.0496826171875, + "epoch": 0.9897209985315712, + "fcm_dpo/beta": 0.0008298626053147018, + "fcm_dpo/delta": -0.09556920826435089, + "fcm_dpo/margin": 570.3719482421875, + "fcm_dpo/q_t": 0.39160943031311035, + "grad_norm": 39.19919204711914, + "learning_rate": 2.1077827798404725e-10, + "logits/chosen": -1.1228229999542236, + "logits/rejected": -1.1515860557556152, + "logps/chosen": -651.5823974609375, + "logps/ref_chosen": -45.445526123046875, + "logps/ref_rejected": -70.04593658447266, + "logps/rejected": -1246.5548095703125, + "loss": 1.0523, + "margin_dpo/margin_mean": 570.3719482421875, + "margin_dpo/margin_std": 732.308349609375, + "step": 674 + }, + { + "KL/chosen_KL_mean": -702.86669921875, + "KL/mean": -980.7730712890625, + "KL/rejected_KL_mean": -1258.679443359375, + "KL/std": 685.6355590820312, + "epoch": 0.9911894273127754, + "fcm_dpo/beta": 0.0007509513525292277, + "fcm_dpo/delta": -0.030970744788646698, + "fcm_dpo/margin": 555.8126220703125, + "fcm_dpo/q_t": 0.404343843460083, + "grad_norm": 24.052825927734375, + "learning_rate": 1.6138243485910863e-10, + "logits/chosen": -1.1530423164367676, + "logits/rejected": -1.1733019351959229, + "logps/chosen": -747.04296875, + "logps/ref_chosen": -44.17628479003906, + "logps/ref_rejected": -74.09197998046875, + "logps/rejected": -1332.7713623046875, + "loss": 1.0862, + "margin_dpo/margin_mean": 555.8126831054688, + "margin_dpo/margin_std": 725.0802001953125, + "step": 675 + }, + { + "KL/chosen_KL_mean": -762.6793823242188, + "KL/mean": -1022.1181030273438, + "KL/rejected_KL_mean": -1281.556884765625, + "KL/std": 637.0001220703125, + "epoch": 0.9926578560939795, + "fcm_dpo/beta": 0.0007771970704197884, + "fcm_dpo/delta": -0.0041960906237363815, + "fcm_dpo/margin": 518.8775024414062, + "fcm_dpo/q_t": 0.40673425793647766, + "grad_norm": 22.74205780029297, + "learning_rate": 1.1857007165852472e-10, + "logits/chosen": -1.0892168283462524, + "logits/rejected": -1.10811448097229, + "logps/chosen": -834.077880859375, + "logps/ref_chosen": -71.39852905273438, + "logps/ref_rejected": -88.3587646484375, + "logps/rejected": -1369.9156494140625, + "loss": 1.0849, + "margin_dpo/margin_mean": 518.8775024414062, + "margin_dpo/margin_std": 657.755859375, + "step": 676 + }, + { + "KL/chosen_KL_mean": -713.2413940429688, + "KL/mean": -988.1437377929688, + "KL/rejected_KL_mean": -1263.046142578125, + "KL/std": 681.3233032226562, + "epoch": 0.9941262848751835, + "fcm_dpo/beta": 0.0007561305537819862, + "fcm_dpo/delta": -0.020175673067569733, + "fcm_dpo/margin": 549.8046875, + "fcm_dpo/q_t": 0.40893790125846863, + "grad_norm": 24.729129791259766, + "learning_rate": 8.23423165278725e-11, + "logits/chosen": -1.1915206909179688, + "logits/rejected": -1.1915239095687866, + "logps/chosen": -769.768798828125, + "logps/ref_chosen": -56.527435302734375, + "logps/ref_rejected": -78.22654724121094, + "logps/rejected": -1341.2725830078125, + "loss": 1.0947, + "margin_dpo/margin_mean": 549.8046875, + "margin_dpo/margin_std": 822.973876953125, + "step": 677 + }, + { + "KL/chosen_KL_mean": -606.6762084960938, + "KL/mean": -909.5905151367188, + "KL/rejected_KL_mean": -1212.5048828125, + "KL/std": 725.3612060546875, + "epoch": 0.9955947136563876, + "fcm_dpo/beta": 0.000741387193556875, + "fcm_dpo/delta": -0.06460094451904297, + "fcm_dpo/margin": 605.8285522460938, + "fcm_dpo/q_t": 0.396203875541687, + "grad_norm": 33.879825592041016, + "learning_rate": 5.270012410216185e-11, + "logits/chosen": -1.1325280666351318, + "logits/rejected": -1.1821608543395996, + "logps/chosen": -652.8106689453125, + "logps/ref_chosen": -46.13447570800781, + "logps/ref_rejected": -80.60462951660156, + "logps/rejected": -1293.109375, + "loss": 1.0706, + "margin_dpo/margin_mean": 605.82861328125, + "margin_dpo/margin_std": 814.54296875, + "step": 678 + }, + { + "KL/chosen_KL_mean": -715.3004150390625, + "KL/mean": -929.665771484375, + "KL/rejected_KL_mean": -1144.03125, + "KL/std": 562.1583251953125, + "epoch": 0.9970631424375918, + "fcm_dpo/beta": 0.0007575766649097204, + "fcm_dpo/delta": 0.0897865891456604, + "fcm_dpo/margin": 428.7308349609375, + "fcm_dpo/q_t": 0.4284992218017578, + "grad_norm": 30.023422241210938, + "learning_rate": 2.9644275480772416e-11, + "logits/chosen": -1.1566033363342285, + "logits/rejected": -1.156817078590393, + "logps/chosen": -765.5953369140625, + "logps/ref_chosen": -50.294921875, + "logps/ref_rejected": -76.59813690185547, + "logps/rejected": -1220.62939453125, + "loss": 1.1495, + "margin_dpo/margin_mean": 428.7308349609375, + "margin_dpo/margin_std": 679.2525634765625, + "step": 679 + }, + { + "KL/chosen_KL_mean": -714.9935302734375, + "KL/mean": -986.1622314453125, + "KL/rejected_KL_mean": -1257.3310546875, + "KL/std": 710.556884765625, + "epoch": 0.9985315712187959, + "fcm_dpo/beta": 0.0007529498543590307, + "fcm_dpo/delta": -0.012800686061382294, + "fcm_dpo/margin": 542.3373413085938, + "fcm_dpo/q_t": 0.4060339331626892, + "grad_norm": 35.72113800048828, + "learning_rate": 1.31753782067201e-11, + "logits/chosen": -1.134705901145935, + "logits/rejected": -1.1759812831878662, + "logps/chosen": -791.9093017578125, + "logps/ref_chosen": -76.91569519042969, + "logps/ref_rejected": -112.384765625, + "logps/rejected": -1369.7158203125, + "loss": 1.1036, + "margin_dpo/margin_mean": 542.3373413085938, + "margin_dpo/margin_std": 799.07666015625, + "step": 680 + }, + { + "KL/chosen_KL_mean": -731.4549560546875, + "KL/mean": -950.778076171875, + "KL/rejected_KL_mean": -1170.10107421875, + "KL/std": 619.8388671875, + "epoch": 1.0, + "fcm_dpo/beta": 0.000749021302908659, + "fcm_dpo/delta": -0.04601984843611717, + "fcm_dpo/margin": 438.6461181640625, + "fcm_dpo/q_t": 0.4235824942588806, + "grad_norm": 26.493305206298828, + "learning_rate": 3.2938662507808745e-12, + "logits/chosen": -1.2273634672164917, + "logits/rejected": -1.2576552629470825, + "logps/chosen": -792.4122314453125, + "logps/ref_chosen": -60.957279205322266, + "logps/ref_rejected": -88.55797576904297, + "logps/rejected": -1258.6590576171875, + "loss": 1.1554, + "margin_dpo/margin_mean": 438.6461486816406, + "margin_dpo/margin_std": 686.915771484375, + "step": 681 + }, + { + "epoch": 1.0, + "step": 681, + "total_flos": 0.0, + "train_loss": 1.1259927408285604, + "train_runtime": 1736.5167, + "train_samples_per_second": 25.107, + "train_steps_per_second": 0.392 + } + ], + "logging_steps": 1, + "max_steps": 681, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}