From 44db78de8e613da7d6d342dafed2687fd19d8232 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 1 Jun 2026 07:43:18 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.48 Source: Original Platform --- .gitattributes | 36 + README.md | 62 + all_results.json | 9 + config.json | 29 + generation_config.json | 9 + model-00001-of-00007.safetensors | 3 + model-00002-of-00007.safetensors | 3 + model-00003-of-00007.safetensors | 3 + model-00004-of-00007.safetensors | 3 + model-00005-of-00007.safetensors | 3 + model-00006-of-00007.safetensors | 3 + model-00007-of-00007.safetensors | 3 + model.safetensors.index.json | 298 + special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2064 ++++ train_results.json | 9 + trainer_state.json | 15246 +++++++++++++++++++++++++++++ 18 files changed, 17809 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00007.safetensors create mode 100644 model-00002-of-00007.safetensors create mode 100644 model-00003-of-00007.safetensors create mode 100644 model-00004-of-00007.safetensors create mode 100644 model-00005-of-00007.safetensors create mode 100644 model-00006-of-00007.safetensors create mode 100644 model-00007-of-00007.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..3364d50 --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-harmless-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.48 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.48 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-harmless-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-harmless-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..69ae41b --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.999244142101285, + "total_flos": 0.0, + "train_loss": 1.1488945873710643, + "train_runtime": 1653.0477, + "train_samples": 42336, + "train_samples_per_second": 25.611, + "train_steps_per_second": 0.4 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..f6ddc67 --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ff6968383c7e13e105e7644b80d5af3c651f6df2a9a5b5617263cfdf49bdb69 +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..946a670 --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d6d66ea8af43117af3535a461c7ee56e5f11545f9e2ee7983cf8182c8f936f1 +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..a5e12eb --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b16d7b60145e1c9b956449c240c4c0b2afb7d6817551f3aa326ba2f766cb6101 +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..1a5c1d7 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:299531e0a325122c492d4ff06854c85387a08cd2e5ba25bb9593e91e9e957ec7 +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..4736e28 --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:714f7e4450581ebeaebe64a17c8db1a5f2bbf2951769668f33047d96dbc87724 +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..f6d171e --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:432a582310760879a9bab2af72f7fb2e8b3c296e73e1f6c3d04db675dc6d8076 +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..e96dee7 --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6e93f9d5d47dc808cc590f1a9bd459d129d3c5a43b539ffc9c206f08dde5d6d +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..69ae41b --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.999244142101285, + "total_flos": 0.0, + "train_loss": 1.1488945873710643, + "train_runtime": 1653.0477, + "train_samples": 42336, + "train_samples_per_second": 25.611, + "train_steps_per_second": 0.4 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..f307254 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15246 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.999244142101285, + "eval_steps": 200, + "global_step": 661, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.02867889404296875, + "KL/mean": 0.029354453086853027, + "KL/rejected_KL_mean": 0.030029296875, + "KL/std": 0.2071000635623932, + "epoch": 0.0015117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0013532638549804688, + "fcm_dpo/q_t": 0.5000336766242981, + "grad_norm": 28.21779441833496, + "learning_rate": 0.0, + "logits/chosen": 0.13337239623069763, + "logits/rejected": 0.12492949515581131, + "logps/chosen": -64.5841293334961, + "logps/ref_chosen": -64.61280822753906, + "logps/ref_rejected": -64.17195129394531, + "logps/rejected": -64.14192199707031, + "loss": 1.3866, + "margin_dpo/margin_mean": -0.0013527870178222656, + "margin_dpo/margin_std": 0.2561596930027008, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.00289154052734375, + "KL/mean": -0.021616414189338684, + "KL/rejected_KL_mean": -0.04033660888671875, + "KL/std": 0.19624735414981842, + "epoch": 0.0030234315948601664, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.037450045347213745, + "fcm_dpo/q_t": 0.4990639090538025, + "grad_norm": 27.82764434814453, + "learning_rate": 7.462686567164179e-09, + "logits/chosen": 0.09414851665496826, + "logits/rejected": 0.07363267242908478, + "logps/chosen": -56.101890563964844, + "logps/ref_chosen": -56.0989990234375, + "logps/ref_rejected": -66.59971618652344, + "logps/rejected": -66.64006042480469, + "loss": 1.3828, + "margin_dpo/margin_mean": 0.03744968771934509, + "margin_dpo/margin_std": 0.27811938524246216, + "step": 2 + }, + { + "KL/chosen_KL_mean": 0.017974853515625, + "KL/mean": 0.01571372151374817, + "KL/rejected_KL_mean": 0.013454437255859375, + "KL/std": 0.2552720606327057, + "epoch": 0.0045351473922902496, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.004527091979980469, + "fcm_dpo/q_t": 0.4998869001865387, + "grad_norm": 31.184932708740234, + "learning_rate": 1.4925373134328357e-08, + "logits/chosen": 0.09944942593574524, + "logits/rejected": 0.06144290789961815, + "logps/chosen": -65.43928527832031, + "logps/ref_chosen": -65.45726013183594, + "logps/ref_rejected": -90.82853698730469, + "logps/rejected": -90.81507873535156, + "loss": 1.3861, + "margin_dpo/margin_mean": 0.004526615142822266, + "margin_dpo/margin_std": 0.3228065073490143, + "step": 3 + }, + { + "KL/chosen_KL_mean": 0.021240234375, + "KL/mean": 0.024831712245941162, + "KL/rejected_KL_mean": 0.028415679931640625, + "KL/std": 0.23518507182598114, + "epoch": 0.006046863189720333, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.007175266742706299, + "fcm_dpo/q_t": 0.5001801252365112, + "grad_norm": 34.333335876464844, + "learning_rate": 2.2388059701492534e-08, + "logits/chosen": 0.10089954733848572, + "logits/rejected": 0.08478045463562012, + "logps/chosen": -76.83894348144531, + "logps/ref_chosen": -76.86018371582031, + "logps/ref_rejected": -79.91523742675781, + "logps/rejected": -79.88682556152344, + "loss": 1.3873, + "margin_dpo/margin_mean": -0.007175803184509277, + "margin_dpo/margin_std": 0.3401098847389221, + "step": 4 + }, + { + "KL/chosen_KL_mean": 0.015611648559570312, + "KL/mean": 0.014620736241340637, + "KL/rejected_KL_mean": 0.01363372802734375, + "KL/std": 0.22797781229019165, + "epoch": 0.007558578987150416, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0019778311252593994, + "fcm_dpo/q_t": 0.499950647354126, + "grad_norm": 29.629215240478516, + "learning_rate": 2.9850746268656714e-08, + "logits/chosen": 0.08228473365306854, + "logits/rejected": 0.04336842894554138, + "logps/chosen": -62.955726623535156, + "logps/ref_chosen": -62.97134017944336, + "logps/ref_rejected": -79.9192123413086, + "logps/rejected": -79.90557861328125, + "loss": 1.3863, + "margin_dpo/margin_mean": 0.001978069543838501, + "margin_dpo/margin_std": 0.30833548307418823, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.02263641357421875, + "KL/mean": 0.01097261905670166, + "KL/rejected_KL_mean": 0.04458427429199219, + "KL/std": 0.2150484025478363, + "epoch": 0.009070294784580499, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06721735000610352, + "fcm_dpo/q_t": 0.5016798377037048, + "grad_norm": 29.89048194885254, + "learning_rate": 3.731343283582089e-08, + "logits/chosen": 0.1416800618171692, + "logits/rejected": 0.10234207659959793, + "logps/chosen": -51.33000183105469, + "logps/ref_chosen": -51.30736541748047, + "logps/ref_rejected": -82.77239227294922, + "logps/rejected": -82.72781372070312, + "loss": 1.3932, + "margin_dpo/margin_mean": -0.06721735000610352, + "margin_dpo/margin_std": 0.28968173265457153, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.03011322021484375, + "KL/mean": 0.0288875550031662, + "KL/rejected_KL_mean": 0.027660369873046875, + "KL/std": 0.21546083688735962, + "epoch": 0.010582010582010581, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0024529993534088135, + "fcm_dpo/q_t": 0.4999385476112366, + "grad_norm": 27.15511131286621, + "learning_rate": 4.477611940298507e-08, + "logits/chosen": 0.01981586217880249, + "logits/rejected": -0.024220559746026993, + "logps/chosen": -51.429298400878906, + "logps/ref_chosen": -51.45941162109375, + "logps/ref_rejected": -66.3828125, + "logps/rejected": -66.35514831542969, + "loss": 1.3862, + "margin_dpo/margin_mean": 0.0024527907371520996, + "margin_dpo/margin_std": 0.26524245738983154, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.02264404296875, + "KL/mean": -0.0015153884887695312, + "KL/rejected_KL_mean": -0.025671005249023438, + "KL/std": 0.23731042444705963, + "epoch": 0.012093726379440665, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04831033945083618, + "fcm_dpo/q_t": 0.49879276752471924, + "grad_norm": 28.646804809570312, + "learning_rate": 5.223880597014925e-08, + "logits/chosen": 0.07039310038089752, + "logits/rejected": 0.04824473336338997, + "logps/chosen": -62.174903869628906, + "logps/ref_chosen": -62.197547912597656, + "logps/ref_rejected": -74.66180419921875, + "logps/rejected": -74.6874771118164, + "loss": 1.3818, + "margin_dpo/margin_mean": 0.04831072688102722, + "margin_dpo/margin_std": 0.3311997950077057, + "step": 8 + }, + { + "KL/chosen_KL_mean": -0.04302787780761719, + "KL/mean": -0.051038309931755066, + "KL/rejected_KL_mean": -0.059051513671875, + "KL/std": 0.2540060877799988, + "epoch": 0.013605442176870748, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.016026049852371216, + "fcm_dpo/q_t": 0.4995996654033661, + "grad_norm": 31.397911071777344, + "learning_rate": 5.970149253731343e-08, + "logits/chosen": 0.15405794978141785, + "logits/rejected": 0.09594576060771942, + "logps/chosen": -55.67274856567383, + "logps/ref_chosen": -55.629722595214844, + "logps/ref_rejected": -86.21221923828125, + "logps/rejected": -86.27127075195312, + "loss": 1.385, + "margin_dpo/margin_mean": 0.01602587103843689, + "margin_dpo/margin_std": 0.33587515354156494, + "step": 9 + }, + { + "KL/chosen_KL_mean": -0.009862899780273438, + "KL/mean": 0.005285277962684631, + "KL/rejected_KL_mean": 0.020427703857421875, + "KL/std": 0.22252368927001953, + "epoch": 0.015117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.030300527811050415, + "fcm_dpo/q_t": 0.5007571578025818, + "grad_norm": 29.809185028076172, + "learning_rate": 6.71641791044776e-08, + "logits/chosen": 0.13334016501903534, + "logits/rejected": 0.10230215638875961, + "logps/chosen": -62.700462341308594, + "logps/ref_chosen": -62.69060134887695, + "logps/ref_rejected": -90.610107421875, + "logps/rejected": -90.58968353271484, + "loss": 1.3896, + "margin_dpo/margin_mean": -0.03030034899711609, + "margin_dpo/margin_std": 0.34233033657073975, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.016397476196289062, + "KL/mean": 0.004511818289756775, + "KL/rejected_KL_mean": -0.00738525390625, + "KL/std": 0.20762629806995392, + "epoch": 0.016628873771730914, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.023783236742019653, + "fcm_dpo/q_t": 0.4994049668312073, + "grad_norm": 29.40945816040039, + "learning_rate": 7.462686567164178e-08, + "logits/chosen": 0.10654198378324509, + "logits/rejected": 0.09964478015899658, + "logps/chosen": -65.75071716308594, + "logps/ref_chosen": -65.76712036132812, + "logps/ref_rejected": -72.4764633178711, + "logps/rejected": -72.48384857177734, + "loss": 1.3842, + "margin_dpo/margin_mean": 0.023783445358276367, + "margin_dpo/margin_std": 0.3237120509147644, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.018245697021484375, + "KL/mean": -0.011372998356819153, + "KL/rejected_KL_mean": -0.0045013427734375, + "KL/std": 0.23007062077522278, + "epoch": 0.018140589569160998, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.01374778151512146, + "fcm_dpo/q_t": 0.5003437995910645, + "grad_norm": 28.194849014282227, + "learning_rate": 8.208955223880596e-08, + "logits/chosen": 0.029350608587265015, + "logits/rejected": 0.013005432672798634, + "logps/chosen": -60.72313690185547, + "logps/ref_chosen": -60.704891204833984, + "logps/ref_rejected": -69.41564178466797, + "logps/rejected": -69.4201431274414, + "loss": 1.388, + "margin_dpo/margin_mean": -0.013747721910476685, + "margin_dpo/margin_std": 0.33297550678253174, + "step": 12 + }, + { + "KL/chosen_KL_mean": -0.0060176849365234375, + "KL/mean": 0.006569623947143555, + "KL/rejected_KL_mean": 0.019161224365234375, + "KL/std": 0.20004111528396606, + "epoch": 0.019652305366591082, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02517753839492798, + "fcm_dpo/q_t": 0.5006293654441833, + "grad_norm": 29.047027587890625, + "learning_rate": 8.955223880597014e-08, + "logits/chosen": 0.10595913231372833, + "logits/rejected": 0.04388178139925003, + "logps/chosen": -49.91527557373047, + "logps/ref_chosen": -49.90925598144531, + "logps/ref_rejected": -92.37818145751953, + "logps/rejected": -92.35902404785156, + "loss": 1.389, + "margin_dpo/margin_mean": -0.025177836418151855, + "margin_dpo/margin_std": 0.27132922410964966, + "step": 13 + }, + { + "KL/chosen_KL_mean": 0.0030002593994140625, + "KL/mean": -0.004290089011192322, + "KL/rejected_KL_mean": -0.0115814208984375, + "KL/std": 0.1868174970149994, + "epoch": 0.021164021164021163, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.014579087495803833, + "fcm_dpo/q_t": 0.4996357560157776, + "grad_norm": 29.573348999023438, + "learning_rate": 9.701492537313432e-08, + "logits/chosen": 0.09998003393411636, + "logits/rejected": 0.08181308954954147, + "logps/chosen": -60.61579132080078, + "logps/ref_chosen": -60.61879348754883, + "logps/ref_rejected": -71.79306030273438, + "logps/rejected": -71.80464172363281, + "loss": 1.385, + "margin_dpo/margin_mean": 0.014579474925994873, + "margin_dpo/margin_std": 0.2635040581226349, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.02704620361328125, + "KL/mean": -0.035765036940574646, + "KL/rejected_KL_mean": -0.0444793701171875, + "KL/std": 0.24218779802322388, + "epoch": 0.022675736961451247, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.017432361841201782, + "fcm_dpo/q_t": 0.4995632767677307, + "grad_norm": 33.17366027832031, + "learning_rate": 1.044776119402985e-07, + "logits/chosen": 0.05620576813817024, + "logits/rejected": 0.013878353871405125, + "logps/chosen": -63.49658203125, + "logps/ref_chosen": -63.46953582763672, + "logps/ref_rejected": -88.88951110839844, + "logps/rejected": -88.93399047851562, + "loss": 1.3849, + "margin_dpo/margin_mean": 0.017432481050491333, + "margin_dpo/margin_std": 0.349947452545166, + "step": 15 + }, + { + "KL/chosen_KL_mean": -0.0045413970947265625, + "KL/mean": -0.01623951643705368, + "KL/rejected_KL_mean": -0.027942657470703125, + "KL/std": 0.2111794650554657, + "epoch": 0.02418745275888133, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.023399069905281067, + "fcm_dpo/q_t": 0.49941548705101013, + "grad_norm": 26.882421493530273, + "learning_rate": 1.1194029850746268e-07, + "logits/chosen": 0.12233875691890717, + "logits/rejected": 0.08475068211555481, + "logps/chosen": -46.536842346191406, + "logps/ref_chosen": -46.53229904174805, + "logps/ref_rejected": -74.27533721923828, + "logps/rejected": -74.30328369140625, + "loss": 1.3842, + "margin_dpo/margin_mean": 0.02339915931224823, + "margin_dpo/margin_std": 0.29788079857826233, + "step": 16 + }, + { + "KL/chosen_KL_mean": -0.008623123168945312, + "KL/mean": -0.014116168022155762, + "KL/rejected_KL_mean": -0.019603729248046875, + "KL/std": 0.23058179020881653, + "epoch": 0.025699168556311415, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.010984718799591064, + "fcm_dpo/q_t": 0.49972665309906006, + "grad_norm": 32.859825134277344, + "learning_rate": 1.1940298507462686e-07, + "logits/chosen": 0.059348031878471375, + "logits/rejected": 0.04041973873972893, + "logps/chosen": -64.08645629882812, + "logps/ref_chosen": -64.07783508300781, + "logps/ref_rejected": -86.40876770019531, + "logps/rejected": -86.42837524414062, + "loss": 1.3855, + "margin_dpo/margin_mean": 0.010984480381011963, + "margin_dpo/margin_std": 0.3624400794506073, + "step": 17 + }, + { + "KL/chosen_KL_mean": 0.042514801025390625, + "KL/mean": 0.020855039358139038, + "KL/rejected_KL_mean": -0.0008106231689453125, + "KL/std": 0.22396549582481384, + "epoch": 0.027210884353741496, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0433235764503479, + "fcm_dpo/q_t": 0.4989171028137207, + "grad_norm": 27.754545211791992, + "learning_rate": 1.2686567164179106e-07, + "logits/chosen": 0.08723866194486618, + "logits/rejected": 0.042168259620666504, + "logps/chosen": -44.83182144165039, + "logps/ref_chosen": -44.87433624267578, + "logps/ref_rejected": -70.97604370117188, + "logps/rejected": -70.97686004638672, + "loss": 1.3822, + "margin_dpo/margin_mean": 0.04332330822944641, + "margin_dpo/margin_std": 0.31900152564048767, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.01389312744140625, + "KL/mean": -0.016233228147029877, + "KL/rejected_KL_mean": -0.04636383056640625, + "KL/std": 0.2271110713481903, + "epoch": 0.02872260015117158, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06026040017604828, + "fcm_dpo/q_t": 0.4984937906265259, + "grad_norm": 30.69112777709961, + "learning_rate": 1.343283582089552e-07, + "logits/chosen": 0.058250539004802704, + "logits/rejected": 0.044945910573005676, + "logps/chosen": -68.14591979980469, + "logps/ref_chosen": -68.1598129272461, + "logps/ref_rejected": -81.17138671875, + "logps/rejected": -81.21775817871094, + "loss": 1.3805, + "margin_dpo/margin_mean": 0.06026081740856171, + "margin_dpo/margin_std": 0.2684270441532135, + "step": 19 + }, + { + "KL/chosen_KL_mean": 0.018886566162109375, + "KL/mean": 0.013330504298210144, + "KL/rejected_KL_mean": 0.007778167724609375, + "KL/std": 0.21853777766227722, + "epoch": 0.030234315948601664, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.011106997728347778, + "fcm_dpo/q_t": 0.49972179532051086, + "grad_norm": 29.30516815185547, + "learning_rate": 1.4179104477611938e-07, + "logits/chosen": 0.1435462087392807, + "logits/rejected": 0.12017589807510376, + "logps/chosen": -53.65967559814453, + "logps/ref_chosen": -53.67856216430664, + "logps/ref_rejected": -74.16911315917969, + "logps/rejected": -74.16133880615234, + "loss": 1.3854, + "margin_dpo/margin_mean": 0.011106759309768677, + "margin_dpo/margin_std": 0.2792432904243469, + "step": 20 + }, + { + "KL/chosen_KL_mean": 0.0058422088623046875, + "KL/mean": 0.0030414238572120667, + "KL/rejected_KL_mean": 0.000247955322265625, + "KL/std": 0.22705534100532532, + "epoch": 0.031746031746031744, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.005594924092292786, + "fcm_dpo/q_t": 0.4998602867126465, + "grad_norm": 29.399574279785156, + "learning_rate": 1.4925373134328355e-07, + "logits/chosen": 0.11058309674263, + "logits/rejected": 0.08474206924438477, + "logps/chosen": -64.69570922851562, + "logps/ref_chosen": -64.70155334472656, + "logps/ref_rejected": -81.02095031738281, + "logps/rejected": -81.02069854736328, + "loss": 1.386, + "margin_dpo/margin_mean": 0.005594193935394287, + "margin_dpo/margin_std": 0.3037152886390686, + "step": 21 + }, + { + "KL/chosen_KL_mean": -0.030012130737304688, + "KL/mean": -0.02553868293762207, + "KL/rejected_KL_mean": -0.02106475830078125, + "KL/std": 0.23182955384254456, + "epoch": 0.03325774754346183, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.008944094181060791, + "fcm_dpo/q_t": 0.5002233386039734, + "grad_norm": 29.012664794921875, + "learning_rate": 1.5671641791044775e-07, + "logits/chosen": -0.010908234864473343, + "logits/rejected": -0.03109363466501236, + "logps/chosen": -58.066001892089844, + "logps/ref_chosen": -58.03599166870117, + "logps/ref_rejected": -80.72721862792969, + "logps/rejected": -80.74828338623047, + "loss": 1.3875, + "margin_dpo/margin_mean": -0.008943915367126465, + "margin_dpo/margin_std": 0.3591557741165161, + "step": 22 + }, + { + "KL/chosen_KL_mean": -0.003086090087890625, + "KL/mean": -0.006322100758552551, + "KL/rejected_KL_mean": -0.009563446044921875, + "KL/std": 0.22721782326698303, + "epoch": 0.03476946334089191, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.006478816270828247, + "fcm_dpo/q_t": 0.4998379647731781, + "grad_norm": 33.13698959350586, + "learning_rate": 1.6417910447761193e-07, + "logits/chosen": 0.12168412655591965, + "logits/rejected": 0.09707251936197281, + "logps/chosen": -66.35917663574219, + "logps/ref_chosen": -66.35608673095703, + "logps/ref_rejected": -93.02769470214844, + "logps/rejected": -93.0372543334961, + "loss": 1.3859, + "margin_dpo/margin_mean": 0.0064778923988342285, + "margin_dpo/margin_std": 0.31190019845962524, + "step": 23 + }, + { + "KL/chosen_KL_mean": -0.03143882751464844, + "KL/mean": 0.0080418661236763, + "KL/rejected_KL_mean": 0.047527313232421875, + "KL/std": 0.231191486120224, + "epoch": 0.036281179138321996, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.07896275818347931, + "fcm_dpo/q_t": 0.5019730925559998, + "grad_norm": 26.388242721557617, + "learning_rate": 1.716417910447761e-07, + "logits/chosen": 0.15333206951618195, + "logits/rejected": 0.11985106766223907, + "logps/chosen": -54.49267578125, + "logps/ref_chosen": -54.461238861083984, + "logps/ref_rejected": -68.33817291259766, + "logps/rejected": -68.2906494140625, + "loss": 1.3945, + "margin_dpo/margin_mean": -0.0789632499217987, + "margin_dpo/margin_std": 0.3113078773021698, + "step": 24 + }, + { + "KL/chosen_KL_mean": -0.02866363525390625, + "KL/mean": -0.041737914085388184, + "KL/rejected_KL_mean": -0.0548095703125, + "KL/std": 0.24598127603530884, + "epoch": 0.03779289493575208, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.026146292686462402, + "fcm_dpo/q_t": 0.4993473291397095, + "grad_norm": 29.624902725219727, + "learning_rate": 1.7910447761194027e-07, + "logits/chosen": 0.1234542727470398, + "logits/rejected": 0.07062897086143494, + "logps/chosen": -60.032867431640625, + "logps/ref_chosen": -60.00420379638672, + "logps/ref_rejected": -90.47376251220703, + "logps/rejected": -90.52857208251953, + "loss": 1.384, + "margin_dpo/margin_mean": 0.0261460542678833, + "margin_dpo/margin_std": 0.33622923493385315, + "step": 25 + }, + { + "KL/chosen_KL_mean": -0.029012680053710938, + "KL/mean": -0.014886699616909027, + "KL/rejected_KL_mean": -0.000759124755859375, + "KL/std": 0.2382136434316635, + "epoch": 0.039304610733182165, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02825431525707245, + "fcm_dpo/q_t": 0.5007061958312988, + "grad_norm": 29.823959350585938, + "learning_rate": 1.8656716417910447e-07, + "logits/chosen": 0.12276704609394073, + "logits/rejected": 0.10391613841056824, + "logps/chosen": -56.848167419433594, + "logps/ref_chosen": -56.81915283203125, + "logps/ref_rejected": -77.84333038330078, + "logps/rejected": -77.84408569335938, + "loss": 1.3894, + "margin_dpo/margin_mean": -0.028254270553588867, + "margin_dpo/margin_std": 0.33982354402542114, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.036243438720703125, + "KL/mean": 0.017910704016685486, + "KL/rejected_KL_mean": -0.0004253387451171875, + "KL/std": 0.24093255400657654, + "epoch": 0.04081632653061224, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03667256236076355, + "fcm_dpo/q_t": 0.49908387660980225, + "grad_norm": 28.84993553161621, + "learning_rate": 1.9402985074626865e-07, + "logits/chosen": 0.09245912730693817, + "logits/rejected": 0.06795473396778107, + "logps/chosen": -62.840782165527344, + "logps/ref_chosen": -62.87702560424805, + "logps/ref_rejected": -71.34437561035156, + "logps/rejected": -71.34480285644531, + "loss": 1.3829, + "margin_dpo/margin_mean": 0.0366722047328949, + "margin_dpo/margin_std": 0.32028186321258545, + "step": 27 + }, + { + "KL/chosen_KL_mean": -0.02478790283203125, + "KL/mean": -0.04612994194030762, + "KL/rejected_KL_mean": -0.06747055053710938, + "KL/std": 0.22664371132850647, + "epoch": 0.042328042328042326, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04268306493759155, + "fcm_dpo/q_t": 0.4989331364631653, + "grad_norm": 27.578332901000977, + "learning_rate": 2.0149253731343282e-07, + "logits/chosen": 0.06610069423913956, + "logits/rejected": 0.05711779743432999, + "logps/chosen": -59.85816192626953, + "logps/ref_chosen": -59.8333740234375, + "logps/ref_rejected": -70.39804077148438, + "logps/rejected": -70.46550750732422, + "loss": 1.3822, + "margin_dpo/margin_mean": 0.04268324375152588, + "margin_dpo/margin_std": 0.27869123220443726, + "step": 28 + }, + { + "KL/chosen_KL_mean": -0.0189361572265625, + "KL/mean": -0.04419657588005066, + "KL/rejected_KL_mean": -0.0694580078125, + "KL/std": 0.19601967930793762, + "epoch": 0.04383975812547241, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05052053928375244, + "fcm_dpo/q_t": 0.4987371861934662, + "grad_norm": 32.570743560791016, + "learning_rate": 2.08955223880597e-07, + "logits/chosen": 0.1362171769142151, + "logits/rejected": 0.11860013008117676, + "logps/chosen": -74.1391372680664, + "logps/ref_chosen": -74.12020111083984, + "logps/ref_rejected": -83.33099365234375, + "logps/rejected": -83.40045166015625, + "loss": 1.3814, + "margin_dpo/margin_mean": 0.05052059888839722, + "margin_dpo/margin_std": 0.26012739539146423, + "step": 29 + }, + { + "KL/chosen_KL_mean": -0.040557861328125, + "KL/mean": -0.06584726274013519, + "KL/rejected_KL_mean": -0.09113693237304688, + "KL/std": 0.2571483552455902, + "epoch": 0.045351473922902494, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05058065056800842, + "fcm_dpo/q_t": 0.49873781204223633, + "grad_norm": 30.18358612060547, + "learning_rate": 2.1641791044776117e-07, + "logits/chosen": 0.09700868278741837, + "logits/rejected": 0.04471509903669357, + "logps/chosen": -50.791847229003906, + "logps/ref_chosen": -50.75128936767578, + "logps/ref_rejected": -89.29063415527344, + "logps/rejected": -89.38177490234375, + "loss": 1.3816, + "margin_dpo/margin_mean": 0.05057990550994873, + "margin_dpo/margin_std": 0.3860338032245636, + "step": 30 + }, + { + "KL/chosen_KL_mean": -0.060115814208984375, + "KL/mean": -0.05566447973251343, + "KL/rejected_KL_mean": -0.05120849609375, + "KL/std": 0.2631988823413849, + "epoch": 0.04686318972033258, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.008907794952392578, + "fcm_dpo/q_t": 0.5002219676971436, + "grad_norm": 34.45195388793945, + "learning_rate": 2.2388059701492537e-07, + "logits/chosen": 0.12011389434337616, + "logits/rejected": 0.07308140397071838, + "logps/chosen": -65.39686584472656, + "logps/ref_chosen": -65.33675384521484, + "logps/ref_rejected": -100.76666259765625, + "logps/rejected": -100.81787109375, + "loss": 1.3875, + "margin_dpo/margin_mean": -0.008907884359359741, + "margin_dpo/margin_std": 0.37602365016937256, + "step": 31 + }, + { + "KL/chosen_KL_mean": -0.0176239013671875, + "KL/mean": -0.02053038775920868, + "KL/rejected_KL_mean": -0.023433685302734375, + "KL/std": 0.2514227628707886, + "epoch": 0.04837490551776266, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0058145225048065186, + "fcm_dpo/q_t": 0.4998547434806824, + "grad_norm": 30.088085174560547, + "learning_rate": 2.3134328358208954e-07, + "logits/chosen": 0.0881882905960083, + "logits/rejected": 0.08028921484947205, + "logps/chosen": -67.20095825195312, + "logps/ref_chosen": -67.18333435058594, + "logps/ref_rejected": -82.80763244628906, + "logps/rejected": -82.83106994628906, + "loss": 1.3859, + "margin_dpo/margin_mean": 0.005814671516418457, + "margin_dpo/margin_std": 0.274589866399765, + "step": 32 + }, + { + "KL/chosen_KL_mean": -0.0117340087890625, + "KL/mean": -0.026652701199054718, + "KL/rejected_KL_mean": -0.041568756103515625, + "KL/std": 0.25098901987075806, + "epoch": 0.049886621315192746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.029830649495124817, + "fcm_dpo/q_t": 0.49925386905670166, + "grad_norm": 30.986589431762695, + "learning_rate": 2.388059701492537e-07, + "logits/chosen": 0.021548572927713394, + "logits/rejected": -0.0038994159549474716, + "logps/chosen": -64.05121612548828, + "logps/ref_chosen": -64.03948211669922, + "logps/ref_rejected": -75.68357849121094, + "logps/rejected": -75.72515106201172, + "loss": 1.3836, + "margin_dpo/margin_mean": 0.029831483960151672, + "margin_dpo/margin_std": 0.33458536863327026, + "step": 33 + }, + { + "KL/chosen_KL_mean": -0.012973785400390625, + "KL/mean": -0.04527957737445831, + "KL/rejected_KL_mean": -0.07758522033691406, + "KL/std": 0.2115345597267151, + "epoch": 0.05139833711262283, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06461545825004578, + "fcm_dpo/q_t": 0.4983849823474884, + "grad_norm": 28.43404769897461, + "learning_rate": 2.4626865671641786e-07, + "logits/chosen": 0.067808136343956, + "logits/rejected": 0.03934960439801216, + "logps/chosen": -53.677268981933594, + "logps/ref_chosen": -53.6642951965332, + "logps/ref_rejected": -65.77989959716797, + "logps/rejected": -65.85748291015625, + "loss": 1.3801, + "margin_dpo/margin_mean": 0.06461548805236816, + "margin_dpo/margin_std": 0.3139703869819641, + "step": 34 + }, + { + "KL/chosen_KL_mean": -0.08212852478027344, + "KL/mean": -0.07559476792812347, + "KL/rejected_KL_mean": -0.06905746459960938, + "KL/std": 0.21326494216918945, + "epoch": 0.05291005291005291, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.013070791959762573, + "fcm_dpo/q_t": 0.5003267526626587, + "grad_norm": 27.895584106445312, + "learning_rate": 2.537313432835821e-07, + "logits/chosen": 0.06519872695207596, + "logits/rejected": 0.0424288734793663, + "logps/chosen": -61.09899139404297, + "logps/ref_chosen": -61.01686096191406, + "logps/ref_rejected": -72.78598022460938, + "logps/rejected": -72.85504150390625, + "loss": 1.3878, + "margin_dpo/margin_mean": -0.013071000576019287, + "margin_dpo/margin_std": 0.27590203285217285, + "step": 35 + }, + { + "KL/chosen_KL_mean": -0.08506011962890625, + "KL/mean": -0.0908443033695221, + "KL/rejected_KL_mean": -0.09663009643554688, + "KL/std": 0.21762478351593018, + "epoch": 0.05442176870748299, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.011570632457733154, + "fcm_dpo/q_t": 0.4997110962867737, + "grad_norm": 28.65784454345703, + "learning_rate": 2.611940298507462e-07, + "logits/chosen": 0.12134528160095215, + "logits/rejected": 0.06698712706565857, + "logps/chosen": -50.62242126464844, + "logps/ref_chosen": -50.53736114501953, + "logps/ref_rejected": -78.11678314208984, + "logps/rejected": -78.21340942382812, + "loss": 1.3854, + "margin_dpo/margin_mean": 0.011571049690246582, + "margin_dpo/margin_std": 0.31866317987442017, + "step": 36 + }, + { + "KL/chosen_KL_mean": -0.04827880859375, + "KL/mean": -0.10174515843391418, + "KL/rejected_KL_mean": -0.15521621704101562, + "KL/std": 0.3025971055030823, + "epoch": 0.055933484504913075, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.10692453384399414, + "fcm_dpo/q_t": 0.4973316192626953, + "grad_norm": 37.73421096801758, + "learning_rate": 2.686567164179104e-07, + "logits/chosen": 0.10500356554985046, + "logits/rejected": 0.02512005902826786, + "logps/chosen": -59.60222625732422, + "logps/ref_chosen": -59.55394744873047, + "logps/ref_rejected": -108.27702331542969, + "logps/rejected": -108.43224334716797, + "loss": 1.3761, + "margin_dpo/margin_mean": 0.10692387819290161, + "margin_dpo/margin_std": 0.4470498561859131, + "step": 37 + }, + { + "KL/chosen_KL_mean": -0.05261421203613281, + "KL/mean": -0.06032392382621765, + "KL/rejected_KL_mean": -0.06803512573242188, + "KL/std": 0.2631487548351288, + "epoch": 0.05744520030234316, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.015420258045196533, + "fcm_dpo/q_t": 0.4996144771575928, + "grad_norm": 29.76226806640625, + "learning_rate": 2.761194029850746e-07, + "logits/chosen": 0.06649500131607056, + "logits/rejected": 0.05258432775735855, + "logps/chosen": -65.84097290039062, + "logps/ref_chosen": -65.78836059570312, + "logps/ref_rejected": -76.1619873046875, + "logps/rejected": -76.23002624511719, + "loss": 1.3851, + "margin_dpo/margin_mean": 0.015420526266098022, + "margin_dpo/margin_std": 0.3901858329772949, + "step": 38 + }, + { + "KL/chosen_KL_mean": -0.056888580322265625, + "KL/mean": -0.08803270757198334, + "KL/rejected_KL_mean": -0.119171142578125, + "KL/std": 0.255765825510025, + "epoch": 0.05895691609977324, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06227746605873108, + "fcm_dpo/q_t": 0.4984431862831116, + "grad_norm": 29.257625579833984, + "learning_rate": 2.8358208955223876e-07, + "logits/chosen": 0.1489247977733612, + "logits/rejected": 0.12244988977909088, + "logps/chosen": -57.23369598388672, + "logps/ref_chosen": -57.17681121826172, + "logps/ref_rejected": -79.486328125, + "logps/rejected": -79.60549926757812, + "loss": 1.3804, + "margin_dpo/margin_mean": 0.06227770447731018, + "margin_dpo/margin_std": 0.36542096734046936, + "step": 39 + }, + { + "KL/chosen_KL_mean": -0.08425140380859375, + "KL/mean": -0.1100742444396019, + "KL/rejected_KL_mean": -0.13589859008789062, + "KL/std": 0.2669486403465271, + "epoch": 0.06046863189720333, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05164425075054169, + "fcm_dpo/q_t": 0.49870917201042175, + "grad_norm": 31.260446548461914, + "learning_rate": 2.9104477611940296e-07, + "logits/chosen": 0.13100269436836243, + "logits/rejected": 0.0802154690027237, + "logps/chosen": -61.418418884277344, + "logps/ref_chosen": -61.33416748046875, + "logps/ref_rejected": -79.10697174072266, + "logps/rejected": -79.24287414550781, + "loss": 1.3815, + "margin_dpo/margin_mean": 0.05164472758769989, + "margin_dpo/margin_std": 0.3377959430217743, + "step": 40 + }, + { + "KL/chosen_KL_mean": -0.14268875122070312, + "KL/mean": -0.15808598697185516, + "KL/rejected_KL_mean": -0.1734790802001953, + "KL/std": 0.2604654133319855, + "epoch": 0.06198034769463341, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.030783504247665405, + "fcm_dpo/q_t": 0.49923068284988403, + "grad_norm": 30.55925750732422, + "learning_rate": 2.985074626865671e-07, + "logits/chosen": 0.06479241698980331, + "logits/rejected": 0.044267427176237106, + "logps/chosen": -67.68942260742188, + "logps/ref_chosen": -67.5467300415039, + "logps/ref_rejected": -83.87788391113281, + "logps/rejected": -84.0513687133789, + "loss": 1.3835, + "margin_dpo/margin_mean": 0.03078368306159973, + "margin_dpo/margin_std": 0.3066967725753784, + "step": 41 + }, + { + "KL/chosen_KL_mean": -0.127410888671875, + "KL/mean": -0.15839272737503052, + "KL/rejected_KL_mean": -0.18938064575195312, + "KL/std": 0.28839781880378723, + "epoch": 0.06349206349206349, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06197001039981842, + "fcm_dpo/q_t": 0.49845150113105774, + "grad_norm": 29.170963287353516, + "learning_rate": 3.059701492537313e-07, + "logits/chosen": 0.06191250681877136, + "logits/rejected": 0.04002637043595314, + "logps/chosen": -61.392269134521484, + "logps/ref_chosen": -61.26485824584961, + "logps/ref_rejected": -76.3629150390625, + "logps/rejected": -76.55229187011719, + "loss": 1.3805, + "margin_dpo/margin_mean": 0.06196974217891693, + "margin_dpo/margin_std": 0.3553955554962158, + "step": 42 + }, + { + "KL/chosen_KL_mean": -0.09712600708007812, + "KL/mean": -0.12862205505371094, + "KL/rejected_KL_mean": -0.16011428833007812, + "KL/std": 0.2960539758205414, + "epoch": 0.06500377928949358, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06299221515655518, + "fcm_dpo/q_t": 0.4984261393547058, + "grad_norm": 34.61469268798828, + "learning_rate": 3.134328358208955e-07, + "logits/chosen": 0.09886477142572403, + "logits/rejected": 0.08766770362854004, + "logps/chosen": -71.90615844726562, + "logps/ref_chosen": -71.80902862548828, + "logps/ref_rejected": -81.12464141845703, + "logps/rejected": -81.28475952148438, + "loss": 1.3805, + "margin_dpo/margin_mean": 0.06299227476119995, + "margin_dpo/margin_std": 0.4519028663635254, + "step": 43 + }, + { + "KL/chosen_KL_mean": -0.1671600341796875, + "KL/mean": -0.1565622091293335, + "KL/rejected_KL_mean": -0.14596939086914062, + "KL/std": 0.26872149109840393, + "epoch": 0.06651549508692366, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02118849754333496, + "fcm_dpo/q_t": 0.5005298256874084, + "grad_norm": 32.638671875, + "learning_rate": 3.2089552238805965e-07, + "logits/chosen": 0.055923182517290115, + "logits/rejected": 0.025120796635746956, + "logps/chosen": -66.71759033203125, + "logps/ref_chosen": -66.55043029785156, + "logps/ref_rejected": -85.06198120117188, + "logps/rejected": -85.20794677734375, + "loss": 1.3888, + "margin_dpo/margin_mean": -0.02118849754333496, + "margin_dpo/margin_std": 0.39402931928634644, + "step": 44 + }, + { + "KL/chosen_KL_mean": -0.1441822052001953, + "KL/mean": -0.19521594047546387, + "KL/rejected_KL_mean": -0.24625015258789062, + "KL/std": 0.28586143255233765, + "epoch": 0.06802721088435375, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.10206963121891022, + "fcm_dpo/q_t": 0.49744898080825806, + "grad_norm": 31.806509017944336, + "learning_rate": 3.2835820895522385e-07, + "logits/chosen": 0.12358801066875458, + "logits/rejected": 0.07059746980667114, + "logps/chosen": -62.388038635253906, + "logps/ref_chosen": -62.24385452270508, + "logps/ref_rejected": -92.96665954589844, + "logps/rejected": -93.2129135131836, + "loss": 1.3765, + "margin_dpo/margin_mean": 0.10207107663154602, + "margin_dpo/margin_std": 0.39976954460144043, + "step": 45 + }, + { + "KL/chosen_KL_mean": -0.10940742492675781, + "KL/mean": -0.17646794021129608, + "KL/rejected_KL_mean": -0.2435302734375, + "KL/std": 0.31598347425460815, + "epoch": 0.06953892668178382, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.13412192463874817, + "fcm_dpo/q_t": 0.49664801359176636, + "grad_norm": 31.140886306762695, + "learning_rate": 3.3582089552238805e-07, + "logits/chosen": 0.12344063818454742, + "logits/rejected": 0.07757961750030518, + "logps/chosen": -61.608314514160156, + "logps/ref_chosen": -61.498905181884766, + "logps/ref_rejected": -78.91172790527344, + "logps/rejected": -79.15525817871094, + "loss": 1.3733, + "margin_dpo/margin_mean": 0.13412132859230042, + "margin_dpo/margin_std": 0.38167721033096313, + "step": 46 + }, + { + "KL/chosen_KL_mean": -0.12112808227539062, + "KL/mean": -0.21411283314228058, + "KL/rejected_KL_mean": -0.307098388671875, + "KL/std": 0.2846325635910034, + "epoch": 0.0710506424792139, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.18596699833869934, + "fcm_dpo/q_t": 0.4953530430793762, + "grad_norm": 28.34612464904785, + "learning_rate": 3.432835820895522e-07, + "logits/chosen": 0.03395320102572441, + "logits/rejected": -0.008879796601831913, + "logps/chosen": -51.6994743347168, + "logps/ref_chosen": -51.578346252441406, + "logps/ref_rejected": -68.2215576171875, + "logps/rejected": -68.52865600585938, + "loss": 1.3682, + "margin_dpo/margin_mean": 0.18596723675727844, + "margin_dpo/margin_std": 0.3997984528541565, + "step": 47 + }, + { + "KL/chosen_KL_mean": -0.23851966857910156, + "KL/mean": -0.25627630949020386, + "KL/rejected_KL_mean": -0.2740345001220703, + "KL/std": 0.2957780957221985, + "epoch": 0.07256235827664399, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03551402688026428, + "fcm_dpo/q_t": 0.49911266565322876, + "grad_norm": 26.784517288208008, + "learning_rate": 3.507462686567164e-07, + "logits/chosen": 0.14234454929828644, + "logits/rejected": 0.11283800005912781, + "logps/chosen": -52.03217315673828, + "logps/ref_chosen": -51.79365158081055, + "logps/ref_rejected": -64.22503662109375, + "logps/rejected": -64.49906921386719, + "loss": 1.3832, + "margin_dpo/margin_mean": 0.03551414608955383, + "margin_dpo/margin_std": 0.3996293544769287, + "step": 48 + }, + { + "KL/chosen_KL_mean": -0.2007770538330078, + "KL/mean": -0.25442013144493103, + "KL/rejected_KL_mean": -0.3080596923828125, + "KL/std": 0.3600367307662964, + "epoch": 0.07407407407407407, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1072855144739151, + "fcm_dpo/q_t": 0.4973185062408447, + "grad_norm": 27.355039596557617, + "learning_rate": 3.5820895522388055e-07, + "logits/chosen": 0.009659804403781891, + "logits/rejected": -0.011214490979909897, + "logps/chosen": -58.33538055419922, + "logps/ref_chosen": -58.13460159301758, + "logps/ref_rejected": -64.63206481933594, + "logps/rejected": -64.94012451171875, + "loss": 1.3762, + "margin_dpo/margin_mean": 0.10728609561920166, + "margin_dpo/margin_std": 0.48972445726394653, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.2860908508300781, + "KL/mean": -0.32769298553466797, + "KL/rejected_KL_mean": -0.3692970275878906, + "KL/std": 0.34533897042274475, + "epoch": 0.07558578987150416, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08320507407188416, + "fcm_dpo/q_t": 0.4979206919670105, + "grad_norm": 28.02340316772461, + "learning_rate": 3.6567164179104475e-07, + "logits/chosen": 0.10904943943023682, + "logits/rejected": 0.07938901335000992, + "logps/chosen": -53.14253234863281, + "logps/ref_chosen": -52.85643768310547, + "logps/ref_rejected": -72.17460632324219, + "logps/rejected": -72.54389953613281, + "loss": 1.3784, + "margin_dpo/margin_mean": 0.08320528268814087, + "margin_dpo/margin_std": 0.41352373361587524, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.30113792419433594, + "KL/mean": -0.38790732622146606, + "KL/rejected_KL_mean": -0.47467803955078125, + "KL/std": 0.3827604651451111, + "epoch": 0.07709750566893424, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.17354324460029602, + "fcm_dpo/q_t": 0.4956646263599396, + "grad_norm": 31.038726806640625, + "learning_rate": 3.7313432835820895e-07, + "logits/chosen": 0.0878702700138092, + "logits/rejected": 0.06008681282401085, + "logps/chosen": -63.95758056640625, + "logps/ref_chosen": -63.65644073486328, + "logps/ref_rejected": -86.13229370117188, + "logps/rejected": -86.60697937011719, + "loss": 1.3696, + "margin_dpo/margin_mean": 0.173543781042099, + "margin_dpo/margin_std": 0.4818759560585022, + "step": 51 + }, + { + "KL/chosen_KL_mean": -0.2929725646972656, + "KL/mean": -0.40666550397872925, + "KL/rejected_KL_mean": -0.520355224609375, + "KL/std": 0.4088834524154663, + "epoch": 0.07860922146636433, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.22738106548786163, + "fcm_dpo/q_t": 0.4943210184574127, + "grad_norm": 32.647708892822266, + "learning_rate": 3.805970149253731e-07, + "logits/chosen": 0.1057782769203186, + "logits/rejected": 0.05464206635951996, + "logps/chosen": -68.13318634033203, + "logps/ref_chosen": -67.8402099609375, + "logps/ref_rejected": -96.97090911865234, + "logps/rejected": -97.49126434326172, + "loss": 1.3644, + "margin_dpo/margin_mean": 0.2273811250925064, + "margin_dpo/margin_std": 0.5355072021484375, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.38391876220703125, + "KL/mean": -0.4516035318374634, + "KL/rejected_KL_mean": -0.519287109375, + "KL/std": 0.3600003123283386, + "epoch": 0.0801209372637944, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.13536784052848816, + "fcm_dpo/q_t": 0.4966163635253906, + "grad_norm": 27.55713653564453, + "learning_rate": 3.880597014925373e-07, + "logits/chosen": 0.09621694684028625, + "logits/rejected": 0.08538716286420822, + "logps/chosen": -57.26205825805664, + "logps/ref_chosen": -56.87813949584961, + "logps/ref_rejected": -60.75569152832031, + "logps/rejected": -61.27497863769531, + "loss": 1.3734, + "margin_dpo/margin_mean": 0.1353682279586792, + "margin_dpo/margin_std": 0.47499287128448486, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.37387847900390625, + "KL/mean": -0.44698044657707214, + "KL/rejected_KL_mean": -0.5200824737548828, + "KL/std": 0.37565189599990845, + "epoch": 0.08163265306122448, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.14620435237884521, + "fcm_dpo/q_t": 0.4963470697402954, + "grad_norm": 26.632923126220703, + "learning_rate": 3.9552238805970144e-07, + "logits/chosen": 0.042465463280677795, + "logits/rejected": 0.02737961709499359, + "logps/chosen": -47.64080047607422, + "logps/ref_chosen": -47.26692199707031, + "logps/ref_rejected": -62.19426727294922, + "logps/rejected": -62.71434783935547, + "loss": 1.3723, + "margin_dpo/margin_mean": 0.14620399475097656, + "margin_dpo/margin_std": 0.4815361499786377, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.37229156494140625, + "KL/mean": -0.5114315152168274, + "KL/rejected_KL_mean": -0.65057373046875, + "KL/std": 0.49368512630462646, + "epoch": 0.08314436885865457, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2782818675041199, + "fcm_dpo/q_t": 0.493056058883667, + "grad_norm": 31.412118911743164, + "learning_rate": 4.0298507462686564e-07, + "logits/chosen": 0.03549818694591522, + "logits/rejected": -0.04075536131858826, + "logps/chosen": -50.698482513427734, + "logps/ref_chosen": -50.32619094848633, + "logps/ref_rejected": -92.44389343261719, + "logps/rejected": -93.09446716308594, + "loss": 1.3598, + "margin_dpo/margin_mean": 0.27828145027160645, + "margin_dpo/margin_std": 0.6260495185852051, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.3755607604980469, + "KL/mean": -0.4541851282119751, + "KL/rejected_KL_mean": -0.5328025817871094, + "KL/std": 0.4594634771347046, + "epoch": 0.08465608465608465, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1572437584400177, + "fcm_dpo/q_t": 0.49606892466545105, + "grad_norm": 27.55105972290039, + "learning_rate": 4.1044776119402984e-07, + "logits/chosen": 0.12995021045207977, + "logits/rejected": 0.10750794410705566, + "logps/chosen": -57.14253234863281, + "logps/ref_chosen": -56.766971588134766, + "logps/ref_rejected": -66.30504608154297, + "logps/rejected": -66.83784484863281, + "loss": 1.3715, + "margin_dpo/margin_mean": 0.15724411606788635, + "margin_dpo/margin_std": 0.583028256893158, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.5324649810791016, + "KL/mean": -0.6625369787216187, + "KL/rejected_KL_mean": -0.7926063537597656, + "KL/std": 0.5992856025695801, + "epoch": 0.08616780045351474, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2601401209831238, + "fcm_dpo/q_t": 0.4935043454170227, + "grad_norm": 30.670913696289062, + "learning_rate": 4.17910447761194e-07, + "logits/chosen": 0.14049838483333588, + "logits/rejected": 0.07349002361297607, + "logps/chosen": -58.300209045410156, + "logps/ref_chosen": -57.76774597167969, + "logps/ref_rejected": -82.75698852539062, + "logps/rejected": -83.54959106445312, + "loss": 1.3617, + "margin_dpo/margin_mean": 0.26013994216918945, + "margin_dpo/margin_std": 0.6875753402709961, + "step": 57 + }, + { + "KL/chosen_KL_mean": -0.6022109985351562, + "KL/mean": -0.6994897723197937, + "KL/rejected_KL_mean": -0.7967681884765625, + "KL/std": 0.6317234635353088, + "epoch": 0.08767951625094482, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1945592164993286, + "fcm_dpo/q_t": 0.4951819181442261, + "grad_norm": 30.232378005981445, + "learning_rate": 4.253731343283582e-07, + "logits/chosen": 0.08454892039299011, + "logits/rejected": 0.0682353526353836, + "logps/chosen": -73.36630249023438, + "logps/ref_chosen": -72.76408386230469, + "logps/ref_rejected": -84.49275207519531, + "logps/rejected": -85.28952026367188, + "loss": 1.3693, + "margin_dpo/margin_mean": 0.19455870985984802, + "margin_dpo/margin_std": 0.9174859523773193, + "step": 58 + }, + { + "KL/chosen_KL_mean": -0.5136947631835938, + "KL/mean": -0.6560180187225342, + "KL/rejected_KL_mean": -0.7983455657958984, + "KL/std": 0.5859674215316772, + "epoch": 0.08919123204837491, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.28464627265930176, + "fcm_dpo/q_t": 0.49290651082992554, + "grad_norm": 26.60697364807129, + "learning_rate": 4.3283582089552234e-07, + "logits/chosen": 0.13293355703353882, + "logits/rejected": 0.06686300039291382, + "logps/chosen": -50.33447265625, + "logps/ref_chosen": -49.820777893066406, + "logps/ref_rejected": -77.14368438720703, + "logps/rejected": -77.94203186035156, + "loss": 1.3597, + "margin_dpo/margin_mean": 0.28464648127555847, + "margin_dpo/margin_std": 0.7471631169319153, + "step": 59 + }, + { + "KL/chosen_KL_mean": -0.6310577392578125, + "KL/mean": -0.6649550199508667, + "KL/rejected_KL_mean": -0.6988525390625, + "KL/std": 0.6437369585037231, + "epoch": 0.09070294784580499, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0677938461303711, + "fcm_dpo/q_t": 0.4983041286468506, + "grad_norm": 29.712261199951172, + "learning_rate": 4.4029850746268654e-07, + "logits/chosen": 0.1172294169664383, + "logits/rejected": 0.1158132255077362, + "logps/chosen": -63.855831146240234, + "logps/ref_chosen": -63.22477340698242, + "logps/ref_rejected": -61.360477447509766, + "logps/rejected": -62.059329986572266, + "loss": 1.3814, + "margin_dpo/margin_mean": 0.0677938461303711, + "margin_dpo/margin_std": 0.8459521532058716, + "step": 60 + }, + { + "KL/chosen_KL_mean": -0.7134819030761719, + "KL/mean": -0.7604533433914185, + "KL/rejected_KL_mean": -0.8074264526367188, + "KL/std": 0.7058064341545105, + "epoch": 0.09221466364323508, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0939413458108902, + "fcm_dpo/q_t": 0.4976520538330078, + "grad_norm": 27.93014907836914, + "learning_rate": 4.4776119402985074e-07, + "logits/chosen": 0.14893561601638794, + "logits/rejected": 0.11584499478340149, + "logps/chosen": -49.73027801513672, + "logps/ref_chosen": -49.01679992675781, + "logps/ref_rejected": -74.90817260742188, + "logps/rejected": -75.71559143066406, + "loss": 1.3793, + "margin_dpo/margin_mean": 0.09394136071205139, + "margin_dpo/margin_std": 0.9773008823394775, + "step": 61 + }, + { + "KL/chosen_KL_mean": -0.7257556915283203, + "KL/mean": -0.8533656001091003, + "KL/rejected_KL_mean": -0.9809722900390625, + "KL/std": 0.6654196977615356, + "epoch": 0.09372637944066516, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.25521424412727356, + "fcm_dpo/q_t": 0.4936321973800659, + "grad_norm": 28.620866775512695, + "learning_rate": 4.552238805970149e-07, + "logits/chosen": 0.11153241246938705, + "logits/rejected": 0.07246184349060059, + "logps/chosen": -63.477622985839844, + "logps/ref_chosen": -62.751869201660156, + "logps/ref_rejected": -78.93360900878906, + "logps/rejected": -79.91458129882812, + "loss": 1.3632, + "margin_dpo/margin_mean": 0.2552138566970825, + "margin_dpo/margin_std": 0.9336051940917969, + "step": 62 + }, + { + "KL/chosen_KL_mean": -0.5593242645263672, + "KL/mean": -0.8448190689086914, + "KL/rejected_KL_mean": -1.1303176879882812, + "KL/std": 0.6812584400177002, + "epoch": 0.09523809523809523, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5709925889968872, + "fcm_dpo/q_t": 0.4857600927352905, + "grad_norm": 31.421058654785156, + "learning_rate": 4.626865671641791e-07, + "logits/chosen": 0.16452708840370178, + "logits/rejected": 0.1406918168067932, + "logps/chosen": -61.0745735168457, + "logps/ref_chosen": -60.51525115966797, + "logps/ref_rejected": -85.11021423339844, + "logps/rejected": -86.24053192138672, + "loss": 1.3317, + "margin_dpo/margin_mean": 0.5709917545318604, + "margin_dpo/margin_std": 0.8313959836959839, + "step": 63 + }, + { + "KL/chosen_KL_mean": -0.8067245483398438, + "KL/mean": -0.849244236946106, + "KL/rejected_KL_mean": -0.8917655944824219, + "KL/std": 0.7516355514526367, + "epoch": 0.09674981103552532, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08503898978233337, + "fcm_dpo/q_t": 0.49786609411239624, + "grad_norm": 26.175302505493164, + "learning_rate": 4.701492537313433e-07, + "logits/chosen": 0.07753998041152954, + "logits/rejected": 0.05309564620256424, + "logps/chosen": -52.013572692871094, + "logps/ref_chosen": -51.20684814453125, + "logps/ref_rejected": -66.93081665039062, + "logps/rejected": -67.82258605957031, + "loss": 1.3799, + "margin_dpo/margin_mean": 0.08503925800323486, + "margin_dpo/margin_std": 0.9009600281715393, + "step": 64 + }, + { + "KL/chosen_KL_mean": -0.8779640197753906, + "KL/mean": -1.1214277744293213, + "KL/rejected_KL_mean": -1.3648948669433594, + "KL/std": 0.9085813760757446, + "epoch": 0.0982615268329554, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4869285821914673, + "fcm_dpo/q_t": 0.48788049817085266, + "grad_norm": 30.840694427490234, + "learning_rate": 4.776119402985074e-07, + "logits/chosen": 0.20080530643463135, + "logits/rejected": 0.17102986574172974, + "logps/chosen": -68.16665649414062, + "logps/ref_chosen": -67.2886962890625, + "logps/ref_rejected": -74.44281005859375, + "logps/rejected": -75.80770111083984, + "loss": 1.3418, + "margin_dpo/margin_mean": 0.48692822456359863, + "margin_dpo/margin_std": 1.1947252750396729, + "step": 65 + }, + { + "KL/chosen_KL_mean": -0.9242038726806641, + "KL/mean": -1.0837883949279785, + "KL/rejected_KL_mean": -1.243377685546875, + "KL/std": 0.8412731885910034, + "epoch": 0.09977324263038549, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.31917569041252136, + "fcm_dpo/q_t": 0.4920502305030823, + "grad_norm": 29.339235305786133, + "learning_rate": 4.850746268656717e-07, + "logits/chosen": 0.09757672995328903, + "logits/rejected": 0.07372169941663742, + "logps/chosen": -71.66761779785156, + "logps/ref_chosen": -70.743408203125, + "logps/ref_rejected": -77.26499938964844, + "logps/rejected": -78.50837707519531, + "loss": 1.3578, + "margin_dpo/margin_mean": 0.31917527318000793, + "margin_dpo/margin_std": 1.1150388717651367, + "step": 66 + }, + { + "KL/chosen_KL_mean": -0.7873554229736328, + "KL/mean": -0.9470667839050293, + "KL/rejected_KL_mean": -1.106781005859375, + "KL/std": 0.7722550630569458, + "epoch": 0.10128495842781557, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3194220960140228, + "fcm_dpo/q_t": 0.49203306436538696, + "grad_norm": 29.272001266479492, + "learning_rate": 4.925373134328357e-07, + "logits/chosen": 0.0839434489607811, + "logits/rejected": 0.028710132464766502, + "logps/chosen": -61.389957427978516, + "logps/ref_chosen": -60.60260009765625, + "logps/ref_rejected": -75.22235870361328, + "logps/rejected": -76.32913970947266, + "loss": 1.3569, + "margin_dpo/margin_mean": 0.31942254304885864, + "margin_dpo/margin_std": 0.9561138153076172, + "step": 67 + }, + { + "KL/chosen_KL_mean": -1.2437705993652344, + "KL/mean": -1.405458927154541, + "KL/rejected_KL_mean": -1.5671539306640625, + "KL/std": 0.949453592300415, + "epoch": 0.10279667422524566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3233872652053833, + "fcm_dpo/q_t": 0.49194926023483276, + "grad_norm": 31.313793182373047, + "learning_rate": 5e-07, + "logits/chosen": 0.04201754555106163, + "logits/rejected": 0.013082325458526611, + "logps/chosen": -78.77214050292969, + "logps/ref_chosen": -77.52836608886719, + "logps/ref_rejected": -93.17778015136719, + "logps/rejected": -94.74493408203125, + "loss": 1.3583, + "margin_dpo/margin_mean": 0.32338738441467285, + "margin_dpo/margin_std": 1.2540662288665771, + "step": 68 + }, + { + "KL/chosen_KL_mean": -1.1451244354248047, + "KL/mean": -1.4814937114715576, + "KL/rejected_KL_mean": -1.8178558349609375, + "KL/std": 1.0125997066497803, + "epoch": 0.10430839002267574, + "fcm_dpo/beta": 0.10156106948852539, + "fcm_dpo/delta": 0.15371915698051453, + "fcm_dpo/margin": 0.6727360486984253, + "fcm_dpo/q_t": 0.4832885265350342, + "grad_norm": 31.382776260375977, + "learning_rate": 4.999965034812934e-07, + "logits/chosen": 0.11837112903594971, + "logits/rejected": 0.07452549785375595, + "logps/chosen": -67.08818054199219, + "logps/ref_chosen": -65.94305419921875, + "logps/ref_rejected": -89.7735595703125, + "logps/rejected": -91.59141540527344, + "loss": 1.3227, + "margin_dpo/margin_mean": 0.672736644744873, + "margin_dpo/margin_std": 1.206894874572754, + "step": 69 + }, + { + "KL/chosen_KL_mean": -1.2427654266357422, + "KL/mean": -1.400121808052063, + "KL/rejected_KL_mean": -1.5574722290039062, + "KL/std": 0.8860043883323669, + "epoch": 0.10582010582010581, + "fcm_dpo/beta": 0.10312213003635406, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3147037923336029, + "fcm_dpo/q_t": 0.4919206500053406, + "grad_norm": 29.573619842529297, + "learning_rate": 4.999860140229787e-07, + "logits/chosen": 0.1305362433195114, + "logits/rejected": 0.10781893879175186, + "logps/chosen": -63.20067596435547, + "logps/ref_chosen": -61.95791244506836, + "logps/ref_rejected": -75.80945587158203, + "logps/rejected": -77.36692810058594, + "loss": 1.3577, + "margin_dpo/margin_mean": 0.3147040009498596, + "margin_dpo/margin_std": 1.1622765064239502, + "step": 70 + }, + { + "KL/chosen_KL_mean": -1.4353160858154297, + "KL/mean": -1.5389230251312256, + "KL/rejected_KL_mean": -1.6425323486328125, + "KL/std": 1.0423591136932373, + "epoch": 0.1073318216175359, + "fcm_dpo/beta": 0.10312213003635406, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2072136104106903, + "fcm_dpo/q_t": 0.49464237689971924, + "grad_norm": 29.52690315246582, + "learning_rate": 4.999685319184688e-07, + "logits/chosen": 0.08314691483974457, + "logits/rejected": 0.0680352970957756, + "logps/chosen": -64.78289794921875, + "logps/ref_chosen": -63.34757995605469, + "logps/ref_rejected": -67.49658203125, + "logps/rejected": -69.13911437988281, + "loss": 1.3712, + "margin_dpo/margin_mean": 0.2072138488292694, + "margin_dpo/margin_std": 1.4971086978912354, + "step": 71 + }, + { + "KL/chosen_KL_mean": -1.2093849182128906, + "KL/mean": -1.5765079259872437, + "KL/rejected_KL_mean": -1.943634033203125, + "KL/std": 1.095198154449463, + "epoch": 0.10884353741496598, + "fcm_dpo/beta": 0.10635857284069061, + "fcm_dpo/delta": 0.15451082587242126, + "fcm_dpo/margin": 0.7342473268508911, + "fcm_dpo/q_t": 0.4809272885322571, + "grad_norm": 31.995201110839844, + "learning_rate": 4.999440576567755e-07, + "logits/chosen": 0.10724207758903503, + "logits/rejected": 0.044822730123996735, + "logps/chosen": -57.06868362426758, + "logps/ref_chosen": -55.85929870605469, + "logps/ref_rejected": -68.45423889160156, + "logps/rejected": -70.39787292480469, + "loss": 1.3149, + "margin_dpo/margin_mean": 0.7342471480369568, + "margin_dpo/margin_std": 1.330765962600708, + "step": 72 + }, + { + "KL/chosen_KL_mean": -1.806976318359375, + "KL/mean": -1.8687390089035034, + "KL/rejected_KL_mean": -1.9305057525634766, + "KL/std": 1.2052420377731323, + "epoch": 0.11035525321239607, + "fcm_dpo/beta": 0.10635857284069061, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.12353459000587463, + "fcm_dpo/q_t": 0.4968770742416382, + "grad_norm": 34.1959342956543, + "learning_rate": 4.999125919224965e-07, + "logits/chosen": 0.12511396408081055, + "logits/rejected": 0.11073576658964157, + "logps/chosen": -70.94578552246094, + "logps/ref_chosen": -69.13880920410156, + "logps/ref_rejected": -79.04586791992188, + "logps/rejected": -80.97637939453125, + "loss": 1.3805, + "margin_dpo/margin_mean": 0.12353500723838806, + "margin_dpo/margin_std": 1.5384290218353271, + "step": 73 + }, + { + "KL/chosen_KL_mean": -1.2401103973388672, + "KL/mean": -1.5870051383972168, + "KL/rejected_KL_mean": -1.9338951110839844, + "KL/std": 0.9800302982330322, + "epoch": 0.11186696900982615, + "fcm_dpo/beta": 0.10799010097980499, + "fcm_dpo/delta": 0.15109321475028992, + "fcm_dpo/margin": 0.6937872171401978, + "fcm_dpo/q_t": 0.4816681742668152, + "grad_norm": 29.913087844848633, + "learning_rate": 4.998741355957963e-07, + "logits/chosen": 0.13613180816173553, + "logits/rejected": 0.08324501663446426, + "logps/chosen": -51.163848876953125, + "logps/ref_chosen": -49.923736572265625, + "logps/ref_rejected": -81.73213958740234, + "logps/rejected": -83.66603088378906, + "loss": 1.3172, + "margin_dpo/margin_mean": 0.6937870383262634, + "margin_dpo/margin_std": 1.2566819190979004, + "step": 74 + }, + { + "KL/chosen_KL_mean": -1.4187679290771484, + "KL/mean": -1.7926573753356934, + "KL/rejected_KL_mean": -2.166543960571289, + "KL/std": 1.1323838233947754, + "epoch": 0.11337868480725624, + "fcm_dpo/beta": 0.11300066858530045, + "fcm_dpo/delta": 0.1517949253320694, + "fcm_dpo/margin": 0.7477781772613525, + "fcm_dpo/q_t": 0.4794332981109619, + "grad_norm": 29.2504940032959, + "learning_rate": 4.998286897523808e-07, + "logits/chosen": 0.10926586389541626, + "logits/rejected": 0.07744550704956055, + "logps/chosen": -47.487518310546875, + "logps/ref_chosen": -46.06875228881836, + "logps/ref_rejected": -66.1181411743164, + "logps/rejected": -68.28468322753906, + "loss": 1.3102, + "margin_dpo/margin_mean": 0.7477779388427734, + "margin_dpo/margin_std": 1.4347933530807495, + "step": 75 + }, + { + "KL/chosen_KL_mean": -1.6109638214111328, + "KL/mean": -1.770002007484436, + "KL/rejected_KL_mean": -1.9290390014648438, + "KL/std": 1.1657469272613525, + "epoch": 0.11489040060468632, + "fcm_dpo/beta": 0.11300066858530045, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3180782198905945, + "fcm_dpo/q_t": 0.4910624921321869, + "grad_norm": 33.03213119506836, + "learning_rate": 4.997762556634679e-07, + "logits/chosen": 0.1351306140422821, + "logits/rejected": 0.09061160683631897, + "logps/chosen": -55.67371368408203, + "logps/ref_chosen": -54.06275177001953, + "logps/ref_rejected": -74.87464141845703, + "logps/rejected": -76.80368041992188, + "loss": 1.3583, + "margin_dpo/margin_mean": 0.31807827949523926, + "margin_dpo/margin_std": 1.5470020771026611, + "step": 76 + }, + { + "KL/chosen_KL_mean": -1.6516189575195312, + "KL/mean": -2.0099892616271973, + "KL/rejected_KL_mean": -2.368358612060547, + "KL/std": 1.238326072692871, + "epoch": 0.1164021164021164, + "fcm_dpo/beta": 0.11631464213132858, + "fcm_dpo/delta": 0.1445261687040329, + "fcm_dpo/margin": 0.7167394161224365, + "fcm_dpo/q_t": 0.47974491119384766, + "grad_norm": 33.77786636352539, + "learning_rate": 4.99716834795752e-07, + "logits/chosen": 0.15451249480247498, + "logits/rejected": 0.1131986528635025, + "logps/chosen": -54.72771072387695, + "logps/ref_chosen": -53.07609176635742, + "logps/ref_rejected": -74.45601654052734, + "logps/rejected": -76.82437133789062, + "loss": 1.3121, + "margin_dpo/margin_mean": 0.7167395353317261, + "margin_dpo/margin_std": 1.4588513374328613, + "step": 77 + }, + { + "KL/chosen_KL_mean": -1.8221549987792969, + "KL/mean": -2.0393378734588623, + "KL/rejected_KL_mean": -2.256519317626953, + "KL/std": 1.2632840871810913, + "epoch": 0.11791383219954649, + "fcm_dpo/beta": 0.11631464213132858, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4343580901622772, + "fcm_dpo/q_t": 0.48744601011276245, + "grad_norm": 34.19994354248047, + "learning_rate": 4.996504288113623e-07, + "logits/chosen": 0.11588499695062637, + "logits/rejected": 0.09554223716259003, + "logps/chosen": -69.54757690429688, + "logps/ref_chosen": -67.72541809082031, + "logps/ref_rejected": -79.03926849365234, + "logps/rejected": -81.29579162597656, + "loss": 1.3433, + "margin_dpo/margin_mean": 0.4343578517436981, + "margin_dpo/margin_std": 1.4269845485687256, + "step": 78 + }, + { + "KL/chosen_KL_mean": -1.7330493927001953, + "KL/mean": -2.276017189025879, + "KL/rejected_KL_mean": -2.8189849853515625, + "KL/std": 1.5086700916290283, + "epoch": 0.11942554799697656, + "fcm_dpo/beta": 0.12106219679117203, + "fcm_dpo/delta": 0.27233055233955383, + "fcm_dpo/margin": 1.0859367847442627, + "fcm_dpo/q_t": 0.46888160705566406, + "grad_norm": 36.362300872802734, + "learning_rate": 4.995770395678171e-07, + "logits/chosen": 0.19241735339164734, + "logits/rejected": 0.1304788440465927, + "logps/chosen": -53.89369201660156, + "logps/ref_chosen": -52.16064453125, + "logps/ref_rejected": -83.31062316894531, + "logps/rejected": -86.12960815429688, + "loss": 1.2734, + "margin_dpo/margin_mean": 1.0859365463256836, + "margin_dpo/margin_std": 1.9711155891418457, + "step": 79 + }, + { + "KL/chosen_KL_mean": -2.051755905151367, + "KL/mean": -2.335285186767578, + "KL/rejected_KL_mean": -2.6188125610351562, + "KL/std": 1.3859307765960693, + "epoch": 0.12093726379440665, + "fcm_dpo/beta": 0.12457603216171265, + "fcm_dpo/delta": 0.14052413403987885, + "fcm_dpo/margin": 0.567061722278595, + "fcm_dpo/q_t": 0.48296597599983215, + "grad_norm": 35.15907669067383, + "learning_rate": 4.994966691179711e-07, + "logits/chosen": 0.14136065542697906, + "logits/rejected": 0.0816778764128685, + "logps/chosen": -63.462318420410156, + "logps/ref_chosen": -61.410560607910156, + "logps/ref_rejected": -78.66004943847656, + "logps/rejected": -81.27886199951172, + "loss": 1.3314, + "margin_dpo/margin_mean": 0.5670617818832397, + "margin_dpo/margin_std": 1.904353141784668, + "step": 80 + }, + { + "KL/chosen_KL_mean": -2.106597900390625, + "KL/mean": -2.5513858795166016, + "KL/rejected_KL_mean": -2.9961700439453125, + "KL/std": 1.4156627655029297, + "epoch": 0.12244897959183673, + "fcm_dpo/beta": 0.12924838066101074, + "fcm_dpo/delta": 0.11433006823062897, + "fcm_dpo/margin": 0.8895747661590576, + "fcm_dpo/q_t": 0.4723474383354187, + "grad_norm": 36.73268508911133, + "learning_rate": 4.994093197099587e-07, + "logits/chosen": 0.13766348361968994, + "logits/rejected": 0.10336862504482269, + "logps/chosen": -65.91096496582031, + "logps/ref_chosen": -63.80437088012695, + "logps/ref_rejected": -79.3484115600586, + "logps/rejected": -82.3445816040039, + "loss": 1.2874, + "margin_dpo/margin_mean": 0.8895748853683472, + "margin_dpo/margin_std": 1.6920909881591797, + "step": 81 + }, + { + "KL/chosen_KL_mean": -1.7257709503173828, + "KL/mean": -2.2757372856140137, + "KL/rejected_KL_mean": -2.8257064819335938, + "KL/std": 1.3814198970794678, + "epoch": 0.12396069538926682, + "fcm_dpo/beta": 0.13408434391021729, + "fcm_dpo/delta": 0.25668978691101074, + "fcm_dpo/margin": 1.0999317169189453, + "fcm_dpo/q_t": 0.46460217237472534, + "grad_norm": 34.45591354370117, + "learning_rate": 4.993149937871306e-07, + "logits/chosen": 0.124167300760746, + "logits/rejected": 0.06017389893531799, + "logps/chosen": -50.543663024902344, + "logps/ref_chosen": -48.817893981933594, + "logps/ref_rejected": -70.31497955322266, + "logps/rejected": -73.14068603515625, + "loss": 1.2565, + "margin_dpo/margin_mean": 1.0999321937561035, + "margin_dpo/margin_std": 1.6014728546142578, + "step": 82 + }, + { + "KL/chosen_KL_mean": -2.0304203033447266, + "KL/mean": -2.6201720237731934, + "KL/rejected_KL_mean": -3.2099266052246094, + "KL/std": 1.4840151071548462, + "epoch": 0.1254724111866969, + "fcm_dpo/beta": 0.14089030027389526, + "fcm_dpo/delta": 0.23794013261795044, + "fcm_dpo/margin": 1.1795077323913574, + "fcm_dpo/q_t": 0.46056699752807617, + "grad_norm": 39.07538986206055, + "learning_rate": 4.992136939879856e-07, + "logits/chosen": 0.1855728030204773, + "logits/rejected": 0.13604822754859924, + "logps/chosen": -59.18119430541992, + "logps/ref_chosen": -57.15077209472656, + "logps/ref_rejected": -75.1710205078125, + "logps/rejected": -78.38095092773438, + "loss": 1.2443, + "margin_dpo/margin_mean": 1.1795073747634888, + "margin_dpo/margin_std": 1.8788938522338867, + "step": 83 + }, + { + "KL/chosen_KL_mean": -2.489429473876953, + "KL/mean": -2.9202234745025635, + "KL/rejected_KL_mean": -3.351024627685547, + "KL/std": 1.8019747734069824, + "epoch": 0.12698412698412698, + "fcm_dpo/beta": 0.14852453768253326, + "fcm_dpo/delta": 0.2758869230747223, + "fcm_dpo/margin": 0.8615986108779907, + "fcm_dpo/q_t": 0.4696010649204254, + "grad_norm": 44.80130386352539, + "learning_rate": 4.991054231460969e-07, + "logits/chosen": 0.16139745712280273, + "logits/rejected": 0.11974325776100159, + "logps/chosen": -67.26673126220703, + "logps/ref_chosen": -64.77729797363281, + "logps/ref_rejected": -84.71949768066406, + "logps/rejected": -88.07052612304688, + "loss": 1.2825, + "margin_dpo/margin_mean": 0.8615986108779907, + "margin_dpo/margin_std": 1.8895180225372314, + "step": 84 + }, + { + "KL/chosen_KL_mean": -2.2373600006103516, + "KL/mean": -2.8613109588623047, + "KL/rejected_KL_mean": -3.485260009765625, + "KL/std": 1.7679171562194824, + "epoch": 0.12849584278155707, + "fcm_dpo/beta": 0.15582939982414246, + "fcm_dpo/delta": 0.20922988653182983, + "fcm_dpo/margin": 1.247900128364563, + "fcm_dpo/q_t": 0.45365017652511597, + "grad_norm": 39.90945053100586, + "learning_rate": 4.989901842900325e-07, + "logits/chosen": 0.1625782549381256, + "logits/rejected": 0.11865755915641785, + "logps/chosen": -52.48905563354492, + "logps/ref_chosen": -50.25169372558594, + "logps/ref_rejected": -66.55439758300781, + "logps/rejected": -70.03965759277344, + "loss": 1.2256, + "margin_dpo/margin_mean": 1.2479002475738525, + "margin_dpo/margin_std": 1.9751851558685303, + "step": 85 + }, + { + "KL/chosen_KL_mean": -2.76788330078125, + "KL/mean": -3.2456436157226562, + "KL/rejected_KL_mean": -3.7234020233154297, + "KL/std": 1.8651580810546875, + "epoch": 0.13000755857898716, + "fcm_dpo/beta": 0.16335007548332214, + "fcm_dpo/delta": 0.2474798560142517, + "fcm_dpo/margin": 0.9555222988128662, + "fcm_dpo/q_t": 0.4628984332084656, + "grad_norm": 42.055274963378906, + "learning_rate": 4.988679806432711e-07, + "logits/chosen": 0.18018244206905365, + "logits/rejected": 0.16184869408607483, + "logps/chosen": -63.49706268310547, + "logps/ref_chosen": -60.72917938232422, + "logps/ref_rejected": -72.30961608886719, + "logps/rejected": -76.03301239013672, + "loss": 1.2629, + "margin_dpo/margin_mean": 0.955522894859314, + "margin_dpo/margin_std": 2.0158450603485107, + "step": 86 + }, + { + "KL/chosen_KL_mean": -2.952251434326172, + "KL/mean": -3.472977638244629, + "KL/rejected_KL_mean": -3.9937057495117188, + "KL/std": 1.9867116212844849, + "epoch": 0.13151927437641722, + "fcm_dpo/beta": 0.17066358029842377, + "fcm_dpo/delta": 0.2265702486038208, + "fcm_dpo/margin": 1.0414538383483887, + "fcm_dpo/q_t": 0.458102822303772, + "grad_norm": 51.16208267211914, + "learning_rate": 4.987388156241114e-07, + "logits/chosen": 0.17016345262527466, + "logits/rejected": 0.1106840968132019, + "logps/chosen": -68.71022033691406, + "logps/ref_chosen": -65.75796508789062, + "logps/ref_rejected": -84.81159973144531, + "logps/rejected": -88.8052978515625, + "loss": 1.2572, + "margin_dpo/margin_mean": 1.0414538383483887, + "margin_dpo/margin_std": 2.364213466644287, + "step": 87 + }, + { + "KL/chosen_KL_mean": -2.958810806274414, + "KL/mean": -3.3786325454711914, + "KL/rejected_KL_mean": -3.7984542846679688, + "KL/std": 2.2283806800842285, + "epoch": 0.1330309901738473, + "fcm_dpo/beta": 0.17520461976528168, + "fcm_dpo/delta": 0.06729626655578613, + "fcm_dpo/margin": 0.8396420478820801, + "fcm_dpo/q_t": 0.46605074405670166, + "grad_norm": 54.265357971191406, + "learning_rate": 4.986026928455767e-07, + "logits/chosen": 0.24361751973628998, + "logits/rejected": 0.2162962704896927, + "logps/chosen": -65.7828369140625, + "logps/ref_chosen": -62.82402801513672, + "logps/ref_rejected": -74.9607162475586, + "logps/rejected": -78.75917053222656, + "loss": 1.3089, + "margin_dpo/margin_mean": 0.8396422863006592, + "margin_dpo/margin_std": 2.872804641723633, + "step": 88 + }, + { + "KL/chosen_KL_mean": -2.457714080810547, + "KL/mean": -3.142911672592163, + "KL/rejected_KL_mean": -3.8281097412109375, + "KL/std": 2.082172393798828, + "epoch": 0.1345427059712774, + "fcm_dpo/beta": 0.18018333613872528, + "fcm_dpo/delta": 0.1557571142911911, + "fcm_dpo/margin": 1.370391845703125, + "fcm_dpo/q_t": 0.44298940896987915, + "grad_norm": 48.1218376159668, + "learning_rate": 4.984596161153135e-07, + "logits/chosen": 0.2617233097553253, + "logits/rejected": 0.17753329873085022, + "logps/chosen": -43.649147033691406, + "logps/ref_chosen": -41.191436767578125, + "logps/ref_rejected": -85.44769287109375, + "logps/rejected": -89.27580261230469, + "loss": 1.2025, + "margin_dpo/margin_mean": 1.370391845703125, + "margin_dpo/margin_std": 2.4326417446136475, + "step": 89 + }, + { + "KL/chosen_KL_mean": -2.9885406494140625, + "KL/mean": -3.592991828918457, + "KL/rejected_KL_mean": -4.19744873046875, + "KL/std": 2.275628089904785, + "epoch": 0.1360544217687075, + "fcm_dpo/beta": 0.18507075309753418, + "fcm_dpo/delta": 0.18068405985832214, + "fcm_dpo/margin": 1.208900809288025, + "fcm_dpo/q_t": 0.4476126432418823, + "grad_norm": 50.888099670410156, + "learning_rate": 4.983095894354857e-07, + "logits/chosen": 0.13500599563121796, + "logits/rejected": 0.08118491619825363, + "logps/chosen": -59.57244873046875, + "logps/ref_chosen": -56.58390808105469, + "logps/ref_rejected": -86.86978149414062, + "logps/rejected": -91.06723022460938, + "loss": 1.2211, + "margin_dpo/margin_mean": 1.2089004516601562, + "margin_dpo/margin_std": 2.3021106719970703, + "step": 90 + }, + { + "KL/chosen_KL_mean": -2.9596328735351562, + "KL/mean": -3.510220527648926, + "KL/rejected_KL_mean": -4.0608062744140625, + "KL/std": 2.125333309173584, + "epoch": 0.13756613756613756, + "fcm_dpo/beta": 0.19327741861343384, + "fcm_dpo/delta": 0.1906760036945343, + "fcm_dpo/margin": 1.1011730432510376, + "fcm_dpo/q_t": 0.45179492235183716, + "grad_norm": 48.79199981689453, + "learning_rate": 4.98152617002662e-07, + "logits/chosen": 0.16606345772743225, + "logits/rejected": 0.12146725505590439, + "logps/chosen": -55.341976165771484, + "logps/ref_chosen": -52.38234329223633, + "logps/ref_rejected": -72.17642211914062, + "logps/rejected": -76.23722839355469, + "loss": 1.2445, + "margin_dpo/margin_mean": 1.101172924041748, + "margin_dpo/margin_std": 2.5953307151794434, + "step": 91 + }, + { + "KL/chosen_KL_mean": -3.0597572326660156, + "KL/mean": -3.790630340576172, + "KL/rejected_KL_mean": -4.521511077880859, + "KL/std": 2.433997631072998, + "epoch": 0.13907785336356765, + "fcm_dpo/beta": 0.19871735572814941, + "fcm_dpo/delta": 0.11208958178758621, + "fcm_dpo/margin": 1.4617501497268677, + "fcm_dpo/q_t": 0.43508481979370117, + "grad_norm": 47.62337875366211, + "learning_rate": 4.979887032076988e-07, + "logits/chosen": 0.23759877681732178, + "logits/rejected": 0.195323646068573, + "logps/chosen": -56.068458557128906, + "logps/ref_chosen": -53.00870132446289, + "logps/ref_rejected": -79.77812957763672, + "logps/rejected": -84.29963684082031, + "loss": 1.1943, + "margin_dpo/margin_mean": 1.4617503881454468, + "margin_dpo/margin_std": 2.8890533447265625, + "step": 92 + }, + { + "KL/chosen_KL_mean": -3.3475303649902344, + "KL/mean": -3.8043527603149414, + "KL/rejected_KL_mean": -4.26116943359375, + "KL/std": 2.362985849380493, + "epoch": 0.14058956916099774, + "fcm_dpo/beta": 0.2060101330280304, + "fcm_dpo/delta": 0.21536466479301453, + "fcm_dpo/margin": 0.9136420488357544, + "fcm_dpo/q_t": 0.45910483598709106, + "grad_norm": 49.81450271606445, + "learning_rate": 4.978178526356172e-07, + "logits/chosen": 0.20445819199085236, + "logits/rejected": 0.17469534277915955, + "logps/chosen": -48.25458526611328, + "logps/ref_chosen": -44.90705108642578, + "logps/ref_rejected": -58.7879524230957, + "logps/rejected": -63.04912185668945, + "loss": 1.2869, + "margin_dpo/margin_mean": 0.9136421084403992, + "margin_dpo/margin_std": 2.857240676879883, + "step": 93 + }, + { + "KL/chosen_KL_mean": -2.9746627807617188, + "KL/mean": -3.8127119541168213, + "KL/rejected_KL_mean": -4.650768280029297, + "KL/std": 2.772942543029785, + "epoch": 0.1421012849584278, + "fcm_dpo/beta": 0.20925289392471313, + "fcm_dpo/delta": 0.051092229783535004, + "fcm_dpo/margin": 1.6761001348495483, + "fcm_dpo/q_t": 0.4200799763202667, + "grad_norm": 50.38736343383789, + "learning_rate": 4.976400700654751e-07, + "logits/chosen": 0.2535303235054016, + "logits/rejected": 0.21268761157989502, + "logps/chosen": -62.91243362426758, + "logps/ref_chosen": -59.93777084350586, + "logps/ref_rejected": -79.3138427734375, + "logps/rejected": -83.96460723876953, + "loss": 1.1828, + "margin_dpo/margin_mean": 1.6761000156402588, + "margin_dpo/margin_std": 3.414187431335449, + "step": 94 + }, + { + "KL/chosen_KL_mean": -3.832956314086914, + "KL/mean": -4.543184757232666, + "KL/rejected_KL_mean": -5.253414154052734, + "KL/std": 2.3406383991241455, + "epoch": 0.1436130007558579, + "fcm_dpo/beta": 0.21277236938476562, + "fcm_dpo/delta": 0.10093221068382263, + "fcm_dpo/margin": 1.4204583168029785, + "fcm_dpo/q_t": 0.4322485625743866, + "grad_norm": 59.44841384887695, + "learning_rate": 4.974553604702332e-07, + "logits/chosen": 0.13966067135334015, + "logits/rejected": 0.07558946311473846, + "logps/chosen": -64.00144958496094, + "logps/ref_chosen": -60.168487548828125, + "logps/ref_rejected": -90.73665618896484, + "logps/rejected": -95.99006652832031, + "loss": 1.198, + "margin_dpo/margin_mean": 1.4204577207565308, + "margin_dpo/margin_std": 2.919649124145508, + "step": 95 + }, + { + "KL/chosen_KL_mean": -3.674968719482422, + "KL/mean": -4.511168479919434, + "KL/rejected_KL_mean": -5.3473663330078125, + "KL/std": 2.884733200073242, + "epoch": 0.14512471655328799, + "fcm_dpo/beta": 0.21607878804206848, + "fcm_dpo/delta": 0.03986040875315666, + "fcm_dpo/margin": 1.672404408454895, + "fcm_dpo/q_t": 0.42547816038131714, + "grad_norm": 57.87812042236328, + "learning_rate": 4.972637290166157e-07, + "logits/chosen": 0.16100308299064636, + "logits/rejected": 0.11939145624637604, + "logps/chosen": -64.34375, + "logps/ref_chosen": -60.66877746582031, + "logps/ref_rejected": -88.30673217773438, + "logps/rejected": -93.65409851074219, + "loss": 1.2024, + "margin_dpo/margin_mean": 1.6724035739898682, + "margin_dpo/margin_std": 3.720932722091675, + "step": 96 + }, + { + "KL/chosen_KL_mean": -4.3227691650390625, + "KL/mean": -4.781896114349365, + "KL/rejected_KL_mean": -5.24102783203125, + "KL/std": 2.8743977546691895, + "epoch": 0.14663643235071808, + "fcm_dpo/beta": 0.22158443927764893, + "fcm_dpo/delta": 0.20110613107681274, + "fcm_dpo/margin": 0.9182647466659546, + "fcm_dpo/q_t": 0.45814549922943115, + "grad_norm": 86.01407623291016, + "learning_rate": 4.970651810649666e-07, + "logits/chosen": 0.11800533533096313, + "logits/rejected": 0.07440978288650513, + "logps/chosen": -69.36688995361328, + "logps/ref_chosen": -65.04412078857422, + "logps/ref_rejected": -78.42092895507812, + "logps/rejected": -83.66195678710938, + "loss": 1.3629, + "margin_dpo/margin_mean": 0.9182642698287964, + "margin_dpo/margin_std": 3.942337989807129, + "step": 97 + }, + { + "KL/chosen_KL_mean": -3.8495330810546875, + "KL/mean": -4.211657524108887, + "KL/rejected_KL_mean": -4.573783874511719, + "KL/std": 2.5712876319885254, + "epoch": 0.14814814814814814, + "fcm_dpo/beta": 0.2320966124534607, + "fcm_dpo/delta": 0.23647648096084595, + "fcm_dpo/margin": 0.7242507934570312, + "fcm_dpo/q_t": 0.4608962833881378, + "grad_norm": 72.66352081298828, + "learning_rate": 4.968597221690985e-07, + "logits/chosen": 0.20253103971481323, + "logits/rejected": 0.175521582365036, + "logps/chosen": -59.35276412963867, + "logps/ref_chosen": -55.503231048583984, + "logps/ref_rejected": -72.81553649902344, + "logps/rejected": -77.38932800292969, + "loss": 1.3091, + "margin_dpo/margin_mean": 0.7242506742477417, + "margin_dpo/margin_std": 2.5094985961914062, + "step": 98 + }, + { + "KL/chosen_KL_mean": -3.8535900115966797, + "KL/mean": -4.35680627822876, + "KL/rejected_KL_mean": -4.860027313232422, + "KL/std": 2.769084930419922, + "epoch": 0.14965986394557823, + "fcm_dpo/beta": 0.24122494459152222, + "fcm_dpo/delta": 0.16129615902900696, + "fcm_dpo/margin": 1.0064336061477661, + "fcm_dpo/q_t": 0.44870877265930176, + "grad_norm": 85.34445190429688, + "learning_rate": 4.966473580761389e-07, + "logits/chosen": 0.2370331585407257, + "logits/rejected": 0.1967862844467163, + "logps/chosen": -62.429229736328125, + "logps/ref_chosen": -58.57563781738281, + "logps/ref_rejected": -78.693603515625, + "logps/rejected": -83.55363464355469, + "loss": 1.3245, + "margin_dpo/margin_mean": 1.006433129310608, + "margin_dpo/margin_std": 3.5495963096618652, + "step": 99 + }, + { + "KL/chosen_KL_mean": -4.311008453369141, + "KL/mean": -4.901121139526367, + "KL/rejected_KL_mean": -5.491233825683594, + "KL/std": 3.0689799785614014, + "epoch": 0.15117157974300832, + "fcm_dpo/beta": 0.2495937943458557, + "fcm_dpo/delta": 0.10603199899196625, + "fcm_dpo/margin": 1.180228352546692, + "fcm_dpo/q_t": 0.4410404562950134, + "grad_norm": 86.84514617919922, + "learning_rate": 4.964280947263676e-07, + "logits/chosen": 0.21110297739505768, + "logits/rejected": 0.2032468616962433, + "logps/chosen": -83.89443969726562, + "logps/ref_chosen": -79.58343505859375, + "logps/ref_rejected": -92.152587890625, + "logps/rejected": -97.64381408691406, + "loss": 1.3697, + "margin_dpo/margin_mean": 1.1802282333374023, + "margin_dpo/margin_std": 4.390231132507324, + "step": 100 + }, + { + "KL/chosen_KL_mean": -3.517528533935547, + "KL/mean": -4.3843183517456055, + "KL/rejected_KL_mean": -5.251110076904297, + "KL/std": 2.818021059036255, + "epoch": 0.15268329554043839, + "fcm_dpo/beta": 0.24841973185539246, + "fcm_dpo/delta": -0.03219292312860489, + "fcm_dpo/margin": 1.7335816621780396, + "fcm_dpo/q_t": 0.40388786792755127, + "grad_norm": 62.59505844116211, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": 0.21590465307235718, + "logits/rejected": 0.1761970967054367, + "logps/chosen": -55.85031509399414, + "logps/ref_chosen": -52.332786560058594, + "logps/ref_rejected": -69.55589294433594, + "logps/rejected": -74.8070068359375, + "loss": 1.1389, + "margin_dpo/margin_mean": 1.7335816621780396, + "margin_dpo/margin_std": 3.1004440784454346, + "step": 101 + }, + { + "KL/chosen_KL_mean": -4.192995071411133, + "KL/mean": -4.606147766113281, + "KL/rejected_KL_mean": -5.019306182861328, + "KL/std": 2.706145763397217, + "epoch": 0.15419501133786848, + "fcm_dpo/beta": 0.2551325857639313, + "fcm_dpo/delta": 0.19242900609970093, + "fcm_dpo/margin": 0.8263107538223267, + "fcm_dpo/q_t": 0.46237632632255554, + "grad_norm": 89.90907287597656, + "learning_rate": 4.959688949822748e-07, + "logits/chosen": 0.12861177325248718, + "logits/rejected": 0.08954156935214996, + "logps/chosen": -68.93647766113281, + "logps/ref_chosen": -64.74348449707031, + "logps/ref_rejected": -69.06132507324219, + "logps/rejected": -74.08063507080078, + "loss": 1.3995, + "margin_dpo/margin_mean": 0.8263108730316162, + "margin_dpo/margin_std": 3.9148406982421875, + "step": 102 + }, + { + "KL/chosen_KL_mean": -3.756814956665039, + "KL/mean": -4.423976898193359, + "KL/rejected_KL_mean": -5.0911407470703125, + "KL/std": 2.8079519271850586, + "epoch": 0.15570672713529857, + "fcm_dpo/beta": 0.2602066993713379, + "fcm_dpo/delta": 0.053878165781497955, + "fcm_dpo/margin": 1.3343229293823242, + "fcm_dpo/q_t": 0.4253695607185364, + "grad_norm": 76.40733337402344, + "learning_rate": 4.957289714327572e-07, + "logits/chosen": 0.23914015293121338, + "logits/rejected": 0.20504045486450195, + "logps/chosen": -67.59346008300781, + "logps/ref_chosen": -63.83664321899414, + "logps/ref_rejected": -79.32362365722656, + "logps/rejected": -84.41476440429688, + "loss": 1.1951, + "margin_dpo/margin_mean": 1.3343226909637451, + "margin_dpo/margin_std": 2.8255200386047363, + "step": 103 + }, + { + "KL/chosen_KL_mean": -3.706247329711914, + "KL/mean": -4.552823066711426, + "KL/rejected_KL_mean": -5.399406433105469, + "KL/std": 3.520082950592041, + "epoch": 0.15721844293272866, + "fcm_dpo/beta": 0.25901108980178833, + "fcm_dpo/delta": -0.04045912250876427, + "fcm_dpo/margin": 1.6931569576263428, + "fcm_dpo/q_t": 0.41955292224884033, + "grad_norm": 84.37599182128906, + "learning_rate": 4.954821743156767e-07, + "logits/chosen": 0.23270666599273682, + "logits/rejected": 0.14419865608215332, + "logps/chosen": -64.70545959472656, + "logps/ref_chosen": -60.99920654296875, + "logps/ref_rejected": -98.84645080566406, + "logps/rejected": -104.24586486816406, + "loss": 1.243, + "margin_dpo/margin_mean": 1.6931579113006592, + "margin_dpo/margin_std": 4.295382499694824, + "step": 104 + }, + { + "KL/chosen_KL_mean": -3.97216796875, + "KL/mean": -4.591570854187012, + "KL/rejected_KL_mean": -5.210971832275391, + "KL/std": 2.5960726737976074, + "epoch": 0.15873015873015872, + "fcm_dpo/beta": 0.26095521450042725, + "fcm_dpo/delta": 0.07911492139101028, + "fcm_dpo/margin": 1.238802194595337, + "fcm_dpo/q_t": 0.4329737424850464, + "grad_norm": 80.13673400878906, + "learning_rate": 4.952285105344791e-07, + "logits/chosen": 0.15176521241664886, + "logits/rejected": 0.09743764251470566, + "logps/chosen": -74.92243957519531, + "logps/ref_chosen": -70.95027160644531, + "logps/ref_rejected": -87.88340759277344, + "logps/rejected": -93.0943832397461, + "loss": 1.2338, + "margin_dpo/margin_mean": 1.2388018369674683, + "margin_dpo/margin_std": 3.063159227371216, + "step": 105 + }, + { + "KL/chosen_KL_mean": -3.693246841430664, + "KL/mean": -4.331310749053955, + "KL/rejected_KL_mean": -4.969371795654297, + "KL/std": 2.6151795387268066, + "epoch": 0.1602418745275888, + "fcm_dpo/beta": 0.2641947269439697, + "fcm_dpo/delta": 0.06503366678953171, + "fcm_dpo/margin": 1.276125192642212, + "fcm_dpo/q_t": 0.4248006343841553, + "grad_norm": 84.72764587402344, + "learning_rate": 4.949679871846857e-07, + "logits/chosen": 0.2076093852519989, + "logits/rejected": 0.193533256649971, + "logps/chosen": -66.15258026123047, + "logps/ref_chosen": -62.45933151245117, + "logps/ref_rejected": -67.00595092773438, + "logps/rejected": -71.9753189086914, + "loss": 1.2583, + "margin_dpo/margin_mean": 1.276125192642212, + "margin_dpo/margin_std": 3.409266710281372, + "step": 106 + }, + { + "KL/chosen_KL_mean": -4.450431823730469, + "KL/mean": -4.831617832183838, + "KL/rejected_KL_mean": -5.212799072265625, + "KL/std": 3.1901092529296875, + "epoch": 0.1617535903250189, + "fcm_dpo/beta": 0.2652558982372284, + "fcm_dpo/delta": 0.004532136023044586, + "fcm_dpo/margin": 0.7623693943023682, + "fcm_dpo/q_t": 0.45172134041786194, + "grad_norm": 115.32234191894531, + "learning_rate": 4.947006115536947e-07, + "logits/chosen": 0.12762564420700073, + "logits/rejected": 0.10460235178470612, + "logps/chosen": -80.28839874267578, + "logps/ref_chosen": -75.83796691894531, + "logps/ref_rejected": -87.74038696289062, + "logps/rejected": -92.95318603515625, + "loss": 1.434, + "margin_dpo/margin_mean": 0.7623691558837891, + "margin_dpo/margin_std": 3.885159492492676, + "step": 107 + }, + { + "KL/chosen_KL_mean": -3.832916259765625, + "KL/mean": -4.541226863861084, + "KL/rejected_KL_mean": -5.249538421630859, + "KL/std": 2.5288538932800293, + "epoch": 0.16326530612244897, + "fcm_dpo/beta": 0.2664978504180908, + "fcm_dpo/delta": 0.02329789660871029, + "fcm_dpo/margin": 1.4166216850280762, + "fcm_dpo/q_t": 0.42314743995666504, + "grad_norm": 75.23424530029297, + "learning_rate": 4.944263911205772e-07, + "logits/chosen": 0.11665268242359161, + "logits/rejected": 0.08691516518592834, + "logps/chosen": -72.22615051269531, + "logps/ref_chosen": -68.39323425292969, + "logps/ref_rejected": -83.24267578125, + "logps/rejected": -88.4922103881836, + "loss": 1.2263, + "margin_dpo/margin_mean": 1.416621446609497, + "margin_dpo/margin_std": 3.3912062644958496, + "step": 108 + }, + { + "KL/chosen_KL_mean": -3.450235366821289, + "KL/mean": -4.449607849121094, + "KL/rejected_KL_mean": -5.448982238769531, + "KL/std": 2.9472644329071045, + "epoch": 0.16477702191987906, + "fcm_dpo/beta": 0.26176708936691284, + "fcm_dpo/delta": -0.12986820936203003, + "fcm_dpo/margin": 1.9987459182739258, + "fcm_dpo/q_t": 0.3933427035808563, + "grad_norm": 65.35698699951172, + "learning_rate": 4.941453335558681e-07, + "logits/chosen": 0.15471996366977692, + "logits/rejected": 0.09994255751371384, + "logps/chosen": -58.97771453857422, + "logps/ref_chosen": -55.52748107910156, + "logps/ref_rejected": -83.55218505859375, + "logps/rejected": -89.00117492675781, + "loss": 1.0639, + "margin_dpo/margin_mean": 1.9987448453903198, + "margin_dpo/margin_std": 3.154825210571289, + "step": 109 + }, + { + "KL/chosen_KL_mean": -4.1198272705078125, + "KL/mean": -4.419821739196777, + "KL/rejected_KL_mean": -4.719810485839844, + "KL/std": 2.4698705673217773, + "epoch": 0.16628873771730915, + "fcm_dpo/beta": 0.26964741945266724, + "fcm_dpo/delta": 0.24171972274780273, + "fcm_dpo/margin": 0.5999836921691895, + "fcm_dpo/q_t": 0.46288520097732544, + "grad_norm": 109.37680053710938, + "learning_rate": 4.938574467213517e-07, + "logits/chosen": 0.13163237273693085, + "logits/rejected": 0.14082229137420654, + "logps/chosen": -85.27857208251953, + "logps/ref_chosen": -81.15874481201172, + "logps/ref_rejected": -72.56021118164062, + "logps/rejected": -77.28002166748047, + "loss": 1.3957, + "margin_dpo/margin_mean": 0.5999835729598999, + "margin_dpo/margin_std": 3.180210590362549, + "step": 110 + }, + { + "KL/chosen_KL_mean": -3.9667015075683594, + "KL/mean": -4.667611122131348, + "KL/rejected_KL_mean": -5.368522644042969, + "KL/std": 2.5095343589782715, + "epoch": 0.16780045351473924, + "fcm_dpo/beta": 0.27373117208480835, + "fcm_dpo/delta": 0.016820773482322693, + "fcm_dpo/margin": 1.4018200635910034, + "fcm_dpo/q_t": 0.4187893867492676, + "grad_norm": 69.44522857666016, + "learning_rate": 4.935627386698418e-07, + "logits/chosen": 0.24480414390563965, + "logits/rejected": 0.20698174834251404, + "logps/chosen": -56.32568359375, + "logps/ref_chosen": -52.358985900878906, + "logps/ref_rejected": -77.06150817871094, + "logps/rejected": -82.43002319335938, + "loss": 1.1883, + "margin_dpo/margin_mean": 1.4018198251724243, + "margin_dpo/margin_std": 2.955274820327759, + "step": 111 + }, + { + "KL/chosen_KL_mean": -3.5427474975585938, + "KL/mean": -4.380864143371582, + "KL/rejected_KL_mean": -5.218982696533203, + "KL/std": 2.5112671852111816, + "epoch": 0.1693121693121693, + "fcm_dpo/beta": 0.2712618112564087, + "fcm_dpo/delta": -0.05728024244308472, + "fcm_dpo/margin": 1.6762354373931885, + "fcm_dpo/q_t": 0.39894920587539673, + "grad_norm": 79.7928237915039, + "learning_rate": 4.932612176449559e-07, + "logits/chosen": 0.15141016244888306, + "logits/rejected": 0.08643672615289688, + "logps/chosen": -66.56281280517578, + "logps/ref_chosen": -63.02006530761719, + "logps/ref_rejected": -111.36941528320312, + "logps/rejected": -116.58839416503906, + "loss": 1.1195, + "margin_dpo/margin_mean": 1.6762347221374512, + "margin_dpo/margin_std": 2.8349223136901855, + "step": 112 + }, + { + "KL/chosen_KL_mean": -3.744342803955078, + "KL/mean": -4.343097686767578, + "KL/rejected_KL_mean": -4.941852569580078, + "KL/std": 2.3790249824523926, + "epoch": 0.1708238851095994, + "fcm_dpo/beta": 0.27505138516426086, + "fcm_dpo/delta": 0.07165899127721786, + "fcm_dpo/margin": 1.1975092887878418, + "fcm_dpo/q_t": 0.42461806535720825, + "grad_norm": 87.63443756103516, + "learning_rate": 4.929528920808854e-07, + "logits/chosen": 0.16694726049900055, + "logits/rejected": 0.12760739028453827, + "logps/chosen": -59.552005767822266, + "logps/ref_chosen": -55.80766296386719, + "logps/ref_rejected": -69.84014129638672, + "logps/rejected": -74.78199768066406, + "loss": 1.2267, + "margin_dpo/margin_mean": 1.1975088119506836, + "margin_dpo/margin_std": 2.7592105865478516, + "step": 113 + }, + { + "KL/chosen_KL_mean": -3.324331283569336, + "KL/mean": -4.2867326736450195, + "KL/rejected_KL_mean": -5.249141693115234, + "KL/std": 2.531193733215332, + "epoch": 0.17233560090702948, + "fcm_dpo/beta": 0.2680574655532837, + "fcm_dpo/delta": -0.12321934103965759, + "fcm_dpo/margin": 1.9248015880584717, + "fcm_dpo/q_t": 0.3910922408103943, + "grad_norm": 68.17572784423828, + "learning_rate": 4.92637770602159e-07, + "logits/chosen": 0.2011614441871643, + "logits/rejected": 0.14068567752838135, + "logps/chosen": -69.6571044921875, + "logps/ref_chosen": -66.33277130126953, + "logps/ref_rejected": -71.61489868164062, + "logps/rejected": -76.8640365600586, + "loss": 1.0673, + "margin_dpo/margin_mean": 1.9248019456863403, + "margin_dpo/margin_std": 2.92665958404541, + "step": 114 + }, + { + "KL/chosen_KL_mean": -3.7208175659179688, + "KL/mean": -4.421115875244141, + "KL/rejected_KL_mean": -5.121416091918945, + "KL/std": 2.2531988620758057, + "epoch": 0.17384731670445955, + "fcm_dpo/beta": 0.2685314416885376, + "fcm_dpo/delta": 0.024830270558595657, + "fcm_dpo/margin": 1.4005990028381348, + "fcm_dpo/q_t": 0.4187403619289398, + "grad_norm": 74.76184844970703, + "learning_rate": 4.923158620234019e-07, + "logits/chosen": 0.18886233866214752, + "logits/rejected": 0.12659035623073578, + "logps/chosen": -59.46985626220703, + "logps/ref_chosen": -55.74903869628906, + "logps/ref_rejected": -79.59849548339844, + "logps/rejected": -84.71990966796875, + "loss": 1.1603, + "margin_dpo/margin_mean": 1.4005998373031616, + "margin_dpo/margin_std": 2.6480507850646973, + "step": 115 + }, + { + "KL/chosen_KL_mean": -3.4787425994873047, + "KL/mean": -4.41444206237793, + "KL/rejected_KL_mean": -5.350147247314453, + "KL/std": 2.367363452911377, + "epoch": 0.17535903250188964, + "fcm_dpo/beta": 0.2657136917114258, + "fcm_dpo/delta": -0.10223374515771866, + "fcm_dpo/margin": 1.8714066743850708, + "fcm_dpo/q_t": 0.38647329807281494, + "grad_norm": 58.41557312011719, + "learning_rate": 4.91987175349089e-07, + "logits/chosen": 0.19088421761989594, + "logits/rejected": 0.12268730252981186, + "logps/chosen": -52.843910217285156, + "logps/ref_chosen": -49.36516571044922, + "logps/ref_rejected": -72.84671020507812, + "logps/rejected": -78.19685363769531, + "loss": 1.0349, + "margin_dpo/margin_mean": 1.8714067935943604, + "margin_dpo/margin_std": 2.270500659942627, + "step": 116 + }, + { + "KL/chosen_KL_mean": -3.357513427734375, + "KL/mean": -4.0659332275390625, + "KL/rejected_KL_mean": -4.77435302734375, + "KL/std": 2.205228328704834, + "epoch": 0.17687074829931973, + "fcm_dpo/beta": 0.261690229177475, + "fcm_dpo/delta": 0.02821500599384308, + "fcm_dpo/margin": 1.4168295860290527, + "fcm_dpo/q_t": 0.41594117879867554, + "grad_norm": 67.27315521240234, + "learning_rate": 4.916517197732933e-07, + "logits/chosen": 0.16953285038471222, + "logits/rejected": 0.1338912546634674, + "logps/chosen": -61.06841278076172, + "logps/ref_chosen": -57.710899353027344, + "logps/ref_rejected": -69.77253723144531, + "logps/rejected": -74.54689025878906, + "loss": 1.1649, + "margin_dpo/margin_mean": 1.4168298244476318, + "margin_dpo/margin_std": 2.5413122177124023, + "step": 117 + }, + { + "KL/chosen_KL_mean": -3.2425918579101562, + "KL/mean": -4.178920745849609, + "KL/rejected_KL_mean": -5.115245819091797, + "KL/std": 2.3707337379455566, + "epoch": 0.17838246409674982, + "fcm_dpo/beta": 0.2604265809059143, + "fcm_dpo/delta": -0.09273022413253784, + "fcm_dpo/margin": 1.8726553916931152, + "fcm_dpo/q_t": 0.3897615075111389, + "grad_norm": 65.98107147216797, + "learning_rate": 4.913095046794281e-07, + "logits/chosen": 0.24106436967849731, + "logits/rejected": 0.20028874278068542, + "logps/chosen": -55.72248840332031, + "logps/ref_chosen": -52.479896545410156, + "logps/ref_rejected": -81.359130859375, + "logps/rejected": -86.47437286376953, + "loss": 1.0587, + "margin_dpo/margin_mean": 1.872654914855957, + "margin_dpo/margin_std": 2.531564712524414, + "step": 118 + }, + { + "KL/chosen_KL_mean": -4.147886276245117, + "KL/mean": -4.786754608154297, + "KL/rejected_KL_mean": -5.425624847412109, + "KL/std": 2.504222869873047, + "epoch": 0.17989417989417988, + "fcm_dpo/beta": 0.26322388648986816, + "fcm_dpo/delta": 0.06562946736812592, + "fcm_dpo/margin": 1.2777378559112549, + "fcm_dpo/q_t": 0.4259907603263855, + "grad_norm": 72.88529968261719, + "learning_rate": 4.909605396399855e-07, + "logits/chosen": 0.16665619611740112, + "logits/rejected": 0.12804852426052094, + "logps/chosen": -65.50556182861328, + "logps/ref_chosen": -61.35767364501953, + "logps/ref_rejected": -75.71510314941406, + "logps/rejected": -81.1407241821289, + "loss": 1.2262, + "margin_dpo/margin_mean": 1.2777388095855713, + "margin_dpo/margin_std": 3.0291521549224854, + "step": 119 + }, + { + "KL/chosen_KL_mean": -3.4492645263671875, + "KL/mean": -4.388361930847168, + "KL/rejected_KL_mean": -5.327457427978516, + "KL/std": 2.3779149055480957, + "epoch": 0.18140589569160998, + "fcm_dpo/beta": 0.2602246403694153, + "fcm_dpo/delta": -0.09322664141654968, + "fcm_dpo/margin": 1.8781991004943848, + "fcm_dpo/q_t": 0.39164602756500244, + "grad_norm": 64.43611907958984, + "learning_rate": 4.906048344162676e-07, + "logits/chosen": 0.17029550671577454, + "logits/rejected": 0.11013634502887726, + "logps/chosen": -63.356834411621094, + "logps/ref_chosen": -59.907569885253906, + "logps/ref_rejected": -79.6910629272461, + "logps/rejected": -85.01852416992188, + "loss": 1.0494, + "margin_dpo/margin_mean": 1.8781991004943848, + "margin_dpo/margin_std": 2.4651622772216797, + "step": 120 + }, + { + "KL/chosen_KL_mean": -3.789203643798828, + "KL/mean": -4.682255268096924, + "KL/rejected_KL_mean": -5.575309753417969, + "KL/std": 2.5316545963287354, + "epoch": 0.18291761148904007, + "fcm_dpo/beta": 0.2558312714099884, + "fcm_dpo/delta": -0.05986927077174187, + "fcm_dpo/margin": 1.786109209060669, + "fcm_dpo/q_t": 0.401405930519104, + "grad_norm": 59.51321792602539, + "learning_rate": 4.902423989581143e-07, + "logits/chosen": 0.2673395872116089, + "logits/rejected": 0.18181562423706055, + "logps/chosen": -59.45524597167969, + "logps/ref_chosen": -55.66604232788086, + "logps/ref_rejected": -101.56233978271484, + "logps/rejected": -107.13764953613281, + "loss": 1.0761, + "margin_dpo/margin_mean": 1.7861084938049316, + "margin_dpo/margin_std": 2.6160924434661865, + "step": 121 + }, + { + "KL/chosen_KL_mean": -3.9761314392089844, + "KL/mean": -4.964360237121582, + "KL/rejected_KL_mean": -5.9525909423828125, + "KL/std": 2.7434768676757812, + "epoch": 0.18442932728647016, + "fcm_dpo/beta": 0.25225332379341125, + "fcm_dpo/delta": -0.10360904037952423, + "fcm_dpo/margin": 1.976454734802246, + "fcm_dpo/q_t": 0.394494891166687, + "grad_norm": 70.2988510131836, + "learning_rate": 4.898732434036243e-07, + "logits/chosen": 0.17670899629592896, + "logits/rejected": 0.1416880190372467, + "logps/chosen": -67.31050109863281, + "logps/ref_chosen": -63.334373474121094, + "logps/ref_rejected": -73.67523193359375, + "logps/rejected": -79.62782287597656, + "loss": 1.0731, + "margin_dpo/margin_mean": 1.9764546155929565, + "margin_dpo/margin_std": 3.0363006591796875, + "step": 122 + }, + { + "KL/chosen_KL_mean": -3.6747913360595703, + "KL/mean": -4.590383529663086, + "KL/rejected_KL_mean": -5.505973815917969, + "KL/std": 2.4913928508758545, + "epoch": 0.18594104308390022, + "fcm_dpo/beta": 0.2495909333229065, + "fcm_dpo/delta": -0.0601581409573555, + "fcm_dpo/margin": 1.8311854600906372, + "fcm_dpo/q_t": 0.39790278673171997, + "grad_norm": 61.7016487121582, + "learning_rate": 4.894973780788722e-07, + "logits/chosen": 0.18975844979286194, + "logits/rejected": 0.14888188242912292, + "logps/chosen": -60.57353210449219, + "logps/ref_chosen": -56.89874267578125, + "logps/ref_rejected": -78.97028350830078, + "logps/rejected": -84.47625732421875, + "loss": 1.0911, + "margin_dpo/margin_mean": 1.8311853408813477, + "margin_dpo/margin_std": 2.7468996047973633, + "step": 123 + }, + { + "KL/chosen_KL_mean": -3.9783401489257812, + "KL/mean": -5.216902256011963, + "KL/rejected_KL_mean": -6.455467224121094, + "KL/std": 2.7113471031188965, + "epoch": 0.1874527588813303, + "fcm_dpo/beta": 0.2392091453075409, + "fcm_dpo/delta": -0.20544171333312988, + "fcm_dpo/margin": 2.4771270751953125, + "fcm_dpo/q_t": 0.3674274981021881, + "grad_norm": 53.000423431396484, + "learning_rate": 4.89114813497619e-07, + "logits/chosen": 0.21945105493068695, + "logits/rejected": 0.15901131927967072, + "logps/chosen": -61.094425201416016, + "logps/ref_chosen": -57.116085052490234, + "logps/ref_rejected": -87.93074035644531, + "logps/rejected": -94.38621520996094, + "loss": 0.9766, + "margin_dpo/margin_mean": 2.4771275520324707, + "margin_dpo/margin_std": 2.7539939880371094, + "step": 124 + }, + { + "KL/chosen_KL_mean": -4.249895095825195, + "KL/mean": -5.264259338378906, + "KL/rejected_KL_mean": -6.278621673583984, + "KL/std": 2.9378597736358643, + "epoch": 0.1889644746787604, + "fcm_dpo/beta": 0.2334146797657013, + "fcm_dpo/delta": -0.07813793420791626, + "fcm_dpo/margin": 2.028723955154419, + "fcm_dpo/q_t": 0.39546871185302734, + "grad_norm": 61.0987548828125, + "learning_rate": 4.887255603610184e-07, + "logits/chosen": 0.23440095782279968, + "logits/rejected": 0.17516063153743744, + "logps/chosen": -69.95606994628906, + "logps/ref_chosen": -65.7061767578125, + "logps/ref_rejected": -91.72711944580078, + "logps/rejected": -98.0057373046875, + "loss": 1.0701, + "margin_dpo/margin_mean": 2.0287246704101562, + "margin_dpo/margin_std": 2.8646774291992188, + "step": 125 + }, + { + "KL/chosen_KL_mean": -3.9711647033691406, + "KL/mean": -4.703935623168945, + "KL/rejected_KL_mean": -5.436708450317383, + "KL/std": 3.10610032081604, + "epoch": 0.19047619047619047, + "fcm_dpo/beta": 0.23558691143989563, + "fcm_dpo/delta": 0.05671154335141182, + "fcm_dpo/margin": 1.4655463695526123, + "fcm_dpo/q_t": 0.4278152883052826, + "grad_norm": 60.16349792480469, + "learning_rate": 4.883296295573176e-07, + "logits/chosen": 0.04985351115465164, + "logits/rejected": 0.0437270849943161, + "logps/chosen": -72.14725494384766, + "logps/ref_chosen": -68.17608642578125, + "logps/ref_rejected": -65.1175537109375, + "logps/rejected": -70.55426025390625, + "loss": 1.2067, + "margin_dpo/margin_mean": 1.4655463695526123, + "margin_dpo/margin_std": 3.381613254547119, + "step": 126 + }, + { + "KL/chosen_KL_mean": -4.2151336669921875, + "KL/mean": -5.367134094238281, + "KL/rejected_KL_mean": -6.519136428833008, + "KL/std": 2.6317625045776367, + "epoch": 0.19198790627362056, + "fcm_dpo/beta": 0.2297043651342392, + "fcm_dpo/delta": -0.13883370161056519, + "fcm_dpo/margin": 2.3039979934692383, + "fcm_dpo/q_t": 0.3804101347923279, + "grad_norm": 55.27705001831055, + "learning_rate": 4.87927032161552e-07, + "logits/chosen": 0.15548180043697357, + "logits/rejected": 0.12521973252296448, + "logps/chosen": -66.09536743164062, + "logps/ref_chosen": -61.88023376464844, + "logps/ref_rejected": -68.46012878417969, + "logps/rejected": -74.97926330566406, + "loss": 1.0161, + "margin_dpo/margin_mean": 2.3039979934692383, + "margin_dpo/margin_std": 2.670165538787842, + "step": 127 + }, + { + "KL/chosen_KL_mean": -4.582956314086914, + "KL/mean": -5.595683574676514, + "KL/rejected_KL_mean": -6.6084136962890625, + "KL/std": 3.187983512878418, + "epoch": 0.19349962207105065, + "fcm_dpo/beta": 0.22704312205314636, + "fcm_dpo/delta": -0.06312703341245651, + "fcm_dpo/margin": 2.02545166015625, + "fcm_dpo/q_t": 0.40157029032707214, + "grad_norm": 59.90847396850586, + "learning_rate": 4.875177794352363e-07, + "logits/chosen": 0.2170131355524063, + "logits/rejected": 0.15855154395103455, + "logps/chosen": -71.29193878173828, + "logps/ref_chosen": -66.708984375, + "logps/ref_rejected": -94.97969055175781, + "logps/rejected": -101.58810424804688, + "loss": 1.1385, + "margin_dpo/margin_mean": 2.025451421737671, + "margin_dpo/margin_std": 3.7224786281585693, + "step": 128 + }, + { + "KL/chosen_KL_mean": -4.919008255004883, + "KL/mean": -5.729151248931885, + "KL/rejected_KL_mean": -6.539299011230469, + "KL/std": 2.8172764778137207, + "epoch": 0.19501133786848074, + "fcm_dpo/beta": 0.2280907928943634, + "fcm_dpo/delta": 0.031591035425662994, + "fcm_dpo/margin": 1.6202887296676636, + "fcm_dpo/q_t": 0.4201734662055969, + "grad_norm": 67.37427520751953, + "learning_rate": 4.871018828260491e-07, + "logits/chosen": 0.1873515546321869, + "logits/rejected": 0.18076498806476593, + "logps/chosen": -70.2578353881836, + "logps/ref_chosen": -65.33882904052734, + "logps/ref_rejected": -68.06109619140625, + "logps/rejected": -74.60038757324219, + "loss": 1.1711, + "margin_dpo/margin_mean": 1.6202887296676636, + "margin_dpo/margin_std": 3.2078969478607178, + "step": 129 + }, + { + "KL/chosen_KL_mean": -4.648164749145508, + "KL/mean": -5.663567543029785, + "KL/rejected_KL_mean": -6.6789703369140625, + "KL/std": 2.797410011291504, + "epoch": 0.1965230536659108, + "fcm_dpo/beta": 0.22417128086090088, + "fcm_dpo/delta": -0.06046818196773529, + "fcm_dpo/margin": 2.0308070182800293, + "fcm_dpo/q_t": 0.39721450209617615, + "grad_norm": 64.58387756347656, + "learning_rate": 4.866793539675126e-07, + "logits/chosen": 0.13751313090324402, + "logits/rejected": 0.08786692470312119, + "logps/chosen": -63.30891036987305, + "logps/ref_chosen": -58.660743713378906, + "logps/ref_rejected": -79.24510192871094, + "logps/rejected": -85.924072265625, + "loss": 1.0734, + "margin_dpo/margin_mean": 2.0308070182800293, + "margin_dpo/margin_std": 2.682551383972168, + "step": 130 + }, + { + "KL/chosen_KL_mean": -4.569456100463867, + "KL/mean": -5.825868606567383, + "KL/rejected_KL_mean": -7.082279205322266, + "KL/std": 3.4104576110839844, + "epoch": 0.1980347694633409, + "fcm_dpo/beta": 0.2204248309135437, + "fcm_dpo/delta": -0.16282838582992554, + "fcm_dpo/margin": 2.5128283500671387, + "fcm_dpo/q_t": 0.38101017475128174, + "grad_norm": 53.8778190612793, + "learning_rate": 4.86250204678667e-07, + "logits/chosen": 0.1606750786304474, + "logits/rejected": 0.09683164209127426, + "logps/chosen": -57.08399200439453, + "logps/ref_chosen": -52.51453399658203, + "logps/ref_rejected": -85.18299865722656, + "logps/rejected": -92.26527404785156, + "loss": 1.0654, + "margin_dpo/margin_mean": 2.5128278732299805, + "margin_dpo/margin_std": 3.8156890869140625, + "step": 131 + }, + { + "KL/chosen_KL_mean": -4.946624755859375, + "KL/mean": -6.04380464553833, + "KL/rejected_KL_mean": -7.140985488891602, + "KL/std": 3.199896812438965, + "epoch": 0.19954648526077098, + "fcm_dpo/beta": 0.215684711933136, + "fcm_dpo/delta": -0.07706592977046967, + "fcm_dpo/margin": 2.194359064102173, + "fcm_dpo/q_t": 0.39275842905044556, + "grad_norm": 56.11330795288086, + "learning_rate": 4.858144469637408e-07, + "logits/chosen": 0.2194550335407257, + "logits/rejected": 0.18725933134555817, + "logps/chosen": -70.63175964355469, + "logps/ref_chosen": -65.68513488769531, + "logps/ref_rejected": -69.54120635986328, + "logps/rejected": -76.68218994140625, + "loss": 1.0823, + "margin_dpo/margin_mean": 2.1943588256835938, + "margin_dpo/margin_std": 3.272285223007202, + "step": 132 + }, + { + "KL/chosen_KL_mean": -4.965156555175781, + "KL/mean": -6.00357723236084, + "KL/rejected_KL_mean": -7.041996002197266, + "KL/std": 3.207258701324463, + "epoch": 0.20105820105820105, + "fcm_dpo/beta": 0.21359238028526306, + "fcm_dpo/delta": -0.045637097209692, + "fcm_dpo/margin": 2.076840400695801, + "fcm_dpo/q_t": 0.40116482973098755, + "grad_norm": 58.3845100402832, + "learning_rate": 4.853720930118138e-07, + "logits/chosen": 0.15236756205558777, + "logits/rejected": 0.14272542297840118, + "logps/chosen": -68.56327056884766, + "logps/ref_chosen": -63.598114013671875, + "logps/ref_rejected": -73.72798156738281, + "logps/rejected": -80.76997375488281, + "loss": 1.1045, + "margin_dpo/margin_mean": 2.0768399238586426, + "margin_dpo/margin_std": 3.3041024208068848, + "step": 133 + }, + { + "KL/chosen_KL_mean": -4.748861312866211, + "KL/mean": -6.207903861999512, + "KL/rejected_KL_mean": -7.6669464111328125, + "KL/std": 3.6748008728027344, + "epoch": 0.20256991685563114, + "fcm_dpo/beta": 0.20595017075538635, + "fcm_dpo/delta": -0.21452711522579193, + "fcm_dpo/margin": 2.918086290359497, + "fcm_dpo/q_t": 0.36868858337402344, + "grad_norm": 47.815277099609375, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": 0.2514156401157379, + "logits/rejected": 0.19562683999538422, + "logps/chosen": -58.543434143066406, + "logps/ref_chosen": -53.79457092285156, + "logps/ref_rejected": -74.16741943359375, + "logps/rejected": -81.83436584472656, + "loss": 0.9909, + "margin_dpo/margin_mean": 2.9180867671966553, + "margin_dpo/margin_std": 3.5976271629333496, + "step": 134 + }, + { + "KL/chosen_KL_mean": -4.821933746337891, + "KL/mean": -5.825990676879883, + "KL/rejected_KL_mean": -6.830051422119141, + "KL/std": 3.4957146644592285, + "epoch": 0.20408163265306123, + "fcm_dpo/beta": 0.2038605511188507, + "fcm_dpo/delta": -0.00977998971939087, + "fcm_dpo/margin": 2.0081186294555664, + "fcm_dpo/q_t": 0.41021621227264404, + "grad_norm": 47.532005310058594, + "learning_rate": 4.844676460754862e-07, + "logits/chosen": 0.1996041089296341, + "logits/rejected": 0.16683346033096313, + "logps/chosen": -54.26300811767578, + "logps/ref_chosen": -49.441078186035156, + "logps/ref_rejected": -65.96878051757812, + "logps/rejected": -72.798828125, + "loss": 1.1397, + "margin_dpo/margin_mean": 2.008118152618408, + "margin_dpo/margin_std": 3.5981740951538086, + "step": 135 + }, + { + "KL/chosen_KL_mean": -5.761157989501953, + "KL/mean": -6.998971939086914, + "KL/rejected_KL_mean": -8.23678970336914, + "KL/std": 3.8275370597839355, + "epoch": 0.20559334845049132, + "fcm_dpo/beta": 0.20050784945487976, + "fcm_dpo/delta": -0.10157489776611328, + "fcm_dpo/margin": 2.475637912750244, + "fcm_dpo/q_t": 0.40083396434783936, + "grad_norm": 59.76097869873047, + "learning_rate": 4.840055783904106e-07, + "logits/chosen": 0.1858748495578766, + "logits/rejected": 0.1138669028878212, + "logps/chosen": -72.52041625976562, + "logps/ref_chosen": -66.75926208496094, + "logps/ref_rejected": -94.61787414550781, + "logps/rejected": -102.85466003417969, + "loss": 1.1558, + "margin_dpo/margin_mean": 2.4756383895874023, + "margin_dpo/margin_std": 4.930904388427734, + "step": 136 + }, + { + "KL/chosen_KL_mean": -5.439434051513672, + "KL/mean": -6.702852249145508, + "KL/rejected_KL_mean": -7.966270446777344, + "KL/std": 3.578794002532959, + "epoch": 0.20710506424792138, + "fcm_dpo/beta": 0.19725792109966278, + "fcm_dpo/delta": -0.10347731411457062, + "fcm_dpo/margin": 2.5268397331237793, + "fcm_dpo/q_t": 0.3930894136428833, + "grad_norm": 47.14102554321289, + "learning_rate": 4.835369650662767e-07, + "logits/chosen": 0.20675092935562134, + "logits/rejected": 0.17818483710289001, + "logps/chosen": -62.22323226928711, + "logps/ref_chosen": -56.78379821777344, + "logps/ref_rejected": -69.89952087402344, + "logps/rejected": -77.86579132080078, + "loss": 1.0813, + "margin_dpo/margin_mean": 2.5268397331237793, + "margin_dpo/margin_std": 3.943085193634033, + "step": 137 + }, + { + "KL/chosen_KL_mean": -6.369855880737305, + "KL/mean": -7.403163909912109, + "KL/rejected_KL_mean": -8.436473846435547, + "KL/std": 3.7773261070251465, + "epoch": 0.20861678004535147, + "fcm_dpo/beta": 0.19536003470420837, + "fcm_dpo/delta": -0.0039053638465702534, + "fcm_dpo/margin": 2.066615104675293, + "fcm_dpo/q_t": 0.41097143292427063, + "grad_norm": 50.57745361328125, + "learning_rate": 4.830618192112065e-07, + "logits/chosen": 0.20486241579055786, + "logits/rejected": 0.16861580312252045, + "logps/chosen": -65.1358642578125, + "logps/ref_chosen": -58.766014099121094, + "logps/ref_rejected": -68.12371826171875, + "logps/rejected": -76.56019592285156, + "loss": 1.1632, + "margin_dpo/margin_mean": 2.066614866256714, + "margin_dpo/margin_std": 4.009616374969482, + "step": 138 + }, + { + "KL/chosen_KL_mean": -6.473136901855469, + "KL/mean": -7.848593235015869, + "KL/rejected_KL_mean": -9.224044799804688, + "KL/std": 3.7480998039245605, + "epoch": 0.21012849584278157, + "fcm_dpo/beta": 0.19235269725322723, + "fcm_dpo/delta": -0.13614793121814728, + "fcm_dpo/margin": 2.7509093284606934, + "fcm_dpo/q_t": 0.38339757919311523, + "grad_norm": 51.38783264160156, + "learning_rate": 4.825801541160509e-07, + "logits/chosen": 0.15835514664649963, + "logits/rejected": 0.12959414720535278, + "logps/chosen": -77.69873046875, + "logps/ref_chosen": -71.2255859375, + "logps/ref_rejected": -82.1834716796875, + "logps/rejected": -91.40751647949219, + "loss": 1.0374, + "margin_dpo/margin_mean": 2.7509093284606934, + "margin_dpo/margin_std": 3.6548454761505127, + "step": 139 + }, + { + "KL/chosen_KL_mean": -6.044340133666992, + "KL/mean": -7.782115459442139, + "KL/rejected_KL_mean": -9.519889831542969, + "KL/std": 4.871986389160156, + "epoch": 0.21164021164021163, + "fcm_dpo/beta": 0.1829332858324051, + "fcm_dpo/delta": -0.2522822618484497, + "fcm_dpo/margin": 3.4755468368530273, + "fcm_dpo/q_t": 0.36386656761169434, + "grad_norm": 53.137977600097656, + "learning_rate": 4.820919832540181e-07, + "logits/chosen": 0.16880294680595398, + "logits/rejected": 0.12310120463371277, + "logps/chosen": -69.32200622558594, + "logps/ref_chosen": -63.27766418457031, + "logps/ref_rejected": -83.30647277832031, + "logps/rejected": -92.82636260986328, + "loss": 1.0477, + "margin_dpo/margin_mean": 3.4755468368530273, + "margin_dpo/margin_std": 5.251531600952148, + "step": 140 + }, + { + "KL/chosen_KL_mean": -6.465095520019531, + "KL/mean": -8.143421173095703, + "KL/rejected_KL_mean": -9.821746826171875, + "KL/std": 4.110562324523926, + "epoch": 0.21315192743764172, + "fcm_dpo/beta": 0.17414629459381104, + "fcm_dpo/delta": -0.199050173163414, + "fcm_dpo/margin": 3.3566455841064453, + "fcm_dpo/q_t": 0.37942415475845337, + "grad_norm": 47.3781852722168, + "learning_rate": 4.815973202802966e-07, + "logits/chosen": 0.2222299873828888, + "logits/rejected": 0.17720073461532593, + "logps/chosen": -68.23186492919922, + "logps/ref_chosen": -61.76676940917969, + "logps/ref_rejected": -88.60601806640625, + "logps/rejected": -98.42776489257812, + "loss": 1.0593, + "margin_dpo/margin_mean": 3.356645345687866, + "margin_dpo/margin_std": 5.109691619873047, + "step": 141 + }, + { + "KL/chosen_KL_mean": -6.469383239746094, + "KL/mean": -7.785819053649902, + "KL/rejected_KL_mean": -9.102256774902344, + "KL/std": 4.037965774536133, + "epoch": 0.2146636432350718, + "fcm_dpo/beta": 0.17250174283981323, + "fcm_dpo/delta": -0.05671734735369682, + "fcm_dpo/margin": 2.6328725814819336, + "fcm_dpo/q_t": 0.3999059796333313, + "grad_norm": 45.11027526855469, + "learning_rate": 4.810961790316729e-07, + "logits/chosen": 0.231331467628479, + "logits/rejected": 0.20561900734901428, + "logps/chosen": -71.74415588378906, + "logps/ref_chosen": -65.2747802734375, + "logps/ref_rejected": -81.1378173828125, + "logps/rejected": -90.24007415771484, + "loss": 1.0967, + "margin_dpo/margin_mean": 2.6328725814819336, + "margin_dpo/margin_std": 4.121742248535156, + "step": 142 + }, + { + "KL/chosen_KL_mean": -6.877973556518555, + "KL/mean": -8.075854301452637, + "KL/rejected_KL_mean": -9.273731231689453, + "KL/std": 4.413012981414795, + "epoch": 0.2161753590325019, + "fcm_dpo/beta": 0.17143571376800537, + "fcm_dpo/delta": -0.011194366030395031, + "fcm_dpo/margin": 2.3957576751708984, + "fcm_dpo/q_t": 0.4077395796775818, + "grad_norm": 55.14004135131836, + "learning_rate": 4.805885735261454e-07, + "logits/chosen": 0.21375364065170288, + "logits/rejected": 0.1969037652015686, + "logps/chosen": -69.49580383300781, + "logps/ref_chosen": -62.617828369140625, + "logps/ref_rejected": -70.39239501953125, + "logps/rejected": -79.66612243652344, + "loss": 1.19, + "margin_dpo/margin_mean": 2.3957576751708984, + "margin_dpo/margin_std": 5.123311996459961, + "step": 143 + }, + { + "KL/chosen_KL_mean": -7.226917266845703, + "KL/mean": -8.757612228393555, + "KL/rejected_KL_mean": -10.28830337524414, + "KL/std": 4.669466972351074, + "epoch": 0.21768707482993196, + "fcm_dpo/beta": 0.16772940754890442, + "fcm_dpo/delta": -0.12045253813266754, + "fcm_dpo/margin": 3.0613865852355957, + "fcm_dpo/q_t": 0.39098232984542847, + "grad_norm": 46.48661422729492, + "learning_rate": 4.800745179625307e-07, + "logits/chosen": 0.21085073053836823, + "logits/rejected": 0.18012139201164246, + "logps/chosen": -68.02960205078125, + "logps/ref_chosen": -60.80268859863281, + "logps/ref_rejected": -79.07284545898438, + "logps/rejected": -89.36114501953125, + "loss": 1.1012, + "margin_dpo/margin_mean": 3.061386823654175, + "margin_dpo/margin_std": 5.1614203453063965, + "step": 144 + }, + { + "KL/chosen_KL_mean": -6.772495269775391, + "KL/mean": -8.340176582336426, + "KL/rejected_KL_mean": -9.907852172851562, + "KL/std": 4.697710037231445, + "epoch": 0.21919879062736206, + "fcm_dpo/beta": 0.16429109871387482, + "fcm_dpo/delta": -0.1214357241988182, + "fcm_dpo/margin": 3.1353633403778076, + "fcm_dpo/q_t": 0.3892659544944763, + "grad_norm": 49.442745208740234, + "learning_rate": 4.795540267200686e-07, + "logits/chosen": 0.15911483764648438, + "logits/rejected": 0.1758701205253601, + "logps/chosen": -81.38395690917969, + "logps/ref_chosen": -74.61146545410156, + "logps/ref_rejected": -83.24461364746094, + "logps/rejected": -93.1524658203125, + "loss": 1.103, + "margin_dpo/margin_mean": 3.135362148284912, + "margin_dpo/margin_std": 5.358179092407227, + "step": 145 + }, + { + "KL/chosen_KL_mean": -6.513679504394531, + "KL/mean": -8.18224048614502, + "KL/rejected_KL_mean": -9.85080337524414, + "KL/std": 4.615451812744141, + "epoch": 0.22071050642479215, + "fcm_dpo/beta": 0.16036675870418549, + "fcm_dpo/delta": -0.14257195591926575, + "fcm_dpo/margin": 3.3371238708496094, + "fcm_dpo/q_t": 0.38525545597076416, + "grad_norm": 42.53998565673828, + "learning_rate": 4.790271143580173e-07, + "logits/chosen": 0.16977742314338684, + "logits/rejected": 0.15491747856140137, + "logps/chosen": -64.35466003417969, + "logps/ref_chosen": -57.84098434448242, + "logps/ref_rejected": -67.47422790527344, + "logps/rejected": -77.32502746582031, + "loss": 1.0618, + "margin_dpo/margin_mean": 3.3371238708496094, + "margin_dpo/margin_std": 4.9862470626831055, + "step": 146 + }, + { + "KL/chosen_KL_mean": -7.826589584350586, + "KL/mean": -9.213622093200684, + "KL/rejected_KL_mean": -10.600662231445312, + "KL/std": 5.016104698181152, + "epoch": 0.2222222222222222, + "fcm_dpo/beta": 0.15783822536468506, + "fcm_dpo/delta": -0.03958672657608986, + "fcm_dpo/margin": 2.7740743160247803, + "fcm_dpo/q_t": 0.4058781862258911, + "grad_norm": 52.86528778076172, + "learning_rate": 4.784937956152489e-07, + "logits/chosen": 0.1791575849056244, + "logits/rejected": 0.13689681887626648, + "logps/chosen": -74.64006042480469, + "logps/ref_chosen": -66.81346893310547, + "logps/ref_rejected": -81.1796875, + "logps/rejected": -91.78034973144531, + "loss": 1.1557, + "margin_dpo/margin_mean": 2.774074077606201, + "margin_dpo/margin_std": 5.396799564361572, + "step": 147 + }, + { + "KL/chosen_KL_mean": -7.366302490234375, + "KL/mean": -9.069774627685547, + "KL/rejected_KL_mean": -10.773239135742188, + "KL/std": 4.651217460632324, + "epoch": 0.2237339380196523, + "fcm_dpo/beta": 0.15506836771965027, + "fcm_dpo/delta": -0.13526862859725952, + "fcm_dpo/margin": 3.40694260597229, + "fcm_dpo/q_t": 0.38584333658218384, + "grad_norm": 35.693355560302734, + "learning_rate": 4.779540854098347e-07, + "logits/chosen": 0.3457987308502197, + "logits/rejected": 0.27079546451568604, + "logps/chosen": -56.05405807495117, + "logps/ref_chosen": -48.6877555847168, + "logps/ref_rejected": -67.50503540039062, + "logps/rejected": -78.27827453613281, + "loss": 1.0782, + "margin_dpo/margin_mean": 3.406942367553711, + "margin_dpo/margin_std": 5.257372856140137, + "step": 148 + }, + { + "KL/chosen_KL_mean": -9.045269012451172, + "KL/mean": -11.011503219604492, + "KL/rejected_KL_mean": -12.977741241455078, + "KL/std": 5.339559555053711, + "epoch": 0.2252456538170824, + "fcm_dpo/beta": 0.1490006297826767, + "fcm_dpo/delta": -0.1973770558834076, + "fcm_dpo/margin": 3.9324727058410645, + "fcm_dpo/q_t": 0.3704410195350647, + "grad_norm": 36.19922637939453, + "learning_rate": 4.774079988386296e-07, + "logits/chosen": 0.1623860001564026, + "logits/rejected": 0.11575397104024887, + "logps/chosen": -64.18904876708984, + "logps/ref_chosen": -55.143775939941406, + "logps/ref_rejected": -64.79888916015625, + "logps/rejected": -77.7766342163086, + "loss": 1.0031, + "margin_dpo/margin_mean": 3.9324722290039062, + "margin_dpo/margin_std": 4.933730125427246, + "step": 149 + }, + { + "KL/chosen_KL_mean": -7.397439956665039, + "KL/mean": -9.875222206115723, + "KL/rejected_KL_mean": -12.35300064086914, + "KL/std": 5.53492546081543, + "epoch": 0.22675736961451248, + "fcm_dpo/beta": 0.14036396145820618, + "fcm_dpo/delta": -0.31827008724212646, + "fcm_dpo/margin": 4.955565929412842, + "fcm_dpo/q_t": 0.3498973846435547, + "grad_norm": 34.423675537109375, + "learning_rate": 4.768555511768486e-07, + "logits/chosen": 0.21100425720214844, + "logits/rejected": 0.1672147810459137, + "logps/chosen": -74.8681869506836, + "logps/ref_chosen": -67.47074890136719, + "logps/ref_rejected": -89.21170806884766, + "logps/rejected": -101.56471252441406, + "loss": 0.9365, + "margin_dpo/margin_mean": 4.95556640625, + "margin_dpo/margin_std": 5.431443691253662, + "step": 150 + }, + { + "KL/chosen_KL_mean": -6.885496139526367, + "KL/mean": -9.645720481872559, + "KL/rejected_KL_mean": -12.405941009521484, + "KL/std": 5.8176798820495605, + "epoch": 0.22826908541194255, + "fcm_dpo/beta": 0.1314392387866974, + "fcm_dpo/delta": -0.3507463037967682, + "fcm_dpo/margin": 5.520442962646484, + "fcm_dpo/q_t": 0.3430783152580261, + "grad_norm": 30.11147689819336, + "learning_rate": 4.762967578776406e-07, + "logits/chosen": 0.2235882431268692, + "logits/rejected": 0.16875801980495453, + "logps/chosen": -59.34503936767578, + "logps/ref_chosen": -52.45954132080078, + "logps/ref_rejected": -79.0630111694336, + "logps/rejected": -91.46894836425781, + "loss": 0.9129, + "margin_dpo/margin_mean": 5.520442962646484, + "margin_dpo/margin_std": 5.8820390701293945, + "step": 151 + }, + { + "KL/chosen_KL_mean": -8.967247009277344, + "KL/mean": -10.84051513671875, + "KL/rejected_KL_mean": -12.713790893554688, + "KL/std": 5.6841278076171875, + "epoch": 0.22978080120937264, + "fcm_dpo/beta": 0.1281639039516449, + "fcm_dpo/delta": -0.08456161618232727, + "fcm_dpo/margin": 3.746544361114502, + "fcm_dpo/q_t": 0.3974611461162567, + "grad_norm": 35.68574905395508, + "learning_rate": 4.757316345716553e-07, + "logits/chosen": 0.25782787799835205, + "logits/rejected": 0.20684176683425903, + "logps/chosen": -65.52108001708984, + "logps/ref_chosen": -56.5538330078125, + "logps/ref_rejected": -76.55074310302734, + "logps/rejected": -89.26453399658203, + "loss": 1.1185, + "margin_dpo/margin_mean": 3.746544122695923, + "margin_dpo/margin_std": 6.411558151245117, + "step": 152 + }, + { + "KL/chosen_KL_mean": -8.4833984375, + "KL/mean": -10.582864761352539, + "KL/rejected_KL_mean": -12.682331085205078, + "KL/std": 5.9477057456970215, + "epoch": 0.23129251700680273, + "fcm_dpo/beta": 0.12414808571338654, + "fcm_dpo/delta": -0.12824231386184692, + "fcm_dpo/margin": 4.198927402496338, + "fcm_dpo/q_t": 0.3833044469356537, + "grad_norm": 32.4552001953125, + "learning_rate": 4.751601970666064e-07, + "logits/chosen": 0.18484419584274292, + "logits/rejected": 0.14812716841697693, + "logps/chosen": -76.49029541015625, + "logps/ref_chosen": -68.00689697265625, + "logps/ref_rejected": -74.83482360839844, + "logps/rejected": -87.51715087890625, + "loss": 1.033, + "margin_dpo/margin_mean": 4.198927402496338, + "margin_dpo/margin_std": 5.387975692749023, + "step": 153 + }, + { + "KL/chosen_KL_mean": -9.598503112792969, + "KL/mean": -11.168582916259766, + "KL/rejected_KL_mean": -12.738666534423828, + "KL/std": 6.069890975952148, + "epoch": 0.2328042328042328, + "fcm_dpo/beta": 0.12375746667385101, + "fcm_dpo/delta": 0.011842047795653343, + "fcm_dpo/margin": 3.1401634216308594, + "fcm_dpo/q_t": 0.4137424826622009, + "grad_norm": 33.3686637878418, + "learning_rate": 4.745824613468292e-07, + "logits/chosen": 0.2683183252811432, + "logits/rejected": 0.26516079902648926, + "logps/chosen": -68.821044921875, + "logps/ref_chosen": -59.222537994384766, + "logps/ref_rejected": -64.19131469726562, + "logps/rejected": -76.92998504638672, + "loss": 1.1731, + "margin_dpo/margin_mean": 3.1401634216308594, + "margin_dpo/margin_std": 6.250120162963867, + "step": 154 + }, + { + "KL/chosen_KL_mean": -9.947959899902344, + "KL/mean": -12.126672744750977, + "KL/rejected_KL_mean": -14.305377960205078, + "KL/std": 6.030064582824707, + "epoch": 0.23431594860166288, + "fcm_dpo/beta": 0.12072409689426422, + "fcm_dpo/delta": -0.13440188765525818, + "fcm_dpo/margin": 4.357418537139893, + "fcm_dpo/q_t": 0.389107346534729, + "grad_norm": 33.98651885986328, + "learning_rate": 4.7399844357283393e-07, + "logits/chosen": 0.24776160717010498, + "logits/rejected": 0.22814488410949707, + "logps/chosen": -78.40265655517578, + "logps/ref_chosen": -68.45469665527344, + "logps/ref_rejected": -77.91763305664062, + "logps/rejected": -92.22301483154297, + "loss": 1.1193, + "margin_dpo/margin_mean": 4.357418060302734, + "margin_dpo/margin_std": 7.67659854888916, + "step": 155 + }, + { + "KL/chosen_KL_mean": -10.01881217956543, + "KL/mean": -12.622465133666992, + "KL/rejected_KL_mean": -15.226116180419922, + "KL/std": 6.6650285720825195, + "epoch": 0.23582766439909297, + "fcm_dpo/beta": 0.11669893562793732, + "fcm_dpo/delta": -0.2210705578327179, + "fcm_dpo/margin": 5.207304954528809, + "fcm_dpo/q_t": 0.3674676716327667, + "grad_norm": 32.13863754272461, + "learning_rate": 4.7340816008085305e-07, + "logits/chosen": 0.22416064143180847, + "logits/rejected": 0.17890284955501556, + "logps/chosen": -77.28840637207031, + "logps/ref_chosen": -67.26959991455078, + "logps/ref_rejected": -86.95914459228516, + "logps/rejected": -102.18525695800781, + "loss": 0.9856, + "margin_dpo/margin_mean": 5.207304954528809, + "margin_dpo/margin_std": 6.2471513748168945, + "step": 156 + }, + { + "KL/chosen_KL_mean": -9.303003311157227, + "KL/mean": -11.44937515258789, + "KL/rejected_KL_mean": -13.595748901367188, + "KL/std": 6.676199436187744, + "epoch": 0.23733938019652306, + "fcm_dpo/beta": 0.1125432625412941, + "fcm_dpo/delta": -0.09084660559892654, + "fcm_dpo/margin": 4.292743682861328, + "fcm_dpo/q_t": 0.39273664355278015, + "grad_norm": 29.455463409423828, + "learning_rate": 4.728116273823847e-07, + "logits/chosen": 0.21543173491954803, + "logits/rejected": 0.19620484113693237, + "logps/chosen": -64.07588195800781, + "logps/ref_chosen": -54.77287292480469, + "logps/ref_rejected": -63.87866973876953, + "logps/rejected": -77.47441864013672, + "loss": 1.0827, + "margin_dpo/margin_mean": 4.292743682861328, + "margin_dpo/margin_std": 6.249413967132568, + "step": 157 + }, + { + "KL/chosen_KL_mean": -10.352668762207031, + "KL/mean": -12.572793960571289, + "KL/rejected_KL_mean": -14.792915344238281, + "KL/std": 6.630293846130371, + "epoch": 0.23885109599395313, + "fcm_dpo/beta": 0.11155691742897034, + "fcm_dpo/delta": -0.1003374308347702, + "fcm_dpo/margin": 4.440248489379883, + "fcm_dpo/q_t": 0.3917720317840576, + "grad_norm": 31.280303955078125, + "learning_rate": 4.7220886216373085e-07, + "logits/chosen": 0.27400046586990356, + "logits/rejected": 0.2366476058959961, + "logps/chosen": -75.275390625, + "logps/ref_chosen": -64.92271423339844, + "logps/ref_rejected": -82.23789978027344, + "logps/rejected": -97.03080749511719, + "loss": 1.0824, + "margin_dpo/margin_mean": 4.440248489379883, + "margin_dpo/margin_std": 6.843707084655762, + "step": 158 + }, + { + "KL/chosen_KL_mean": -10.615558624267578, + "KL/mean": -13.190221786499023, + "KL/rejected_KL_mean": -15.764892578125, + "KL/std": 7.167794704437256, + "epoch": 0.24036281179138322, + "fcm_dpo/beta": 0.1089281514286995, + "fcm_dpo/delta": -0.1702059954404831, + "fcm_dpo/margin": 5.149332046508789, + "fcm_dpo/q_t": 0.3772723078727722, + "grad_norm": 33.383506774902344, + "learning_rate": 4.715998812855304e-07, + "logits/chosen": 0.2682814598083496, + "logits/rejected": 0.22989416122436523, + "logps/chosen": -67.66255187988281, + "logps/ref_chosen": -57.046993255615234, + "logps/ref_rejected": -73.32441711425781, + "logps/rejected": -89.08930969238281, + "loss": 1.0526, + "margin_dpo/margin_mean": 5.149332046508789, + "margin_dpo/margin_std": 7.4767351150512695, + "step": 159 + }, + { + "KL/chosen_KL_mean": -11.970067977905273, + "KL/mean": -14.173985481262207, + "KL/rejected_KL_mean": -16.377899169921875, + "KL/std": 6.7857513427734375, + "epoch": 0.2418745275888133, + "fcm_dpo/beta": 0.10626037418842316, + "fcm_dpo/delta": -0.07168857008218765, + "fcm_dpo/margin": 4.407832622528076, + "fcm_dpo/q_t": 0.40258103609085083, + "grad_norm": 27.9632625579834, + "learning_rate": 4.7098470178228755e-07, + "logits/chosen": 0.1273048371076584, + "logits/rejected": 0.0839085727930069, + "logps/chosen": -61.776981353759766, + "logps/ref_chosen": -49.806915283203125, + "logps/ref_rejected": -68.3370132446289, + "logps/rejected": -84.71491241455078, + "loss": 1.1236, + "margin_dpo/margin_mean": 4.407833099365234, + "margin_dpo/margin_std": 7.865055084228516, + "step": 160 + }, + { + "KL/chosen_KL_mean": -11.96878433227539, + "KL/mean": -14.375494003295898, + "KL/rejected_KL_mean": -16.782207489013672, + "KL/std": 6.705003261566162, + "epoch": 0.24338624338624337, + "fcm_dpo/beta": 0.10418327152729034, + "fcm_dpo/delta": -0.1067676991224289, + "fcm_dpo/margin": 4.813423156738281, + "fcm_dpo/q_t": 0.3899020552635193, + "grad_norm": 26.78690528869629, + "learning_rate": 4.703633408618955e-07, + "logits/chosen": 0.27834972739219666, + "logits/rejected": 0.23936110734939575, + "logps/chosen": -64.46926879882812, + "logps/ref_chosen": -52.50048828125, + "logps/ref_rejected": -66.04540252685547, + "logps/rejected": -82.82760620117188, + "loss": 1.0822, + "margin_dpo/margin_mean": 4.813423156738281, + "margin_dpo/margin_std": 7.380710601806641, + "step": 161 + }, + { + "KL/chosen_KL_mean": -12.701950073242188, + "KL/mean": -15.949088096618652, + "KL/rejected_KL_mean": -19.19622802734375, + "KL/std": 7.516228675842285, + "epoch": 0.24489795918367346, + "fcm_dpo/beta": 0.09950388967990875, + "fcm_dpo/delta": -0.26341742277145386, + "fcm_dpo/margin": 6.494287014007568, + "fcm_dpo/q_t": 0.3561316728591919, + "grad_norm": 27.878267288208008, + "learning_rate": 4.697358159051549e-07, + "logits/chosen": 0.3174731135368347, + "logits/rejected": 0.267278790473938, + "logps/chosen": -82.171142578125, + "logps/ref_chosen": -69.46919250488281, + "logps/ref_rejected": -92.00952911376953, + "logps/rejected": -111.20575714111328, + "loss": 0.9581, + "margin_dpo/margin_mean": 6.494287490844727, + "margin_dpo/margin_std": 7.305610656738281, + "step": 162 + }, + { + "KL/chosen_KL_mean": -11.61453628540039, + "KL/mean": -14.817399978637695, + "KL/rejected_KL_mean": -18.020267486572266, + "KL/std": 7.376770973205566, + "epoch": 0.24640967498110355, + "fcm_dpo/beta": 0.09595300257205963, + "fcm_dpo/delta": -0.22854195535182953, + "fcm_dpo/margin": 6.405729293823242, + "fcm_dpo/q_t": 0.36570775508880615, + "grad_norm": 25.580001831054688, + "learning_rate": 4.691021444652876e-07, + "logits/chosen": 0.2480703890323639, + "logits/rejected": 0.2000638246536255, + "logps/chosen": -62.228370666503906, + "logps/ref_chosen": -50.613834381103516, + "logps/ref_rejected": -74.62033081054688, + "logps/rejected": -92.64059448242188, + "loss": 1.004, + "margin_dpo/margin_mean": 6.405730247497559, + "margin_dpo/margin_std": 7.875078201293945, + "step": 163 + }, + { + "KL/chosen_KL_mean": -12.40074348449707, + "KL/mean": -15.593101501464844, + "KL/rejected_KL_mean": -18.78546142578125, + "KL/std": 7.785859107971191, + "epoch": 0.24792139077853365, + "fcm_dpo/beta": 0.09157611429691315, + "fcm_dpo/delta": -0.1956927627325058, + "fcm_dpo/margin": 6.384723663330078, + "fcm_dpo/q_t": 0.3727573752403259, + "grad_norm": 24.026826858520508, + "learning_rate": 4.6846234426744624e-07, + "logits/chosen": 0.2387039214372635, + "logits/rejected": 0.17490170896053314, + "logps/chosen": -67.24885559082031, + "logps/ref_chosen": -54.848114013671875, + "logps/ref_rejected": -79.0630111694336, + "logps/rejected": -97.84847259521484, + "loss": 1.0249, + "margin_dpo/margin_mean": 6.384724140167236, + "margin_dpo/margin_std": 8.432074546813965, + "step": 164 + }, + { + "KL/chosen_KL_mean": -13.576322555541992, + "KL/mean": -16.508556365966797, + "KL/rejected_KL_mean": -19.44078826904297, + "KL/std": 7.981330871582031, + "epoch": 0.2494331065759637, + "fcm_dpo/beta": 0.0890810489654541, + "fcm_dpo/delta": -0.12915945053100586, + "fcm_dpo/margin": 5.864466667175293, + "fcm_dpo/q_t": 0.3837765157222748, + "grad_norm": 25.287494659423828, + "learning_rate": 4.678164332082175e-07, + "logits/chosen": 0.3062145709991455, + "logits/rejected": 0.25015687942504883, + "logps/chosen": -64.66553497314453, + "logps/ref_chosen": -51.089210510253906, + "logps/ref_rejected": -71.23370361328125, + "logps/rejected": -90.67448425292969, + "loss": 1.0654, + "margin_dpo/margin_mean": 5.864466667175293, + "margin_dpo/margin_std": 8.354823112487793, + "step": 165 + }, + { + "KL/chosen_KL_mean": -13.206365585327148, + "KL/mean": -15.502958297729492, + "KL/rejected_KL_mean": -17.799549102783203, + "KL/std": 7.9299445152282715, + "epoch": 0.2509448223733938, + "fcm_dpo/beta": 0.08768243342638016, + "fcm_dpo/delta": -0.002980044111609459, + "fcm_dpo/margin": 4.593182563781738, + "fcm_dpo/q_t": 0.41203340888023376, + "grad_norm": 28.28586196899414, + "learning_rate": 4.6716442935512214e-07, + "logits/chosen": 0.2627357840538025, + "logits/rejected": 0.1715720295906067, + "logps/chosen": -76.39717102050781, + "logps/ref_chosen": -63.19081115722656, + "logps/ref_rejected": -93.8402099609375, + "logps/rejected": -111.63975524902344, + "loss": 1.1261, + "margin_dpo/margin_mean": 4.593182563781738, + "margin_dpo/margin_std": 7.637331962585449, + "step": 166 + }, + { + "KL/chosen_KL_mean": -11.713014602661133, + "KL/mean": -15.292600631713867, + "KL/rejected_KL_mean": -18.872188568115234, + "KL/std": 8.240446090698242, + "epoch": 0.25245653817082386, + "fcm_dpo/beta": 0.08448252081871033, + "fcm_dpo/delta": -0.22041411697864532, + "fcm_dpo/margin": 7.1591691970825195, + "fcm_dpo/q_t": 0.36489737033843994, + "grad_norm": 22.19241714477539, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": 0.2110971212387085, + "logits/rejected": 0.1757899820804596, + "logps/chosen": -70.63728332519531, + "logps/ref_chosen": -58.92427062988281, + "logps/ref_rejected": -72.97377014160156, + "logps/rejected": -91.84596252441406, + "loss": 0.9721, + "margin_dpo/margin_mean": 7.1591691970825195, + "margin_dpo/margin_std": 7.9116530418396, + "step": 167 + }, + { + "KL/chosen_KL_mean": -13.643762588500977, + "KL/mean": -16.358333587646484, + "KL/rejected_KL_mean": -19.07291030883789, + "KL/std": 8.718094825744629, + "epoch": 0.25396825396825395, + "fcm_dpo/beta": 0.08331099897623062, + "fcm_dpo/delta": -0.05487321317195892, + "fcm_dpo/margin": 5.429144859313965, + "fcm_dpo/q_t": 0.40054255723953247, + "grad_norm": 25.996004104614258, + "learning_rate": 4.6584221638904767e-07, + "logits/chosen": 0.2337023764848709, + "logits/rejected": 0.19847044348716736, + "logps/chosen": -79.2951431274414, + "logps/ref_chosen": -65.65138244628906, + "logps/ref_rejected": -79.71418762207031, + "logps/rejected": -98.78709411621094, + "loss": 1.085, + "margin_dpo/margin_mean": 5.429144859313965, + "margin_dpo/margin_std": 7.938119411468506, + "step": 168 + }, + { + "KL/chosen_KL_mean": -11.592302322387695, + "KL/mean": -14.968731880187988, + "KL/rejected_KL_mean": -18.34516143798828, + "KL/std": 8.936497688293457, + "epoch": 0.25547996976568405, + "fcm_dpo/beta": 0.08121495693922043, + "fcm_dpo/delta": -0.15712596476078033, + "fcm_dpo/margin": 6.7528581619262695, + "fcm_dpo/q_t": 0.38491493463516235, + "grad_norm": 24.72463607788086, + "learning_rate": 4.651720442612075e-07, + "logits/chosen": 0.3097034692764282, + "logits/rejected": 0.27537956833839417, + "logps/chosen": -73.01817321777344, + "logps/ref_chosen": -61.425865173339844, + "logps/ref_rejected": -76.09590148925781, + "logps/rejected": -94.44105529785156, + "loss": 1.0647, + "margin_dpo/margin_mean": 6.7528581619262695, + "margin_dpo/margin_std": 10.517898559570312, + "step": 169 + }, + { + "KL/chosen_KL_mean": -11.574005126953125, + "KL/mean": -14.787891387939453, + "KL/rejected_KL_mean": -18.001773834228516, + "KL/std": 8.793878555297852, + "epoch": 0.25699168556311414, + "fcm_dpo/beta": 0.07942621409893036, + "fcm_dpo/delta": -0.11631622910499573, + "fcm_dpo/margin": 6.427769184112549, + "fcm_dpo/q_t": 0.3889446258544922, + "grad_norm": 20.754453659057617, + "learning_rate": 4.6449585330874425e-07, + "logits/chosen": 0.24759146571159363, + "logits/rejected": 0.24484430253505707, + "logps/chosen": -68.2271957397461, + "logps/ref_chosen": -56.65319061279297, + "logps/ref_rejected": -63.45965576171875, + "logps/rejected": -81.46142578125, + "loss": 1.0987, + "margin_dpo/margin_mean": 6.427768707275391, + "margin_dpo/margin_std": 10.505061149597168, + "step": 170 + }, + { + "KL/chosen_KL_mean": -12.49212646484375, + "KL/mean": -16.14303970336914, + "KL/rejected_KL_mean": -19.793956756591797, + "KL/std": 9.996442794799805, + "epoch": 0.2585034013605442, + "fcm_dpo/beta": 0.07601982355117798, + "fcm_dpo/delta": -0.16807925701141357, + "fcm_dpo/margin": 7.301830768585205, + "fcm_dpo/q_t": 0.3793303966522217, + "grad_norm": 23.09961700439453, + "learning_rate": 4.6381366244617224e-07, + "logits/chosen": 0.3099227845668793, + "logits/rejected": 0.2590000331401825, + "logps/chosen": -76.22688293457031, + "logps/ref_chosen": -63.73476028442383, + "logps/ref_rejected": -78.50328063964844, + "logps/rejected": -98.29723358154297, + "loss": 1.075, + "margin_dpo/margin_mean": 7.3018317222595215, + "margin_dpo/margin_std": 11.233263969421387, + "step": 171 + }, + { + "KL/chosen_KL_mean": -13.715980529785156, + "KL/mean": -17.212726593017578, + "KL/rejected_KL_mean": -20.709468841552734, + "KL/std": 10.043166160583496, + "epoch": 0.2600151171579743, + "fcm_dpo/beta": 0.07525793462991714, + "fcm_dpo/delta": -0.13359381258487701, + "fcm_dpo/margin": 6.993487358093262, + "fcm_dpo/q_t": 0.38472551107406616, + "grad_norm": 22.1481876373291, + "learning_rate": 4.631254907558365e-07, + "logits/chosen": 0.3433062434196472, + "logits/rejected": 0.2836194634437561, + "logps/chosen": -65.91773986816406, + "logps/ref_chosen": -52.201759338378906, + "logps/ref_rejected": -82.85285949707031, + "logps/rejected": -103.56233215332031, + "loss": 1.0655, + "margin_dpo/margin_mean": 6.9934868812561035, + "margin_dpo/margin_std": 10.196330070495605, + "step": 172 + }, + { + "KL/chosen_KL_mean": -13.024921417236328, + "KL/mean": -16.835899353027344, + "KL/rejected_KL_mean": -20.64687728881836, + "KL/std": 10.502202987670898, + "epoch": 0.2615268329554044, + "fcm_dpo/beta": 0.07153315842151642, + "fcm_dpo/delta": -0.15856504440307617, + "fcm_dpo/margin": 7.621954917907715, + "fcm_dpo/q_t": 0.3873851001262665, + "grad_norm": 21.773149490356445, + "learning_rate": 4.624313574873786e-07, + "logits/chosen": 0.3005369305610657, + "logits/rejected": 0.21291136741638184, + "logps/chosen": -68.45964813232422, + "logps/ref_chosen": -55.434722900390625, + "logps/ref_rejected": -77.81967163085938, + "logps/rejected": -98.466552734375, + "loss": 1.1175, + "margin_dpo/margin_mean": 7.6219563484191895, + "margin_dpo/margin_std": 13.109130859375, + "step": 173 + }, + { + "KL/chosen_KL_mean": -14.391660690307617, + "KL/mean": -18.400985717773438, + "KL/rejected_KL_mean": -22.41031265258789, + "KL/std": 10.439952850341797, + "epoch": 0.26303854875283444, + "fcm_dpo/beta": 0.06982284039258957, + "fcm_dpo/delta": -0.17004986107349396, + "fcm_dpo/margin": 8.018655776977539, + "fcm_dpo/q_t": 0.3786467909812927, + "grad_norm": 22.08015251159668, + "learning_rate": 4.61731282057198e-07, + "logits/chosen": 0.285661518573761, + "logits/rejected": 0.2192431539297104, + "logps/chosen": -71.56361389160156, + "logps/ref_chosen": -57.17195129394531, + "logps/ref_rejected": -85.47578430175781, + "logps/rejected": -107.88609313964844, + "loss": 1.0428, + "margin_dpo/margin_mean": 8.018654823303223, + "margin_dpo/margin_std": 11.424968719482422, + "step": 174 + }, + { + "KL/chosen_KL_mean": -13.223051071166992, + "KL/mean": -17.665573120117188, + "KL/rejected_KL_mean": -22.10810089111328, + "KL/std": 10.922414779663086, + "epoch": 0.26455026455026454, + "fcm_dpo/beta": 0.06739327311515808, + "fcm_dpo/delta": -0.21115991473197937, + "fcm_dpo/margin": 8.885052680969238, + "fcm_dpo/q_t": 0.3742350935935974, + "grad_norm": 21.33765411376953, + "learning_rate": 4.6102528404790965e-07, + "logits/chosen": 0.34087687730789185, + "logits/rejected": 0.3097060024738312, + "logps/chosen": -80.88867950439453, + "logps/ref_chosen": -67.6656265258789, + "logps/ref_rejected": -84.36766815185547, + "logps/rejected": -106.47576904296875, + "loss": 1.0334, + "margin_dpo/margin_mean": 8.885052680969238, + "margin_dpo/margin_std": 12.609004974365234, + "step": 175 + }, + { + "KL/chosen_KL_mean": -15.523975372314453, + "KL/mean": -18.484966278076172, + "KL/rejected_KL_mean": -21.44595718383789, + "KL/std": 11.551431655883789, + "epoch": 0.2660619803476946, + "fcm_dpo/beta": 0.06620974093675613, + "fcm_dpo/delta": 0.007484421133995056, + "fcm_dpo/margin": 5.921982765197754, + "fcm_dpo/q_t": 0.41712304949760437, + "grad_norm": 25.927288055419922, + "learning_rate": 4.603133832077953e-07, + "logits/chosen": 0.28880423307418823, + "logits/rejected": 0.26152804493904114, + "logps/chosen": -93.38273620605469, + "logps/ref_chosen": -77.8587646484375, + "logps/ref_rejected": -81.08732604980469, + "logps/rejected": -102.53327941894531, + "loss": 1.1915, + "margin_dpo/margin_mean": 5.921982765197754, + "margin_dpo/margin_std": 12.589743614196777, + "step": 176 + }, + { + "KL/chosen_KL_mean": -12.250782012939453, + "KL/mean": -18.0947265625, + "KL/rejected_KL_mean": -23.938678741455078, + "KL/std": 11.902139663696289, + "epoch": 0.2675736961451247, + "fcm_dpo/beta": 0.06285828351974487, + "fcm_dpo/delta": -0.36344408988952637, + "fcm_dpo/margin": 11.687898635864258, + "fcm_dpo/q_t": 0.339341938495636, + "grad_norm": 23.54242515563965, + "learning_rate": 4.5959559945025183e-07, + "logits/chosen": 0.41696828603744507, + "logits/rejected": 0.3189684748649597, + "logps/chosen": -67.47117614746094, + "logps/ref_chosen": -55.22039794921875, + "logps/ref_rejected": -92.54973602294922, + "logps/rejected": -116.48841857910156, + "loss": 0.9092, + "margin_dpo/margin_mean": 11.687898635864258, + "margin_dpo/margin_std": 12.291168212890625, + "step": 177 + }, + { + "KL/chosen_KL_mean": -13.977239608764648, + "KL/mean": -17.54144287109375, + "KL/rejected_KL_mean": -21.10564422607422, + "KL/std": 11.356380462646484, + "epoch": 0.2690854119425548, + "fcm_dpo/beta": 0.06110500544309616, + "fcm_dpo/delta": -0.03895438462495804, + "fcm_dpo/margin": 7.128399848937988, + "fcm_dpo/q_t": 0.4032604992389679, + "grad_norm": 22.13608741760254, + "learning_rate": 4.588719528532341e-07, + "logits/chosen": 0.26632487773895264, + "logits/rejected": 0.2168237566947937, + "logps/chosen": -74.78773498535156, + "logps/ref_chosen": -60.81049346923828, + "logps/ref_rejected": -81.12973022460938, + "logps/rejected": -102.23536682128906, + "loss": 1.1164, + "margin_dpo/margin_mean": 7.128399848937988, + "margin_dpo/margin_std": 11.491066932678223, + "step": 178 + }, + { + "KL/chosen_KL_mean": -14.837139129638672, + "KL/mean": -19.02896499633789, + "KL/rejected_KL_mean": -23.22079086303711, + "KL/std": 11.86774730682373, + "epoch": 0.2705971277399849, + "fcm_dpo/beta": 0.06048303842544556, + "fcm_dpo/delta": -0.11276848614215851, + "fcm_dpo/margin": 8.383651733398438, + "fcm_dpo/q_t": 0.390702486038208, + "grad_norm": 21.742908477783203, + "learning_rate": 4.581424636586928e-07, + "logits/chosen": 0.3228716254234314, + "logits/rejected": 0.30639001727104187, + "logps/chosen": -80.50885009765625, + "logps/ref_chosen": -65.67171478271484, + "logps/ref_rejected": -75.32586669921875, + "logps/rejected": -98.54666137695312, + "loss": 1.1012, + "margin_dpo/margin_mean": 8.383651733398438, + "margin_dpo/margin_std": 14.014520645141602, + "step": 179 + }, + { + "KL/chosen_KL_mean": -11.49049186706543, + "KL/mean": -15.210617065429688, + "KL/rejected_KL_mean": -18.930740356445312, + "KL/std": 11.852537155151367, + "epoch": 0.272108843537415, + "fcm_dpo/beta": 0.059869199991226196, + "fcm_dpo/delta": -0.047591034322977066, + "fcm_dpo/margin": 7.440252304077148, + "fcm_dpo/q_t": 0.4054303765296936, + "grad_norm": 19.28363037109375, + "learning_rate": 4.5740715227200897e-07, + "logits/chosen": 0.13881587982177734, + "logits/rejected": 0.11996544897556305, + "logps/chosen": -68.17330169677734, + "logps/ref_chosen": -56.68280792236328, + "logps/ref_rejected": -64.94414520263672, + "logps/rejected": -83.87488555908203, + "loss": 1.1436, + "margin_dpo/margin_mean": 7.440252304077148, + "margin_dpo/margin_std": 13.85991096496582, + "step": 180 + }, + { + "KL/chosen_KL_mean": -10.01158332824707, + "KL/mean": -15.477460861206055, + "KL/rejected_KL_mean": -20.94333267211914, + "KL/std": 12.780189514160156, + "epoch": 0.273620559334845, + "fcm_dpo/beta": 0.05729272961616516, + "fcm_dpo/delta": -0.2422669529914856, + "fcm_dpo/margin": 10.931753158569336, + "fcm_dpo/q_t": 0.3609431982040405, + "grad_norm": 19.187829971313477, + "learning_rate": 4.566660392614228e-07, + "logits/chosen": 0.32175952196121216, + "logits/rejected": 0.2816872298717499, + "logps/chosen": -70.78762817382812, + "logps/ref_chosen": -60.77604675292969, + "logps/ref_rejected": -83.98361206054688, + "logps/rejected": -104.92694854736328, + "loss": 0.9572, + "margin_dpo/margin_mean": 10.931753158569336, + "margin_dpo/margin_std": 11.934650421142578, + "step": 181 + }, + { + "KL/chosen_KL_mean": -13.245315551757812, + "KL/mean": -19.128177642822266, + "KL/rejected_KL_mean": -25.011043548583984, + "KL/std": 13.650596618652344, + "epoch": 0.2751322751322751, + "fcm_dpo/beta": 0.05450388044118881, + "fcm_dpo/delta": -0.2583681643009186, + "fcm_dpo/margin": 11.765732765197754, + "fcm_dpo/q_t": 0.3621079921722412, + "grad_norm": 19.192251205444336, + "learning_rate": 4.5591914535745817e-07, + "logits/chosen": 0.31452324986457825, + "logits/rejected": 0.2316857874393463, + "logps/chosen": -73.49909973144531, + "logps/ref_chosen": -60.2537841796875, + "logps/ref_rejected": -89.7706298828125, + "logps/rejected": -114.78167724609375, + "loss": 1.0017, + "margin_dpo/margin_mean": 11.76573371887207, + "margin_dpo/margin_std": 15.48458480834961, + "step": 182 + }, + { + "KL/chosen_KL_mean": -16.417619705200195, + "KL/mean": -18.79208755493164, + "KL/rejected_KL_mean": -21.16655731201172, + "KL/std": 13.141827583312988, + "epoch": 0.2766439909297052, + "fcm_dpo/beta": 0.05492541193962097, + "fcm_dpo/delta": 0.14310485124588013, + "fcm_dpo/margin": 4.748943328857422, + "fcm_dpo/q_t": 0.44205474853515625, + "grad_norm": 20.398983001708984, + "learning_rate": 4.551664914523433e-07, + "logits/chosen": 0.29353460669517517, + "logits/rejected": 0.2715173363685608, + "logps/chosen": -78.17903900146484, + "logps/ref_chosen": -61.76142120361328, + "logps/ref_rejected": -72.54627990722656, + "logps/rejected": -93.71284484863281, + "loss": 1.2654, + "margin_dpo/margin_mean": 4.748943328857422, + "margin_dpo/margin_std": 13.215112686157227, + "step": 183 + }, + { + "KL/chosen_KL_mean": -10.467620849609375, + "KL/mean": -14.862070083618164, + "KL/rejected_KL_mean": -19.256515502929688, + "KL/std": 11.553922653198242, + "epoch": 0.2781557067271353, + "fcm_dpo/beta": 0.054390810430049896, + "fcm_dpo/delta": -0.08345025777816772, + "fcm_dpo/margin": 8.788891792297363, + "fcm_dpo/q_t": 0.39447537064552307, + "grad_norm": 18.428091049194336, + "learning_rate": 4.544080985994258e-07, + "logits/chosen": 0.40536069869995117, + "logits/rejected": 0.339550644159317, + "logps/chosen": -57.30834197998047, + "logps/ref_chosen": -46.840721130371094, + "logps/ref_rejected": -69.3609390258789, + "logps/rejected": -88.6174545288086, + "loss": 1.0607, + "margin_dpo/margin_mean": 8.788890838623047, + "margin_dpo/margin_std": 11.725053787231445, + "step": 184 + }, + { + "KL/chosen_KL_mean": -13.35588264465332, + "KL/mean": -18.210655212402344, + "KL/rejected_KL_mean": -23.065425872802734, + "KL/std": 13.737922668457031, + "epoch": 0.2796674225245654, + "fcm_dpo/beta": 0.0531897135078907, + "fcm_dpo/delta": -0.12439177185297012, + "fcm_dpo/margin": 9.709543228149414, + "fcm_dpo/q_t": 0.39072197675704956, + "grad_norm": 17.2995548248291, + "learning_rate": 4.5364398801258394e-07, + "logits/chosen": 0.3512234091758728, + "logits/rejected": 0.3028402030467987, + "logps/chosen": -65.6770248413086, + "logps/ref_chosen": -52.32114028930664, + "logps/ref_rejected": -68.3885726928711, + "logps/rejected": -91.45399475097656, + "loss": 1.1187, + "margin_dpo/margin_mean": 9.70954418182373, + "margin_dpo/margin_std": 16.909622192382812, + "step": 185 + }, + { + "KL/chosen_KL_mean": -11.858358383178711, + "KL/mean": -17.2672176361084, + "KL/rejected_KL_mean": -22.676071166992188, + "KL/std": 13.716083526611328, + "epoch": 0.2811791383219955, + "fcm_dpo/beta": 0.05181612819433212, + "fcm_dpo/delta": -0.17022064328193665, + "fcm_dpo/margin": 10.817716598510742, + "fcm_dpo/q_t": 0.38037991523742676, + "grad_norm": 20.882518768310547, + "learning_rate": 4.5287418106563354e-07, + "logits/chosen": 0.25211092829704285, + "logits/rejected": 0.20962807536125183, + "logps/chosen": -79.27848815917969, + "logps/ref_chosen": -67.42012786865234, + "logps/ref_rejected": -82.50968933105469, + "logps/rejected": -105.18576049804688, + "loss": 1.0654, + "margin_dpo/margin_mean": 10.817716598510742, + "margin_dpo/margin_std": 16.649192810058594, + "step": 186 + }, + { + "KL/chosen_KL_mean": -14.678447723388672, + "KL/mean": -19.588550567626953, + "KL/rejected_KL_mean": -24.4986572265625, + "KL/std": 14.59811019897461, + "epoch": 0.28269085411942557, + "fcm_dpo/beta": 0.050496138632297516, + "fcm_dpo/delta": -0.10132630914449692, + "fcm_dpo/margin": 9.820213317871094, + "fcm_dpo/q_t": 0.39432644844055176, + "grad_norm": 20.352502822875977, + "learning_rate": 4.520986992917297e-07, + "logits/chosen": 0.3316032886505127, + "logits/rejected": 0.2751980423927307, + "logps/chosen": -90.20394134521484, + "logps/ref_chosen": -75.52549743652344, + "logps/ref_rejected": -94.76289367675781, + "logps/rejected": -119.26155090332031, + "loss": 1.1129, + "margin_dpo/margin_mean": 9.820212364196777, + "margin_dpo/margin_std": 16.846996307373047, + "step": 187 + }, + { + "KL/chosen_KL_mean": -13.222888946533203, + "KL/mean": -18.66843032836914, + "KL/rejected_KL_mean": -24.113971710205078, + "KL/std": 15.047087669372559, + "epoch": 0.2842025699168556, + "fcm_dpo/beta": 0.049329958856105804, + "fcm_dpo/delta": -0.14497430622577667, + "fcm_dpo/margin": 10.891085624694824, + "fcm_dpo/q_t": 0.3854219317436218, + "grad_norm": 18.09861946105957, + "learning_rate": 4.5131756438276466e-07, + "logits/chosen": 0.36570295691490173, + "logits/rejected": 0.3198516368865967, + "logps/chosen": -84.7462158203125, + "logps/ref_chosen": -71.52333068847656, + "logps/ref_rejected": -78.29949951171875, + "logps/rejected": -102.41346740722656, + "loss": 1.0563, + "margin_dpo/margin_mean": 10.89108657836914, + "margin_dpo/margin_std": 16.060014724731445, + "step": 188 + }, + { + "KL/chosen_KL_mean": -12.530065536499023, + "KL/mean": -17.523853302001953, + "KL/rejected_KL_mean": -22.51763916015625, + "KL/std": 14.935396194458008, + "epoch": 0.2857142857142857, + "fcm_dpo/beta": 0.04785740375518799, + "fcm_dpo/delta": -0.08463504910469055, + "fcm_dpo/margin": 9.98757266998291, + "fcm_dpo/q_t": 0.3948071599006653, + "grad_norm": 20.050233840942383, + "learning_rate": 4.5053079818876096e-07, + "logits/chosen": 0.35361284017562866, + "logits/rejected": 0.36745667457580566, + "logps/chosen": -84.70632934570312, + "logps/ref_chosen": -72.17626953125, + "logps/ref_rejected": -75.26313781738281, + "logps/rejected": -97.78077697753906, + "loss": 1.102, + "margin_dpo/margin_mean": 9.987573623657227, + "margin_dpo/margin_std": 15.788932800292969, + "step": 189 + }, + { + "KL/chosen_KL_mean": -11.82335090637207, + "KL/mean": -18.92856788635254, + "KL/rejected_KL_mean": -26.033790588378906, + "KL/std": 15.521673202514648, + "epoch": 0.2872260015117158, + "fcm_dpo/beta": 0.046157702803611755, + "fcm_dpo/delta": -0.27415594458580017, + "fcm_dpo/margin": 14.210436820983887, + "fcm_dpo/q_t": 0.35759294033050537, + "grad_norm": 22.220190048217773, + "learning_rate": 4.4973842271726024e-07, + "logits/chosen": 0.39344653487205505, + "logits/rejected": 0.24719932675361633, + "logps/chosen": -66.44762420654297, + "logps/ref_chosen": -54.624271392822266, + "logps/ref_rejected": -101.47068786621094, + "logps/rejected": -127.50447845458984, + "loss": 0.9607, + "margin_dpo/margin_mean": 14.210436820983887, + "margin_dpo/margin_std": 16.54659652709961, + "step": 190 + }, + { + "KL/chosen_KL_mean": -16.818561553955078, + "KL/mean": -22.14589500427246, + "KL/rejected_KL_mean": -27.473224639892578, + "KL/std": 15.95837688446045, + "epoch": 0.2887377173091459, + "fcm_dpo/beta": 0.04499708116054535, + "fcm_dpo/delta": -0.08340764790773392, + "fcm_dpo/margin": 10.6546630859375, + "fcm_dpo/q_t": 0.39313390851020813, + "grad_norm": 19.53053092956543, + "learning_rate": 4.48940460132708e-07, + "logits/chosen": 0.4083729386329651, + "logits/rejected": 0.3787025213241577, + "logps/chosen": -89.7510757446289, + "logps/ref_chosen": -72.93251037597656, + "logps/ref_rejected": -89.95103454589844, + "logps/rejected": -117.42425537109375, + "loss": 1.1007, + "margin_dpo/margin_mean": 10.6546630859375, + "margin_dpo/margin_std": 17.215587615966797, + "step": 191 + }, + { + "KL/chosen_KL_mean": -18.174646377563477, + "KL/mean": -21.69927215576172, + "KL/rejected_KL_mean": -25.223896026611328, + "KL/std": 16.775699615478516, + "epoch": 0.29024943310657597, + "fcm_dpo/beta": 0.045455966144800186, + "fcm_dpo/delta": 0.08187517523765564, + "fcm_dpo/margin": 7.049251556396484, + "fcm_dpo/q_t": 0.4282623529434204, + "grad_norm": 16.56797981262207, + "learning_rate": 4.481369327558329e-07, + "logits/chosen": 0.405287504196167, + "logits/rejected": 0.37762707471847534, + "logps/chosen": -72.17576599121094, + "logps/ref_chosen": -54.001121520996094, + "logps/ref_rejected": -63.531551361083984, + "logps/rejected": -88.75544738769531, + "loss": 1.1946, + "margin_dpo/margin_mean": 7.049251556396484, + "margin_dpo/margin_std": 14.657774925231934, + "step": 192 + }, + { + "KL/chosen_KL_mean": -14.546079635620117, + "KL/mean": -21.136138916015625, + "KL/rejected_KL_mean": -27.726200103759766, + "KL/std": 15.746472358703613, + "epoch": 0.29176114890400606, + "fcm_dpo/beta": 0.04400428384542465, + "fcm_dpo/delta": -0.1927540898323059, + "fcm_dpo/margin": 13.180118560791016, + "fcm_dpo/q_t": 0.3722004294395447, + "grad_norm": 15.471779823303223, + "learning_rate": 4.47327863063023e-07, + "logits/chosen": 0.3393857181072235, + "logits/rejected": 0.3160160183906555, + "logps/chosen": -71.29535675048828, + "logps/ref_chosen": -56.74927520751953, + "logps/ref_rejected": -58.80629348754883, + "logps/rejected": -86.5324935913086, + "loss": 1.0014, + "margin_dpo/margin_mean": 13.180119514465332, + "margin_dpo/margin_std": 16.18925666809082, + "step": 193 + }, + { + "KL/chosen_KL_mean": -15.568546295166016, + "KL/mean": -20.744672775268555, + "KL/rejected_KL_mean": -25.920806884765625, + "KL/std": 15.98554801940918, + "epoch": 0.29327286470143615, + "fcm_dpo/beta": 0.043137095868587494, + "fcm_dpo/delta": -0.05045357346534729, + "fcm_dpo/margin": 10.352262496948242, + "fcm_dpo/q_t": 0.40236881375312805, + "grad_norm": 18.62397575378418, + "learning_rate": 4.4651327368569684e-07, + "logits/chosen": 0.3981412351131439, + "logits/rejected": 0.3676302433013916, + "logps/chosen": -72.21798706054688, + "logps/ref_chosen": -56.64944076538086, + "logps/ref_rejected": -69.98954772949219, + "logps/rejected": -95.91035461425781, + "loss": 1.1542, + "margin_dpo/margin_mean": 10.352262496948242, + "margin_dpo/margin_std": 19.430416107177734, + "step": 194 + }, + { + "KL/chosen_KL_mean": -18.541385650634766, + "KL/mean": -25.322803497314453, + "KL/rejected_KL_mean": -32.10423278808594, + "KL/std": 18.18044090270996, + "epoch": 0.2947845804988662, + "fcm_dpo/beta": 0.04202111437916756, + "fcm_dpo/delta": -0.18148893117904663, + "fcm_dpo/margin": 13.562843322753906, + "fcm_dpo/q_t": 0.37634384632110596, + "grad_norm": 18.791791915893555, + "learning_rate": 4.4569318740967043e-07, + "logits/chosen": 0.30191123485565186, + "logits/rejected": 0.30708640813827515, + "logps/chosen": -88.95116424560547, + "logps/ref_chosen": -70.40977478027344, + "logps/ref_rejected": -74.39448547363281, + "logps/rejected": -106.49871826171875, + "loss": 1.036, + "margin_dpo/margin_mean": 13.562843322753906, + "margin_dpo/margin_std": 18.799701690673828, + "step": 195 + }, + { + "KL/chosen_KL_mean": -17.88593864440918, + "KL/mean": -23.226863861083984, + "KL/rejected_KL_mean": -28.567790985107422, + "KL/std": 18.183853149414062, + "epoch": 0.2962962962962963, + "fcm_dpo/beta": 0.04170439392328262, + "fcm_dpo/delta": -0.04776248335838318, + "fcm_dpo/margin": 10.68185043334961, + "fcm_dpo/q_t": 0.39945489168167114, + "grad_norm": 18.276790618896484, + "learning_rate": 4.448676271745197e-07, + "logits/chosen": 0.37690073251724243, + "logits/rejected": 0.3343625068664551, + "logps/chosen": -77.11351013183594, + "logps/ref_chosen": -59.227577209472656, + "logps/ref_rejected": -83.54757690429688, + "logps/rejected": -112.11536407470703, + "loss": 1.0993, + "margin_dpo/margin_mean": 10.68185043334961, + "margin_dpo/margin_std": 16.383975982666016, + "step": 196 + }, + { + "KL/chosen_KL_mean": -15.438322067260742, + "KL/mean": -22.16175651550293, + "KL/rejected_KL_mean": -28.88519287109375, + "KL/std": 17.83734130859375, + "epoch": 0.29780801209372637, + "fcm_dpo/beta": 0.040766216814517975, + "fcm_dpo/delta": -0.15669496357440948, + "fcm_dpo/margin": 13.446867942810059, + "fcm_dpo/q_t": 0.38244131207466125, + "grad_norm": 17.88084602355957, + "learning_rate": 4.440366160729392e-07, + "logits/chosen": 0.4925154447555542, + "logits/rejected": 0.4390217363834381, + "logps/chosen": -66.96745300292969, + "logps/ref_chosen": -51.52912902832031, + "logps/ref_rejected": -73.70631408691406, + "logps/rejected": -102.59150695800781, + "loss": 1.1024, + "margin_dpo/margin_mean": 13.446868896484375, + "margin_dpo/margin_std": 22.26742935180664, + "step": 197 + }, + { + "KL/chosen_KL_mean": -16.668659210205078, + "KL/mean": -23.898550033569336, + "KL/rejected_KL_mean": -31.12844467163086, + "KL/std": 17.481998443603516, + "epoch": 0.29931972789115646, + "fcm_dpo/beta": 0.039299823343753815, + "fcm_dpo/delta": -0.17802850902080536, + "fcm_dpo/margin": 14.459779739379883, + "fcm_dpo/q_t": 0.37265992164611816, + "grad_norm": 17.161579132080078, + "learning_rate": 4.432001773500957e-07, + "logits/chosen": 0.44338977336883545, + "logits/rejected": 0.401350200176239, + "logps/chosen": -76.45133972167969, + "logps/ref_chosen": -59.78268051147461, + "logps/ref_rejected": -72.24533081054688, + "logps/rejected": -103.37377166748047, + "loss": 1.0073, + "margin_dpo/margin_mean": 14.459779739379883, + "margin_dpo/margin_std": 17.56760025024414, + "step": 198 + }, + { + "KL/chosen_KL_mean": -20.50527000427246, + "KL/mean": -26.135860443115234, + "KL/rejected_KL_mean": -31.76644515991211, + "KL/std": 18.526927947998047, + "epoch": 0.30083144368858655, + "fcm_dpo/beta": 0.03888273611664772, + "fcm_dpo/delta": -0.04109527915716171, + "fcm_dpo/margin": 11.26118278503418, + "fcm_dpo/q_t": 0.40396648645401, + "grad_norm": 16.385759353637695, + "learning_rate": 4.4235833440297856e-07, + "logits/chosen": 0.41728949546813965, + "logits/rejected": 0.3250621557235718, + "logps/chosen": -76.89204406738281, + "logps/ref_chosen": -56.38677215576172, + "logps/ref_rejected": -74.56779479980469, + "logps/rejected": -106.33424377441406, + "loss": 1.1651, + "margin_dpo/margin_mean": 11.261181831359863, + "margin_dpo/margin_std": 21.53316879272461, + "step": 199 + }, + { + "KL/chosen_KL_mean": -15.848258972167969, + "KL/mean": -23.894195556640625, + "KL/rejected_KL_mean": -31.940135955810547, + "KL/std": 19.302684783935547, + "epoch": 0.30234315948601664, + "fcm_dpo/beta": 0.03706369549036026, + "fcm_dpo/delta": -0.2104116529226303, + "fcm_dpo/margin": 16.091880798339844, + "fcm_dpo/q_t": 0.37147045135498047, + "grad_norm": 15.661738395690918, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": 0.4774281978607178, + "logits/rejected": 0.3997184634208679, + "logps/chosen": -73.6725845336914, + "logps/ref_chosen": -57.82432556152344, + "logps/ref_rejected": -89.28246307373047, + "logps/rejected": -121.22259521484375, + "loss": 1.0369, + "margin_dpo/margin_mean": 16.091880798339844, + "margin_dpo/margin_std": 22.954631805419922, + "step": 200 + }, + { + "KL/chosen_KL_mean": -22.75146484375, + "KL/mean": -29.275501251220703, + "KL/rejected_KL_mean": -35.799537658691406, + "KL/std": 20.26548957824707, + "epoch": 0.30385487528344673, + "fcm_dpo/beta": 0.036228105425834656, + "fcm_dpo/delta": -0.07731332629919052, + "fcm_dpo/margin": 13.04806900024414, + "fcm_dpo/q_t": 0.3976749777793884, + "grad_norm": 17.741933822631836, + "learning_rate": 4.4065853017905953e-07, + "logits/chosen": 0.47501319646835327, + "logits/rejected": 0.426998496055603, + "logps/chosen": -81.751220703125, + "logps/ref_chosen": -58.999759674072266, + "logps/ref_rejected": -84.67575073242188, + "logps/rejected": -120.47528076171875, + "loss": 1.1085, + "margin_dpo/margin_mean": 13.04806900024414, + "margin_dpo/margin_std": 21.63117218017578, + "step": 201 + }, + { + "KL/chosen_KL_mean": -18.69136619567871, + "KL/mean": -26.704818725585938, + "KL/rejected_KL_mean": -34.71826934814453, + "KL/std": 19.59726333618164, + "epoch": 0.30536659108087677, + "fcm_dpo/beta": 0.03546854481101036, + "fcm_dpo/delta": -0.1782231628894806, + "fcm_dpo/margin": 16.026901245117188, + "fcm_dpo/q_t": 0.37232154607772827, + "grad_norm": 17.821508407592773, + "learning_rate": 4.3980061644943575e-07, + "logits/chosen": 0.35442060232162476, + "logits/rejected": 0.2809903621673584, + "logps/chosen": -66.35201263427734, + "logps/ref_chosen": -47.660648345947266, + "logps/ref_rejected": -73.63249969482422, + "logps/rejected": -108.35076904296875, + "loss": 1.0311, + "margin_dpo/margin_mean": 16.026901245117188, + "margin_dpo/margin_std": 21.388687133789062, + "step": 202 + }, + { + "KL/chosen_KL_mean": -23.500404357910156, + "KL/mean": -30.568737030029297, + "KL/rejected_KL_mean": -37.63706970214844, + "KL/std": 19.979999542236328, + "epoch": 0.30687830687830686, + "fcm_dpo/beta": 0.03459791839122772, + "fcm_dpo/delta": -0.09357498586177826, + "fcm_dpo/margin": 14.136667251586914, + "fcm_dpo/q_t": 0.3933200538158417, + "grad_norm": 19.208473205566406, + "learning_rate": 4.3893739358856455e-07, + "logits/chosen": 0.46573925018310547, + "logits/rejected": 0.3855220079421997, + "logps/chosen": -85.82594299316406, + "logps/ref_chosen": -62.32553482055664, + "logps/ref_rejected": -99.37226104736328, + "logps/rejected": -137.00933837890625, + "loss": 1.0852, + "margin_dpo/margin_mean": 14.136667251586914, + "margin_dpo/margin_std": 21.907751083374023, + "step": 203 + }, + { + "KL/chosen_KL_mean": -21.444040298461914, + "KL/mean": -29.211734771728516, + "KL/rejected_KL_mean": -36.979427337646484, + "KL/std": 21.172447204589844, + "epoch": 0.30839002267573695, + "fcm_dpo/beta": 0.033351823687553406, + "fcm_dpo/delta": -0.12830305099487305, + "fcm_dpo/margin": 15.535384178161621, + "fcm_dpo/q_t": 0.38586777448654175, + "grad_norm": 17.047832489013672, + "learning_rate": 4.380688857426449e-07, + "logits/chosen": 0.4149578809738159, + "logits/rejected": 0.3435972034931183, + "logps/chosen": -72.07335662841797, + "logps/ref_chosen": -50.62931823730469, + "logps/ref_rejected": -66.60475158691406, + "logps/rejected": -103.58418273925781, + "loss": 1.058, + "margin_dpo/margin_mean": 15.535385131835938, + "margin_dpo/margin_std": 21.494243621826172, + "step": 204 + }, + { + "KL/chosen_KL_mean": -24.892501831054688, + "KL/mean": -32.499061584472656, + "KL/rejected_KL_mean": -40.10561752319336, + "KL/std": 21.863265991210938, + "epoch": 0.30990173847316704, + "fcm_dpo/beta": 0.03289718180894852, + "fcm_dpo/delta": -0.10606454312801361, + "fcm_dpo/margin": 15.213113784790039, + "fcm_dpo/q_t": 0.39138591289520264, + "grad_norm": 20.646154403686523, + "learning_rate": 4.3719511720570814e-07, + "logits/chosen": 0.4746994972229004, + "logits/rejected": 0.40480706095695496, + "logps/chosen": -95.2486801147461, + "logps/ref_chosen": -70.3561782836914, + "logps/ref_rejected": -93.39848327636719, + "logps/rejected": -133.50408935546875, + "loss": 1.092, + "margin_dpo/margin_mean": 15.213114738464355, + "margin_dpo/margin_std": 24.427967071533203, + "step": 205 + }, + { + "KL/chosen_KL_mean": -27.019397735595703, + "KL/mean": -32.2818489074707, + "KL/rejected_KL_mean": -37.54429626464844, + "KL/std": 21.710651397705078, + "epoch": 0.31141345427059713, + "fcm_dpo/beta": 0.033220645040273666, + "fcm_dpo/delta": 0.05127355828881264, + "fcm_dpo/margin": 10.524890899658203, + "fcm_dpo/q_t": 0.42648404836654663, + "grad_norm": 18.123523712158203, + "learning_rate": 4.363161124189387e-07, + "logits/chosen": 0.496069073677063, + "logits/rejected": 0.47948509454727173, + "logps/chosen": -94.66487884521484, + "logps/ref_chosen": -67.64547729492188, + "logps/ref_rejected": -79.89584350585938, + "logps/rejected": -117.44013977050781, + "loss": 1.2423, + "margin_dpo/margin_mean": 10.52489185333252, + "margin_dpo/margin_std": 26.20848274230957, + "step": 206 + }, + { + "KL/chosen_KL_mean": -31.685041427612305, + "KL/mean": -39.493553161621094, + "KL/rejected_KL_mean": -47.30206298828125, + "KL/std": 22.86937713623047, + "epoch": 0.3129251700680272, + "fcm_dpo/beta": 0.032525695860385895, + "fcm_dpo/delta": -0.1139114499092102, + "fcm_dpo/margin": 15.617015838623047, + "fcm_dpo/q_t": 0.39173316955566406, + "grad_norm": 15.79115104675293, + "learning_rate": 4.3543189596998986e-07, + "logits/chosen": 0.42408090829849243, + "logits/rejected": 0.35405462980270386, + "logps/chosen": -99.34923553466797, + "logps/ref_chosen": -67.66419219970703, + "logps/ref_rejected": -85.10249328613281, + "logps/rejected": -132.40455627441406, + "loss": 1.0747, + "margin_dpo/margin_mean": 15.61701774597168, + "margin_dpo/margin_std": 23.907955169677734, + "step": 207 + }, + { + "KL/chosen_KL_mean": -25.132305145263672, + "KL/mean": -29.537364959716797, + "KL/rejected_KL_mean": -33.942420959472656, + "KL/std": 21.290771484375, + "epoch": 0.3144368858654573, + "fcm_dpo/beta": 0.032919660210609436, + "fcm_dpo/delta": 0.11329277604818344, + "fcm_dpo/margin": 8.810117721557617, + "fcm_dpo/q_t": 0.4377458691596985, + "grad_norm": 22.17745018005371, + "learning_rate": 4.3454249259229664e-07, + "logits/chosen": 0.4305967688560486, + "logits/rejected": 0.4044501781463623, + "logps/chosen": -82.864013671875, + "logps/ref_chosen": -57.731712341308594, + "logps/ref_rejected": -74.19276428222656, + "logps/rejected": -108.13518524169922, + "loss": 1.2647, + "margin_dpo/margin_mean": 8.810117721557617, + "margin_dpo/margin_std": 23.99683952331543, + "step": 208 + }, + { + "KL/chosen_KL_mean": -26.022214889526367, + "KL/mean": -35.826934814453125, + "KL/rejected_KL_mean": -45.631656646728516, + "KL/std": 24.46694564819336, + "epoch": 0.31594860166288735, + "fcm_dpo/beta": 0.03194945305585861, + "fcm_dpo/delta": -0.24169307947158813, + "fcm_dpo/margin": 19.609439849853516, + "fcm_dpo/q_t": 0.3655078411102295, + "grad_norm": 18.581724166870117, + "learning_rate": 4.336479271643833e-07, + "logits/chosen": 0.382016658782959, + "logits/rejected": 0.327199250459671, + "logps/chosen": -94.57229614257812, + "logps/ref_chosen": -68.55007934570312, + "logps/ref_rejected": -87.90541076660156, + "logps/rejected": -133.53707885742188, + "loss": 1.0394, + "margin_dpo/margin_mean": 19.609439849853516, + "margin_dpo/margin_std": 28.908449172973633, + "step": 209 + }, + { + "KL/chosen_KL_mean": -25.5579891204834, + "KL/mean": -34.911468505859375, + "KL/rejected_KL_mean": -44.26494598388672, + "KL/std": 26.126291275024414, + "epoch": 0.31746031746031744, + "fcm_dpo/beta": 0.030834760516881943, + "fcm_dpo/delta": -0.18722277879714966, + "fcm_dpo/margin": 18.706954956054688, + "fcm_dpo/q_t": 0.3771812915802002, + "grad_norm": 16.534156799316406, + "learning_rate": 4.327482247091679e-07, + "logits/chosen": 0.5074554681777954, + "logits/rejected": 0.40456104278564453, + "logps/chosen": -82.82626342773438, + "logps/ref_chosen": -57.268272399902344, + "logps/ref_rejected": -85.72807312011719, + "logps/rejected": -129.99302673339844, + "loss": 1.0423, + "margin_dpo/margin_mean": 18.706954956054688, + "margin_dpo/margin_std": 26.688194274902344, + "step": 210 + }, + { + "KL/chosen_KL_mean": -23.136999130249023, + "KL/mean": -32.110008239746094, + "KL/rejected_KL_mean": -41.0830192565918, + "KL/std": 23.694541931152344, + "epoch": 0.31897203325774753, + "fcm_dpo/beta": 0.029924146831035614, + "fcm_dpo/delta": -0.14464184641838074, + "fcm_dpo/margin": 17.946014404296875, + "fcm_dpo/q_t": 0.38534092903137207, + "grad_norm": 20.95856285095215, + "learning_rate": 4.3184341039326217e-07, + "logits/chosen": 0.5217145681381226, + "logits/rejected": 0.4215095043182373, + "logps/chosen": -76.7777099609375, + "logps/ref_chosen": -53.640708923339844, + "logps/ref_rejected": -93.0387954711914, + "logps/rejected": -134.12181091308594, + "loss": 1.0483, + "margin_dpo/margin_mean": 17.946014404296875, + "margin_dpo/margin_std": 25.320432662963867, + "step": 211 + }, + { + "KL/chosen_KL_mean": -27.168684005737305, + "KL/mean": -36.86266326904297, + "KL/rejected_KL_mean": -46.556636810302734, + "KL/std": 25.865568161010742, + "epoch": 0.3204837490551776, + "fcm_dpo/beta": 0.02877327799797058, + "fcm_dpo/delta": -0.16739435493946075, + "fcm_dpo/margin": 19.387954711914062, + "fcm_dpo/q_t": 0.3769418001174927, + "grad_norm": 14.226759910583496, + "learning_rate": 4.309335095262675e-07, + "logits/chosen": 0.5224308967590332, + "logits/rejected": 0.4461041986942291, + "logps/chosen": -84.53543090820312, + "logps/ref_chosen": -57.36674499511719, + "logps/ref_rejected": -79.89643096923828, + "logps/rejected": -126.45306396484375, + "loss": 1.0449, + "margin_dpo/margin_mean": 19.387954711914062, + "margin_dpo/margin_std": 27.3609619140625, + "step": 212 + }, + { + "KL/chosen_KL_mean": -20.49267578125, + "KL/mean": -31.65011215209961, + "KL/rejected_KL_mean": -42.80754852294922, + "KL/std": 26.360469818115234, + "epoch": 0.3219954648526077, + "fcm_dpo/beta": 0.02747097983956337, + "fcm_dpo/delta": -0.22853778302669525, + "fcm_dpo/margin": 22.314865112304688, + "fcm_dpo/q_t": 0.3668210506439209, + "grad_norm": 13.588920593261719, + "learning_rate": 4.3001854756006724e-07, + "logits/chosen": 0.48388439416885376, + "logits/rejected": 0.4597597122192383, + "logps/chosen": -85.71379089355469, + "logps/ref_chosen": -65.22111511230469, + "logps/ref_rejected": -80.1810302734375, + "logps/rejected": -122.98857116699219, + "loss": 1.0092, + "margin_dpo/margin_mean": 22.314865112304688, + "margin_dpo/margin_std": 29.59410858154297, + "step": 213 + }, + { + "KL/chosen_KL_mean": -24.507936477661133, + "KL/mean": -35.89101791381836, + "KL/rejected_KL_mean": -47.27410125732422, + "KL/std": 26.936935424804688, + "epoch": 0.3235071806500378, + "fcm_dpo/beta": 0.02660961076617241, + "fcm_dpo/delta": -0.2185751348733902, + "fcm_dpo/margin": 22.76616668701172, + "fcm_dpo/q_t": 0.3699638247489929, + "grad_norm": 17.4736270904541, + "learning_rate": 4.290985500881143e-07, + "logits/chosen": 0.36848658323287964, + "logits/rejected": 0.34894949197769165, + "logps/chosen": -85.80026245117188, + "logps/ref_chosen": -61.292327880859375, + "logps/ref_rejected": -67.69841003417969, + "logps/rejected": -114.97250366210938, + "loss": 1.0248, + "margin_dpo/margin_mean": 22.76616668701172, + "margin_dpo/margin_std": 30.800565719604492, + "step": 214 + }, + { + "KL/chosen_KL_mean": -30.14183235168457, + "KL/mean": -41.4921875, + "KL/rejected_KL_mean": -52.84253692626953, + "KL/std": 28.326629638671875, + "epoch": 0.3250188964474679, + "fcm_dpo/beta": 0.025595759972929955, + "fcm_dpo/delta": -0.19203141331672668, + "fcm_dpo/margin": 22.700714111328125, + "fcm_dpo/q_t": 0.3764479160308838, + "grad_norm": 15.755528450012207, + "learning_rate": 4.281735428447157e-07, + "logits/chosen": 0.36972734332084656, + "logits/rejected": 0.2591157853603363, + "logps/chosen": -94.01097106933594, + "logps/ref_chosen": -63.869136810302734, + "logps/ref_rejected": -98.7657241821289, + "logps/rejected": -151.60826110839844, + "loss": 1.0497, + "margin_dpo/margin_mean": 22.700716018676758, + "margin_dpo/margin_std": 32.51849365234375, + "step": 215 + }, + { + "KL/chosen_KL_mean": -26.490917205810547, + "KL/mean": -38.6580810546875, + "KL/rejected_KL_mean": -50.82523727416992, + "KL/std": 28.62359619140625, + "epoch": 0.32653061224489793, + "fcm_dpo/beta": 0.024382300674915314, + "fcm_dpo/delta": -0.2054787278175354, + "fcm_dpo/margin": 24.334320068359375, + "fcm_dpo/q_t": 0.36771076917648315, + "grad_norm": 21.268993377685547, + "learning_rate": 4.2724355170431247e-07, + "logits/chosen": 0.5125446915626526, + "logits/rejected": 0.42290613055229187, + "logps/chosen": -94.31587219238281, + "logps/ref_chosen": -67.824951171875, + "logps/ref_rejected": -96.40231323242188, + "logps/rejected": -147.22755432128906, + "loss": 0.9914, + "margin_dpo/margin_mean": 24.334320068359375, + "margin_dpo/margin_std": 29.2112979888916, + "step": 216 + }, + { + "KL/chosen_KL_mean": -30.6090087890625, + "KL/mean": -43.44352340698242, + "KL/rejected_KL_mean": -56.278038024902344, + "KL/std": 30.038503646850586, + "epoch": 0.328042328042328, + "fcm_dpo/beta": 0.02329612709581852, + "fcm_dpo/delta": -0.21151894330978394, + "fcm_dpo/margin": 25.669031143188477, + "fcm_dpo/q_t": 0.3687037229537964, + "grad_norm": 13.630374908447266, + "learning_rate": 4.26308602680756e-07, + "logits/chosen": 0.46526721119880676, + "logits/rejected": 0.3592798709869385, + "logps/chosen": -91.11399841308594, + "logps/ref_chosen": -60.5049934387207, + "logps/ref_rejected": -84.26618194580078, + "logps/rejected": -140.54421997070312, + "loss": 0.9964, + "margin_dpo/margin_mean": 25.669031143188477, + "margin_dpo/margin_std": 31.728784561157227, + "step": 217 + }, + { + "KL/chosen_KL_mean": -33.458717346191406, + "KL/mean": -41.555747985839844, + "KL/rejected_KL_mean": -49.65278625488281, + "KL/std": 29.070512771606445, + "epoch": 0.3295540438397581, + "fcm_dpo/beta": 0.02337442710995674, + "fcm_dpo/delta": 0.020912915468215942, + "fcm_dpo/margin": 16.194072723388672, + "fcm_dpo/q_t": 0.4174070358276367, + "grad_norm": 16.084028244018555, + "learning_rate": 4.253687219265803e-07, + "logits/chosen": 0.3525770306587219, + "logits/rejected": 0.3516993820667267, + "logps/chosen": -104.05303192138672, + "logps/ref_chosen": -70.59431457519531, + "logps/ref_rejected": -73.89038848876953, + "logps/rejected": -123.54317474365234, + "loss": 1.2067, + "margin_dpo/margin_mean": 16.194074630737305, + "margin_dpo/margin_std": 35.17280960083008, + "step": 218 + }, + { + "KL/chosen_KL_mean": -32.29864501953125, + "KL/mean": -40.812965393066406, + "KL/rejected_KL_mean": -49.32728576660156, + "KL/std": 30.129287719726562, + "epoch": 0.3310657596371882, + "fcm_dpo/beta": 0.023134605959057808, + "fcm_dpo/delta": 0.0059198178350925446, + "fcm_dpo/margin": 17.028644561767578, + "fcm_dpo/q_t": 0.4128570556640625, + "grad_norm": 16.333072662353516, + "learning_rate": 4.2442393573227043e-07, + "logits/chosen": 0.44589486718177795, + "logits/rejected": 0.40339186787605286, + "logps/chosen": -92.78958892822266, + "logps/ref_chosen": -60.490943908691406, + "logps/ref_rejected": -75.85001373291016, + "logps/rejected": -125.17729187011719, + "loss": 1.1414, + "margin_dpo/margin_mean": 17.028642654418945, + "margin_dpo/margin_std": 29.682361602783203, + "step": 219 + }, + { + "KL/chosen_KL_mean": -27.624902725219727, + "KL/mean": -37.65066909790039, + "KL/rejected_KL_mean": -47.67643356323242, + "KL/std": 30.960710525512695, + "epoch": 0.3325774754346183, + "fcm_dpo/beta": 0.02288152649998665, + "fcm_dpo/delta": -0.06287384033203125, + "fcm_dpo/margin": 20.051528930664062, + "fcm_dpo/q_t": 0.40230315923690796, + "grad_norm": 13.608770370483398, + "learning_rate": 4.234742705255272e-07, + "logits/chosen": 0.5344582796096802, + "logits/rejected": 0.46789342164993286, + "logps/chosen": -72.6383056640625, + "logps/ref_chosen": -45.013397216796875, + "logps/ref_rejected": -70.49369812011719, + "logps/rejected": -118.17013549804688, + "loss": 1.1337, + "margin_dpo/margin_mean": 20.05152702331543, + "margin_dpo/margin_std": 35.65072250366211, + "step": 220 + }, + { + "KL/chosen_KL_mean": -27.55103874206543, + "KL/mean": -38.84452438354492, + "KL/rejected_KL_mean": -50.13801574707031, + "KL/std": 30.538475036621094, + "epoch": 0.3340891912320484, + "fcm_dpo/beta": 0.022582344710826874, + "fcm_dpo/delta": -0.11589628458023071, + "fcm_dpo/margin": 22.58698081970215, + "fcm_dpo/q_t": 0.3906116783618927, + "grad_norm": 16.422992706298828, + "learning_rate": 4.22519752870528e-07, + "logits/chosen": 0.512248158454895, + "logits/rejected": 0.4338856339454651, + "logps/chosen": -86.64688110351562, + "logps/ref_chosen": -59.09584045410156, + "logps/ref_rejected": -88.64388275146484, + "logps/rejected": -138.78189086914062, + "loss": 1.0787, + "margin_dpo/margin_mean": 22.58698081970215, + "margin_dpo/margin_std": 35.058292388916016, + "step": 221 + }, + { + "KL/chosen_KL_mean": -28.761634826660156, + "KL/mean": -43.88484573364258, + "KL/rejected_KL_mean": -59.00804901123047, + "KL/std": 34.609222412109375, + "epoch": 0.3356009070294785, + "fcm_dpo/beta": 0.02151619829237461, + "fcm_dpo/delta": -0.2687666416168213, + "fcm_dpo/margin": 30.246416091918945, + "fcm_dpo/q_t": 0.3583294153213501, + "grad_norm": 15.557254791259766, + "learning_rate": 4.2156040946718343e-07, + "logits/chosen": 0.5562437772750854, + "logits/rejected": 0.46473515033721924, + "logps/chosen": -84.75932312011719, + "logps/ref_chosen": -55.9976921081543, + "logps/ref_rejected": -111.94727325439453, + "logps/rejected": -170.955322265625, + "loss": 0.9796, + "margin_dpo/margin_mean": 30.246417999267578, + "margin_dpo/margin_std": 37.03383255004883, + "step": 222 + }, + { + "KL/chosen_KL_mean": -34.243709564208984, + "KL/mean": -47.062828063964844, + "KL/rejected_KL_mean": -59.88194274902344, + "KL/std": 34.03950500488281, + "epoch": 0.3371126228269085, + "fcm_dpo/beta": 0.020738139748573303, + "fcm_dpo/delta": -0.13991129398345947, + "fcm_dpo/margin": 25.63823699951172, + "fcm_dpo/q_t": 0.3834461569786072, + "grad_norm": 15.027467727661133, + "learning_rate": 4.2059626715039065e-07, + "logits/chosen": 0.5505378246307373, + "logits/rejected": 0.4921608865261078, + "logps/chosen": -94.1351318359375, + "logps/ref_chosen": -59.891422271728516, + "logps/ref_rejected": -86.28954315185547, + "logps/rejected": -146.17147827148438, + "loss": 1.0376, + "margin_dpo/margin_mean": 25.63823699951172, + "margin_dpo/margin_std": 34.131752014160156, + "step": 223 + }, + { + "KL/chosen_KL_mean": -39.61954116821289, + "KL/mean": -47.04735565185547, + "KL/rejected_KL_mean": -54.475162506103516, + "KL/std": 33.57848358154297, + "epoch": 0.3386243386243386, + "fcm_dpo/beta": 0.02092786133289337, + "fcm_dpo/delta": 0.0920458734035492, + "fcm_dpo/margin": 14.855623245239258, + "fcm_dpo/q_t": 0.4327733814716339, + "grad_norm": 21.00340461730957, + "learning_rate": 4.1962735288928304e-07, + "logits/chosen": 0.5788267850875854, + "logits/rejected": 0.5559718608856201, + "logps/chosen": -103.66417694091797, + "logps/ref_chosen": -64.04463195800781, + "logps/ref_rejected": -75.05450439453125, + "logps/rejected": -129.5296630859375, + "loss": 1.2273, + "margin_dpo/margin_mean": 14.855621337890625, + "margin_dpo/margin_std": 35.59134292602539, + "step": 224 + }, + { + "KL/chosen_KL_mean": -37.78761291503906, + "KL/mean": -51.74939727783203, + "KL/rejected_KL_mean": -65.711181640625, + "KL/std": 37.48065185546875, + "epoch": 0.3401360544217687, + "fcm_dpo/beta": 0.020355746150016785, + "fcm_dpo/delta": -0.18072660267353058, + "fcm_dpo/margin": 27.923568725585938, + "fcm_dpo/q_t": 0.3758217692375183, + "grad_norm": 14.574810028076172, + "learning_rate": 4.186536937864752e-07, + "logits/chosen": 0.5684969425201416, + "logits/rejected": 0.4474506974220276, + "logps/chosen": -103.88343811035156, + "logps/ref_chosen": -66.0958251953125, + "logps/ref_rejected": -97.68675231933594, + "logps/rejected": -163.39793395996094, + "loss": 1.0172, + "margin_dpo/margin_mean": 27.923568725585938, + "margin_dpo/margin_std": 35.680519104003906, + "step": 225 + }, + { + "KL/chosen_KL_mean": -35.753868103027344, + "KL/mean": -46.9656982421875, + "KL/rejected_KL_mean": -58.17752456665039, + "KL/std": 34.91864776611328, + "epoch": 0.3416477702191988, + "fcm_dpo/beta": 0.020231008529663086, + "fcm_dpo/delta": -0.056395478546619415, + "fcm_dpo/margin": 22.42365837097168, + "fcm_dpo/q_t": 0.40082597732543945, + "grad_norm": 14.549175262451172, + "learning_rate": 4.176753170773052e-07, + "logits/chosen": 0.5831120610237122, + "logits/rejected": 0.5354971885681152, + "logps/chosen": -87.17073822021484, + "logps/ref_chosen": -51.4168701171875, + "logps/ref_rejected": -66.30068969726562, + "logps/rejected": -124.47821044921875, + "loss": 1.1442, + "margin_dpo/margin_mean": 22.42365837097168, + "margin_dpo/margin_std": 41.22904586791992, + "step": 226 + }, + { + "KL/chosen_KL_mean": -38.926422119140625, + "KL/mean": -50.359954833984375, + "KL/rejected_KL_mean": -61.79349136352539, + "KL/std": 38.167694091796875, + "epoch": 0.3431594860166289, + "fcm_dpo/beta": 0.019856732338666916, + "fcm_dpo/delta": -0.05691502243280411, + "fcm_dpo/margin": 22.86707305908203, + "fcm_dpo/q_t": 0.4031808376312256, + "grad_norm": 16.369155883789062, + "learning_rate": 4.166922501290729e-07, + "logits/chosen": 0.6283693313598633, + "logits/rejected": 0.5883187651634216, + "logps/chosen": -96.91619873046875, + "logps/ref_chosen": -57.989776611328125, + "logps/ref_rejected": -75.05464172363281, + "logps/rejected": -136.84814453125, + "loss": 1.1451, + "margin_dpo/margin_mean": 22.867074966430664, + "margin_dpo/margin_std": 42.62117004394531, + "step": 227 + }, + { + "KL/chosen_KL_mean": -42.122718811035156, + "KL/mean": -54.82220458984375, + "KL/rejected_KL_mean": -67.52168273925781, + "KL/std": 36.266571044921875, + "epoch": 0.34467120181405897, + "fcm_dpo/beta": 0.019605718553066254, + "fcm_dpo/delta": -0.10297306627035141, + "fcm_dpo/margin": 25.39896011352539, + "fcm_dpo/q_t": 0.39041581749916077, + "grad_norm": 15.85113525390625, + "learning_rate": 4.1570452044027405e-07, + "logits/chosen": 0.5884374976158142, + "logits/rejected": 0.5055756568908691, + "logps/chosen": -97.68208312988281, + "logps/ref_chosen": -55.55936813354492, + "logps/ref_rejected": -77.02364349365234, + "logps/rejected": -144.54531860351562, + "loss": 1.0724, + "margin_dpo/margin_mean": 25.39896011352539, + "margin_dpo/margin_std": 37.421138763427734, + "step": 228 + }, + { + "KL/chosen_KL_mean": -34.708621978759766, + "KL/mean": -46.20718765258789, + "KL/rejected_KL_mean": -57.70575714111328, + "KL/std": 35.467079162597656, + "epoch": 0.34618291761148906, + "fcm_dpo/beta": 0.019299227744340897, + "fcm_dpo/delta": -0.04590361565351486, + "fcm_dpo/margin": 22.997133255004883, + "fcm_dpo/q_t": 0.4034169912338257, + "grad_norm": 22.405696868896484, + "learning_rate": 4.147121556398312e-07, + "logits/chosen": 0.6827352046966553, + "logits/rejected": 0.6061787605285645, + "logps/chosen": -85.50328063964844, + "logps/ref_chosen": -50.79466247558594, + "logps/ref_rejected": -78.4474105834961, + "logps/rejected": -136.15316772460938, + "loss": 1.1506, + "margin_dpo/margin_mean": 22.997133255004883, + "margin_dpo/margin_std": 43.55659484863281, + "step": 229 + }, + { + "KL/chosen_KL_mean": -38.71097946166992, + "KL/mean": -52.039764404296875, + "KL/rejected_KL_mean": -65.36854553222656, + "KL/std": 39.001365661621094, + "epoch": 0.3476946334089191, + "fcm_dpo/beta": 0.01911795511841774, + "fcm_dpo/delta": -0.1162446141242981, + "fcm_dpo/margin": 26.657567977905273, + "fcm_dpo/q_t": 0.3882319927215576, + "grad_norm": 14.44546890258789, + "learning_rate": 4.137151834863213e-07, + "logits/chosen": 0.5673795938491821, + "logits/rejected": 0.5688091516494751, + "logps/chosen": -95.44020080566406, + "logps/ref_chosen": -56.729225158691406, + "logps/ref_rejected": -62.99180603027344, + "logps/rejected": -128.3603515625, + "loss": 1.0652, + "margin_dpo/margin_mean": 26.65756607055664, + "margin_dpo/margin_std": 37.82829284667969, + "step": 230 + }, + { + "KL/chosen_KL_mean": -44.53082275390625, + "KL/mean": -64.40026092529297, + "KL/rejected_KL_mean": -84.26969909667969, + "KL/std": 38.41732406616211, + "epoch": 0.3492063492063492, + "fcm_dpo/beta": 0.017824435606598854, + "fcm_dpo/delta": -0.3340993821620941, + "fcm_dpo/margin": 39.738887786865234, + "fcm_dpo/q_t": 0.3404316008090973, + "grad_norm": 14.944037437438965, + "learning_rate": 4.1271363186719835e-07, + "logits/chosen": 0.49198731780052185, + "logits/rejected": 0.4875026047229767, + "logps/chosen": -117.12792205810547, + "logps/ref_chosen": -72.59709930419922, + "logps/ref_rejected": -86.2322998046875, + "logps/rejected": -170.5019989013672, + "loss": 0.9208, + "margin_dpo/margin_mean": 39.73888397216797, + "margin_dpo/margin_std": 41.09081268310547, + "step": 231 + }, + { + "KL/chosen_KL_mean": -44.682891845703125, + "KL/mean": -58.00080108642578, + "KL/rejected_KL_mean": -71.31871032714844, + "KL/std": 41.824562072753906, + "epoch": 0.3507180650037793, + "fcm_dpo/beta": 0.017421916127204895, + "fcm_dpo/delta": -0.06721210479736328, + "fcm_dpo/margin": 26.635814666748047, + "fcm_dpo/q_t": 0.4011406898498535, + "grad_norm": 15.53917407989502, + "learning_rate": 4.1170752879801436e-07, + "logits/chosen": 0.539695143699646, + "logits/rejected": 0.5095845460891724, + "logps/chosen": -112.80142211914062, + "logps/ref_chosen": -68.1185302734375, + "logps/ref_rejected": -83.79415893554688, + "logps/rejected": -155.1128692626953, + "loss": 1.1295, + "margin_dpo/margin_mean": 26.635812759399414, + "margin_dpo/margin_std": 47.687801361083984, + "step": 232 + }, + { + "KL/chosen_KL_mean": -51.6092643737793, + "KL/mean": -61.849483489990234, + "KL/rejected_KL_mean": -72.08970642089844, + "KL/std": 39.40514373779297, + "epoch": 0.35222978080120937, + "fcm_dpo/beta": 0.01731543242931366, + "fcm_dpo/delta": 0.04540575295686722, + "fcm_dpo/margin": 20.48044204711914, + "fcm_dpo/q_t": 0.4230949878692627, + "grad_norm": 14.722317695617676, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": 0.599013090133667, + "logits/rejected": 0.5388975739479065, + "logps/chosen": -106.67941284179688, + "logps/ref_chosen": -55.070152282714844, + "logps/ref_rejected": -66.61845397949219, + "logps/rejected": -138.70816040039062, + "loss": 1.1796, + "margin_dpo/margin_mean": 20.48044204711914, + "margin_dpo/margin_std": 39.12367248535156, + "step": 233 + }, + { + "KL/chosen_KL_mean": -46.10203170776367, + "KL/mean": -56.328800201416016, + "KL/rejected_KL_mean": -66.5555648803711, + "KL/std": 38.45304489135742, + "epoch": 0.35374149659863946, + "fcm_dpo/beta": 0.017331501469016075, + "fcm_dpo/delta": -0.11514575034379959, + "fcm_dpo/margin": 20.453533172607422, + "fcm_dpo/q_t": 0.42306917905807495, + "grad_norm": 20.012310028076172, + "learning_rate": 4.09681781007452e-07, + "logits/chosen": 0.5130677819252014, + "logits/rejected": 0.506496787071228, + "logps/chosen": -102.02792358398438, + "logps/ref_chosen": -55.92589569091797, + "logps/ref_rejected": -51.11608123779297, + "logps/rejected": -117.67164611816406, + "loss": 1.2025, + "margin_dpo/margin_mean": 20.45353126525879, + "margin_dpo/margin_std": 41.53302001953125, + "step": 234 + }, + { + "KL/chosen_KL_mean": -40.315879821777344, + "KL/mean": -57.65726089477539, + "KL/rejected_KL_mean": -74.99864196777344, + "KL/std": 41.41876220703125, + "epoch": 0.35525321239606955, + "fcm_dpo/beta": 0.016705987975001335, + "fcm_dpo/delta": -0.1900615096092224, + "fcm_dpo/margin": 34.68276596069336, + "fcm_dpo/q_t": 0.36982935667037964, + "grad_norm": 14.829582214355469, + "learning_rate": 4.08662192950594e-07, + "logits/chosen": 0.6136411428451538, + "logits/rejected": 0.5992604494094849, + "logps/chosen": -104.85560607910156, + "logps/ref_chosen": -64.53972625732422, + "logps/ref_rejected": -77.69151306152344, + "logps/rejected": -152.69015502929688, + "loss": 0.9932, + "margin_dpo/margin_mean": 34.682762145996094, + "margin_dpo/margin_std": 40.54000473022461, + "step": 235 + }, + { + "KL/chosen_KL_mean": -61.0404052734375, + "KL/mean": -74.00381469726562, + "KL/rejected_KL_mean": -86.96721649169922, + "KL/std": 40.741188049316406, + "epoch": 0.35676492819349964, + "fcm_dpo/beta": 0.016382815316319466, + "fcm_dpo/delta": -0.02608785405755043, + "fcm_dpo/margin": 25.926815032958984, + "fcm_dpo/q_t": 0.4062482714653015, + "grad_norm": 14.39378547668457, + "learning_rate": 4.076381667711306e-07, + "logits/chosen": 0.5750887393951416, + "logits/rejected": 0.5613845586776733, + "logps/chosen": -132.1951446533203, + "logps/ref_chosen": -71.15473937988281, + "logps/ref_rejected": -84.88541412353516, + "logps/rejected": -171.85263061523438, + "loss": 1.1396, + "margin_dpo/margin_mean": 25.926815032958984, + "margin_dpo/margin_std": 46.30166244506836, + "step": 236 + }, + { + "KL/chosen_KL_mean": -51.05704116821289, + "KL/mean": -66.93670654296875, + "KL/rejected_KL_mean": -82.8163833618164, + "KL/std": 40.99628448486328, + "epoch": 0.35827664399092973, + "fcm_dpo/beta": 0.01615855097770691, + "fcm_dpo/delta": -0.1191396713256836, + "fcm_dpo/margin": 31.75934410095215, + "fcm_dpo/q_t": 0.3859434723854065, + "grad_norm": 17.10631561279297, + "learning_rate": 4.066097311132753e-07, + "logits/chosen": 0.5949573516845703, + "logits/rejected": 0.5852821469306946, + "logps/chosen": -127.19905090332031, + "logps/ref_chosen": -76.14201354980469, + "logps/ref_rejected": -80.88479614257812, + "logps/rejected": -163.701171875, + "loss": 1.0751, + "margin_dpo/margin_mean": 31.759342193603516, + "margin_dpo/margin_std": 47.10854721069336, + "step": 237 + }, + { + "KL/chosen_KL_mean": -45.8564567565918, + "KL/mean": -61.5413932800293, + "KL/rejected_KL_mean": -77.22633361816406, + "KL/std": 42.1279411315918, + "epoch": 0.35978835978835977, + "fcm_dpo/beta": 0.015698228031396866, + "fcm_dpo/delta": -0.09846552461385727, + "fcm_dpo/margin": 31.36986541748047, + "fcm_dpo/q_t": 0.39032959938049316, + "grad_norm": 19.96809959411621, + "learning_rate": 4.0557691474458414e-07, + "logits/chosen": 0.5656386017799377, + "logits/rejected": 0.5574653148651123, + "logps/chosen": -114.74130249023438, + "logps/ref_chosen": -68.88484954833984, + "logps/ref_rejected": -75.8946304321289, + "logps/rejected": -153.1209716796875, + "loss": 1.0689, + "margin_dpo/margin_mean": 31.36986541748047, + "margin_dpo/margin_std": 44.51061248779297, + "step": 238 + }, + { + "KL/chosen_KL_mean": -56.033790588378906, + "KL/mean": -71.40078735351562, + "KL/rejected_KL_mean": -86.76777648925781, + "KL/std": 45.64823913574219, + "epoch": 0.36130007558578986, + "fcm_dpo/beta": 0.015608660876750946, + "fcm_dpo/delta": -0.08397899568080902, + "fcm_dpo/margin": 30.733985900878906, + "fcm_dpo/q_t": 0.39459508657455444, + "grad_norm": 18.36031723022461, + "learning_rate": 4.045397465551513e-07, + "logits/chosen": 0.6763525009155273, + "logits/rejected": 0.5366228222846985, + "logps/chosen": -112.80561828613281, + "logps/ref_chosen": -56.771827697753906, + "logps/ref_rejected": -116.23050689697266, + "logps/rejected": -202.998291015625, + "loss": 1.0954, + "margin_dpo/margin_mean": 30.733989715576172, + "margin_dpo/margin_std": 47.917232513427734, + "step": 239 + }, + { + "KL/chosen_KL_mean": -54.066680908203125, + "KL/mean": -73.76835632324219, + "KL/rejected_KL_mean": -93.47004699707031, + "KL/std": 45.91557312011719, + "epoch": 0.36281179138321995, + "fcm_dpo/beta": 0.014995868317782879, + "fcm_dpo/delta": -0.2027919590473175, + "fcm_dpo/margin": 39.40336608886719, + "fcm_dpo/q_t": 0.36883121728897095, + "grad_norm": 13.07312297821045, + "learning_rate": 4.0349825555680045e-07, + "logits/chosen": 0.5906857848167419, + "logits/rejected": 0.49216535687446594, + "logps/chosen": -107.42079162597656, + "logps/ref_chosen": -53.35411071777344, + "logps/ref_rejected": -80.12019348144531, + "logps/rejected": -173.59024047851562, + "loss": 0.996, + "margin_dpo/margin_mean": 39.40336608886719, + "margin_dpo/margin_std": 47.73066711425781, + "step": 240 + }, + { + "KL/chosen_KL_mean": -50.90334701538086, + "KL/mean": -63.605812072753906, + "KL/rejected_KL_mean": -76.30827331542969, + "KL/std": 42.9580078125, + "epoch": 0.36432350718065004, + "fcm_dpo/beta": 0.0149228535592556, + "fcm_dpo/delta": 0.021481888368725777, + "fcm_dpo/margin": 25.404935836791992, + "fcm_dpo/q_t": 0.4165813624858856, + "grad_norm": 17.74489402770996, + "learning_rate": 4.0245247088227377e-07, + "logits/chosen": 0.5852094292640686, + "logits/rejected": 0.5466130375862122, + "logps/chosen": -122.79875946044922, + "logps/ref_chosen": -71.89541625976562, + "logps/ref_rejected": -83.03492736816406, + "logps/rejected": -159.34320068359375, + "loss": 1.1587, + "margin_dpo/margin_mean": 25.404937744140625, + "margin_dpo/margin_std": 47.308311462402344, + "step": 241 + }, + { + "KL/chosen_KL_mean": -52.51380920410156, + "KL/mean": -69.51727294921875, + "KL/rejected_KL_mean": -86.52073669433594, + "KL/std": 45.80069351196289, + "epoch": 0.36583522297808013, + "fcm_dpo/beta": 0.014580689370632172, + "fcm_dpo/delta": -0.10227706283330917, + "fcm_dpo/margin": 34.006919860839844, + "fcm_dpo/q_t": 0.3903062343597412, + "grad_norm": 13.116487503051758, + "learning_rate": 4.0140242178441665e-07, + "logits/chosen": 0.5915254354476929, + "logits/rejected": 0.5709429383277893, + "logps/chosen": -110.44123840332031, + "logps/ref_chosen": -57.927433013916016, + "logps/ref_rejected": -67.838623046875, + "logps/rejected": -154.35934448242188, + "loss": 1.0696, + "margin_dpo/margin_mean": 34.00692367553711, + "margin_dpo/margin_std": 49.14616394042969, + "step": 242 + }, + { + "KL/chosen_KL_mean": -51.1470832824707, + "KL/mean": -66.16419982910156, + "KL/rejected_KL_mean": -81.18131256103516, + "KL/std": 45.306007385253906, + "epoch": 0.3673469387755102, + "fcm_dpo/beta": 0.014499957673251629, + "fcm_dpo/delta": -0.03716906160116196, + "fcm_dpo/margin": 30.034236907958984, + "fcm_dpo/q_t": 0.40246307849884033, + "grad_norm": 17.26784896850586, + "learning_rate": 4.003481376353596e-07, + "logits/chosen": 0.5729988813400269, + "logits/rejected": 0.5832550525665283, + "logps/chosen": -125.42375946044922, + "logps/ref_chosen": -74.27667236328125, + "logps/ref_rejected": -73.24340057373047, + "logps/rejected": -154.42471313476562, + "loss": 1.104, + "margin_dpo/margin_mean": 30.034236907958984, + "margin_dpo/margin_std": 46.801856994628906, + "step": 243 + }, + { + "KL/chosen_KL_mean": -48.39063262939453, + "KL/mean": -68.6911849975586, + "KL/rejected_KL_mean": -88.99174499511719, + "KL/std": 45.00447082519531, + "epoch": 0.3688586545729403, + "fcm_dpo/beta": 0.01415896974503994, + "fcm_dpo/delta": -0.18511970341205597, + "fcm_dpo/margin": 40.60111999511719, + "fcm_dpo/q_t": 0.3705122470855713, + "grad_norm": 13.084891319274902, + "learning_rate": 3.9928964792569654e-07, + "logits/chosen": 0.6256475448608398, + "logits/rejected": 0.5371542572975159, + "logps/chosen": -101.75453186035156, + "logps/ref_chosen": -53.36390686035156, + "logps/ref_rejected": -71.10276794433594, + "logps/rejected": -160.09451293945312, + "loss": 0.993, + "margin_dpo/margin_mean": 40.60111999511719, + "margin_dpo/margin_std": 46.31639099121094, + "step": 244 + }, + { + "KL/chosen_KL_mean": -53.580116271972656, + "KL/mean": -76.88776397705078, + "KL/rejected_KL_mean": -100.19540405273438, + "KL/std": 46.22126770019531, + "epoch": 0.37037037037037035, + "fcm_dpo/beta": 0.01345006562769413, + "fcm_dpo/delta": -0.2425159513950348, + "fcm_dpo/margin": 46.61528015136719, + "fcm_dpo/q_t": 0.35643166303634644, + "grad_norm": 17.690349578857422, + "learning_rate": 3.982269822636601e-07, + "logits/chosen": 0.6387679576873779, + "logits/rejected": 0.6115980744361877, + "logps/chosen": -124.77522277832031, + "logps/ref_chosen": -71.19510650634766, + "logps/ref_rejected": -80.76235961914062, + "logps/rejected": -180.957763671875, + "loss": 0.9451, + "margin_dpo/margin_mean": 46.61528015136719, + "margin_dpo/margin_std": 46.82879638671875, + "step": 245 + }, + { + "KL/chosen_KL_mean": -63.579872131347656, + "KL/mean": -81.17933654785156, + "KL/rejected_KL_mean": -98.77880859375, + "KL/std": 46.690460205078125, + "epoch": 0.37188208616780044, + "fcm_dpo/beta": 0.013229056261479855, + "fcm_dpo/delta": -0.06901253014802933, + "fcm_dpo/margin": 35.198936462402344, + "fcm_dpo/q_t": 0.3967069089412689, + "grad_norm": 14.13879108428955, + "learning_rate": 3.971601703742932e-07, + "logits/chosen": 0.6410825252532959, + "logits/rejected": 0.5778796672821045, + "logps/chosen": -135.200927734375, + "logps/ref_chosen": -71.62104797363281, + "logps/ref_rejected": -94.03392028808594, + "logps/rejected": -192.81272888183594, + "loss": 1.1142, + "margin_dpo/margin_mean": 35.198936462402344, + "margin_dpo/margin_std": 58.479515075683594, + "step": 246 + }, + { + "KL/chosen_KL_mean": -67.45832824707031, + "KL/mean": -78.08027648925781, + "KL/rejected_KL_mean": -88.70221710205078, + "KL/std": 45.65076446533203, + "epoch": 0.37339380196523053, + "fcm_dpo/beta": 0.013414295390248299, + "fcm_dpo/delta": 0.11734433472156525, + "fcm_dpo/margin": 21.243894577026367, + "fcm_dpo/q_t": 0.4357885718345642, + "grad_norm": 17.8750057220459, + "learning_rate": 3.960892420986177e-07, + "logits/chosen": 0.6398141980171204, + "logits/rejected": 0.6296453475952148, + "logps/chosen": -147.48086547851562, + "logps/ref_chosen": -80.02254486083984, + "logps/ref_rejected": -89.22705841064453, + "logps/rejected": -177.9292755126953, + "loss": 1.2307, + "margin_dpo/margin_mean": 21.243894577026367, + "margin_dpo/margin_std": 49.95774841308594, + "step": 247 + }, + { + "KL/chosen_KL_mean": -57.298828125, + "KL/mean": -76.10064697265625, + "KL/rejected_KL_mean": -94.9024658203125, + "KL/std": 50.01144790649414, + "epoch": 0.3749055177626606, + "fcm_dpo/beta": 0.013268327340483665, + "fcm_dpo/delta": -0.10401751101016998, + "fcm_dpo/margin": 37.6036491394043, + "fcm_dpo/q_t": 0.3911561369895935, + "grad_norm": 15.191692352294922, + "learning_rate": 3.9501422739279953e-07, + "logits/chosen": 0.6122466325759888, + "logits/rejected": 0.6708812117576599, + "logps/chosen": -122.67678833007812, + "logps/ref_chosen": -65.37796020507812, + "logps/ref_rejected": -61.365787506103516, + "logps/rejected": -156.26824951171875, + "loss": 1.0769, + "margin_dpo/margin_mean": 37.6036491394043, + "margin_dpo/margin_std": 56.18958282470703, + "step": 248 + }, + { + "KL/chosen_KL_mean": -72.68350219726562, + "KL/mean": -80.0612564086914, + "KL/rejected_KL_mean": -87.43902587890625, + "KL/std": 46.41590118408203, + "epoch": 0.3764172335600907, + "fcm_dpo/beta": 0.013199391774833202, + "fcm_dpo/delta": 0.035816043615341187, + "fcm_dpo/margin": 14.755523681640625, + "fcm_dpo/q_t": 0.4553859233856201, + "grad_norm": 17.512218475341797, + "learning_rate": 3.9393515632731094e-07, + "logits/chosen": 0.6221470236778259, + "logits/rejected": 0.6636273860931396, + "logps/chosen": -147.2849578857422, + "logps/ref_chosen": -74.60145568847656, + "logps/ref_rejected": -63.79338455200195, + "logps/rejected": -151.23240661621094, + "loss": 1.3242, + "margin_dpo/margin_mean": 14.755523681640625, + "margin_dpo/margin_std": 53.123321533203125, + "step": 249 + }, + { + "KL/chosen_KL_mean": -62.269371032714844, + "KL/mean": -81.58802032470703, + "KL/rejected_KL_mean": -100.90668487548828, + "KL/std": 50.64720153808594, + "epoch": 0.3779289493575208, + "fcm_dpo/beta": 0.013095343485474586, + "fcm_dpo/delta": -0.111575648188591, + "fcm_dpo/margin": 38.637306213378906, + "fcm_dpo/q_t": 0.3866844177246094, + "grad_norm": 13.674488067626953, + "learning_rate": 3.9285205908608934e-07, + "logits/chosen": 0.7028018236160278, + "logits/rejected": 0.6593612432479858, + "logps/chosen": -124.20758056640625, + "logps/ref_chosen": -61.938209533691406, + "logps/ref_rejected": -72.21602630615234, + "logps/rejected": -173.12271118164062, + "loss": 1.0556, + "margin_dpo/margin_mean": 38.63731002807617, + "margin_dpo/margin_std": 53.086402893066406, + "step": 250 + }, + { + "KL/chosen_KL_mean": -68.74446105957031, + "KL/mean": -80.33143615722656, + "KL/rejected_KL_mean": -91.91839599609375, + "KL/std": 46.451534271240234, + "epoch": 0.3794406651549509, + "fcm_dpo/beta": 0.013123005628585815, + "fcm_dpo/delta": 0.09900989383459091, + "fcm_dpo/margin": 23.173925399780273, + "fcm_dpo/q_t": 0.4319212734699249, + "grad_norm": 21.88115119934082, + "learning_rate": 3.9176496596569265e-07, + "logits/chosen": 0.6787819862365723, + "logits/rejected": 0.6353781223297119, + "logps/chosen": -135.60140991210938, + "logps/ref_chosen": -66.85694885253906, + "logps/ref_rejected": -84.83396911621094, + "logps/rejected": -176.7523651123047, + "loss": 1.2205, + "margin_dpo/margin_mean": 23.173927307128906, + "margin_dpo/margin_std": 53.315067291259766, + "step": 251 + }, + { + "KL/chosen_KL_mean": -63.414527893066406, + "KL/mean": -73.9803466796875, + "KL/rejected_KL_mean": -84.54615783691406, + "KL/std": 48.33477020263672, + "epoch": 0.38095238095238093, + "fcm_dpo/beta": 0.013572916388511658, + "fcm_dpo/delta": 0.1134091168642044, + "fcm_dpo/margin": 21.131629943847656, + "fcm_dpo/q_t": 0.4354143738746643, + "grad_norm": 22.012300491333008, + "learning_rate": 3.9067390737445254e-07, + "logits/chosen": 0.6178069114685059, + "logits/rejected": 0.5614415407180786, + "logps/chosen": -119.63845825195312, + "logps/ref_chosen": -56.22393035888672, + "logps/ref_rejected": -77.1136245727539, + "logps/rejected": -161.6597900390625, + "loss": 1.2487, + "margin_dpo/margin_mean": 21.131629943847656, + "margin_dpo/margin_std": 52.747032165527344, + "step": 252 + }, + { + "KL/chosen_KL_mean": -63.61356735229492, + "KL/mean": -77.61599731445312, + "KL/rejected_KL_mean": -91.61842346191406, + "KL/std": 47.1943244934082, + "epoch": 0.382464096749811, + "fcm_dpo/beta": 0.013471575453877449, + "fcm_dpo/delta": 0.0228419192135334, + "fcm_dpo/margin": 28.00486183166504, + "fcm_dpo/q_t": 0.4155396521091461, + "grad_norm": 14.961233139038086, + "learning_rate": 3.8957891383162304e-07, + "logits/chosen": 0.7191380262374878, + "logits/rejected": 0.6768438220024109, + "logps/chosen": -115.82357788085938, + "logps/ref_chosen": -52.21001434326172, + "logps/ref_rejected": -58.75764846801758, + "logps/rejected": -150.37606811523438, + "loss": 1.1456, + "margin_dpo/margin_mean": 28.004863739013672, + "margin_dpo/margin_std": 47.672386169433594, + "step": 253 + }, + { + "KL/chosen_KL_mean": -68.19625854492188, + "KL/mean": -83.46250915527344, + "KL/rejected_KL_mean": -98.72874450683594, + "KL/std": 49.697425842285156, + "epoch": 0.3839758125472411, + "fcm_dpo/beta": 0.01349942572414875, + "fcm_dpo/delta": -0.013055309653282166, + "fcm_dpo/margin": 30.532485961914062, + "fcm_dpo/q_t": 0.40936195850372314, + "grad_norm": 14.441313743591309, + "learning_rate": 3.884800159665276e-07, + "logits/chosen": 0.6286872625350952, + "logits/rejected": 0.5743802189826965, + "logps/chosen": -133.83258056640625, + "logps/ref_chosen": -65.63632202148438, + "logps/ref_rejected": -82.34425354003906, + "logps/rejected": -181.072998046875, + "loss": 1.1219, + "margin_dpo/margin_mean": 30.532485961914062, + "margin_dpo/margin_std": 49.943382263183594, + "step": 254 + }, + { + "KL/chosen_KL_mean": -60.290618896484375, + "KL/mean": -79.58023071289062, + "KL/rejected_KL_mean": -98.86984252929688, + "KL/std": 49.87480926513672, + "epoch": 0.3854875283446712, + "fcm_dpo/beta": 0.013280254788696766, + "fcm_dpo/delta": -0.11871804296970367, + "fcm_dpo/margin": 38.579219818115234, + "fcm_dpo/q_t": 0.38657641410827637, + "grad_norm": 18.216447830200195, + "learning_rate": 3.873772445177015e-07, + "logits/chosen": 0.5999297499656677, + "logits/rejected": 0.5706052184104919, + "logps/chosen": -128.2017059326172, + "logps/ref_chosen": -67.91108703613281, + "logps/ref_rejected": -83.89114379882812, + "logps/rejected": -182.760986328125, + "loss": 1.0558, + "margin_dpo/margin_mean": 38.57921600341797, + "margin_dpo/margin_std": 54.264957427978516, + "step": 255 + }, + { + "KL/chosen_KL_mean": -73.21519470214844, + "KL/mean": -90.58259582519531, + "KL/rejected_KL_mean": -107.94999694824219, + "KL/std": 46.76201629638672, + "epoch": 0.3869992441421013, + "fcm_dpo/beta": 0.013069174252450466, + "fcm_dpo/delta": -0.05695026367902756, + "fcm_dpo/margin": 34.73479080200195, + "fcm_dpo/q_t": 0.40087705850601196, + "grad_norm": 17.64289093017578, + "learning_rate": 3.862706303320329e-07, + "logits/chosen": 0.6389260292053223, + "logits/rejected": 0.570476770401001, + "logps/chosen": -136.71517944335938, + "logps/ref_chosen": -63.49998474121094, + "logps/ref_rejected": -90.77104187011719, + "logps/rejected": -198.72103881835938, + "loss": 1.1229, + "margin_dpo/margin_mean": 34.73479080200195, + "margin_dpo/margin_std": 59.88385772705078, + "step": 256 + }, + { + "KL/chosen_KL_mean": -69.81033325195312, + "KL/mean": -91.40444946289062, + "KL/rejected_KL_mean": -112.9985580444336, + "KL/std": 52.70113754272461, + "epoch": 0.3885109599395314, + "fcm_dpo/beta": 0.012741890735924244, + "fcm_dpo/delta": -0.15949344635009766, + "fcm_dpo/margin": 43.188232421875, + "fcm_dpo/q_t": 0.37954503297805786, + "grad_norm": 14.254016876220703, + "learning_rate": 3.851602043638994e-07, + "logits/chosen": 0.6155068874359131, + "logits/rejected": 0.5465556383132935, + "logps/chosen": -140.41098022460938, + "logps/ref_chosen": -70.60064697265625, + "logps/ref_rejected": -108.58313751220703, + "logps/rejected": -221.58169555664062, + "loss": 1.0363, + "margin_dpo/margin_mean": 43.188232421875, + "margin_dpo/margin_std": 59.28783416748047, + "step": 257 + }, + { + "KL/chosen_KL_mean": -70.38436889648438, + "KL/mean": -86.61546325683594, + "KL/rejected_KL_mean": -102.84657287597656, + "KL/std": 45.8340950012207, + "epoch": 0.3900226757369615, + "fcm_dpo/beta": 0.012654589489102364, + "fcm_dpo/delta": -0.01126368809491396, + "fcm_dpo/margin": 32.46220397949219, + "fcm_dpo/q_t": 0.40449464321136475, + "grad_norm": 16.840524673461914, + "learning_rate": 3.840459976743023e-07, + "logits/chosen": 0.6729997396469116, + "logits/rejected": 0.6183003187179565, + "logps/chosen": -129.63853454589844, + "logps/ref_chosen": -59.25416564941406, + "logps/ref_rejected": -85.58709716796875, + "logps/rejected": -188.43365478515625, + "loss": 1.0877, + "margin_dpo/margin_mean": 32.46220397949219, + "margin_dpo/margin_std": 43.45000457763672, + "step": 258 + }, + { + "KL/chosen_KL_mean": -60.93222427368164, + "KL/mean": -85.822998046875, + "KL/rejected_KL_mean": -110.71377563476562, + "KL/std": 51.75875473022461, + "epoch": 0.3915343915343915, + "fcm_dpo/beta": 0.012163689360022545, + "fcm_dpo/delta": -0.2205704152584076, + "fcm_dpo/margin": 49.78155517578125, + "fcm_dpo/q_t": 0.36391448974609375, + "grad_norm": 14.892366409301758, + "learning_rate": 3.8292804142999796e-07, + "logits/chosen": 0.5799644589424133, + "logits/rejected": 0.47284597158432007, + "logps/chosen": -126.36709594726562, + "logps/ref_chosen": -65.43487548828125, + "logps/ref_rejected": -95.41731262207031, + "logps/rejected": -206.13108825683594, + "loss": 0.9932, + "margin_dpo/margin_mean": 49.78155517578125, + "margin_dpo/margin_std": 59.923851013183594, + "step": 259 + }, + { + "KL/chosen_KL_mean": -61.398841857910156, + "KL/mean": -81.24398803710938, + "KL/rejected_KL_mean": -101.08912658691406, + "KL/std": 49.07374572753906, + "epoch": 0.3930461073318216, + "fcm_dpo/beta": 0.011900585144758224, + "fcm_dpo/delta": -0.07667770981788635, + "fcm_dpo/margin": 39.69029235839844, + "fcm_dpo/q_t": 0.3952261209487915, + "grad_norm": 15.002617835998535, + "learning_rate": 3.818063669026256e-07, + "logits/chosen": 0.6235231161117554, + "logits/rejected": 0.5295395851135254, + "logps/chosen": -110.48841857910156, + "logps/ref_chosen": -49.08958435058594, + "logps/ref_rejected": -79.01708221435547, + "logps/rejected": -180.106201171875, + "loss": 1.0902, + "margin_dpo/margin_mean": 39.69029235839844, + "margin_dpo/margin_std": 60.59455871582031, + "step": 260 + }, + { + "KL/chosen_KL_mean": -60.26841354370117, + "KL/mean": -77.18064880371094, + "KL/rejected_KL_mean": -94.09290313720703, + "KL/std": 47.547637939453125, + "epoch": 0.3945578231292517, + "fcm_dpo/beta": 0.011928768828511238, + "fcm_dpo/delta": -0.0037367884069681168, + "fcm_dpo/margin": 33.82448959350586, + "fcm_dpo/q_t": 0.40984904766082764, + "grad_norm": 16.326772689819336, + "learning_rate": 3.806810054678331e-07, + "logits/chosen": 0.5075196027755737, + "logits/rejected": 0.5455510020256042, + "logps/chosen": -131.14080810546875, + "logps/ref_chosen": -70.87239074707031, + "logps/ref_rejected": -65.01522064208984, + "logps/rejected": -159.10812377929688, + "loss": 1.1194, + "margin_dpo/margin_mean": 33.824493408203125, + "margin_dpo/margin_std": 53.84626007080078, + "step": 261 + }, + { + "KL/chosen_KL_mean": -65.84159851074219, + "KL/mean": -81.57791900634766, + "KL/rejected_KL_mean": -97.31422424316406, + "KL/std": 46.80791091918945, + "epoch": 0.3960695389266818, + "fcm_dpo/beta": 0.011966339312493801, + "fcm_dpo/delta": 0.02416188269853592, + "fcm_dpo/margin": 31.47262191772461, + "fcm_dpo/q_t": 0.41486579179763794, + "grad_norm": 15.958047866821289, + "learning_rate": 3.7955198860439887e-07, + "logits/chosen": 0.6846475601196289, + "logits/rejected": 0.6157461404800415, + "logps/chosen": -133.71221923828125, + "logps/ref_chosen": -67.8706283569336, + "logps/ref_rejected": -88.7205810546875, + "logps/rejected": -186.03480529785156, + "loss": 1.1325, + "margin_dpo/margin_mean": 31.472623825073242, + "margin_dpo/margin_std": 50.958343505859375, + "step": 262 + }, + { + "KL/chosen_KL_mean": -61.87702941894531, + "KL/mean": -77.3545913696289, + "KL/rejected_KL_mean": -92.8321533203125, + "KL/std": 47.284324645996094, + "epoch": 0.3975812547241119, + "fcm_dpo/beta": 0.012026961892843246, + "fcm_dpo/delta": 0.028671840205788612, + "fcm_dpo/margin": 30.955129623413086, + "fcm_dpo/q_t": 0.41534680128097534, + "grad_norm": 14.962888717651367, + "learning_rate": 3.784193478933516e-07, + "logits/chosen": 0.5917189121246338, + "logits/rejected": 0.47899508476257324, + "logps/chosen": -117.07160949707031, + "logps/ref_chosen": -55.194583892822266, + "logps/ref_rejected": -80.54048156738281, + "logps/rejected": -173.37265014648438, + "loss": 1.141, + "margin_dpo/margin_mean": 30.95513153076172, + "margin_dpo/margin_std": 52.228553771972656, + "step": 263 + }, + { + "KL/chosen_KL_mean": -64.47840881347656, + "KL/mean": -81.5421142578125, + "KL/rejected_KL_mean": -98.60582733154297, + "KL/std": 51.82183074951172, + "epoch": 0.39909297052154197, + "fcm_dpo/beta": 0.011982956901192665, + "fcm_dpo/delta": -0.009435471147298813, + "fcm_dpo/margin": 34.127403259277344, + "fcm_dpo/q_t": 0.40906399488449097, + "grad_norm": 14.751063346862793, + "learning_rate": 3.7728311501708674e-07, + "logits/chosen": 0.5228692889213562, + "logits/rejected": 0.47892439365386963, + "logps/chosen": -147.6490936279297, + "logps/ref_chosen": -83.17068481445312, + "logps/ref_rejected": -88.33625793457031, + "logps/rejected": -186.94207763671875, + "loss": 1.13, + "margin_dpo/margin_mean": 34.127403259277344, + "margin_dpo/margin_std": 57.67744445800781, + "step": 264 + }, + { + "KL/chosen_KL_mean": -62.788230895996094, + "KL/mean": -81.142822265625, + "KL/rejected_KL_mean": -99.49742889404297, + "KL/std": 52.152587890625, + "epoch": 0.40060468631897206, + "fcm_dpo/beta": 0.011970747262239456, + "fcm_dpo/delta": -0.04135889559984207, + "fcm_dpo/margin": 36.709197998046875, + "fcm_dpo/q_t": 0.40274354815483093, + "grad_norm": 12.999602317810059, + "learning_rate": 3.7614332175848027e-07, + "logits/chosen": 0.6839252710342407, + "logits/rejected": 0.6158575415611267, + "logps/chosen": -114.45108032226562, + "logps/ref_chosen": -51.66284942626953, + "logps/ref_rejected": -67.1720962524414, + "logps/rejected": -166.66952514648438, + "loss": 1.1362, + "margin_dpo/margin_mean": 36.709197998046875, + "margin_dpo/margin_std": 64.98656463623047, + "step": 265 + }, + { + "KL/chosen_KL_mean": -59.46649169921875, + "KL/mean": -77.75037384033203, + "KL/rejected_KL_mean": -96.03425598144531, + "KL/std": 52.215267181396484, + "epoch": 0.4021164021164021, + "fcm_dpo/beta": 0.011831636540591717, + "fcm_dpo/delta": -0.03418291360139847, + "fcm_dpo/margin": 36.56776428222656, + "fcm_dpo/q_t": 0.4037179946899414, + "grad_norm": 17.84242057800293, + "learning_rate": 3.75e-07, + "logits/chosen": 0.5979915261268616, + "logits/rejected": 0.5204529762268066, + "logps/chosen": -116.91698455810547, + "logps/ref_chosen": -57.45049285888672, + "logps/ref_rejected": -77.60826110839844, + "logps/rejected": -173.64251708984375, + "loss": 1.108, + "margin_dpo/margin_mean": 36.56776809692383, + "margin_dpo/margin_std": 58.076812744140625, + "step": 266 + }, + { + "KL/chosen_KL_mean": -54.2578239440918, + "KL/mean": -68.69037628173828, + "KL/rejected_KL_mean": -83.1229248046875, + "KL/std": 48.732933044433594, + "epoch": 0.4036281179138322, + "fcm_dpo/beta": 0.012029076926410198, + "fcm_dpo/delta": 0.052326202392578125, + "fcm_dpo/margin": 28.865095138549805, + "fcm_dpo/q_t": 0.4219139814376831, + "grad_norm": 15.21697998046875, + "learning_rate": 3.738531817228131e-07, + "logits/chosen": 0.66780024766922, + "logits/rejected": 0.6482948064804077, + "logps/chosen": -109.29317474365234, + "logps/ref_chosen": -55.03535079956055, + "logps/ref_rejected": -66.0953369140625, + "logps/rejected": -149.21826171875, + "loss": 1.1956, + "margin_dpo/margin_mean": 28.865097045898438, + "margin_dpo/margin_std": 58.339569091796875, + "step": 267 + }, + { + "KL/chosen_KL_mean": -54.20028305053711, + "KL/mean": -71.07929992675781, + "KL/rejected_KL_mean": -87.95832061767578, + "KL/std": 51.59467315673828, + "epoch": 0.4051398337112623, + "fcm_dpo/beta": 0.01196706760674715, + "fcm_dpo/delta": -0.004256328567862511, + "fcm_dpo/margin": 33.75802993774414, + "fcm_dpo/q_t": 0.4097514748573303, + "grad_norm": 13.947291374206543, + "learning_rate": 3.7270289900589204e-07, + "logits/chosen": 0.5228180289268494, + "logits/rejected": 0.5083709955215454, + "logps/chosen": -119.27202606201172, + "logps/ref_chosen": -65.07174682617188, + "logps/ref_rejected": -71.42485809326172, + "logps/rejected": -159.3831787109375, + "loss": 1.1071, + "margin_dpo/margin_mean": 33.75802993774414, + "margin_dpo/margin_std": 50.174339294433594, + "step": 268 + }, + { + "KL/chosen_KL_mean": -60.456459045410156, + "KL/mean": -81.71235656738281, + "KL/rejected_KL_mean": -102.96824645996094, + "KL/std": 54.69612503051758, + "epoch": 0.40665154950869237, + "fcm_dpo/beta": 0.01182098500430584, + "fcm_dpo/delta": -0.10809072107076645, + "fcm_dpo/margin": 42.51179122924805, + "fcm_dpo/q_t": 0.38763895630836487, + "grad_norm": 13.600680351257324, + "learning_rate": 3.7154918402511714e-07, + "logits/chosen": 0.7140573263168335, + "logits/rejected": 0.6650851964950562, + "logps/chosen": -127.59266662597656, + "logps/ref_chosen": -67.1362075805664, + "logps/ref_rejected": -82.55778503417969, + "logps/rejected": -185.52603149414062, + "loss": 1.0495, + "margin_dpo/margin_mean": 42.51179122924805, + "margin_dpo/margin_std": 55.717506408691406, + "step": 269 + }, + { + "KL/chosen_KL_mean": -62.103599548339844, + "KL/mean": -78.8128433227539, + "KL/rejected_KL_mean": -95.52208709716797, + "KL/std": 53.32160949707031, + "epoch": 0.40816326530612246, + "fcm_dpo/beta": 0.011670410633087158, + "fcm_dpo/delta": 0.01030159741640091, + "fcm_dpo/margin": 33.41849899291992, + "fcm_dpo/q_t": 0.4117770791053772, + "grad_norm": 14.696253776550293, + "learning_rate": 3.7039206905237656e-07, + "logits/chosen": 0.6845616102218628, + "logits/rejected": 0.5967895984649658, + "logps/chosen": -128.79229736328125, + "logps/ref_chosen": -66.6886978149414, + "logps/ref_rejected": -85.16129302978516, + "logps/rejected": -180.68338012695312, + "loss": 1.1368, + "margin_dpo/margin_mean": 33.41849899291992, + "margin_dpo/margin_std": 57.01043701171875, + "step": 270 + }, + { + "KL/chosen_KL_mean": -63.44163513183594, + "KL/mean": -76.80795288085938, + "KL/rejected_KL_mean": -90.17426300048828, + "KL/std": 55.60846710205078, + "epoch": 0.40967498110355255, + "fcm_dpo/beta": 0.011818543076515198, + "fcm_dpo/delta": 0.08686529844999313, + "fcm_dpo/margin": 26.732616424560547, + "fcm_dpo/q_t": 0.434378057718277, + "grad_norm": 17.31071662902832, + "learning_rate": 3.692315864546635e-07, + "logits/chosen": 0.6486247777938843, + "logits/rejected": 0.58237624168396, + "logps/chosen": -135.84918212890625, + "logps/ref_chosen": -72.40754699707031, + "logps/ref_rejected": -92.06311798095703, + "logps/rejected": -182.2373809814453, + "loss": 1.234, + "margin_dpo/margin_mean": 26.732616424560547, + "margin_dpo/margin_std": 65.48910522460938, + "step": 271 + }, + { + "KL/chosen_KL_mean": -53.223960876464844, + "KL/mean": -79.23175811767578, + "KL/rejected_KL_mean": -105.23955535888672, + "KL/std": 51.41781997680664, + "epoch": 0.41118669690098264, + "fcm_dpo/beta": 0.011572392657399178, + "fcm_dpo/delta": -0.21442440152168274, + "fcm_dpo/margin": 52.015586853027344, + "fcm_dpo/q_t": 0.3613682985305786, + "grad_norm": 15.099222183227539, + "learning_rate": 3.6806776869317067e-07, + "logits/chosen": 0.6465578079223633, + "logits/rejected": 0.6684205532073975, + "logps/chosen": -119.82536315917969, + "logps/ref_chosen": -66.60140228271484, + "logps/ref_rejected": -67.74340057373047, + "logps/rejected": -172.9829559326172, + "loss": 0.9487, + "margin_dpo/margin_mean": 52.015586853027344, + "margin_dpo/margin_std": 49.51817321777344, + "step": 272 + }, + { + "KL/chosen_KL_mean": -73.37745666503906, + "KL/mean": -92.10945129394531, + "KL/rejected_KL_mean": -110.8414306640625, + "KL/std": 55.56861114501953, + "epoch": 0.4126984126984127, + "fcm_dpo/beta": 0.011334793642163277, + "fcm_dpo/delta": -0.025967109948396683, + "fcm_dpo/margin": 37.463966369628906, + "fcm_dpo/q_t": 0.40638309717178345, + "grad_norm": 17.103038787841797, + "learning_rate": 3.669006483223828e-07, + "logits/chosen": 0.6392084360122681, + "logits/rejected": 0.5658600926399231, + "logps/chosen": -130.73233032226562, + "logps/ref_chosen": -57.35487747192383, + "logps/ref_rejected": -84.17168426513672, + "logps/rejected": -195.01312255859375, + "loss": 1.1537, + "margin_dpo/margin_mean": 37.46397018432617, + "margin_dpo/margin_std": 70.37014770507812, + "step": 273 + }, + { + "KL/chosen_KL_mean": -71.62263488769531, + "KL/mean": -91.7503890991211, + "KL/rejected_KL_mean": -111.87814331054688, + "KL/std": 55.358795166015625, + "epoch": 0.41421012849584277, + "fcm_dpo/beta": 0.011266610585153103, + "fcm_dpo/delta": -0.05605369806289673, + "fcm_dpo/margin": 40.25550842285156, + "fcm_dpo/q_t": 0.3987223207950592, + "grad_norm": 13.076324462890625, + "learning_rate": 3.657302579891656e-07, + "logits/chosen": 0.5529523491859436, + "logits/rejected": 0.5354233980178833, + "logps/chosen": -131.26412963867188, + "logps/ref_chosen": -59.64149475097656, + "logps/ref_rejected": -68.29348754882812, + "logps/rejected": -180.171630859375, + "loss": 1.1094, + "margin_dpo/margin_mean": 40.25550842285156, + "margin_dpo/margin_std": 65.55795288085938, + "step": 274 + }, + { + "KL/chosen_KL_mean": -70.55653381347656, + "KL/mean": -90.97258758544922, + "KL/rejected_KL_mean": -111.38862609863281, + "KL/std": 53.519981384277344, + "epoch": 0.41572184429327286, + "fcm_dpo/beta": 0.011159954592585564, + "fcm_dpo/delta": -0.05834575742483139, + "fcm_dpo/margin": 40.83209991455078, + "fcm_dpo/q_t": 0.39650213718414307, + "grad_norm": 14.841992378234863, + "learning_rate": 3.645566304318526e-07, + "logits/chosen": 0.6303668022155762, + "logits/rejected": 0.545132040977478, + "logps/chosen": -123.82318115234375, + "logps/ref_chosen": -53.26664352416992, + "logps/ref_rejected": -73.84062194824219, + "logps/rejected": -185.229248046875, + "loss": 1.0729, + "margin_dpo/margin_mean": 40.83209991455078, + "margin_dpo/margin_std": 55.855072021484375, + "step": 275 + }, + { + "KL/chosen_KL_mean": -66.35244750976562, + "KL/mean": -86.4322509765625, + "KL/rejected_KL_mean": -106.51205444335938, + "KL/std": 50.89011764526367, + "epoch": 0.41723356009070295, + "fcm_dpo/beta": 0.011026125401258469, + "fcm_dpo/delta": -0.04477095603942871, + "fcm_dpo/margin": 40.15960693359375, + "fcm_dpo/q_t": 0.3993651270866394, + "grad_norm": 15.211221694946289, + "learning_rate": 3.633797984793294e-07, + "logits/chosen": 0.5870778560638428, + "logits/rejected": 0.5541011095046997, + "logps/chosen": -119.37323760986328, + "logps/ref_chosen": -53.02079772949219, + "logps/ref_rejected": -61.56678771972656, + "logps/rejected": -168.07884216308594, + "loss": 1.0821, + "margin_dpo/margin_mean": 40.15961456298828, + "margin_dpo/margin_std": 56.424461364746094, + "step": 276 + }, + { + "KL/chosen_KL_mean": -76.11033630371094, + "KL/mean": -86.88145446777344, + "KL/rejected_KL_mean": -97.652587890625, + "KL/std": 54.547332763671875, + "epoch": 0.41874527588813304, + "fcm_dpo/beta": 0.011240396648645401, + "fcm_dpo/delta": 0.16203390061855316, + "fcm_dpo/margin": 21.542251586914062, + "fcm_dpo/q_t": 0.4464568793773651, + "grad_norm": 20.440866470336914, + "learning_rate": 3.6219979505011555e-07, + "logits/chosen": 0.6785659790039062, + "logits/rejected": 0.7067118287086487, + "logps/chosen": -147.54331970214844, + "logps/ref_chosen": -71.43299102783203, + "logps/ref_rejected": -67.65852355957031, + "logps/rejected": -165.3111114501953, + "loss": 1.2769, + "margin_dpo/margin_mean": 21.542253494262695, + "margin_dpo/margin_std": 62.680477142333984, + "step": 277 + }, + { + "KL/chosen_KL_mean": -78.5838623046875, + "KL/mean": -98.24070739746094, + "KL/rejected_KL_mean": -117.89755249023438, + "KL/std": 58.49961853027344, + "epoch": 0.42025699168556313, + "fcm_dpo/beta": 0.01136256754398346, + "fcm_dpo/delta": -0.04974301904439926, + "fcm_dpo/margin": 39.31369400024414, + "fcm_dpo/q_t": 0.39944133162498474, + "grad_norm": 20.18327522277832, + "learning_rate": 3.6101665315144353e-07, + "logits/chosen": 0.573406994342804, + "logits/rejected": 0.5146926641464233, + "logps/chosen": -145.6946258544922, + "logps/ref_chosen": -67.11076354980469, + "logps/ref_rejected": -88.74851989746094, + "logps/rejected": -206.64608764648438, + "loss": 1.1067, + "margin_dpo/margin_mean": 39.31369400024414, + "margin_dpo/margin_std": 61.28262710571289, + "step": 278 + }, + { + "KL/chosen_KL_mean": -61.78923797607422, + "KL/mean": -88.9472885131836, + "KL/rejected_KL_mean": -116.1053466796875, + "KL/std": 57.12919998168945, + "epoch": 0.4217687074829932, + "fcm_dpo/beta": 0.0109342560172081, + "fcm_dpo/delta": -0.20577079057693481, + "fcm_dpo/margin": 54.31610107421875, + "fcm_dpo/q_t": 0.36407333612442017, + "grad_norm": 14.203146934509277, + "learning_rate": 3.5983040587833563e-07, + "logits/chosen": 0.5994449853897095, + "logits/rejected": 0.5615173578262329, + "logps/chosen": -116.28672790527344, + "logps/ref_chosen": -54.49748611450195, + "logps/ref_rejected": -70.42373657226562, + "logps/rejected": -186.52908325195312, + "loss": 0.9612, + "margin_dpo/margin_mean": 54.31610107421875, + "margin_dpo/margin_std": 54.377685546875, + "step": 279 + }, + { + "KL/chosen_KL_mean": -67.08258056640625, + "KL/mean": -94.1540298461914, + "KL/rejected_KL_mean": -121.2254867553711, + "KL/std": 59.60696792602539, + "epoch": 0.42328042328042326, + "fcm_dpo/beta": 0.010509304702281952, + "fcm_dpo/delta": -0.17919118702411652, + "fcm_dpo/margin": 54.14289855957031, + "fcm_dpo/q_t": 0.3707593083381653, + "grad_norm": 11.810566902160645, + "learning_rate": 3.586410864126781e-07, + "logits/chosen": 0.6714847087860107, + "logits/rejected": 0.6250628232955933, + "logps/chosen": -127.51538848876953, + "logps/ref_chosen": -60.43281173706055, + "logps/ref_rejected": -78.39051818847656, + "logps/rejected": -199.61599731445312, + "loss": 0.9715, + "margin_dpo/margin_mean": 54.14289855957031, + "margin_dpo/margin_std": 55.1671257019043, + "step": 280 + }, + { + "KL/chosen_KL_mean": -70.63385772705078, + "KL/mean": -94.31147766113281, + "KL/rejected_KL_mean": -117.98908996582031, + "KL/std": 58.223365783691406, + "epoch": 0.42479213907785335, + "fcm_dpo/beta": 0.010295085608959198, + "fcm_dpo/delta": -0.09190942347049713, + "fcm_dpo/margin": 47.35523986816406, + "fcm_dpo/q_t": 0.39007121324539185, + "grad_norm": 13.856719017028809, + "learning_rate": 3.574487280222929e-07, + "logits/chosen": 0.6418502926826477, + "logits/rejected": 0.6694193482398987, + "logps/chosen": -130.91595458984375, + "logps/ref_chosen": -60.2820930480957, + "logps/ref_rejected": -62.04009246826172, + "logps/rejected": -180.0291748046875, + "loss": 1.0588, + "margin_dpo/margin_mean": 47.35523986816406, + "margin_dpo/margin_std": 64.03118896484375, + "step": 281 + }, + { + "KL/chosen_KL_mean": -77.45885467529297, + "KL/mean": -99.65185546875, + "KL/rejected_KL_mean": -121.8448715209961, + "KL/std": 58.28876876831055, + "epoch": 0.42630385487528344, + "fcm_dpo/beta": 0.010223493911325932, + "fcm_dpo/delta": -0.057356491684913635, + "fcm_dpo/margin": 44.386009216308594, + "fcm_dpo/q_t": 0.3992747664451599, + "grad_norm": 16.32402992248535, + "learning_rate": 3.562533640600075e-07, + "logits/chosen": 0.5796880722045898, + "logits/rejected": 0.532652735710144, + "logps/chosen": -138.08277893066406, + "logps/ref_chosen": -60.623924255371094, + "logps/ref_rejected": -68.67400360107422, + "logps/rejected": -190.5188751220703, + "loss": 1.1023, + "margin_dpo/margin_mean": 44.386009216308594, + "margin_dpo/margin_std": 67.81919860839844, + "step": 282 + }, + { + "KL/chosen_KL_mean": -77.25079345703125, + "KL/mean": -98.12049102783203, + "KL/rejected_KL_mean": -118.99020385742188, + "KL/std": 59.12428283691406, + "epoch": 0.42781557067271353, + "fcm_dpo/beta": 0.010089964605867863, + "fcm_dpo/delta": -0.02210230752825737, + "fcm_dpo/margin": 41.739402770996094, + "fcm_dpo/q_t": 0.4049845337867737, + "grad_norm": 16.064359664916992, + "learning_rate": 3.550550279627215e-07, + "logits/chosen": 0.6336187720298767, + "logits/rejected": 0.5247352123260498, + "logps/chosen": -144.89854431152344, + "logps/ref_chosen": -67.64775085449219, + "logps/ref_rejected": -99.96835327148438, + "logps/rejected": -218.95855712890625, + "loss": 1.1101, + "margin_dpo/margin_mean": 41.739402770996094, + "margin_dpo/margin_std": 65.20516204833984, + "step": 283 + }, + { + "KL/chosen_KL_mean": -72.93248748779297, + "KL/mean": -95.26510620117188, + "KL/rejected_KL_mean": -117.59771728515625, + "KL/std": 61.540306091308594, + "epoch": 0.4293272864701436, + "fcm_dpo/beta": 0.009952565655112267, + "fcm_dpo/delta": -0.04708627983927727, + "fcm_dpo/margin": 44.66523742675781, + "fcm_dpo/q_t": 0.3982611894607544, + "grad_norm": 12.55010986328125, + "learning_rate": 3.5385375325047163e-07, + "logits/chosen": 0.6949187517166138, + "logits/rejected": 0.6278550028800964, + "logps/chosen": -129.89991760253906, + "logps/ref_chosen": -56.96742630004883, + "logps/ref_rejected": -86.36236572265625, + "logps/rejected": -203.9600830078125, + "loss": 1.0738, + "margin_dpo/margin_mean": 44.66523742675781, + "margin_dpo/margin_std": 59.5830078125, + "step": 284 + }, + { + "KL/chosen_KL_mean": -85.50302124023438, + "KL/mean": -103.44571685791016, + "KL/rejected_KL_mean": -121.38841247558594, + "KL/std": 56.31591796875, + "epoch": 0.4308390022675737, + "fcm_dpo/beta": 0.010065186768770218, + "fcm_dpo/delta": 0.039796821773052216, + "fcm_dpo/margin": 35.885398864746094, + "fcm_dpo/q_t": 0.4194878935813904, + "grad_norm": 17.57463264465332, + "learning_rate": 3.5264957352549375e-07, + "logits/chosen": 0.6707309484481812, + "logits/rejected": 0.6428732872009277, + "logps/chosen": -157.15911865234375, + "logps/ref_chosen": -71.65611267089844, + "logps/ref_rejected": -81.63829803466797, + "logps/rejected": -203.02670288085938, + "loss": 1.1526, + "margin_dpo/margin_mean": 35.88539505004883, + "margin_dpo/margin_std": 62.94104766845703, + "step": 285 + }, + { + "KL/chosen_KL_mean": -80.76374816894531, + "KL/mean": -109.85708618164062, + "KL/rejected_KL_mean": -138.950439453125, + "KL/std": 60.531402587890625, + "epoch": 0.4323507180650038, + "fcm_dpo/beta": 0.00978100299835205, + "fcm_dpo/delta": -0.17936107516288757, + "fcm_dpo/margin": 58.18669128417969, + "fcm_dpo/q_t": 0.3713955581188202, + "grad_norm": 13.661906242370605, + "learning_rate": 3.514425224712835e-07, + "logits/chosen": 0.5708847641944885, + "logits/rejected": 0.47632908821105957, + "logps/chosen": -141.84326171875, + "logps/ref_chosen": -61.07952117919922, + "logps/ref_rejected": -91.28128051757812, + "logps/rejected": -230.23171997070312, + "loss": 0.9846, + "margin_dpo/margin_mean": 58.18669509887695, + "margin_dpo/margin_std": 64.16620635986328, + "step": 286 + }, + { + "KL/chosen_KL_mean": -67.0542221069336, + "KL/mean": -94.71749114990234, + "KL/rejected_KL_mean": -122.38075256347656, + "KL/std": 61.530731201171875, + "epoch": 0.43386243386243384, + "fcm_dpo/beta": 0.009508990682661533, + "fcm_dpo/delta": -0.1330246478319168, + "fcm_dpo/margin": 55.32652282714844, + "fcm_dpo/q_t": 0.3804447650909424, + "grad_norm": 12.890028953552246, + "learning_rate": 3.502326338516534e-07, + "logits/chosen": 0.7085520029067993, + "logits/rejected": 0.6656246185302734, + "logps/chosen": -113.09001159667969, + "logps/ref_chosen": -46.035789489746094, + "logps/ref_rejected": -59.95293426513672, + "logps/rejected": -182.33367919921875, + "loss": 1.0121, + "margin_dpo/margin_mean": 55.32652282714844, + "margin_dpo/margin_std": 63.78333282470703, + "step": 287 + }, + { + "KL/chosen_KL_mean": -84.3800048828125, + "KL/mean": -105.1005630493164, + "KL/rejected_KL_mean": -125.82112121582031, + "KL/std": 59.49192810058594, + "epoch": 0.43537414965986393, + "fcm_dpo/beta": 0.00945931114256382, + "fcm_dpo/delta": 0.008291497826576233, + "fcm_dpo/margin": 41.441131591796875, + "fcm_dpo/q_t": 0.4110769033432007, + "grad_norm": 17.03542137145996, + "learning_rate": 3.490199415097892e-07, + "logits/chosen": 0.5387293100357056, + "logits/rejected": 0.478929340839386, + "logps/chosen": -149.77084350585938, + "logps/ref_chosen": -65.3908462524414, + "logps/ref_rejected": -88.53607940673828, + "logps/rejected": -214.35720825195312, + "loss": 1.1204, + "margin_dpo/margin_mean": 41.44112777709961, + "margin_dpo/margin_std": 64.76750183105469, + "step": 288 + }, + { + "KL/chosen_KL_mean": -86.93623352050781, + "KL/mean": -105.4552993774414, + "KL/rejected_KL_mean": -123.974365234375, + "KL/std": 60.03764724731445, + "epoch": 0.436885865457294, + "fcm_dpo/beta": 0.009551433846354485, + "fcm_dpo/delta": 0.047566674649715424, + "fcm_dpo/margin": 37.038116455078125, + "fcm_dpo/q_t": 0.4222422242164612, + "grad_norm": 13.532853126525879, + "learning_rate": 3.4780447936730247e-07, + "logits/chosen": 0.7148054242134094, + "logits/rejected": 0.6745719909667969, + "logps/chosen": -141.5298614501953, + "logps/ref_chosen": -54.5936279296875, + "logps/ref_rejected": -67.20855712890625, + "logps/rejected": -191.18292236328125, + "loss": 1.171, + "margin_dpo/margin_mean": 37.038116455078125, + "margin_dpo/margin_std": 70.36173248291016, + "step": 289 + }, + { + "KL/chosen_KL_mean": -96.75265502929688, + "KL/mean": -120.16667175292969, + "KL/rejected_KL_mean": -143.58067321777344, + "KL/std": 58.69277572631836, + "epoch": 0.4383975812547241, + "fcm_dpo/beta": 0.00943966954946518, + "fcm_dpo/delta": -0.04456711933016777, + "fcm_dpo/margin": 46.82802200317383, + "fcm_dpo/q_t": 0.3998667299747467, + "grad_norm": 16.931427001953125, + "learning_rate": 3.465862814232821e-07, + "logits/chosen": 0.7094553709030151, + "logits/rejected": 0.6341379284858704, + "logps/chosen": -158.13723754882812, + "logps/ref_chosen": -61.38457489013672, + "logps/ref_rejected": -91.92778015136719, + "logps/rejected": -235.50845336914062, + "loss": 1.0928, + "margin_dpo/margin_mean": 46.82802200317383, + "margin_dpo/margin_std": 69.21977233886719, + "step": 290 + }, + { + "KL/chosen_KL_mean": -90.10990142822266, + "KL/mean": -117.04533386230469, + "KL/rejected_KL_mean": -143.98077392578125, + "KL/std": 59.01616668701172, + "epoch": 0.4399092970521542, + "fcm_dpo/beta": 0.009374039247632027, + "fcm_dpo/delta": -0.11086567491292953, + "fcm_dpo/margin": 53.87086868286133, + "fcm_dpo/q_t": 0.3872455060482025, + "grad_norm": 14.803458213806152, + "learning_rate": 3.4536538175334343e-07, + "logits/chosen": 0.8204612731933594, + "logits/rejected": 0.741672933101654, + "logps/chosen": -140.97293090820312, + "logps/ref_chosen": -50.863037109375, + "logps/ref_rejected": -82.20868682861328, + "logps/rejected": -226.189453125, + "loss": 1.0403, + "margin_dpo/margin_mean": 53.87086486816406, + "margin_dpo/margin_std": 67.89382934570312, + "step": 291 + }, + { + "KL/chosen_KL_mean": -89.85836791992188, + "KL/mean": -109.6369857788086, + "KL/rejected_KL_mean": -129.41561889648438, + "KL/std": 62.5750732421875, + "epoch": 0.4414210128495843, + "fcm_dpo/beta": 0.009300924837589264, + "fcm_dpo/delta": 0.03329315781593323, + "fcm_dpo/margin": 39.557254791259766, + "fcm_dpo/q_t": 0.41779154539108276, + "grad_norm": 15.200834274291992, + "learning_rate": 3.4414181450867465e-07, + "logits/chosen": 0.6663825511932373, + "logits/rejected": 0.6153430938720703, + "logps/chosen": -154.20724487304688, + "logps/ref_chosen": -64.34888458251953, + "logps/ref_rejected": -72.86434173583984, + "logps/rejected": -202.27996826171875, + "loss": 1.1589, + "margin_dpo/margin_mean": 39.5572509765625, + "margin_dpo/margin_std": 72.9423599243164, + "step": 292 + }, + { + "KL/chosen_KL_mean": -89.22817993164062, + "KL/mean": -118.22019958496094, + "KL/rejected_KL_mean": -147.21221923828125, + "KL/std": 63.70530700683594, + "epoch": 0.4429327286470144, + "fcm_dpo/beta": 0.009160864166915417, + "fcm_dpo/delta": -0.13831757009029388, + "fcm_dpo/margin": 57.984046936035156, + "fcm_dpo/q_t": 0.3815430700778961, + "grad_norm": 12.239211082458496, + "learning_rate": 3.4291561391508185e-07, + "logits/chosen": 0.7890902757644653, + "logits/rejected": 0.6947133541107178, + "logps/chosen": -144.09764099121094, + "logps/ref_chosen": -54.869468688964844, + "logps/ref_rejected": -81.858642578125, + "logps/rejected": -229.07086181640625, + "loss": 1.043, + "margin_dpo/margin_mean": 57.984046936035156, + "margin_dpo/margin_std": 77.6739501953125, + "step": 293 + }, + { + "KL/chosen_KL_mean": -94.31707763671875, + "KL/mean": -113.79631042480469, + "KL/rejected_KL_mean": -133.27554321289062, + "KL/std": 64.919921875, + "epoch": 0.4444444444444444, + "fcm_dpo/beta": 0.009092319756746292, + "fcm_dpo/delta": 0.04737778753042221, + "fcm_dpo/margin": 38.958457946777344, + "fcm_dpo/q_t": 0.4201389253139496, + "grad_norm": 12.569846153259277, + "learning_rate": 3.4168681427203153e-07, + "logits/chosen": 0.6751217246055603, + "logits/rejected": 0.6271165609359741, + "logps/chosen": -150.98797607421875, + "logps/ref_chosen": -56.670902252197266, + "logps/ref_rejected": -70.32819366455078, + "logps/rejected": -203.60372924804688, + "loss": 1.1334, + "margin_dpo/margin_mean": 38.958457946777344, + "margin_dpo/margin_std": 60.146690368652344, + "step": 294 + }, + { + "KL/chosen_KL_mean": -99.99388122558594, + "KL/mean": -118.2969970703125, + "KL/rejected_KL_mean": -136.60012817382812, + "KL/std": 61.911251068115234, + "epoch": 0.4459561602418745, + "fcm_dpo/beta": 0.009216606616973877, + "fcm_dpo/delta": 0.06484965234994888, + "fcm_dpo/margin": 36.60624694824219, + "fcm_dpo/q_t": 0.42461496591567993, + "grad_norm": 16.885400772094727, + "learning_rate": 3.4045544995169125e-07, + "logits/chosen": 0.6669223308563232, + "logits/rejected": 0.5581985116004944, + "logps/chosen": -150.394775390625, + "logps/ref_chosen": -50.40088653564453, + "logps/ref_rejected": -83.43521881103516, + "logps/rejected": -220.03533935546875, + "loss": 1.1682, + "margin_dpo/margin_mean": 36.60624694824219, + "margin_dpo/margin_std": 67.60258483886719, + "step": 295 + }, + { + "KL/chosen_KL_mean": -102.53762817382812, + "KL/mean": -124.98433685302734, + "KL/rejected_KL_mean": -147.4310302734375, + "KL/std": 65.74832153320312, + "epoch": 0.4474678760393046, + "fcm_dpo/beta": 0.009207111783325672, + "fcm_dpo/delta": -0.014326311647891998, + "fcm_dpo/margin": 44.89341735839844, + "fcm_dpo/q_t": 0.40750962495803833, + "grad_norm": 13.373530387878418, + "learning_rate": 3.392215553979679e-07, + "logits/chosen": 0.6259567737579346, + "logits/rejected": 0.5756511092185974, + "logps/chosen": -171.68797302246094, + "logps/ref_chosen": -69.15034484863281, + "logps/ref_rejected": -89.60166931152344, + "logps/rejected": -237.03271484375, + "loss": 1.1283, + "margin_dpo/margin_mean": 44.89341735839844, + "margin_dpo/margin_std": 75.29107666015625, + "step": 296 + }, + { + "KL/chosen_KL_mean": -106.02978515625, + "KL/mean": -130.51258850097656, + "KL/rejected_KL_mean": -154.99539184570312, + "KL/std": 57.90068817138672, + "epoch": 0.4489795918367347, + "fcm_dpo/beta": 0.009207883849740028, + "fcm_dpo/delta": -0.05345804616808891, + "fcm_dpo/margin": 48.965606689453125, + "fcm_dpo/q_t": 0.39636701345443726, + "grad_norm": 13.408093452453613, + "learning_rate": 3.3798516512554485e-07, + "logits/chosen": 0.6601380705833435, + "logits/rejected": 0.6036698818206787, + "logps/chosen": -164.04608154296875, + "logps/ref_chosen": -58.01630401611328, + "logps/ref_rejected": -69.95780944824219, + "logps/rejected": -224.95321655273438, + "loss": 1.0602, + "margin_dpo/margin_mean": 48.965606689453125, + "margin_dpo/margin_std": 60.579673767089844, + "step": 297 + }, + { + "KL/chosen_KL_mean": -103.89498901367188, + "KL/mean": -122.39523315429688, + "KL/rejected_KL_mean": -140.89549255371094, + "KL/std": 61.32954406738281, + "epoch": 0.4504913076341648, + "fcm_dpo/beta": 0.009213869459927082, + "fcm_dpo/delta": 0.06121082603931427, + "fcm_dpo/margin": 37.00048828125, + "fcm_dpo/q_t": 0.4239236116409302, + "grad_norm": 12.870407104492188, + "learning_rate": 3.367463137189156e-07, + "logits/chosen": 0.7620357275009155, + "logits/rejected": 0.7042220830917358, + "logps/chosen": -160.06430053710938, + "logps/ref_chosen": -56.1693115234375, + "logps/ref_rejected": -68.55052185058594, + "logps/rejected": -209.44601440429688, + "loss": 1.1781, + "margin_dpo/margin_mean": 37.00048828125, + "margin_dpo/margin_std": 71.90211486816406, + "step": 298 + }, + { + "KL/chosen_KL_mean": -104.1708755493164, + "KL/mean": -120.55952453613281, + "KL/rejected_KL_mean": -136.9481658935547, + "KL/std": 60.09037780761719, + "epoch": 0.4520030234315949, + "fcm_dpo/beta": 0.009359677322208881, + "fcm_dpo/delta": 0.09627296775579453, + "fcm_dpo/margin": 32.777286529541016, + "fcm_dpo/q_t": 0.4320846498012543, + "grad_norm": 18.66891098022461, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": 0.541263222694397, + "logits/rejected": 0.5110787153244019, + "logps/chosen": -166.48867797851562, + "logps/ref_chosen": -62.31780242919922, + "logps/ref_rejected": -72.60028839111328, + "logps/rejected": -209.5484619140625, + "loss": 1.2256, + "margin_dpo/margin_mean": 32.777286529541016, + "margin_dpo/margin_std": 76.10530090332031, + "step": 299 + }, + { + "KL/chosen_KL_mean": -105.92295837402344, + "KL/mean": -126.46493530273438, + "KL/rejected_KL_mean": -147.00689697265625, + "KL/std": 60.57496643066406, + "epoch": 0.45351473922902497, + "fcm_dpo/beta": 0.009462611749768257, + "fcm_dpo/delta": 0.011636088602244854, + "fcm_dpo/margin": 41.083927154541016, + "fcm_dpo/q_t": 0.41320398449897766, + "grad_norm": 15.427064895629883, + "learning_rate": 3.3426136618426043e-07, + "logits/chosen": 0.6564747095108032, + "logits/rejected": 0.5867733359336853, + "logps/chosen": -166.30453491210938, + "logps/ref_chosen": -60.38157653808594, + "logps/ref_rejected": -75.45442199707031, + "logps/rejected": -222.46131896972656, + "loss": 1.1443, + "margin_dpo/margin_mean": 41.08393096923828, + "margin_dpo/margin_std": 71.81248474121094, + "step": 300 + }, + { + "KL/chosen_KL_mean": -104.05447387695312, + "KL/mean": -121.67876434326172, + "KL/rejected_KL_mean": -139.30307006835938, + "KL/std": 59.82817077636719, + "epoch": 0.455026455026455, + "fcm_dpo/beta": 0.009520202875137329, + "fcm_dpo/delta": 0.06667570024728775, + "fcm_dpo/margin": 35.24860382080078, + "fcm_dpo/q_t": 0.42482560873031616, + "grad_norm": 13.765838623046875, + "learning_rate": 3.3301533956555885e-07, + "logits/chosen": 0.713027834892273, + "logits/rejected": 0.6836607456207275, + "logps/chosen": -156.90536499023438, + "logps/ref_chosen": -52.85089111328125, + "logps/ref_rejected": -69.97584533691406, + "logps/rejected": -209.27891540527344, + "loss": 1.182, + "margin_dpo/margin_mean": 35.24860382080078, + "margin_dpo/margin_std": 69.6445083618164, + "step": 301 + }, + { + "KL/chosen_KL_mean": -108.92539978027344, + "KL/mean": -123.26554870605469, + "KL/rejected_KL_mean": -137.605712890625, + "KL/std": 61.54811096191406, + "epoch": 0.4565381708238851, + "fcm_dpo/beta": 0.009755447506904602, + "fcm_dpo/delta": 0.12373100966215134, + "fcm_dpo/margin": 28.680299758911133, + "fcm_dpo/q_t": 0.43714994192123413, + "grad_norm": 18.384824752807617, + "learning_rate": 3.317669908293554e-07, + "logits/chosen": 0.5518735647201538, + "logits/rejected": 0.4927511215209961, + "logps/chosen": -175.89190673828125, + "logps/ref_chosen": -66.96650695800781, + "logps/ref_rejected": -88.09510803222656, + "logps/rejected": -225.7008056640625, + "loss": 1.2291, + "margin_dpo/margin_mean": 28.680301666259766, + "margin_dpo/margin_std": 68.15020751953125, + "step": 302 + }, + { + "KL/chosen_KL_mean": -97.18345642089844, + "KL/mean": -122.90306091308594, + "KL/rejected_KL_mean": -148.62265014648438, + "KL/std": 62.800289154052734, + "epoch": 0.4580498866213152, + "fcm_dpo/beta": 0.009679718874394894, + "fcm_dpo/delta": -0.10298407077789307, + "fcm_dpo/margin": 51.439186096191406, + "fcm_dpo/q_t": 0.38958287239074707, + "grad_norm": 12.350512504577637, + "learning_rate": 3.3051635489464793e-07, + "logits/chosen": 0.657049298286438, + "logits/rejected": 0.5822213888168335, + "logps/chosen": -159.30499267578125, + "logps/ref_chosen": -62.12152862548828, + "logps/ref_rejected": -90.31204223632812, + "logps/rejected": -238.9346923828125, + "loss": 1.082, + "margin_dpo/margin_mean": 51.439186096191406, + "margin_dpo/margin_std": 78.26869201660156, + "step": 303 + }, + { + "KL/chosen_KL_mean": -85.36961364746094, + "KL/mean": -112.04019165039062, + "KL/rejected_KL_mean": -138.7107696533203, + "KL/std": 66.2973403930664, + "epoch": 0.4595616024187453, + "fcm_dpo/beta": 0.009458931162953377, + "fcm_dpo/delta": -0.11018989235162735, + "fcm_dpo/margin": 53.34115219116211, + "fcm_dpo/q_t": 0.383327841758728, + "grad_norm": 12.269899368286133, + "learning_rate": 3.292634667444117e-07, + "logits/chosen": 0.6840123534202576, + "logits/rejected": 0.6246076822280884, + "logps/chosen": -146.064697265625, + "logps/ref_chosen": -60.695091247558594, + "logps/ref_rejected": -78.2525405883789, + "logps/rejected": -216.96331787109375, + "loss": 1.0185, + "margin_dpo/margin_mean": 53.341156005859375, + "margin_dpo/margin_std": 59.39166259765625, + "step": 304 + }, + { + "KL/chosen_KL_mean": -98.3131103515625, + "KL/mean": -116.91204833984375, + "KL/rejected_KL_mean": -135.51097106933594, + "KL/std": 61.239044189453125, + "epoch": 0.46107331821617537, + "fcm_dpo/beta": 0.009442286565899849, + "fcm_dpo/delta": 0.05038090795278549, + "fcm_dpo/margin": 37.19786834716797, + "fcm_dpo/q_t": 0.4206337034702301, + "grad_norm": 14.711054801940918, + "learning_rate": 3.280083614246217e-07, + "logits/chosen": 0.6261643171310425, + "logits/rejected": 0.6597653031349182, + "logps/chosen": -171.01223754882812, + "logps/ref_chosen": -72.69914245605469, + "logps/ref_rejected": -65.65670776367188, + "logps/rejected": -201.16769409179688, + "loss": 1.1829, + "margin_dpo/margin_mean": 37.19786834716797, + "margin_dpo/margin_std": 73.96134185791016, + "step": 305 + }, + { + "KL/chosen_KL_mean": -92.8389663696289, + "KL/mean": -113.00958251953125, + "KL/rejected_KL_mean": -133.18020629882812, + "KL/std": 58.908294677734375, + "epoch": 0.46258503401360546, + "fcm_dpo/beta": 0.009471280500292778, + "fcm_dpo/delta": 0.017888210713863373, + "fcm_dpo/margin": 40.34123229980469, + "fcm_dpo/q_t": 0.4124301075935364, + "grad_norm": 13.316874504089355, + "learning_rate": 3.267510740432719e-07, + "logits/chosen": 0.7246155738830566, + "logits/rejected": 0.6098471879959106, + "logps/chosen": -146.8094940185547, + "logps/ref_chosen": -53.97052764892578, + "logps/ref_rejected": -71.02423095703125, + "logps/rejected": -204.20443725585938, + "loss": 1.1139, + "margin_dpo/margin_mean": 40.34123229980469, + "margin_dpo/margin_std": 57.43824005126953, + "step": 306 + }, + { + "KL/chosen_KL_mean": -87.2799301147461, + "KL/mean": -96.95132446289062, + "KL/rejected_KL_mean": -106.62272644042969, + "KL/std": 58.196372985839844, + "epoch": 0.46409674981103555, + "fcm_dpo/beta": 0.009879468940198421, + "fcm_dpo/delta": 0.21212655305862427, + "fcm_dpo/margin": 19.342792510986328, + "fcm_dpo/q_t": 0.4589805006980896, + "grad_norm": 18.865333557128906, + "learning_rate": 3.2549163976939285e-07, + "logits/chosen": 0.6826301217079163, + "logits/rejected": 0.632476806640625, + "logps/chosen": -144.6930389404297, + "logps/ref_chosen": -57.413108825683594, + "logps/ref_rejected": -68.68010711669922, + "logps/rejected": -175.30282592773438, + "loss": 1.3379, + "margin_dpo/margin_mean": 19.342790603637695, + "margin_dpo/margin_std": 75.95152282714844, + "step": 307 + }, + { + "KL/chosen_KL_mean": -86.84735870361328, + "KL/mean": -106.39009094238281, + "KL/rejected_KL_mean": -125.93280029296875, + "KL/std": 60.022674560546875, + "epoch": 0.4656084656084656, + "fcm_dpo/beta": 0.009924834594130516, + "fcm_dpo/delta": 0.012152183800935745, + "fcm_dpo/margin": 39.08544158935547, + "fcm_dpo/q_t": 0.41299301385879517, + "grad_norm": 11.46269702911377, + "learning_rate": 3.2423009383206874e-07, + "logits/chosen": 0.647024929523468, + "logits/rejected": 0.6352015733718872, + "logps/chosen": -153.44615173339844, + "logps/ref_chosen": -66.59879302978516, + "logps/ref_rejected": -74.337158203125, + "logps/rejected": -200.26995849609375, + "loss": 1.1458, + "margin_dpo/margin_mean": 39.08544158935547, + "margin_dpo/margin_std": 68.68582153320312, + "step": 308 + }, + { + "KL/chosen_KL_mean": -98.74766540527344, + "KL/mean": -118.67848205566406, + "KL/rejected_KL_mean": -138.6092987060547, + "KL/std": 57.710323333740234, + "epoch": 0.4671201814058957, + "fcm_dpo/beta": 0.00993155688047409, + "fcm_dpo/delta": 0.003720022737979889, + "fcm_dpo/margin": 39.86164093017578, + "fcm_dpo/q_t": 0.4107176661491394, + "grad_norm": 12.478151321411133, + "learning_rate": 3.229664715194511e-07, + "logits/chosen": 0.6989116668701172, + "logits/rejected": 0.6386100053787231, + "logps/chosen": -164.14239501953125, + "logps/ref_chosen": -65.39474487304688, + "logps/ref_rejected": -75.70930480957031, + "logps/rejected": -214.318603515625, + "loss": 1.1265, + "margin_dpo/margin_mean": 39.86164093017578, + "margin_dpo/margin_std": 63.787353515625, + "step": 309 + }, + { + "KL/chosen_KL_mean": -101.77289581298828, + "KL/mean": -113.84951782226562, + "KL/rejected_KL_mean": -125.9261474609375, + "KL/std": 53.13010025024414, + "epoch": 0.46863189720332576, + "fcm_dpo/beta": 0.01020658016204834, + "fcm_dpo/delta": 0.15757328271865845, + "fcm_dpo/margin": 24.153255462646484, + "fcm_dpo/q_t": 0.4470118284225464, + "grad_norm": 14.50604248046875, + "learning_rate": 3.2170080817777257e-07, + "logits/chosen": 0.6527221202850342, + "logits/rejected": 0.6363627314567566, + "logps/chosen": -176.441162109375, + "logps/ref_chosen": -74.66827392578125, + "logps/ref_rejected": -80.5689697265625, + "logps/rejected": -206.4951171875, + "loss": 1.2556, + "margin_dpo/margin_mean": 24.153257369995117, + "margin_dpo/margin_std": 63.91498565673828, + "step": 310 + }, + { + "KL/chosen_KL_mean": -81.11943054199219, + "KL/mean": -102.99217224121094, + "KL/rejected_KL_mean": -124.86492156982422, + "KL/std": 66.24354553222656, + "epoch": 0.47014361300075586, + "fcm_dpo/beta": 0.01017935760319233, + "fcm_dpo/delta": -0.048217982053756714, + "fcm_dpo/margin": 43.74549102783203, + "fcm_dpo/q_t": 0.4035261273384094, + "grad_norm": 13.386900901794434, + "learning_rate": 3.204331392103574e-07, + "logits/chosen": 0.5761426091194153, + "logits/rejected": 0.4230411648750305, + "logps/chosen": -140.8574676513672, + "logps/ref_chosen": -59.738033294677734, + "logps/ref_rejected": -93.60757446289062, + "logps/rejected": -218.47250366210938, + "loss": 1.1018, + "margin_dpo/margin_mean": 43.74549102783203, + "margin_dpo/margin_std": 69.48627471923828, + "step": 311 + }, + { + "KL/chosen_KL_mean": -82.9559097290039, + "KL/mean": -107.94522857666016, + "KL/rejected_KL_mean": -132.93453979492188, + "KL/std": 58.801048278808594, + "epoch": 0.47165532879818595, + "fcm_dpo/beta": 0.010070530697703362, + "fcm_dpo/delta": -0.10864290595054626, + "fcm_dpo/margin": 49.97863006591797, + "fcm_dpo/q_t": 0.38455379009246826, + "grad_norm": 11.932977676391602, + "learning_rate": 3.1916350007663176e-07, + "logits/chosen": 0.6558113098144531, + "logits/rejected": 0.558535099029541, + "logps/chosen": -136.7723388671875, + "logps/ref_chosen": -53.816436767578125, + "logps/ref_rejected": -68.6575698852539, + "logps/rejected": -201.59210205078125, + "loss": 1.0222, + "margin_dpo/margin_mean": 49.97863006591797, + "margin_dpo/margin_std": 57.203495025634766, + "step": 312 + }, + { + "KL/chosen_KL_mean": -86.02486419677734, + "KL/mean": -98.96048736572266, + "KL/rejected_KL_mean": -111.8961181640625, + "KL/std": 57.0450439453125, + "epoch": 0.47316704459561604, + "fcm_dpo/beta": 0.010199690237641335, + "fcm_dpo/delta": 0.139798104763031, + "fcm_dpo/margin": 25.871265411376953, + "fcm_dpo/q_t": 0.44218185544013977, + "grad_norm": 13.864587783813477, + "learning_rate": 3.178919262911314e-07, + "logits/chosen": 0.6773887872695923, + "logits/rejected": 0.6556931734085083, + "logps/chosen": -145.9822235107422, + "logps/ref_chosen": -59.957359313964844, + "logps/ref_rejected": -69.31729888916016, + "logps/rejected": -181.21340942382812, + "loss": 1.237, + "margin_dpo/margin_mean": 25.871265411376953, + "margin_dpo/margin_std": 63.11532974243164, + "step": 313 + }, + { + "KL/chosen_KL_mean": -80.63925170898438, + "KL/mean": -105.95073699951172, + "KL/rejected_KL_mean": -131.26222229003906, + "KL/std": 57.55366134643555, + "epoch": 0.47467876039304613, + "fcm_dpo/beta": 0.010059855878353119, + "fcm_dpo/delta": -0.11579519510269165, + "fcm_dpo/margin": 50.62295913696289, + "fcm_dpo/q_t": 0.3863465487957001, + "grad_norm": 12.481680870056152, + "learning_rate": 3.166184534225087e-07, + "logits/chosen": 0.6313918828964233, + "logits/rejected": 0.6649610996246338, + "logps/chosen": -150.90740966796875, + "logps/ref_chosen": -70.26815795898438, + "logps/ref_rejected": -69.23971557617188, + "logps/rejected": -200.50193786621094, + "loss": 1.0367, + "margin_dpo/margin_mean": 50.622955322265625, + "margin_dpo/margin_std": 63.79005432128906, + "step": 314 + }, + { + "KL/chosen_KL_mean": -86.05230712890625, + "KL/mean": -105.64132690429688, + "KL/rejected_KL_mean": -125.23036193847656, + "KL/std": 58.11652374267578, + "epoch": 0.47619047619047616, + "fcm_dpo/beta": 0.010089458897709846, + "fcm_dpo/delta": 0.004446987062692642, + "fcm_dpo/margin": 39.17805099487305, + "fcm_dpo/q_t": 0.40984123945236206, + "grad_norm": 13.194849014282227, + "learning_rate": 3.1534311709253723e-07, + "logits/chosen": 0.543517529964447, + "logits/rejected": 0.5082773566246033, + "logps/chosen": -153.8470001220703, + "logps/ref_chosen": -67.79469299316406, + "logps/ref_rejected": -74.55148315429688, + "logps/rejected": -199.78184509277344, + "loss": 1.1191, + "margin_dpo/margin_mean": 39.17805480957031, + "margin_dpo/margin_std": 60.50445556640625, + "step": 315 + }, + { + "KL/chosen_KL_mean": -80.4795150756836, + "KL/mean": -107.60636901855469, + "KL/rejected_KL_mean": -134.7332000732422, + "KL/std": 60.75721740722656, + "epoch": 0.47770219198790626, + "fcm_dpo/beta": 0.009939778596162796, + "fcm_dpo/delta": -0.14816725254058838, + "fcm_dpo/margin": 54.25369644165039, + "fcm_dpo/q_t": 0.3790704011917114, + "grad_norm": 13.798815727233887, + "learning_rate": 3.1406595297511564e-07, + "logits/chosen": 0.5641090273857117, + "logits/rejected": 0.41707244515419006, + "logps/chosen": -135.76800537109375, + "logps/ref_chosen": -55.288482666015625, + "logps/ref_rejected": -96.15723419189453, + "logps/rejected": -230.89044189453125, + "loss": 1.0166, + "margin_dpo/margin_mean": 54.25369644165039, + "margin_dpo/margin_std": 61.49580383300781, + "step": 316 + }, + { + "KL/chosen_KL_mean": -75.87356567382812, + "KL/mean": -102.3594970703125, + "KL/rejected_KL_mean": -128.84542846679688, + "KL/std": 56.36368179321289, + "epoch": 0.47921390778533635, + "fcm_dpo/beta": 0.009550904855132103, + "fcm_dpo/delta": -0.11237458139657974, + "fcm_dpo/margin": 52.97185516357422, + "fcm_dpo/q_t": 0.3837541937828064, + "grad_norm": 16.33446502685547, + "learning_rate": 3.1278699679526975e-07, + "logits/chosen": 0.6727269887924194, + "logits/rejected": 0.6263071298599243, + "logps/chosen": -130.45494079589844, + "logps/ref_chosen": -54.58137512207031, + "logps/ref_rejected": -72.77232360839844, + "logps/rejected": -201.6177520751953, + "loss": 1.0265, + "margin_dpo/margin_mean": 52.97185516357422, + "margin_dpo/margin_std": 62.5837516784668, + "step": 317 + }, + { + "KL/chosen_KL_mean": -83.718994140625, + "KL/mean": -103.1109848022461, + "KL/rejected_KL_mean": -122.50297546386719, + "KL/std": 60.44317626953125, + "epoch": 0.48072562358276644, + "fcm_dpo/beta": 0.009556062519550323, + "fcm_dpo/delta": 0.030469154939055443, + "fcm_dpo/margin": 38.78398895263672, + "fcm_dpo/q_t": 0.42044275999069214, + "grad_norm": 12.4547700881958, + "learning_rate": 3.1150628432815336e-07, + "logits/chosen": 0.684049129486084, + "logits/rejected": 0.6094720959663391, + "logps/chosen": -136.6072235107422, + "logps/ref_chosen": -52.88822937011719, + "logps/ref_rejected": -80.63988494873047, + "logps/rejected": -203.1428680419922, + "loss": 1.1845, + "margin_dpo/margin_mean": 38.78398513793945, + "margin_dpo/margin_std": 80.1749038696289, + "step": 318 + }, + { + "KL/chosen_KL_mean": -80.73106384277344, + "KL/mean": -107.32977294921875, + "KL/rejected_KL_mean": -133.928466796875, + "KL/std": 62.93165969848633, + "epoch": 0.48223733938019653, + "fcm_dpo/beta": 0.009435629472136497, + "fcm_dpo/delta": -0.10740967094898224, + "fcm_dpo/margin": 53.197418212890625, + "fcm_dpo/q_t": 0.3889296054840088, + "grad_norm": 12.824899673461914, + "learning_rate": 3.1022385139804707e-07, + "logits/chosen": 0.6069538593292236, + "logits/rejected": 0.5893139839172363, + "logps/chosen": -145.09439086914062, + "logps/ref_chosen": -64.36333465576172, + "logps/ref_rejected": -79.47296142578125, + "logps/rejected": -213.40142822265625, + "loss": 1.0517, + "margin_dpo/margin_mean": 53.197418212890625, + "margin_dpo/margin_std": 72.72633361816406, + "step": 319 + }, + { + "KL/chosen_KL_mean": -75.99618530273438, + "KL/mean": -97.96165466308594, + "KL/rejected_KL_mean": -119.9271240234375, + "KL/std": 63.95793914794922, + "epoch": 0.4837490551776266, + "fcm_dpo/beta": 0.009483279660344124, + "fcm_dpo/delta": -0.02047700434923172, + "fcm_dpo/margin": 43.93092727661133, + "fcm_dpo/q_t": 0.40889182686805725, + "grad_norm": 14.631209373474121, + "learning_rate": 3.0893973387735683e-07, + "logits/chosen": 0.5133136510848999, + "logits/rejected": 0.4730033278465271, + "logps/chosen": -125.554931640625, + "logps/ref_chosen": -49.558746337890625, + "logps/ref_rejected": -71.23444366455078, + "logps/rejected": -191.16156005859375, + "loss": 1.1239, + "margin_dpo/margin_mean": 43.93092346191406, + "margin_dpo/margin_std": 68.785400390625, + "step": 320 + }, + { + "KL/chosen_KL_mean": -81.69332885742188, + "KL/mean": -105.12799072265625, + "KL/rejected_KL_mean": -128.56268310546875, + "KL/std": 57.98059844970703, + "epoch": 0.4852607709750567, + "fcm_dpo/beta": 0.009233346208930016, + "fcm_dpo/delta": -0.03561249375343323, + "fcm_dpo/margin": 46.869346618652344, + "fcm_dpo/q_t": 0.40108391642570496, + "grad_norm": 19.632156372070312, + "learning_rate": 3.0765396768561004e-07, + "logits/chosen": 0.626788854598999, + "logits/rejected": 0.6063965559005737, + "logps/chosen": -133.77859497070312, + "logps/ref_chosen": -52.08526611328125, + "logps/ref_rejected": -55.58674621582031, + "logps/rejected": -184.1494140625, + "loss": 1.1006, + "margin_dpo/margin_mean": 46.869346618652344, + "margin_dpo/margin_std": 68.83665466308594, + "step": 321 + }, + { + "KL/chosen_KL_mean": -92.54100799560547, + "KL/mean": -121.60884094238281, + "KL/rejected_KL_mean": -150.67666625976562, + "KL/std": 64.49148559570312, + "epoch": 0.48677248677248675, + "fcm_dpo/beta": 0.009122923016548157, + "fcm_dpo/delta": -0.13744433224201202, + "fcm_dpo/margin": 58.13566970825195, + "fcm_dpo/q_t": 0.37799185514450073, + "grad_norm": 12.562987327575684, + "learning_rate": 3.063665887884511e-07, + "logits/chosen": 0.6769028902053833, + "logits/rejected": 0.5882055759429932, + "logps/chosen": -139.9451141357422, + "logps/ref_chosen": -47.404109954833984, + "logps/ref_rejected": -73.4260025024414, + "logps/rejected": -224.1026611328125, + "loss": 1.0043, + "margin_dpo/margin_mean": 58.13567352294922, + "margin_dpo/margin_std": 63.69596862792969, + "step": 322 + }, + { + "KL/chosen_KL_mean": -94.64340209960938, + "KL/mean": -114.46276092529297, + "KL/rejected_KL_mean": -134.28213500976562, + "KL/std": 62.43727111816406, + "epoch": 0.48828420256991684, + "fcm_dpo/beta": 0.009102044627070427, + "fcm_dpo/delta": 0.040557216852903366, + "fcm_dpo/margin": 39.63875198364258, + "fcm_dpo/q_t": 0.421923965215683, + "grad_norm": 13.756513595581055, + "learning_rate": 3.0507763319663517e-07, + "logits/chosen": 0.5732111930847168, + "logits/rejected": 0.49360191822052, + "logps/chosen": -164.64968872070312, + "logps/ref_chosen": -70.00630187988281, + "logps/ref_rejected": -86.96690368652344, + "logps/rejected": -221.24905395507812, + "loss": 1.1802, + "margin_dpo/margin_mean": 39.63875198364258, + "margin_dpo/margin_std": 80.1014633178711, + "step": 323 + }, + { + "KL/chosen_KL_mean": -84.78605651855469, + "KL/mean": -111.35850524902344, + "KL/rejected_KL_mean": -137.93096923828125, + "KL/std": 70.51113891601562, + "epoch": 0.4897959183673469, + "fcm_dpo/beta": 0.008961576968431473, + "fcm_dpo/delta": -0.08072944730520248, + "fcm_dpo/margin": 53.1449089050293, + "fcm_dpo/q_t": 0.39130061864852905, + "grad_norm": 17.810380935668945, + "learning_rate": 3.0378713696502097e-07, + "logits/chosen": 0.6615591645240784, + "logits/rejected": 0.6014628410339355, + "logps/chosen": -140.67486572265625, + "logps/ref_chosen": -55.88882064819336, + "logps/ref_rejected": -75.23088073730469, + "logps/rejected": -213.16184997558594, + "loss": 1.0472, + "margin_dpo/margin_mean": 53.1449089050293, + "margin_dpo/margin_std": 65.71917724609375, + "step": 324 + }, + { + "KL/chosen_KL_mean": -104.56004333496094, + "KL/mean": -129.3292236328125, + "KL/rejected_KL_mean": -154.098388671875, + "KL/std": 67.77610778808594, + "epoch": 0.491307634164777, + "fcm_dpo/beta": 0.008865425363183022, + "fcm_dpo/delta": -0.04157440736889839, + "fcm_dpo/margin": 49.538360595703125, + "fcm_dpo/q_t": 0.4014624059200287, + "grad_norm": 14.201213836669922, + "learning_rate": 3.0249513619156206e-07, + "logits/chosen": 0.606431245803833, + "logits/rejected": 0.5380966067314148, + "logps/chosen": -168.70706176757812, + "logps/ref_chosen": -64.14701843261719, + "logps/ref_rejected": -79.91143798828125, + "logps/rejected": -234.00982666015625, + "loss": 1.1141, + "margin_dpo/margin_mean": 49.53835678100586, + "margin_dpo/margin_std": 81.64930725097656, + "step": 325 + }, + { + "KL/chosen_KL_mean": -121.84555053710938, + "KL/mean": -132.28628540039062, + "KL/rejected_KL_mean": -142.72705078125, + "KL/std": 65.33851623535156, + "epoch": 0.4928193499622071, + "fcm_dpo/beta": 0.009159738197922707, + "fcm_dpo/delta": 0.21309442818164825, + "fcm_dpo/margin": 20.88149070739746, + "fcm_dpo/q_t": 0.4580095410346985, + "grad_norm": 15.675823211669922, + "learning_rate": 3.012016670162977e-07, + "logits/chosen": 0.5414941310882568, + "logits/rejected": 0.5495314002037048, + "logps/chosen": -197.37686157226562, + "logps/ref_chosen": -75.53131103515625, + "logps/ref_rejected": -76.5898666381836, + "logps/rejected": -219.31692504882812, + "loss": 1.312, + "margin_dpo/margin_mean": 20.881492614746094, + "margin_dpo/margin_std": 72.97334289550781, + "step": 326 + }, + { + "KL/chosen_KL_mean": -109.90470886230469, + "KL/mean": -129.81394958496094, + "KL/rejected_KL_mean": -149.72317504882812, + "KL/std": 68.6119384765625, + "epoch": 0.4943310657596372, + "fcm_dpo/beta": 0.009345386177301407, + "fcm_dpo/delta": 0.028610462322831154, + "fcm_dpo/margin": 39.81847381591797, + "fcm_dpo/q_t": 0.4180833101272583, + "grad_norm": 16.183422088623047, + "learning_rate": 2.99906765620341e-07, + "logits/chosen": 0.5542730689048767, + "logits/rejected": 0.5263379216194153, + "logps/chosen": -179.24188232421875, + "logps/ref_chosen": -69.33717346191406, + "logps/ref_rejected": -73.37751770019531, + "logps/rejected": -223.1007080078125, + "loss": 1.1743, + "margin_dpo/margin_mean": 39.81846618652344, + "margin_dpo/margin_std": 78.34571838378906, + "step": 327 + }, + { + "KL/chosen_KL_mean": -96.4388427734375, + "KL/mean": -120.63749694824219, + "KL/rejected_KL_mean": -144.83615112304688, + "KL/std": 68.9698486328125, + "epoch": 0.4958427815570673, + "fcm_dpo/beta": 0.009272612631320953, + "fcm_dpo/delta": -0.05103527009487152, + "fcm_dpo/margin": 48.39728927612305, + "fcm_dpo/q_t": 0.39925065636634827, + "grad_norm": 14.83205795288086, + "learning_rate": 2.9861046822486766e-07, + "logits/chosen": 0.5593709349632263, + "logits/rejected": 0.519730269908905, + "logps/chosen": -158.14508056640625, + "logps/ref_chosen": -61.70623016357422, + "logps/ref_rejected": -83.73808288574219, + "logps/rejected": -228.57421875, + "loss": 1.0764, + "margin_dpo/margin_mean": 48.39728546142578, + "margin_dpo/margin_std": 66.56529998779297, + "step": 328 + }, + { + "KL/chosen_KL_mean": -107.68646240234375, + "KL/mean": -132.2223358154297, + "KL/rejected_KL_mean": -156.75819396972656, + "KL/std": 71.65765380859375, + "epoch": 0.4973544973544973, + "fcm_dpo/beta": 0.009159384295344353, + "fcm_dpo/delta": -0.05180327221751213, + "fcm_dpo/margin": 49.07172393798828, + "fcm_dpo/q_t": 0.4012608826160431, + "grad_norm": 14.937561988830566, + "learning_rate": 2.9731281109010253e-07, + "logits/chosen": 0.6999276876449585, + "logits/rejected": 0.6358014345169067, + "logps/chosen": -172.18487548828125, + "logps/ref_chosen": -64.4984130859375, + "logps/ref_rejected": -83.6591796875, + "logps/rejected": -240.4173583984375, + "loss": 1.0909, + "margin_dpo/margin_mean": 49.071720123291016, + "margin_dpo/margin_std": 73.95594787597656, + "step": 329 + }, + { + "KL/chosen_KL_mean": -92.5936508178711, + "KL/mean": -118.45382690429688, + "KL/rejected_KL_mean": -144.31399536132812, + "KL/std": 67.32807922363281, + "epoch": 0.4988662131519274, + "fcm_dpo/beta": 0.009068884886801243, + "fcm_dpo/delta": -0.07242908328771591, + "fcm_dpo/margin": 51.72032928466797, + "fcm_dpo/q_t": 0.3960675597190857, + "grad_norm": 14.681183815002441, + "learning_rate": 2.9601383051430505e-07, + "logits/chosen": 0.6028814315795898, + "logits/rejected": 0.53467857837677, + "logps/chosen": -147.39828491210938, + "logps/ref_chosen": -54.80464172363281, + "logps/ref_rejected": -75.3194351196289, + "logps/rejected": -219.6334228515625, + "loss": 1.1115, + "margin_dpo/margin_mean": 51.720333099365234, + "margin_dpo/margin_std": 85.41941833496094, + "step": 330 + }, + { + "KL/chosen_KL_mean": -100.29885864257812, + "KL/mean": -132.77366638183594, + "KL/rejected_KL_mean": -165.24847412109375, + "KL/std": 71.77218627929688, + "epoch": 0.5003779289493575, + "fcm_dpo/beta": 0.008779097348451614, + "fcm_dpo/delta": -0.1802905797958374, + "fcm_dpo/margin": 64.94962310791016, + "fcm_dpo/q_t": 0.37083327770233154, + "grad_norm": 12.52709674835205, + "learning_rate": 2.947135628327544e-07, + "logits/chosen": 0.7146250009536743, + "logits/rejected": 0.6876611709594727, + "logps/chosen": -159.54144287109375, + "logps/ref_chosen": -59.242584228515625, + "logps/ref_rejected": -69.87483215332031, + "logps/rejected": -235.12330627441406, + "loss": 1.003, + "margin_dpo/margin_mean": 64.94961547851562, + "margin_dpo/margin_std": 78.14910888671875, + "step": 331 + }, + { + "KL/chosen_KL_mean": -101.7713623046875, + "KL/mean": -129.58682250976562, + "KL/rejected_KL_mean": -157.40225219726562, + "KL/std": 68.52835083007812, + "epoch": 0.5018896447467877, + "fcm_dpo/beta": 0.008652852848172188, + "fcm_dpo/delta": -0.08633846044540405, + "fcm_dpo/margin": 55.63090896606445, + "fcm_dpo/q_t": 0.3918297290802002, + "grad_norm": 12.344429016113281, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": 0.5712667107582092, + "logits/rejected": 0.5271207690238953, + "logps/chosen": -168.88111877441406, + "logps/ref_chosen": -67.10975646972656, + "logps/ref_rejected": -77.11839294433594, + "logps/rejected": -234.52066040039062, + "loss": 1.0523, + "margin_dpo/margin_mean": 55.63090515136719, + "margin_dpo/margin_std": 69.46024322509766, + "step": 332 + }, + { + "KL/chosen_KL_mean": -109.85867309570312, + "KL/mean": -139.08721923828125, + "KL/rejected_KL_mean": -168.31573486328125, + "KL/std": 68.70384216308594, + "epoch": 0.5034013605442177, + "fcm_dpo/beta": 0.008428534492850304, + "fcm_dpo/delta": -0.09740786254405975, + "fcm_dpo/margin": 58.457061767578125, + "fcm_dpo/q_t": 0.38885655999183655, + "grad_norm": 12.776857376098633, + "learning_rate": 2.921093116725076e-07, + "logits/chosen": 0.6594347953796387, + "logits/rejected": 0.5772568583488464, + "logps/chosen": -168.23980712890625, + "logps/ref_chosen": -58.381134033203125, + "logps/ref_rejected": -85.02839660644531, + "logps/rejected": -253.34413146972656, + "loss": 1.0402, + "margin_dpo/margin_mean": 58.457061767578125, + "margin_dpo/margin_std": 73.11105346679688, + "step": 333 + }, + { + "KL/chosen_KL_mean": -104.70248413085938, + "KL/mean": -125.80743408203125, + "KL/rejected_KL_mean": -146.91238403320312, + "KL/std": 70.07809448242188, + "epoch": 0.5049130763416477, + "fcm_dpo/beta": 0.008443513885140419, + "fcm_dpo/delta": 0.045077141374349594, + "fcm_dpo/margin": 42.209896087646484, + "fcm_dpo/q_t": 0.42087602615356445, + "grad_norm": 13.54383659362793, + "learning_rate": 2.9080540104031484e-07, + "logits/chosen": 0.6578313112258911, + "logits/rejected": 0.6086193919181824, + "logps/chosen": -171.594482421875, + "logps/ref_chosen": -66.89199829101562, + "logps/ref_rejected": -91.83695220947266, + "logps/rejected": -238.74932861328125, + "loss": 1.1727, + "margin_dpo/margin_mean": 42.209896087646484, + "margin_dpo/margin_std": 81.92105865478516, + "step": 334 + }, + { + "KL/chosen_KL_mean": -106.31073760986328, + "KL/mean": -129.83917236328125, + "KL/rejected_KL_mean": -153.36761474609375, + "KL/std": 69.8459701538086, + "epoch": 0.5064247921390779, + "fcm_dpo/beta": 0.008487050421535969, + "fcm_dpo/delta": 0.00023875758051872253, + "fcm_dpo/margin": 47.05687713623047, + "fcm_dpo/q_t": 0.4113461375236511, + "grad_norm": 17.41133689880371, + "learning_rate": 2.895003489933375e-07, + "logits/chosen": 0.6846311688423157, + "logits/rejected": 0.6434398889541626, + "logps/chosen": -167.8251953125, + "logps/ref_chosen": -61.51445770263672, + "logps/ref_rejected": -75.68916320800781, + "logps/rejected": -229.05677795410156, + "loss": 1.1342, + "margin_dpo/margin_mean": 47.05687713623047, + "margin_dpo/margin_std": 79.80413818359375, + "step": 335 + }, + { + "KL/chosen_KL_mean": -117.79705810546875, + "KL/mean": -142.47894287109375, + "KL/rejected_KL_mean": -167.1608428955078, + "KL/std": 72.92349243164062, + "epoch": 0.5079365079365079, + "fcm_dpo/beta": 0.00837024487555027, + "fcm_dpo/delta": -0.014911554753780365, + "fcm_dpo/margin": 49.36377716064453, + "fcm_dpo/q_t": 0.4105210602283478, + "grad_norm": 13.49311637878418, + "learning_rate": 2.8819419203668675e-07, + "logits/chosen": 0.557252049446106, + "logits/rejected": 0.5326744914054871, + "logps/chosen": -186.64712524414062, + "logps/ref_chosen": -68.85006713867188, + "logps/ref_rejected": -92.99603271484375, + "logps/rejected": -260.1568603515625, + "loss": 1.1244, + "margin_dpo/margin_mean": 49.36377716064453, + "margin_dpo/margin_std": 81.3376693725586, + "step": 336 + }, + { + "KL/chosen_KL_mean": -122.55076599121094, + "KL/mean": -142.1136016845703, + "KL/rejected_KL_mean": -161.67642211914062, + "KL/std": 71.70664978027344, + "epoch": 0.509448223733938, + "fcm_dpo/beta": 0.00851651281118393, + "fcm_dpo/delta": 0.06903576850891113, + "fcm_dpo/margin": 39.125640869140625, + "fcm_dpo/q_t": 0.42501017451286316, + "grad_norm": 12.491925239562988, + "learning_rate": 2.8688696670638053e-07, + "logits/chosen": 0.5466808080673218, + "logits/rejected": 0.5131638050079346, + "logps/chosen": -195.7386016845703, + "logps/ref_chosen": -73.18783569335938, + "logps/ref_rejected": -86.89118957519531, + "logps/rejected": -248.56761169433594, + "loss": 1.1792, + "margin_dpo/margin_mean": 39.125640869140625, + "margin_dpo/margin_std": 76.22183227539062, + "step": 337 + }, + { + "KL/chosen_KL_mean": -118.58080291748047, + "KL/mean": -139.35684204101562, + "KL/rejected_KL_mean": -160.13287353515625, + "KL/std": 68.07583618164062, + "epoch": 0.5109599395313681, + "fcm_dpo/beta": 0.008585257455706596, + "fcm_dpo/delta": 0.04488629475235939, + "fcm_dpo/margin": 41.552066802978516, + "fcm_dpo/q_t": 0.41984856128692627, + "grad_norm": 12.422159194946289, + "learning_rate": 2.8557870956832133e-07, + "logits/chosen": 0.6132569313049316, + "logits/rejected": 0.5857648849487305, + "logps/chosen": -182.52041625976562, + "logps/ref_chosen": -63.939613342285156, + "logps/ref_rejected": -75.34243774414062, + "logps/rejected": -235.47531127929688, + "loss": 1.1626, + "margin_dpo/margin_mean": 41.552066802978516, + "margin_dpo/margin_std": 76.6851806640625, + "step": 338 + }, + { + "KL/chosen_KL_mean": -96.97520446777344, + "KL/mean": -120.72698974609375, + "KL/rejected_KL_mean": -144.478759765625, + "KL/std": 67.07359313964844, + "epoch": 0.5124716553287982, + "fcm_dpo/beta": 0.00859010498970747, + "fcm_dpo/delta": -0.008500054478645325, + "fcm_dpo/margin": 47.50355529785156, + "fcm_dpo/q_t": 0.4071218967437744, + "grad_norm": 13.649033546447754, + "learning_rate": 2.842694572172736e-07, + "logits/chosen": 0.7820595502853394, + "logits/rejected": 0.6895301342010498, + "logps/chosen": -142.5243377685547, + "logps/ref_chosen": -45.54913330078125, + "logps/ref_rejected": -67.0482177734375, + "logps/rejected": -211.5269775390625, + "loss": 1.1103, + "margin_dpo/margin_mean": 47.5035514831543, + "margin_dpo/margin_std": 71.7981185913086, + "step": 339 + }, + { + "KL/chosen_KL_mean": -109.6189193725586, + "KL/mean": -132.94656372070312, + "KL/rejected_KL_mean": -156.27420043945312, + "KL/std": 73.74958801269531, + "epoch": 0.5139833711262283, + "fcm_dpo/beta": 0.008629849180579185, + "fcm_dpo/delta": -0.002979576587677002, + "fcm_dpo/margin": 46.65528106689453, + "fcm_dpo/q_t": 0.41186124086380005, + "grad_norm": 12.804910659790039, + "learning_rate": 2.8295924627584004e-07, + "logits/chosen": 0.6743656396865845, + "logits/rejected": 0.6509165167808533, + "logps/chosen": -163.62457275390625, + "logps/ref_chosen": -54.00564956665039, + "logps/ref_rejected": -61.314430236816406, + "logps/rejected": -217.58863830566406, + "loss": 1.1527, + "margin_dpo/margin_mean": 46.65528106689453, + "margin_dpo/margin_std": 86.06853485107422, + "step": 340 + }, + { + "KL/chosen_KL_mean": -108.90359497070312, + "KL/mean": -135.95693969726562, + "KL/rejected_KL_mean": -163.01025390625, + "KL/std": 72.9114990234375, + "epoch": 0.5154950869236583, + "fcm_dpo/beta": 0.00841558538377285, + "fcm_dpo/delta": -0.06184224784374237, + "fcm_dpo/margin": 54.10666275024414, + "fcm_dpo/q_t": 0.3978581130504608, + "grad_norm": 14.011096000671387, + "learning_rate": 2.816481133934373e-07, + "logits/chosen": 0.6594457626342773, + "logits/rejected": 0.6075119972229004, + "logps/chosen": -172.29869079589844, + "logps/ref_chosen": -63.39509582519531, + "logps/ref_rejected": -76.20973205566406, + "logps/rejected": -239.22000122070312, + "loss": 1.0938, + "margin_dpo/margin_mean": 54.10666275024414, + "margin_dpo/margin_std": 77.17527770996094, + "step": 341 + }, + { + "KL/chosen_KL_mean": -108.45635986328125, + "KL/mean": -135.4060821533203, + "KL/rejected_KL_mean": -162.35580444335938, + "KL/std": 76.76370239257812, + "epoch": 0.5170068027210885, + "fcm_dpo/beta": 0.008395740762352943, + "fcm_dpo/delta": -0.05540557950735092, + "fcm_dpo/margin": 53.899452209472656, + "fcm_dpo/q_t": 0.3998180329799652, + "grad_norm": 12.18912410736084, + "learning_rate": 2.8033609524527046e-07, + "logits/chosen": 0.7089950442314148, + "logits/rejected": 0.6643673181533813, + "logps/chosen": -161.50418090820312, + "logps/ref_chosen": -53.047813415527344, + "logps/ref_rejected": -68.2854232788086, + "logps/rejected": -230.6412353515625, + "loss": 1.0868, + "margin_dpo/margin_mean": 53.899452209472656, + "margin_dpo/margin_std": 78.99366760253906, + "step": 342 + }, + { + "KL/chosen_KL_mean": -101.50072479248047, + "KL/mean": -121.11677551269531, + "KL/rejected_KL_mean": -140.73281860351562, + "KL/std": 71.01953125, + "epoch": 0.5185185185185185, + "fcm_dpo/beta": 0.00842662900686264, + "fcm_dpo/delta": 0.07119348645210266, + "fcm_dpo/margin": 39.232112884521484, + "fcm_dpo/q_t": 0.4252588152885437, + "grad_norm": 11.982053756713867, + "learning_rate": 2.7902322853130753e-07, + "logits/chosen": 0.5230769515037537, + "logits/rejected": 0.5136237740516663, + "logps/chosen": -172.07925415039062, + "logps/ref_chosen": -70.57852935791016, + "logps/ref_rejected": -84.73873901367188, + "logps/rejected": -225.4715576171875, + "loss": 1.1768, + "margin_dpo/margin_mean": 39.23210906982422, + "margin_dpo/margin_std": 73.24249267578125, + "step": 343 + }, + { + "KL/chosen_KL_mean": -111.83555603027344, + "KL/mean": -140.12460327148438, + "KL/rejected_KL_mean": -168.41366577148438, + "KL/std": 71.88983154296875, + "epoch": 0.5200302343159486, + "fcm_dpo/beta": 0.008404587395489216, + "fcm_dpo/delta": -0.07948686927556992, + "fcm_dpo/margin": 56.57810592651367, + "fcm_dpo/q_t": 0.3939194977283478, + "grad_norm": 15.187760353088379, + "learning_rate": 2.7770954997525274e-07, + "logits/chosen": 0.6720702648162842, + "logits/rejected": 0.5977091789245605, + "logps/chosen": -167.6465606689453, + "logps/ref_chosen": -55.811004638671875, + "logps/ref_rejected": -84.77637481689453, + "logps/rejected": -253.19003295898438, + "loss": 1.0683, + "margin_dpo/margin_mean": 56.57810974121094, + "margin_dpo/margin_std": 79.43333435058594, + "step": 344 + }, + { + "KL/chosen_KL_mean": -90.22946166992188, + "KL/mean": -114.63905334472656, + "KL/rejected_KL_mean": -139.0486602783203, + "KL/std": 65.07427978515625, + "epoch": 0.5215419501133787, + "fcm_dpo/beta": 0.008393699303269386, + "fcm_dpo/delta": -0.01029902882874012, + "fcm_dpo/margin": 48.81920623779297, + "fcm_dpo/q_t": 0.40778762102127075, + "grad_norm": 13.599929809570312, + "learning_rate": 2.7639509632351927e-07, + "logits/chosen": 0.691046416759491, + "logits/rejected": 0.6441917419433594, + "logps/chosen": -148.01556396484375, + "logps/ref_chosen": -57.78609848022461, + "logps/ref_rejected": -78.91847229003906, + "logps/rejected": -217.96713256835938, + "loss": 1.1166, + "margin_dpo/margin_mean": 48.81920623779297, + "margin_dpo/margin_std": 77.43244171142578, + "step": 345 + }, + { + "KL/chosen_KL_mean": -98.14485168457031, + "KL/mean": -127.07212829589844, + "KL/rejected_KL_mean": -155.99942016601562, + "KL/std": 75.85902404785156, + "epoch": 0.5230536659108088, + "fcm_dpo/beta": 0.008301769383251667, + "fcm_dpo/delta": -0.08439122140407562, + "fcm_dpo/margin": 57.854549407958984, + "fcm_dpo/q_t": 0.3912527859210968, + "grad_norm": 14.651946067810059, + "learning_rate": 2.7507990434420123e-07, + "logits/chosen": 0.674179196357727, + "logits/rejected": 0.5830097198486328, + "logps/chosen": -154.4299774169922, + "logps/ref_chosen": -56.285125732421875, + "logps/ref_rejected": -91.15303039550781, + "logps/rejected": -247.15243530273438, + "loss": 1.0674, + "margin_dpo/margin_mean": 57.854549407958984, + "margin_dpo/margin_std": 78.946533203125, + "step": 346 + }, + { + "KL/chosen_KL_mean": -107.67420959472656, + "KL/mean": -130.18055725097656, + "KL/rejected_KL_mean": -152.6868896484375, + "KL/std": 72.40330505371094, + "epoch": 0.5245653817082389, + "fcm_dpo/beta": 0.008253240026533604, + "fcm_dpo/delta": 0.02960244007408619, + "fcm_dpo/margin": 45.0127067565918, + "fcm_dpo/q_t": 0.4166538715362549, + "grad_norm": 15.322473526000977, + "learning_rate": 2.737640108260456e-07, + "logits/chosen": 0.8045604228973389, + "logits/rejected": 0.7491985559463501, + "logps/chosen": -161.1737518310547, + "logps/ref_chosen": -53.499542236328125, + "logps/ref_rejected": -72.52565002441406, + "logps/rejected": -225.21255493164062, + "loss": 1.1389, + "margin_dpo/margin_mean": 45.0127067565918, + "margin_dpo/margin_std": 75.2705078125, + "step": 347 + }, + { + "KL/chosen_KL_mean": -96.49987030029297, + "KL/mean": -123.30168914794922, + "KL/rejected_KL_mean": -150.10350036621094, + "KL/std": 69.12874603271484, + "epoch": 0.5260770975056689, + "fcm_dpo/beta": 0.008198895491659641, + "fcm_dpo/delta": -0.04165411740541458, + "fcm_dpo/margin": 53.603633880615234, + "fcm_dpo/q_t": 0.4039544463157654, + "grad_norm": 13.738903999328613, + "learning_rate": 2.724474525774229e-07, + "logits/chosen": 0.7492630481719971, + "logits/rejected": 0.7167302370071411, + "logps/chosen": -147.28671264648438, + "logps/ref_chosen": -50.78684997558594, + "logps/ref_rejected": -68.63732147216797, + "logps/rejected": -218.74081420898438, + "loss": 1.1037, + "margin_dpo/margin_mean": 53.6036376953125, + "margin_dpo/margin_std": 84.41233825683594, + "step": 348 + }, + { + "KL/chosen_KL_mean": -95.79183959960938, + "KL/mean": -123.4986572265625, + "KL/rejected_KL_mean": -151.2054443359375, + "KL/std": 74.21245574951172, + "epoch": 0.527588813303099, + "fcm_dpo/beta": 0.00817472767084837, + "fcm_dpo/delta": -0.0556333102285862, + "fcm_dpo/margin": 55.413612365722656, + "fcm_dpo/q_t": 0.399311363697052, + "grad_norm": 13.029300689697266, + "learning_rate": 2.711302664252973e-07, + "logits/chosen": 0.6797126531600952, + "logits/rejected": 0.5815830230712891, + "logps/chosen": -149.11685180664062, + "logps/ref_chosen": -53.325008392333984, + "logps/ref_rejected": -83.21236419677734, + "logps/rejected": -234.41781616210938, + "loss": 1.0839, + "margin_dpo/margin_mean": 55.413612365722656, + "margin_dpo/margin_std": 79.81897735595703, + "step": 349 + }, + { + "KL/chosen_KL_mean": -102.28199005126953, + "KL/mean": -136.58529663085938, + "KL/rejected_KL_mean": -170.88861083984375, + "KL/std": 79.92459869384766, + "epoch": 0.5291005291005291, + "fcm_dpo/beta": 0.007922561839222908, + "fcm_dpo/delta": -0.1521030217409134, + "fcm_dpo/margin": 68.60662841796875, + "fcm_dpo/q_t": 0.376028448343277, + "grad_norm": 16.23141098022461, + "learning_rate": 2.698124892141971e-07, + "logits/chosen": 0.6232413053512573, + "logits/rejected": 0.5423790216445923, + "logps/chosen": -163.9077606201172, + "logps/ref_chosen": -61.625770568847656, + "logps/ref_rejected": -87.63627624511719, + "logps/rejected": -258.52490234375, + "loss": 1.0016, + "margin_dpo/margin_mean": 68.60662841796875, + "margin_dpo/margin_std": 77.74540710449219, + "step": 350 + }, + { + "KL/chosen_KL_mean": -97.53581237792969, + "KL/mean": -122.48731231689453, + "KL/rejected_KL_mean": -147.43881225585938, + "KL/std": 68.1795883178711, + "epoch": 0.5306122448979592, + "fcm_dpo/beta": 0.007859094068408012, + "fcm_dpo/delta": 0.007957909256219864, + "fcm_dpo/margin": 49.903011322021484, + "fcm_dpo/q_t": 0.4095124900341034, + "grad_norm": 13.02753734588623, + "learning_rate": 2.6849415780518357e-07, + "logits/chosen": 0.5934625267982483, + "logits/rejected": 0.515455424785614, + "logps/chosen": -153.79214477539062, + "logps/ref_chosen": -56.2563362121582, + "logps/ref_rejected": -79.11589813232422, + "logps/rejected": -226.55471801757812, + "loss": 1.1386, + "margin_dpo/margin_mean": 49.90301513671875, + "margin_dpo/margin_std": 84.64741516113281, + "step": 351 + }, + { + "KL/chosen_KL_mean": -97.65863037109375, + "KL/mean": -126.34355163574219, + "KL/rejected_KL_mean": -155.02847290039062, + "KL/std": 74.74703979492188, + "epoch": 0.5321239606953893, + "fcm_dpo/beta": 0.007809111848473549, + "fcm_dpo/delta": -0.0504891499876976, + "fcm_dpo/margin": 57.369850158691406, + "fcm_dpo/q_t": 0.3992422819137573, + "grad_norm": 12.689568519592285, + "learning_rate": 2.6717530907482024e-07, + "logits/chosen": 0.6787396669387817, + "logits/rejected": 0.6228320598602295, + "logps/chosen": -160.7105712890625, + "logps/ref_chosen": -63.05195236206055, + "logps/ref_rejected": -85.52035522460938, + "logps/rejected": -240.548828125, + "loss": 1.0786, + "margin_dpo/margin_mean": 57.369850158691406, + "margin_dpo/margin_std": 80.87751007080078, + "step": 352 + }, + { + "KL/chosen_KL_mean": -95.69207763671875, + "KL/mean": -122.7952880859375, + "KL/rejected_KL_mean": -149.8984832763672, + "KL/std": 70.47689819335938, + "epoch": 0.5336356764928194, + "fcm_dpo/beta": 0.007794972509145737, + "fcm_dpo/delta": -0.023562796413898468, + "fcm_dpo/margin": 54.2064094543457, + "fcm_dpo/q_t": 0.40339455008506775, + "grad_norm": 12.208295822143555, + "learning_rate": 2.658559799141411e-07, + "logits/chosen": 0.6846418976783752, + "logits/rejected": 0.6894683837890625, + "logps/chosen": -164.70126342773438, + "logps/ref_chosen": -69.00918579101562, + "logps/ref_rejected": -72.65840148925781, + "logps/rejected": -222.556884765625, + "loss": 1.0904, + "margin_dpo/margin_mean": 54.20640563964844, + "margin_dpo/margin_std": 75.77108764648438, + "step": 353 + }, + { + "KL/chosen_KL_mean": -96.5929183959961, + "KL/mean": -127.48405456542969, + "KL/rejected_KL_mean": -158.37518310546875, + "KL/std": 68.72752380371094, + "epoch": 0.5351473922902494, + "fcm_dpo/beta": 0.007651031948626041, + "fcm_dpo/delta": -0.07695139944553375, + "fcm_dpo/margin": 61.78227615356445, + "fcm_dpo/q_t": 0.39242392778396606, + "grad_norm": 13.139801025390625, + "learning_rate": 2.6453620722761895e-07, + "logits/chosen": 0.7695657014846802, + "logits/rejected": 0.6296084523200989, + "logps/chosen": -136.38125610351562, + "logps/ref_chosen": -39.78833770751953, + "logps/ref_rejected": -69.56885528564453, + "logps/rejected": -227.94406127929688, + "loss": 1.0635, + "margin_dpo/margin_mean": 61.78227996826172, + "margin_dpo/margin_std": 82.7366943359375, + "step": 354 + }, + { + "KL/chosen_KL_mean": -103.11363220214844, + "KL/mean": -134.59609985351562, + "KL/rejected_KL_mean": -166.0785675048828, + "KL/std": 77.29907989501953, + "epoch": 0.5366591080876795, + "fcm_dpo/beta": 0.007575302384793758, + "fcm_dpo/delta": -0.0807543694972992, + "fcm_dpo/margin": 62.96492004394531, + "fcm_dpo/q_t": 0.3925732970237732, + "grad_norm": 15.88890552520752, + "learning_rate": 2.632160279321328e-07, + "logits/chosen": 0.7179785966873169, + "logits/rejected": 0.5774426460266113, + "logps/chosen": -149.3690185546875, + "logps/ref_chosen": -46.25537872314453, + "logps/ref_rejected": -78.20236206054688, + "logps/rejected": -244.28091430664062, + "loss": 1.0732, + "margin_dpo/margin_mean": 62.96492385864258, + "margin_dpo/margin_std": 89.31265258789062, + "step": 355 + }, + { + "KL/chosen_KL_mean": -97.25872802734375, + "KL/mean": -123.20895385742188, + "KL/rejected_KL_mean": -149.1591796875, + "KL/std": 73.4035873413086, + "epoch": 0.5381708238851096, + "fcm_dpo/beta": 0.00750363152474165, + "fcm_dpo/delta": 0.010530386120080948, + "fcm_dpo/margin": 51.900447845458984, + "fcm_dpo/q_t": 0.41315823793411255, + "grad_norm": 12.564486503601074, + "learning_rate": 2.618954789559356e-07, + "logits/chosen": 0.660346269607544, + "logits/rejected": 0.5757850408554077, + "logps/chosen": -145.16488647460938, + "logps/ref_chosen": -47.906158447265625, + "logps/ref_rejected": -74.29397583007812, + "logps/rejected": -223.45315551757812, + "loss": 1.1549, + "margin_dpo/margin_mean": 51.90044403076172, + "margin_dpo/margin_std": 94.7741470336914, + "step": 356 + }, + { + "KL/chosen_KL_mean": -115.6062240600586, + "KL/mean": -137.5577392578125, + "KL/rejected_KL_mean": -159.50926208496094, + "KL/std": 72.64031982421875, + "epoch": 0.5396825396825397, + "fcm_dpo/beta": 0.007546389475464821, + "fcm_dpo/delta": 0.06960046291351318, + "fcm_dpo/margin": 43.90301513671875, + "fcm_dpo/q_t": 0.4237821698188782, + "grad_norm": 13.361268997192383, + "learning_rate": 2.6057459723762076e-07, + "logits/chosen": 0.6856166124343872, + "logits/rejected": 0.6604490280151367, + "logps/chosen": -178.24122619628906, + "logps/ref_chosen": -62.63500213623047, + "logps/ref_rejected": -65.11399841308594, + "logps/rejected": -224.62326049804688, + "loss": 1.1781, + "margin_dpo/margin_mean": 43.90301513671875, + "margin_dpo/margin_std": 79.18133544921875, + "step": 357 + }, + { + "KL/chosen_KL_mean": -111.6898193359375, + "KL/mean": -142.54754638671875, + "KL/rejected_KL_mean": -173.40528869628906, + "KL/std": 76.13407897949219, + "epoch": 0.5411942554799698, + "fcm_dpo/beta": 0.007627150975167751, + "fcm_dpo/delta": -0.07491104304790497, + "fcm_dpo/margin": 61.71546173095703, + "fcm_dpo/q_t": 0.3949502408504486, + "grad_norm": 16.380239486694336, + "learning_rate": 2.5925341972508954e-07, + "logits/chosen": 0.589666485786438, + "logits/rejected": 0.6041041612625122, + "logps/chosen": -178.89944458007812, + "logps/ref_chosen": -67.20960998535156, + "logps/ref_rejected": -69.34715270996094, + "logps/rejected": -242.75244140625, + "loss": 1.0715, + "margin_dpo/margin_mean": 61.71546173095703, + "margin_dpo/margin_std": 84.65841674804688, + "step": 358 + }, + { + "KL/chosen_KL_mean": -126.52949523925781, + "KL/mean": -141.81393432617188, + "KL/rejected_KL_mean": -157.098388671875, + "KL/std": 74.38471984863281, + "epoch": 0.5427059712773998, + "fcm_dpo/beta": 0.007773904595524073, + "fcm_dpo/delta": 0.16444545984268188, + "fcm_dpo/margin": 30.568878173828125, + "fcm_dpo/q_t": 0.44677430391311646, + "grad_norm": 15.262069702148438, + "learning_rate": 2.579319833745169e-07, + "logits/chosen": 0.628871500492096, + "logits/rejected": 0.5945923328399658, + "logps/chosen": -189.05528259277344, + "logps/ref_chosen": -62.52578353881836, + "logps/ref_rejected": -76.63114929199219, + "logps/rejected": -233.72952270507812, + "loss": 1.2533, + "margin_dpo/margin_mean": 30.56888198852539, + "margin_dpo/margin_std": 77.31507110595703, + "step": 359 + }, + { + "KL/chosen_KL_mean": -121.8115234375, + "KL/mean": -148.08889770507812, + "KL/rejected_KL_mean": -174.36624145507812, + "KL/std": 79.44476318359375, + "epoch": 0.54421768707483, + "fcm_dpo/beta": 0.007763609290122986, + "fcm_dpo/delta": -0.008455626666545868, + "fcm_dpo/margin": 52.55471420288086, + "fcm_dpo/q_t": 0.4087502360343933, + "grad_norm": 12.670330047607422, + "learning_rate": 2.5661032514931834e-07, + "logits/chosen": 0.5793747901916504, + "logits/rejected": 0.48438405990600586, + "logps/chosen": -185.29925537109375, + "logps/ref_chosen": -63.48772048950195, + "logps/ref_rejected": -90.6891098022461, + "logps/rejected": -265.05535888671875, + "loss": 1.108, + "margin_dpo/margin_mean": 52.554718017578125, + "margin_dpo/margin_std": 79.45843505859375, + "step": 360 + }, + { + "KL/chosen_KL_mean": -116.59442901611328, + "KL/mean": -147.58245849609375, + "KL/rejected_KL_mean": -178.57049560546875, + "KL/std": 77.82331848144531, + "epoch": 0.54572940287226, + "fcm_dpo/beta": 0.007699761539697647, + "fcm_dpo/delta": -0.08099059760570526, + "fcm_dpo/margin": 61.97608184814453, + "fcm_dpo/q_t": 0.3913147449493408, + "grad_norm": 12.27685546875, + "learning_rate": 2.552884820191154e-07, + "logits/chosen": 0.7169207334518433, + "logits/rejected": 0.6682182550430298, + "logps/chosen": -174.51156616210938, + "logps/ref_chosen": -57.917144775390625, + "logps/ref_rejected": -72.39089965820312, + "logps/rejected": -250.96139526367188, + "loss": 1.0528, + "margin_dpo/margin_mean": 61.97608184814453, + "margin_dpo/margin_std": 79.43408203125, + "step": 361 + }, + { + "KL/chosen_KL_mean": -118.05259704589844, + "KL/mean": -151.82545471191406, + "KL/rejected_KL_mean": -185.5983123779297, + "KL/std": 84.501220703125, + "epoch": 0.54724111866969, + "fcm_dpo/beta": 0.0075636012479662895, + "fcm_dpo/delta": -0.11687410622835159, + "fcm_dpo/margin": 67.54571533203125, + "fcm_dpo/q_t": 0.3894726037979126, + "grad_norm": 13.32232666015625, + "learning_rate": 2.53966490958702e-07, + "logits/chosen": 0.7364634275436401, + "logits/rejected": 0.6219512224197388, + "logps/chosen": -181.49606323242188, + "logps/ref_chosen": -63.4434700012207, + "logps/ref_rejected": -103.45516967773438, + "logps/rejected": -289.053466796875, + "loss": 1.0648, + "margin_dpo/margin_mean": 67.54571533203125, + "margin_dpo/margin_std": 99.06629180908203, + "step": 362 + }, + { + "KL/chosen_KL_mean": -123.294921875, + "KL/mean": -155.24533081054688, + "KL/rejected_KL_mean": -187.1957550048828, + "KL/std": 75.24211883544922, + "epoch": 0.5487528344671202, + "fcm_dpo/beta": 0.007382534444332123, + "fcm_dpo/delta": -0.07541361451148987, + "fcm_dpo/margin": 63.90083312988281, + "fcm_dpo/q_t": 0.39400172233581543, + "grad_norm": 14.595932006835938, + "learning_rate": 2.526443889470099e-07, + "logits/chosen": 0.7039732933044434, + "logits/rejected": 0.5658940076828003, + "logps/chosen": -171.94674682617188, + "logps/ref_chosen": -48.65182876586914, + "logps/ref_rejected": -88.65904235839844, + "logps/rejected": -275.85479736328125, + "loss": 1.0634, + "margin_dpo/margin_mean": 63.90083694458008, + "margin_dpo/margin_std": 87.35243225097656, + "step": 363 + }, + { + "KL/chosen_KL_mean": -109.17286682128906, + "KL/mean": -144.913330078125, + "KL/rejected_KL_mean": -180.65379333496094, + "KL/std": 83.70720672607422, + "epoch": 0.5502645502645502, + "fcm_dpo/beta": 0.007225670851767063, + "fcm_dpo/delta": -0.12290657311677933, + "fcm_dpo/margin": 71.4809341430664, + "fcm_dpo/q_t": 0.3865968585014343, + "grad_norm": 11.887857437133789, + "learning_rate": 2.513222129660744e-07, + "logits/chosen": 0.603469729423523, + "logits/rejected": 0.5128209590911865, + "logps/chosen": -167.0439453125, + "logps/ref_chosen": -57.87107467651367, + "logps/ref_rejected": -80.95503234863281, + "logps/rejected": -261.60882568359375, + "loss": 1.0645, + "margin_dpo/margin_mean": 71.4809341430664, + "margin_dpo/margin_std": 104.8623046875, + "step": 364 + }, + { + "KL/chosen_KL_mean": -102.30352020263672, + "KL/mean": -135.4107208251953, + "KL/rejected_KL_mean": -168.51791381835938, + "KL/std": 87.6494369506836, + "epoch": 0.5517762660619804, + "fcm_dpo/beta": 0.007087586913257837, + "fcm_dpo/delta": -0.07303847372531891, + "fcm_dpo/margin": 66.21439361572266, + "fcm_dpo/q_t": 0.39243191480636597, + "grad_norm": 11.069833755493164, + "learning_rate": 2.5e-07, + "logits/chosen": 0.651065468788147, + "logits/rejected": 0.6449978351593018, + "logps/chosen": -167.24569702148438, + "logps/ref_chosen": -64.94217681884766, + "logps/ref_rejected": -74.8599853515625, + "logps/rejected": -243.37789916992188, + "loss": 1.039, + "margin_dpo/margin_mean": 66.21439361572266, + "margin_dpo/margin_std": 77.24095153808594, + "step": 365 + }, + { + "KL/chosen_KL_mean": -101.74160766601562, + "KL/mean": -129.18167114257812, + "KL/rejected_KL_mean": -156.62171936035156, + "KL/std": 77.08753967285156, + "epoch": 0.5532879818594104, + "fcm_dpo/beta": 0.007117936387658119, + "fcm_dpo/delta": 0.009379461407661438, + "fcm_dpo/margin": 54.8801155090332, + "fcm_dpo/q_t": 0.41335535049438477, + "grad_norm": 13.69518756866455, + "learning_rate": 2.486777870339255e-07, + "logits/chosen": 0.6271668672561646, + "logits/rejected": 0.6168335676193237, + "logps/chosen": -156.9075927734375, + "logps/ref_chosen": -55.16598129272461, + "logps/ref_rejected": -65.26121520996094, + "logps/rejected": -221.8829345703125, + "loss": 1.1506, + "margin_dpo/margin_mean": 54.88011169433594, + "margin_dpo/margin_std": 99.16736602783203, + "step": 366 + }, + { + "KL/chosen_KL_mean": -116.36273193359375, + "KL/mean": -143.30245971679688, + "KL/rejected_KL_mean": -170.2421875, + "KL/std": 78.35556030273438, + "epoch": 0.5547996976568406, + "fcm_dpo/beta": 0.007099668495357037, + "fcm_dpo/delta": 0.01813933067023754, + "fcm_dpo/margin": 53.87945556640625, + "fcm_dpo/q_t": 0.41123294830322266, + "grad_norm": 12.775952339172363, + "learning_rate": 2.4735561105299014e-07, + "logits/chosen": 0.663223147392273, + "logits/rejected": 0.5544114112854004, + "logps/chosen": -172.37319946289062, + "logps/ref_chosen": -56.01046371459961, + "logps/ref_rejected": -77.31010437011719, + "logps/rejected": -247.5522918701172, + "loss": 1.1225, + "margin_dpo/margin_mean": 53.879451751708984, + "margin_dpo/margin_std": 83.09695434570312, + "step": 367 + }, + { + "KL/chosen_KL_mean": -128.00091552734375, + "KL/mean": -153.79397583007812, + "KL/rejected_KL_mean": -179.5870361328125, + "KL/std": 77.07600402832031, + "epoch": 0.5563114134542706, + "fcm_dpo/beta": 0.007140608504414558, + "fcm_dpo/delta": 0.03284718468785286, + "fcm_dpo/margin": 51.58610534667969, + "fcm_dpo/q_t": 0.41517072916030884, + "grad_norm": 13.476099014282227, + "learning_rate": 2.46033509041298e-07, + "logits/chosen": 0.5109145045280457, + "logits/rejected": 0.5121912360191345, + "logps/chosen": -202.8302001953125, + "logps/ref_chosen": -74.82927703857422, + "logps/ref_rejected": -76.11680603027344, + "logps/rejected": -255.70384216308594, + "loss": 1.1322, + "margin_dpo/margin_mean": 51.58610534667969, + "margin_dpo/margin_std": 81.55989837646484, + "step": 368 + }, + { + "KL/chosen_KL_mean": -119.14222717285156, + "KL/mean": -142.11587524414062, + "KL/rejected_KL_mean": -165.08953857421875, + "KL/std": 77.98490905761719, + "epoch": 0.5578231292517006, + "fcm_dpo/beta": 0.007270177826285362, + "fcm_dpo/delta": 0.0675322413444519, + "fcm_dpo/margin": 45.947303771972656, + "fcm_dpo/q_t": 0.42452847957611084, + "grad_norm": 12.494619369506836, + "learning_rate": 2.447115179808846e-07, + "logits/chosen": 0.6886367797851562, + "logits/rejected": 0.6335114240646362, + "logps/chosen": -177.46844482421875, + "logps/ref_chosen": -58.32621765136719, + "logps/ref_rejected": -80.92183685302734, + "logps/rejected": -246.01138305664062, + "loss": 1.1678, + "margin_dpo/margin_mean": 45.947303771972656, + "margin_dpo/margin_std": 83.54072570800781, + "step": 369 + }, + { + "KL/chosen_KL_mean": -114.87617492675781, + "KL/mean": -147.30535888671875, + "KL/rejected_KL_mean": -179.7345733642578, + "KL/std": 81.04230499267578, + "epoch": 0.5593348450491308, + "fcm_dpo/beta": 0.007172431331127882, + "fcm_dpo/delta": -0.06864120811223984, + "fcm_dpo/margin": 64.8583984375, + "fcm_dpo/q_t": 0.393879234790802, + "grad_norm": 12.52617073059082, + "learning_rate": 2.4338967485068164e-07, + "logits/chosen": 0.7644656896591187, + "logits/rejected": 0.6927477717399597, + "logps/chosen": -167.7598876953125, + "logps/ref_chosen": -52.88372039794922, + "logps/ref_rejected": -79.43692016601562, + "logps/rejected": -259.1714782714844, + "loss": 1.0801, + "margin_dpo/margin_mean": 64.85840606689453, + "margin_dpo/margin_std": 94.23211669921875, + "step": 370 + }, + { + "KL/chosen_KL_mean": -115.05244445800781, + "KL/mean": -142.79544067382812, + "KL/rejected_KL_mean": -170.53843688964844, + "KL/std": 77.92510986328125, + "epoch": 0.5608465608465608, + "fcm_dpo/beta": 0.007202905602753162, + "fcm_dpo/delta": -0.000254802405834198, + "fcm_dpo/margin": 55.48599624633789, + "fcm_dpo/q_t": 0.4096566438674927, + "grad_norm": 15.842639923095703, + "learning_rate": 2.420680166254831e-07, + "logits/chosen": 0.8232964277267456, + "logits/rejected": 0.788852870464325, + "logps/chosen": -164.2766571044922, + "logps/ref_chosen": -49.224212646484375, + "logps/ref_rejected": -63.348472595214844, + "logps/rejected": -233.88690185546875, + "loss": 1.1171, + "margin_dpo/margin_mean": 55.486000061035156, + "margin_dpo/margin_std": 85.15362548828125, + "step": 371 + }, + { + "KL/chosen_KL_mean": -123.14059448242188, + "KL/mean": -139.93470764160156, + "KL/rejected_KL_mean": -156.72882080078125, + "KL/std": 77.38941192626953, + "epoch": 0.562358276643991, + "fcm_dpo/beta": 0.007378925569355488, + "fcm_dpo/delta": 0.1541157066822052, + "fcm_dpo/margin": 33.58821487426758, + "fcm_dpo/q_t": 0.4454403519630432, + "grad_norm": 16.654294967651367, + "learning_rate": 2.4074658027491044e-07, + "logits/chosen": 0.6877930760383606, + "logits/rejected": 0.5879219770431519, + "logps/chosen": -175.41015625, + "logps/ref_chosen": -52.269554138183594, + "logps/ref_rejected": -72.99522399902344, + "logps/rejected": -229.72402954101562, + "loss": 1.2892, + "margin_dpo/margin_mean": 33.588218688964844, + "margin_dpo/margin_std": 101.15355682373047, + "step": 372 + }, + { + "KL/chosen_KL_mean": -136.29205322265625, + "KL/mean": -160.035888671875, + "KL/rejected_KL_mean": -183.77972412109375, + "KL/std": 79.77000427246094, + "epoch": 0.563869992441421, + "fcm_dpo/beta": 0.007445330731570721, + "fcm_dpo/delta": 0.048104479908943176, + "fcm_dpo/margin": 47.48767852783203, + "fcm_dpo/q_t": 0.4214223027229309, + "grad_norm": 13.432877540588379, + "learning_rate": 2.394254027623792e-07, + "logits/chosen": 0.6861114501953125, + "logits/rejected": 0.6134101152420044, + "logps/chosen": -197.40504455566406, + "logps/ref_chosen": -61.112998962402344, + "logps/ref_rejected": -76.24851989746094, + "logps/rejected": -260.0282287597656, + "loss": 1.1955, + "margin_dpo/margin_mean": 47.48767852783203, + "margin_dpo/margin_std": 100.75148010253906, + "step": 373 + }, + { + "KL/chosen_KL_mean": -116.14297485351562, + "KL/mean": -153.75967407226562, + "KL/rejected_KL_mean": -191.37637329101562, + "KL/std": 82.49403381347656, + "epoch": 0.5653817082388511, + "fcm_dpo/beta": 0.007278250530362129, + "fcm_dpo/delta": -0.1563607156276703, + "fcm_dpo/margin": 75.23340606689453, + "fcm_dpo/q_t": 0.37690961360931396, + "grad_norm": 13.282299995422363, + "learning_rate": 2.381045210440644e-07, + "logits/chosen": 0.6100819110870361, + "logits/rejected": 0.6109589338302612, + "logps/chosen": -188.8121795654297, + "logps/ref_chosen": -72.66920471191406, + "logps/ref_rejected": -76.83158874511719, + "logps/rejected": -268.20794677734375, + "loss": 1.0268, + "margin_dpo/margin_mean": 75.23340606689453, + "margin_dpo/margin_std": 97.93537902832031, + "step": 374 + }, + { + "KL/chosen_KL_mean": -113.97125244140625, + "KL/mean": -142.1385040283203, + "KL/rejected_KL_mean": -170.3057403564453, + "KL/std": 88.80377197265625, + "epoch": 0.5668934240362812, + "fcm_dpo/beta": 0.007202597334980965, + "fcm_dpo/delta": -0.006175890564918518, + "fcm_dpo/margin": 56.33448791503906, + "fcm_dpo/q_t": 0.40949082374572754, + "grad_norm": 15.551325798034668, + "learning_rate": 2.3678397206786715e-07, + "logits/chosen": 0.7072441577911377, + "logits/rejected": 0.6454247236251831, + "logps/chosen": -171.65455627441406, + "logps/ref_chosen": -57.68330383300781, + "logps/ref_rejected": -79.34097290039062, + "logps/rejected": -249.64671325683594, + "loss": 1.133, + "margin_dpo/margin_mean": 56.33448028564453, + "margin_dpo/margin_std": 96.07463073730469, + "step": 375 + }, + { + "KL/chosen_KL_mean": -122.6119384765625, + "KL/mean": -156.71728515625, + "KL/rejected_KL_mean": -190.8226318359375, + "KL/std": 88.48606872558594, + "epoch": 0.5684051398337112, + "fcm_dpo/beta": 0.007107567973434925, + "fcm_dpo/delta": -0.0894772857427597, + "fcm_dpo/margin": 68.2106704711914, + "fcm_dpo/q_t": 0.3942412734031677, + "grad_norm": 13.224767684936523, + "learning_rate": 2.3546379277238103e-07, + "logits/chosen": 0.7721911668777466, + "logits/rejected": 0.6945620775222778, + "logps/chosen": -174.2860107421875, + "logps/ref_chosen": -51.674072265625, + "logps/ref_rejected": -75.69713592529297, + "logps/rejected": -266.519775390625, + "loss": 1.0832, + "margin_dpo/margin_mean": 68.2106704711914, + "margin_dpo/margin_std": 104.27572631835938, + "step": 376 + }, + { + "KL/chosen_KL_mean": -124.97073364257812, + "KL/mean": -148.96510314941406, + "KL/rejected_KL_mean": -172.95948791503906, + "KL/std": 75.81988525390625, + "epoch": 0.5699168556311414, + "fcm_dpo/beta": 0.007183787412941456, + "fcm_dpo/delta": 0.05659983307123184, + "fcm_dpo/margin": 47.98875045776367, + "fcm_dpo/q_t": 0.42107149958610535, + "grad_norm": 13.312004089355469, + "learning_rate": 2.3414402008585886e-07, + "logits/chosen": 0.6740518808364868, + "logits/rejected": 0.6495604515075684, + "logps/chosen": -171.14926147460938, + "logps/ref_chosen": -46.17853546142578, + "logps/ref_rejected": -57.756500244140625, + "logps/rejected": -230.71597290039062, + "loss": 1.1638, + "margin_dpo/margin_mean": 47.988746643066406, + "margin_dpo/margin_std": 86.49845886230469, + "step": 377 + }, + { + "KL/chosen_KL_mean": -125.39057922363281, + "KL/mean": -147.92562866210938, + "KL/rejected_KL_mean": -170.46072387695312, + "KL/std": 82.615234375, + "epoch": 0.5714285714285714, + "fcm_dpo/beta": 0.0072835953906178474, + "fcm_dpo/delta": 0.07346655428409576, + "fcm_dpo/margin": 45.07012176513672, + "fcm_dpo/q_t": 0.42542755603790283, + "grad_norm": 13.203572273254395, + "learning_rate": 2.3282469092517977e-07, + "logits/chosen": 0.7316404581069946, + "logits/rejected": 0.6816238164901733, + "logps/chosen": -184.6094512939453, + "logps/ref_chosen": -59.21887969970703, + "logps/ref_rejected": -71.24818420410156, + "logps/rejected": -241.70889282226562, + "loss": 1.1769, + "margin_dpo/margin_mean": 45.07012176513672, + "margin_dpo/margin_std": 84.25788116455078, + "step": 378 + }, + { + "KL/chosen_KL_mean": -121.52671813964844, + "KL/mean": -154.25457763671875, + "KL/rejected_KL_mean": -186.98245239257812, + "KL/std": 85.5152587890625, + "epoch": 0.5729402872260015, + "fcm_dpo/beta": 0.007202798500657082, + "fcm_dpo/delta": -0.07496453821659088, + "fcm_dpo/margin": 65.45574951171875, + "fcm_dpo/q_t": 0.3955724537372589, + "grad_norm": 14.618745803833008, + "learning_rate": 2.3150584219481643e-07, + "logits/chosen": 0.671619176864624, + "logits/rejected": 0.5937076807022095, + "logps/chosen": -197.84329223632812, + "logps/ref_chosen": -76.31658935546875, + "logps/ref_rejected": -104.26200103759766, + "logps/rejected": -291.24444580078125, + "loss": 1.074, + "margin_dpo/margin_mean": 65.45574951171875, + "margin_dpo/margin_std": 94.66348266601562, + "step": 379 + }, + { + "KL/chosen_KL_mean": -105.07964324951172, + "KL/mean": -143.19851684570312, + "KL/rejected_KL_mean": -181.31741333007812, + "KL/std": 78.93547058105469, + "epoch": 0.5744520030234316, + "fcm_dpo/beta": 0.007026575040072203, + "fcm_dpo/delta": -0.14325784146785736, + "fcm_dpo/margin": 76.23777770996094, + "fcm_dpo/q_t": 0.37770792841911316, + "grad_norm": 11.684196472167969, + "learning_rate": 2.3018751078580283e-07, + "logits/chosen": 0.7029041051864624, + "logits/rejected": 0.6619795560836792, + "logps/chosen": -166.36280822753906, + "logps/ref_chosen": -61.283164978027344, + "logps/ref_rejected": -72.38892364501953, + "logps/rejected": -253.7063446044922, + "loss": 1.023, + "margin_dpo/margin_mean": 76.23777770996094, + "margin_dpo/margin_std": 94.23179626464844, + "step": 380 + }, + { + "KL/chosen_KL_mean": -129.57518005371094, + "KL/mean": -144.18365478515625, + "KL/rejected_KL_mean": -158.7921142578125, + "KL/std": 81.25999450683594, + "epoch": 0.5759637188208617, + "fcm_dpo/beta": 0.007202141918241978, + "fcm_dpo/delta": 0.1924881935119629, + "fcm_dpo/margin": 29.21695327758789, + "fcm_dpo/q_t": 0.45364683866500854, + "grad_norm": 12.961562156677246, + "learning_rate": 2.288697335747027e-07, + "logits/chosen": 0.6437864303588867, + "logits/rejected": 0.6247032880783081, + "logps/chosen": -187.78916931152344, + "logps/ref_chosen": -58.2139892578125, + "logps/ref_rejected": -60.78669357299805, + "logps/rejected": -219.5788116455078, + "loss": 1.3017, + "margin_dpo/margin_mean": 29.21695327758789, + "margin_dpo/margin_std": 97.07288360595703, + "step": 381 + }, + { + "KL/chosen_KL_mean": -129.46324157714844, + "KL/mean": -156.5772705078125, + "KL/rejected_KL_mean": -183.69131469726562, + "KL/std": 81.28971862792969, + "epoch": 0.5774754346182918, + "fcm_dpo/beta": 0.007276223041117191, + "fcm_dpo/delta": 0.005313534289598465, + "fcm_dpo/margin": 54.228084564208984, + "fcm_dpo/q_t": 0.41020467877388, + "grad_norm": 13.063127517700195, + "learning_rate": 2.2755254742257706e-07, + "logits/chosen": 0.6734954118728638, + "logits/rejected": 0.6170482635498047, + "logps/chosen": -191.28855895996094, + "logps/ref_chosen": -61.82532501220703, + "logps/ref_rejected": -83.0452880859375, + "logps/rejected": -266.7366027832031, + "loss": 1.1109, + "margin_dpo/margin_mean": 54.228084564208984, + "margin_dpo/margin_std": 79.92002868652344, + "step": 382 + }, + { + "KL/chosen_KL_mean": -130.00021362304688, + "KL/mean": -154.45675659179688, + "KL/rejected_KL_mean": -178.91326904296875, + "KL/std": 81.607421875, + "epoch": 0.5789871504157218, + "fcm_dpo/beta": 0.0072705792263150215, + "fcm_dpo/delta": 0.0458693653345108, + "fcm_dpo/margin": 48.91305923461914, + "fcm_dpo/q_t": 0.42183059453964233, + "grad_norm": 15.296006202697754, + "learning_rate": 2.2623598917395436e-07, + "logits/chosen": 0.5694981813430786, + "logits/rejected": 0.6031885147094727, + "logps/chosen": -210.5634765625, + "logps/ref_chosen": -80.56326293945312, + "logps/ref_rejected": -74.62922668457031, + "logps/rejected": -253.54249572753906, + "loss": 1.1927, + "margin_dpo/margin_mean": 48.91305923461914, + "margin_dpo/margin_std": 102.63812255859375, + "step": 383 + }, + { + "KL/chosen_KL_mean": -124.91947937011719, + "KL/mean": -152.39324951171875, + "KL/rejected_KL_mean": -179.86700439453125, + "KL/std": 80.42024993896484, + "epoch": 0.5804988662131519, + "fcm_dpo/beta": 0.0073145655915141106, + "fcm_dpo/delta": -0.00199760589748621, + "fcm_dpo/margin": 54.94753646850586, + "fcm_dpo/q_t": 0.40669381618499756, + "grad_norm": 15.579803466796875, + "learning_rate": 2.2492009565579875e-07, + "logits/chosen": 0.7384841442108154, + "logits/rejected": 0.6944015622138977, + "logps/chosen": -190.39462280273438, + "logps/ref_chosen": -65.47514343261719, + "logps/ref_rejected": -79.67378234863281, + "logps/rejected": -259.5408020019531, + "loss": 1.1154, + "margin_dpo/margin_mean": 54.94753646850586, + "margin_dpo/margin_std": 85.66532135009766, + "step": 384 + }, + { + "KL/chosen_KL_mean": -123.13201904296875, + "KL/mean": -159.01617431640625, + "KL/rejected_KL_mean": -194.9003143310547, + "KL/std": 82.08739471435547, + "epoch": 0.582010582010582, + "fcm_dpo/beta": 0.0072093624621629715, + "fcm_dpo/delta": -0.12364254891872406, + "fcm_dpo/margin": 71.76829528808594, + "fcm_dpo/q_t": 0.383453369140625, + "grad_norm": 12.84723949432373, + "learning_rate": 2.2360490367648084e-07, + "logits/chosen": 0.6259031295776367, + "logits/rejected": 0.5812070965766907, + "logps/chosen": -189.18853759765625, + "logps/ref_chosen": -66.0565185546875, + "logps/ref_rejected": -86.68023681640625, + "logps/rejected": -281.58056640625, + "loss": 1.0275, + "margin_dpo/margin_mean": 71.76829528808594, + "margin_dpo/margin_std": 87.22917175292969, + "step": 385 + }, + { + "KL/chosen_KL_mean": -142.86434936523438, + "KL/mean": -166.49163818359375, + "KL/rejected_KL_mean": -190.11892700195312, + "KL/std": 85.83015441894531, + "epoch": 0.5835222978080121, + "fcm_dpo/beta": 0.007186328526586294, + "fcm_dpo/delta": 0.06257650256156921, + "fcm_dpo/margin": 47.25459289550781, + "fcm_dpo/q_t": 0.42166996002197266, + "grad_norm": 13.215150833129883, + "learning_rate": 2.2229045002474724e-07, + "logits/chosen": 0.6145610809326172, + "logits/rejected": 0.5554145574569702, + "logps/chosen": -218.48800659179688, + "logps/ref_chosen": -75.6236572265625, + "logps/ref_rejected": -92.62330627441406, + "logps/rejected": -282.74224853515625, + "loss": 1.1694, + "margin_dpo/margin_mean": 47.25458908081055, + "margin_dpo/margin_std": 87.60594940185547, + "step": 386 + }, + { + "KL/chosen_KL_mean": -125.47482299804688, + "KL/mean": -159.42498779296875, + "KL/rejected_KL_mean": -193.3751678466797, + "KL/std": 79.78838348388672, + "epoch": 0.5850340136054422, + "fcm_dpo/beta": 0.007123415358364582, + "fcm_dpo/delta": -0.08804966509342194, + "fcm_dpo/margin": 67.90034484863281, + "fcm_dpo/q_t": 0.39012467861175537, + "grad_norm": 13.807373046875, + "learning_rate": 2.209767714686924e-07, + "logits/chosen": 0.7212274670600891, + "logits/rejected": 0.6039391756057739, + "logps/chosen": -172.696533203125, + "logps/ref_chosen": -47.22170639038086, + "logps/ref_rejected": -87.338134765625, + "logps/rejected": -280.71331787109375, + "loss": 1.0358, + "margin_dpo/margin_mean": 67.90033721923828, + "margin_dpo/margin_std": 80.91610717773438, + "step": 387 + }, + { + "KL/chosen_KL_mean": -128.02667236328125, + "KL/mean": -149.42562866210938, + "KL/rejected_KL_mean": -170.8245849609375, + "KL/std": 84.12289428710938, + "epoch": 0.5865457294028723, + "fcm_dpo/beta": 0.007163255475461483, + "fcm_dpo/delta": 0.09631352126598358, + "fcm_dpo/margin": 42.79789352416992, + "fcm_dpo/q_t": 0.4336731433868408, + "grad_norm": 13.67048168182373, + "learning_rate": 2.1966390475472954e-07, + "logits/chosen": 0.6444123983383179, + "logits/rejected": 0.639690101146698, + "logps/chosen": -202.60614013671875, + "logps/ref_chosen": -74.5794677734375, + "logps/ref_rejected": -79.92558288574219, + "logps/rejected": -250.7501678466797, + "loss": 1.2238, + "margin_dpo/margin_mean": 42.797889709472656, + "margin_dpo/margin_std": 99.72714233398438, + "step": 388 + }, + { + "KL/chosen_KL_mean": -121.66131591796875, + "KL/mean": -155.45494079589844, + "KL/rejected_KL_mean": -189.24856567382812, + "KL/std": 81.51905059814453, + "epoch": 0.5880574452003023, + "fcm_dpo/beta": 0.007154286373406649, + "fcm_dpo/delta": -0.0876961201429367, + "fcm_dpo/margin": 67.58724212646484, + "fcm_dpo/q_t": 0.3913712501525879, + "grad_norm": 24.15449333190918, + "learning_rate": 2.1835188660656265e-07, + "logits/chosen": 0.7004282474517822, + "logits/rejected": 0.6616201400756836, + "logps/chosen": -183.28567504882812, + "logps/ref_chosen": -61.624366760253906, + "logps/ref_rejected": -76.50978088378906, + "logps/rejected": -265.75836181640625, + "loss": 1.0559, + "margin_dpo/margin_mean": 67.58724212646484, + "margin_dpo/margin_std": 90.30506896972656, + "step": 389 + }, + { + "KL/chosen_KL_mean": -109.01703643798828, + "KL/mean": -135.18896484375, + "KL/rejected_KL_mean": -161.36090087890625, + "KL/std": 78.40666198730469, + "epoch": 0.5895691609977324, + "fcm_dpo/beta": 0.0071219149976968765, + "fcm_dpo/delta": 0.028195127844810486, + "fcm_dpo/margin": 52.34385681152344, + "fcm_dpo/q_t": 0.41536635160446167, + "grad_norm": 11.698328971862793, + "learning_rate": 2.170407537241599e-07, + "logits/chosen": 0.7919474840164185, + "logits/rejected": 0.7127720713615417, + "logps/chosen": -154.88890075683594, + "logps/ref_chosen": -45.871864318847656, + "logps/ref_rejected": -61.305999755859375, + "logps/rejected": -222.66690063476562, + "loss": 1.1296, + "margin_dpo/margin_mean": 52.34385681152344, + "margin_dpo/margin_std": 82.52626037597656, + "step": 390 + }, + { + "KL/chosen_KL_mean": -122.2738037109375, + "KL/mean": -153.88844299316406, + "KL/rejected_KL_mean": -185.50308227539062, + "KL/std": 80.52210235595703, + "epoch": 0.5910808767951625, + "fcm_dpo/beta": 0.007075528148561716, + "fcm_dpo/delta": -0.049968231469392776, + "fcm_dpo/margin": 63.22930145263672, + "fcm_dpo/q_t": 0.39903032779693604, + "grad_norm": 12.636250495910645, + "learning_rate": 2.1573054278272636e-07, + "logits/chosen": 0.6821379065513611, + "logits/rejected": 0.6111657619476318, + "logps/chosen": -180.4608154296875, + "logps/ref_chosen": -58.18701171875, + "logps/ref_rejected": -83.63442993164062, + "logps/rejected": -269.13751220703125, + "loss": 1.1019, + "margin_dpo/margin_mean": 63.22930145263672, + "margin_dpo/margin_std": 98.30987548828125, + "step": 391 + }, + { + "KL/chosen_KL_mean": -109.65264892578125, + "KL/mean": -144.0840606689453, + "KL/rejected_KL_mean": -178.51547241210938, + "KL/std": 86.02153015136719, + "epoch": 0.5925925925925926, + "fcm_dpo/beta": 0.007030046544969082, + "fcm_dpo/delta": -0.0886000394821167, + "fcm_dpo/margin": 68.86282348632812, + "fcm_dpo/q_t": 0.3925461173057556, + "grad_norm": 11.746094703674316, + "learning_rate": 2.1442129043167873e-07, + "logits/chosen": 0.6962657570838928, + "logits/rejected": 0.6370902061462402, + "logps/chosen": -179.39718627929688, + "logps/ref_chosen": -69.7445297241211, + "logps/ref_rejected": -94.05877685546875, + "logps/rejected": -272.5742492675781, + "loss": 1.0767, + "margin_dpo/margin_mean": 68.86282348632812, + "margin_dpo/margin_std": 101.075439453125, + "step": 392 + }, + { + "KL/chosen_KL_mean": -123.72262573242188, + "KL/mean": -159.5321044921875, + "KL/rejected_KL_mean": -195.34158325195312, + "KL/std": 83.92768859863281, + "epoch": 0.5941043083900227, + "fcm_dpo/beta": 0.006823638454079628, + "fcm_dpo/delta": -0.09428704530000687, + "fcm_dpo/margin": 71.61897277832031, + "fcm_dpo/q_t": 0.3880813419818878, + "grad_norm": 11.2382230758667, + "learning_rate": 2.131130332936195e-07, + "logits/chosen": 0.6831046342849731, + "logits/rejected": 0.6424210071563721, + "logps/chosen": -176.05752563476562, + "logps/ref_chosen": -52.33489990234375, + "logps/ref_rejected": -74.33809661865234, + "logps/rejected": -269.6796875, + "loss": 1.0332, + "margin_dpo/margin_mean": 71.61896514892578, + "margin_dpo/margin_std": 83.35065460205078, + "step": 393 + }, + { + "KL/chosen_KL_mean": -119.21342468261719, + "KL/mean": -149.92340087890625, + "KL/rejected_KL_mean": -180.6333770751953, + "KL/std": 77.53596496582031, + "epoch": 0.5956160241874527, + "fcm_dpo/beta": 0.006829938385635614, + "fcm_dpo/delta": -0.020546775311231613, + "fcm_dpo/margin": 61.419960021972656, + "fcm_dpo/q_t": 0.40248793363571167, + "grad_norm": 11.583659172058105, + "learning_rate": 2.1180580796331323e-07, + "logits/chosen": 0.7224094867706299, + "logits/rejected": 0.6902562379837036, + "logps/chosen": -179.88955688476562, + "logps/ref_chosen": -60.6761360168457, + "logps/ref_rejected": -71.36074829101562, + "logps/rejected": -251.994140625, + "loss": 1.0789, + "margin_dpo/margin_mean": 61.419960021972656, + "margin_dpo/margin_std": 77.19164276123047, + "step": 394 + }, + { + "KL/chosen_KL_mean": -122.50703430175781, + "KL/mean": -150.58975219726562, + "KL/rejected_KL_mean": -178.67247009277344, + "KL/std": 80.42475891113281, + "epoch": 0.5971277399848829, + "fcm_dpo/beta": 0.0068355584517121315, + "fcm_dpo/delta": 0.016474729403853416, + "fcm_dpo/margin": 56.165428161621094, + "fcm_dpo/q_t": 0.4139012396335602, + "grad_norm": 13.971101760864258, + "learning_rate": 2.104996510066625e-07, + "logits/chosen": 0.7060272693634033, + "logits/rejected": 0.6042532920837402, + "logps/chosen": -173.11135864257812, + "logps/ref_chosen": -50.60432434082031, + "logps/ref_rejected": -77.08731079101562, + "logps/rejected": -255.75978088378906, + "loss": 1.1185, + "margin_dpo/margin_mean": 56.165428161621094, + "margin_dpo/margin_std": 85.66535949707031, + "step": 395 + }, + { + "KL/chosen_KL_mean": -116.39704132080078, + "KL/mean": -146.19480895996094, + "KL/rejected_KL_mean": -175.99258422851562, + "KL/std": 89.64775085449219, + "epoch": 0.5986394557823129, + "fcm_dpo/beta": 0.0067558130249381065, + "fcm_dpo/delta": -0.0043720826506614685, + "fcm_dpo/margin": 59.59553146362305, + "fcm_dpo/q_t": 0.4080699682235718, + "grad_norm": 11.438410758972168, + "learning_rate": 2.0919459895968517e-07, + "logits/chosen": 0.7179821729660034, + "logits/rejected": 0.6186529994010925, + "logps/chosen": -167.75665283203125, + "logps/ref_chosen": -51.35961151123047, + "logps/ref_rejected": -79.89360046386719, + "logps/rejected": -255.88616943359375, + "loss": 1.0956, + "margin_dpo/margin_mean": 59.59553146362305, + "margin_dpo/margin_std": 77.72926330566406, + "step": 396 + }, + { + "KL/chosen_KL_mean": -133.34695434570312, + "KL/mean": -150.32850646972656, + "KL/rejected_KL_mean": -167.31004333496094, + "KL/std": 81.386474609375, + "epoch": 0.600151171579743, + "fcm_dpo/beta": 0.006991279777139425, + "fcm_dpo/delta": 0.1663690060377121, + "fcm_dpo/margin": 33.96311950683594, + "fcm_dpo/q_t": 0.4465146064758301, + "grad_norm": 12.702045440673828, + "learning_rate": 2.078906883274924e-07, + "logits/chosen": 0.6114163994789124, + "logits/rejected": 0.5590552687644958, + "logps/chosen": -199.8031768798828, + "logps/ref_chosen": -66.45622253417969, + "logps/ref_rejected": -85.74736785888672, + "logps/rejected": -253.0574188232422, + "loss": 1.2715, + "margin_dpo/margin_mean": 33.96311950683594, + "margin_dpo/margin_std": 96.10679626464844, + "step": 397 + }, + { + "KL/chosen_KL_mean": -113.849853515625, + "KL/mean": -151.95046997070312, + "KL/rejected_KL_mean": -190.05108642578125, + "KL/std": 85.21319580078125, + "epoch": 0.6016628873771731, + "fcm_dpo/beta": 0.0068751135841012, + "fcm_dpo/delta": -0.13184233009815216, + "fcm_dpo/margin": 76.20124053955078, + "fcm_dpo/q_t": 0.38280850648880005, + "grad_norm": 10.512614250183105, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": 0.6477320194244385, + "logits/rejected": 0.5816516876220703, + "logps/chosen": -163.09410095214844, + "logps/ref_chosen": -49.244239807128906, + "logps/ref_rejected": -75.18949127197266, + "logps/rejected": -265.2405700683594, + "loss": 1.0135, + "margin_dpo/margin_mean": 76.20124053955078, + "margin_dpo/margin_std": 89.14450073242188, + "step": 398 + }, + { + "KL/chosen_KL_mean": -132.7601776123047, + "KL/mean": -176.23660278320312, + "KL/rejected_KL_mean": -219.71304321289062, + "KL/std": 94.06497192382812, + "epoch": 0.6031746031746031, + "fcm_dpo/beta": 0.006639732047915459, + "fcm_dpo/delta": -0.1895592212677002, + "fcm_dpo/margin": 86.952880859375, + "fcm_dpo/q_t": 0.3730096220970154, + "grad_norm": 12.22734260559082, + "learning_rate": 2.052864371672457e-07, + "logits/chosen": 0.6234591007232666, + "logits/rejected": 0.4737226963043213, + "logps/chosen": -201.06695556640625, + "logps/ref_chosen": -68.30679321289062, + "logps/ref_rejected": -113.2708511352539, + "logps/rejected": -332.98388671875, + "loss": 0.9958, + "margin_dpo/margin_mean": 86.95288848876953, + "margin_dpo/margin_std": 105.41049194335938, + "step": 399 + }, + { + "KL/chosen_KL_mean": -147.748779296875, + "KL/mean": -172.35479736328125, + "KL/rejected_KL_mean": -196.96080017089844, + "KL/std": 86.04580688476562, + "epoch": 0.6046863189720333, + "fcm_dpo/beta": 0.006632579956203699, + "fcm_dpo/delta": 0.07553811371326447, + "fcm_dpo/margin": 49.212013244628906, + "fcm_dpo/q_t": 0.42484426498413086, + "grad_norm": 17.143085479736328, + "learning_rate": 2.0398616948569493e-07, + "logits/chosen": 0.6802634000778198, + "logits/rejected": 0.6172671318054199, + "logps/chosen": -219.37527465820312, + "logps/ref_chosen": -71.62649536132812, + "logps/ref_rejected": -90.98765563964844, + "logps/rejected": -287.9484558105469, + "loss": 1.167, + "margin_dpo/margin_mean": 49.212013244628906, + "margin_dpo/margin_std": 84.21356201171875, + "step": 400 + }, + { + "KL/chosen_KL_mean": -112.90580749511719, + "KL/mean": -147.96231079101562, + "KL/rejected_KL_mean": -183.01881408691406, + "KL/std": 92.42097473144531, + "epoch": 0.6061980347694633, + "fcm_dpo/beta": 0.006614885292947292, + "fcm_dpo/delta": -0.0674346387386322, + "fcm_dpo/margin": 70.11300659179688, + "fcm_dpo/q_t": 0.3951302170753479, + "grad_norm": 9.65842342376709, + "learning_rate": 2.0268718890989752e-07, + "logits/chosen": 0.7155077457427979, + "logits/rejected": 0.6205871105194092, + "logps/chosen": -166.63076782226562, + "logps/ref_chosen": -53.72495651245117, + "logps/ref_rejected": -75.06304931640625, + "logps/rejected": -258.08184814453125, + "loss": 1.0542, + "margin_dpo/margin_mean": 70.11299896240234, + "margin_dpo/margin_std": 88.1304702758789, + "step": 401 + }, + { + "KL/chosen_KL_mean": -123.09654235839844, + "KL/mean": -150.04502868652344, + "KL/rejected_KL_mean": -176.9935302734375, + "KL/std": 79.02203369140625, + "epoch": 0.6077097505668935, + "fcm_dpo/beta": 0.006622655317187309, + "fcm_dpo/delta": 0.04423164948821068, + "fcm_dpo/margin": 53.89698791503906, + "fcm_dpo/q_t": 0.41926831007003784, + "grad_norm": 13.637972831726074, + "learning_rate": 2.013895317751323e-07, + "logits/chosen": 0.6671075820922852, + "logits/rejected": 0.646820068359375, + "logps/chosen": -184.970458984375, + "logps/ref_chosen": -61.873931884765625, + "logps/ref_rejected": -66.15198516845703, + "logps/rejected": -243.1455078125, + "loss": 1.1591, + "margin_dpo/margin_mean": 53.89698028564453, + "margin_dpo/margin_std": 95.33618927001953, + "step": 402 + }, + { + "KL/chosen_KL_mean": -129.99464416503906, + "KL/mean": -165.91558837890625, + "KL/rejected_KL_mean": -201.83653259277344, + "KL/std": 91.60020446777344, + "epoch": 0.6092214663643235, + "fcm_dpo/beta": 0.006587062031030655, + "fcm_dpo/delta": -0.07708509266376495, + "fcm_dpo/margin": 71.84190368652344, + "fcm_dpo/q_t": 0.39418938755989075, + "grad_norm": 10.517227172851562, + "learning_rate": 2.0009323437965898e-07, + "logits/chosen": 0.8143857717514038, + "logits/rejected": 0.722244143486023, + "logps/chosen": -181.31614685058594, + "logps/ref_chosen": -51.321502685546875, + "logps/ref_rejected": -86.54010772705078, + "logps/rejected": -288.37664794921875, + "loss": 1.0617, + "margin_dpo/margin_mean": 71.84190368652344, + "margin_dpo/margin_std": 96.05760192871094, + "step": 403 + }, + { + "KL/chosen_KL_mean": -122.32210540771484, + "KL/mean": -158.37295532226562, + "KL/rejected_KL_mean": -194.42381286621094, + "KL/std": 92.72088623046875, + "epoch": 0.6107331821617535, + "fcm_dpo/beta": 0.006459000986069441, + "fcm_dpo/delta": -0.07044952362775803, + "fcm_dpo/margin": 72.10169982910156, + "fcm_dpo/q_t": 0.3962360620498657, + "grad_norm": 14.719085693359375, + "learning_rate": 1.9879833298370237e-07, + "logits/chosen": 0.6476539969444275, + "logits/rejected": 0.5492456555366516, + "logps/chosen": -184.58499145507812, + "logps/ref_chosen": -62.26288604736328, + "logps/ref_rejected": -95.19029998779297, + "logps/rejected": -289.6141052246094, + "loss": 1.0753, + "margin_dpo/margin_mean": 72.10169982910156, + "margin_dpo/margin_std": 100.69046020507812, + "step": 404 + }, + { + "KL/chosen_KL_mean": -125.48126220703125, + "KL/mean": -154.83218383789062, + "KL/rejected_KL_mean": -184.18310546875, + "KL/std": 83.02670288085938, + "epoch": 0.6122448979591837, + "fcm_dpo/beta": 0.006452477071434259, + "fcm_dpo/delta": 0.02102312445640564, + "fcm_dpo/margin": 58.70184326171875, + "fcm_dpo/q_t": 0.41416507959365845, + "grad_norm": 11.768880844116211, + "learning_rate": 1.975048638084379e-07, + "logits/chosen": 0.7292817831039429, + "logits/rejected": 0.6814388036727905, + "logps/chosen": -176.06561279296875, + "logps/ref_chosen": -50.5843391418457, + "logps/ref_rejected": -65.43156433105469, + "logps/rejected": -249.6146697998047, + "loss": 1.1197, + "margin_dpo/margin_mean": 58.70184326171875, + "margin_dpo/margin_std": 84.7135009765625, + "step": 405 + }, + { + "KL/chosen_KL_mean": -120.27396392822266, + "KL/mean": -158.66598510742188, + "KL/rejected_KL_mean": -197.05801391601562, + "KL/std": 89.59160614013672, + "epoch": 0.6137566137566137, + "fcm_dpo/beta": 0.006429762579500675, + "fcm_dpo/delta": -0.09845130890607834, + "fcm_dpo/margin": 76.78402709960938, + "fcm_dpo/q_t": 0.3877302408218384, + "grad_norm": 12.265458106994629, + "learning_rate": 1.9621286303497914e-07, + "logits/chosen": 0.7168718576431274, + "logits/rejected": 0.5531203746795654, + "logps/chosen": -169.26956176757812, + "logps/ref_chosen": -48.99560546875, + "logps/ref_rejected": -92.47774505615234, + "logps/rejected": -289.5357666015625, + "loss": 1.0502, + "margin_dpo/margin_mean": 76.78402709960938, + "margin_dpo/margin_std": 100.21034240722656, + "step": 406 + }, + { + "KL/chosen_KL_mean": -151.61819458007812, + "KL/mean": -181.9169921875, + "KL/rejected_KL_mean": -212.21580505371094, + "KL/std": 101.87554168701172, + "epoch": 0.6152683295540439, + "fcm_dpo/beta": 0.006411860231310129, + "fcm_dpo/delta": 0.011711766943335533, + "fcm_dpo/margin": 60.59760665893555, + "fcm_dpo/q_t": 0.41196078062057495, + "grad_norm": 13.501534461975098, + "learning_rate": 1.9492236680336483e-07, + "logits/chosen": 0.558795690536499, + "logits/rejected": 0.49235716462135315, + "logps/chosen": -241.0187530517578, + "logps/ref_chosen": -89.40056610107422, + "logps/ref_rejected": -99.28775024414062, + "logps/rejected": -311.5035400390625, + "loss": 1.1392, + "margin_dpo/margin_mean": 60.59761047363281, + "margin_dpo/margin_std": 103.35305786132812, + "step": 407 + }, + { + "KL/chosen_KL_mean": -113.6641845703125, + "KL/mean": -156.4038848876953, + "KL/rejected_KL_mean": -199.1436004638672, + "KL/std": 84.85147094726562, + "epoch": 0.6167800453514739, + "fcm_dpo/beta": 0.006298558786511421, + "fcm_dpo/delta": -0.1461481899023056, + "fcm_dpo/margin": 85.47942352294922, + "fcm_dpo/q_t": 0.37603700160980225, + "grad_norm": 10.623373985290527, + "learning_rate": 1.9363341121154895e-07, + "logits/chosen": 0.6735584735870361, + "logits/rejected": 0.5957342386245728, + "logps/chosen": -168.36810302734375, + "logps/ref_chosen": -54.70391845703125, + "logps/ref_rejected": -73.98648834228516, + "logps/rejected": -273.1300964355469, + "loss": 0.9966, + "margin_dpo/margin_mean": 85.47941589355469, + "margin_dpo/margin_std": 89.57831573486328, + "step": 408 + }, + { + "KL/chosen_KL_mean": -140.25930786132812, + "KL/mean": -159.85809326171875, + "KL/rejected_KL_mean": -179.4569091796875, + "KL/std": 73.4033203125, + "epoch": 0.618291761148904, + "fcm_dpo/beta": 0.006353562697768211, + "fcm_dpo/delta": 0.15491655468940735, + "fcm_dpo/margin": 39.19761657714844, + "fcm_dpo/q_t": 0.44314515590667725, + "grad_norm": 12.572002410888672, + "learning_rate": 1.9234603231438994e-07, + "logits/chosen": 0.6764421463012695, + "logits/rejected": 0.6854976415634155, + "logps/chosen": -202.37753295898438, + "logps/ref_chosen": -62.11822509765625, + "logps/ref_rejected": -61.933509826660156, + "logps/rejected": -241.39041137695312, + "loss": 1.2212, + "margin_dpo/margin_mean": 39.19761657714844, + "margin_dpo/margin_std": 83.82356262207031, + "step": 409 + }, + { + "KL/chosen_KL_mean": -131.65499877929688, + "KL/mean": -166.28590393066406, + "KL/rejected_KL_mean": -200.9167938232422, + "KL/std": 82.87063598632812, + "epoch": 0.6198034769463341, + "fcm_dpo/beta": 0.006320708431303501, + "fcm_dpo/delta": -0.041026365011930466, + "fcm_dpo/margin": 69.26181030273438, + "fcm_dpo/q_t": 0.3986797332763672, + "grad_norm": 11.368917465209961, + "learning_rate": 1.9106026612264315e-07, + "logits/chosen": 0.6804258823394775, + "logits/rejected": 0.6558456420898438, + "logps/chosen": -193.45767211914062, + "logps/ref_chosen": -61.80266189575195, + "logps/ref_rejected": -76.60002136230469, + "logps/rejected": -277.5168151855469, + "loss": 1.0623, + "margin_dpo/margin_mean": 69.2618179321289, + "margin_dpo/margin_std": 81.38176727294922, + "step": 410 + }, + { + "KL/chosen_KL_mean": -138.29071044921875, + "KL/mean": -171.49337768554688, + "KL/rejected_KL_mean": -204.696044921875, + "KL/std": 89.67030334472656, + "epoch": 0.6213151927437641, + "fcm_dpo/beta": 0.0063368408009409904, + "fcm_dpo/delta": -0.021709755063056946, + "fcm_dpo/margin": 66.40533447265625, + "fcm_dpo/q_t": 0.4044159948825836, + "grad_norm": 11.313436508178711, + "learning_rate": 1.8977614860195296e-07, + "logits/chosen": 0.7408417463302612, + "logits/rejected": 0.6807792782783508, + "logps/chosen": -192.736083984375, + "logps/ref_chosen": -54.44539260864258, + "logps/ref_rejected": -74.5650863647461, + "logps/rejected": -279.2611083984375, + "loss": 1.0935, + "margin_dpo/margin_mean": 66.40534210205078, + "margin_dpo/margin_std": 95.49729919433594, + "step": 411 + }, + { + "KL/chosen_KL_mean": -141.05941772460938, + "KL/mean": -171.43373107910156, + "KL/rejected_KL_mean": -201.80804443359375, + "KL/std": 77.73248291015625, + "epoch": 0.6228269085411943, + "fcm_dpo/beta": 0.006320852320641279, + "fcm_dpo/delta": 0.016446245834231377, + "fcm_dpo/margin": 60.74862289428711, + "fcm_dpo/q_t": 0.41138458251953125, + "grad_norm": 11.807281494140625, + "learning_rate": 1.8849371567184662e-07, + "logits/chosen": 0.690960168838501, + "logits/rejected": 0.6279210448265076, + "logps/chosen": -196.3074951171875, + "logps/ref_chosen": -55.248085021972656, + "logps/ref_rejected": -68.96623229980469, + "logps/rejected": -270.7742614746094, + "loss": 1.1047, + "margin_dpo/margin_mean": 60.748619079589844, + "margin_dpo/margin_std": 82.77706909179688, + "step": 412 + }, + { + "KL/chosen_KL_mean": -156.57864379882812, + "KL/mean": -185.267822265625, + "KL/rejected_KL_mean": -213.95700073242188, + "KL/std": 90.07394409179688, + "epoch": 0.6243386243386243, + "fcm_dpo/beta": 0.006390274967998266, + "fcm_dpo/delta": 0.03452521562576294, + "fcm_dpo/margin": 57.37835693359375, + "fcm_dpo/q_t": 0.41872984170913696, + "grad_norm": 14.247014045715332, + "learning_rate": 1.872130032047302e-07, + "logits/chosen": 0.531366229057312, + "logits/rejected": 0.49875760078430176, + "logps/chosen": -225.29937744140625, + "logps/ref_chosen": -68.72074890136719, + "logps/ref_rejected": -78.76539611816406, + "logps/rejected": -292.722412109375, + "loss": 1.1705, + "margin_dpo/margin_mean": 57.37835693359375, + "margin_dpo/margin_std": 110.43424224853516, + "step": 413 + }, + { + "KL/chosen_KL_mean": -138.19000244140625, + "KL/mean": -172.3620147705078, + "KL/rejected_KL_mean": -206.5340576171875, + "KL/std": 96.0737075805664, + "epoch": 0.6258503401360545, + "fcm_dpo/beta": 0.006364773027598858, + "fcm_dpo/delta": -0.03656965494155884, + "fcm_dpo/margin": 68.34403991699219, + "fcm_dpo/q_t": 0.3993681073188782, + "grad_norm": 10.928391456604004, + "learning_rate": 1.8593404702488436e-07, + "logits/chosen": 0.7208206653594971, + "logits/rejected": 0.6589357852935791, + "logps/chosen": -192.32821655273438, + "logps/ref_chosen": -54.138214111328125, + "logps/ref_rejected": -74.65741729736328, + "logps/rejected": -281.19146728515625, + "loss": 1.0749, + "margin_dpo/margin_mean": 68.34403991699219, + "margin_dpo/margin_std": 88.88926696777344, + "step": 414 + }, + { + "KL/chosen_KL_mean": -136.01686096191406, + "KL/mean": -164.8328857421875, + "KL/rejected_KL_mean": -193.64889526367188, + "KL/std": 89.09803771972656, + "epoch": 0.6273620559334845, + "fcm_dpo/beta": 0.006370305083692074, + "fcm_dpo/delta": 0.03412775322794914, + "fcm_dpo/margin": 57.63203811645508, + "fcm_dpo/q_t": 0.4166450798511505, + "grad_norm": 11.460617065429688, + "learning_rate": 1.846568829074628e-07, + "logits/chosen": 0.7241143584251404, + "logits/rejected": 0.7067319750785828, + "logps/chosen": -191.9354248046875, + "logps/ref_chosen": -55.91856002807617, + "logps/ref_rejected": -61.747703552246094, + "logps/rejected": -255.3966064453125, + "loss": 1.1407, + "margin_dpo/margin_mean": 57.632041931152344, + "margin_dpo/margin_std": 96.32302856445312, + "step": 415 + }, + { + "KL/chosen_KL_mean": -146.26498413085938, + "KL/mean": -170.3350830078125, + "KL/rejected_KL_mean": -194.40518188476562, + "KL/std": 93.53575134277344, + "epoch": 0.6288737717309146, + "fcm_dpo/beta": 0.006337391212582588, + "fcm_dpo/delta": -0.08305186033248901, + "fcm_dpo/margin": 48.14020919799805, + "fcm_dpo/q_t": 0.43126845359802246, + "grad_norm": 14.00360107421875, + "learning_rate": 1.8338154657749128e-07, + "logits/chosen": 0.6633551716804504, + "logits/rejected": 0.6139250993728638, + "logps/chosen": -200.98806762695312, + "logps/ref_chosen": -54.72308349609375, + "logps/ref_rejected": -69.17388916015625, + "logps/rejected": -263.5790710449219, + "loss": 1.1968, + "margin_dpo/margin_mean": 48.14020919799805, + "margin_dpo/margin_std": 90.29344177246094, + "step": 416 + }, + { + "KL/chosen_KL_mean": -151.7848358154297, + "KL/mean": -183.6634521484375, + "KL/rejected_KL_mean": -215.5420684814453, + "KL/std": 87.87062072753906, + "epoch": 0.6303854875283447, + "fcm_dpo/beta": 0.006316634826362133, + "fcm_dpo/delta": -0.003457318991422653, + "fcm_dpo/margin": 63.75724411010742, + "fcm_dpo/q_t": 0.4064716696739197, + "grad_norm": 12.411998748779297, + "learning_rate": 1.8210807370886849e-07, + "logits/chosen": 0.8052752017974854, + "logits/rejected": 0.7394411563873291, + "logps/chosen": -208.5760955810547, + "logps/ref_chosen": -56.791259765625, + "logps/ref_rejected": -68.7791748046875, + "logps/rejected": -284.32122802734375, + "loss": 1.1314, + "margin_dpo/margin_mean": 63.75724792480469, + "margin_dpo/margin_std": 106.73062133789062, + "step": 417 + }, + { + "KL/chosen_KL_mean": -160.93739318847656, + "KL/mean": -187.81124877929688, + "KL/rejected_KL_mean": -214.68505859375, + "KL/std": 98.37525939941406, + "epoch": 0.6318972033257747, + "fcm_dpo/beta": 0.006278286688029766, + "fcm_dpo/delta": 0.06338182091712952, + "fcm_dpo/margin": 53.74766540527344, + "fcm_dpo/q_t": 0.42482703924179077, + "grad_norm": 13.669818878173828, + "learning_rate": 1.8083649992336825e-07, + "logits/chosen": 0.7154676914215088, + "logits/rejected": 0.7190610766410828, + "logps/chosen": -230.04537963867188, + "logps/ref_chosen": -69.10798645019531, + "logps/ref_rejected": -75.09132385253906, + "logps/rejected": -289.7763977050781, + "loss": 1.1789, + "margin_dpo/margin_mean": 53.74766540527344, + "margin_dpo/margin_std": 99.94847106933594, + "step": 418 + }, + { + "KL/chosen_KL_mean": -130.17111206054688, + "KL/mean": -168.118408203125, + "KL/rejected_KL_mean": -206.0657196044922, + "KL/std": 90.11360168457031, + "epoch": 0.6334089191232048, + "fcm_dpo/beta": 0.006277072709053755, + "fcm_dpo/delta": -0.08035100251436234, + "fcm_dpo/margin": 75.89460754394531, + "fcm_dpo/q_t": 0.39397668838500977, + "grad_norm": 11.692562103271484, + "learning_rate": 1.7956686078964255e-07, + "logits/chosen": 0.6163479089736938, + "logits/rejected": 0.5674476027488708, + "logps/chosen": -188.34288024902344, + "logps/ref_chosen": -58.1717643737793, + "logps/ref_rejected": -71.67066955566406, + "logps/rejected": -277.73638916015625, + "loss": 1.0612, + "margin_dpo/margin_mean": 75.89460754394531, + "margin_dpo/margin_std": 103.62324523925781, + "step": 419 + }, + { + "KL/chosen_KL_mean": -160.61224365234375, + "KL/mean": -180.34158325195312, + "KL/rejected_KL_mean": -200.0709228515625, + "KL/std": 93.16465759277344, + "epoch": 0.6349206349206349, + "fcm_dpo/beta": 0.00638390239328146, + "fcm_dpo/delta": 0.15220247209072113, + "fcm_dpo/margin": 39.45867156982422, + "fcm_dpo/q_t": 0.4451296329498291, + "grad_norm": 12.861660957336426, + "learning_rate": 1.782991918222275e-07, + "logits/chosen": 0.7003469467163086, + "logits/rejected": 0.655767560005188, + "logps/chosen": -217.66575622558594, + "logps/ref_chosen": -57.05351257324219, + "logps/ref_rejected": -62.670982360839844, + "logps/rejected": -262.7419128417969, + "loss": 1.2613, + "margin_dpo/margin_mean": 39.45867156982422, + "margin_dpo/margin_std": 106.66671752929688, + "step": 420 + }, + { + "KL/chosen_KL_mean": -147.0243377685547, + "KL/mean": -176.02650451660156, + "KL/rejected_KL_mean": -205.02865600585938, + "KL/std": 92.48200988769531, + "epoch": 0.636432350718065, + "fcm_dpo/beta": 0.006444813683629036, + "fcm_dpo/delta": 0.026773784309625626, + "fcm_dpo/margin": 58.004310607910156, + "fcm_dpo/q_t": 0.41775447130203247, + "grad_norm": 14.737412452697754, + "learning_rate": 1.7703352848054887e-07, + "logits/chosen": 0.6828116178512573, + "logits/rejected": 0.6210600137710571, + "logps/chosen": -204.34759521484375, + "logps/ref_chosen": -57.32324981689453, + "logps/ref_rejected": -75.33782958984375, + "logps/rejected": -280.3664855957031, + "loss": 1.1888, + "margin_dpo/margin_mean": 58.004310607910156, + "margin_dpo/margin_std": 120.49269104003906, + "step": 421 + }, + { + "KL/chosen_KL_mean": -130.69232177734375, + "KL/mean": -168.12051391601562, + "KL/rejected_KL_mean": -205.54869079589844, + "KL/std": 91.55455780029297, + "epoch": 0.6379440665154951, + "fcm_dpo/beta": 0.0064270165748894215, + "fcm_dpo/delta": -0.08514492213726044, + "fcm_dpo/margin": 74.85635375976562, + "fcm_dpo/q_t": 0.39063405990600586, + "grad_norm": 13.418085098266602, + "learning_rate": 1.7576990616793137e-07, + "logits/chosen": 0.6663640737533569, + "logits/rejected": 0.6602794528007507, + "logps/chosen": -197.74990844726562, + "logps/ref_chosen": -67.05757141113281, + "logps/ref_rejected": -72.12803649902344, + "logps/rejected": -277.6767272949219, + "loss": 1.0445, + "margin_dpo/margin_mean": 74.85636138916016, + "margin_dpo/margin_std": 92.13258361816406, + "step": 422 + }, + { + "KL/chosen_KL_mean": -132.84735107421875, + "KL/mean": -173.277099609375, + "KL/rejected_KL_mean": -213.7068328857422, + "KL/std": 93.94609832763672, + "epoch": 0.6394557823129252, + "fcm_dpo/beta": 0.006261817179620266, + "fcm_dpo/delta": -0.1122402548789978, + "fcm_dpo/margin": 80.85948944091797, + "fcm_dpo/q_t": 0.3863791525363922, + "grad_norm": 11.530719757080078, + "learning_rate": 1.745083602306071e-07, + "logits/chosen": 0.7256453633308411, + "logits/rejected": 0.657960057258606, + "logps/chosen": -186.90902709960938, + "logps/ref_chosen": -54.06167221069336, + "logps/ref_rejected": -76.64092254638672, + "logps/rejected": -290.3477783203125, + "loss": 1.0317, + "margin_dpo/margin_mean": 80.85948181152344, + "margin_dpo/margin_std": 100.18473815917969, + "step": 423 + }, + { + "KL/chosen_KL_mean": -141.92503356933594, + "KL/mean": -180.5269012451172, + "KL/rejected_KL_mean": -219.12875366210938, + "KL/std": 88.34001159667969, + "epoch": 0.6409674981103552, + "fcm_dpo/beta": 0.00612823199480772, + "fcm_dpo/delta": -0.07791505008935928, + "fcm_dpo/margin": 77.20375061035156, + "fcm_dpo/q_t": 0.3932613730430603, + "grad_norm": 16.036226272583008, + "learning_rate": 1.7324892595672804e-07, + "logits/chosen": 0.6484436392784119, + "logits/rejected": 0.6037266254425049, + "logps/chosen": -195.53390502929688, + "logps/ref_chosen": -53.60887145996094, + "logps/ref_rejected": -79.2139892578125, + "logps/rejected": -298.3427734375, + "loss": 1.0579, + "margin_dpo/margin_mean": 77.20375061035156, + "margin_dpo/margin_std": 101.07964324951172, + "step": 424 + }, + { + "KL/chosen_KL_mean": -141.54779052734375, + "KL/mean": -171.28265380859375, + "KL/rejected_KL_mean": -201.0175323486328, + "KL/std": 85.41214752197266, + "epoch": 0.6424792139077853, + "fcm_dpo/beta": 0.006170147098600864, + "fcm_dpo/delta": 0.03432762995362282, + "fcm_dpo/margin": 59.469749450683594, + "fcm_dpo/q_t": 0.416858047246933, + "grad_norm": 12.468502044677734, + "learning_rate": 1.7199163857537824e-07, + "logits/chosen": 0.7201306819915771, + "logits/rejected": 0.6925822496414185, + "logps/chosen": -199.96246337890625, + "logps/ref_chosen": -58.41468048095703, + "logps/ref_rejected": -66.59054565429688, + "logps/rejected": -267.60809326171875, + "loss": 1.1375, + "margin_dpo/margin_mean": 59.469749450683594, + "margin_dpo/margin_std": 96.95804595947266, + "step": 425 + }, + { + "KL/chosen_KL_mean": -170.64788818359375, + "KL/mean": -188.68301391601562, + "KL/rejected_KL_mean": -206.7181396484375, + "KL/std": 91.15631866455078, + "epoch": 0.6439909297052154, + "fcm_dpo/beta": 0.006349620409309864, + "fcm_dpo/delta": 0.17486482858657837, + "fcm_dpo/margin": 36.070255279541016, + "fcm_dpo/q_t": 0.4480932652950287, + "grad_norm": 16.96051788330078, + "learning_rate": 1.7073653325558828e-07, + "logits/chosen": 0.6682687997817993, + "logits/rejected": 0.6715260744094849, + "logps/chosen": -242.35610961914062, + "logps/ref_chosen": -71.70822143554688, + "logps/ref_rejected": -73.57725524902344, + "logps/rejected": -280.29541015625, + "loss": 1.2922, + "margin_dpo/margin_mean": 36.07025146484375, + "margin_dpo/margin_std": 114.49520874023438, + "step": 426 + }, + { + "KL/chosen_KL_mean": -161.13392639160156, + "KL/mean": -192.15673828125, + "KL/rejected_KL_mean": -223.17955017089844, + "KL/std": 99.90629577636719, + "epoch": 0.6455026455026455, + "fcm_dpo/beta": 0.0063988445326685905, + "fcm_dpo/delta": 0.0030871136114001274, + "fcm_dpo/margin": 62.045616149902344, + "fcm_dpo/q_t": 0.4128350615501404, + "grad_norm": 13.89861011505127, + "learning_rate": 1.6948364510535218e-07, + "logits/chosen": 0.7114277482032776, + "logits/rejected": 0.6444870233535767, + "logps/chosen": -219.77670288085938, + "logps/ref_chosen": -58.64276885986328, + "logps/ref_rejected": -86.25437927246094, + "logps/rejected": -309.4339294433594, + "loss": 1.1435, + "margin_dpo/margin_mean": 62.045623779296875, + "margin_dpo/margin_std": 111.41607666015625, + "step": 427 + }, + { + "KL/chosen_KL_mean": -155.09945678710938, + "KL/mean": -188.89183044433594, + "KL/rejected_KL_mean": -222.6842041015625, + "KL/std": 100.60108947753906, + "epoch": 0.6470143613000756, + "fcm_dpo/beta": 0.006401218008249998, + "fcm_dpo/delta": -0.03433738648891449, + "fcm_dpo/margin": 67.5847396850586, + "fcm_dpo/q_t": 0.4036714434623718, + "grad_norm": 12.783305168151855, + "learning_rate": 1.6823300917064458e-07, + "logits/chosen": 0.6407492160797119, + "logits/rejected": 0.5998574495315552, + "logps/chosen": -221.6955108642578, + "logps/ref_chosen": -66.5960464477539, + "logps/ref_rejected": -82.3941650390625, + "logps/rejected": -305.078369140625, + "loss": 1.1121, + "margin_dpo/margin_mean": 67.5847396850586, + "margin_dpo/margin_std": 107.8612060546875, + "step": 428 + }, + { + "KL/chosen_KL_mean": -157.85076904296875, + "KL/mean": -183.78836059570312, + "KL/rejected_KL_mean": -209.72593688964844, + "KL/std": 85.93083190917969, + "epoch": 0.6485260770975056, + "fcm_dpo/beta": 0.00645467359572649, + "fcm_dpo/delta": 0.06671077758073807, + "fcm_dpo/margin": 51.87518310546875, + "fcm_dpo/q_t": 0.42263728380203247, + "grad_norm": 14.143959999084473, + "learning_rate": 1.669846604344412e-07, + "logits/chosen": 0.6520216464996338, + "logits/rejected": 0.6711635589599609, + "logps/chosen": -214.8604736328125, + "logps/ref_chosen": -57.00970458984375, + "logps/ref_rejected": -59.86549377441406, + "logps/rejected": -269.5914306640625, + "loss": 1.1735, + "margin_dpo/margin_mean": 51.87518310546875, + "margin_dpo/margin_std": 95.87495422363281, + "step": 429 + }, + { + "KL/chosen_KL_mean": -140.60528564453125, + "KL/mean": -183.13192749023438, + "KL/rejected_KL_mean": -225.65859985351562, + "KL/std": 87.80361938476562, + "epoch": 0.6500377928949358, + "fcm_dpo/beta": 0.006307076197117567, + "fcm_dpo/delta": -0.14426840841770172, + "fcm_dpo/margin": 85.05331420898438, + "fcm_dpo/q_t": 0.3759458661079407, + "grad_norm": 13.13545036315918, + "learning_rate": 1.6573863381573954e-07, + "logits/chosen": 0.5855617523193359, + "logits/rejected": 0.5838299989700317, + "logps/chosen": -200.16848754882812, + "logps/ref_chosen": -59.563194274902344, + "logps/ref_rejected": -70.52289581298828, + "logps/rejected": -296.1814880371094, + "loss": 1.015, + "margin_dpo/margin_mean": 85.05331420898438, + "margin_dpo/margin_std": 100.49716186523438, + "step": 430 + }, + { + "KL/chosen_KL_mean": -140.24456787109375, + "KL/mean": -170.9658203125, + "KL/rejected_KL_mean": -201.68710327148438, + "KL/std": 89.18963623046875, + "epoch": 0.6515495086923658, + "fcm_dpo/beta": 0.006269997917115688, + "fcm_dpo/delta": 0.015329739078879356, + "fcm_dpo/margin": 61.44251251220703, + "fcm_dpo/q_t": 0.41269272565841675, + "grad_norm": 12.90273380279541, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": 0.6572583913803101, + "logits/rejected": 0.6016641855239868, + "logps/chosen": -190.44488525390625, + "logps/ref_chosen": -50.20032501220703, + "logps/ref_rejected": -77.81680297851562, + "logps/rejected": -279.50390625, + "loss": 1.1252, + "margin_dpo/margin_mean": 61.4425163269043, + "margin_dpo/margin_std": 97.04786682128906, + "step": 431 + }, + { + "KL/chosen_KL_mean": -147.359619140625, + "KL/mean": -179.90625, + "KL/rejected_KL_mean": -212.45291137695312, + "KL/std": 88.27741241455078, + "epoch": 0.6530612244897959, + "fcm_dpo/beta": 0.006269288249313831, + "fcm_dpo/delta": -0.008461249992251396, + "fcm_dpo/margin": 65.09325408935547, + "fcm_dpo/q_t": 0.40682950615882874, + "grad_norm": 13.348393440246582, + "learning_rate": 1.632536862810844e-07, + "logits/chosen": 0.6946985721588135, + "logits/rejected": 0.6414108276367188, + "logps/chosen": -209.02236938476562, + "logps/ref_chosen": -61.662757873535156, + "logps/ref_rejected": -83.94496154785156, + "logps/rejected": -296.3978576660156, + "loss": 1.1148, + "margin_dpo/margin_mean": 65.09325408935547, + "margin_dpo/margin_std": 101.81596374511719, + "step": 432 + }, + { + "KL/chosen_KL_mean": -145.7971954345703, + "KL/mean": -183.37362670898438, + "KL/rejected_KL_mean": -220.95008850097656, + "KL/std": 83.93692016601562, + "epoch": 0.654572940287226, + "fcm_dpo/beta": 0.006223015021532774, + "fcm_dpo/delta": -0.07095064222812653, + "fcm_dpo/margin": 75.15287017822266, + "fcm_dpo/q_t": 0.3921838700771332, + "grad_norm": 12.827315330505371, + "learning_rate": 1.6201483487445515e-07, + "logits/chosen": 0.7911877632141113, + "logits/rejected": 0.7912458181381226, + "logps/chosen": -209.5263671875, + "logps/ref_chosen": -63.72917938232422, + "logps/ref_rejected": -65.8391342163086, + "logps/rejected": -286.7892150878906, + "loss": 1.0583, + "margin_dpo/margin_mean": 75.15287017822266, + "margin_dpo/margin_std": 96.3397216796875, + "step": 433 + }, + { + "KL/chosen_KL_mean": -124.45191192626953, + "KL/mean": -166.2886962890625, + "KL/rejected_KL_mean": -208.12547302246094, + "KL/std": 93.03741455078125, + "epoch": 0.656084656084656, + "fcm_dpo/beta": 0.00603717565536499, + "fcm_dpo/delta": -0.1126817911863327, + "fcm_dpo/margin": 83.67355346679688, + "fcm_dpo/q_t": 0.3845771253108978, + "grad_norm": 11.618966102600098, + "learning_rate": 1.6077844460203204e-07, + "logits/chosen": 0.8206672072410583, + "logits/rejected": 0.7543665170669556, + "logps/chosen": -172.42523193359375, + "logps/ref_chosen": -47.97331619262695, + "logps/ref_rejected": -72.51132202148438, + "logps/rejected": -280.63677978515625, + "loss": 1.0545, + "margin_dpo/margin_mean": 83.67355346679688, + "margin_dpo/margin_std": 112.55625915527344, + "step": 434 + }, + { + "KL/chosen_KL_mean": -150.77972412109375, + "KL/mean": -181.7442169189453, + "KL/rejected_KL_mean": -212.7086944580078, + "KL/std": 88.39662170410156, + "epoch": 0.6575963718820862, + "fcm_dpo/beta": 0.0060953618958592415, + "fcm_dpo/delta": 0.02286442741751671, + "fcm_dpo/margin": 61.92898941040039, + "fcm_dpo/q_t": 0.41460686922073364, + "grad_norm": 13.56441879272461, + "learning_rate": 1.5954455004830878e-07, + "logits/chosen": 0.7735519409179688, + "logits/rejected": 0.7344120144844055, + "logps/chosen": -207.8399658203125, + "logps/ref_chosen": -57.06024932861328, + "logps/ref_rejected": -71.69146728515625, + "logps/rejected": -284.400146484375, + "loss": 1.1378, + "margin_dpo/margin_mean": 61.92898941040039, + "margin_dpo/margin_std": 102.48014068603516, + "step": 435 + }, + { + "KL/chosen_KL_mean": -149.90362548828125, + "KL/mean": -175.8869171142578, + "KL/rejected_KL_mean": -201.87020874023438, + "KL/std": 87.14476776123047, + "epoch": 0.6591080876795162, + "fcm_dpo/beta": 0.006144003942608833, + "fcm_dpo/delta": 0.08344481885433197, + "fcm_dpo/margin": 51.96659469604492, + "fcm_dpo/q_t": 0.4281252324581146, + "grad_norm": 14.366814613342285, + "learning_rate": 1.5831318572796847e-07, + "logits/chosen": 0.6878930926322937, + "logits/rejected": 0.6309799551963806, + "logps/chosen": -206.06167602539062, + "logps/ref_chosen": -56.158050537109375, + "logps/ref_rejected": -67.63787841796875, + "logps/rejected": -269.5080871582031, + "loss": 1.1935, + "margin_dpo/margin_mean": 51.96659469604492, + "margin_dpo/margin_std": 106.5383529663086, + "step": 436 + }, + { + "KL/chosen_KL_mean": -156.62506103515625, + "KL/mean": -187.46817016601562, + "KL/rejected_KL_mean": -218.31126403808594, + "KL/std": 94.74977111816406, + "epoch": 0.6606198034769464, + "fcm_dpo/beta": 0.006142673082649708, + "fcm_dpo/delta": 0.020575210452079773, + "fcm_dpo/margin": 61.68621063232422, + "fcm_dpo/q_t": 0.41576099395751953, + "grad_norm": 15.018247604370117, + "learning_rate": 1.5708438608491815e-07, + "logits/chosen": 0.6854249238967896, + "logits/rejected": 0.5601704716682434, + "logps/chosen": -213.61083984375, + "logps/ref_chosen": -56.98578643798828, + "logps/ref_rejected": -85.61524963378906, + "logps/rejected": -303.926513671875, + "loss": 1.1762, + "margin_dpo/margin_mean": 61.68621063232422, + "margin_dpo/margin_std": 119.7267074584961, + "step": 437 + }, + { + "KL/chosen_KL_mean": -134.8008270263672, + "KL/mean": -176.47152709960938, + "KL/rejected_KL_mean": -218.1422119140625, + "KL/std": 98.8070297241211, + "epoch": 0.6621315192743764, + "fcm_dpo/beta": 0.006119017489254475, + "fcm_dpo/delta": -0.11572910845279694, + "fcm_dpo/margin": 83.3414077758789, + "fcm_dpo/q_t": 0.3877474069595337, + "grad_norm": 12.523723602294922, + "learning_rate": 1.558581854913253e-07, + "logits/chosen": 0.7682843804359436, + "logits/rejected": 0.7010859847068787, + "logps/chosen": -176.07861328125, + "logps/ref_chosen": -41.27777862548828, + "logps/ref_rejected": -65.33840942382812, + "logps/rejected": -283.48065185546875, + "loss": 1.0372, + "margin_dpo/margin_mean": 83.3414077758789, + "margin_dpo/margin_std": 108.39314270019531, + "step": 438 + }, + { + "KL/chosen_KL_mean": -153.8817138671875, + "KL/mean": -191.04409790039062, + "KL/rejected_KL_mean": -228.2064666748047, + "KL/std": 100.76587677001953, + "epoch": 0.6636432350718064, + "fcm_dpo/beta": 0.006057861726731062, + "fcm_dpo/delta": -0.05337338522076607, + "fcm_dpo/margin": 74.32474517822266, + "fcm_dpo/q_t": 0.3977718949317932, + "grad_norm": 12.527008056640625, + "learning_rate": 1.5463461824665658e-07, + "logits/chosen": 0.6251957416534424, + "logits/rejected": 0.5903706550598145, + "logps/chosen": -235.2993621826172, + "logps/ref_chosen": -81.41764831542969, + "logps/ref_rejected": -94.72309875488281, + "logps/rejected": -322.9295654296875, + "loss": 1.0785, + "margin_dpo/margin_mean": 74.32474517822266, + "margin_dpo/margin_std": 101.76370239257812, + "step": 439 + }, + { + "KL/chosen_KL_mean": -132.0679931640625, + "KL/mean": -169.42111206054688, + "KL/rejected_KL_mean": -206.7742462158203, + "KL/std": 89.31532287597656, + "epoch": 0.6651549508692366, + "fcm_dpo/beta": 0.005974017083644867, + "fcm_dpo/delta": -0.04854672774672508, + "fcm_dpo/margin": 74.70626831054688, + "fcm_dpo/q_t": 0.39923959970474243, + "grad_norm": 20.631868362426758, + "learning_rate": 1.534137185767178e-07, + "logits/chosen": 0.6927276849746704, + "logits/rejected": 0.5932461619377136, + "logps/chosen": -174.60617065429688, + "logps/ref_chosen": -42.538185119628906, + "logps/ref_rejected": -69.78813934326172, + "logps/rejected": -276.5623779296875, + "loss": 1.0803, + "margin_dpo/margin_mean": 74.70626831054688, + "margin_dpo/margin_std": 103.83875274658203, + "step": 440 + }, + { + "KL/chosen_KL_mean": -133.71435546875, + "KL/mean": -175.30303955078125, + "KL/rejected_KL_mean": -216.8917236328125, + "KL/std": 96.92250061035156, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.005834102630615234, + "fcm_dpo/delta": -0.09035193175077438, + "fcm_dpo/margin": 83.17737579345703, + "fcm_dpo/q_t": 0.3867751955986023, + "grad_norm": 13.054012298583984, + "learning_rate": 1.521955206326976e-07, + "logits/chosen": 0.6799192428588867, + "logits/rejected": 0.5854415893554688, + "logps/chosen": -191.30758666992188, + "logps/ref_chosen": -57.593223571777344, + "logps/ref_rejected": -84.82878875732422, + "logps/rejected": -301.72052001953125, + "loss": 1.0199, + "margin_dpo/margin_mean": 83.1773681640625, + "margin_dpo/margin_std": 86.29299926757812, + "step": 441 + }, + { + "KL/chosen_KL_mean": -161.66006469726562, + "KL/mean": -201.89044189453125, + "KL/rejected_KL_mean": -242.12081909179688, + "KL/std": 97.12081146240234, + "epoch": 0.6681783824640968, + "fcm_dpo/beta": 0.005757060833275318, + "fcm_dpo/delta": -0.06665948033332825, + "fcm_dpo/margin": 80.46076202392578, + "fcm_dpo/q_t": 0.3939310908317566, + "grad_norm": 13.893742561340332, + "learning_rate": 1.5098005849021078e-07, + "logits/chosen": 0.7006994485855103, + "logits/rejected": 0.6529697179794312, + "logps/chosen": -229.12127685546875, + "logps/ref_chosen": -67.46121978759766, + "logps/ref_rejected": -89.0693588256836, + "logps/rejected": -331.190185546875, + "loss": 1.0552, + "margin_dpo/margin_mean": 80.46076965332031, + "margin_dpo/margin_std": 100.84950256347656, + "step": 442 + }, + { + "KL/chosen_KL_mean": -133.28994750976562, + "KL/mean": -182.92572021484375, + "KL/rejected_KL_mean": -232.5614776611328, + "KL/std": 102.47624206542969, + "epoch": 0.6696900982615268, + "fcm_dpo/beta": 0.005609867163002491, + "fcm_dpo/delta": -0.16628825664520264, + "fcm_dpo/margin": 99.27154541015625, + "fcm_dpo/q_t": 0.3736429810523987, + "grad_norm": 14.203608512878418, + "learning_rate": 1.4976736614834662e-07, + "logits/chosen": 0.7221025228500366, + "logits/rejected": 0.6557480096817017, + "logps/chosen": -188.0860595703125, + "logps/ref_chosen": -54.79610061645508, + "logps/ref_rejected": -77.80781555175781, + "logps/rejected": -310.3692932128906, + "loss": 0.9957, + "margin_dpo/margin_mean": 99.27154541015625, + "margin_dpo/margin_std": 114.22992706298828, + "step": 443 + }, + { + "KL/chosen_KL_mean": -164.04290771484375, + "KL/mean": -186.19549560546875, + "KL/rejected_KL_mean": -208.34808349609375, + "KL/std": 101.94633483886719, + "epoch": 0.671201814058957, + "fcm_dpo/beta": 0.005667264573276043, + "fcm_dpo/delta": 0.15300863981246948, + "fcm_dpo/margin": 44.3051643371582, + "fcm_dpo/q_t": 0.4450858235359192, + "grad_norm": 16.502891540527344, + "learning_rate": 1.4855747752871654e-07, + "logits/chosen": 0.6830138564109802, + "logits/rejected": 0.5934052467346191, + "logps/chosen": -222.79196166992188, + "logps/ref_chosen": -58.749061584472656, + "logps/ref_rejected": -86.87396240234375, + "logps/rejected": -295.2220458984375, + "loss": 1.2501, + "margin_dpo/margin_mean": 44.30516052246094, + "margin_dpo/margin_std": 112.28422546386719, + "step": 444 + }, + { + "KL/chosen_KL_mean": -155.0587921142578, + "KL/mean": -196.34661865234375, + "KL/rejected_KL_mean": -237.63446044921875, + "KL/std": 93.90274047851562, + "epoch": 0.672713529856387, + "fcm_dpo/beta": 0.0056798215955495834, + "fcm_dpo/delta": -0.0723859965801239, + "fcm_dpo/margin": 82.57566833496094, + "fcm_dpo/q_t": 0.3913373351097107, + "grad_norm": 12.268744468688965, + "learning_rate": 1.473504264745062e-07, + "logits/chosen": 0.6730467081069946, + "logits/rejected": 0.6653146743774414, + "logps/chosen": -215.97622680664062, + "logps/ref_chosen": -60.91743850708008, + "logps/ref_rejected": -71.5637435913086, + "logps/rejected": -309.19818115234375, + "loss": 1.0459, + "margin_dpo/margin_mean": 82.57566833496094, + "margin_dpo/margin_std": 99.127197265625, + "step": 445 + }, + { + "KL/chosen_KL_mean": -141.07745361328125, + "KL/mean": -183.24526977539062, + "KL/rejected_KL_mean": -225.4130859375, + "KL/std": 95.9248275756836, + "epoch": 0.674225245653817, + "fcm_dpo/beta": 0.005640149116516113, + "fcm_dpo/delta": -0.08037254214286804, + "fcm_dpo/margin": 84.33563232421875, + "fcm_dpo/q_t": 0.3905682861804962, + "grad_norm": 11.658855438232422, + "learning_rate": 1.461462467495284e-07, + "logits/chosen": 0.7361161708831787, + "logits/rejected": 0.6562565565109253, + "logps/chosen": -189.876708984375, + "logps/ref_chosen": -48.79924774169922, + "logps/ref_rejected": -71.8719482421875, + "logps/rejected": -297.2850341796875, + "loss": 1.0373, + "margin_dpo/margin_mean": 84.33563232421875, + "margin_dpo/margin_std": 91.992919921875, + "step": 446 + }, + { + "KL/chosen_KL_mean": -137.0552978515625, + "KL/mean": -186.58033752441406, + "KL/rejected_KL_mean": -236.10537719726562, + "KL/std": 95.55780029296875, + "epoch": 0.6757369614512472, + "fcm_dpo/beta": 0.005425992421805859, + "fcm_dpo/delta": -0.14611226320266724, + "fcm_dpo/margin": 99.0500717163086, + "fcm_dpo/q_t": 0.37507176399230957, + "grad_norm": 12.766183853149414, + "learning_rate": 1.4494497203727843e-07, + "logits/chosen": 0.6704369783401489, + "logits/rejected": 0.565066397190094, + "logps/chosen": -190.73802185058594, + "logps/ref_chosen": -53.682716369628906, + "logps/ref_rejected": -88.17315673828125, + "logps/rejected": -324.2785339355469, + "loss": 1.0021, + "margin_dpo/margin_mean": 99.05007934570312, + "margin_dpo/margin_std": 108.26829528808594, + "step": 447 + }, + { + "KL/chosen_KL_mean": -149.04644775390625, + "KL/mean": -187.95932006835938, + "KL/rejected_KL_mean": -226.8721923828125, + "KL/std": 98.18172454833984, + "epoch": 0.6772486772486772, + "fcm_dpo/beta": 0.005390051286667585, + "fcm_dpo/delta": -0.020336728543043137, + "fcm_dpo/margin": 77.82575988769531, + "fcm_dpo/q_t": 0.40321606397628784, + "grad_norm": 9.62354564666748, + "learning_rate": 1.4374663593999256e-07, + "logits/chosen": 0.7249032258987427, + "logits/rejected": 0.6728522777557373, + "logps/chosen": -202.79769897460938, + "logps/ref_chosen": -53.75125503540039, + "logps/ref_rejected": -77.17623901367188, + "logps/rejected": -304.0484313964844, + "loss": 1.085, + "margin_dpo/margin_mean": 77.82575988769531, + "margin_dpo/margin_std": 105.24398803710938, + "step": 448 + }, + { + "KL/chosen_KL_mean": -178.3193359375, + "KL/mean": -197.23353576660156, + "KL/rejected_KL_mean": -216.14773559570312, + "KL/std": 100.89761352539062, + "epoch": 0.6787603930461074, + "fcm_dpo/beta": 0.005525457672774792, + "fcm_dpo/delta": 0.19548821449279785, + "fcm_dpo/margin": 37.82838821411133, + "fcm_dpo/q_t": 0.452498197555542, + "grad_norm": 18.4531307220459, + "learning_rate": 1.4255127197770707e-07, + "logits/chosen": 0.5460859537124634, + "logits/rejected": 0.545451283454895, + "logps/chosen": -254.14671325683594, + "logps/ref_chosen": -75.82737731933594, + "logps/ref_rejected": -82.20687866210938, + "logps/rejected": -298.3546142578125, + "loss": 1.2628, + "margin_dpo/margin_mean": 37.82838821411133, + "margin_dpo/margin_std": 99.50639343261719, + "step": 449 + }, + { + "KL/chosen_KL_mean": -146.93153381347656, + "KL/mean": -179.05615234375, + "KL/rejected_KL_mean": -211.1807861328125, + "KL/std": 101.90434265136719, + "epoch": 0.6802721088435374, + "fcm_dpo/beta": 0.005613807588815689, + "fcm_dpo/delta": 0.04069120064377785, + "fcm_dpo/margin": 64.24925231933594, + "fcm_dpo/q_t": 0.4215589761734009, + "grad_norm": 12.490607261657715, + "learning_rate": 1.4135891358732205e-07, + "logits/chosen": 0.8319960832595825, + "logits/rejected": 0.7168232798576355, + "logps/chosen": -194.0472412109375, + "logps/ref_chosen": -47.11572265625, + "logps/ref_rejected": -78.7546615600586, + "logps/rejected": -289.9354553222656, + "loss": 1.16, + "margin_dpo/margin_mean": 64.2492446899414, + "margin_dpo/margin_std": 117.5666275024414, + "step": 450 + }, + { + "KL/chosen_KL_mean": -148.942138671875, + "KL/mean": -177.62315368652344, + "KL/rejected_KL_mean": -206.30418395996094, + "KL/std": 96.27726745605469, + "epoch": 0.6817838246409675, + "fcm_dpo/beta": 0.005715922452509403, + "fcm_dpo/delta": 0.07441489398479462, + "fcm_dpo/margin": 57.362030029296875, + "fcm_dpo/q_t": 0.42512810230255127, + "grad_norm": 13.263283729553223, + "learning_rate": 1.4016959412166437e-07, + "logits/chosen": 0.6437816619873047, + "logits/rejected": 0.5958889722824097, + "logps/chosen": -212.29258728027344, + "logps/ref_chosen": -63.350440979003906, + "logps/ref_rejected": -76.28530883789062, + "logps/rejected": -282.5894775390625, + "loss": 1.1622, + "margin_dpo/margin_mean": 57.362030029296875, + "margin_dpo/margin_std": 100.08808898925781, + "step": 451 + }, + { + "KL/chosen_KL_mean": -147.36083984375, + "KL/mean": -180.61264038085938, + "KL/rejected_KL_mean": -213.86444091796875, + "KL/std": 92.54578399658203, + "epoch": 0.6832955404383976, + "fcm_dpo/beta": 0.005742376670241356, + "fcm_dpo/delta": 0.018834762275218964, + "fcm_dpo/margin": 66.50358581542969, + "fcm_dpo/q_t": 0.4136967360973358, + "grad_norm": 14.197269439697266, + "learning_rate": 1.3898334684855645e-07, + "logits/chosen": 0.6498645544052124, + "logits/rejected": 0.5683034658432007, + "logps/chosen": -202.94668579101562, + "logps/ref_chosen": -55.58583450317383, + "logps/ref_rejected": -77.68738555908203, + "logps/rejected": -291.55181884765625, + "loss": 1.1394, + "margin_dpo/margin_mean": 66.50358581542969, + "margin_dpo/margin_std": 112.99519348144531, + "step": 452 + }, + { + "KL/chosen_KL_mean": -147.13145446777344, + "KL/mean": -179.03643798828125, + "KL/rejected_KL_mean": -210.94140625, + "KL/std": 96.53543090820312, + "epoch": 0.6848072562358276, + "fcm_dpo/beta": 0.005770553834736347, + "fcm_dpo/delta": 0.033000145107507706, + "fcm_dpo/margin": 63.80995178222656, + "fcm_dpo/q_t": 0.41716498136520386, + "grad_norm": 14.38304615020752, + "learning_rate": 1.3780020494988445e-07, + "logits/chosen": 0.637764573097229, + "logits/rejected": 0.6154556274414062, + "logps/chosen": -208.90966796875, + "logps/ref_chosen": -61.778202056884766, + "logps/ref_rejected": -71.51403045654297, + "logps/rejected": -282.4554443359375, + "loss": 1.1463, + "margin_dpo/margin_mean": 63.80995178222656, + "margin_dpo/margin_std": 109.52471160888672, + "step": 453 + }, + { + "KL/chosen_KL_mean": -136.54115295410156, + "KL/mean": -172.95950317382812, + "KL/rejected_KL_mean": -209.37783813476562, + "KL/std": 96.98989868164062, + "epoch": 0.6863189720332578, + "fcm_dpo/beta": 0.005766098387539387, + "fcm_dpo/delta": -0.020890843123197556, + "fcm_dpo/margin": 72.83668518066406, + "fcm_dpo/q_t": 0.4038616418838501, + "grad_norm": 12.730914115905762, + "learning_rate": 1.366202015206706e-07, + "logits/chosen": 0.6938184499740601, + "logits/rejected": 0.6552758812904358, + "logps/chosen": -188.13632202148438, + "logps/ref_chosen": -51.59515380859375, + "logps/ref_rejected": -63.96732711791992, + "logps/rejected": -273.34515380859375, + "loss": 1.1007, + "margin_dpo/margin_mean": 72.836669921875, + "margin_dpo/margin_std": 109.37074279785156, + "step": 454 + }, + { + "KL/chosen_KL_mean": -156.71600341796875, + "KL/mean": -192.44114685058594, + "KL/rejected_KL_mean": -228.16629028320312, + "KL/std": 96.54014587402344, + "epoch": 0.6878306878306878, + "fcm_dpo/beta": 0.0057487934827804565, + "fcm_dpo/delta": -0.011250181123614311, + "fcm_dpo/margin": 71.4502944946289, + "fcm_dpo/q_t": 0.4079058766365051, + "grad_norm": 13.477120399475098, + "learning_rate": 1.354433695681474e-07, + "logits/chosen": 0.5650009512901306, + "logits/rejected": 0.5374402403831482, + "logps/chosen": -227.36770629882812, + "logps/ref_chosen": -70.65170288085938, + "logps/ref_rejected": -77.44276428222656, + "logps/rejected": -305.60906982421875, + "loss": 1.0988, + "margin_dpo/margin_mean": 71.45030212402344, + "margin_dpo/margin_std": 103.77528381347656, + "step": 455 + }, + { + "KL/chosen_KL_mean": -153.38815307617188, + "KL/mean": -186.91957092285156, + "KL/rejected_KL_mean": -220.4510040283203, + "KL/std": 96.58705139160156, + "epoch": 0.6893424036281179, + "fcm_dpo/beta": 0.005753940902650356, + "fcm_dpo/delta": 0.014651095494627953, + "fcm_dpo/margin": 67.0628433227539, + "fcm_dpo/q_t": 0.41437584161758423, + "grad_norm": 15.417052268981934, + "learning_rate": 1.3426974201083439e-07, + "logits/chosen": 0.6084275245666504, + "logits/rejected": 0.5453929901123047, + "logps/chosen": -209.78643798828125, + "logps/ref_chosen": -56.398284912109375, + "logps/ref_rejected": -82.61642456054688, + "logps/rejected": -303.06744384765625, + "loss": 1.1289, + "margin_dpo/margin_mean": 67.0628433227539, + "margin_dpo/margin_std": 110.33543395996094, + "step": 456 + }, + { + "KL/chosen_KL_mean": -151.1236572265625, + "KL/mean": -186.5624237060547, + "KL/rejected_KL_mean": -222.00119018554688, + "KL/std": 98.57406616210938, + "epoch": 0.690854119425548, + "fcm_dpo/beta": 0.005745270289480686, + "fcm_dpo/delta": -0.007712500169873238, + "fcm_dpo/margin": 70.87751770019531, + "fcm_dpo/q_t": 0.4060364365577698, + "grad_norm": 12.802936553955078, + "learning_rate": 1.3309935167761717e-07, + "logits/chosen": 0.7779738306999207, + "logits/rejected": 0.6983845233917236, + "logps/chosen": -195.84423828125, + "logps/ref_chosen": -44.72057342529297, + "logps/ref_rejected": -68.1158676147461, + "logps/rejected": -290.1170654296875, + "loss": 1.09, + "margin_dpo/margin_mean": 70.87751770019531, + "margin_dpo/margin_std": 94.67393493652344, + "step": 457 + }, + { + "KL/chosen_KL_mean": -144.48300170898438, + "KL/mean": -183.05062866210938, + "KL/rejected_KL_mean": -221.61825561523438, + "KL/std": 101.5442886352539, + "epoch": 0.6923658352229781, + "fcm_dpo/beta": 0.005730716977268457, + "fcm_dpo/delta": -0.04396446794271469, + "fcm_dpo/margin": 77.13526153564453, + "fcm_dpo/q_t": 0.3995509445667267, + "grad_norm": 12.134772300720215, + "learning_rate": 1.3193223130682936e-07, + "logits/chosen": 0.6931065917015076, + "logits/rejected": 0.5775718688964844, + "logps/chosen": -194.48867797851562, + "logps/ref_chosen": -50.00569152832031, + "logps/ref_rejected": -87.50015258789062, + "logps/rejected": -309.118408203125, + "loss": 1.0893, + "margin_dpo/margin_mean": 77.13526916503906, + "margin_dpo/margin_std": 113.13229370117188, + "step": 458 + }, + { + "KL/chosen_KL_mean": -140.49830627441406, + "KL/mean": -185.78858947753906, + "KL/rejected_KL_mean": -231.07887268066406, + "KL/std": 115.9324951171875, + "epoch": 0.6938775510204082, + "fcm_dpo/beta": 0.0056596603244543076, + "fcm_dpo/delta": -0.11930438876152039, + "fcm_dpo/margin": 90.58056640625, + "fcm_dpo/q_t": 0.38196009397506714, + "grad_norm": 12.148503303527832, + "learning_rate": 1.3076841354533658e-07, + "logits/chosen": 0.7065992951393127, + "logits/rejected": 0.6742749214172363, + "logps/chosen": -205.87625122070312, + "logps/ref_chosen": -65.37794494628906, + "logps/ref_rejected": -88.19244384765625, + "logps/rejected": -319.27130126953125, + "loss": 1.0332, + "margin_dpo/margin_mean": 90.58056640625, + "margin_dpo/margin_std": 108.33981323242188, + "step": 459 + }, + { + "KL/chosen_KL_mean": -152.5089111328125, + "KL/mean": -196.7735595703125, + "KL/rejected_KL_mean": -241.03823852539062, + "KL/std": 101.97077941894531, + "epoch": 0.6953892668178382, + "fcm_dpo/beta": 0.0054806992411613464, + "fcm_dpo/delta": -0.0901375487446785, + "fcm_dpo/margin": 88.52931213378906, + "fcm_dpo/q_t": 0.3906601667404175, + "grad_norm": 11.680279731750488, + "learning_rate": 1.2960793094762345e-07, + "logits/chosen": 0.7258821129798889, + "logits/rejected": 0.6057487726211548, + "logps/chosen": -217.07057189941406, + "logps/ref_chosen": -64.5616683959961, + "logps/ref_rejected": -88.67890167236328, + "logps/rejected": -329.7171325683594, + "loss": 1.0422, + "margin_dpo/margin_mean": 88.52931213378906, + "margin_dpo/margin_std": 110.12898254394531, + "step": 460 + }, + { + "KL/chosen_KL_mean": -127.51901245117188, + "KL/mean": -172.11732482910156, + "KL/rejected_KL_mean": -216.71563720703125, + "KL/std": 94.23216247558594, + "epoch": 0.6969009826152683, + "fcm_dpo/beta": 0.005357278976589441, + "fcm_dpo/delta": -0.08428196609020233, + "fcm_dpo/margin": 89.1966323852539, + "fcm_dpo/q_t": 0.38994884490966797, + "grad_norm": 11.4501371383667, + "learning_rate": 1.2845081597488286e-07, + "logits/chosen": 0.7885807752609253, + "logits/rejected": 0.7069375514984131, + "logps/chosen": -176.99691772460938, + "logps/ref_chosen": -49.4779167175293, + "logps/ref_rejected": -72.65262603759766, + "logps/rejected": -289.3682861328125, + "loss": 1.0401, + "margin_dpo/margin_mean": 89.19664001464844, + "margin_dpo/margin_std": 101.12211608886719, + "step": 461 + }, + { + "KL/chosen_KL_mean": -140.79742431640625, + "KL/mean": -184.84768676757812, + "KL/rejected_KL_mean": -228.89794921875, + "KL/std": 92.286865234375, + "epoch": 0.6984126984126984, + "fcm_dpo/beta": 0.00531365629285574, + "fcm_dpo/delta": -0.07177212089300156, + "fcm_dpo/margin": 88.10053253173828, + "fcm_dpo/q_t": 0.390781968832016, + "grad_norm": 12.073556900024414, + "learning_rate": 1.27297100994108e-07, + "logits/chosen": 0.6879956722259521, + "logits/rejected": 0.6383634209632874, + "logps/chosen": -201.29254150390625, + "logps/ref_chosen": -60.4951171875, + "logps/ref_rejected": -74.82136535644531, + "logps/rejected": -303.71929931640625, + "loss": 1.0442, + "margin_dpo/margin_mean": 88.10052490234375, + "margin_dpo/margin_std": 103.91917419433594, + "step": 462 + }, + { + "KL/chosen_KL_mean": -168.54791259765625, + "KL/mean": -195.15968322753906, + "KL/rejected_KL_mean": -221.77145385742188, + "KL/std": 93.11166381835938, + "epoch": 0.6999244142101285, + "fcm_dpo/beta": 0.005378012545406818, + "fcm_dpo/delta": 0.11724771559238434, + "fcm_dpo/margin": 53.223541259765625, + "fcm_dpo/q_t": 0.4336676001548767, + "grad_norm": 17.376508712768555, + "learning_rate": 1.2614681827718695e-07, + "logits/chosen": 0.6730971336364746, + "logits/rejected": 0.6703752279281616, + "logps/chosen": -236.2330322265625, + "logps/ref_chosen": -67.68511962890625, + "logps/ref_rejected": -71.32196044921875, + "logps/rejected": -293.0934143066406, + "loss": 1.1962, + "margin_dpo/margin_mean": 53.223541259765625, + "margin_dpo/margin_std": 102.88514709472656, + "step": 463 + }, + { + "KL/chosen_KL_mean": -152.6397705078125, + "KL/mean": -194.53070068359375, + "KL/rejected_KL_mean": -236.421630859375, + "KL/std": 104.98362731933594, + "epoch": 0.7014361300075586, + "fcm_dpo/beta": 0.005408740136772394, + "fcm_dpo/delta": -0.05576523393392563, + "fcm_dpo/margin": 83.78184509277344, + "fcm_dpo/q_t": 0.3968367576599121, + "grad_norm": 11.669844627380371, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": 0.6964342594146729, + "logits/rejected": 0.6669223308563232, + "logps/chosen": -211.805419921875, + "logps/ref_chosen": -59.16564178466797, + "logps/ref_rejected": -69.56146240234375, + "logps/rejected": -305.98309326171875, + "loss": 1.0841, + "margin_dpo/margin_mean": 83.78184509277344, + "margin_dpo/margin_std": 118.7095718383789, + "step": 464 + }, + { + "KL/chosen_KL_mean": -160.89891052246094, + "KL/mean": -196.22000122070312, + "KL/rejected_KL_mean": -231.54110717773438, + "KL/std": 99.98161315917969, + "epoch": 0.7029478458049887, + "fcm_dpo/beta": 0.00537415686994791, + "fcm_dpo/delta": 0.021056313067674637, + "fcm_dpo/margin": 70.64216613769531, + "fcm_dpo/q_t": 0.41352707147598267, + "grad_norm": 12.136297225952148, + "learning_rate": 1.238566782415197e-07, + "logits/chosen": 0.752052903175354, + "logits/rejected": 0.6917625665664673, + "logps/chosen": -219.41258239746094, + "logps/ref_chosen": -58.513671875, + "logps/ref_rejected": -84.31745910644531, + "logps/rejected": -315.85858154296875, + "loss": 1.1299, + "margin_dpo/margin_mean": 70.64217376708984, + "margin_dpo/margin_std": 112.321044921875, + "step": 465 + }, + { + "KL/chosen_KL_mean": -174.26275634765625, + "KL/mean": -194.21337890625, + "KL/rejected_KL_mean": -214.16400146484375, + "KL/std": 107.25249481201172, + "epoch": 0.7044595616024187, + "fcm_dpo/beta": 0.005527378059923649, + "fcm_dpo/delta": 0.18388685584068298, + "fcm_dpo/margin": 39.90126037597656, + "fcm_dpo/q_t": 0.4488188624382019, + "grad_norm": 18.92510414123535, + "learning_rate": 1.2271688498291334e-07, + "logits/chosen": 0.682281494140625, + "logits/rejected": 0.6877008676528931, + "logps/chosen": -247.528564453125, + "logps/ref_chosen": -73.26580810546875, + "logps/ref_rejected": -74.83621215820312, + "logps/rejected": -289.0002136230469, + "loss": 1.2685, + "margin_dpo/margin_mean": 39.90126037597656, + "margin_dpo/margin_std": 110.06634521484375, + "step": 466 + }, + { + "KL/chosen_KL_mean": -153.88006591796875, + "KL/mean": -187.2439422607422, + "KL/rejected_KL_mean": -220.60781860351562, + "KL/std": 103.33351135253906, + "epoch": 0.7059712773998488, + "fcm_dpo/beta": 0.005615277215838432, + "fcm_dpo/delta": 0.026298683136701584, + "fcm_dpo/margin": 66.72773742675781, + "fcm_dpo/q_t": 0.4163486957550049, + "grad_norm": 11.329875946044922, + "learning_rate": 1.2158065210664848e-07, + "logits/chosen": 0.7394921779632568, + "logits/rejected": 0.5978328585624695, + "logps/chosen": -201.4595489501953, + "logps/ref_chosen": -47.57947540283203, + "logps/ref_rejected": -78.68522644042969, + "logps/rejected": -299.29302978515625, + "loss": 1.1286, + "margin_dpo/margin_mean": 66.72773742675781, + "margin_dpo/margin_std": 107.79798889160156, + "step": 467 + }, + { + "KL/chosen_KL_mean": -146.56996154785156, + "KL/mean": -190.59567260742188, + "KL/rejected_KL_mean": -234.62139892578125, + "KL/std": 101.60633850097656, + "epoch": 0.7074829931972789, + "fcm_dpo/beta": 0.005549922585487366, + "fcm_dpo/delta": -0.09322632849216461, + "fcm_dpo/margin": 88.05142974853516, + "fcm_dpo/q_t": 0.38789719343185425, + "grad_norm": 16.00092887878418, + "learning_rate": 1.204480113956011e-07, + "logits/chosen": 0.6605847477912903, + "logits/rejected": 0.6465221643447876, + "logps/chosen": -210.49774169921875, + "logps/ref_chosen": -63.92778778076172, + "logps/ref_rejected": -76.51626586914062, + "logps/rejected": -311.1376647949219, + "loss": 1.0514, + "margin_dpo/margin_mean": 88.05143737792969, + "margin_dpo/margin_std": 115.202880859375, + "step": 468 + }, + { + "KL/chosen_KL_mean": -149.39480590820312, + "KL/mean": -189.18585205078125, + "KL/rejected_KL_mean": -228.97686767578125, + "KL/std": 100.38656616210938, + "epoch": 0.708994708994709, + "fcm_dpo/beta": 0.005461008753627539, + "fcm_dpo/delta": -0.037061676383018494, + "fcm_dpo/margin": 79.58206939697266, + "fcm_dpo/q_t": 0.3996996283531189, + "grad_norm": 13.444028854370117, + "learning_rate": 1.1931899453216697e-07, + "logits/chosen": 0.7412175536155701, + "logits/rejected": 0.7253165245056152, + "logps/chosen": -208.4530029296875, + "logps/ref_chosen": -59.05818176269531, + "logps/ref_rejected": -75.67672729492188, + "logps/rejected": -304.6535949707031, + "loss": 1.0622, + "margin_dpo/margin_mean": 79.58206939697266, + "margin_dpo/margin_std": 93.98836517333984, + "step": 469 + }, + { + "KL/chosen_KL_mean": -142.4019317626953, + "KL/mean": -181.78958129882812, + "KL/rejected_KL_mean": -221.17721557617188, + "KL/std": 96.90911865234375, + "epoch": 0.7105064247921391, + "fcm_dpo/beta": 0.005472735036164522, + "fcm_dpo/delta": -0.03261677548289299, + "fcm_dpo/margin": 78.77527618408203, + "fcm_dpo/q_t": 0.4012529253959656, + "grad_norm": 12.413467407226562, + "learning_rate": 1.1819363309737438e-07, + "logits/chosen": 0.7318305969238281, + "logits/rejected": 0.665635883808136, + "logps/chosen": -190.26937866210938, + "logps/ref_chosen": -47.86743927001953, + "logps/ref_rejected": -65.96859741210938, + "logps/rejected": -287.14581298828125, + "loss": 1.0847, + "margin_dpo/margin_mean": 78.77527618408203, + "margin_dpo/margin_std": 108.5361328125, + "step": 470 + }, + { + "KL/chosen_KL_mean": -138.75448608398438, + "KL/mean": -181.73556518554688, + "KL/rejected_KL_mean": -224.7166290283203, + "KL/std": 96.12451171875, + "epoch": 0.7120181405895691, + "fcm_dpo/beta": 0.0053847962990403175, + "fcm_dpo/delta": -0.06609291583299637, + "fcm_dpo/margin": 85.96214294433594, + "fcm_dpo/q_t": 0.3922724723815918, + "grad_norm": 13.146032333374023, + "learning_rate": 1.1707195857000215e-07, + "logits/chosen": 0.7092275619506836, + "logits/rejected": 0.6556600332260132, + "logps/chosen": -196.53233337402344, + "logps/ref_chosen": -57.777854919433594, + "logps/ref_rejected": -73.81172180175781, + "logps/rejected": -298.5283508300781, + "loss": 1.0531, + "margin_dpo/margin_mean": 85.96214294433594, + "margin_dpo/margin_std": 105.41901397705078, + "step": 471 + }, + { + "KL/chosen_KL_mean": -139.62513732910156, + "KL/mean": -173.63473510742188, + "KL/rejected_KL_mean": -207.6443328857422, + "KL/std": 100.2338638305664, + "epoch": 0.7135298563869993, + "fcm_dpo/beta": 0.00541897676885128, + "fcm_dpo/delta": 0.03227098286151886, + "fcm_dpo/margin": 68.01919555664062, + "fcm_dpo/q_t": 0.41764307022094727, + "grad_norm": 13.090045928955078, + "learning_rate": 1.1595400232569768e-07, + "logits/chosen": 0.7192651629447937, + "logits/rejected": 0.6732028722763062, + "logps/chosen": -195.5338134765625, + "logps/ref_chosen": -55.908668518066406, + "logps/ref_rejected": -74.70294189453125, + "logps/rejected": -282.3472900390625, + "loss": 1.159, + "margin_dpo/margin_mean": 68.01919555664062, + "margin_dpo/margin_std": 125.8399658203125, + "step": 472 + }, + { + "KL/chosen_KL_mean": -143.20843505859375, + "KL/mean": -185.14447021484375, + "KL/rejected_KL_mean": -227.0805206298828, + "KL/std": 115.70045471191406, + "epoch": 0.7150415721844293, + "fcm_dpo/beta": 0.005365893244743347, + "fcm_dpo/delta": -0.052427202463150024, + "fcm_dpo/margin": 83.87207794189453, + "fcm_dpo/q_t": 0.4017806649208069, + "grad_norm": 14.348088264465332, + "learning_rate": 1.1483979563610069e-07, + "logits/chosen": 0.8000465631484985, + "logits/rejected": 0.6861811876296997, + "logps/chosen": -197.36932373046875, + "logps/ref_chosen": -54.16088104248047, + "logps/ref_rejected": -92.76789855957031, + "logps/rejected": -319.8484191894531, + "loss": 1.1088, + "margin_dpo/margin_mean": 83.87207794189453, + "margin_dpo/margin_std": 137.078125, + "step": 473 + }, + { + "KL/chosen_KL_mean": -145.8909912109375, + "KL/mean": -181.9802703857422, + "KL/rejected_KL_mean": -218.06954956054688, + "KL/std": 103.30743408203125, + "epoch": 0.7165532879818595, + "fcm_dpo/beta": 0.0053678592666983604, + "fcm_dpo/delta": 0.013039542362093925, + "fcm_dpo/margin": 72.17857360839844, + "fcm_dpo/q_t": 0.4138396084308624, + "grad_norm": 18.02506446838379, + "learning_rate": 1.1372936966796709e-07, + "logits/chosen": 0.7723579406738281, + "logits/rejected": 0.699033260345459, + "logps/chosen": -192.57669067382812, + "logps/ref_chosen": -46.685707092285156, + "logps/ref_rejected": -71.44731903076172, + "logps/rejected": -289.5168762207031, + "loss": 1.1372, + "margin_dpo/margin_mean": 72.17857360839844, + "margin_dpo/margin_std": 123.31417846679688, + "step": 474 + }, + { + "KL/chosen_KL_mean": -139.6383514404297, + "KL/mean": -190.87100219726562, + "KL/rejected_KL_mean": -242.1036376953125, + "KL/std": 104.505615234375, + "epoch": 0.7180650037792895, + "fcm_dpo/beta": 0.005235554650425911, + "fcm_dpo/delta": -0.14490701258182526, + "fcm_dpo/margin": 102.46532440185547, + "fcm_dpo/q_t": 0.3785374164581299, + "grad_norm": 9.661964416503906, + "learning_rate": 1.126227554822985e-07, + "logits/chosen": 0.7241721153259277, + "logits/rejected": 0.6713173389434814, + "logps/chosen": -198.1256561279297, + "logps/ref_chosen": -58.4873046875, + "logps/ref_rejected": -87.00187683105469, + "logps/rejected": -329.10552978515625, + "loss": 1.0044, + "margin_dpo/margin_mean": 102.46531677246094, + "margin_dpo/margin_std": 115.88256072998047, + "step": 475 + }, + { + "KL/chosen_KL_mean": -170.72518920898438, + "KL/mean": -202.345703125, + "KL/rejected_KL_mean": -233.9662322998047, + "KL/std": 108.97540283203125, + "epoch": 0.7195767195767195, + "fcm_dpo/beta": 0.005290007218718529, + "fcm_dpo/delta": 0.06701076030731201, + "fcm_dpo/margin": 63.24106979370117, + "fcm_dpo/q_t": 0.424893856048584, + "grad_norm": 14.140975952148438, + "learning_rate": 1.1151998403347243e-07, + "logits/chosen": 0.6015628576278687, + "logits/rejected": 0.6035597324371338, + "logps/chosen": -246.1068115234375, + "logps/ref_chosen": -75.38162231445312, + "logps/ref_rejected": -76.99822235107422, + "logps/rejected": -310.9644775390625, + "loss": 1.1701, + "margin_dpo/margin_mean": 63.24106979370117, + "margin_dpo/margin_std": 116.38935852050781, + "step": 476 + }, + { + "KL/chosen_KL_mean": -169.3343048095703, + "KL/mean": -205.39453125, + "KL/rejected_KL_mean": -241.45474243164062, + "KL/std": 109.87100219726562, + "epoch": 0.7210884353741497, + "fcm_dpo/beta": 0.005296539049595594, + "fcm_dpo/delta": 0.018730882555246353, + "fcm_dpo/margin": 72.12043762207031, + "fcm_dpo/q_t": 0.41515541076660156, + "grad_norm": 13.771434783935547, + "learning_rate": 1.1042108616837692e-07, + "logits/chosen": 0.7227048277854919, + "logits/rejected": 0.6724196672439575, + "logps/chosen": -230.40768432617188, + "logps/ref_chosen": -61.073387145996094, + "logps/ref_rejected": -81.34375, + "logps/rejected": -322.7984924316406, + "loss": 1.1689, + "margin_dpo/margin_mean": 72.12043762207031, + "margin_dpo/margin_std": 140.45468139648438, + "step": 477 + }, + { + "KL/chosen_KL_mean": -149.87953186035156, + "KL/mean": -177.552490234375, + "KL/rejected_KL_mean": -205.22547912597656, + "KL/std": 93.2808837890625, + "epoch": 0.7226001511715797, + "fcm_dpo/beta": 0.005373564548790455, + "fcm_dpo/delta": 0.10587666183710098, + "fcm_dpo/margin": 55.34593963623047, + "fcm_dpo/q_t": 0.43196284770965576, + "grad_norm": 15.536688804626465, + "learning_rate": 1.0932609262554746e-07, + "logits/chosen": 0.655882716178894, + "logits/rejected": 0.6669564247131348, + "logps/chosen": -207.04684448242188, + "logps/ref_chosen": -57.16731643676758, + "logps/ref_rejected": -53.30917739868164, + "logps/rejected": -258.53466796875, + "loss": 1.2018, + "margin_dpo/margin_mean": 55.34593963623047, + "margin_dpo/margin_std": 114.13899230957031, + "step": 478 + }, + { + "KL/chosen_KL_mean": -154.94833374023438, + "KL/mean": -179.95150756835938, + "KL/rejected_KL_mean": -204.95465087890625, + "KL/std": 93.98583984375, + "epoch": 0.7241118669690099, + "fcm_dpo/beta": 0.005500415340065956, + "fcm_dpo/delta": 0.12866447865962982, + "fcm_dpo/margin": 50.00631332397461, + "fcm_dpo/q_t": 0.4381939172744751, + "grad_norm": 15.030978202819824, + "learning_rate": 1.0823503403430734e-07, + "logits/chosen": 0.6659060716629028, + "logits/rejected": 0.627450704574585, + "logps/chosen": -213.86166381835938, + "logps/ref_chosen": -58.91331481933594, + "logps/ref_rejected": -63.7403450012207, + "logps/rejected": -268.69500732421875, + "loss": 1.2205, + "margin_dpo/margin_mean": 50.006309509277344, + "margin_dpo/margin_std": 111.1731185913086, + "step": 479 + }, + { + "KL/chosen_KL_mean": -159.55169677734375, + "KL/mean": -198.3495330810547, + "KL/rejected_KL_mean": -237.14736938476562, + "KL/std": 99.51255798339844, + "epoch": 0.7256235827664399, + "fcm_dpo/beta": 0.005575342103838921, + "fcm_dpo/delta": -0.035166725516319275, + "fcm_dpo/margin": 77.59567260742188, + "fcm_dpo/q_t": 0.40224170684814453, + "grad_norm": 15.844165802001953, + "learning_rate": 1.0714794091391072e-07, + "logits/chosen": 0.6764963269233704, + "logits/rejected": 0.6669754981994629, + "logps/chosen": -222.352294921875, + "logps/ref_chosen": -62.80061340332031, + "logps/ref_rejected": -67.58859252929688, + "logps/rejected": -304.7359619140625, + "loss": 1.1007, + "margin_dpo/margin_mean": 77.59567260742188, + "margin_dpo/margin_std": 114.90257263183594, + "step": 480 + }, + { + "KL/chosen_KL_mean": -156.21351623535156, + "KL/mean": -191.5591583251953, + "KL/rejected_KL_mean": -226.90480041503906, + "KL/std": 102.61424255371094, + "epoch": 0.72713529856387, + "fcm_dpo/beta": 0.005516592413187027, + "fcm_dpo/delta": 0.010359540581703186, + "fcm_dpo/margin": 70.6912841796875, + "fcm_dpo/q_t": 0.41247764229774475, + "grad_norm": 14.178512573242188, + "learning_rate": 1.0606484367268906e-07, + "logits/chosen": 0.6627799272537231, + "logits/rejected": 0.6644883751869202, + "logps/chosen": -221.5, + "logps/ref_chosen": -65.28649139404297, + "logps/ref_rejected": -70.78668212890625, + "logps/rejected": -297.69146728515625, + "loss": 1.1258, + "margin_dpo/margin_mean": 70.69129180908203, + "margin_dpo/margin_std": 114.8702392578125, + "step": 481 + }, + { + "KL/chosen_KL_mean": -175.76876831054688, + "KL/mean": -212.71697998046875, + "KL/rejected_KL_mean": -249.66519165039062, + "KL/std": 110.07460021972656, + "epoch": 0.7286470143613001, + "fcm_dpo/beta": 0.005537115037441254, + "fcm_dpo/delta": -0.009665248915553093, + "fcm_dpo/margin": 73.89642333984375, + "fcm_dpo/q_t": 0.41188326478004456, + "grad_norm": 16.9229679107666, + "learning_rate": 1.0498577260720048e-07, + "logits/chosen": 0.6482515335083008, + "logits/rejected": 0.4943665862083435, + "logps/chosen": -236.67495727539062, + "logps/ref_chosen": -60.906185150146484, + "logps/ref_rejected": -103.44656372070312, + "logps/rejected": -353.11175537109375, + "loss": 1.1497, + "margin_dpo/margin_mean": 73.89642333984375, + "margin_dpo/margin_std": 137.73287963867188, + "step": 482 + }, + { + "KL/chosen_KL_mean": -147.17604064941406, + "KL/mean": -191.10537719726562, + "KL/rejected_KL_mean": -235.03472900390625, + "KL/std": 99.93904113769531, + "epoch": 0.7301587301587301, + "fcm_dpo/beta": 0.005463513545691967, + "fcm_dpo/delta": -0.08396557718515396, + "fcm_dpo/margin": 87.85865783691406, + "fcm_dpo/q_t": 0.39078426361083984, + "grad_norm": 12.708328247070312, + "learning_rate": 1.0391075790138232e-07, + "logits/chosen": 0.7652244567871094, + "logits/rejected": 0.6555371284484863, + "logps/chosen": -200.36805725097656, + "logps/ref_chosen": -53.192012786865234, + "logps/ref_rejected": -81.83927154541016, + "logps/rejected": -316.8739929199219, + "loss": 1.0521, + "margin_dpo/margin_mean": 87.85865783691406, + "margin_dpo/margin_std": 113.67207336425781, + "step": 483 + }, + { + "KL/chosen_KL_mean": -154.01959228515625, + "KL/mean": -181.98313903808594, + "KL/rejected_KL_mean": -209.94671630859375, + "KL/std": 97.48323059082031, + "epoch": 0.7316704459561603, + "fcm_dpo/beta": 0.005525670014321804, + "fcm_dpo/delta": 0.09322261810302734, + "fcm_dpo/margin": 55.9271240234375, + "fcm_dpo/q_t": 0.42791566252708435, + "grad_norm": 17.04726219177246, + "learning_rate": 1.0283982962570681e-07, + "logits/chosen": 0.772758960723877, + "logits/rejected": 0.7413250207901001, + "logps/chosen": -211.78904724121094, + "logps/ref_chosen": -57.76945877075195, + "logps/ref_rejected": -71.6829833984375, + "logps/rejected": -281.62969970703125, + "loss": 1.1562, + "margin_dpo/margin_mean": 55.9271240234375, + "margin_dpo/margin_std": 86.11868286132812, + "step": 484 + }, + { + "KL/chosen_KL_mean": -156.07125854492188, + "KL/mean": -188.55908203125, + "KL/rejected_KL_mean": -221.0469207763672, + "KL/std": 99.03124237060547, + "epoch": 0.7331821617535903, + "fcm_dpo/beta": 0.005521639250218868, + "fcm_dpo/delta": 0.04180263727903366, + "fcm_dpo/margin": 64.97565460205078, + "fcm_dpo/q_t": 0.41826534271240234, + "grad_norm": 12.852514266967773, + "learning_rate": 1.0177301773633992e-07, + "logits/chosen": 0.733189582824707, + "logits/rejected": 0.7117427587509155, + "logps/chosen": -212.7071075439453, + "logps/ref_chosen": -56.63584899902344, + "logps/ref_rejected": -70.85614013671875, + "logps/rejected": -291.9030456542969, + "loss": 1.1325, + "margin_dpo/margin_mean": 64.97565460205078, + "margin_dpo/margin_std": 96.84291076660156, + "step": 485 + }, + { + "KL/chosen_KL_mean": -181.23599243164062, + "KL/mean": -211.62835693359375, + "KL/rejected_KL_mean": -242.020751953125, + "KL/std": 114.32650756835938, + "epoch": 0.7346938775510204, + "fcm_dpo/beta": 0.00562657555565238, + "fcm_dpo/delta": 0.060053564608097076, + "fcm_dpo/margin": 60.78474426269531, + "fcm_dpo/q_t": 0.42665895819664, + "grad_norm": 11.995468139648438, + "learning_rate": 1.007103520743035e-07, + "logits/chosen": 0.7314083576202393, + "logits/rejected": 0.6161884069442749, + "logps/chosen": -237.5830078125, + "logps/ref_chosen": -56.347023010253906, + "logps/ref_rejected": -85.97221374511719, + "logps/rejected": -327.9929504394531, + "loss": 1.2017, + "margin_dpo/margin_mean": 60.78474426269531, + "margin_dpo/margin_std": 133.8424072265625, + "step": 486 + }, + { + "KL/chosen_KL_mean": -162.27162170410156, + "KL/mean": -199.17288208007812, + "KL/rejected_KL_mean": -236.07412719726562, + "KL/std": 104.9486083984375, + "epoch": 0.7362055933484505, + "fcm_dpo/beta": 0.005627620965242386, + "fcm_dpo/delta": -0.016016241163015366, + "fcm_dpo/margin": 73.8025131225586, + "fcm_dpo/q_t": 0.4103911519050598, + "grad_norm": 14.136743545532227, + "learning_rate": 9.965186236464046e-08, + "logits/chosen": 0.8066281676292419, + "logits/rejected": 0.7472410202026367, + "logps/chosen": -222.8888397216797, + "logps/ref_chosen": -60.617218017578125, + "logps/ref_rejected": -82.50975036621094, + "logps/rejected": -318.5838623046875, + "loss": 1.1205, + "margin_dpo/margin_mean": 73.80250549316406, + "margin_dpo/margin_std": 125.0999755859375, + "step": 487 + }, + { + "KL/chosen_KL_mean": -152.14353942871094, + "KL/mean": -192.02452087402344, + "KL/rejected_KL_mean": -231.905517578125, + "KL/std": 99.62962341308594, + "epoch": 0.7377173091458806, + "fcm_dpo/beta": 0.005611489526927471, + "fcm_dpo/delta": -0.05003175511956215, + "fcm_dpo/margin": 79.76195526123047, + "fcm_dpo/q_t": 0.39986562728881836, + "grad_norm": 16.106365203857422, + "learning_rate": 9.859757821558337e-08, + "logits/chosen": 0.7334238290786743, + "logits/rejected": 0.6768772006034851, + "logps/chosen": -215.25259399414062, + "logps/ref_chosen": -63.10905075073242, + "logps/ref_rejected": -82.49348449707031, + "logps/rejected": -314.39898681640625, + "loss": 1.0765, + "margin_dpo/margin_mean": 79.76195526123047, + "margin_dpo/margin_std": 109.84991455078125, + "step": 488 + }, + { + "KL/chosen_KL_mean": -178.01058959960938, + "KL/mean": -202.5729522705078, + "KL/rejected_KL_mean": -227.13528442382812, + "KL/std": 110.450927734375, + "epoch": 0.7392290249433107, + "fcm_dpo/beta": 0.005673965439200401, + "fcm_dpo/delta": 0.12477852404117584, + "fcm_dpo/margin": 49.12468719482422, + "fcm_dpo/q_t": 0.4373658299446106, + "grad_norm": 14.4613618850708, + "learning_rate": 9.754752911772615e-08, + "logits/chosen": 0.6765942573547363, + "logits/rejected": 0.6304404735565186, + "logps/chosen": -242.9995574951172, + "logps/ref_chosen": -64.98896026611328, + "logps/ref_rejected": -84.39607238769531, + "logps/rejected": -311.5313720703125, + "loss": 1.2361, + "margin_dpo/margin_mean": 49.12468719482422, + "margin_dpo/margin_std": 119.76895141601562, + "step": 489 + }, + { + "KL/chosen_KL_mean": -160.54306030273438, + "KL/mean": -189.7637481689453, + "KL/rejected_KL_mean": -218.9844512939453, + "KL/std": 106.82121276855469, + "epoch": 0.7407407407407407, + "fcm_dpo/beta": 0.005743181798607111, + "fcm_dpo/delta": 0.06649347394704819, + "fcm_dpo/margin": 58.441383361816406, + "fcm_dpo/q_t": 0.4260715842247009, + "grad_norm": 12.848529815673828, + "learning_rate": 9.650174444319956e-08, + "logits/chosen": 0.755155086517334, + "logits/rejected": 0.7366271018981934, + "logps/chosen": -222.45181274414062, + "logps/ref_chosen": -61.90874481201172, + "logps/ref_rejected": -70.58566284179688, + "logps/rejected": -289.57012939453125, + "loss": 1.217, + "margin_dpo/margin_mean": 58.441383361816406, + "margin_dpo/margin_std": 134.05650329589844, + "step": 490 + }, + { + "KL/chosen_KL_mean": -153.87290954589844, + "KL/mean": -185.72323608398438, + "KL/rejected_KL_mean": -217.57354736328125, + "KL/std": 97.37249755859375, + "epoch": 0.7422524565381708, + "fcm_dpo/beta": 0.0057761832140386105, + "fcm_dpo/delta": 0.03252832964062691, + "fcm_dpo/margin": 63.70063400268555, + "fcm_dpo/q_t": 0.4171232581138611, + "grad_norm": 13.742348670959473, + "learning_rate": 9.546025344484868e-08, + "logits/chosen": 0.6527000069618225, + "logits/rejected": 0.5926094055175781, + "logps/chosen": -209.34860229492188, + "logps/ref_chosen": -55.47570037841797, + "logps/ref_rejected": -78.70318603515625, + "logps/rejected": -296.2767333984375, + "loss": 1.1383, + "margin_dpo/margin_mean": 63.70063781738281, + "margin_dpo/margin_std": 103.06165313720703, + "step": 491 + }, + { + "KL/chosen_KL_mean": -182.7808837890625, + "KL/mean": -209.8721923828125, + "KL/rejected_KL_mean": -236.9635009765625, + "KL/std": 109.27008819580078, + "epoch": 0.7437641723356009, + "fcm_dpo/beta": 0.005924704950302839, + "fcm_dpo/delta": 0.080745629966259, + "fcm_dpo/margin": 54.182613372802734, + "fcm_dpo/q_t": 0.4289284348487854, + "grad_norm": 16.213777542114258, + "learning_rate": 9.442308525541589e-08, + "logits/chosen": 0.7147728800773621, + "logits/rejected": 0.6443384885787964, + "logps/chosen": -250.06727600097656, + "logps/ref_chosen": -67.28638458251953, + "logps/ref_rejected": -82.78628540039062, + "logps/rejected": -319.74981689453125, + "loss": 1.2201, + "margin_dpo/margin_mean": 54.182613372802734, + "margin_dpo/margin_std": 122.61883544921875, + "step": 492 + }, + { + "KL/chosen_KL_mean": -149.8064727783203, + "KL/mean": -190.75125122070312, + "KL/rejected_KL_mean": -231.696044921875, + "KL/std": 106.93426513671875, + "epoch": 0.745275888133031, + "fcm_dpo/beta": 0.005893126130104065, + "fcm_dpo/delta": -0.08745664358139038, + "fcm_dpo/margin": 81.88956451416016, + "fcm_dpo/q_t": 0.3927948474884033, + "grad_norm": 14.668304443359375, + "learning_rate": 9.339026888672468e-08, + "logits/chosen": 0.7042903304100037, + "logits/rejected": 0.6280442476272583, + "logps/chosen": -205.73397827148438, + "logps/ref_chosen": -55.92750549316406, + "logps/ref_rejected": -79.12149810791016, + "logps/rejected": -310.8175354003906, + "loss": 1.0729, + "margin_dpo/margin_mean": 81.88956451416016, + "margin_dpo/margin_std": 115.11085510253906, + "step": 493 + }, + { + "KL/chosen_KL_mean": -155.3618621826172, + "KL/mean": -190.2035369873047, + "KL/rejected_KL_mean": -225.04522705078125, + "KL/std": 106.14227294921875, + "epoch": 0.7467876039304611, + "fcm_dpo/beta": 0.005800245329737663, + "fcm_dpo/delta": -0.004481932148337364, + "fcm_dpo/margin": 69.683349609375, + "fcm_dpo/q_t": 0.41056621074676514, + "grad_norm": 15.236351013183594, + "learning_rate": 9.236183322886945e-08, + "logits/chosen": 0.623336672782898, + "logits/rejected": 0.572545051574707, + "logps/chosen": -223.31597900390625, + "logps/ref_chosen": -67.95410919189453, + "logps/ref_rejected": -90.50865173339844, + "logps/rejected": -315.55389404296875, + "loss": 1.154, + "margin_dpo/margin_mean": 69.683349609375, + "margin_dpo/margin_std": 131.35519409179688, + "step": 494 + }, + { + "KL/chosen_KL_mean": -153.0796661376953, + "KL/mean": -181.11660766601562, + "KL/rejected_KL_mean": -209.15353393554688, + "KL/std": 104.95536804199219, + "epoch": 0.7482993197278912, + "fcm_dpo/beta": 0.005901433527469635, + "fcm_dpo/delta": 0.0706893801689148, + "fcm_dpo/margin": 56.073883056640625, + "fcm_dpo/q_t": 0.43131160736083984, + "grad_norm": 17.017921447753906, + "learning_rate": 9.133780704940594e-08, + "logits/chosen": 0.7580794095993042, + "logits/rejected": 0.6966645121574402, + "logps/chosen": -205.7051239013672, + "logps/ref_chosen": -52.62546157836914, + "logps/ref_rejected": -72.06781005859375, + "logps/rejected": -281.2213439941406, + "loss": 1.2098, + "margin_dpo/margin_mean": 56.073883056640625, + "margin_dpo/margin_std": 128.44505310058594, + "step": 495 + }, + { + "KL/chosen_KL_mean": -167.5390625, + "KL/mean": -206.25115966796875, + "KL/rejected_KL_mean": -244.96324157714844, + "KL/std": 114.25956726074219, + "epoch": 0.7498110355253212, + "fcm_dpo/beta": 0.005806931294500828, + "fcm_dpo/delta": -0.05337368696928024, + "fcm_dpo/margin": 77.4241943359375, + "fcm_dpo/q_t": 0.4092605710029602, + "grad_norm": 13.090649604797363, + "learning_rate": 9.031821899254797e-08, + "logits/chosen": 0.7403356432914734, + "logits/rejected": 0.6223543882369995, + "logps/chosen": -225.13638305664062, + "logps/ref_chosen": -57.597320556640625, + "logps/ref_rejected": -94.36127471923828, + "logps/rejected": -339.32452392578125, + "loss": 1.1346, + "margin_dpo/margin_mean": 77.4241943359375, + "margin_dpo/margin_std": 145.4561004638672, + "step": 496 + }, + { + "KL/chosen_KL_mean": -168.29544067382812, + "KL/mean": -207.5030975341797, + "KL/rejected_KL_mean": -246.71075439453125, + "KL/std": 105.35246276855469, + "epoch": 0.7513227513227513, + "fcm_dpo/beta": 0.005777341779321432, + "fcm_dpo/delta": -0.05565295368432999, + "fcm_dpo/margin": 78.41531372070312, + "fcm_dpo/q_t": 0.3961220383644104, + "grad_norm": 13.204294204711914, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": 0.7045537829399109, + "logits/rejected": 0.6701431274414062, + "logps/chosen": -241.08538818359375, + "logps/ref_chosen": -72.78994750976562, + "logps/ref_rejected": -89.48483276367188, + "logps/rejected": -336.1955871582031, + "loss": 1.0859, + "margin_dpo/margin_mean": 78.41531372070312, + "margin_dpo/margin_std": 114.64138793945312, + "step": 497 + }, + { + "KL/chosen_KL_mean": -151.4757537841797, + "KL/mean": -191.3174591064453, + "KL/rejected_KL_mean": -231.15916442871094, + "KL/std": 99.70770263671875, + "epoch": 0.7528344671201814, + "fcm_dpo/beta": 0.005714482627809048, + "fcm_dpo/delta": -0.05804353579878807, + "fcm_dpo/margin": 79.68342590332031, + "fcm_dpo/q_t": 0.39497846364974976, + "grad_norm": 14.724855422973633, + "learning_rate": 8.829247120198563e-08, + "logits/chosen": 0.6854042410850525, + "logits/rejected": 0.6628051996231079, + "logps/chosen": -219.84146118164062, + "logps/ref_chosen": -68.36572265625, + "logps/ref_rejected": -71.28846740722656, + "logps/rejected": -302.4476318359375, + "loss": 1.068, + "margin_dpo/margin_mean": 79.68342590332031, + "margin_dpo/margin_std": 106.51513671875, + "step": 498 + }, + { + "KL/chosen_KL_mean": -151.89300537109375, + "KL/mean": -192.9249267578125, + "KL/rejected_KL_mean": -233.95684814453125, + "KL/std": 109.68285369873047, + "epoch": 0.7543461829176115, + "fcm_dpo/beta": 0.00564592145383358, + "fcm_dpo/delta": -0.06639964133501053, + "fcm_dpo/margin": 82.06385040283203, + "fcm_dpo/q_t": 0.399397611618042, + "grad_norm": 18.094348907470703, + "learning_rate": 8.728636813280163e-08, + "logits/chosen": 0.7410115003585815, + "logits/rejected": 0.672810435295105, + "logps/chosen": -213.80181884765625, + "logps/ref_chosen": -61.90882873535156, + "logps/ref_rejected": -91.9411392211914, + "logps/rejected": -325.8979797363281, + "loss": 1.1248, + "margin_dpo/margin_mean": 82.0638427734375, + "margin_dpo/margin_std": 142.8561553955078, + "step": 499 + }, + { + "KL/chosen_KL_mean": -158.30862426757812, + "KL/mean": -192.1842041015625, + "KL/rejected_KL_mean": -226.05978393554688, + "KL/std": 97.90144348144531, + "epoch": 0.7558578987150416, + "fcm_dpo/beta": 0.005628244951367378, + "fcm_dpo/delta": 0.019335078075528145, + "fcm_dpo/margin": 67.75115966796875, + "fcm_dpo/q_t": 0.4112476706504822, + "grad_norm": 15.212784767150879, + "learning_rate": 8.628481651367875e-08, + "logits/chosen": 0.6629618406295776, + "logits/rejected": 0.6684551239013672, + "logps/chosen": -228.53445434570312, + "logps/ref_chosen": -70.225830078125, + "logps/ref_rejected": -71.72203063964844, + "logps/rejected": -297.78179931640625, + "loss": 1.1625, + "margin_dpo/margin_mean": 67.75115966796875, + "margin_dpo/margin_std": 127.54959869384766, + "step": 500 + }, + { + "KL/chosen_KL_mean": -159.2582550048828, + "KL/mean": -189.79246520996094, + "KL/rejected_KL_mean": -220.32667541503906, + "KL/std": 104.34779357910156, + "epoch": 0.7573696145124716, + "fcm_dpo/beta": 0.005706685595214367, + "fcm_dpo/delta": 0.05312148109078407, + "fcm_dpo/margin": 61.06843185424805, + "fcm_dpo/q_t": 0.4191363751888275, + "grad_norm": 11.982666015625, + "learning_rate": 8.528784436016878e-08, + "logits/chosen": 0.7013833522796631, + "logits/rejected": 0.7047612071037292, + "logps/chosen": -223.8570556640625, + "logps/ref_chosen": -64.59880828857422, + "logps/ref_rejected": -70.59329223632812, + "logps/rejected": -290.91998291015625, + "loss": 1.1253, + "margin_dpo/margin_mean": 61.06842803955078, + "margin_dpo/margin_std": 85.38192749023438, + "step": 501 + }, + { + "KL/chosen_KL_mean": -159.037109375, + "KL/mean": -192.675537109375, + "KL/rejected_KL_mean": -226.31393432617188, + "KL/std": 110.60833740234375, + "epoch": 0.7588813303099018, + "fcm_dpo/beta": 0.005739234387874603, + "fcm_dpo/delta": 0.014180617406964302, + "fcm_dpo/margin": 67.27684020996094, + "fcm_dpo/q_t": 0.4120279550552368, + "grad_norm": 17.2750301361084, + "learning_rate": 8.4295479559726e-08, + "logits/chosen": 0.7115000486373901, + "logits/rejected": 0.6628080606460571, + "logps/chosen": -224.5037384033203, + "logps/ref_chosen": -65.46662902832031, + "logps/ref_rejected": -90.22233581542969, + "logps/rejected": -316.5362548828125, + "loss": 1.1198, + "margin_dpo/margin_mean": 67.27684020996094, + "margin_dpo/margin_std": 103.1072769165039, + "step": 502 + }, + { + "KL/chosen_KL_mean": -142.4674072265625, + "KL/mean": -176.8408203125, + "KL/rejected_KL_mean": -211.2142333984375, + "KL/std": 96.57347106933594, + "epoch": 0.7603930461073318, + "fcm_dpo/beta": 0.0057237460277974606, + "fcm_dpo/delta": 0.0067562200129032135, + "fcm_dpo/margin": 68.746826171875, + "fcm_dpo/q_t": 0.4091821312904358, + "grad_norm": 12.451175689697266, + "learning_rate": 8.330774987092712e-08, + "logits/chosen": 0.6891748905181885, + "logits/rejected": 0.6937826871871948, + "logps/chosen": -194.3021697998047, + "logps/ref_chosen": -51.83476257324219, + "logps/ref_rejected": -57.62522506713867, + "logps/rejected": -268.8394775390625, + "loss": 1.1277, + "margin_dpo/margin_mean": 68.746826171875, + "margin_dpo/margin_std": 110.9075698852539, + "step": 503 + }, + { + "KL/chosen_KL_mean": -147.31979370117188, + "KL/mean": -192.40731811523438, + "KL/rejected_KL_mean": -237.49481201171875, + "KL/std": 96.30535888671875, + "epoch": 0.7619047619047619, + "fcm_dpo/beta": 0.005670842714607716, + "fcm_dpo/delta": -0.11748392879962921, + "fcm_dpo/margin": 90.17503356933594, + "fcm_dpo/q_t": 0.38196149468421936, + "grad_norm": 13.738269805908203, + "learning_rate": 8.232468292269479e-08, + "logits/chosen": 0.7036927342414856, + "logits/rejected": 0.6851463317871094, + "logps/chosen": -215.97097778320312, + "logps/ref_chosen": -68.65119934082031, + "logps/ref_rejected": -77.91394805908203, + "logps/rejected": -315.40875244140625, + "loss": 1.0149, + "margin_dpo/margin_mean": 90.17504119873047, + "margin_dpo/margin_std": 97.8731689453125, + "step": 504 + }, + { + "KL/chosen_KL_mean": -155.90220642089844, + "KL/mean": -186.30569458007812, + "KL/rejected_KL_mean": -216.70916748046875, + "KL/std": 111.57499694824219, + "epoch": 0.763416477702192, + "fcm_dpo/beta": 0.005609945394098759, + "fcm_dpo/delta": 0.060340505093336105, + "fcm_dpo/margin": 60.806968688964844, + "fcm_dpo/q_t": 0.42532825469970703, + "grad_norm": 13.912813186645508, + "learning_rate": 8.134630621352483e-08, + "logits/chosen": 0.7275354266166687, + "logits/rejected": 0.6875624656677246, + "logps/chosen": -215.90106201171875, + "logps/ref_chosen": -59.99884796142578, + "logps/ref_rejected": -76.88048553466797, + "logps/rejected": -293.58966064453125, + "loss": 1.1904, + "margin_dpo/margin_mean": 60.806968688964844, + "margin_dpo/margin_std": 123.60678100585938, + "step": 505 + }, + { + "KL/chosen_KL_mean": -154.84149169921875, + "KL/mean": -187.8001708984375, + "KL/rejected_KL_mean": -220.75881958007812, + "KL/std": 99.20973205566406, + "epoch": 0.764928193499622, + "fcm_dpo/beta": 0.005677978508174419, + "fcm_dpo/delta": 0.026687901467084885, + "fcm_dpo/margin": 65.91732025146484, + "fcm_dpo/q_t": 0.4146183431148529, + "grad_norm": 14.703615188598633, + "learning_rate": 8.037264711071698e-08, + "logits/chosen": 0.6911933422088623, + "logits/rejected": 0.6756948232650757, + "logps/chosen": -224.9127960205078, + "logps/ref_chosen": -70.07130432128906, + "logps/ref_rejected": -82.03775024414062, + "logps/rejected": -302.79656982421875, + "loss": 1.1628, + "margin_dpo/margin_mean": 65.91732025146484, + "margin_dpo/margin_std": 123.7665023803711, + "step": 506 + }, + { + "KL/chosen_KL_mean": -166.98780822753906, + "KL/mean": -203.90573120117188, + "KL/rejected_KL_mean": -240.82363891601562, + "KL/std": 111.98727416992188, + "epoch": 0.7664399092970522, + "fcm_dpo/beta": 0.0056532504968345165, + "fcm_dpo/delta": -0.01878700777888298, + "fcm_dpo/margin": 73.83582305908203, + "fcm_dpo/q_t": 0.41168636083602905, + "grad_norm": 14.66965389251709, + "learning_rate": 7.940373284960933e-08, + "logits/chosen": 0.6715726852416992, + "logits/rejected": 0.6238397359848022, + "logps/chosen": -238.99484252929688, + "logps/ref_chosen": -72.00703430175781, + "logps/ref_rejected": -93.94987487792969, + "logps/rejected": -334.77349853515625, + "loss": 1.1429, + "margin_dpo/margin_mean": 73.83582305908203, + "margin_dpo/margin_std": 132.9780731201172, + "step": 507 + }, + { + "KL/chosen_KL_mean": -153.32525634765625, + "KL/mean": -196.36053466796875, + "KL/rejected_KL_mean": -239.39578247070312, + "KL/std": 116.38261413574219, + "epoch": 0.7679516250944822, + "fcm_dpo/beta": 0.005641060881316662, + "fcm_dpo/delta": -0.09029045701026917, + "fcm_dpo/margin": 86.07052612304688, + "fcm_dpo/q_t": 0.3928912281990051, + "grad_norm": 16.511768341064453, + "learning_rate": 7.843959053281663e-08, + "logits/chosen": 0.660283625125885, + "logits/rejected": 0.5391038656234741, + "logps/chosen": -213.54518127441406, + "logps/ref_chosen": -60.21992492675781, + "logps/ref_rejected": -95.9200668334961, + "logps/rejected": -335.31585693359375, + "loss": 1.0785, + "margin_dpo/margin_mean": 86.07052612304688, + "margin_dpo/margin_std": 125.32095336914062, + "step": 508 + }, + { + "KL/chosen_KL_mean": -163.14154052734375, + "KL/mean": -195.06103515625, + "KL/rejected_KL_mean": -226.98052978515625, + "KL/std": 103.42306518554688, + "epoch": 0.7694633408919124, + "fcm_dpo/beta": 0.0055953748524188995, + "fcm_dpo/delta": 0.04433656483888626, + "fcm_dpo/margin": 63.8390007019043, + "fcm_dpo/q_t": 0.41883113980293274, + "grad_norm": 16.704587936401367, + "learning_rate": 7.748024712947204e-08, + "logits/chosen": 0.6394075155258179, + "logits/rejected": 0.6166356205940247, + "logps/chosen": -229.4116973876953, + "logps/ref_chosen": -66.27017211914062, + "logps/ref_rejected": -71.73065185546875, + "logps/rejected": -298.711181640625, + "loss": 1.1671, + "margin_dpo/margin_mean": 63.83899688720703, + "margin_dpo/margin_std": 119.53250122070312, + "step": 509 + }, + { + "KL/chosen_KL_mean": -161.60655212402344, + "KL/mean": -203.13580322265625, + "KL/rejected_KL_mean": -244.6650390625, + "KL/std": 113.58114624023438, + "epoch": 0.7709750566893424, + "fcm_dpo/beta": 0.005545272957533598, + "fcm_dpo/delta": -0.06424017250537872, + "fcm_dpo/margin": 83.0584716796875, + "fcm_dpo/q_t": 0.4023531377315521, + "grad_norm": 14.228266716003418, + "learning_rate": 7.652572947447272e-08, + "logits/chosen": 0.7977691888809204, + "logits/rejected": 0.7011072039604187, + "logps/chosen": -215.15142822265625, + "logps/ref_chosen": -53.54487609863281, + "logps/ref_rejected": -91.36648559570312, + "logps/rejected": -336.0315246582031, + "loss": 1.1203, + "margin_dpo/margin_mean": 83.0584716796875, + "margin_dpo/margin_std": 144.74627685546875, + "step": 510 + }, + { + "KL/chosen_KL_mean": -152.7357177734375, + "KL/mean": -199.5233154296875, + "KL/rejected_KL_mean": -246.31092834472656, + "KL/std": 102.5706787109375, + "epoch": 0.7724867724867724, + "fcm_dpo/beta": 0.005463335197418928, + "fcm_dpo/delta": -0.11711655557155609, + "fcm_dpo/margin": 93.5752182006836, + "fcm_dpo/q_t": 0.38272392749786377, + "grad_norm": 17.340545654296875, + "learning_rate": 7.557606426772961e-08, + "logits/chosen": 0.7229855060577393, + "logits/rejected": 0.6641731262207031, + "logps/chosen": -208.58009338378906, + "logps/ref_chosen": -55.844383239746094, + "logps/ref_rejected": -86.49819946289062, + "logps/rejected": -332.80914306640625, + "loss": 1.0249, + "margin_dpo/margin_mean": 93.57521057128906, + "margin_dpo/margin_std": 109.5538330078125, + "step": 511 + }, + { + "KL/chosen_KL_mean": -158.43212890625, + "KL/mean": -187.49783325195312, + "KL/rejected_KL_mean": -216.56353759765625, + "KL/std": 95.57821655273438, + "epoch": 0.7739984882842026, + "fcm_dpo/beta": 0.005476620979607105, + "fcm_dpo/delta": 0.08439172059297562, + "fcm_dpo/margin": 58.131404876708984, + "fcm_dpo/q_t": 0.4264548420906067, + "grad_norm": 19.954294204711914, + "learning_rate": 7.463127807341966e-08, + "logits/chosen": 0.5845282673835754, + "logits/rejected": 0.5833408236503601, + "logps/chosen": -220.08517456054688, + "logps/ref_chosen": -61.653038024902344, + "logps/ref_rejected": -72.83148193359375, + "logps/rejected": -289.39501953125, + "loss": 1.1921, + "margin_dpo/margin_mean": 58.131404876708984, + "margin_dpo/margin_std": 116.42222595214844, + "step": 512 + }, + { + "KL/chosen_KL_mean": -141.0041046142578, + "KL/mean": -178.875244140625, + "KL/rejected_KL_mean": -216.7464141845703, + "KL/std": 98.06171417236328, + "epoch": 0.7755102040816326, + "fcm_dpo/beta": 0.005491101182997227, + "fcm_dpo/delta": -0.016725288704037666, + "fcm_dpo/margin": 75.74229431152344, + "fcm_dpo/q_t": 0.40563303232192993, + "grad_norm": 11.389507293701172, + "learning_rate": 7.369139731924401e-08, + "logits/chosen": 0.8551111221313477, + "logits/rejected": 0.7970899343490601, + "logps/chosen": -191.85665893554688, + "logps/ref_chosen": -50.85256576538086, + "logps/ref_rejected": -69.21754455566406, + "logps/rejected": -285.9639587402344, + "loss": 1.0901, + "margin_dpo/margin_mean": 75.74230194091797, + "margin_dpo/margin_std": 104.52362060546875, + "step": 513 + }, + { + "KL/chosen_KL_mean": -157.754150390625, + "KL/mean": -200.53826904296875, + "KL/rejected_KL_mean": -243.3223876953125, + "KL/std": 106.9458999633789, + "epoch": 0.7770219198790628, + "fcm_dpo/beta": 0.005463584326207638, + "fcm_dpo/delta": -0.0708877444267273, + "fcm_dpo/margin": 85.56825256347656, + "fcm_dpo/q_t": 0.3941824734210968, + "grad_norm": 15.855218887329102, + "learning_rate": 7.275644829568747e-08, + "logits/chosen": 0.7480206489562988, + "logits/rejected": 0.7150290608406067, + "logps/chosen": -227.13906860351562, + "logps/ref_chosen": -69.38493347167969, + "logps/ref_rejected": -83.32447814941406, + "logps/rejected": -326.6468505859375, + "loss": 1.0743, + "margin_dpo/margin_mean": 85.56825256347656, + "margin_dpo/margin_std": 121.04784393310547, + "step": 514 + }, + { + "KL/chosen_KL_mean": -165.98629760742188, + "KL/mean": -199.73580932617188, + "KL/rejected_KL_mean": -233.4853515625, + "KL/std": 100.55045318603516, + "epoch": 0.7785336356764928, + "fcm_dpo/beta": 0.005442567635327578, + "fcm_dpo/delta": 0.033889930695295334, + "fcm_dpo/margin": 67.4990234375, + "fcm_dpo/q_t": 0.41551756858825684, + "grad_norm": 14.63206672668457, + "learning_rate": 7.182645715528435e-08, + "logits/chosen": 0.7209761142730713, + "logits/rejected": 0.6388729810714722, + "logps/chosen": -219.67333984375, + "logps/ref_chosen": -53.687034606933594, + "logps/ref_rejected": -83.59614562988281, + "logps/rejected": -317.08148193359375, + "loss": 1.1453, + "margin_dpo/margin_mean": 67.4990234375, + "margin_dpo/margin_std": 115.27774047851562, + "step": 515 + }, + { + "KL/chosen_KL_mean": -143.6598358154297, + "KL/mean": -175.19981384277344, + "KL/rejected_KL_mean": -206.7397918701172, + "KL/std": 99.82717895507812, + "epoch": 0.780045351473923, + "fcm_dpo/beta": 0.005517736077308655, + "fcm_dpo/delta": 0.05346924811601639, + "fcm_dpo/margin": 63.079952239990234, + "fcm_dpo/q_t": 0.41942089796066284, + "grad_norm": 15.73136043548584, + "learning_rate": 7.090144991188568e-08, + "logits/chosen": 0.710132896900177, + "logits/rejected": 0.6714646816253662, + "logps/chosen": -200.56155395507812, + "logps/ref_chosen": -56.9017219543457, + "logps/ref_rejected": -67.83477783203125, + "logps/rejected": -274.5745849609375, + "loss": 1.1707, + "margin_dpo/margin_mean": 63.079952239990234, + "margin_dpo/margin_std": 118.81394958496094, + "step": 516 + }, + { + "KL/chosen_KL_mean": -171.2911376953125, + "KL/mean": -196.329345703125, + "KL/rejected_KL_mean": -221.36752319335938, + "KL/std": 105.70317077636719, + "epoch": 0.781557067271353, + "fcm_dpo/beta": 0.005636849440634251, + "fcm_dpo/delta": 0.1204327642917633, + "fcm_dpo/margin": 50.076385498046875, + "fcm_dpo/q_t": 0.43720489740371704, + "grad_norm": 15.20931625366211, + "learning_rate": 6.998145243993284e-08, + "logits/chosen": 0.7693038582801819, + "logits/rejected": 0.7707263827323914, + "logps/chosen": -233.0662841796875, + "logps/ref_chosen": -61.775142669677734, + "logps/ref_rejected": -62.88270950317383, + "logps/rejected": -284.250244140625, + "loss": 1.2151, + "margin_dpo/margin_mean": 50.076385498046875, + "margin_dpo/margin_std": 110.11534118652344, + "step": 517 + }, + { + "KL/chosen_KL_mean": -145.9417724609375, + "KL/mean": -181.2034912109375, + "KL/rejected_KL_mean": -216.46519470214844, + "KL/std": 102.95506286621094, + "epoch": 0.783068783068783, + "fcm_dpo/beta": 0.005652305670082569, + "fcm_dpo/delta": 0.0014371161814779043, + "fcm_dpo/margin": 70.5234146118164, + "fcm_dpo/q_t": 0.4121934175491333, + "grad_norm": 12.902327537536621, + "learning_rate": 6.906649047373245e-08, + "logits/chosen": 0.6915863752365112, + "logits/rejected": 0.6487727165222168, + "logps/chosen": -207.96701049804688, + "logps/ref_chosen": -62.02523422241211, + "logps/ref_rejected": -79.06085205078125, + "logps/rejected": -295.52606201171875, + "loss": 1.1253, + "margin_dpo/margin_mean": 70.52342224121094, + "margin_dpo/margin_std": 116.34848022460938, + "step": 518 + }, + { + "KL/chosen_KL_mean": -176.3029022216797, + "KL/mean": -196.67068481445312, + "KL/rejected_KL_mean": -217.03846740722656, + "KL/std": 105.22140502929688, + "epoch": 0.7845804988662132, + "fcm_dpo/beta": 0.005807607434689999, + "fcm_dpo/delta": 0.16692955791950226, + "fcm_dpo/margin": 40.735565185546875, + "fcm_dpo/q_t": 0.4469537138938904, + "grad_norm": 22.4091854095459, + "learning_rate": 6.815658960673781e-08, + "logits/chosen": 0.7130922675132751, + "logits/rejected": 0.6659531593322754, + "logps/chosen": -237.90927124023438, + "logps/ref_chosen": -61.60636901855469, + "logps/ref_rejected": -74.50727844238281, + "logps/rejected": -291.5457458496094, + "loss": 1.3159, + "margin_dpo/margin_mean": 40.735565185546875, + "margin_dpo/margin_std": 138.74868774414062, + "step": 519 + }, + { + "KL/chosen_KL_mean": -162.3382110595703, + "KL/mean": -193.408447265625, + "KL/rejected_KL_mean": -224.47869873046875, + "KL/std": 105.3907470703125, + "epoch": 0.7860922146636432, + "fcm_dpo/beta": 0.005840703845024109, + "fcm_dpo/delta": 0.03788800537586212, + "fcm_dpo/margin": 62.14047622680664, + "fcm_dpo/q_t": 0.4192347824573517, + "grad_norm": 15.117642402648926, + "learning_rate": 6.725177529083209e-08, + "logits/chosen": 0.7774897813796997, + "logits/rejected": 0.720142662525177, + "logps/chosen": -225.21163940429688, + "logps/ref_chosen": -62.87343215942383, + "logps/ref_rejected": -76.505615234375, + "logps/rejected": -300.98431396484375, + "loss": 1.1514, + "margin_dpo/margin_mean": 62.140480041503906, + "margin_dpo/margin_std": 108.19155883789062, + "step": 520 + }, + { + "KL/chosen_KL_mean": -160.02740478515625, + "KL/mean": -204.9112091064453, + "KL/rejected_KL_mean": -249.79498291015625, + "KL/std": 104.34115600585938, + "epoch": 0.7876039304610734, + "fcm_dpo/beta": 0.005790054798126221, + "fcm_dpo/delta": -0.12616199254989624, + "fcm_dpo/margin": 89.76756286621094, + "fcm_dpo/q_t": 0.3843619227409363, + "grad_norm": 12.815384864807129, + "learning_rate": 6.63520728356167e-08, + "logits/chosen": 0.5885381698608398, + "logits/rejected": 0.5134579539299011, + "logps/chosen": -224.23410034179688, + "logps/ref_chosen": -64.20668029785156, + "logps/ref_rejected": -92.28083038330078, + "logps/rejected": -342.0758056640625, + "loss": 1.0372, + "margin_dpo/margin_mean": 89.76756286621094, + "margin_dpo/margin_std": 118.39604187011719, + "step": 521 + }, + { + "KL/chosen_KL_mean": -160.57598876953125, + "KL/mean": -187.0106658935547, + "KL/rejected_KL_mean": -213.4453582763672, + "KL/std": 104.2094955444336, + "epoch": 0.7891156462585034, + "fcm_dpo/beta": 0.005809691734611988, + "fcm_dpo/delta": 0.09589952230453491, + "fcm_dpo/margin": 52.869354248046875, + "fcm_dpo/q_t": 0.43191057443618774, + "grad_norm": 15.976807594299316, + "learning_rate": 6.545750740770336e-08, + "logits/chosen": 0.7014885544776917, + "logits/rejected": 0.6930610537528992, + "logps/chosen": -218.94570922851562, + "logps/ref_chosen": -58.369720458984375, + "logps/ref_rejected": -68.79248046875, + "logps/rejected": -282.23785400390625, + "loss": 1.2483, + "margin_dpo/margin_mean": 52.869354248046875, + "margin_dpo/margin_std": 136.038330078125, + "step": 522 + }, + { + "KL/chosen_KL_mean": -165.72467041015625, + "KL/mean": -197.77651977539062, + "KL/rejected_KL_mean": -229.82833862304688, + "KL/std": 105.67681121826172, + "epoch": 0.7906273620559335, + "fcm_dpo/beta": 0.005870272405445576, + "fcm_dpo/delta": 0.024627383798360825, + "fcm_dpo/margin": 64.10369873046875, + "fcm_dpo/q_t": 0.4130624532699585, + "grad_norm": 17.706695556640625, + "learning_rate": 6.456810403001012e-08, + "logits/chosen": 0.6971518397331238, + "logits/rejected": 0.5737546682357788, + "logps/chosen": -231.43789672851562, + "logps/ref_chosen": -65.71324157714844, + "logps/ref_rejected": -91.98896789550781, + "logps/rejected": -321.81732177734375, + "loss": 1.1611, + "margin_dpo/margin_mean": 64.10369873046875, + "margin_dpo/margin_std": 120.08074188232422, + "step": 523 + }, + { + "KL/chosen_KL_mean": -143.75306701660156, + "KL/mean": -177.9951171875, + "KL/rejected_KL_mean": -212.2371826171875, + "KL/std": 101.63809204101562, + "epoch": 0.7921390778533636, + "fcm_dpo/beta": 0.00589970126748085, + "fcm_dpo/delta": -0.00446426123380661, + "fcm_dpo/margin": 68.48410034179688, + "fcm_dpo/q_t": 0.40784668922424316, + "grad_norm": 15.014260292053223, + "learning_rate": 6.368388758106134e-08, + "logits/chosen": 0.6233705878257751, + "logits/rejected": 0.5993573665618896, + "logps/chosen": -220.10430908203125, + "logps/ref_chosen": -76.35124969482422, + "logps/ref_rejected": -89.96072387695312, + "logps/rejected": -302.1979064941406, + "loss": 1.1119, + "margin_dpo/margin_mean": 68.48410034179688, + "margin_dpo/margin_std": 104.5627212524414, + "step": 524 + }, + { + "KL/chosen_KL_mean": -163.55609130859375, + "KL/mean": -190.743408203125, + "KL/rejected_KL_mean": -217.9307098388672, + "KL/std": 100.99457550048828, + "epoch": 0.7936507936507936, + "fcm_dpo/beta": 0.00594139751046896, + "fcm_dpo/delta": 0.07956387102603912, + "fcm_dpo/margin": 54.37462615966797, + "fcm_dpo/q_t": 0.42932847142219543, + "grad_norm": 19.195297241210938, + "learning_rate": 6.280488279429185e-08, + "logits/chosen": 0.5288229584693909, + "logits/rejected": 0.5252171754837036, + "logps/chosen": -239.0518798828125, + "logps/ref_chosen": -75.49578857421875, + "logps/ref_rejected": -84.04852294921875, + "logps/rejected": -301.979248046875, + "loss": 1.209, + "margin_dpo/margin_mean": 54.37462615966797, + "margin_dpo/margin_std": 121.72251892089844, + "step": 525 + }, + { + "KL/chosen_KL_mean": -170.69439697265625, + "KL/mean": -195.64962768554688, + "KL/rejected_KL_mean": -220.6048583984375, + "KL/std": 103.0473403930664, + "epoch": 0.7951625094482238, + "fcm_dpo/beta": 0.006015341263264418, + "fcm_dpo/delta": 0.10238677263259888, + "fcm_dpo/margin": 49.910484313964844, + "fcm_dpo/q_t": 0.4324687719345093, + "grad_norm": 15.048983573913574, + "learning_rate": 6.193111425735515e-08, + "logits/chosen": 0.7008933424949646, + "logits/rejected": 0.6313944458961487, + "logps/chosen": -231.98681640625, + "logps/ref_chosen": -61.29241943359375, + "logps/ref_rejected": -82.47763061523438, + "logps/rejected": -303.0824890136719, + "loss": 1.211, + "margin_dpo/margin_mean": 49.910484313964844, + "margin_dpo/margin_std": 105.97373962402344, + "step": 526 + }, + { + "KL/chosen_KL_mean": -178.74819946289062, + "KL/mean": -200.55226135253906, + "KL/rejected_KL_mean": -222.35635375976562, + "KL/std": 101.82447814941406, + "epoch": 0.7966742252456538, + "fcm_dpo/beta": 0.006244257092475891, + "fcm_dpo/delta": 0.13016949594020844, + "fcm_dpo/margin": 43.60816192626953, + "fcm_dpo/q_t": 0.4404991865158081, + "grad_norm": 17.9735050201416, + "learning_rate": 6.106260641143546e-08, + "logits/chosen": 0.7643716335296631, + "logits/rejected": 0.6801888346672058, + "logps/chosen": -240.2208251953125, + "logps/ref_chosen": -61.472625732421875, + "logps/ref_rejected": -90.52831268310547, + "logps/rejected": -312.8846435546875, + "loss": 1.2555, + "margin_dpo/margin_mean": 43.60816192626953, + "margin_dpo/margin_std": 115.52593994140625, + "step": 527 + }, + { + "KL/chosen_KL_mean": -157.19871520996094, + "KL/mean": -182.855224609375, + "KL/rejected_KL_mean": -208.51173400878906, + "KL/std": 98.76786804199219, + "epoch": 0.7981859410430839, + "fcm_dpo/beta": 0.006307326257228851, + "fcm_dpo/delta": 0.07893209159374237, + "fcm_dpo/margin": 51.313026428222656, + "fcm_dpo/q_t": 0.42957398295402527, + "grad_norm": 18.043712615966797, + "learning_rate": 6.019938355056422e-08, + "logits/chosen": 0.6514958143234253, + "logits/rejected": 0.5767842531204224, + "logps/chosen": -215.99073791503906, + "logps/ref_chosen": -58.792015075683594, + "logps/ref_rejected": -71.82516479492188, + "logps/rejected": -280.3369140625, + "loss": 1.2402, + "margin_dpo/margin_mean": 51.313026428222656, + "margin_dpo/margin_std": 126.47222900390625, + "step": 528 + }, + { + "KL/chosen_KL_mean": -150.249267578125, + "KL/mean": -200.34048461914062, + "KL/rejected_KL_mean": -250.43173217773438, + "KL/std": 97.72628021240234, + "epoch": 0.799697656840514, + "fcm_dpo/beta": 0.006136591546237469, + "fcm_dpo/delta": -0.2290957272052765, + "fcm_dpo/margin": 100.18245697021484, + "fcm_dpo/q_t": 0.36025696992874146, + "grad_norm": 17.034433364868164, + "learning_rate": 5.934146982094049e-08, + "logits/chosen": 0.6221810579299927, + "logits/rejected": 0.5700123310089111, + "logps/chosen": -205.3202362060547, + "logps/ref_chosen": -55.070960998535156, + "logps/ref_rejected": -75.44007873535156, + "logps/rejected": -325.8717956542969, + "loss": 0.9612, + "margin_dpo/margin_mean": 100.18244934082031, + "margin_dpo/margin_std": 106.74053192138672, + "step": 529 + }, + { + "KL/chosen_KL_mean": -152.41957092285156, + "KL/mean": -184.40634155273438, + "KL/rejected_KL_mean": -216.39312744140625, + "KL/std": 104.05105590820312, + "epoch": 0.8012093726379441, + "fcm_dpo/beta": 0.006076472345739603, + "fcm_dpo/delta": 0.011720176786184311, + "fcm_dpo/margin": 63.97355270385742, + "fcm_dpo/q_t": 0.41505563259124756, + "grad_norm": 19.411815643310547, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": 0.742800235748291, + "logits/rejected": 0.6958895921707153, + "logps/chosen": -209.16339111328125, + "logps/ref_chosen": -56.743812561035156, + "logps/ref_rejected": -76.6692123413086, + "logps/rejected": -293.06231689453125, + "loss": 1.1371, + "margin_dpo/margin_mean": 63.97355651855469, + "margin_dpo/margin_std": 108.51207733154297, + "step": 530 + }, + { + "KL/chosen_KL_mean": -151.02877807617188, + "KL/mean": -184.73265075683594, + "KL/rejected_KL_mean": -218.4365234375, + "KL/std": 102.89830017089844, + "epoch": 0.8027210884353742, + "fcm_dpo/beta": 0.00608763936907053, + "fcm_dpo/delta": -0.010867763310670853, + "fcm_dpo/margin": 67.40776062011719, + "fcm_dpo/q_t": 0.40898647904396057, + "grad_norm": 15.34810733795166, + "learning_rate": 5.7641665597021435e-08, + "logits/chosen": 0.6947374939918518, + "logits/rejected": 0.6132571697235107, + "logps/chosen": -202.14523315429688, + "logps/ref_chosen": -51.116455078125, + "logps/ref_rejected": -79.52884674072266, + "logps/rejected": -297.9653625488281, + "loss": 1.1247, + "margin_dpo/margin_mean": 67.40776824951172, + "margin_dpo/margin_std": 111.83642578125, + "step": 531 + }, + { + "KL/chosen_KL_mean": -175.77328491210938, + "KL/mean": -211.45449829101562, + "KL/rejected_KL_mean": -247.13571166992188, + "KL/std": 101.09457397460938, + "epoch": 0.8042328042328042, + "fcm_dpo/beta": 0.006043744273483753, + "fcm_dpo/delta": -0.03269847482442856, + "fcm_dpo/margin": 71.36245727539062, + "fcm_dpo/q_t": 0.40316763520240784, + "grad_norm": 15.20186710357666, + "learning_rate": 5.679982264990424e-08, + "logits/chosen": 0.6399841904640198, + "logits/rejected": 0.5887705087661743, + "logps/chosen": -234.05322265625, + "logps/ref_chosen": -58.279945373535156, + "logps/ref_rejected": -78.05426788330078, + "logps/rejected": -325.19000244140625, + "loss": 1.1102, + "margin_dpo/margin_mean": 71.36245727539062, + "margin_dpo/margin_std": 113.26620483398438, + "step": 532 + }, + { + "KL/chosen_KL_mean": -125.95477294921875, + "KL/mean": -164.83905029296875, + "KL/rejected_KL_mean": -203.7233123779297, + "KL/std": 106.76790618896484, + "epoch": 0.8057445200302343, + "fcm_dpo/beta": 0.005985047202557325, + "fcm_dpo/delta": -0.06859785318374634, + "fcm_dpo/margin": 77.76854705810547, + "fcm_dpo/q_t": 0.39635133743286133, + "grad_norm": 14.38045883178711, + "learning_rate": 5.596338392706076e-08, + "logits/chosen": 0.8273634314537048, + "logits/rejected": 0.7599341869354248, + "logps/chosen": -182.37278747558594, + "logps/ref_chosen": -56.41801071166992, + "logps/ref_rejected": -73.89324951171875, + "logps/rejected": -277.6165771484375, + "loss": 1.0808, + "margin_dpo/margin_mean": 77.76854705810547, + "margin_dpo/margin_std": 113.24247741699219, + "step": 533 + }, + { + "KL/chosen_KL_mean": -157.62551879882812, + "KL/mean": -189.51998901367188, + "KL/rejected_KL_mean": -221.4144287109375, + "KL/std": 99.86007690429688, + "epoch": 0.8072562358276644, + "fcm_dpo/beta": 0.005947187077254057, + "fcm_dpo/delta": 0.02126619592308998, + "fcm_dpo/margin": 63.78891372680664, + "fcm_dpo/q_t": 0.41670671105384827, + "grad_norm": 14.632553100585938, + "learning_rate": 5.513237282548033e-08, + "logits/chosen": 0.6824707984924316, + "logits/rejected": 0.6461671590805054, + "logps/chosen": -218.37420654296875, + "logps/ref_chosen": -60.748687744140625, + "logps/ref_rejected": -73.8623046875, + "logps/rejected": -295.2767333984375, + "loss": 1.1674, + "margin_dpo/margin_mean": 63.78891372680664, + "margin_dpo/margin_std": 123.30252075195312, + "step": 534 + }, + { + "KL/chosen_KL_mean": -166.86672973632812, + "KL/mean": -197.24063110351562, + "KL/rejected_KL_mean": -227.61453247070312, + "KL/std": 106.24088287353516, + "epoch": 0.8087679516250945, + "fcm_dpo/beta": 0.006018165964633226, + "fcm_dpo/delta": 0.035570912063121796, + "fcm_dpo/margin": 60.74782943725586, + "fcm_dpo/q_t": 0.41900062561035156, + "grad_norm": 15.982062339782715, + "learning_rate": 5.430681259032957e-08, + "logits/chosen": 0.6182019710540771, + "logits/rejected": 0.557220995426178, + "logps/chosen": -228.504150390625, + "logps/ref_chosen": -61.637413024902344, + "logps/ref_rejected": -80.93138885498047, + "logps/rejected": -308.5459289550781, + "loss": 1.1686, + "margin_dpo/margin_mean": 60.74782943725586, + "margin_dpo/margin_std": 115.64102172851562, + "step": 535 + }, + { + "KL/chosen_KL_mean": -145.83583068847656, + "KL/mean": -192.2183837890625, + "KL/rejected_KL_mean": -238.6009521484375, + "KL/std": 106.7518539428711, + "epoch": 0.8102796674225246, + "fcm_dpo/beta": 0.005869024433195591, + "fcm_dpo/delta": -0.1532009094953537, + "fcm_dpo/margin": 92.76512908935547, + "fcm_dpo/q_t": 0.3770672678947449, + "grad_norm": 12.833633422851562, + "learning_rate": 5.3486726314303175e-08, + "logits/chosen": 0.7487941384315491, + "logits/rejected": 0.662542462348938, + "logps/chosen": -197.7248077392578, + "logps/ref_chosen": -51.88897705078125, + "logps/ref_rejected": -73.34864044189453, + "logps/rejected": -311.9495849609375, + "loss": 0.9974, + "margin_dpo/margin_mean": 92.76513671875, + "margin_dpo/margin_std": 103.5233154296875, + "step": 536 + }, + { + "KL/chosen_KL_mean": -165.7203369140625, + "KL/mean": -204.83944702148438, + "KL/rejected_KL_mean": -243.9585418701172, + "KL/std": 113.77987670898438, + "epoch": 0.8117913832199547, + "fcm_dpo/beta": 0.00576906418427825, + "fcm_dpo/delta": -0.05421897768974304, + "fcm_dpo/margin": 78.23818969726562, + "fcm_dpo/q_t": 0.4007411599159241, + "grad_norm": 14.385953903198242, + "learning_rate": 5.267213693697695e-08, + "logits/chosen": 0.7646770477294922, + "logits/rejected": 0.6726725101470947, + "logps/chosen": -219.96896362304688, + "logps/ref_chosen": -54.248619079589844, + "logps/ref_rejected": -94.94343566894531, + "logps/rejected": -338.9019775390625, + "loss": 1.1018, + "margin_dpo/margin_mean": 78.23818969726562, + "margin_dpo/margin_std": 123.42788696289062, + "step": 537 + }, + { + "KL/chosen_KL_mean": -165.12217712402344, + "KL/mean": -204.04302978515625, + "KL/rejected_KL_mean": -242.96392822265625, + "KL/std": 109.24994659423828, + "epoch": 0.8133030990173847, + "fcm_dpo/beta": 0.0057380604557693005, + "fcm_dpo/delta": -0.04881321266293526, + "fcm_dpo/margin": 77.84176635742188, + "fcm_dpo/q_t": 0.39897143840789795, + "grad_norm": 14.58580493927002, + "learning_rate": 5.1863067244167144e-08, + "logits/chosen": 0.7028927206993103, + "logits/rejected": 0.6771712303161621, + "logps/chosen": -235.21571350097656, + "logps/ref_chosen": -70.09353637695312, + "logps/ref_rejected": -79.49833679199219, + "logps/rejected": -322.4622802734375, + "loss": 1.0787, + "margin_dpo/margin_mean": 77.84175872802734, + "margin_dpo/margin_std": 109.56571960449219, + "step": 538 + }, + { + "KL/chosen_KL_mean": -172.00189208984375, + "KL/mean": -201.82867431640625, + "KL/rejected_KL_mean": -231.6554718017578, + "KL/std": 103.12753295898438, + "epoch": 0.8148148148148148, + "fcm_dpo/beta": 0.005756544414907694, + "fcm_dpo/delta": 0.05865339934825897, + "fcm_dpo/margin": 59.653564453125, + "fcm_dpo/q_t": 0.4250602722167969, + "grad_norm": 15.257790565490723, + "learning_rate": 5.105953986729195e-08, + "logits/chosen": 0.6645068526268005, + "logits/rejected": 0.5794214606285095, + "logps/chosen": -233.93359375, + "logps/ref_chosen": -61.93169403076172, + "logps/ref_rejected": -84.08946228027344, + "logps/rejected": -315.74493408203125, + "loss": 1.1636, + "margin_dpo/margin_mean": 59.653564453125, + "margin_dpo/margin_std": 110.09439086914062, + "step": 539 + }, + { + "KL/chosen_KL_mean": -157.47537231445312, + "KL/mean": -203.33316040039062, + "KL/rejected_KL_mean": -249.19097900390625, + "KL/std": 116.36079406738281, + "epoch": 0.8163265306122449, + "fcm_dpo/beta": 0.005698896944522858, + "fcm_dpo/delta": -0.12926650047302246, + "fcm_dpo/margin": 91.71561431884766, + "fcm_dpo/q_t": 0.3814542591571808, + "grad_norm": 14.18432331085205, + "learning_rate": 5.026157728273966e-08, + "logits/chosen": 0.7328705787658691, + "logits/rejected": 0.6384468078613281, + "logps/chosen": -220.1796112060547, + "logps/ref_chosen": -62.704254150390625, + "logps/ref_rejected": -95.63597106933594, + "logps/rejected": -344.82696533203125, + "loss": 1.0221, + "margin_dpo/margin_mean": 91.71561431884766, + "margin_dpo/margin_std": 109.49424743652344, + "step": 540 + }, + { + "KL/chosen_KL_mean": -153.7994384765625, + "KL/mean": -191.71060180664062, + "KL/rejected_KL_mean": -229.62179565429688, + "KL/std": 102.88682556152344, + "epoch": 0.817838246409675, + "fcm_dpo/beta": 0.005589553155004978, + "fcm_dpo/delta": -0.025420255959033966, + "fcm_dpo/margin": 75.82234191894531, + "fcm_dpo/q_t": 0.4019586443901062, + "grad_norm": 13.555953025817871, + "learning_rate": 4.9469201811239035e-08, + "logits/chosen": 0.7742887139320374, + "logits/rejected": 0.8011547923088074, + "logps/chosen": -216.2802734375, + "logps/ref_chosen": -62.48084259033203, + "logps/ref_rejected": -57.55541229248047, + "logps/rejected": -287.17718505859375, + "loss": 1.0834, + "margin_dpo/margin_mean": 75.82233428955078, + "margin_dpo/margin_std": 100.54383850097656, + "step": 541 + }, + { + "KL/chosen_KL_mean": -133.67481994628906, + "KL/mean": -176.20965576171875, + "KL/rejected_KL_mean": -218.7445068359375, + "KL/std": 99.16780090332031, + "epoch": 0.8193499622071051, + "fcm_dpo/beta": 0.0055374205112457275, + "fcm_dpo/delta": -0.07474702596664429, + "fcm_dpo/margin": 85.06968688964844, + "fcm_dpo/q_t": 0.39279815554618835, + "grad_norm": 14.162556648254395, + "learning_rate": 4.868243561723534e-08, + "logits/chosen": 0.7516461610794067, + "logits/rejected": 0.7043957114219666, + "logps/chosen": -183.1297149658203, + "logps/ref_chosen": -49.454891204833984, + "logps/ref_rejected": -65.33275604248047, + "logps/rejected": -284.0772705078125, + "loss": 1.0731, + "margin_dpo/margin_mean": 85.06968688964844, + "margin_dpo/margin_std": 120.65644073486328, + "step": 542 + }, + { + "KL/chosen_KL_mean": -144.69039916992188, + "KL/mean": -186.76333618164062, + "KL/rejected_KL_mean": -228.83628845214844, + "KL/std": 102.19773864746094, + "epoch": 0.8208616780045351, + "fcm_dpo/beta": 0.005488088354468346, + "fcm_dpo/delta": -0.06477857381105423, + "fcm_dpo/margin": 84.1458969116211, + "fcm_dpo/q_t": 0.3943963348865509, + "grad_norm": 11.421889305114746, + "learning_rate": 4.790130070827028e-08, + "logits/chosen": 0.7144954204559326, + "logits/rejected": 0.6286009550094604, + "logps/chosen": -195.791259765625, + "logps/ref_chosen": -51.100860595703125, + "logps/ref_rejected": -76.06130981445312, + "logps/rejected": -304.8975830078125, + "loss": 1.0675, + "margin_dpo/margin_mean": 84.14590454101562, + "margin_dpo/margin_std": 111.68165588378906, + "step": 543 + }, + { + "KL/chosen_KL_mean": -155.3533172607422, + "KL/mean": -202.8394012451172, + "KL/rejected_KL_mean": -250.32546997070312, + "KL/std": 111.51242065429688, + "epoch": 0.8223733938019653, + "fcm_dpo/beta": 0.005360930226743221, + "fcm_dpo/delta": -0.11500416696071625, + "fcm_dpo/margin": 94.97218322753906, + "fcm_dpo/q_t": 0.38585346937179565, + "grad_norm": 14.714296340942383, + "learning_rate": 4.7125818934366454e-08, + "logits/chosen": 0.6963378190994263, + "logits/rejected": 0.6182718276977539, + "logps/chosen": -215.63055419921875, + "logps/ref_chosen": -60.2772331237793, + "logps/ref_rejected": -88.40553283691406, + "logps/rejected": -338.73101806640625, + "loss": 1.0491, + "margin_dpo/margin_mean": 94.97218322753906, + "margin_dpo/margin_std": 128.6180877685547, + "step": 544 + }, + { + "KL/chosen_KL_mean": -170.68997192382812, + "KL/mean": -197.08816528320312, + "KL/rejected_KL_mean": -223.48635864257812, + "KL/std": 103.76513671875, + "epoch": 0.8238851095993953, + "fcm_dpo/beta": 0.005427246913313866, + "fcm_dpo/delta": 0.11671829223632812, + "fcm_dpo/margin": 52.796348571777344, + "fcm_dpo/q_t": 0.43493160605430603, + "grad_norm": 14.415922164916992, + "learning_rate": 4.635601198741607e-08, + "logits/chosen": 0.6525213718414307, + "logits/rejected": 0.5933715105056763, + "logps/chosen": -232.30523681640625, + "logps/ref_chosen": -61.61524963378906, + "logps/ref_rejected": -78.71266174316406, + "logps/rejected": -302.1990051269531, + "loss": 1.2087, + "margin_dpo/margin_mean": 52.796348571777344, + "margin_dpo/margin_std": 112.15919494628906, + "step": 545 + }, + { + "KL/chosen_KL_mean": -160.33554077148438, + "KL/mean": -190.79754638671875, + "KL/rejected_KL_mean": -221.25955200195312, + "KL/std": 99.437255859375, + "epoch": 0.8253968253968254, + "fcm_dpo/beta": 0.005504027009010315, + "fcm_dpo/delta": 0.0669669657945633, + "fcm_dpo/margin": 60.92401123046875, + "fcm_dpo/q_t": 0.4233905076980591, + "grad_norm": 17.242345809936523, + "learning_rate": 4.559190140057428e-08, + "logits/chosen": 0.7818703651428223, + "logits/rejected": 0.7760703563690186, + "logps/chosen": -219.6488037109375, + "logps/ref_chosen": -59.313262939453125, + "logps/ref_rejected": -64.73631286621094, + "logps/rejected": -285.995849609375, + "loss": 1.1879, + "margin_dpo/margin_mean": 60.92401123046875, + "margin_dpo/margin_std": 123.54962158203125, + "step": 546 + }, + { + "KL/chosen_KL_mean": -139.52874755859375, + "KL/mean": -184.1389617919922, + "KL/rejected_KL_mean": -228.74916076660156, + "KL/std": 104.88829040527344, + "epoch": 0.8269085411942555, + "fcm_dpo/beta": 0.005440082401037216, + "fcm_dpo/delta": -0.09014324843883514, + "fcm_dpo/margin": 89.22044372558594, + "fcm_dpo/q_t": 0.38920462131500244, + "grad_norm": 14.522475242614746, + "learning_rate": 4.483350854765672e-08, + "logits/chosen": 0.6299358606338501, + "logits/rejected": 0.5683769583702087, + "logps/chosen": -194.5054931640625, + "logps/ref_chosen": -54.97674560546875, + "logps/ref_rejected": -75.35922241210938, + "logps/rejected": -304.1083984375, + "loss": 1.0546, + "margin_dpo/margin_mean": 89.2204360961914, + "margin_dpo/margin_std": 118.66877746582031, + "step": 547 + }, + { + "KL/chosen_KL_mean": -165.9710693359375, + "KL/mean": -192.84939575195312, + "KL/rejected_KL_mean": -219.7277374267578, + "KL/std": 104.1871109008789, + "epoch": 0.8284202569916855, + "fcm_dpo/beta": 0.0055316779762506485, + "fcm_dpo/delta": 0.10533760488033295, + "fcm_dpo/margin": 53.756656646728516, + "fcm_dpo/q_t": 0.43300092220306396, + "grad_norm": 16.411643981933594, + "learning_rate": 4.4080854642541826e-08, + "logits/chosen": 0.6215442419052124, + "logits/rejected": 0.5646119117736816, + "logps/chosen": -229.18174743652344, + "logps/ref_chosen": -63.21067428588867, + "logps/ref_rejected": -81.23347473144531, + "logps/rejected": -300.9612121582031, + "loss": 1.2022, + "margin_dpo/margin_mean": 53.756656646728516, + "margin_dpo/margin_std": 111.9897232055664, + "step": 548 + }, + { + "KL/chosen_KL_mean": -162.5404815673828, + "KL/mean": -196.65106201171875, + "KL/rejected_KL_mean": -230.76162719726562, + "KL/std": 113.05363464355469, + "epoch": 0.8299319727891157, + "fcm_dpo/beta": 0.005542172584682703, + "fcm_dpo/delta": 0.02252175845205784, + "fcm_dpo/margin": 68.22113800048828, + "fcm_dpo/q_t": 0.41690272092819214, + "grad_norm": 16.461366653442383, + "learning_rate": 4.333396073857723e-08, + "logits/chosen": 0.7623779773712158, + "logits/rejected": 0.6952296495437622, + "logps/chosen": -226.81399536132812, + "logps/ref_chosen": -64.27351379394531, + "logps/ref_rejected": -92.31663513183594, + "logps/rejected": -323.0782470703125, + "loss": 1.1704, + "margin_dpo/margin_mean": 68.22114562988281, + "margin_dpo/margin_std": 133.75875854492188, + "step": 549 + }, + { + "KL/chosen_KL_mean": -168.4662322998047, + "KL/mean": -191.34432983398438, + "KL/rejected_KL_mean": -214.22242736816406, + "KL/std": 100.89875793457031, + "epoch": 0.8314436885865457, + "fcm_dpo/beta": 0.0056635793298482895, + "fcm_dpo/delta": 0.1448059380054474, + "fcm_dpo/margin": 45.75617980957031, + "fcm_dpo/q_t": 0.44192641973495483, + "grad_norm": 15.279605865478516, + "learning_rate": 4.259284772799099e-08, + "logits/chosen": 0.7094156742095947, + "logits/rejected": 0.6828978061676025, + "logps/chosen": -224.69667053222656, + "logps/ref_chosen": -56.230438232421875, + "logps/ref_rejected": -62.59788513183594, + "logps/rejected": -276.8203125, + "loss": 1.2405, + "margin_dpo/margin_mean": 45.75617599487305, + "margin_dpo/margin_std": 109.83035278320312, + "step": 550 + }, + { + "KL/chosen_KL_mean": -172.52247619628906, + "KL/mean": -201.90504455566406, + "KL/rejected_KL_mean": -231.28762817382812, + "KL/std": 108.69143676757812, + "epoch": 0.8329554043839759, + "fcm_dpo/beta": 0.005799220874905586, + "fcm_dpo/delta": 0.06108397990465164, + "fcm_dpo/margin": 58.76515197753906, + "fcm_dpo/q_t": 0.4258086681365967, + "grad_norm": 15.20448112487793, + "learning_rate": 4.1857536341307176e-08, + "logits/chosen": 0.7232894897460938, + "logits/rejected": 0.6899063587188721, + "logps/chosen": -240.26968383789062, + "logps/ref_chosen": -67.74720764160156, + "logps/ref_rejected": -87.04285430908203, + "logps/rejected": -318.3304748535156, + "loss": 1.1593, + "margin_dpo/margin_mean": 58.76515197753906, + "margin_dpo/margin_std": 106.95538330078125, + "step": 551 + }, + { + "KL/chosen_KL_mean": -158.82626342773438, + "KL/mean": -192.85626220703125, + "KL/rejected_KL_mean": -226.88626098632812, + "KL/std": 108.2763900756836, + "epoch": 0.8344671201814059, + "fcm_dpo/beta": 0.00583791546523571, + "fcm_dpo/delta": 0.002204202115535736, + "fcm_dpo/margin": 68.06000518798828, + "fcm_dpo/q_t": 0.4077278971672058, + "grad_norm": 16.67183494567871, + "learning_rate": 4.112804714676593e-08, + "logits/chosen": 0.6853049993515015, + "logits/rejected": 0.631269097328186, + "logps/chosen": -221.75253295898438, + "logps/ref_chosen": -62.92625427246094, + "logps/ref_rejected": -82.98365783691406, + "logps/rejected": -309.86993408203125, + "loss": 1.1141, + "margin_dpo/margin_mean": 68.06000518798828, + "margin_dpo/margin_std": 102.41045379638672, + "step": 552 + }, + { + "KL/chosen_KL_mean": -172.24127197265625, + "KL/mean": -205.81686401367188, + "KL/rejected_KL_mean": -239.3924560546875, + "KL/std": 105.17045593261719, + "epoch": 0.8359788359788359, + "fcm_dpo/beta": 0.005816556513309479, + "fcm_dpo/delta": 0.009793659672141075, + "fcm_dpo/margin": 67.15115356445312, + "fcm_dpo/q_t": 0.4150969982147217, + "grad_norm": 15.538284301757812, + "learning_rate": 4.0404400549748144e-08, + "logits/chosen": 0.6806514859199524, + "logits/rejected": 0.5783441066741943, + "logps/chosen": -228.27976989746094, + "logps/ref_chosen": -56.038490295410156, + "logps/ref_rejected": -84.48454284667969, + "logps/rejected": -323.8769836425781, + "loss": 1.1563, + "margin_dpo/margin_mean": 67.15115356445312, + "margin_dpo/margin_std": 126.45013427734375, + "step": 553 + }, + { + "KL/chosen_KL_mean": -155.46331787109375, + "KL/mean": -195.21212768554688, + "KL/rejected_KL_mean": -234.9609375, + "KL/std": 101.96823120117188, + "epoch": 0.8374905517762661, + "fcm_dpo/beta": 0.005765959620475769, + "fcm_dpo/delta": -0.06119035556912422, + "fcm_dpo/margin": 79.49761199951172, + "fcm_dpo/q_t": 0.39454004168510437, + "grad_norm": 14.239871978759766, + "learning_rate": 3.968661679220467e-08, + "logits/chosen": 0.6696711778640747, + "logits/rejected": 0.6575784683227539, + "logps/chosen": -219.99392700195312, + "logps/ref_chosen": -64.53059387207031, + "logps/ref_rejected": -71.2155990600586, + "logps/rejected": -306.1765441894531, + "loss": 1.0772, + "margin_dpo/margin_mean": 79.49761199951172, + "margin_dpo/margin_std": 111.55782318115234, + "step": 554 + }, + { + "KL/chosen_KL_mean": -170.21527099609375, + "KL/mean": -201.37120056152344, + "KL/rejected_KL_mean": -232.527099609375, + "KL/std": 102.17310333251953, + "epoch": 0.8390022675736961, + "fcm_dpo/beta": 0.00582374632358551, + "fcm_dpo/delta": 0.03709454834461212, + "fcm_dpo/margin": 62.311851501464844, + "fcm_dpo/q_t": 0.4159049093723297, + "grad_norm": 14.529513359069824, + "learning_rate": 3.89747159520904e-08, + "logits/chosen": 0.68208909034729, + "logits/rejected": 0.6605509519577026, + "logps/chosen": -236.8671875, + "logps/ref_chosen": -66.65191650390625, + "logps/ref_rejected": -68.6667251586914, + "logps/rejected": -301.19384765625, + "loss": 1.1831, + "margin_dpo/margin_mean": 62.31185531616211, + "margin_dpo/margin_std": 122.26577758789062, + "step": 555 + }, + { + "KL/chosen_KL_mean": -168.04849243164062, + "KL/mean": -197.58963012695312, + "KL/rejected_KL_mean": -227.13076782226562, + "KL/std": 106.78985595703125, + "epoch": 0.8405139833711263, + "fcm_dpo/beta": 0.0058286068961024284, + "fcm_dpo/delta": 0.057645734399557114, + "fcm_dpo/margin": 59.082279205322266, + "fcm_dpo/q_t": 0.42424604296684265, + "grad_norm": 15.20958423614502, + "learning_rate": 3.826871794280192e-08, + "logits/chosen": 0.7351358532905579, + "logits/rejected": 0.6876686811447144, + "logps/chosen": -220.880859375, + "logps/ref_chosen": -52.832366943359375, + "logps/ref_rejected": -64.49044036865234, + "logps/rejected": -291.6212158203125, + "loss": 1.1902, + "margin_dpo/margin_mean": 59.082279205322266, + "margin_dpo/margin_std": 121.4478759765625, + "step": 556 + }, + { + "KL/chosen_KL_mean": -166.92498779296875, + "KL/mean": -209.52706909179688, + "KL/rejected_KL_mean": -252.129150390625, + "KL/std": 104.20233154296875, + "epoch": 0.8420256991685563, + "fcm_dpo/beta": 0.005754388403147459, + "fcm_dpo/delta": -0.09558665752410889, + "fcm_dpo/margin": 85.20416259765625, + "fcm_dpo/q_t": 0.388553261756897, + "grad_norm": 11.867389678955078, + "learning_rate": 3.756864251262143e-08, + "logits/chosen": 0.8205718398094177, + "logits/rejected": 0.7426509857177734, + "logps/chosen": -221.96096801757812, + "logps/ref_chosen": -55.03598403930664, + "logps/ref_rejected": -75.80644989013672, + "logps/rejected": -327.93560791015625, + "loss": 1.0353, + "margin_dpo/margin_mean": 85.20416259765625, + "margin_dpo/margin_std": 100.88729095458984, + "step": 557 + }, + { + "KL/chosen_KL_mean": -154.03515625, + "KL/mean": -201.06326293945312, + "KL/rejected_KL_mean": -248.09136962890625, + "KL/std": 111.77014923095703, + "epoch": 0.8435374149659864, + "fcm_dpo/beta": 0.005608959123492241, + "fcm_dpo/delta": -0.13560570776462555, + "fcm_dpo/margin": 94.05621337890625, + "fcm_dpo/q_t": 0.38192081451416016, + "grad_norm": 11.089186668395996, + "learning_rate": 3.687450924416341e-08, + "logits/chosen": 0.7711827754974365, + "logits/rejected": 0.7159498929977417, + "logps/chosen": -217.26150512695312, + "logps/ref_chosen": -63.226348876953125, + "logps/ref_rejected": -91.46881866455078, + "logps/rejected": -339.5601806640625, + "loss": 1.0224, + "margin_dpo/margin_mean": 94.05621337890625, + "margin_dpo/margin_std": 115.32173156738281, + "step": 558 + }, + { + "KL/chosen_KL_mean": -156.6865234375, + "KL/mean": -198.32675170898438, + "KL/rejected_KL_mean": -239.96702575683594, + "KL/std": 109.83236694335938, + "epoch": 0.8450491307634165, + "fcm_dpo/beta": 0.005508118309080601, + "fcm_dpo/delta": -0.06268289685249329, + "fcm_dpo/margin": 83.280517578125, + "fcm_dpo/q_t": 0.4005870819091797, + "grad_norm": 11.729891777038574, + "learning_rate": 3.6186337553827743e-08, + "logits/chosen": 0.6707146167755127, + "logits/rejected": 0.6091177463531494, + "logps/chosen": -218.20816040039062, + "logps/ref_chosen": -61.521644592285156, + "logps/ref_rejected": -82.83859252929688, + "logps/rejected": -322.80560302734375, + "loss": 1.0981, + "margin_dpo/margin_mean": 83.280517578125, + "margin_dpo/margin_std": 130.9115753173828, + "step": 559 + }, + { + "KL/chosen_KL_mean": -171.60235595703125, + "KL/mean": -209.61856079101562, + "KL/rejected_KL_mean": -247.63478088378906, + "KL/std": 106.89784240722656, + "epoch": 0.8465608465608465, + "fcm_dpo/beta": 0.00552861113101244, + "fcm_dpo/delta": -0.02149122953414917, + "fcm_dpo/margin": 76.03242492675781, + "fcm_dpo/q_t": 0.4038241505622864, + "grad_norm": 16.61585235595703, + "learning_rate": 3.550414669125573e-08, + "logits/chosen": 0.705360472202301, + "logits/rejected": 0.6663854122161865, + "logps/chosen": -232.24359130859375, + "logps/ref_chosen": -60.64122009277344, + "logps/ref_rejected": -78.75474548339844, + "logps/rejected": -326.3895263671875, + "loss": 1.0842, + "margin_dpo/margin_mean": 76.03242492675781, + "margin_dpo/margin_std": 100.37289428710938, + "step": 560 + }, + { + "KL/chosen_KL_mean": -156.08416748046875, + "KL/mean": -191.4915771484375, + "KL/rejected_KL_mean": -226.89898681640625, + "KL/std": 104.07613372802734, + "epoch": 0.8480725623582767, + "fcm_dpo/beta": 0.005500419996678829, + "fcm_dpo/delta": 0.010878851637244225, + "fcm_dpo/margin": 70.81483459472656, + "fcm_dpo/q_t": 0.41301605105400085, + "grad_norm": 12.828431129455566, + "learning_rate": 3.482795573879241e-08, + "logits/chosen": 0.6696001291275024, + "logits/rejected": 0.6357676982879639, + "logps/chosen": -218.582763671875, + "logps/ref_chosen": -62.49859619140625, + "logps/ref_rejected": -78.72064208984375, + "logps/rejected": -305.61962890625, + "loss": 1.1241, + "margin_dpo/margin_mean": 70.81483459472656, + "margin_dpo/margin_std": 112.45053100585938, + "step": 561 + }, + { + "KL/chosen_KL_mean": -167.10365295410156, + "KL/mean": -212.86196899414062, + "KL/rejected_KL_mean": -258.62030029296875, + "KL/std": 120.87919616699219, + "epoch": 0.8495842781557067, + "fcm_dpo/beta": 0.005394667387008667, + "fcm_dpo/delta": -0.10014957934617996, + "fcm_dpo/margin": 91.51663970947266, + "fcm_dpo/q_t": 0.3899437189102173, + "grad_norm": 16.289958953857422, + "learning_rate": 3.415778361095226e-08, + "logits/chosen": 0.6985616683959961, + "logits/rejected": 0.6643628478050232, + "logps/chosen": -241.88539123535156, + "logps/ref_chosen": -74.78173828125, + "logps/ref_rejected": -92.63499450683594, + "logps/rejected": -351.25531005859375, + "loss": 1.0459, + "margin_dpo/margin_mean": 91.51663970947266, + "margin_dpo/margin_std": 117.03136444091797, + "step": 562 + }, + { + "KL/chosen_KL_mean": -139.36734008789062, + "KL/mean": -178.2583770751953, + "KL/rejected_KL_mean": -217.1494140625, + "KL/std": 93.23200225830078, + "epoch": 0.8510959939531368, + "fcm_dpo/beta": 0.0053891572169959545, + "fcm_dpo/delta": -0.02001415193080902, + "fcm_dpo/margin": 77.78207397460938, + "fcm_dpo/q_t": 0.4036242961883545, + "grad_norm": 18.518709182739258, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": 0.7896828651428223, + "logits/rejected": 0.7366276979446411, + "logps/chosen": -189.5658416748047, + "logps/ref_chosen": -50.19850158691406, + "logps/ref_rejected": -66.76687622070312, + "logps/rejected": -283.9162902832031, + "loss": 1.1081, + "margin_dpo/margin_mean": 77.78207397460938, + "margin_dpo/margin_std": 119.42278289794922, + "step": 563 + }, + { + "KL/chosen_KL_mean": -142.17361450195312, + "KL/mean": -191.52587890625, + "KL/rejected_KL_mean": -240.87814331054688, + "KL/std": 101.90835571289062, + "epoch": 0.8526077097505669, + "fcm_dpo/beta": 0.0052757966332137585, + "fcm_dpo/delta": -0.1275056004524231, + "fcm_dpo/margin": 98.70454406738281, + "fcm_dpo/q_t": 0.3802381753921509, + "grad_norm": 12.03218936920166, + "learning_rate": 3.283557064487785e-08, + "logits/chosen": 0.679800271987915, + "logits/rejected": 0.6504189968109131, + "logps/chosen": -197.91445922851562, + "logps/ref_chosen": -55.7408447265625, + "logps/ref_rejected": -74.82323455810547, + "logps/rejected": -315.7013854980469, + "loss": 1.0277, + "margin_dpo/margin_mean": 98.70453643798828, + "margin_dpo/margin_std": 121.65130615234375, + "step": 564 + }, + { + "KL/chosen_KL_mean": -172.18453979492188, + "KL/mean": -207.91888427734375, + "KL/rejected_KL_mean": -243.65325927734375, + "KL/std": 105.38109588623047, + "epoch": 0.854119425547997, + "fcm_dpo/beta": 0.005283844657242298, + "fcm_dpo/delta": 0.02286720462143421, + "fcm_dpo/margin": 71.46871948242188, + "fcm_dpo/q_t": 0.41272348165512085, + "grad_norm": 14.575935363769531, + "learning_rate": 3.218356679178252e-08, + "logits/chosen": 0.7471562623977661, + "logits/rejected": 0.6933939456939697, + "logps/chosen": -230.52191162109375, + "logps/ref_chosen": -58.33738327026367, + "logps/ref_rejected": -78.31776428222656, + "logps/rejected": -321.97100830078125, + "loss": 1.1176, + "margin_dpo/margin_mean": 71.46871185302734, + "margin_dpo/margin_std": 105.04327392578125, + "step": 565 + }, + { + "KL/chosen_KL_mean": -162.09071350097656, + "KL/mean": -194.7822265625, + "KL/rejected_KL_mean": -227.47377014160156, + "KL/std": 108.80207824707031, + "epoch": 0.8556311413454271, + "fcm_dpo/beta": 0.0053364988416433334, + "fcm_dpo/delta": 0.05221934616565704, + "fcm_dpo/margin": 65.383056640625, + "fcm_dpo/q_t": 0.4226508140563965, + "grad_norm": 14.741025924682617, + "learning_rate": 3.1537655732553764e-08, + "logits/chosen": 0.7219111919403076, + "logits/rejected": 0.7103610634803772, + "logps/chosen": -233.314453125, + "logps/ref_chosen": -71.22373962402344, + "logps/ref_rejected": -71.11601257324219, + "logps/rejected": -298.58978271484375, + "loss": 1.1933, + "margin_dpo/margin_mean": 65.383056640625, + "margin_dpo/margin_std": 137.35252380371094, + "step": 566 + }, + { + "KL/chosen_KL_mean": -155.47418212890625, + "KL/mean": -196.4601593017578, + "KL/rejected_KL_mean": -237.4461212158203, + "KL/std": 102.49577331542969, + "epoch": 0.8571428571428571, + "fcm_dpo/beta": 0.0052682156674563885, + "fcm_dpo/delta": -0.034140560775995255, + "fcm_dpo/margin": 81.97193908691406, + "fcm_dpo/q_t": 0.4000481367111206, + "grad_norm": 11.802513122558594, + "learning_rate": 3.089785553471233e-08, + "logits/chosen": 0.7189158201217651, + "logits/rejected": 0.6299164295196533, + "logps/chosen": -208.14346313476562, + "logps/ref_chosen": -52.669273376464844, + "logps/ref_rejected": -74.34785461425781, + "logps/rejected": -311.7939758300781, + "loss": 1.0831, + "margin_dpo/margin_mean": 81.97193908691406, + "margin_dpo/margin_std": 110.23538208007812, + "step": 567 + }, + { + "KL/chosen_KL_mean": -142.93621826171875, + "KL/mean": -193.83016967773438, + "KL/rejected_KL_mean": -244.72412109375, + "KL/std": 109.18955993652344, + "epoch": 0.8586545729402872, + "fcm_dpo/beta": 0.005216483026742935, + "fcm_dpo/delta": -0.1383085399866104, + "fcm_dpo/margin": 101.78787231445312, + "fcm_dpo/q_t": 0.3798676133155823, + "grad_norm": 15.167051315307617, + "learning_rate": 3.026418409484513e-08, + "logits/chosen": 0.7353081107139587, + "logits/rejected": 0.6538002490997314, + "logps/chosen": -195.11422729492188, + "logps/ref_chosen": -52.178001403808594, + "logps/ref_rejected": -85.8277587890625, + "logps/rejected": -330.5518798828125, + "loss": 1.005, + "margin_dpo/margin_mean": 101.78787231445312, + "margin_dpo/margin_std": 109.98161315917969, + "step": 568 + }, + { + "KL/chosen_KL_mean": -168.43011474609375, + "KL/mean": -193.41647338867188, + "KL/rejected_KL_mean": -218.40283203125, + "KL/std": 110.10757446289062, + "epoch": 0.8601662887377173, + "fcm_dpo/beta": 0.0052167270332574844, + "fcm_dpo/delta": 0.14295698702335358, + "fcm_dpo/margin": 49.97270202636719, + "fcm_dpo/q_t": 0.4411056339740753, + "grad_norm": 14.588390350341797, + "learning_rate": 2.963665913810451e-08, + "logits/chosen": 0.6284128427505493, + "logits/rejected": 0.5956965684890747, + "logps/chosen": -231.07937622070312, + "logps/ref_chosen": -62.649261474609375, + "logps/ref_rejected": -75.4298324584961, + "logps/rejected": -293.8326721191406, + "loss": 1.2268, + "margin_dpo/margin_mean": 49.97270202636719, + "margin_dpo/margin_std": 109.44187927246094, + "step": 569 + }, + { + "KL/chosen_KL_mean": -147.50064086914062, + "KL/mean": -195.2564697265625, + "KL/rejected_KL_mean": -243.01231384277344, + "KL/std": 101.05863189697266, + "epoch": 0.8616780045351474, + "fcm_dpo/beta": 0.005238240119069815, + "fcm_dpo/delta": -0.1055883839726448, + "fcm_dpo/margin": 95.51168060302734, + "fcm_dpo/q_t": 0.38603097200393677, + "grad_norm": 13.221871376037598, + "learning_rate": 2.9015298217712453e-08, + "logits/chosen": 0.70135498046875, + "logits/rejected": 0.6157346963882446, + "logps/chosen": -197.54244995117188, + "logps/ref_chosen": -50.04179382324219, + "logps/ref_rejected": -78.27146911621094, + "logps/rejected": -321.2837829589844, + "loss": 1.0312, + "margin_dpo/margin_mean": 95.51167297363281, + "margin_dpo/margin_std": 113.90409851074219, + "step": 570 + }, + { + "KL/chosen_KL_mean": -161.2565155029297, + "KL/mean": -188.04335021972656, + "KL/rejected_KL_mean": -214.83018493652344, + "KL/std": 99.07657623291016, + "epoch": 0.8631897203325775, + "fcm_dpo/beta": 0.0052953860722482204, + "fcm_dpo/delta": 0.11900515854358673, + "fcm_dpo/margin": 53.57366943359375, + "fcm_dpo/q_t": 0.43476077914237976, + "grad_norm": 13.77877140045166, + "learning_rate": 2.840011871446962e-08, + "logits/chosen": 0.7282629609107971, + "logits/rejected": 0.7028162479400635, + "logps/chosen": -214.913330078125, + "logps/ref_chosen": -53.65681457519531, + "logps/ref_rejected": -66.13298034667969, + "logps/rejected": -280.9631652832031, + "loss": 1.2133, + "margin_dpo/margin_mean": 53.573665618896484, + "margin_dpo/margin_std": 114.42227172851562, + "step": 571 + }, + { + "KL/chosen_KL_mean": -161.99856567382812, + "KL/mean": -193.23593139648438, + "KL/rejected_KL_mean": -224.47332763671875, + "KL/std": 105.49080657958984, + "epoch": 0.8647014361300076, + "fcm_dpo/beta": 0.005384175106883049, + "fcm_dpo/delta": 0.06513378024101257, + "fcm_dpo/margin": 62.474761962890625, + "fcm_dpo/q_t": 0.42261409759521484, + "grad_norm": 14.15194034576416, + "learning_rate": 2.7791137836269158e-08, + "logits/chosen": 0.6913356781005859, + "logits/rejected": 0.733739972114563, + "logps/chosen": -236.8164825439453, + "logps/ref_chosen": -74.81792449951172, + "logps/ref_rejected": -65.88681030273438, + "logps/rejected": -290.3601379394531, + "loss": 1.1585, + "margin_dpo/margin_mean": 62.47476577758789, + "margin_dpo/margin_std": 106.80880737304688, + "step": 572 + }, + { + "KL/chosen_KL_mean": -176.1700897216797, + "KL/mean": -211.75979614257812, + "KL/rejected_KL_mean": -247.34950256347656, + "KL/std": 114.86283111572266, + "epoch": 0.8662131519274376, + "fcm_dpo/beta": 0.005393402185291052, + "fcm_dpo/delta": 0.016733139753341675, + "fcm_dpo/margin": 71.179443359375, + "fcm_dpo/q_t": 0.41635024547576904, + "grad_norm": 15.528578758239746, + "learning_rate": 2.718837261761528e-08, + "logits/chosen": 0.7083475589752197, + "logits/rejected": 0.6636322736740112, + "logps/chosen": -244.89572143554688, + "logps/ref_chosen": -68.72564697265625, + "logps/ref_rejected": -88.16201782226562, + "logps/rejected": -335.51153564453125, + "loss": 1.1631, + "margin_dpo/margin_mean": 71.179443359375, + "margin_dpo/margin_std": 137.14370727539062, + "step": 573 + }, + { + "KL/chosen_KL_mean": -156.84823608398438, + "KL/mean": -202.28956604003906, + "KL/rejected_KL_mean": -247.73086547851562, + "KL/std": 105.29086303710938, + "epoch": 0.8677248677248677, + "fcm_dpo/beta": 0.005354342050850391, + "fcm_dpo/delta": -0.09120815247297287, + "fcm_dpo/margin": 90.88264465332031, + "fcm_dpo/q_t": 0.3886836767196655, + "grad_norm": 11.730754852294922, + "learning_rate": 2.659183991914696e-08, + "logits/chosen": 0.7415429949760437, + "logits/rejected": 0.6785948276519775, + "logps/chosen": -213.16165161132812, + "logps/ref_chosen": -56.31340026855469, + "logps/ref_rejected": -83.91553497314453, + "logps/rejected": -331.64642333984375, + "loss": 1.0255, + "margin_dpo/margin_mean": 90.88263702392578, + "margin_dpo/margin_std": 99.63211059570312, + "step": 574 + }, + { + "KL/chosen_KL_mean": -161.31875610351562, + "KL/mean": -193.75637817382812, + "KL/rejected_KL_mean": -226.19400024414062, + "KL/std": 107.47755432128906, + "epoch": 0.8692365835222978, + "fcm_dpo/beta": 0.005301470868289471, + "fcm_dpo/delta": 0.05722144991159439, + "fcm_dpo/margin": 64.87521362304688, + "fcm_dpo/q_t": 0.4250301122665405, + "grad_norm": 13.5238676071167, + "learning_rate": 2.600155642716606e-08, + "logits/chosen": 0.7329334616661072, + "logits/rejected": 0.6541694402694702, + "logps/chosen": -225.90289306640625, + "logps/ref_chosen": -64.5841293334961, + "logps/ref_rejected": -93.47034454345703, + "logps/rejected": -319.6643371582031, + "loss": 1.1879, + "margin_dpo/margin_mean": 64.87521362304688, + "margin_dpo/margin_std": 128.93638610839844, + "step": 575 + }, + { + "KL/chosen_KL_mean": -143.23312377929688, + "KL/mean": -190.56591796875, + "KL/rejected_KL_mean": -237.89871215820312, + "KL/std": 107.76126861572266, + "epoch": 0.8707482993197279, + "fcm_dpo/beta": 0.005239981692284346, + "fcm_dpo/delta": -0.10286588966846466, + "fcm_dpo/margin": 94.66560363769531, + "fcm_dpo/q_t": 0.38725581765174866, + "grad_norm": 12.78931999206543, + "learning_rate": 2.5417538653170754e-08, + "logits/chosen": 0.716622531414032, + "logits/rejected": 0.6150184869766235, + "logps/chosen": -196.51364135742188, + "logps/ref_chosen": -53.28052520751953, + "logps/ref_rejected": -84.2000503540039, + "logps/rejected": -322.0987548828125, + "loss": 1.0446, + "margin_dpo/margin_mean": 94.66560363769531, + "margin_dpo/margin_std": 119.16376495361328, + "step": 576 + }, + { + "KL/chosen_KL_mean": -160.7834930419922, + "KL/mean": -191.5000762939453, + "KL/rejected_KL_mean": -222.2166748046875, + "KL/std": 105.21707916259766, + "epoch": 0.872260015117158, + "fcm_dpo/beta": 0.005305111408233643, + "fcm_dpo/delta": 0.07664323598146439, + "fcm_dpo/margin": 61.43318176269531, + "fcm_dpo/q_t": 0.42474716901779175, + "grad_norm": 13.350493431091309, + "learning_rate": 2.4839802933393607e-08, + "logits/chosen": 0.6843061447143555, + "logits/rejected": 0.6764715909957886, + "logps/chosen": -223.10818481445312, + "logps/ref_chosen": -62.32468795776367, + "logps/ref_rejected": -67.300537109375, + "logps/rejected": -289.5172119140625, + "loss": 1.1767, + "margin_dpo/margin_mean": 61.43318176269531, + "margin_dpo/margin_std": 112.92008972167969, + "step": 577 + }, + { + "KL/chosen_KL_mean": -151.0160369873047, + "KL/mean": -181.88204956054688, + "KL/rejected_KL_mean": -212.748046875, + "KL/std": 109.35968017578125, + "epoch": 0.873771730914588, + "fcm_dpo/beta": 0.005385764874517918, + "fcm_dpo/delta": 0.06986413896083832, + "fcm_dpo/margin": 61.73200225830078, + "fcm_dpo/q_t": 0.42697471380233765, + "grad_norm": 12.926276206970215, + "learning_rate": 2.4268365428344733e-08, + "logits/chosen": 0.7761894464492798, + "logits/rejected": 0.7603508234024048, + "logps/chosen": -207.67161560058594, + "logps/ref_chosen": -56.65557861328125, + "logps/ref_rejected": -68.21835327148438, + "logps/rejected": -280.9664001464844, + "loss": 1.1874, + "margin_dpo/margin_mean": 61.73200988769531, + "margin_dpo/margin_std": 125.3292236328125, + "step": 578 + }, + { + "KL/chosen_KL_mean": -157.9138946533203, + "KL/mean": -202.1259765625, + "KL/rejected_KL_mean": -246.3380584716797, + "KL/std": 101.47502136230469, + "epoch": 0.8752834467120182, + "fcm_dpo/beta": 0.005336810369044542, + "fcm_dpo/delta": -0.07564752548933029, + "fcm_dpo/margin": 88.4241714477539, + "fcm_dpo/q_t": 0.3900591731071472, + "grad_norm": 12.923798561096191, + "learning_rate": 2.3703242122359357e-08, + "logits/chosen": 0.6561988592147827, + "logits/rejected": 0.6358931064605713, + "logps/chosen": -214.72354125976562, + "logps/ref_chosen": -56.809661865234375, + "logps/ref_rejected": -68.09613037109375, + "logps/rejected": -314.4342041015625, + "loss": 1.0415, + "margin_dpo/margin_mean": 88.4241714477539, + "margin_dpo/margin_std": 104.05563354492188, + "step": 579 + }, + { + "KL/chosen_KL_mean": -158.97811889648438, + "KL/mean": -197.81932067871094, + "KL/rejected_KL_mean": -236.66053771972656, + "KL/std": 118.46055603027344, + "epoch": 0.8767951625094482, + "fcm_dpo/beta": 0.005337144713848829, + "fcm_dpo/delta": -0.015590623021125793, + "fcm_dpo/margin": 77.68240356445312, + "fcm_dpo/q_t": 0.4092617630958557, + "grad_norm": 14.614388465881348, + "learning_rate": 2.3144448823151392e-08, + "logits/chosen": 0.6761208772659302, + "logits/rejected": 0.6266993880271912, + "logps/chosen": -216.67822265625, + "logps/ref_chosen": -57.70011520385742, + "logps/ref_rejected": -77.90664672851562, + "logps/rejected": -314.56719970703125, + "loss": 1.1279, + "margin_dpo/margin_mean": 77.68241119384766, + "margin_dpo/margin_std": 131.71633911132812, + "step": 580 + }, + { + "KL/chosen_KL_mean": -174.16661071777344, + "KL/mean": -209.36288452148438, + "KL/rejected_KL_mean": -244.55918884277344, + "KL/std": 105.65506744384766, + "epoch": 0.8783068783068783, + "fcm_dpo/beta": 0.005308878142386675, + "fcm_dpo/delta": 0.02710052765905857, + "fcm_dpo/margin": 70.39259338378906, + "fcm_dpo/q_t": 0.4171040952205658, + "grad_norm": 13.96402359008789, + "learning_rate": 2.259200116137039e-08, + "logits/chosen": 0.7344849109649658, + "logits/rejected": 0.6715294122695923, + "logps/chosen": -233.49896240234375, + "logps/ref_chosen": -59.332359313964844, + "logps/ref_rejected": -83.64482116699219, + "logps/rejected": -328.2040100097656, + "loss": 1.151, + "margin_dpo/margin_mean": 70.39259338378906, + "margin_dpo/margin_std": 125.0219955444336, + "step": 581 + }, + { + "KL/chosen_KL_mean": -159.0695037841797, + "KL/mean": -192.51058959960938, + "KL/rejected_KL_mean": -225.95169067382812, + "KL/std": 99.99513244628906, + "epoch": 0.8798185941043084, + "fcm_dpo/beta": 0.0053823720663785934, + "fcm_dpo/delta": 0.04131370037794113, + "fcm_dpo/margin": 66.88218688964844, + "fcm_dpo/q_t": 0.41894370317459106, + "grad_norm": 11.404556274414062, + "learning_rate": 2.204591459016525e-08, + "logits/chosen": 0.7167221903800964, + "logits/rejected": 0.7457035779953003, + "logps/chosen": -223.23236083984375, + "logps/ref_chosen": -64.16285705566406, + "logps/ref_rejected": -58.632896423339844, + "logps/rejected": -284.5845947265625, + "loss": 1.1493, + "margin_dpo/margin_mean": 66.88218688964844, + "margin_dpo/margin_std": 114.8166275024414, + "step": 582 + }, + { + "KL/chosen_KL_mean": -156.87049865722656, + "KL/mean": -200.28271484375, + "KL/rejected_KL_mean": -243.6949462890625, + "KL/std": 113.34555053710938, + "epoch": 0.8813303099017384, + "fcm_dpo/beta": 0.005323405377566814, + "fcm_dpo/delta": -0.06531860679388046, + "fcm_dpo/margin": 86.82443237304688, + "fcm_dpo/q_t": 0.3956996500492096, + "grad_norm": 14.661458015441895, + "learning_rate": 2.1506204384751064e-08, + "logits/chosen": 0.791956901550293, + "logits/rejected": 0.6918524503707886, + "logps/chosen": -208.74288940429688, + "logps/ref_chosen": -51.87239456176758, + "logps/ref_rejected": -83.86331176757812, + "logps/rejected": -327.5582580566406, + "loss": 1.0855, + "margin_dpo/margin_mean": 86.82443237304688, + "margin_dpo/margin_std": 128.30801391601562, + "step": 583 + }, + { + "KL/chosen_KL_mean": -145.236083984375, + "KL/mean": -181.6009979248047, + "KL/rejected_KL_mean": -217.96591186523438, + "KL/std": 104.1064453125, + "epoch": 0.8828420256991686, + "fcm_dpo/beta": 0.005305691156536341, + "fcm_dpo/delta": 0.014509003609418869, + "fcm_dpo/margin": 72.72981262207031, + "fcm_dpo/q_t": 0.41576987504959106, + "grad_norm": 14.06952953338623, + "learning_rate": 2.09728856419826e-08, + "logits/chosen": 0.8287737965583801, + "logits/rejected": 0.7238653898239136, + "logps/chosen": -191.80746459960938, + "logps/ref_chosen": -46.571388244628906, + "logps/ref_rejected": -80.67969512939453, + "logps/rejected": -298.6455993652344, + "loss": 1.1556, + "margin_dpo/margin_mean": 72.72981262207031, + "margin_dpo/margin_std": 134.8087158203125, + "step": 584 + }, + { + "KL/chosen_KL_mean": -169.44070434570312, + "KL/mean": -196.7550506591797, + "KL/rejected_KL_mean": -224.0694122314453, + "KL/std": 114.88101196289062, + "epoch": 0.8843537414965986, + "fcm_dpo/beta": 0.005436314269900322, + "fcm_dpo/delta": 0.10527031868696213, + "fcm_dpo/margin": 54.628692626953125, + "fcm_dpo/q_t": 0.4324593245983124, + "grad_norm": 13.1787691116333, + "learning_rate": 2.044597327993153e-08, + "logits/chosen": 0.6715638637542725, + "logits/rejected": 0.6183586716651917, + "logps/chosen": -227.56524658203125, + "logps/ref_chosen": -58.124534606933594, + "logps/ref_rejected": -79.00538635253906, + "logps/rejected": -303.0747985839844, + "loss": 1.2024, + "margin_dpo/margin_mean": 54.628700256347656, + "margin_dpo/margin_std": 113.292724609375, + "step": 585 + }, + { + "KL/chosen_KL_mean": -155.8141632080078, + "KL/mean": -190.7159423828125, + "KL/rejected_KL_mean": -225.61773681640625, + "KL/std": 95.33213806152344, + "epoch": 0.8858654572940288, + "fcm_dpo/beta": 0.005444392561912537, + "fcm_dpo/delta": 0.020684288814663887, + "fcm_dpo/margin": 69.80355834960938, + "fcm_dpo/q_t": 0.41235822439193726, + "grad_norm": 13.345340728759766, + "learning_rate": 1.9925482037469187e-08, + "logits/chosen": 0.7436306476593018, + "logits/rejected": 0.7003189325332642, + "logps/chosen": -209.91580200195312, + "logps/ref_chosen": -54.10163879394531, + "logps/ref_rejected": -63.72113037109375, + "logps/rejected": -289.3388671875, + "loss": 1.097, + "margin_dpo/margin_mean": 69.8035659790039, + "margin_dpo/margin_std": 88.85975646972656, + "step": 586 + }, + { + "KL/chosen_KL_mean": -164.1700439453125, + "KL/mean": -198.61328125, + "KL/rejected_KL_mean": -233.05653381347656, + "KL/std": 112.59260559082031, + "epoch": 0.8873771730914588, + "fcm_dpo/beta": 0.005474920384585857, + "fcm_dpo/delta": 0.023741472512483597, + "fcm_dpo/margin": 68.88650512695312, + "fcm_dpo/q_t": 0.40968871116638184, + "grad_norm": 15.52365779876709, + "learning_rate": 1.9411426473854687e-08, + "logits/chosen": 0.7410461902618408, + "logits/rejected": 0.7377340793609619, + "logps/chosen": -227.58721923828125, + "logps/ref_chosen": -63.41719436645508, + "logps/ref_rejected": -63.47003936767578, + "logps/rejected": -296.5265808105469, + "loss": 1.1936, + "margin_dpo/margin_mean": 68.88650512695312, + "margin_dpo/margin_std": 151.58599853515625, + "step": 587 + }, + { + "KL/chosen_KL_mean": -166.9088592529297, + "KL/mean": -206.42413330078125, + "KL/rejected_KL_mean": -245.9394073486328, + "KL/std": 111.02951049804688, + "epoch": 0.8888888888888888, + "fcm_dpo/beta": 0.005442941095679998, + "fcm_dpo/delta": -0.03191109001636505, + "fcm_dpo/margin": 79.03054809570312, + "fcm_dpo/q_t": 0.4048862159252167, + "grad_norm": 17.76612091064453, + "learning_rate": 1.890382096832699e-08, + "logits/chosen": 0.7640427947044373, + "logits/rejected": 0.7215005159378052, + "logps/chosen": -229.10989379882812, + "logps/ref_chosen": -62.20103454589844, + "logps/ref_rejected": -82.10249328613281, + "logps/rejected": -328.0419006347656, + "loss": 1.1259, + "margin_dpo/margin_mean": 79.03054809570312, + "margin_dpo/margin_std": 135.61080932617188, + "step": 588 + }, + { + "KL/chosen_KL_mean": -156.30934143066406, + "KL/mean": -196.30322265625, + "KL/rejected_KL_mean": -236.297119140625, + "KL/std": 106.37506103515625, + "epoch": 0.890400604686319, + "fcm_dpo/beta": 0.005437628366053104, + "fcm_dpo/delta": -0.036562640219926834, + "fcm_dpo/margin": 79.98777770996094, + "fcm_dpo/q_t": 0.3991260528564453, + "grad_norm": 12.22568416595459, + "learning_rate": 1.840267971970344e-08, + "logits/chosen": 0.7192363739013672, + "logits/rejected": 0.6883012652397156, + "logps/chosen": -213.02294921875, + "logps/ref_chosen": -56.71361541748047, + "logps/ref_rejected": -76.7366943359375, + "logps/rejected": -313.0338134765625, + "loss": 1.0678, + "margin_dpo/margin_mean": 79.98777770996094, + "margin_dpo/margin_std": 100.07125854492188, + "step": 589 + }, + { + "KL/chosen_KL_mean": -170.75979614257812, + "KL/mean": -207.8297882080078, + "KL/rejected_KL_mean": -244.89979553222656, + "KL/std": 99.0168685913086, + "epoch": 0.891912320483749, + "fcm_dpo/beta": 0.005432046018540859, + "fcm_dpo/delta": -0.003115180879831314, + "fcm_dpo/margin": 74.14000701904297, + "fcm_dpo/q_t": 0.4076809287071228, + "grad_norm": 16.128381729125977, + "learning_rate": 1.7908016745981856e-08, + "logits/chosen": 0.6945627927780151, + "logits/rejected": 0.6577792763710022, + "logps/chosen": -237.2736053466797, + "logps/ref_chosen": -66.5138168334961, + "logps/ref_rejected": -85.70820617675781, + "logps/rejected": -330.6080017089844, + "loss": 1.1049, + "margin_dpo/margin_mean": 74.13999938964844, + "margin_dpo/margin_std": 106.85514831542969, + "step": 590 + }, + { + "KL/chosen_KL_mean": -145.10528564453125, + "KL/mean": -192.39013671875, + "KL/rejected_KL_mean": -239.6749725341797, + "KL/std": 115.98556518554688, + "epoch": 0.8934240362811792, + "fcm_dpo/beta": 0.0052824802696704865, + "fcm_dpo/delta": -0.10702680796384811, + "fcm_dpo/margin": 94.5697021484375, + "fcm_dpo/q_t": 0.3904598355293274, + "grad_norm": 14.471207618713379, + "learning_rate": 1.7419845883949098e-08, + "logits/chosen": 0.8123652338981628, + "logits/rejected": 0.7523353099822998, + "logps/chosen": -205.80245971679688, + "logps/ref_chosen": -60.697181701660156, + "logps/ref_rejected": -86.12278747558594, + "logps/rejected": -325.7977600097656, + "loss": 1.0776, + "margin_dpo/margin_mean": 94.5697021484375, + "margin_dpo/margin_std": 141.56353759765625, + "step": 591 + }, + { + "KL/chosen_KL_mean": -158.5380401611328, + "KL/mean": -194.74591064453125, + "KL/rejected_KL_mean": -230.95382690429688, + "KL/std": 99.0007095336914, + "epoch": 0.8949357520786092, + "fcm_dpo/beta": 0.0052817342802882195, + "fcm_dpo/delta": 0.017707686871290207, + "fcm_dpo/margin": 72.41576385498047, + "fcm_dpo/q_t": 0.4149664640426636, + "grad_norm": 14.247783660888672, + "learning_rate": 1.6938180788793556e-08, + "logits/chosen": 0.7698496580123901, + "logits/rejected": 0.6575839519500732, + "logps/chosen": -209.77536010742188, + "logps/ref_chosen": -51.237327575683594, + "logps/ref_rejected": -81.60242462158203, + "logps/rejected": -312.5562438964844, + "loss": 1.1246, + "margin_dpo/margin_mean": 72.41576385498047, + "margin_dpo/margin_std": 112.3682632446289, + "step": 592 + }, + { + "KL/chosen_KL_mean": -131.92184448242188, + "KL/mean": -170.8389892578125, + "KL/rejected_KL_mean": -209.75611877441406, + "KL/std": 101.70758819580078, + "epoch": 0.8964474678760394, + "fcm_dpo/beta": 0.005294942297041416, + "fcm_dpo/delta": -0.012750206515192986, + "fcm_dpo/margin": 77.83428955078125, + "fcm_dpo/q_t": 0.4051951766014099, + "grad_norm": 15.31247615814209, + "learning_rate": 1.6463034933723336e-08, + "logits/chosen": 0.7372517585754395, + "logits/rejected": 0.6430565118789673, + "logps/chosen": -174.0018310546875, + "logps/ref_chosen": -42.08000183105469, + "logps/ref_rejected": -68.47499084472656, + "logps/rejected": -278.2311096191406, + "loss": 1.1067, + "margin_dpo/margin_mean": 77.83428955078125, + "margin_dpo/margin_std": 117.07554626464844, + "step": 593 + }, + { + "KL/chosen_KL_mean": -166.61257934570312, + "KL/mean": -200.24847412109375, + "KL/rejected_KL_mean": -233.8843994140625, + "KL/std": 100.83384704589844, + "epoch": 0.8979591836734694, + "fcm_dpo/beta": 0.00533599779009819, + "fcm_dpo/delta": 0.04256129637360573, + "fcm_dpo/margin": 67.27182006835938, + "fcm_dpo/q_t": 0.4163721799850464, + "grad_norm": 12.89805793762207, + "learning_rate": 1.5994421609589385e-08, + "logits/chosen": 0.6495029926300049, + "logits/rejected": 0.6361119747161865, + "logps/chosen": -230.271240234375, + "logps/ref_chosen": -63.658668518066406, + "logps/ref_rejected": -70.35597229003906, + "logps/rejected": -304.2403564453125, + "loss": 1.131, + "margin_dpo/margin_mean": 67.27182006835938, + "margin_dpo/margin_std": 102.46228790283203, + "step": 594 + }, + { + "KL/chosen_KL_mean": -154.23065185546875, + "KL/mean": -198.49163818359375, + "KL/rejected_KL_mean": -242.75262451171875, + "KL/std": 107.8331298828125, + "epoch": 0.8994708994708994, + "fcm_dpo/beta": 0.005307753570377827, + "fcm_dpo/delta": -0.07329151034355164, + "fcm_dpo/margin": 88.52198791503906, + "fcm_dpo/q_t": 0.3952780067920685, + "grad_norm": 11.26756763458252, + "learning_rate": 1.553235392451377e-08, + "logits/chosen": 0.8032486438751221, + "logits/rejected": 0.7115143537521362, + "logps/chosen": -210.44940185546875, + "logps/ref_chosen": -56.21875762939453, + "logps/ref_rejected": -83.95773315429688, + "logps/rejected": -326.71038818359375, + "loss": 1.0858, + "margin_dpo/margin_mean": 88.52198791503906, + "margin_dpo/margin_std": 133.67189025878906, + "step": 595 + }, + { + "KL/chosen_KL_mean": -178.541015625, + "KL/mean": -195.6376953125, + "KL/rejected_KL_mean": -212.734375, + "KL/std": 99.39712524414062, + "epoch": 0.9009826152683296, + "fcm_dpo/beta": 0.005426476243883371, + "fcm_dpo/delta": 0.21902009844779968, + "fcm_dpo/margin": 34.19335174560547, + "fcm_dpo/q_t": 0.45916134119033813, + "grad_norm": 16.031299591064453, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": 0.63373863697052, + "logits/rejected": 0.6557974219322205, + "logps/chosen": -247.0218963623047, + "logps/ref_chosen": -68.48088073730469, + "logps/ref_rejected": -61.732967376708984, + "logps/rejected": -274.46734619140625, + "loss": 1.2901, + "margin_dpo/margin_mean": 34.19335174560547, + "margin_dpo/margin_std": 105.52054595947266, + "step": 596 + }, + { + "KL/chosen_KL_mean": -136.6261444091797, + "KL/mean": -170.66302490234375, + "KL/rejected_KL_mean": -204.69989013671875, + "KL/std": 87.61666870117188, + "epoch": 0.9024943310657596, + "fcm_dpo/beta": 0.005537579767405987, + "fcm_dpo/delta": 0.0237730722874403, + "fcm_dpo/margin": 68.0737533569336, + "fcm_dpo/q_t": 0.4140661358833313, + "grad_norm": 11.68276309967041, + "learning_rate": 1.4627906988186111e-08, + "logits/chosen": 0.7150381803512573, + "logits/rejected": 0.6985746622085571, + "logps/chosen": -185.483642578125, + "logps/ref_chosen": -48.85750961303711, + "logps/ref_rejected": -55.068084716796875, + "logps/rejected": -259.7679748535156, + "loss": 1.1229, + "margin_dpo/margin_mean": 68.0737533569336, + "margin_dpo/margin_std": 103.95882415771484, + "step": 597 + }, + { + "KL/chosen_KL_mean": -178.44558715820312, + "KL/mean": -200.84381103515625, + "KL/rejected_KL_mean": -223.24200439453125, + "KL/std": 104.78968811035156, + "epoch": 0.9040060468631897, + "fcm_dpo/beta": 0.005672060884535313, + "fcm_dpo/delta": 0.14913338422775269, + "fcm_dpo/margin": 44.79642105102539, + "fcm_dpo/q_t": 0.44390666484832764, + "grad_norm": 13.62793254852295, + "learning_rate": 1.4185553036259095e-08, + "logits/chosen": 0.7172830104827881, + "logits/rejected": 0.6382390260696411, + "logps/chosen": -237.33274841308594, + "logps/ref_chosen": -58.88715362548828, + "logps/ref_rejected": -81.43145751953125, + "logps/rejected": -304.6734619140625, + "loss": 1.2437, + "margin_dpo/margin_mean": 44.796417236328125, + "margin_dpo/margin_std": 112.09017181396484, + "step": 598 + }, + { + "KL/chosen_KL_mean": -180.0388946533203, + "KL/mean": -206.260009765625, + "KL/rejected_KL_mean": -232.48114013671875, + "KL/std": 100.97914123535156, + "epoch": 0.9055177626606198, + "fcm_dpo/beta": 0.005800665356218815, + "fcm_dpo/delta": 0.09832191467285156, + "fcm_dpo/margin": 52.442222595214844, + "fcm_dpo/q_t": 0.43262118101119995, + "grad_norm": 16.923696517944336, + "learning_rate": 1.3749795321332885e-08, + "logits/chosen": 0.7779073715209961, + "logits/rejected": 0.7366725206375122, + "logps/chosen": -237.64608764648438, + "logps/ref_chosen": -57.60719299316406, + "logps/ref_rejected": -71.80469512939453, + "logps/rejected": -304.28582763671875, + "loss": 1.2083, + "margin_dpo/margin_mean": 52.44222640991211, + "margin_dpo/margin_std": 114.17462158203125, + "step": 599 + }, + { + "KL/chosen_KL_mean": -163.935546875, + "KL/mean": -195.71664428710938, + "KL/rejected_KL_mean": -227.49774169921875, + "KL/std": 111.24919128417969, + "epoch": 0.9070294784580499, + "fcm_dpo/beta": 0.005879203788936138, + "fcm_dpo/delta": 0.02595154196023941, + "fcm_dpo/margin": 63.562198638916016, + "fcm_dpo/q_t": 0.4192517399787903, + "grad_norm": 17.71397590637207, + "learning_rate": 1.3320646032487393e-08, + "logits/chosen": 0.766040563583374, + "logits/rejected": 0.7092505097389221, + "logps/chosen": -222.3778533935547, + "logps/ref_chosen": -58.44231414794922, + "logps/ref_rejected": -83.64639282226562, + "logps/rejected": -311.1441345214844, + "loss": 1.1623, + "margin_dpo/margin_mean": 63.56219482421875, + "margin_dpo/margin_std": 118.50975799560547, + "step": 600 + }, + { + "KL/chosen_KL_mean": -147.6629638671875, + "KL/mean": -190.49981689453125, + "KL/rejected_KL_mean": -233.33670043945312, + "KL/std": 115.73957061767578, + "epoch": 0.90854119425548, + "fcm_dpo/beta": 0.005733265075832605, + "fcm_dpo/delta": -0.09673769026994705, + "fcm_dpo/margin": 85.6737289428711, + "fcm_dpo/q_t": 0.3916672468185425, + "grad_norm": 13.079845428466797, + "learning_rate": 1.2898117173950868e-08, + "logits/chosen": 0.7403521537780762, + "logits/rejected": 0.6603990793228149, + "logps/chosen": -203.2572784423828, + "logps/ref_chosen": -55.59432601928711, + "logps/ref_rejected": -83.68630981445312, + "logps/rejected": -317.02301025390625, + "loss": 1.0838, + "margin_dpo/margin_mean": 85.67372131347656, + "margin_dpo/margin_std": 131.4403076171875, + "step": 601 + }, + { + "KL/chosen_KL_mean": -136.67974853515625, + "KL/mean": -176.0723876953125, + "KL/rejected_KL_mean": -215.46502685546875, + "KL/std": 99.88746643066406, + "epoch": 0.91005291005291, + "fcm_dpo/beta": 0.005706445779651403, + "fcm_dpo/delta": -0.05199021100997925, + "fcm_dpo/margin": 78.78524780273438, + "fcm_dpo/q_t": 0.39741697907447815, + "grad_norm": 13.740655899047852, + "learning_rate": 1.2482220564763667e-08, + "logits/chosen": 0.7226306200027466, + "logits/rejected": 0.6935929656028748, + "logps/chosen": -193.02894592285156, + "logps/ref_chosen": -56.349185943603516, + "logps/ref_rejected": -71.9959716796875, + "logps/rejected": -287.46099853515625, + "loss": 1.0727, + "margin_dpo/margin_mean": 78.7852554321289, + "margin_dpo/margin_std": 106.68989562988281, + "step": 602 + }, + { + "KL/chosen_KL_mean": -149.92149353027344, + "KL/mean": -188.60498046875, + "KL/rejected_KL_mean": -227.28848266601562, + "KL/std": 99.36843872070312, + "epoch": 0.9115646258503401, + "fcm_dpo/beta": 0.005627095699310303, + "fcm_dpo/delta": -0.03714841976761818, + "fcm_dpo/margin": 77.36700439453125, + "fcm_dpo/q_t": 0.40159088373184204, + "grad_norm": 14.270939826965332, + "learning_rate": 1.2072967838448051e-08, + "logits/chosen": 0.6830695867538452, + "logits/rejected": 0.6326911449432373, + "logps/chosen": -203.08987426757812, + "logps/ref_chosen": -53.16838836669922, + "logps/ref_rejected": -73.8604736328125, + "logps/rejected": -301.14898681640625, + "loss": 1.0973, + "margin_dpo/margin_mean": 77.36700439453125, + "margin_dpo/margin_std": 116.27694702148438, + "step": 603 + }, + { + "KL/chosen_KL_mean": -147.6639862060547, + "KL/mean": -180.3508758544922, + "KL/rejected_KL_mean": -213.0377655029297, + "KL/std": 95.46358489990234, + "epoch": 0.9130763416477702, + "fcm_dpo/beta": 0.005667116958647966, + "fcm_dpo/delta": 0.030490772798657417, + "fcm_dpo/margin": 65.37378692626953, + "fcm_dpo/q_t": 0.41794219613075256, + "grad_norm": 16.183061599731445, + "learning_rate": 1.1670370442682459e-08, + "logits/chosen": 0.6764880418777466, + "logits/rejected": 0.6834244132041931, + "logps/chosen": -220.31341552734375, + "logps/ref_chosen": -72.64942169189453, + "logps/ref_rejected": -69.8792724609375, + "logps/rejected": -282.91705322265625, + "loss": 1.1664, + "margin_dpo/margin_mean": 65.37378692626953, + "margin_dpo/margin_std": 125.50735473632812, + "step": 604 + }, + { + "KL/chosen_KL_mean": -166.05267333984375, + "KL/mean": -202.0325469970703, + "KL/rejected_KL_mean": -238.01242065429688, + "KL/std": 99.51739501953125, + "epoch": 0.9145880574452003, + "fcm_dpo/beta": 0.0056418003514409065, + "fcm_dpo/delta": -0.006424885243177414, + "fcm_dpo/margin": 71.95973205566406, + "fcm_dpo/q_t": 0.40800243616104126, + "grad_norm": 15.322138786315918, + "learning_rate": 1.1274439638981532e-08, + "logits/chosen": 0.7631226778030396, + "logits/rejected": 0.7103903293609619, + "logps/chosen": -227.66552734375, + "logps/ref_chosen": -61.61284637451172, + "logps/ref_rejected": -79.34398651123047, + "logps/rejected": -317.3564147949219, + "loss": 1.1273, + "margin_dpo/margin_mean": 71.95973205566406, + "margin_dpo/margin_std": 118.6252212524414, + "step": 605 + }, + { + "KL/chosen_KL_mean": -146.68096923828125, + "KL/mean": -189.8187255859375, + "KL/rejected_KL_mean": -232.95648193359375, + "KL/std": 103.99552917480469, + "epoch": 0.9160997732426304, + "fcm_dpo/beta": 0.005604826845228672, + "fcm_dpo/delta": -0.08777381479740143, + "fcm_dpo/margin": 86.2755126953125, + "fcm_dpo/q_t": 0.39236563444137573, + "grad_norm": 15.880330085754395, + "learning_rate": 1.0885186502381016e-08, + "logits/chosen": 0.6865954995155334, + "logits/rejected": 0.6223019361495972, + "logps/chosen": -201.14520263671875, + "logps/ref_chosen": -54.46424102783203, + "logps/ref_rejected": -79.62708282470703, + "logps/rejected": -312.58355712890625, + "loss": 1.0655, + "margin_dpo/margin_mean": 86.2755126953125, + "margin_dpo/margin_std": 120.58283996582031, + "step": 606 + }, + { + "KL/chosen_KL_mean": -161.45761108398438, + "KL/mean": -198.7611083984375, + "KL/rejected_KL_mean": -236.0645751953125, + "KL/std": 100.57814025878906, + "epoch": 0.9176114890400605, + "fcm_dpo/beta": 0.005495757795870304, + "fcm_dpo/delta": -0.01238175481557846, + "fcm_dpo/margin": 74.60693359375, + "fcm_dpo/q_t": 0.4067836403846741, + "grad_norm": 12.885636329650879, + "learning_rate": 1.0502621921127774e-08, + "logits/chosen": 0.6752293109893799, + "logits/rejected": 0.6519873142242432, + "logps/chosen": -224.3184814453125, + "logps/ref_chosen": -62.86086654663086, + "logps/ref_rejected": -72.5501937866211, + "logps/rejected": -308.61474609375, + "loss": 1.119, + "margin_dpo/margin_mean": 74.60693359375, + "margin_dpo/margin_std": 114.7344970703125, + "step": 607 + }, + { + "KL/chosen_KL_mean": -164.59835815429688, + "KL/mean": -205.1026153564453, + "KL/rejected_KL_mean": -245.6068572998047, + "KL/std": 110.28776550292969, + "epoch": 0.9191232048374905, + "fcm_dpo/beta": 0.005514230579137802, + "fcm_dpo/delta": -0.04886316880583763, + "fcm_dpo/margin": 81.00849914550781, + "fcm_dpo/q_t": 0.3991330862045288, + "grad_norm": 13.691490173339844, + "learning_rate": 1.0126756596375685e-08, + "logits/chosen": 0.7040418982505798, + "logits/rejected": 0.6250287294387817, + "logps/chosen": -227.77908325195312, + "logps/ref_chosen": -63.18071746826172, + "logps/ref_rejected": -99.15888214111328, + "logps/rejected": -344.7657470703125, + "loss": 1.078, + "margin_dpo/margin_mean": 81.00849914550781, + "margin_dpo/margin_std": 112.37265014648438, + "step": 608 + }, + { + "KL/chosen_KL_mean": -146.50006103515625, + "KL/mean": -189.08883666992188, + "KL/rejected_KL_mean": -231.67758178710938, + "KL/std": 99.16598510742188, + "epoch": 0.9206349206349206, + "fcm_dpo/beta": 0.0054123280569911, + "fcm_dpo/delta": -0.06470134854316711, + "fcm_dpo/margin": 85.17752075195312, + "fcm_dpo/q_t": 0.39238014817237854, + "grad_norm": 12.95341682434082, + "learning_rate": 9.757601041885694e-09, + "logits/chosen": 0.785415530204773, + "logits/rejected": 0.7473229169845581, + "logps/chosen": -195.123291015625, + "logps/ref_chosen": -48.62322235107422, + "logps/ref_rejected": -68.28271484375, + "logps/rejected": -299.9602966308594, + "loss": 1.0493, + "margin_dpo/margin_mean": 85.17752075195312, + "margin_dpo/margin_std": 98.3773193359375, + "step": 609 + }, + { + "KL/chosen_KL_mean": -160.15623474121094, + "KL/mean": -201.11212158203125, + "KL/rejected_KL_mean": -242.06800842285156, + "KL/std": 109.5565414428711, + "epoch": 0.9221466364323507, + "fcm_dpo/beta": 0.005365372635424137, + "fcm_dpo/delta": -0.041763849556446075, + "fcm_dpo/margin": 81.91178131103516, + "fcm_dpo/q_t": 0.4016649127006531, + "grad_norm": 12.831808090209961, + "learning_rate": 9.395165583732379e-09, + "logits/chosen": 0.7413656711578369, + "logits/rejected": 0.7341820001602173, + "logps/chosen": -232.82135009765625, + "logps/ref_chosen": -72.66513061523438, + "logps/ref_rejected": -87.15310668945312, + "logps/rejected": -329.22113037109375, + "loss": 1.0972, + "margin_dpo/margin_mean": 81.91178894042969, + "margin_dpo/margin_std": 123.76081085205078, + "step": 610 + }, + { + "KL/chosen_KL_mean": -147.50057983398438, + "KL/mean": -180.25575256347656, + "KL/rejected_KL_mean": -213.01092529296875, + "KL/std": 96.39732360839844, + "epoch": 0.9236583522297808, + "fcm_dpo/beta": 0.005401009693741798, + "fcm_dpo/delta": 0.047877293080091476, + "fcm_dpo/margin": 65.51034545898438, + "fcm_dpo/q_t": 0.4180787205696106, + "grad_norm": 13.849523544311523, + "learning_rate": 9.03946036001449e-09, + "logits/chosen": 0.7488099932670593, + "logits/rejected": 0.7005026340484619, + "logps/chosen": -195.8091583251953, + "logps/ref_chosen": -48.30857849121094, + "logps/ref_rejected": -70.6141128540039, + "logps/rejected": -283.62506103515625, + "loss": 1.1319, + "margin_dpo/margin_mean": 65.51034545898438, + "margin_dpo/margin_std": 98.97489929199219, + "step": 611 + }, + { + "KL/chosen_KL_mean": -162.2694549560547, + "KL/mean": -210.58135986328125, + "KL/rejected_KL_mean": -258.89324951171875, + "KL/std": 104.07249450683594, + "epoch": 0.9251700680272109, + "fcm_dpo/beta": 0.005330703221261501, + "fcm_dpo/delta": -0.1212921291589737, + "fcm_dpo/margin": 96.62378692626953, + "fcm_dpo/q_t": 0.3814992904663086, + "grad_norm": 12.464214324951172, + "learning_rate": 8.690495320571839e-09, + "logits/chosen": 0.6381398439407349, + "logits/rejected": 0.5688310861587524, + "logps/chosen": -223.50100708007812, + "logps/ref_chosen": -61.23155975341797, + "logps/ref_rejected": -94.37979888916016, + "logps/rejected": -353.2730407714844, + "loss": 1.02, + "margin_dpo/margin_mean": 96.62378692626953, + "margin_dpo/margin_std": 113.25186157226562, + "step": 612 + }, + { + "KL/chosen_KL_mean": -140.88748168945312, + "KL/mean": -183.08731079101562, + "KL/rejected_KL_mean": -225.2871551513672, + "KL/std": 110.63365173339844, + "epoch": 0.926681783824641, + "fcm_dpo/beta": 0.005276457406580448, + "fcm_dpo/delta": -0.04751266539096832, + "fcm_dpo/margin": 84.39967346191406, + "fcm_dpo/q_t": 0.39712822437286377, + "grad_norm": 11.47330093383789, + "learning_rate": 8.348280226706722e-09, + "logits/chosen": 0.6416479349136353, + "logits/rejected": 0.6371424198150635, + "logps/chosen": -194.87057495117188, + "logps/ref_chosen": -53.98310852050781, + "logps/ref_rejected": -58.32208251953125, + "logps/rejected": -283.6092529296875, + "loss": 1.0729, + "margin_dpo/margin_mean": 84.39967346191406, + "margin_dpo/margin_std": 112.11402893066406, + "step": 613 + }, + { + "KL/chosen_KL_mean": -160.6935272216797, + "KL/mean": -200.44207763671875, + "KL/rejected_KL_mean": -240.19061279296875, + "KL/std": 94.97077941894531, + "epoch": 0.9281934996220711, + "fcm_dpo/beta": 0.005260440520942211, + "fcm_dpo/delta": -0.019519072026014328, + "fcm_dpo/margin": 79.4970703125, + "fcm_dpo/q_t": 0.4030989706516266, + "grad_norm": 15.34008502960205, + "learning_rate": 8.012824650910937e-09, + "logits/chosen": 0.7439731359481812, + "logits/rejected": 0.7354336977005005, + "logps/chosen": -220.9365692138672, + "logps/ref_chosen": -60.24303436279297, + "logps/ref_rejected": -72.26258850097656, + "logps/rejected": -312.45318603515625, + "loss": 1.0915, + "margin_dpo/margin_mean": 79.4970703125, + "margin_dpo/margin_std": 109.16985321044922, + "step": 614 + }, + { + "KL/chosen_KL_mean": -169.2880859375, + "KL/mean": -212.89573669433594, + "KL/rejected_KL_mean": -256.50341796875, + "KL/std": 123.34716033935547, + "epoch": 0.9297052154195011, + "fcm_dpo/beta": 0.005149473436176777, + "fcm_dpo/delta": -0.052740830928087234, + "fcm_dpo/margin": 87.21531677246094, + "fcm_dpo/q_t": 0.399341881275177, + "grad_norm": 13.794776916503906, + "learning_rate": 7.684137976598088e-09, + "logits/chosen": 0.6767693161964417, + "logits/rejected": 0.618577778339386, + "logps/chosen": -241.38275146484375, + "logps/ref_chosen": -72.09467315673828, + "logps/ref_rejected": -104.02980041503906, + "logps/rejected": -360.533203125, + "loss": 1.1001, + "margin_dpo/margin_mean": 87.21531677246094, + "margin_dpo/margin_std": 134.67889404296875, + "step": 615 + }, + { + "KL/chosen_KL_mean": -161.08840942382812, + "KL/mean": -194.94741821289062, + "KL/rejected_KL_mean": -228.80642700195312, + "KL/std": 113.65570068359375, + "epoch": 0.9312169312169312, + "fcm_dpo/beta": 0.0052018859423696995, + "fcm_dpo/delta": 0.04951094835996628, + "fcm_dpo/margin": 67.71803283691406, + "fcm_dpo/q_t": 0.41964447498321533, + "grad_norm": 12.714033126831055, + "learning_rate": 7.36222939784098e-09, + "logits/chosen": 0.7352438569068909, + "logits/rejected": 0.6616415977478027, + "logps/chosen": -219.61912536621094, + "logps/ref_chosen": -58.530723571777344, + "logps/ref_rejected": -75.48025512695312, + "logps/rejected": -304.28668212890625, + "loss": 1.1425, + "margin_dpo/margin_mean": 67.71803283691406, + "margin_dpo/margin_std": 109.96153259277344, + "step": 616 + }, + { + "KL/chosen_KL_mean": -169.83023071289062, + "KL/mean": -201.72604370117188, + "KL/rejected_KL_mean": -233.621826171875, + "KL/std": 104.35264587402344, + "epoch": 0.9327286470143613, + "fcm_dpo/beta": 0.005237380973994732, + "fcm_dpo/delta": 0.06759772449731827, + "fcm_dpo/margin": 63.79159927368164, + "fcm_dpo/q_t": 0.4250500798225403, + "grad_norm": 16.894447326660156, + "learning_rate": 7.047107919114586e-09, + "logits/chosen": 0.7362958788871765, + "logits/rejected": 0.6854845285415649, + "logps/chosen": -227.4389190673828, + "logps/ref_chosen": -57.608673095703125, + "logps/ref_rejected": -81.22109985351562, + "logps/rejected": -314.8429260253906, + "loss": 1.1671, + "margin_dpo/margin_mean": 63.791595458984375, + "margin_dpo/margin_std": 112.23297882080078, + "step": 617 + }, + { + "KL/chosen_KL_mean": -156.01065063476562, + "KL/mean": -194.25592041015625, + "KL/rejected_KL_mean": -232.50120544433594, + "KL/std": 114.61581420898438, + "epoch": 0.9342403628117913, + "fcm_dpo/beta": 0.005274903029203415, + "fcm_dpo/delta": -0.0038125645369291306, + "fcm_dpo/margin": 76.49055480957031, + "fcm_dpo/q_t": 0.410194456577301, + "grad_norm": 15.224896430969238, + "learning_rate": 6.738782355044048e-09, + "logits/chosen": 0.7007787227630615, + "logits/rejected": 0.6046236753463745, + "logps/chosen": -212.7065887451172, + "logps/ref_chosen": -56.69594192504883, + "logps/ref_rejected": -85.92362976074219, + "logps/rejected": -318.4248352050781, + "loss": 1.1073, + "margin_dpo/margin_mean": 76.49055480957031, + "margin_dpo/margin_std": 115.4001235961914, + "step": 618 + }, + { + "KL/chosen_KL_mean": -149.36265563964844, + "KL/mean": -190.2163543701172, + "KL/rejected_KL_mean": -231.070068359375, + "KL/std": 108.53770446777344, + "epoch": 0.9357520786092215, + "fcm_dpo/beta": 0.005254857242107391, + "fcm_dpo/delta": -0.03081546165049076, + "fcm_dpo/margin": 81.70741271972656, + "fcm_dpo/q_t": 0.4019641876220703, + "grad_norm": 13.710566520690918, + "learning_rate": 6.437261330158206e-09, + "logits/chosen": 0.8129085302352905, + "logits/rejected": 0.7362833619117737, + "logps/chosen": -203.42108154296875, + "logps/ref_chosen": -54.05841827392578, + "logps/ref_rejected": -83.55493927001953, + "logps/rejected": -314.625, + "loss": 1.0929, + "margin_dpo/margin_mean": 81.70740509033203, + "margin_dpo/margin_std": 119.22251892089844, + "step": 619 + }, + { + "KL/chosen_KL_mean": -154.7310333251953, + "KL/mean": -186.06826782226562, + "KL/rejected_KL_mean": -217.40548706054688, + "KL/std": 98.9871826171875, + "epoch": 0.9372637944066515, + "fcm_dpo/beta": 0.005364430136978626, + "fcm_dpo/delta": 0.06292795389890671, + "fcm_dpo/margin": 62.67444610595703, + "fcm_dpo/q_t": 0.4228057861328125, + "grad_norm": 13.484994888305664, + "learning_rate": 6.142553278648238e-09, + "logits/chosen": 0.7374277114868164, + "logits/rejected": 0.7379218339920044, + "logps/chosen": -218.1007537841797, + "logps/ref_chosen": -63.36971664428711, + "logps/ref_rejected": -65.68269348144531, + "logps/rejected": -283.0881652832031, + "loss": 1.1642, + "margin_dpo/margin_mean": 62.67444610595703, + "margin_dpo/margin_std": 104.49517059326172, + "step": 620 + }, + { + "KL/chosen_KL_mean": -151.9742431640625, + "KL/mean": -188.44854736328125, + "KL/rejected_KL_mean": -224.9228515625, + "KL/std": 106.9564208984375, + "epoch": 0.9387755102040817, + "fcm_dpo/beta": 0.0053487373515963554, + "fcm_dpo/delta": 0.009966127574443817, + "fcm_dpo/margin": 72.94859313964844, + "fcm_dpo/q_t": 0.4138706624507904, + "grad_norm": 14.086136817932129, + "learning_rate": 5.854666444131934e-09, + "logits/chosen": 0.7628237009048462, + "logits/rejected": 0.6525827646255493, + "logps/chosen": -204.2954559326172, + "logps/ref_chosen": -52.321224212646484, + "logps/ref_rejected": -88.09001159667969, + "logps/rejected": -313.0128479003906, + "loss": 1.141, + "margin_dpo/margin_mean": 72.94860076904297, + "margin_dpo/margin_std": 126.93898010253906, + "step": 621 + }, + { + "KL/chosen_KL_mean": -167.28756713867188, + "KL/mean": -204.14352416992188, + "KL/rejected_KL_mean": -240.99948120117188, + "KL/std": 111.68397521972656, + "epoch": 0.9402872260015117, + "fcm_dpo/beta": 0.005355824716389179, + "fcm_dpo/delta": 0.005186108872294426, + "fcm_dpo/margin": 73.71192932128906, + "fcm_dpo/q_t": 0.41089174151420593, + "grad_norm": 16.46088409423828, + "learning_rate": 5.573608879422875e-09, + "logits/chosen": 0.6860193610191345, + "logits/rejected": 0.6397637128829956, + "logps/chosen": -227.15301513671875, + "logps/ref_chosen": -59.86545944213867, + "logps/ref_rejected": -81.86668395996094, + "logps/rejected": -322.86614990234375, + "loss": 1.121, + "margin_dpo/margin_mean": 73.71192932128906, + "margin_dpo/margin_std": 115.60111999511719, + "step": 622 + }, + { + "KL/chosen_KL_mean": -176.53656005859375, + "KL/mean": -214.87255859375, + "KL/rejected_KL_mean": -253.20855712890625, + "KL/std": 117.26280212402344, + "epoch": 0.9417989417989417, + "fcm_dpo/beta": 0.005307169631123543, + "fcm_dpo/delta": -0.007740970700979233, + "fcm_dpo/margin": 76.6719970703125, + "fcm_dpo/q_t": 0.40785908699035645, + "grad_norm": 14.635030746459961, + "learning_rate": 5.299388446305342e-09, + "logits/chosen": 0.7361361980438232, + "logits/rejected": 0.6836817264556885, + "logps/chosen": -243.90501403808594, + "logps/ref_chosen": -67.36846160888672, + "logps/ref_rejected": -82.02733612060547, + "logps/rejected": -335.23590087890625, + "loss": 1.1183, + "margin_dpo/margin_mean": 76.67198944091797, + "margin_dpo/margin_std": 121.35376739501953, + "step": 623 + }, + { + "KL/chosen_KL_mean": -149.57037353515625, + "KL/mean": -190.38003540039062, + "KL/rejected_KL_mean": -231.18972778320312, + "KL/std": 110.87216186523438, + "epoch": 0.9433106575963719, + "fcm_dpo/beta": 0.0052911280654370785, + "fcm_dpo/delta": -0.03351764753460884, + "fcm_dpo/margin": 81.61933898925781, + "fcm_dpo/q_t": 0.4038599133491516, + "grad_norm": 15.522960662841797, + "learning_rate": 5.03201281531429e-09, + "logits/chosen": 0.7098270654678345, + "logits/rejected": 0.6187886595726013, + "logps/chosen": -200.596923828125, + "logps/ref_chosen": -51.02655029296875, + "logps/ref_rejected": -76.49203491210938, + "logps/rejected": -307.6817626953125, + "loss": 1.0977, + "margin_dpo/margin_mean": 81.61933898925781, + "margin_dpo/margin_std": 122.6544189453125, + "step": 624 + }, + { + "KL/chosen_KL_mean": -150.91305541992188, + "KL/mean": -183.52633666992188, + "KL/rejected_KL_mean": -216.1396484375, + "KL/std": 105.79248046875, + "epoch": 0.9448223733938019, + "fcm_dpo/beta": 0.005336887203156948, + "fcm_dpo/delta": 0.053777653723955154, + "fcm_dpo/margin": 65.22659301757812, + "fcm_dpo/q_t": 0.42401954531669617, + "grad_norm": 12.910602569580078, + "learning_rate": 4.7714894655209174e-09, + "logits/chosen": 0.8031026124954224, + "logits/rejected": 0.7135155200958252, + "logps/chosen": -205.12066650390625, + "logps/ref_chosen": -54.20761489868164, + "logps/ref_rejected": -84.93669128417969, + "logps/rejected": -301.07635498046875, + "loss": 1.1751, + "margin_dpo/margin_mean": 65.2265853881836, + "margin_dpo/margin_std": 127.71304321289062, + "step": 625 + }, + { + "KL/chosen_KL_mean": -142.59837341308594, + "KL/mean": -187.04966735839844, + "KL/rejected_KL_mean": -231.5009765625, + "KL/std": 111.5535888671875, + "epoch": 0.9463340891912321, + "fcm_dpo/beta": 0.0052881138399243355, + "fcm_dpo/delta": -0.0737067237496376, + "fcm_dpo/margin": 88.90258026123047, + "fcm_dpo/q_t": 0.3975789546966553, + "grad_norm": 13.397624015808105, + "learning_rate": 4.517825684323323e-09, + "logits/chosen": 0.8315858840942383, + "logits/rejected": 0.6983362436294556, + "logps/chosen": -187.660400390625, + "logps/ref_chosen": -45.06201934814453, + "logps/ref_rejected": -89.66368103027344, + "logps/rejected": -321.1646728515625, + "loss": 1.0832, + "margin_dpo/margin_mean": 88.902587890625, + "margin_dpo/margin_std": 135.9188232421875, + "step": 626 + }, + { + "KL/chosen_KL_mean": -164.44793701171875, + "KL/mean": -211.04330444335938, + "KL/rejected_KL_mean": -257.638671875, + "KL/std": 116.6589584350586, + "epoch": 0.9478458049886621, + "fcm_dpo/beta": 0.005192744545638561, + "fcm_dpo/delta": -0.08852745592594147, + "fcm_dpo/margin": 93.19075012207031, + "fcm_dpo/q_t": 0.38831257820129395, + "grad_norm": 14.439326286315918, + "learning_rate": 4.271028567242818e-09, + "logits/chosen": 0.6702276468276978, + "logits/rejected": 0.5530549883842468, + "logps/chosen": -223.23898315429688, + "logps/ref_chosen": -58.791053771972656, + "logps/ref_rejected": -94.90802001953125, + "logps/rejected": -352.54669189453125, + "loss": 1.0465, + "margin_dpo/margin_mean": 93.19074249267578, + "margin_dpo/margin_std": 117.09492492675781, + "step": 627 + }, + { + "KL/chosen_KL_mean": -149.05197143554688, + "KL/mean": -193.82723999023438, + "KL/rejected_KL_mean": -238.60252380371094, + "KL/std": 108.63807678222656, + "epoch": 0.9493575207860923, + "fcm_dpo/beta": 0.005168614909052849, + "fcm_dpo/delta": -0.06649599969387054, + "fcm_dpo/margin": 89.55052947998047, + "fcm_dpo/q_t": 0.3932734727859497, + "grad_norm": 14.440736770629883, + "learning_rate": 4.0311050177251895e-09, + "logits/chosen": 0.7511168718338013, + "logits/rejected": 0.7099975347518921, + "logps/chosen": -201.85556030273438, + "logps/ref_chosen": -52.80357360839844, + "logps/ref_rejected": -76.49468994140625, + "logps/rejected": -315.09722900390625, + "loss": 1.0868, + "margin_dpo/margin_mean": 89.55052185058594, + "margin_dpo/margin_std": 124.22722625732422, + "step": 628 + }, + { + "KL/chosen_KL_mean": -161.83309936523438, + "KL/mean": -194.438720703125, + "KL/rejected_KL_mean": -227.04434204101562, + "KL/std": 101.754150390625, + "epoch": 0.9508692365835223, + "fcm_dpo/beta": 0.005171940661966801, + "fcm_dpo/delta": 0.06475324183702469, + "fcm_dpo/margin": 65.21124267578125, + "fcm_dpo/q_t": 0.4241940975189209, + "grad_norm": 13.463421821594238, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": 0.7015402317047119, + "logits/rejected": 0.7001262307167053, + "logps/chosen": -232.55059814453125, + "logps/ref_chosen": -70.71749877929688, + "logps/ref_rejected": -78.96273803710938, + "logps/rejected": -306.007080078125, + "loss": 1.1524, + "margin_dpo/margin_mean": 65.21124267578125, + "margin_dpo/margin_std": 109.4168930053711, + "step": 629 + }, + { + "KL/chosen_KL_mean": -149.88983154296875, + "KL/mean": -194.91024780273438, + "KL/rejected_KL_mean": -239.9306640625, + "KL/std": 109.47689819335938, + "epoch": 0.9523809523809523, + "fcm_dpo/beta": 0.00514186080545187, + "fcm_dpo/delta": -0.06599334627389908, + "fcm_dpo/margin": 90.04084014892578, + "fcm_dpo/q_t": 0.39410853385925293, + "grad_norm": 11.221573829650879, + "learning_rate": 3.5719052736323806e-09, + "logits/chosen": 0.6838537454605103, + "logits/rejected": 0.6431096792221069, + "logps/chosen": -206.09124755859375, + "logps/ref_chosen": -56.201412200927734, + "logps/ref_rejected": -74.69807434082031, + "logps/rejected": -314.62872314453125, + "loss": 1.061, + "margin_dpo/margin_mean": 90.04084777832031, + "margin_dpo/margin_std": 117.41181945800781, + "step": 630 + }, + { + "KL/chosen_KL_mean": -143.79959106445312, + "KL/mean": -194.85324096679688, + "KL/rejected_KL_mean": -245.9068603515625, + "KL/std": 116.61567687988281, + "epoch": 0.9538926681783825, + "fcm_dpo/beta": 0.004982981365174055, + "fcm_dpo/delta": -0.11698402464389801, + "fcm_dpo/margin": 102.10726928710938, + "fcm_dpo/q_t": 0.38580942153930664, + "grad_norm": 13.442806243896484, + "learning_rate": 3.352641923861144e-09, + "logits/chosen": 0.798941969871521, + "logits/rejected": 0.6885172128677368, + "logps/chosen": -202.62020874023438, + "logps/ref_chosen": -58.82059860229492, + "logps/ref_rejected": -96.51437377929688, + "logps/rejected": -342.4212646484375, + "loss": 1.0399, + "margin_dpo/margin_mean": 102.10726928710938, + "margin_dpo/margin_std": 128.56503295898438, + "step": 631 + }, + { + "KL/chosen_KL_mean": -146.59799194335938, + "KL/mean": -193.16958618164062, + "KL/rejected_KL_mean": -239.74114990234375, + "KL/std": 102.21504211425781, + "epoch": 0.9554043839758125, + "fcm_dpo/beta": 0.004959441255778074, + "fcm_dpo/delta": -0.06491100043058395, + "fcm_dpo/margin": 93.14317321777344, + "fcm_dpo/q_t": 0.39288681745529175, + "grad_norm": 12.798542022705078, + "learning_rate": 3.140277830901428e-09, + "logits/chosen": 0.771345853805542, + "logits/rejected": 0.7523634433746338, + "logps/chosen": -205.38404846191406, + "logps/ref_chosen": -58.786048889160156, + "logps/ref_rejected": -67.21923828125, + "logps/rejected": -306.96038818359375, + "loss": 1.0488, + "margin_dpo/margin_mean": 93.14315795898438, + "margin_dpo/margin_std": 111.02008056640625, + "step": 632 + }, + { + "KL/chosen_KL_mean": -141.06906127929688, + "KL/mean": -177.5758514404297, + "KL/rejected_KL_mean": -214.08261108398438, + "KL/std": 98.3267822265625, + "epoch": 0.9569160997732427, + "fcm_dpo/beta": 0.004975237883627415, + "fcm_dpo/delta": 0.03772956132888794, + "fcm_dpo/margin": 73.01356506347656, + "fcm_dpo/q_t": 0.4177446663379669, + "grad_norm": 13.155255317687988, + "learning_rate": 2.9348189350335007e-09, + "logits/chosen": 0.6996445655822754, + "logits/rejected": 0.6450544595718384, + "logps/chosen": -193.1992645263672, + "logps/ref_chosen": -52.13019561767578, + "logps/ref_rejected": -67.23016357421875, + "logps/rejected": -281.3127746582031, + "loss": 1.1303, + "margin_dpo/margin_mean": 73.01355743408203, + "margin_dpo/margin_std": 112.07791900634766, + "step": 633 + }, + { + "KL/chosen_KL_mean": -169.75709533691406, + "KL/mean": -184.82101440429688, + "KL/rejected_KL_mean": -199.8849334716797, + "KL/std": 102.64701843261719, + "epoch": 0.9584278155706727, + "fcm_dpo/beta": 0.005162340588867664, + "fcm_dpo/delta": 0.24793994426727295, + "fcm_dpo/margin": 30.12784194946289, + "fcm_dpo/q_t": 0.46613961458206177, + "grad_norm": 16.462249755859375, + "learning_rate": 2.736270983384276e-09, + "logits/chosen": 0.7885172963142395, + "logits/rejected": 0.8044285774230957, + "logps/chosen": -230.7368927001953, + "logps/ref_chosen": -60.97979736328125, + "logps/ref_rejected": -58.50825119018555, + "logps/rejected": -258.3931884765625, + "loss": 1.3256, + "margin_dpo/margin_mean": 30.12784194946289, + "margin_dpo/margin_std": 118.00814819335938, + "step": 634 + }, + { + "KL/chosen_KL_mean": -186.54104614257812, + "KL/mean": -216.65505981445312, + "KL/rejected_KL_mean": -246.76907348632812, + "KL/std": 102.26522827148438, + "epoch": 0.9599395313681028, + "fcm_dpo/beta": 0.005301056429743767, + "fcm_dpo/delta": 0.08293192088603973, + "fcm_dpo/margin": 60.2280387878418, + "fcm_dpo/q_t": 0.4280802309513092, + "grad_norm": 13.260807991027832, + "learning_rate": 2.5446395297668287e-09, + "logits/chosen": 0.627880871295929, + "logits/rejected": 0.5735411047935486, + "logps/chosen": -252.51405334472656, + "logps/ref_chosen": -65.9730224609375, + "logps/ref_rejected": -85.61317443847656, + "logps/rejected": -332.3822326660156, + "loss": 1.1937, + "margin_dpo/margin_mean": 60.2280387878418, + "margin_dpo/margin_std": 122.29312896728516, + "step": 635 + }, + { + "KL/chosen_KL_mean": -144.86769104003906, + "KL/mean": -186.0552978515625, + "KL/rejected_KL_mean": -227.24293518066406, + "KL/std": 103.58889770507812, + "epoch": 0.9614512471655329, + "fcm_dpo/beta": 0.005266260355710983, + "fcm_dpo/delta": -0.03554299846291542, + "fcm_dpo/margin": 82.37523651123047, + "fcm_dpo/q_t": 0.39980411529541016, + "grad_norm": 11.369194984436035, + "learning_rate": 2.359929934524829e-09, + "logits/chosen": 0.7130604386329651, + "logits/rejected": 0.6156246662139893, + "logps/chosen": -194.0078582763672, + "logps/ref_chosen": -49.140167236328125, + "logps/ref_rejected": -81.26971435546875, + "logps/rejected": -308.51263427734375, + "loss": 1.0788, + "margin_dpo/margin_mean": 82.37523651123047, + "margin_dpo/margin_std": 110.31642150878906, + "step": 636 + }, + { + "KL/chosen_KL_mean": -194.6241455078125, + "KL/mean": -224.41693115234375, + "KL/rejected_KL_mean": -254.209716796875, + "KL/std": 104.2559814453125, + "epoch": 0.9629629629629629, + "fcm_dpo/beta": 0.005353373475372791, + "fcm_dpo/delta": 0.08315958827733994, + "fcm_dpo/margin": 59.5855712890625, + "fcm_dpo/q_t": 0.4289915859699249, + "grad_norm": 14.297188758850098, + "learning_rate": 2.1821473643827137e-09, + "logits/chosen": 0.6850694417953491, + "logits/rejected": 0.6259503364562988, + "logps/chosen": -268.32073974609375, + "logps/ref_chosen": -73.69658660888672, + "logps/ref_rejected": -83.01487731933594, + "logps/rejected": -337.224609375, + "loss": 1.2052, + "margin_dpo/margin_mean": 59.5855712890625, + "margin_dpo/margin_std": 130.34597778320312, + "step": 637 + }, + { + "KL/chosen_KL_mean": -168.71023559570312, + "KL/mean": -205.56460571289062, + "KL/rejected_KL_mean": -242.4189910888672, + "KL/std": 103.81689453125, + "epoch": 0.9644746787603931, + "fcm_dpo/beta": 0.005360821262001991, + "fcm_dpo/delta": 0.005060809198766947, + "fcm_dpo/margin": 73.70875549316406, + "fcm_dpo/q_t": 0.4109768569469452, + "grad_norm": 13.191207885742188, + "learning_rate": 2.0112967923011646e-09, + "logits/chosen": 0.7055521011352539, + "logits/rejected": 0.6617870330810547, + "logps/chosen": -231.49180603027344, + "logps/ref_chosen": -62.78158187866211, + "logps/ref_rejected": -85.40478515625, + "logps/rejected": -327.82379150390625, + "loss": 1.1151, + "margin_dpo/margin_mean": 73.70875549316406, + "margin_dpo/margin_std": 113.07898712158203, + "step": 638 + }, + { + "KL/chosen_KL_mean": -150.687744140625, + "KL/mean": -193.58897399902344, + "KL/rejected_KL_mean": -236.49020385742188, + "KL/std": 110.87168884277344, + "epoch": 0.9659863945578231, + "fcm_dpo/beta": 0.00532121304422617, + "fcm_dpo/delta": -0.059241220355033875, + "fcm_dpo/margin": 85.80245208740234, + "fcm_dpo/q_t": 0.39776647090911865, + "grad_norm": 16.12027359008789, + "learning_rate": 1.847382997337943e-09, + "logits/chosen": 0.7033539414405823, + "logits/rejected": 0.614127516746521, + "logps/chosen": -204.45433044433594, + "logps/ref_chosen": -53.76658630371094, + "logps/ref_rejected": -72.30009460449219, + "logps/rejected": -308.790283203125, + "loss": 1.0754, + "margin_dpo/margin_mean": 85.80245971679688, + "margin_dpo/margin_std": 120.51637268066406, + "step": 639 + }, + { + "KL/chosen_KL_mean": -157.09376525878906, + "KL/mean": -196.0372772216797, + "KL/rejected_KL_mean": -234.98080444335938, + "KL/std": 106.0317611694336, + "epoch": 0.9674981103552532, + "fcm_dpo/beta": 0.005276700481772423, + "fcm_dpo/delta": -0.011631457135081291, + "fcm_dpo/margin": 77.88703918457031, + "fcm_dpo/q_t": 0.4065815806388855, + "grad_norm": 13.552083015441895, + "learning_rate": 1.690410564514244e-09, + "logits/chosen": 0.7871816158294678, + "logits/rejected": 0.724848747253418, + "logps/chosen": -208.51153564453125, + "logps/ref_chosen": -51.41777801513672, + "logps/ref_rejected": -77.27879333496094, + "logps/rejected": -312.25958251953125, + "loss": 1.1062, + "margin_dpo/margin_mean": 77.88703918457031, + "margin_dpo/margin_std": 116.83767700195312, + "step": 640 + }, + { + "KL/chosen_KL_mean": -164.87841796875, + "KL/mean": -199.95181274414062, + "KL/rejected_KL_mean": -235.02520751953125, + "KL/std": 97.74984741210938, + "epoch": 0.9690098261526833, + "fcm_dpo/beta": 0.0052994233556091785, + "fcm_dpo/delta": 0.02932356297969818, + "fcm_dpo/margin": 70.14678192138672, + "fcm_dpo/q_t": 0.4133235216140747, + "grad_norm": 14.538914680480957, + "learning_rate": 1.5403838846864692e-09, + "logits/chosen": 0.7304531335830688, + "logits/rejected": 0.7157402038574219, + "logps/chosen": -235.93307495117188, + "logps/ref_chosen": -71.0546646118164, + "logps/ref_rejected": -82.2440185546875, + "logps/rejected": -317.26922607421875, + "loss": 1.1067, + "margin_dpo/margin_mean": 70.14678192138672, + "margin_dpo/margin_std": 93.35204315185547, + "step": 641 + }, + { + "KL/chosen_KL_mean": -172.72177124023438, + "KL/mean": -199.08563232421875, + "KL/rejected_KL_mean": -225.44949340820312, + "KL/std": 95.5352783203125, + "epoch": 0.9705215419501134, + "fcm_dpo/beta": 0.005387555807828903, + "fcm_dpo/delta": 0.11940450966358185, + "fcm_dpo/margin": 52.727725982666016, + "fcm_dpo/q_t": 0.43623262643814087, + "grad_norm": 16.527711868286133, + "learning_rate": 1.3973071544233218e-09, + "logits/chosen": 0.6687928438186646, + "logits/rejected": 0.689499020576477, + "logps/chosen": -241.65106201171875, + "logps/ref_chosen": -68.92927551269531, + "logps/ref_rejected": -70.85682678222656, + "logps/rejected": -296.30633544921875, + "loss": 1.2159, + "margin_dpo/margin_mean": 52.727718353271484, + "margin_dpo/margin_std": 114.44686889648438, + "step": 642 + }, + { + "KL/chosen_KL_mean": -157.55726623535156, + "KL/mean": -198.06973266601562, + "KL/rejected_KL_mean": -238.58216857910156, + "KL/std": 103.79234313964844, + "epoch": 0.9720332577475435, + "fcm_dpo/beta": 0.005408759694546461, + "fcm_dpo/delta": -0.04006705433130264, + "fcm_dpo/margin": 81.02490234375, + "fcm_dpo/q_t": 0.4013475775718689, + "grad_norm": 18.932897567749023, + "learning_rate": 1.261184375888541e-09, + "logits/chosen": 0.6582962274551392, + "logits/rejected": 0.5775290727615356, + "logps/chosen": -222.86630249023438, + "logps/ref_chosen": -65.30903625488281, + "logps/ref_rejected": -83.61613464355469, + "logps/rejected": -322.19830322265625, + "loss": 1.0966, + "margin_dpo/margin_mean": 81.02490234375, + "margin_dpo/margin_std": 121.24118041992188, + "step": 643 + }, + { + "KL/chosen_KL_mean": -147.66122436523438, + "KL/mean": -175.08193969726562, + "KL/rejected_KL_mean": -202.50262451171875, + "KL/std": 93.31697082519531, + "epoch": 0.9735449735449735, + "fcm_dpo/beta": 0.005520361475646496, + "fcm_dpo/delta": 0.0987476035952568, + "fcm_dpo/margin": 54.841392517089844, + "fcm_dpo/q_t": 0.4305710196495056, + "grad_norm": 12.613712310791016, + "learning_rate": 1.1320193567288527e-09, + "logits/chosen": 0.7914489507675171, + "logits/rejected": 0.7597800493240356, + "logps/chosen": -198.66384887695312, + "logps/ref_chosen": -51.002601623535156, + "logps/ref_rejected": -64.46372985839844, + "logps/rejected": -266.96636962890625, + "loss": 1.204, + "margin_dpo/margin_mean": 54.841392517089844, + "margin_dpo/margin_std": 113.74671936035156, + "step": 644 + }, + { + "KL/chosen_KL_mean": -161.51620483398438, + "KL/mean": -195.10723876953125, + "KL/rejected_KL_mean": -228.69830322265625, + "KL/std": 101.00505828857422, + "epoch": 0.9750566893424036, + "fcm_dpo/beta": 0.0055358922109007835, + "fcm_dpo/delta": 0.02916993945837021, + "fcm_dpo/margin": 67.1821060180664, + "fcm_dpo/q_t": 0.41418036818504333, + "grad_norm": 16.324495315551758, + "learning_rate": 1.0098157099674987e-09, + "logits/chosen": 0.6692662835121155, + "logits/rejected": 0.6541492938995361, + "logps/chosen": -222.4796142578125, + "logps/ref_chosen": -60.963409423828125, + "logps/ref_rejected": -69.73353576660156, + "logps/rejected": -298.43182373046875, + "loss": 1.1259, + "margin_dpo/margin_mean": 67.1821060180664, + "margin_dpo/margin_std": 103.56056213378906, + "step": 645 + }, + { + "KL/chosen_KL_mean": -177.9462890625, + "KL/mean": -210.1239013671875, + "KL/rejected_KL_mean": -242.301513671875, + "KL/std": 117.13391876220703, + "epoch": 0.9765684051398337, + "fcm_dpo/beta": 0.0055798254907131195, + "fcm_dpo/delta": 0.04242541640996933, + "fcm_dpo/margin": 64.35521697998047, + "fcm_dpo/q_t": 0.41991138458251953, + "grad_norm": 12.960869789123535, + "learning_rate": 8.945768539031783e-10, + "logits/chosen": 0.7192285060882568, + "logits/rejected": 0.6622798442840576, + "logps/chosen": -240.23635864257812, + "logps/ref_chosen": -62.290069580078125, + "logps/ref_rejected": -85.54812622070312, + "logps/rejected": -327.8496398925781, + "loss": 1.1636, + "margin_dpo/margin_mean": 64.35521697998047, + "margin_dpo/margin_std": 120.24072265625, + "step": 646 + }, + { + "KL/chosen_KL_mean": -175.16905212402344, + "KL/mean": -225.5684051513672, + "KL/rejected_KL_mean": -275.9677734375, + "KL/std": 116.21209716796875, + "epoch": 0.9780801209372638, + "fcm_dpo/beta": 0.005468328017741442, + "fcm_dpo/delta": -0.1598251312971115, + "fcm_dpo/margin": 100.79869842529297, + "fcm_dpo/q_t": 0.3744158446788788, + "grad_norm": 15.467833518981934, + "learning_rate": 7.863060120144316e-10, + "logits/chosen": 0.7321820259094238, + "logits/rejected": 0.6378551721572876, + "logps/chosen": -242.6849365234375, + "logps/ref_chosen": -67.515869140625, + "logps/ref_rejected": -101.50871276855469, + "logps/rejected": -377.4764404296875, + "loss": 0.9981, + "margin_dpo/margin_mean": 100.79869079589844, + "margin_dpo/margin_std": 113.80744934082031, + "step": 647 + }, + { + "KL/chosen_KL_mean": -175.15017700195312, + "KL/mean": -209.93740844726562, + "KL/rejected_KL_mean": -244.72459411621094, + "KL/std": 101.48037719726562, + "epoch": 0.9795918367346939, + "fcm_dpo/beta": 0.005409829318523407, + "fcm_dpo/delta": 0.0242181196808815, + "fcm_dpo/margin": 69.57443237304688, + "fcm_dpo/q_t": 0.41655397415161133, + "grad_norm": 14.243342399597168, + "learning_rate": 6.850062128694045e-10, + "logits/chosen": 0.6601248383522034, + "logits/rejected": 0.6005022525787354, + "logps/chosen": -239.74610900878906, + "logps/ref_chosen": -64.59593963623047, + "logps/ref_rejected": -83.384033203125, + "logps/rejected": -328.108642578125, + "loss": 1.1685, + "margin_dpo/margin_mean": 69.57443237304688, + "margin_dpo/margin_std": 131.31674194335938, + "step": 648 + }, + { + "KL/chosen_KL_mean": -156.38809204101562, + "KL/mean": -193.30819702148438, + "KL/rejected_KL_mean": -230.2283172607422, + "KL/std": 100.22056579589844, + "epoch": 0.981103552532124, + "fcm_dpo/beta": 0.0054200924932956696, + "fcm_dpo/delta": -0.0005342736840248108, + "fcm_dpo/margin": 73.8402328491211, + "fcm_dpo/q_t": 0.4088748097419739, + "grad_norm": 17.24158477783203, + "learning_rate": 5.906802900412788e-10, + "logits/chosen": 0.7577875256538391, + "logits/rejected": 0.696466326713562, + "logps/chosen": -205.69773864746094, + "logps/ref_chosen": -49.30964660644531, + "logps/ref_rejected": -73.73710632324219, + "logps/rejected": -303.9654235839844, + "loss": 1.1238, + "margin_dpo/margin_mean": 73.84024047851562, + "margin_dpo/margin_std": 118.47221374511719, + "step": 649 + }, + { + "KL/chosen_KL_mean": -154.02386474609375, + "KL/mean": -191.7683868408203, + "KL/rejected_KL_mean": -229.51287841796875, + "KL/std": 102.37055969238281, + "epoch": 0.982615268329554, + "fcm_dpo/beta": 0.0054198214784264565, + "fcm_dpo/delta": -0.009670298546552658, + "fcm_dpo/margin": 75.4890365600586, + "fcm_dpo/q_t": 0.40784576535224915, + "grad_norm": 13.829277038574219, + "learning_rate": 5.033308820289184e-10, + "logits/chosen": 0.7949092388153076, + "logits/rejected": 0.7321877479553223, + "logps/chosen": -209.08712768554688, + "logps/ref_chosen": -55.06325912475586, + "logps/ref_rejected": -77.39610290527344, + "logps/rejected": -306.90899658203125, + "loss": 1.1243, + "margin_dpo/margin_mean": 75.48902893066406, + "margin_dpo/margin_std": 123.73271179199219, + "step": 650 + }, + { + "KL/chosen_KL_mean": -151.89053344726562, + "KL/mean": -184.58779907226562, + "KL/rejected_KL_mean": -217.28509521484375, + "KL/std": 101.58538818359375, + "epoch": 0.9841269841269841, + "fcm_dpo/beta": 0.005477217026054859, + "fcm_dpo/delta": 0.043183956295251846, + "fcm_dpo/margin": 65.39456939697266, + "fcm_dpo/q_t": 0.42023220658302307, + "grad_norm": 13.305492401123047, + "learning_rate": 4.2296043218295606e-10, + "logits/chosen": 0.775327205657959, + "logits/rejected": 0.703120768070221, + "logps/chosen": -205.9556884765625, + "logps/ref_chosen": -54.065162658691406, + "logps/ref_rejected": -77.79080200195312, + "logps/rejected": -295.0758972167969, + "loss": 1.1409, + "margin_dpo/margin_mean": 65.39457702636719, + "margin_dpo/margin_std": 107.34722900390625, + "step": 651 + }, + { + "KL/chosen_KL_mean": -180.54974365234375, + "KL/mean": -211.1258544921875, + "KL/rejected_KL_mean": -241.70196533203125, + "KL/std": 111.74209594726562, + "epoch": 0.9856386999244142, + "fcm_dpo/beta": 0.0055029550567269325, + "fcm_dpo/delta": 0.06549069285392761, + "fcm_dpo/margin": 61.152244567871094, + "fcm_dpo/q_t": 0.4275883436203003, + "grad_norm": 15.900915145874023, + "learning_rate": 3.4957118863768176e-10, + "logits/chosen": 0.7466914653778076, + "logits/rejected": 0.6997571587562561, + "logps/chosen": -244.19003295898438, + "logps/ref_chosen": -63.64030456542969, + "logps/ref_rejected": -78.86882019042969, + "logps/rejected": -320.57080078125, + "loss": 1.2034, + "margin_dpo/margin_mean": 61.15224075317383, + "margin_dpo/margin_std": 134.0066680908203, + "step": 652 + }, + { + "KL/chosen_KL_mean": -158.57315063476562, + "KL/mean": -195.68978881835938, + "KL/rejected_KL_mean": -232.806396484375, + "KL/std": 99.21732330322266, + "epoch": 0.9871504157218443, + "fcm_dpo/beta": 0.005554167553782463, + "fcm_dpo/delta": -0.012951435521245003, + "fcm_dpo/margin": 74.23324584960938, + "fcm_dpo/q_t": 0.4068150818347931, + "grad_norm": 15.474823951721191, + "learning_rate": 2.831652042480093e-10, + "logits/chosen": 0.677382230758667, + "logits/rejected": 0.6351486444473267, + "logps/chosen": -220.2415313720703, + "logps/ref_chosen": -61.668373107910156, + "logps/ref_rejected": -73.83012390136719, + "logps/rejected": -306.63653564453125, + "loss": 1.1146, + "margin_dpo/margin_mean": 74.23324584960938, + "margin_dpo/margin_std": 116.3241958618164, + "step": 653 + }, + { + "KL/chosen_KL_mean": -166.5314178466797, + "KL/mean": -201.0532989501953, + "KL/rejected_KL_mean": -235.5751953125, + "KL/std": 117.71331787109375, + "epoch": 0.9886621315192744, + "fcm_dpo/beta": 0.0054998332634568214, + "fcm_dpo/delta": 0.019461914896965027, + "fcm_dpo/margin": 69.04376220703125, + "fcm_dpo/q_t": 0.41467058658599854, + "grad_norm": 14.01585865020752, + "learning_rate": 2.2374433653205016e-10, + "logits/chosen": 0.6830437183380127, + "logits/rejected": 0.5889010429382324, + "logps/chosen": -224.0996856689453, + "logps/ref_chosen": -57.568267822265625, + "logps/ref_rejected": -87.74789428710938, + "logps/rejected": -323.3230895996094, + "loss": 1.1355, + "margin_dpo/margin_mean": 69.04376220703125, + "margin_dpo/margin_std": 108.8658447265625, + "step": 654 + }, + { + "KL/chosen_KL_mean": -131.15521240234375, + "KL/mean": -182.16342163085938, + "KL/rejected_KL_mean": -233.171630859375, + "KL/std": 102.94525146484375, + "epoch": 0.9901738473167044, + "fcm_dpo/beta": 0.005375571083277464, + "fcm_dpo/delta": -0.16030044853687286, + "fcm_dpo/margin": 102.01641082763672, + "fcm_dpo/q_t": 0.37344735860824585, + "grad_norm": 12.33569049835205, + "learning_rate": 1.7131024761923852e-10, + "logits/chosen": 0.7126660346984863, + "logits/rejected": 0.627194881439209, + "logps/chosen": -183.3023681640625, + "logps/ref_chosen": -52.14714813232422, + "logps/ref_rejected": -80.85014343261719, + "logps/rejected": -314.0217590332031, + "loss": 0.9847, + "margin_dpo/margin_mean": 102.01641082763672, + "margin_dpo/margin_std": 100.66226196289062, + "step": 655 + }, + { + "KL/chosen_KL_mean": -172.86846923828125, + "KL/mean": -209.77020263671875, + "KL/rejected_KL_mean": -246.67193603515625, + "KL/std": 104.92657470703125, + "epoch": 0.9916855631141346, + "fcm_dpo/beta": 0.0053861262276768684, + "fcm_dpo/delta": 0.0025822424795478582, + "fcm_dpo/margin": 73.80348205566406, + "fcm_dpo/q_t": 0.40978139638900757, + "grad_norm": 11.625679969787598, + "learning_rate": 1.2586440420372934e-10, + "logits/chosen": 0.6522502899169922, + "logits/rejected": 0.6087788343429565, + "logps/chosen": -246.12518310546875, + "logps/ref_chosen": -73.25672912597656, + "logps/ref_rejected": -85.35127258300781, + "logps/rejected": -332.023193359375, + "loss": 1.1199, + "margin_dpo/margin_mean": 73.80347442626953, + "margin_dpo/margin_std": 116.68196105957031, + "step": 656 + }, + { + "KL/chosen_KL_mean": -153.27273559570312, + "KL/mean": -199.0772247314453, + "KL/rejected_KL_mean": -244.88169860839844, + "KL/std": 116.36036682128906, + "epoch": 0.9931972789115646, + "fcm_dpo/beta": 0.005317493341863155, + "fcm_dpo/delta": -0.09151517599821091, + "fcm_dpo/margin": 91.60895538330078, + "fcm_dpo/q_t": 0.3905050456523895, + "grad_norm": 11.552481651306152, + "learning_rate": 8.740807750345913e-11, + "logits/chosen": 0.7955504059791565, + "logits/rejected": 0.7159940004348755, + "logps/chosen": -202.99612426757812, + "logps/ref_chosen": -49.72339630126953, + "logps/ref_rejected": -75.1568603515625, + "logps/rejected": -320.03857421875, + "loss": 1.0617, + "margin_dpo/margin_mean": 91.60896301269531, + "margin_dpo/margin_std": 126.87873840332031, + "step": 657 + }, + { + "KL/chosen_KL_mean": -163.15029907226562, + "KL/mean": -200.5810089111328, + "KL/rejected_KL_mean": -238.01170349121094, + "KL/std": 119.92159271240234, + "epoch": 0.9947089947089947, + "fcm_dpo/beta": 0.005321793258190155, + "fcm_dpo/delta": 0.0008049830794334412, + "fcm_dpo/margin": 74.86140441894531, + "fcm_dpo/q_t": 0.41124677658081055, + "grad_norm": 12.123971939086914, + "learning_rate": 5.594234322453539e-11, + "logits/chosen": 0.7718138098716736, + "logits/rejected": 0.7259960770606995, + "logps/chosen": -226.1966552734375, + "logps/ref_chosen": -63.04634094238281, + "logps/ref_rejected": -83.44963073730469, + "logps/rejected": -321.4613342285156, + "loss": 1.1559, + "margin_dpo/margin_mean": 74.86140441894531, + "margin_dpo/margin_std": 139.01248168945312, + "step": 658 + }, + { + "KL/chosen_KL_mean": -165.0323028564453, + "KL/mean": -194.35629272460938, + "KL/rejected_KL_mean": -223.68023681640625, + "KL/std": 104.22109985351562, + "epoch": 0.9962207105064248, + "fcm_dpo/beta": 0.005332798697054386, + "fcm_dpo/delta": 0.09000739455223083, + "fcm_dpo/margin": 58.64795684814453, + "fcm_dpo/q_t": 0.42909497022628784, + "grad_norm": 17.333219528198242, + "learning_rate": 3.146808153123293e-11, + "logits/chosen": 0.7888455390930176, + "logits/rejected": 0.7255823612213135, + "logps/chosen": -220.1125030517578, + "logps/ref_chosen": -55.0802001953125, + "logps/ref_rejected": -71.91049194335938, + "logps/rejected": -295.59075927734375, + "loss": 1.195, + "margin_dpo/margin_mean": 58.6479606628418, + "margin_dpo/margin_std": 118.41319274902344, + "step": 659 + }, + { + "KL/chosen_KL_mean": -156.80108642578125, + "KL/mean": -203.4031982421875, + "KL/rejected_KL_mean": -250.00535583496094, + "KL/std": 112.88031768798828, + "epoch": 0.9977324263038548, + "fcm_dpo/beta": 0.005306331440806389, + "fcm_dpo/delta": -0.09946230053901672, + "fcm_dpo/margin": 93.20425415039062, + "fcm_dpo/q_t": 0.38910984992980957, + "grad_norm": 13.725273132324219, + "learning_rate": 1.3985977021235829e-11, + "logits/chosen": 0.8498274087905884, + "logits/rejected": 0.7831590175628662, + "logps/chosen": -211.32699584960938, + "logps/ref_chosen": -54.525917053222656, + "logps/ref_rejected": -81.23604583740234, + "logps/rejected": -331.24139404296875, + "loss": 1.04, + "margin_dpo/margin_mean": 93.20425415039062, + "margin_dpo/margin_std": 120.00505828857422, + "step": 660 + }, + { + "KL/chosen_KL_mean": -181.01803588867188, + "KL/mean": -210.42062377929688, + "KL/rejected_KL_mean": -239.8232421875, + "KL/std": 113.02836608886719, + "epoch": 0.999244142101285, + "fcm_dpo/beta": 0.005359668284654617, + "fcm_dpo/delta": 0.08725185692310333, + "fcm_dpo/margin": 58.80520248413086, + "fcm_dpo/q_t": 0.4290045499801636, + "grad_norm": 12.07127857208252, + "learning_rate": 3.4965187065971735e-12, + "logits/chosen": 0.7182981967926025, + "logits/rejected": 0.6424450874328613, + "logps/chosen": -241.3906707763672, + "logps/ref_chosen": -60.37263870239258, + "logps/ref_rejected": -77.42874145507812, + "logps/rejected": -317.25201416015625, + "loss": 1.2084, + "margin_dpo/margin_mean": 58.80520248413086, + "margin_dpo/margin_std": 129.78515625, + "step": 661 + }, + { + "epoch": 0.999244142101285, + "step": 661, + "total_flos": 0.0, + "train_loss": 1.1488945873710643, + "train_runtime": 1653.0477, + "train_samples_per_second": 25.611, + "train_steps_per_second": 0.4 + } + ], + "logging_steps": 1, + "max_steps": 661, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}