From b90154e69ada3e0815dd918f125b60a14ac0ce2b Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Wed, 20 May 2026 02:23:33 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.5 Source: Original Platform --- .gitattributes | 36 + README.md | 62 + all_results.json | 9 + config.json | 29 + generation_config.json | 9 + model-00001-of-00007.safetensors | 3 + model-00002-of-00007.safetensors | 3 + model-00003-of-00007.safetensors | 3 + model-00004-of-00007.safetensors | 3 + model-00005-of-00007.safetensors | 3 + model-00006-of-00007.safetensors | 3 + model-00007-of-00007.safetensors | 3 + model.safetensors.index.json | 298 + special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2064 ++++ train_results.json | 9 + trainer_state.json | 15246 +++++++++++++++++++++++++++++ 18 files changed, 17809 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00007.safetensors create mode 100644 model-00002-of-00007.safetensors create mode 100644 model-00003-of-00007.safetensors create mode 100644 model-00004-of-00007.safetensors create mode 100644 model-00005-of-00007.safetensors create mode 100644 model-00006-of-00007.safetensors create mode 100644 model-00007-of-00007.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..c04cdbe --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-harmless-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.5 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.5 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-harmless-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-harmless-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..2aef38a --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.999244142101285, + "total_flos": 0.0, + "train_loss": 1.132411924234497, + "train_runtime": 1654.1278, + "train_samples": 42336, + "train_samples_per_second": 25.594, + "train_steps_per_second": 0.4 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..21c5ef5 --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e4dcc06be415eddbf076d66d9fab9640aa0237797e9a4592decbdb7c6a9b454 +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..c7f47db --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e2999d56530cf7c96ef653ad5ee9efeef0cd641597603aaf83737756bda7909 +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..8969171 --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40eb1105cb313cd37e4afe2a686699970a3bdd7b8f4f93d729ac50d60a63ce16 +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..573d059 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e871495f2712627b5275794b10e248deb356beff9c13c8cf7271193979372e32 +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..afb3bc8 --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4e024191137048b510609db7e39955be84f37ed1343f0e8f3bf0a3c51fb8e1d +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..7beed45 --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fccfd164793f1e811a58ebca3818218da67dcdf31da54e189071e7742cfef34f +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..3db7242 --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5706100ce31556bebe8f97f19c4ef862a7059babd525ed24e3b5657a5b0e5bd2 +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..2aef38a --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.999244142101285, + "total_flos": 0.0, + "train_loss": 1.132411924234497, + "train_runtime": 1654.1278, + "train_samples": 42336, + "train_samples_per_second": 25.594, + "train_steps_per_second": 0.4 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..84db659 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15246 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.999244142101285, + "eval_steps": 200, + "global_step": 661, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.02867889404296875, + "KL/mean": 0.029354453086853027, + "KL/rejected_KL_mean": 0.030029296875, + "KL/std": 0.2071000635623932, + "epoch": 0.0015117157974300832, + "fcm_dpo/beta": 0.10407507419586182, + "fcm_dpo/delta": 0.19971171021461487, + "fcm_dpo/margin": -0.0013532638549804688, + "fcm_dpo/q_t": 0.5000380277633667, + "grad_norm": 29.368080139160156, + "learning_rate": 0.0, + "logits/chosen": 0.13337239623069763, + "logits/rejected": 0.12492949515581131, + "logps/chosen": -64.5841293334961, + "logps/ref_chosen": -64.61280822753906, + "logps/ref_rejected": -64.17195129394531, + "logps/rejected": -64.14192199707031, + "loss": 1.3866, + "margin_dpo/margin_mean": -0.0013527870178222656, + "margin_dpo/margin_std": 0.2561596930027008, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.00289154052734375, + "KL/mean": -0.021616414189338684, + "KL/rejected_KL_mean": -0.04033660888671875, + "KL/std": 0.19624735414981842, + "epoch": 0.0030234315948601664, + "fcm_dpo/beta": 0.10614679753780365, + "fcm_dpo/delta": 0.19520045816898346, + "fcm_dpo/margin": 0.037450045347213745, + "fcm_dpo/q_t": 0.49902579188346863, + "grad_norm": 29.559764862060547, + "learning_rate": 7.462686567164179e-09, + "logits/chosen": 0.09414851665496826, + "logits/rejected": 0.07363267242908478, + "logps/chosen": -56.101890563964844, + "logps/ref_chosen": -56.0989990234375, + "logps/ref_rejected": -66.59971618652344, + "logps/rejected": -66.64006042480469, + "loss": 1.3824, + "margin_dpo/margin_mean": 0.03744968771934509, + "margin_dpo/margin_std": 0.27811938524246216, + "step": 2 + }, + { + "KL/chosen_KL_mean": -0.014759063720703125, + "KL/mean": 0.016700327396392822, + "KL/rejected_KL_mean": 0.04815673828125, + "KL/std": 0.2423822283744812, + "epoch": 0.0045351473922902496, + "fcm_dpo/beta": 0.10821852833032608, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06291508674621582, + "fcm_dpo/q_t": 0.5017015337944031, + "grad_norm": 33.8881950378418, + "learning_rate": 1.4925373134328357e-08, + "logits/chosen": 0.09933048486709595, + "logits/rejected": 0.061286427080631256, + "logps/chosen": -65.47201538085938, + "logps/ref_chosen": -65.45726013183594, + "logps/ref_rejected": -90.82853698730469, + "logps/rejected": -90.78038024902344, + "loss": 1.3934, + "margin_dpo/margin_mean": -0.06291499733924866, + "margin_dpo/margin_std": 0.32749801874160767, + "step": 3 + }, + { + "KL/chosen_KL_mean": 0.0326690673828125, + "KL/mean": 0.010438680648803711, + "KL/rejected_KL_mean": -0.011798858642578125, + "KL/std": 0.2497587949037552, + "epoch": 0.006046863189720333, + "fcm_dpo/beta": 0.11485374718904495, + "fcm_dpo/delta": 0.39507198333740234, + "fcm_dpo/margin": 0.04446643590927124, + "fcm_dpo/q_t": 0.4987691640853882, + "grad_norm": 38.972347259521484, + "learning_rate": 2.2388059701492534e-08, + "logits/chosen": 0.10525606572628021, + "logits/rejected": 0.08916881680488586, + "logps/chosen": -76.8275146484375, + "logps/ref_chosen": -76.86018371582031, + "logps/ref_rejected": -79.91523742675781, + "logps/rejected": -79.92703247070312, + "loss": 1.3815, + "margin_dpo/margin_mean": 0.044465720653533936, + "margin_dpo/margin_std": 0.3332005739212036, + "step": 4 + }, + { + "KL/chosen_KL_mean": 0.00232696533203125, + "KL/mean": -0.004125773906707764, + "KL/rejected_KL_mean": -0.0105743408203125, + "KL/std": 0.21744784712791443, + "epoch": 0.007558578987150416, + "fcm_dpo/beta": 0.11947908252477646, + "fcm_dpo/delta": 0.19778770208358765, + "fcm_dpo/margin": 0.012903928756713867, + "fcm_dpo/q_t": 0.4996222257614136, + "grad_norm": 35.309425354003906, + "learning_rate": 2.9850746268656714e-08, + "logits/chosen": 0.08232656121253967, + "logits/rejected": 0.043468791991472244, + "logps/chosen": -62.96901321411133, + "logps/ref_chosen": -62.97134017944336, + "logps/ref_rejected": -79.9192123413086, + "logps/rejected": -79.9297866821289, + "loss": 1.3851, + "margin_dpo/margin_mean": 0.012904614210128784, + "margin_dpo/margin_std": 0.3334115147590637, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.06997871398925781, + "KL/mean": -0.020912393927574158, + "KL/rejected_KL_mean": 0.028154373168945312, + "KL/std": 0.25535884499549866, + "epoch": 0.009070294784580499, + "fcm_dpo/beta": 0.1218419224023819, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0981263816356659, + "fcm_dpo/q_t": 0.5029854774475098, + "grad_norm": 36.648460388183594, + "learning_rate": 3.731343283582089e-08, + "logits/chosen": 0.16564564406871796, + "logits/rejected": 0.12472124397754669, + "logps/chosen": -51.377342224121094, + "logps/ref_chosen": -51.30736541748047, + "logps/ref_rejected": -82.77239227294922, + "logps/rejected": -82.7442398071289, + "loss": 1.3989, + "margin_dpo/margin_mean": -0.09812664985656738, + "margin_dpo/margin_std": 0.4009813666343689, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.015993118286132812, + "KL/mean": 0.01990698277950287, + "KL/rejected_KL_mean": 0.0238189697265625, + "KL/std": 0.1881713569164276, + "epoch": 0.010582010582010581, + "fcm_dpo/beta": 0.1268007904291153, + "fcm_dpo/delta": 0.1994636505842209, + "fcm_dpo/margin": -0.007827132940292358, + "fcm_dpo/q_t": 0.5002532005310059, + "grad_norm": 34.524017333984375, + "learning_rate": 4.477611940298507e-08, + "logits/chosen": 0.00693818274885416, + "logits/rejected": -0.03615080937743187, + "logps/chosen": -51.44342041015625, + "logps/ref_chosen": -51.45941162109375, + "logps/ref_rejected": -66.3828125, + "logps/rejected": -66.35899353027344, + "loss": 1.3875, + "margin_dpo/margin_mean": -0.007827192544937134, + "margin_dpo/margin_std": 0.2439565658569336, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.019863128662109375, + "KL/mean": -0.010127946734428406, + "KL/rejected_KL_mean": -0.040111541748046875, + "KL/std": 0.24710315465927124, + "epoch": 0.012093726379440665, + "fcm_dpo/beta": 0.13449162244796753, + "fcm_dpo/delta": 0.3923080265522003, + "fcm_dpo/margin": 0.059976786375045776, + "fcm_dpo/q_t": 0.49807968735694885, + "grad_norm": 38.144073486328125, + "learning_rate": 5.223880597014925e-08, + "logits/chosen": 0.07271274924278259, + "logits/rejected": 0.05057502165436745, + "logps/chosen": -62.17768096923828, + "logps/ref_chosen": -62.197547912597656, + "logps/ref_rejected": -74.66180419921875, + "logps/rejected": -74.70191955566406, + "loss": 1.3789, + "margin_dpo/margin_mean": 0.0599762499332428, + "margin_dpo/margin_std": 0.34887492656707764, + "step": 8 + }, + { + "KL/chosen_KL_mean": -0.031890869140625, + "KL/mean": -0.02231958508491516, + "KL/rejected_KL_mean": -0.012752532958984375, + "KL/std": 0.23765209317207336, + "epoch": 0.013605442176870748, + "fcm_dpo/beta": 0.1427447646856308, + "fcm_dpo/delta": 0.199895977973938, + "fcm_dpo/margin": -0.019134581089019775, + "fcm_dpo/q_t": 0.5006834268569946, + "grad_norm": 45.233642578125, + "learning_rate": 5.970149253731343e-08, + "logits/chosen": 0.16762521862983704, + "logits/rejected": 0.10846076905727386, + "logps/chosen": -55.66161346435547, + "logps/ref_chosen": -55.629722595214844, + "logps/ref_rejected": -86.21221923828125, + "logps/rejected": -86.2249755859375, + "loss": 1.3897, + "margin_dpo/margin_mean": -0.019134342670440674, + "margin_dpo/margin_std": 0.3501278758049011, + "step": 9 + }, + { + "KL/chosen_KL_mean": -0.02667236328125, + "KL/mean": -0.001292884349822998, + "KL/rejected_KL_mean": 0.024082183837890625, + "KL/std": 0.2424069046974182, + "epoch": 0.015117157974300832, + "fcm_dpo/beta": 0.1427447646856308, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.05076020956039429, + "fcm_dpo/q_t": 0.5018121004104614, + "grad_norm": 42.52825927734375, + "learning_rate": 6.71641791044776e-08, + "logits/chosen": 0.1346004456281662, + "logits/rejected": 0.10349121689796448, + "logps/chosen": -62.7172737121582, + "logps/ref_chosen": -62.69060134887695, + "logps/ref_rejected": -90.610107421875, + "logps/rejected": -90.58602905273438, + "loss": 1.3942, + "margin_dpo/margin_mean": -0.05076006054878235, + "margin_dpo/margin_std": 0.36290353536605835, + "step": 10 + }, + { + "KL/chosen_KL_mean": -0.0164031982421875, + "KL/mean": -0.019405141472816467, + "KL/rejected_KL_mean": -0.022411346435546875, + "KL/std": 0.19960369169712067, + "epoch": 0.016628873771730914, + "fcm_dpo/beta": 0.1484941840171814, + "fcm_dpo/delta": 0.19743794202804565, + "fcm_dpo/margin": 0.006010264158248901, + "fcm_dpo/q_t": 0.4998025596141815, + "grad_norm": 43.63276672363281, + "learning_rate": 7.462686567164178e-08, + "logits/chosen": 0.11837198585271835, + "logits/rejected": 0.11128821223974228, + "logps/chosen": -65.78352355957031, + "logps/ref_chosen": -65.76712036132812, + "logps/ref_rejected": -72.4764633178711, + "logps/rejected": -72.49887084960938, + "loss": 1.3859, + "margin_dpo/margin_mean": 0.0060103535652160645, + "margin_dpo/margin_std": 0.3046831786632538, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.016712188720703125, + "KL/mean": -0.02420225739479065, + "KL/rejected_KL_mean": -0.0316925048828125, + "KL/std": 0.19888459146022797, + "epoch": 0.018140589569160998, + "fcm_dpo/beta": 0.15447428822517395, + "fcm_dpo/delta": 0.19740913808345795, + "fcm_dpo/margin": 0.014980614185333252, + "fcm_dpo/q_t": 0.499448299407959, + "grad_norm": 43.49526596069336, + "learning_rate": 8.208955223880596e-08, + "logits/chosen": 0.04357253015041351, + "logits/rejected": 0.02697162888944149, + "logps/chosen": -60.72160339355469, + "logps/ref_chosen": -60.704891204833984, + "logps/ref_rejected": -69.41564178466797, + "logps/rejected": -69.44733428955078, + "loss": 1.3844, + "margin_dpo/margin_mean": 0.014980673789978027, + "margin_dpo/margin_std": 0.2739675045013428, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.03954315185546875, + "KL/mean": 0.020595967769622803, + "KL/rejected_KL_mean": 0.00165557861328125, + "KL/std": 0.19518539309501648, + "epoch": 0.019652305366591082, + "fcm_dpo/beta": 0.15752489864826202, + "fcm_dpo/delta": 0.19368299841880798, + "fcm_dpo/margin": 0.037889957427978516, + "fcm_dpo/q_t": 0.4985363185405731, + "grad_norm": 45.36277770996094, + "learning_rate": 8.955223880597014e-08, + "logits/chosen": 0.09215953946113586, + "logits/rejected": 0.031327713280916214, + "logps/chosen": -49.869712829589844, + "logps/ref_chosen": -49.90925598144531, + "logps/ref_rejected": -92.37818145751953, + "logps/rejected": -92.37652587890625, + "loss": 1.3806, + "margin_dpo/margin_mean": 0.03788921236991882, + "margin_dpo/margin_std": 0.25264811515808105, + "step": 13 + }, + { + "KL/chosen_KL_mean": -0.005786895751953125, + "KL/mean": -0.01808251440525055, + "KL/rejected_KL_mean": -0.030376434326171875, + "KL/std": 0.1731014847755432, + "epoch": 0.021164021164021163, + "fcm_dpo/beta": 0.17045539617538452, + "fcm_dpo/delta": 0.39592236280441284, + "fcm_dpo/margin": 0.024587482213974, + "fcm_dpo/q_t": 0.49898165464401245, + "grad_norm": 50.24029541015625, + "learning_rate": 9.701492537313432e-08, + "logits/chosen": 0.09761021286249161, + "logits/rejected": 0.07951986789703369, + "logps/chosen": -60.62458038330078, + "logps/ref_chosen": -60.61879348754883, + "logps/ref_rejected": -71.79306030273438, + "logps/rejected": -71.82344055175781, + "loss": 1.3825, + "margin_dpo/margin_mean": 0.024587631225585938, + "margin_dpo/margin_std": 0.23990672826766968, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.026651382446289062, + "KL/mean": -0.031851932406425476, + "KL/rejected_KL_mean": -0.037044525146484375, + "KL/std": 0.2409576177597046, + "epoch": 0.022675736961451247, + "fcm_dpo/beta": 0.18080593645572662, + "fcm_dpo/delta": 0.19737780094146729, + "fcm_dpo/margin": 0.010393649339675903, + "fcm_dpo/q_t": 0.49955692887306213, + "grad_norm": 60.253578186035156, + "learning_rate": 1.044776119402985e-07, + "logits/chosen": 0.09280131757259369, + "logits/rejected": 0.04811263829469681, + "logps/chosen": -63.49618911743164, + "logps/ref_chosen": -63.46953582763672, + "logps/ref_rejected": -88.88951110839844, + "logps/rejected": -88.92655944824219, + "loss": 1.3853, + "margin_dpo/margin_mean": 0.010393589735031128, + "margin_dpo/margin_std": 0.32523539662361145, + "step": 15 + }, + { + "KL/chosen_KL_mean": 0.0018177032470703125, + "KL/mean": -0.00604432076215744, + "KL/rejected_KL_mean": -0.01390838623046875, + "KL/std": 0.22720113396644592, + "epoch": 0.02418745275888133, + "fcm_dpo/beta": 0.18434685468673706, + "fcm_dpo/delta": 0.19210314750671387, + "fcm_dpo/margin": 0.01572038233280182, + "fcm_dpo/q_t": 0.49929431080818176, + "grad_norm": 49.553138732910156, + "learning_rate": 1.1194029850746268e-07, + "logits/chosen": 0.13143643736839294, + "logits/rejected": 0.09331878274679184, + "logps/chosen": -46.530479431152344, + "logps/ref_chosen": -46.53229904174805, + "logps/ref_rejected": -74.27533721923828, + "logps/rejected": -74.28924560546875, + "loss": 1.3842, + "margin_dpo/margin_mean": 0.01572073996067047, + "margin_dpo/margin_std": 0.3393726944923401, + "step": 16 + }, + { + "KL/chosen_KL_mean": -0.023601531982421875, + "KL/mean": -0.020472899079322815, + "KL/rejected_KL_mean": -0.01734161376953125, + "KL/std": 0.20937620103359222, + "epoch": 0.025699168556311415, + "fcm_dpo/beta": 0.1953100562095642, + "fcm_dpo/delta": 0.19371725618839264, + "fcm_dpo/margin": -0.006255239248275757, + "fcm_dpo/q_t": 0.5003689527511597, + "grad_norm": 64.56086730957031, + "learning_rate": 1.1940298507462686e-07, + "logits/chosen": 0.056045085191726685, + "logits/rejected": 0.03731035813689232, + "logps/chosen": -64.10143280029297, + "logps/ref_chosen": -64.07783508300781, + "logps/ref_rejected": -86.40876770019531, + "logps/rejected": -86.42610931396484, + "loss": 1.3884, + "margin_dpo/margin_mean": -0.006255537271499634, + "margin_dpo/margin_std": 0.2892005145549774, + "step": 17 + }, + { + "KL/chosen_KL_mean": 0.037494659423828125, + "KL/mean": 0.004080817103385925, + "KL/rejected_KL_mean": -0.029333114624023438, + "KL/std": 0.19442421197891235, + "epoch": 0.027210884353741496, + "fcm_dpo/beta": 0.20706477761268616, + "fcm_dpo/delta": 0.386584997177124, + "fcm_dpo/margin": 0.06682577729225159, + "fcm_dpo/q_t": 0.49664899706840515, + "grad_norm": 57.38557434082031, + "learning_rate": 1.2686567164179106e-07, + "logits/chosen": 0.08666019141674042, + "logits/rejected": 0.04165633022785187, + "logps/chosen": -44.83684158325195, + "logps/ref_chosen": -44.87433624267578, + "logps/ref_rejected": -70.97604370117188, + "logps/rejected": -71.00537872314453, + "loss": 1.3732, + "margin_dpo/margin_mean": 0.06682571768760681, + "margin_dpo/margin_std": 0.26390185952186584, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.015768051147460938, + "KL/mean": 0.011087000370025635, + "KL/rejected_KL_mean": 0.006404876708984375, + "KL/std": 0.2442140132188797, + "epoch": 0.02872260015117158, + "fcm_dpo/beta": 0.22403427958488464, + "fcm_dpo/delta": 0.3979843854904175, + "fcm_dpo/margin": 0.00936955213546753, + "fcm_dpo/q_t": 0.4994935691356659, + "grad_norm": 69.56298065185547, + "learning_rate": 1.343283582089552e-07, + "logits/chosen": 0.07334445416927338, + "logits/rejected": 0.05974186211824417, + "logps/chosen": -68.14404296875, + "logps/ref_chosen": -68.1598129272461, + "logps/ref_rejected": -81.17138671875, + "logps/rejected": -81.16498565673828, + "loss": 1.3856, + "margin_dpo/margin_mean": 0.009370237588882446, + "margin_dpo/margin_std": 0.33312809467315674, + "step": 19 + }, + { + "KL/chosen_KL_mean": -0.0043697357177734375, + "KL/mean": 0.011444643139839172, + "KL/rejected_KL_mean": 0.027263641357421875, + "KL/std": 0.21824133396148682, + "epoch": 0.030234315948601664, + "fcm_dpo/beta": 0.23310576379299164, + "fcm_dpo/delta": 0.19794167578220367, + "fcm_dpo/margin": -0.03163328766822815, + "fcm_dpo/q_t": 0.5018023252487183, + "grad_norm": 68.3042984008789, + "learning_rate": 1.4179104477611938e-07, + "logits/chosen": 0.1361723095178604, + "logits/rejected": 0.1131054013967514, + "logps/chosen": -53.68292999267578, + "logps/ref_chosen": -53.67856216430664, + "logps/ref_rejected": -74.16911315917969, + "logps/rejected": -74.14185333251953, + "loss": 1.3948, + "margin_dpo/margin_mean": -0.031633466482162476, + "margin_dpo/margin_std": 0.3028011620044708, + "step": 20 + }, + { + "KL/chosen_KL_mean": 0.022541046142578125, + "KL/mean": 0.02938457578420639, + "KL/rejected_KL_mean": 0.036224365234375, + "KL/std": 0.23280031979084015, + "epoch": 0.031746031746031744, + "fcm_dpo/beta": 0.24255003035068512, + "fcm_dpo/delta": 0.19919079542160034, + "fcm_dpo/margin": -0.01367737352848053, + "fcm_dpo/q_t": 0.5008128881454468, + "grad_norm": 72.05040740966797, + "learning_rate": 1.4925373134328355e-07, + "logits/chosen": 0.09666962176561356, + "logits/rejected": 0.0715550035238266, + "logps/chosen": -64.67901611328125, + "logps/ref_chosen": -64.70155334472656, + "logps/ref_rejected": -81.02095031738281, + "logps/rejected": -80.98472595214844, + "loss": 1.3912, + "margin_dpo/margin_mean": -0.013677477836608887, + "margin_dpo/margin_std": 0.3375088572502136, + "step": 21 + }, + { + "KL/chosen_KL_mean": -0.03160858154296875, + "KL/mean": -0.027323782444000244, + "KL/rejected_KL_mean": -0.023040771484375, + "KL/std": 0.21388794481754303, + "epoch": 0.03325774754346183, + "fcm_dpo/beta": 0.2573118507862091, + "fcm_dpo/delta": 0.19680048525333405, + "fcm_dpo/margin": -0.008568942546844482, + "fcm_dpo/q_t": 0.5005735158920288, + "grad_norm": 75.60509490966797, + "learning_rate": 1.5671641791044775e-07, + "logits/chosen": 0.006399281322956085, + "logits/rejected": -0.0146299097687006, + "logps/chosen": -58.06760025024414, + "logps/ref_chosen": -58.03599166870117, + "logps/ref_rejected": -80.72721862792969, + "logps/rejected": -80.75025939941406, + "loss": 1.3901, + "margin_dpo/margin_mean": -0.008568882942199707, + "margin_dpo/margin_std": 0.3066539466381073, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.014617919921875, + "KL/mean": 0.009073600172996521, + "KL/rejected_KL_mean": 0.00353240966796875, + "KL/std": 0.2195100635290146, + "epoch": 0.03476946334089191, + "fcm_dpo/beta": 0.26226556301116943, + "fcm_dpo/delta": 0.1889042854309082, + "fcm_dpo/margin": 0.011082977056503296, + "fcm_dpo/q_t": 0.4992814064025879, + "grad_norm": 87.26220703125, + "learning_rate": 1.6417910447761193e-07, + "logits/chosen": 0.11942335218191147, + "logits/rejected": 0.09495219588279724, + "logps/chosen": -66.34146881103516, + "logps/ref_chosen": -66.35608673095703, + "logps/ref_rejected": -93.02769470214844, + "logps/rejected": -93.02415466308594, + "loss": 1.3846, + "margin_dpo/margin_mean": 0.011082679033279419, + "margin_dpo/margin_std": 0.29644426703453064, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.0018310546875, + "KL/mean": 0.010483011603355408, + "KL/rejected_KL_mean": 0.019138336181640625, + "KL/std": 0.19121773540973663, + "epoch": 0.036281179138321996, + "fcm_dpo/beta": 0.2780904471874237, + "fcm_dpo/delta": 0.19938357174396515, + "fcm_dpo/margin": -0.017304211854934692, + "fcm_dpo/q_t": 0.5012048482894897, + "grad_norm": 73.68643188476562, + "learning_rate": 1.716417910447761e-07, + "logits/chosen": 0.10710248351097107, + "logits/rejected": 0.07515440881252289, + "logps/chosen": -54.459407806396484, + "logps/ref_chosen": -54.461238861083984, + "logps/ref_rejected": -68.33817291259766, + "logps/rejected": -68.31903839111328, + "loss": 1.3924, + "margin_dpo/margin_mean": -0.017304420471191406, + "margin_dpo/margin_std": 0.25890859961509705, + "step": 24 + }, + { + "KL/chosen_KL_mean": -0.031110763549804688, + "KL/mean": -0.03343985974788666, + "KL/rejected_KL_mean": -0.035770416259765625, + "KL/std": 0.2234145700931549, + "epoch": 0.03779289493575208, + "fcm_dpo/beta": 0.289237916469574, + "fcm_dpo/delta": 0.19651614129543304, + "fcm_dpo/margin": 0.004654437303543091, + "fcm_dpo/q_t": 0.49969354271888733, + "grad_norm": 85.54322052001953, + "learning_rate": 1.7910447761194027e-07, + "logits/chosen": 0.10172505676746368, + "logits/rejected": 0.05041360855102539, + "logps/chosen": -60.035316467285156, + "logps/ref_chosen": -60.00420379638672, + "logps/ref_rejected": -90.47376251220703, + "logps/rejected": -90.50953674316406, + "loss": 1.3872, + "margin_dpo/margin_mean": 0.004654288291931152, + "margin_dpo/margin_std": 0.32953381538391113, + "step": 25 + }, + { + "KL/chosen_KL_mean": -0.03518486022949219, + "KL/mean": -0.012967824935913086, + "KL/rejected_KL_mean": 0.009246826171875, + "KL/std": 0.20040880143642426, + "epoch": 0.039304610733182165, + "fcm_dpo/beta": 0.289237916469574, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.04443258047103882, + "fcm_dpo/q_t": 0.5032066106796265, + "grad_norm": 87.03143310546875, + "learning_rate": 1.8656716417910447e-07, + "logits/chosen": 0.11158916354179382, + "logits/rejected": 0.09309649467468262, + "logps/chosen": -56.854339599609375, + "logps/ref_chosen": -56.81915283203125, + "logps/ref_rejected": -77.84333038330078, + "logps/rejected": -77.8340835571289, + "loss": 1.4012, + "margin_dpo/margin_mean": -0.04443329572677612, + "margin_dpo/margin_std": 0.31203919649124146, + "step": 26 + }, + { + "KL/chosen_KL_mean": -0.0073394775390625, + "KL/mean": 0.002377226948738098, + "KL/rejected_KL_mean": 0.012094497680664062, + "KL/std": 0.24115195870399475, + "epoch": 0.04081632653061224, + "fcm_dpo/beta": 0.289237916469574, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.019435137510299683, + "fcm_dpo/q_t": 0.5014033317565918, + "grad_norm": 83.701904296875, + "learning_rate": 1.9402985074626865e-07, + "logits/chosen": 0.12982380390167236, + "logits/rejected": 0.10382634401321411, + "logps/chosen": -62.88436508178711, + "logps/ref_chosen": -62.87702560424805, + "logps/ref_rejected": -71.34437561035156, + "logps/rejected": -71.33228302001953, + "loss": 1.3939, + "margin_dpo/margin_mean": -0.019435018301010132, + "margin_dpo/margin_std": 0.3067646324634552, + "step": 27 + }, + { + "KL/chosen_KL_mean": -0.024747848510742188, + "KL/mean": -0.01658952236175537, + "KL/rejected_KL_mean": -0.008432388305664062, + "KL/std": 0.2040315568447113, + "epoch": 0.042328042328042326, + "fcm_dpo/beta": 0.3005712330341339, + "fcm_dpo/delta": 0.19217611849308014, + "fcm_dpo/margin": -0.01631641387939453, + "fcm_dpo/q_t": 0.5012955665588379, + "grad_norm": 83.30447387695312, + "learning_rate": 2.0149253731343282e-07, + "logits/chosen": 0.0741531178355217, + "logits/rejected": 0.06495097279548645, + "logps/chosen": -59.858123779296875, + "logps/ref_chosen": -59.8333740234375, + "logps/ref_rejected": -70.39804077148438, + "logps/rejected": -70.4064712524414, + "loss": 1.393, + "margin_dpo/margin_mean": -0.01631644368171692, + "margin_dpo/margin_std": 0.2692261338233948, + "step": 28 + }, + { + "KL/chosen_KL_mean": -0.01944732666015625, + "KL/mean": -0.053507544100284576, + "KL/rejected_KL_mean": -0.08756256103515625, + "KL/std": 0.23613537847995758, + "epoch": 0.04383975812547241, + "fcm_dpo/beta": 0.3183423578739166, + "fcm_dpo/delta": 0.37896400690078735, + "fcm_dpo/margin": 0.0681174248456955, + "fcm_dpo/q_t": 0.494759202003479, + "grad_norm": 102.14923095703125, + "learning_rate": 2.08955223880597e-07, + "logits/chosen": 0.15540289878845215, + "logits/rejected": 0.1371532380580902, + "logps/chosen": -74.1396484375, + "logps/ref_chosen": -74.12020111083984, + "logps/ref_rejected": -83.33099365234375, + "logps/rejected": -83.41854858398438, + "loss": 1.3673, + "margin_dpo/margin_mean": 0.06811761856079102, + "margin_dpo/margin_std": 0.32512563467025757, + "step": 29 + }, + { + "KL/chosen_KL_mean": -0.025737762451171875, + "KL/mean": -0.03448393940925598, + "KL/rejected_KL_mean": -0.0432281494140625, + "KL/std": 0.1987377107143402, + "epoch": 0.045351473922902494, + "fcm_dpo/beta": 0.344091534614563, + "fcm_dpo/delta": 0.3941713571548462, + "fcm_dpo/margin": 0.017492294311523438, + "fcm_dpo/q_t": 0.49854251742362976, + "grad_norm": 103.57805633544922, + "learning_rate": 2.1641791044776117e-07, + "logits/chosen": 0.11821296066045761, + "logits/rejected": 0.06425055116415024, + "logps/chosen": -50.77702713012695, + "logps/ref_chosen": -50.75128936767578, + "logps/ref_rejected": -89.29063415527344, + "logps/rejected": -89.3338623046875, + "loss": 1.3828, + "margin_dpo/margin_mean": 0.017491638660430908, + "margin_dpo/margin_std": 0.2868330478668213, + "step": 30 + }, + { + "KL/chosen_KL_mean": -0.024738311767578125, + "KL/mean": -0.04767066240310669, + "KL/rejected_KL_mean": -0.0706024169921875, + "KL/std": 0.2183002531528473, + "epoch": 0.04686318972033258, + "fcm_dpo/beta": 0.3570849895477295, + "fcm_dpo/delta": 0.17508627474308014, + "fcm_dpo/margin": 0.04586535692214966, + "fcm_dpo/q_t": 0.49598830938339233, + "grad_norm": 122.07454681396484, + "learning_rate": 2.2388059701492537e-07, + "logits/chosen": 0.10629313439130783, + "logits/rejected": 0.06015738844871521, + "logps/chosen": -65.36149597167969, + "logps/ref_chosen": -65.33675384521484, + "logps/ref_rejected": -100.76666259765625, + "logps/rejected": -100.83726501464844, + "loss": 1.3727, + "margin_dpo/margin_mean": 0.04586589336395264, + "margin_dpo/margin_std": 0.3037866950035095, + "step": 31 + }, + { + "KL/chosen_KL_mean": -0.001934051513671875, + "KL/mean": -0.027067288756370544, + "KL/rejected_KL_mean": -0.052204132080078125, + "KL/std": 0.21004009246826172, + "epoch": 0.04837490551776266, + "fcm_dpo/beta": 0.38511091470718384, + "fcm_dpo/delta": 0.38101980090141296, + "fcm_dpo/margin": 0.050266534090042114, + "fcm_dpo/q_t": 0.49527406692504883, + "grad_norm": 115.42655181884766, + "learning_rate": 2.3134328358208954e-07, + "logits/chosen": 0.09014316648244858, + "logits/rejected": 0.08219875395298004, + "logps/chosen": -67.18526458740234, + "logps/ref_chosen": -67.18333435058594, + "logps/ref_rejected": -82.80763244628906, + "logps/rejected": -82.8598403930664, + "loss": 1.3699, + "margin_dpo/margin_mean": 0.05026650428771973, + "margin_dpo/margin_std": 0.2893450856208801, + "step": 32 + }, + { + "KL/chosen_KL_mean": -0.0030364990234375, + "KL/mean": -0.04211585968732834, + "KL/rejected_KL_mean": -0.08119583129882812, + "KL/std": 0.24913763999938965, + "epoch": 0.049886621315192746, + "fcm_dpo/beta": 0.41478103399276733, + "fcm_dpo/delta": 0.3684595823287964, + "fcm_dpo/margin": 0.07815925776958466, + "fcm_dpo/q_t": 0.492145836353302, + "grad_norm": 127.29991149902344, + "learning_rate": 2.388059701492537e-07, + "logits/chosen": 0.02003825642168522, + "logits/rejected": -0.005349243059754372, + "logps/chosen": -64.04251861572266, + "logps/ref_chosen": -64.03948211669922, + "logps/ref_rejected": -75.68357849121094, + "logps/rejected": -75.76477813720703, + "loss": 1.3588, + "margin_dpo/margin_mean": 0.07816009223461151, + "margin_dpo/margin_std": 0.33464300632476807, + "step": 33 + }, + { + "KL/chosen_KL_mean": -0.04259300231933594, + "KL/mean": -0.06921710819005966, + "KL/rejected_KL_mean": -0.0958404541015625, + "KL/std": 0.20878559350967407, + "epoch": 0.05139833711262283, + "fcm_dpo/beta": 0.4466947317123413, + "fcm_dpo/delta": 0.37713927030563354, + "fcm_dpo/margin": 0.053250834345817566, + "fcm_dpo/q_t": 0.49428844451904297, + "grad_norm": 126.29956817626953, + "learning_rate": 2.4626865671641786e-07, + "logits/chosen": 0.1000206470489502, + "logits/rejected": 0.07004327327013016, + "logps/chosen": -53.706886291503906, + "logps/ref_chosen": -53.6642951965332, + "logps/ref_rejected": -65.77989959716797, + "logps/rejected": -65.87574005126953, + "loss": 1.3668, + "margin_dpo/margin_mean": 0.053250715136528015, + "margin_dpo/margin_std": 0.2848781645298004, + "step": 34 + }, + { + "KL/chosen_KL_mean": -0.08517646789550781, + "KL/mean": -0.09704282879829407, + "KL/rejected_KL_mean": -0.10890960693359375, + "KL/std": 0.22471538186073303, + "epoch": 0.05291005291005291, + "fcm_dpo/beta": 0.472166508436203, + "fcm_dpo/delta": 0.1833934485912323, + "fcm_dpo/margin": 0.02373194694519043, + "fcm_dpo/q_t": 0.49740830063819885, + "grad_norm": 130.8272247314453, + "learning_rate": 2.537313432835821e-07, + "logits/chosen": 0.05964568629860878, + "logits/rejected": 0.037086836993694305, + "logps/chosen": -61.1020393371582, + "logps/ref_chosen": -61.01686096191406, + "logps/ref_rejected": -72.78598022460938, + "logps/rejected": -72.8948974609375, + "loss": 1.3811, + "margin_dpo/margin_mean": 0.023731738328933716, + "margin_dpo/margin_std": 0.321732759475708, + "step": 35 + }, + { + "KL/chosen_KL_mean": -0.08330917358398438, + "KL/mean": -0.09451384842395782, + "KL/rejected_KL_mean": -0.10572052001953125, + "KL/std": 0.24240775406360626, + "epoch": 0.05442176870748299, + "fcm_dpo/beta": 0.4811955988407135, + "fcm_dpo/delta": 0.18766069412231445, + "fcm_dpo/margin": 0.022410601377487183, + "fcm_dpo/q_t": 0.4973487854003906, + "grad_norm": 138.20062255859375, + "learning_rate": 2.611940298507462e-07, + "logits/chosen": 0.09503352642059326, + "logits/rejected": 0.04270517826080322, + "logps/chosen": -50.620670318603516, + "logps/ref_chosen": -50.53736114501953, + "logps/ref_rejected": -78.11678314208984, + "logps/rejected": -78.22250366210938, + "loss": 1.3824, + "margin_dpo/margin_mean": 0.02241101861000061, + "margin_dpo/margin_std": 0.35139748454093933, + "step": 36 + }, + { + "KL/chosen_KL_mean": -0.01982879638671875, + "KL/mean": -0.08735629916191101, + "KL/rejected_KL_mean": -0.15488815307617188, + "KL/std": 0.3164428472518921, + "epoch": 0.055933484504913075, + "fcm_dpo/beta": 0.5152612328529358, + "fcm_dpo/delta": 0.33278989791870117, + "fcm_dpo/margin": 0.13505524396896362, + "fcm_dpo/q_t": 0.48369383811950684, + "grad_norm": 183.72947692871094, + "learning_rate": 2.686567164179104e-07, + "logits/chosen": 0.10293924808502197, + "logits/rejected": 0.023218315094709396, + "logps/chosen": -59.57377624511719, + "logps/ref_chosen": -59.55394744873047, + "logps/ref_rejected": -108.27702331542969, + "logps/rejected": -108.43191528320312, + "loss": 1.3308, + "margin_dpo/margin_mean": 0.13505515456199646, + "margin_dpo/margin_std": 0.4417878985404968, + "step": 37 + }, + { + "KL/chosen_KL_mean": -0.09215354919433594, + "KL/mean": -0.10593746602535248, + "KL/rejected_KL_mean": -0.11971855163574219, + "KL/std": 0.28186145424842834, + "epoch": 0.05744520030234316, + "fcm_dpo/beta": 0.5423237681388855, + "fcm_dpo/delta": 0.17220675945281982, + "fcm_dpo/margin": 0.027556508779525757, + "fcm_dpo/q_t": 0.496552973985672, + "grad_norm": 161.56407165527344, + "learning_rate": 2.761194029850746e-07, + "logits/chosen": 0.05953256040811539, + "logits/rejected": 0.045650843530893326, + "logps/chosen": -65.88050842285156, + "logps/ref_chosen": -65.78836059570312, + "logps/ref_rejected": -76.1619873046875, + "logps/rejected": -76.28170776367188, + "loss": 1.3839, + "margin_dpo/margin_mean": 0.02755632996559143, + "margin_dpo/margin_std": 0.40284162759780884, + "step": 38 + }, + { + "KL/chosen_KL_mean": -0.079742431640625, + "KL/mean": -0.1202605664730072, + "KL/rejected_KL_mean": -0.16077423095703125, + "KL/std": 0.25623682141304016, + "epoch": 0.05895691609977324, + "fcm_dpo/beta": 0.5728709697723389, + "fcm_dpo/delta": 0.35464733839035034, + "fcm_dpo/margin": 0.08103156089782715, + "fcm_dpo/q_t": 0.4886939823627472, + "grad_norm": 166.70558166503906, + "learning_rate": 2.8358208955223876e-07, + "logits/chosen": 0.15896514058113098, + "logits/rejected": 0.13215456902980804, + "logps/chosen": -57.256553649902344, + "logps/ref_chosen": -57.17681121826172, + "logps/ref_rejected": -79.486328125, + "logps/rejected": -79.64710998535156, + "loss": 1.3508, + "margin_dpo/margin_mean": 0.08103141188621521, + "margin_dpo/margin_std": 0.35928279161453247, + "step": 39 + }, + { + "KL/chosen_KL_mean": -0.11428260803222656, + "KL/mean": -0.11064784228801727, + "KL/rejected_KL_mean": -0.10701751708984375, + "KL/std": 0.23384666442871094, + "epoch": 0.06046863189720333, + "fcm_dpo/beta": 0.6046038269996643, + "fcm_dpo/delta": 0.18890410661697388, + "fcm_dpo/margin": -0.007268577814102173, + "fcm_dpo/q_t": 0.5011410117149353, + "grad_norm": 200.42584228515625, + "learning_rate": 2.9104477611940296e-07, + "logits/chosen": 0.12039332836866379, + "logits/rejected": 0.07013371586799622, + "logps/chosen": -61.448448181152344, + "logps/ref_chosen": -61.33416748046875, + "logps/ref_rejected": -79.10697174072266, + "logps/rejected": -79.2139892578125, + "loss": 1.402, + "margin_dpo/margin_mean": -0.007268846035003662, + "margin_dpo/margin_std": 0.35015004873275757, + "step": 40 + }, + { + "KL/chosen_KL_mean": -0.122222900390625, + "KL/mean": -0.16071859002113342, + "KL/rejected_KL_mean": -0.19921493530273438, + "KL/std": 0.26297086477279663, + "epoch": 0.06198034769463341, + "fcm_dpo/beta": 0.6379947662353516, + "fcm_dpo/delta": 0.3523167371749878, + "fcm_dpo/margin": 0.0769854187965393, + "fcm_dpo/q_t": 0.48818713426589966, + "grad_norm": 190.40330505371094, + "learning_rate": 2.985074626865671e-07, + "logits/chosen": 0.05450894311070442, + "logits/rejected": 0.03441740572452545, + "logps/chosen": -67.66895294189453, + "logps/ref_chosen": -67.5467300415039, + "logps/ref_rejected": -83.87788391113281, + "logps/rejected": -84.07710266113281, + "loss": 1.3483, + "margin_dpo/margin_mean": 0.07698613405227661, + "margin_dpo/margin_std": 0.3233705461025238, + "step": 41 + }, + { + "KL/chosen_KL_mean": -0.14746665954589844, + "KL/mean": -0.15399429202079773, + "KL/rejected_KL_mean": -0.1605224609375, + "KL/std": 0.28739774227142334, + "epoch": 0.06349206349206349, + "fcm_dpo/beta": 0.6700236201286316, + "fcm_dpo/delta": 0.16138070821762085, + "fcm_dpo/margin": 0.013055786490440369, + "fcm_dpo/q_t": 0.498212993144989, + "grad_norm": 194.78236389160156, + "learning_rate": 3.059701492537313e-07, + "logits/chosen": 0.07204203307628632, + "logits/rejected": 0.0498071014881134, + "logps/chosen": -61.412322998046875, + "logps/ref_chosen": -61.26485824584961, + "logps/ref_rejected": -76.3629150390625, + "logps/rejected": -76.5234375, + "loss": 1.3974, + "margin_dpo/margin_mean": 0.013056054711341858, + "margin_dpo/margin_std": 0.40859484672546387, + "step": 42 + }, + { + "KL/chosen_KL_mean": -0.13036727905273438, + "KL/mean": -0.14713996648788452, + "KL/rejected_KL_mean": -0.16391372680664062, + "KL/std": 0.3216056525707245, + "epoch": 0.06500377928949358, + "fcm_dpo/beta": 0.7093794345855713, + "fcm_dpo/delta": 0.3769518733024597, + "fcm_dpo/margin": 0.03354591131210327, + "fcm_dpo/q_t": 0.4946945309638977, + "grad_norm": 253.1779327392578, + "learning_rate": 3.134328358208955e-07, + "logits/chosen": 0.08703877031803131, + "logits/rejected": 0.07603750377893448, + "logps/chosen": -71.93939208984375, + "logps/ref_chosen": -71.80902862548828, + "logps/ref_rejected": -81.12464141845703, + "logps/rejected": -81.28855895996094, + "loss": 1.3871, + "margin_dpo/margin_mean": 0.033546000719070435, + "margin_dpo/margin_std": 0.4398638606071472, + "step": 43 + }, + { + "KL/chosen_KL_mean": -0.14861679077148438, + "KL/mean": -0.15190964937210083, + "KL/rejected_KL_mean": -0.15520477294921875, + "KL/std": 0.27536773681640625, + "epoch": 0.06651549508692366, + "fcm_dpo/beta": 0.7668402791023254, + "fcm_dpo/delta": 0.39510226249694824, + "fcm_dpo/margin": 0.006583750247955322, + "fcm_dpo/q_t": 0.49894002079963684, + "grad_norm": 248.98779296875, + "learning_rate": 3.2089552238805965e-07, + "logits/chosen": 0.06032150238752365, + "logits/rejected": 0.029356716200709343, + "logps/chosen": -66.69905090332031, + "logps/ref_chosen": -66.55043029785156, + "logps/ref_rejected": -85.06198120117188, + "logps/rejected": -85.2171859741211, + "loss": 1.404, + "margin_dpo/margin_mean": 0.006583571434020996, + "margin_dpo/margin_std": 0.3948115110397339, + "step": 44 + }, + { + "KL/chosen_KL_mean": -0.16165733337402344, + "KL/mean": -0.22152617573738098, + "KL/rejected_KL_mean": -0.28139495849609375, + "KL/std": 0.27924102544784546, + "epoch": 0.06802721088435375, + "fcm_dpo/beta": 0.8197171688079834, + "fcm_dpo/delta": 0.30430763959884644, + "fcm_dpo/margin": 0.11974488198757172, + "fcm_dpo/q_t": 0.4763890504837036, + "grad_norm": 252.16053771972656, + "learning_rate": 3.2835820895522385e-07, + "logits/chosen": 0.13100901246070862, + "logits/rejected": 0.0773434191942215, + "logps/chosen": -62.40550994873047, + "logps/ref_chosen": -62.24385452270508, + "logps/ref_rejected": -92.96665954589844, + "logps/rejected": -93.24806213378906, + "loss": 1.3108, + "margin_dpo/margin_mean": 0.11974579095840454, + "margin_dpo/margin_std": 0.3502202332019806, + "step": 45 + }, + { + "KL/chosen_KL_mean": -0.0877227783203125, + "KL/mean": -0.1594286412000656, + "KL/rejected_KL_mean": -0.23113632202148438, + "KL/std": 0.3115689754486084, + "epoch": 0.06953892668178382, + "fcm_dpo/beta": 0.8677189350128174, + "fcm_dpo/delta": 0.2785709500312805, + "fcm_dpo/margin": 0.14340588450431824, + "fcm_dpo/q_t": 0.4707869291305542, + "grad_norm": 243.6422119140625, + "learning_rate": 3.3582089552238805e-07, + "logits/chosen": 0.14529743790626526, + "logits/rejected": 0.09837373346090317, + "logps/chosen": -61.58662796020508, + "logps/ref_chosen": -61.498905181884766, + "logps/ref_rejected": -78.91172790527344, + "logps/rejected": -79.14286804199219, + "loss": 1.2945, + "margin_dpo/margin_mean": 0.14340543746948242, + "margin_dpo/margin_std": 0.39542055130004883, + "step": 46 + }, + { + "KL/chosen_KL_mean": -0.14699363708496094, + "KL/mean": -0.20896488428115845, + "KL/rejected_KL_mean": -0.2709388732910156, + "KL/std": 0.2617807388305664, + "epoch": 0.0710506424792139, + "fcm_dpo/beta": 0.9190011024475098, + "fcm_dpo/delta": 0.2888580858707428, + "fcm_dpo/margin": 0.12394267320632935, + "fcm_dpo/q_t": 0.4727897047996521, + "grad_norm": 253.87966918945312, + "learning_rate": 3.432835820895522e-07, + "logits/chosen": 0.026472479104995728, + "logits/rejected": -0.016051139682531357, + "logps/chosen": -51.725341796875, + "logps/ref_chosen": -51.578346252441406, + "logps/ref_rejected": -68.2215576171875, + "logps/rejected": -68.49249267578125, + "loss": 1.3024, + "margin_dpo/margin_mean": 0.12394288182258606, + "margin_dpo/margin_std": 0.35938262939453125, + "step": 47 + }, + { + "KL/chosen_KL_mean": -0.229888916015625, + "KL/mean": -0.2503516674041748, + "KL/rejected_KL_mean": -0.27081871032714844, + "KL/std": 0.30007147789001465, + "epoch": 0.07256235827664399, + "fcm_dpo/beta": 0.9456481337547302, + "fcm_dpo/delta": 0.15676452219486237, + "fcm_dpo/margin": 0.04092279076576233, + "fcm_dpo/q_t": 0.4911009967327118, + "grad_norm": 254.16644287109375, + "learning_rate": 3.507462686567164e-07, + "logits/chosen": 0.12581509351730347, + "logits/rejected": 0.09670676290988922, + "logps/chosen": -52.02354049682617, + "logps/ref_chosen": -51.79365158081055, + "logps/ref_rejected": -64.22503662109375, + "logps/rejected": -64.49585723876953, + "loss": 1.3864, + "margin_dpo/margin_mean": 0.04092302918434143, + "margin_dpo/margin_std": 0.41956618428230286, + "step": 48 + }, + { + "KL/chosen_KL_mean": -0.19176101684570312, + "KL/mean": -0.2539128065109253, + "KL/rejected_KL_mean": -0.31606292724609375, + "KL/std": 0.3132970929145813, + "epoch": 0.07407407407407407, + "fcm_dpo/beta": 1.0028908252716064, + "fcm_dpo/delta": 0.27836254239082336, + "fcm_dpo/margin": 0.12430019676685333, + "fcm_dpo/q_t": 0.4709841310977936, + "grad_norm": 258.2995910644531, + "learning_rate": 3.5820895522388055e-07, + "logits/chosen": 0.0265614353120327, + "logits/rejected": 0.0051910169422626495, + "logps/chosen": -58.32636260986328, + "logps/ref_chosen": -58.13460159301758, + "logps/ref_rejected": -64.63206481933594, + "logps/rejected": -64.94813537597656, + "loss": 1.3091, + "margin_dpo/margin_mean": 0.12430053949356079, + "margin_dpo/margin_std": 0.4250980615615845, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.24220848083496094, + "KL/mean": -0.32140886783599854, + "KL/rejected_KL_mean": -0.40061187744140625, + "KL/std": 0.35282331705093384, + "epoch": 0.07558578987150416, + "fcm_dpo/beta": 1.0499261617660522, + "fcm_dpo/delta": 0.2380904257297516, + "fcm_dpo/margin": 0.15839967131614685, + "fcm_dpo/q_t": 0.4610481262207031, + "grad_norm": 269.3840026855469, + "learning_rate": 3.6567164179104475e-07, + "logits/chosen": 0.08331212401390076, + "logits/rejected": 0.05470721051096916, + "logps/chosen": -53.09864807128906, + "logps/ref_chosen": -52.85643768310547, + "logps/ref_rejected": -72.17460632324219, + "logps/rejected": -72.57521057128906, + "loss": 1.2754, + "margin_dpo/margin_mean": 0.15839970111846924, + "margin_dpo/margin_std": 0.4189513921737671, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.25580406188964844, + "KL/mean": -0.35211101174354553, + "KL/rejected_KL_mean": -0.44841766357421875, + "KL/std": 0.3892095685005188, + "epoch": 0.07709750566893424, + "fcm_dpo/beta": 1.0947012901306152, + "fcm_dpo/delta": 0.1934581845998764, + "fcm_dpo/margin": 0.1926138997077942, + "fcm_dpo/q_t": 0.45432114601135254, + "grad_norm": 290.8935852050781, + "learning_rate": 3.7313432835820895e-07, + "logits/chosen": 0.08420804142951965, + "logits/rejected": 0.056402526795864105, + "logps/chosen": -63.91224670410156, + "logps/ref_chosen": -63.65644073486328, + "logps/ref_rejected": -86.13229370117188, + "logps/rejected": -86.58071899414062, + "loss": 1.2721, + "margin_dpo/margin_mean": 0.1926141083240509, + "margin_dpo/margin_std": 0.547376275062561, + "step": 51 + }, + { + "KL/chosen_KL_mean": -0.30968475341796875, + "KL/mean": -0.39266496896743774, + "KL/rejected_KL_mean": -0.47564697265625, + "KL/std": 0.37696564197540283, + "epoch": 0.07860922146636433, + "fcm_dpo/beta": 1.1430740356445312, + "fcm_dpo/delta": 0.21437308192253113, + "fcm_dpo/margin": 0.16595804691314697, + "fcm_dpo/q_t": 0.45774364471435547, + "grad_norm": 346.4855041503906, + "learning_rate": 3.805970149253731e-07, + "logits/chosen": 0.09279091656208038, + "logits/rejected": 0.04256044328212738, + "logps/chosen": -68.14990234375, + "logps/ref_chosen": -67.8402099609375, + "logps/ref_rejected": -96.97090911865234, + "logps/rejected": -97.4465560913086, + "loss": 1.2806, + "margin_dpo/margin_mean": 0.165958434343338, + "margin_dpo/margin_std": 0.49165236949920654, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.31960487365722656, + "KL/mean": -0.4031504988670349, + "KL/rejected_KL_mean": -0.4866962432861328, + "KL/std": 0.3649352788925171, + "epoch": 0.0801209372637944, + "fcm_dpo/beta": 1.184340238571167, + "fcm_dpo/delta": 0.20668236911296844, + "fcm_dpo/margin": 0.16709211468696594, + "fcm_dpo/q_t": 0.45340898633003235, + "grad_norm": 301.8042907714844, + "learning_rate": 3.880597014925373e-07, + "logits/chosen": 0.06423115730285645, + "logits/rejected": 0.05389813706278801, + "logps/chosen": -57.19774627685547, + "logps/ref_chosen": -56.87813949584961, + "logps/ref_rejected": -60.75569152832031, + "logps/rejected": -61.24238967895508, + "loss": 1.2681, + "margin_dpo/margin_mean": 0.16709229350090027, + "margin_dpo/margin_std": 0.4409226179122925, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.3518028259277344, + "KL/mean": -0.4071524739265442, + "KL/rejected_KL_mean": -0.46249961853027344, + "KL/std": 0.3316037654876709, + "epoch": 0.08163265306122448, + "fcm_dpo/beta": 1.2483994960784912, + "fcm_dpo/delta": 0.26621297001838684, + "fcm_dpo/margin": 0.11069938540458679, + "fcm_dpo/q_t": 0.46753621101379395, + "grad_norm": 328.0696716308594, + "learning_rate": 3.9552238805970144e-07, + "logits/chosen": 0.0451444610953331, + "logits/rejected": 0.029947971925139427, + "logps/chosen": -47.61872100830078, + "logps/ref_chosen": -47.26692199707031, + "logps/ref_rejected": -62.19426727294922, + "logps/rejected": -62.65676498413086, + "loss": 1.3134, + "margin_dpo/margin_mean": 0.11069947481155396, + "margin_dpo/margin_std": 0.3909778594970703, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.33339691162109375, + "KL/mean": -0.4660353660583496, + "KL/rejected_KL_mean": -0.5986747741699219, + "KL/std": 0.46117573976516724, + "epoch": 0.08314436885865457, + "fcm_dpo/beta": 1.2880147695541382, + "fcm_dpo/delta": 0.059539683163166046, + "fcm_dpo/margin": 0.265281081199646, + "fcm_dpo/q_t": 0.4293213486671448, + "grad_norm": 368.900634765625, + "learning_rate": 4.0298507462686564e-07, + "logits/chosen": 0.03194599226117134, + "logits/rejected": -0.044348280876874924, + "logps/chosen": -50.65958786010742, + "logps/ref_chosen": -50.32619094848633, + "logps/ref_rejected": -92.44389343261719, + "logps/rejected": -93.04256439208984, + "loss": 1.2, + "margin_dpo/margin_mean": 0.2652807831764221, + "margin_dpo/margin_std": 0.5791739225387573, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.30806732177734375, + "KL/mean": -0.40079766511917114, + "KL/rejected_KL_mean": -0.49352455139160156, + "KL/std": 0.39821040630340576, + "epoch": 0.08465608465608465, + "fcm_dpo/beta": 1.314407229423523, + "fcm_dpo/delta": 0.1603386551141739, + "fcm_dpo/margin": 0.18546155095100403, + "fcm_dpo/q_t": 0.4450131058692932, + "grad_norm": 353.42315673828125, + "learning_rate": 4.1044776119402984e-07, + "logits/chosen": 0.12745994329452515, + "logits/rejected": 0.10495086014270782, + "logps/chosen": -57.07503890991211, + "logps/ref_chosen": -56.766971588134766, + "logps/ref_rejected": -66.30504608154297, + "logps/rejected": -66.79856872558594, + "loss": 1.2562, + "margin_dpo/margin_mean": 0.1854616105556488, + "margin_dpo/margin_std": 0.4858906865119934, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.3892097473144531, + "KL/mean": -0.5379469394683838, + "KL/rejected_KL_mean": -0.6866836547851562, + "KL/std": 0.4577900171279907, + "epoch": 0.08616780045351474, + "fcm_dpo/beta": 1.320029377937317, + "fcm_dpo/delta": 0.006817173212766647, + "fcm_dpo/margin": 0.29747310280799866, + "fcm_dpo/q_t": 0.412424236536026, + "grad_norm": 309.8792724609375, + "learning_rate": 4.17910447761194e-07, + "logits/chosen": 0.08991461247205734, + "logits/rejected": 0.026150904595851898, + "logps/chosen": -58.156951904296875, + "logps/ref_chosen": -57.76774597167969, + "logps/ref_rejected": -82.75698852539062, + "logps/rejected": -83.44367218017578, + "loss": 1.1477, + "margin_dpo/margin_mean": 0.297473281621933, + "margin_dpo/margin_std": 0.5286021828651428, + "step": 57 + }, + { + "KL/chosen_KL_mean": -0.43707275390625, + "KL/mean": -0.5622156262397766, + "KL/rejected_KL_mean": -0.6873550415039062, + "KL/std": 0.5071960687637329, + "epoch": 0.08767951625094482, + "fcm_dpo/beta": 1.3271738290786743, + "fcm_dpo/delta": 0.06818326562643051, + "fcm_dpo/margin": 0.25028324127197266, + "fcm_dpo/q_t": 0.43941134214401245, + "grad_norm": 388.10906982421875, + "learning_rate": 4.253731343283582e-07, + "logits/chosen": 0.05029626190662384, + "logits/rejected": 0.03480858355760574, + "logps/chosen": -73.20115661621094, + "logps/ref_chosen": -72.76408386230469, + "logps/ref_rejected": -84.49275207519531, + "logps/rejected": -85.18010711669922, + "loss": 1.2563, + "margin_dpo/margin_mean": 0.2502829134464264, + "margin_dpo/margin_std": 0.6835585832595825, + "step": 58 + }, + { + "KL/chosen_KL_mean": -0.4274005889892578, + "KL/mean": -0.531050980091095, + "KL/rejected_KL_mean": -0.6347026824951172, + "KL/std": 0.42696622014045715, + "epoch": 0.08919123204837491, + "fcm_dpo/beta": 1.3373160362243652, + "fcm_dpo/delta": -0.07443632930517197, + "fcm_dpo/margin": 0.20729583501815796, + "fcm_dpo/q_t": 0.44409847259521484, + "grad_norm": 324.54107666015625, + "learning_rate": 4.3283582089552234e-07, + "logits/chosen": 0.11193365603685379, + "logits/rejected": 0.04682060331106186, + "logps/chosen": -50.24817657470703, + "logps/ref_chosen": -49.820777893066406, + "logps/ref_rejected": -77.14368438720703, + "logps/rejected": -77.77838897705078, + "loss": 1.261, + "margin_dpo/margin_mean": 0.20729640126228333, + "margin_dpo/margin_std": 0.5395167469978333, + "step": 59 + }, + { + "KL/chosen_KL_mean": -0.45989227294921875, + "KL/mean": -0.5035088062286377, + "KL/rejected_KL_mean": -0.5471286773681641, + "KL/std": 0.44240349531173706, + "epoch": 0.09070294784580499, + "fcm_dpo/beta": 1.377070665359497, + "fcm_dpo/delta": 0.2842600643634796, + "fcm_dpo/margin": 0.08723104000091553, + "fcm_dpo/q_t": 0.4692135453224182, + "grad_norm": 475.13653564453125, + "learning_rate": 4.4029850746268654e-07, + "logits/chosen": 0.11263720691204071, + "logits/rejected": 0.11117991805076599, + "logps/chosen": -63.68466567993164, + "logps/ref_chosen": -63.22477340698242, + "logps/ref_rejected": -61.360477447509766, + "logps/rejected": -61.90760803222656, + "loss": 1.4386, + "margin_dpo/margin_mean": 0.08723104000091553, + "margin_dpo/margin_std": 0.6071930527687073, + "step": 60 + }, + { + "KL/chosen_KL_mean": -0.5582065582275391, + "KL/mean": -0.6111811399459839, + "KL/rejected_KL_mean": -0.6641578674316406, + "KL/std": 0.5173063278198242, + "epoch": 0.09221466364323508, + "fcm_dpo/beta": 1.4540350437164307, + "fcm_dpo/delta": 0.25040096044540405, + "fcm_dpo/margin": 0.10594852268695831, + "fcm_dpo/q_t": 0.4708403944969177, + "grad_norm": 456.5729064941406, + "learning_rate": 4.4776119402985074e-07, + "logits/chosen": 0.1330236792564392, + "logits/rejected": 0.10015955567359924, + "logps/chosen": -49.57500457763672, + "logps/ref_chosen": -49.01679992675781, + "logps/ref_rejected": -74.90817260742188, + "logps/rejected": -75.57232666015625, + "loss": 1.4719, + "margin_dpo/margin_mean": 0.10594844818115234, + "margin_dpo/margin_std": 0.699350118637085, + "step": 61 + }, + { + "KL/chosen_KL_mean": -0.5634193420410156, + "KL/mean": -0.6708869338035583, + "KL/rejected_KL_mean": -0.7783546447753906, + "KL/std": 0.4782254099845886, + "epoch": 0.09372637944066516, + "fcm_dpo/beta": 1.5046186447143555, + "fcm_dpo/delta": 0.07768788933753967, + "fcm_dpo/margin": 0.2149357795715332, + "fcm_dpo/q_t": 0.4257014989852905, + "grad_norm": 442.23895263671875, + "learning_rate": 4.552238805970149e-07, + "logits/chosen": 0.08146628737449646, + "logits/rejected": 0.04323853552341461, + "logps/chosen": -63.31528854370117, + "logps/ref_chosen": -62.751869201660156, + "logps/ref_rejected": -78.93360900878906, + "logps/rejected": -79.71195983886719, + "loss": 1.3105, + "margin_dpo/margin_mean": 0.21493586897850037, + "margin_dpo/margin_std": 0.6780645847320557, + "step": 62 + }, + { + "KL/chosen_KL_mean": -0.4312572479248047, + "KL/mean": -0.6460943818092346, + "KL/rejected_KL_mean": -0.860931396484375, + "KL/std": 0.46772170066833496, + "epoch": 0.09523809523809523, + "fcm_dpo/beta": 1.4502391815185547, + "fcm_dpo/delta": -0.23768939077854156, + "fcm_dpo/margin": 0.42967158555984497, + "fcm_dpo/q_t": 0.36431044340133667, + "grad_norm": 334.40960693359375, + "learning_rate": 4.626865671641791e-07, + "logits/chosen": 0.19986870884895325, + "logits/rejected": 0.17387685179710388, + "logps/chosen": -60.94650650024414, + "logps/ref_chosen": -60.51525115966797, + "logps/ref_rejected": -85.11021423339844, + "logps/rejected": -85.97114562988281, + "loss": 0.9869, + "margin_dpo/margin_mean": 0.4296703636646271, + "margin_dpo/margin_std": 0.5399540662765503, + "step": 63 + }, + { + "KL/chosen_KL_mean": -0.6149120330810547, + "KL/mean": -0.6417762041091919, + "KL/rejected_KL_mean": -0.6686439514160156, + "KL/std": 0.4757128357887268, + "epoch": 0.09674981103552532, + "fcm_dpo/beta": 1.4943134784698486, + "fcm_dpo/delta": 0.32342731952667236, + "fcm_dpo/margin": 0.05373135209083557, + "fcm_dpo/q_t": 0.4820947051048279, + "grad_norm": 445.0806884765625, + "learning_rate": 4.701492537313433e-07, + "logits/chosen": 0.08707167208194733, + "logits/rejected": 0.06187276542186737, + "logps/chosen": -51.82175827026367, + "logps/ref_chosen": -51.20684814453125, + "logps/ref_rejected": -66.93081665039062, + "logps/rejected": -67.5994644165039, + "loss": 1.4673, + "margin_dpo/margin_mean": 0.05373141169548035, + "margin_dpo/margin_std": 0.5616258382797241, + "step": 64 + }, + { + "KL/chosen_KL_mean": -0.5739707946777344, + "KL/mean": -0.7809446454048157, + "KL/rejected_KL_mean": -0.9879226684570312, + "KL/std": 0.5529853701591492, + "epoch": 0.0982615268329554, + "fcm_dpo/beta": 1.475801944732666, + "fcm_dpo/delta": -0.22470661997795105, + "fcm_dpo/margin": 0.41395121812820435, + "fcm_dpo/q_t": 0.3666613698005676, + "grad_norm": 357.36529541015625, + "learning_rate": 4.776119402985074e-07, + "logits/chosen": 0.19512414932250977, + "logits/rejected": 0.1650513857603073, + "logps/chosen": -67.86266326904297, + "logps/ref_chosen": -67.2886962890625, + "logps/ref_rejected": -74.44281005859375, + "logps/rejected": -75.43072509765625, + "loss": 1.0443, + "margin_dpo/margin_mean": 0.4139505922794342, + "margin_dpo/margin_std": 0.6147488355636597, + "step": 65 + }, + { + "KL/chosen_KL_mean": -0.5845527648925781, + "KL/mean": -0.6933289766311646, + "KL/rejected_KL_mean": -0.8021011352539062, + "KL/std": 0.5075341463088989, + "epoch": 0.09977324263038549, + "fcm_dpo/beta": 1.4587290287017822, + "fcm_dpo/delta": 0.08307772874832153, + "fcm_dpo/margin": 0.21754857897758484, + "fcm_dpo/q_t": 0.43842756748199463, + "grad_norm": 425.11907958984375, + "learning_rate": 4.850746268656717e-07, + "logits/chosen": 0.08232487738132477, + "logits/rejected": 0.058401718735694885, + "logps/chosen": -71.32796478271484, + "logps/ref_chosen": -70.743408203125, + "logps/ref_rejected": -77.26499938964844, + "logps/rejected": -78.06710815429688, + "loss": 1.2968, + "margin_dpo/margin_mean": 0.21754851937294006, + "margin_dpo/margin_std": 0.6472800970077515, + "step": 66 + }, + { + "KL/chosen_KL_mean": -0.48465538024902344, + "KL/mean": -0.5979866981506348, + "KL/rejected_KL_mean": -0.7113189697265625, + "KL/std": 0.5123389959335327, + "epoch": 0.10128495842781557, + "fcm_dpo/beta": 1.4985034465789795, + "fcm_dpo/delta": 0.06241083890199661, + "fcm_dpo/margin": 0.2266659140586853, + "fcm_dpo/q_t": 0.4364425837993622, + "grad_norm": 406.1687927246094, + "learning_rate": 4.925373134328357e-07, + "logits/chosen": 0.08061225712299347, + "logits/rejected": 0.02438163198530674, + "logps/chosen": -61.087257385253906, + "logps/ref_chosen": -60.60260009765625, + "logps/ref_rejected": -75.22235870361328, + "logps/rejected": -75.93367767333984, + "loss": 1.2757, + "margin_dpo/margin_mean": 0.22666651010513306, + "margin_dpo/margin_std": 0.6498265862464905, + "step": 67 + }, + { + "KL/chosen_KL_mean": -0.6951408386230469, + "KL/mean": -0.8220087289810181, + "KL/rejected_KL_mean": -0.9488754272460938, + "KL/std": 0.558422327041626, + "epoch": 0.10279667422524566, + "fcm_dpo/beta": 1.494733214378357, + "fcm_dpo/delta": 0.020372621715068817, + "fcm_dpo/margin": 0.2537391185760498, + "fcm_dpo/q_t": 0.42070841789245605, + "grad_norm": 446.0521545410156, + "learning_rate": 5e-07, + "logits/chosen": 0.051371585577726364, + "logits/rejected": 0.0212722048163414, + "logps/chosen": -78.2235107421875, + "logps/ref_chosen": -77.52836608886719, + "logps/ref_rejected": -93.17778015136719, + "logps/rejected": -94.12664794921875, + "loss": 1.2602, + "margin_dpo/margin_mean": 0.2537391185760498, + "margin_dpo/margin_std": 0.6757462024688721, + "step": 68 + }, + { + "KL/chosen_KL_mean": -0.6366329193115234, + "KL/mean": -0.8170256614685059, + "KL/rejected_KL_mean": -0.9974136352539062, + "KL/std": 0.5379056930541992, + "epoch": 0.10430839002267574, + "fcm_dpo/beta": 1.4895339012145996, + "fcm_dpo/delta": -0.14553788304328918, + "fcm_dpo/margin": 0.3607826232910156, + "fcm_dpo/q_t": 0.39012402296066284, + "grad_norm": 371.2175598144531, + "learning_rate": 4.999965034812934e-07, + "logits/chosen": 0.06672338396310806, + "logits/rejected": 0.024326715618371964, + "logps/chosen": -66.57968139648438, + "logps/ref_chosen": -65.94305419921875, + "logps/ref_rejected": -89.7735595703125, + "logps/rejected": -90.7709732055664, + "loss": 1.1082, + "margin_dpo/margin_mean": 0.36078232526779175, + "margin_dpo/margin_std": 0.6062155365943909, + "step": 69 + }, + { + "KL/chosen_KL_mean": -0.6548347473144531, + "KL/mean": -0.7635809779167175, + "KL/rejected_KL_mean": -0.8723239898681641, + "KL/std": 0.4746021628379822, + "epoch": 0.10582010582010581, + "fcm_dpo/beta": 1.4715263843536377, + "fcm_dpo/delta": 0.08196324110031128, + "fcm_dpo/margin": 0.21749544143676758, + "fcm_dpo/q_t": 0.42556124925613403, + "grad_norm": 401.7762756347656, + "learning_rate": 4.999860140229787e-07, + "logits/chosen": 0.12438544631004333, + "logits/rejected": 0.10102519392967224, + "logps/chosen": -62.61274719238281, + "logps/ref_chosen": -61.95791244506836, + "logps/ref_rejected": -75.80945587158203, + "logps/rejected": -76.68177795410156, + "loss": 1.2699, + "margin_dpo/margin_mean": 0.2174956500530243, + "margin_dpo/margin_std": 0.5834441184997559, + "step": 70 + }, + { + "KL/chosen_KL_mean": -0.7265167236328125, + "KL/mean": -0.8052693605422974, + "KL/rejected_KL_mean": -0.8840236663818359, + "KL/std": 0.5016933679580688, + "epoch": 0.1073318216175359, + "fcm_dpo/beta": 1.532849907875061, + "fcm_dpo/delta": 0.1612829566001892, + "fcm_dpo/margin": 0.15750658512115479, + "fcm_dpo/q_t": 0.4435678720474243, + "grad_norm": 440.24615478515625, + "learning_rate": 4.999685319184688e-07, + "logits/chosen": 0.09006579220294952, + "logits/rejected": 0.07397979497909546, + "logps/chosen": -64.0740966796875, + "logps/ref_chosen": -63.34757995605469, + "logps/ref_rejected": -67.49658203125, + "logps/rejected": -68.38059997558594, + "loss": 1.3472, + "margin_dpo/margin_mean": 0.15750646591186523, + "margin_dpo/margin_std": 0.601201057434082, + "step": 71 + }, + { + "KL/chosen_KL_mean": -0.6553325653076172, + "KL/mean": -0.8184918165206909, + "KL/rejected_KL_mean": -0.9816474914550781, + "KL/std": 0.5385796427726746, + "epoch": 0.10884353741496598, + "fcm_dpo/beta": 1.5117855072021484, + "fcm_dpo/delta": -0.09856587648391724, + "fcm_dpo/margin": 0.3263169527053833, + "fcm_dpo/q_t": 0.3978579342365265, + "grad_norm": 408.5451354980469, + "learning_rate": 4.999440576567755e-07, + "logits/chosen": 0.08719843626022339, + "logits/rejected": 0.02425486594438553, + "logps/chosen": -56.51463317871094, + "logps/ref_chosen": -55.85929870605469, + "logps/ref_rejected": -68.45423889160156, + "logps/rejected": -69.4358901977539, + "loss": 1.1726, + "margin_dpo/margin_mean": 0.3263167440891266, + "margin_dpo/margin_std": 0.6592986583709717, + "step": 72 + }, + { + "KL/chosen_KL_mean": -0.9284744262695312, + "KL/mean": -0.994033694267273, + "KL/rejected_KL_mean": -1.0595951080322266, + "KL/std": 0.6347705125808716, + "epoch": 0.11035525321239607, + "fcm_dpo/beta": 1.561780333518982, + "fcm_dpo/delta": 0.19763168692588806, + "fcm_dpo/margin": 0.1311270296573639, + "fcm_dpo/q_t": 0.4644252061843872, + "grad_norm": 534.00927734375, + "learning_rate": 4.999125919224965e-07, + "logits/chosen": 0.07961059361696243, + "logits/rejected": 0.06521774083375931, + "logps/chosen": -70.06727600097656, + "logps/ref_chosen": -69.13880920410156, + "logps/ref_rejected": -79.04586791992188, + "logps/rejected": -80.10546875, + "loss": 1.4815, + "margin_dpo/margin_mean": 0.13112786412239075, + "margin_dpo/margin_std": 0.736503005027771, + "step": 73 + }, + { + "KL/chosen_KL_mean": -0.6928367614746094, + "KL/mean": -0.9160832166671753, + "KL/rejected_KL_mean": -1.1393318176269531, + "KL/std": 0.5641649961471558, + "epoch": 0.11186696900982615, + "fcm_dpo/beta": 1.5258018970489502, + "fcm_dpo/delta": -0.3024459183216095, + "fcm_dpo/margin": 0.4464932680130005, + "fcm_dpo/q_t": 0.36906197667121887, + "grad_norm": 340.6932373046875, + "learning_rate": 4.998741355957963e-07, + "logits/chosen": 0.10730817168951035, + "logits/rejected": 0.05475199967622757, + "logps/chosen": -50.616573333740234, + "logps/ref_chosen": -49.923736572265625, + "logps/ref_rejected": -81.73213958740234, + "logps/rejected": -82.87147521972656, + "loss": 1.0628, + "margin_dpo/margin_mean": 0.4464929401874542, + "margin_dpo/margin_std": 0.6866585612297058, + "step": 74 + }, + { + "KL/chosen_KL_mean": -0.7750892639160156, + "KL/mean": -0.9859875440597534, + "KL/rejected_KL_mean": -1.1968860626220703, + "KL/std": 0.6085139513015747, + "epoch": 0.11337868480725624, + "fcm_dpo/beta": 1.419792652130127, + "fcm_dpo/delta": -0.21434250473976135, + "fcm_dpo/margin": 0.4217972755432129, + "fcm_dpo/q_t": 0.3794889450073242, + "grad_norm": 300.08807373046875, + "learning_rate": 4.998286897523808e-07, + "logits/chosen": 0.07163898646831512, + "logits/rejected": 0.040832214057445526, + "logps/chosen": -46.843841552734375, + "logps/ref_chosen": -46.06875228881836, + "logps/ref_rejected": -66.1181411743164, + "logps/rejected": -67.31503295898438, + "loss": 1.0958, + "margin_dpo/margin_mean": 0.42179712653160095, + "margin_dpo/margin_std": 0.7199387550354004, + "step": 75 + }, + { + "KL/chosen_KL_mean": -0.7950630187988281, + "KL/mean": -0.9406583309173584, + "KL/rejected_KL_mean": -1.0862560272216797, + "KL/std": 0.6709940433502197, + "epoch": 0.11489040060468632, + "fcm_dpo/beta": 1.4085371494293213, + "fcm_dpo/delta": -0.010789202526211739, + "fcm_dpo/margin": 0.2911915183067322, + "fcm_dpo/q_t": 0.42344167828559875, + "grad_norm": 378.4490966796875, + "learning_rate": 4.997762556634679e-07, + "logits/chosen": 0.09013240784406662, + "logits/rejected": 0.04698844254016876, + "logps/chosen": -54.857810974121094, + "logps/ref_chosen": -54.06275177001953, + "logps/ref_rejected": -74.87464141845703, + "logps/rejected": -75.96089935302734, + "loss": 1.2479, + "margin_dpo/margin_mean": 0.2911912798881531, + "margin_dpo/margin_std": 0.7412244081497192, + "step": 76 + }, + { + "KL/chosen_KL_mean": -0.9427947998046875, + "KL/mean": -1.0837860107421875, + "KL/rejected_KL_mean": -1.2247734069824219, + "KL/std": 0.5913428068161011, + "epoch": 0.1164021164021164, + "fcm_dpo/beta": 1.3965743780136108, + "fcm_dpo/delta": 0.003795936703681946, + "fcm_dpo/margin": 0.2819800078868866, + "fcm_dpo/q_t": 0.41928815841674805, + "grad_norm": 390.9847412109375, + "learning_rate": 4.99716834795752e-07, + "logits/chosen": 0.12903091311454773, + "logits/rejected": 0.08758988976478577, + "logps/chosen": -54.01888656616211, + "logps/ref_chosen": -53.07609176635742, + "logps/ref_rejected": -74.45601654052734, + "logps/rejected": -75.6807861328125, + "loss": 1.2229, + "margin_dpo/margin_mean": 0.28198015689849854, + "margin_dpo/margin_std": 0.6389899253845215, + "step": 77 + }, + { + "KL/chosen_KL_mean": -0.8560752868652344, + "KL/mean": -0.9924242496490479, + "KL/rejected_KL_mean": -1.1287765502929688, + "KL/std": 0.5523614287376404, + "epoch": 0.11791383219954649, + "fcm_dpo/beta": 1.4131134748458862, + "fcm_dpo/delta": 0.015169057063758373, + "fcm_dpo/margin": 0.27270209789276123, + "fcm_dpo/q_t": 0.42009180784225464, + "grad_norm": 419.772216796875, + "learning_rate": 4.996504288113623e-07, + "logits/chosen": 0.07726676762104034, + "logits/rejected": 0.05717945471405983, + "logps/chosen": -68.58149719238281, + "logps/ref_chosen": -67.72541809082031, + "logps/ref_rejected": -79.03926849365234, + "logps/rejected": -80.16804504394531, + "loss": 1.2435, + "margin_dpo/margin_mean": 0.27270200848579407, + "margin_dpo/margin_std": 0.673001766204834, + "step": 78 + }, + { + "KL/chosen_KL_mean": -0.9067211151123047, + "KL/mean": -1.1093153953552246, + "KL/rejected_KL_mean": -1.3119125366210938, + "KL/std": 0.602173924446106, + "epoch": 0.11942554799697656, + "fcm_dpo/beta": 1.38477623462677, + "fcm_dpo/delta": -0.17040225863456726, + "fcm_dpo/margin": 0.4051881730556488, + "fcm_dpo/q_t": 0.37906450033187866, + "grad_norm": 295.46978759765625, + "learning_rate": 4.995770395678171e-07, + "logits/chosen": 0.13399448990821838, + "logits/rejected": 0.07460729777812958, + "logps/chosen": -53.06736755371094, + "logps/ref_chosen": -52.16064453125, + "logps/ref_rejected": -83.31062316894531, + "logps/rejected": -84.62252807617188, + "loss": 1.0786, + "margin_dpo/margin_mean": 0.4051882326602936, + "margin_dpo/margin_std": 0.6499325037002563, + "step": 79 + }, + { + "KL/chosen_KL_mean": -0.9616222381591797, + "KL/mean": -1.1492910385131836, + "KL/rejected_KL_mean": -1.3369598388671875, + "KL/std": 0.6497531533241272, + "epoch": 0.12093726379440665, + "fcm_dpo/beta": 1.3587586879730225, + "fcm_dpo/delta": -0.11657415330410004, + "fcm_dpo/margin": 0.3753415048122406, + "fcm_dpo/q_t": 0.39891305565834045, + "grad_norm": 363.4054260253906, + "learning_rate": 4.994966691179711e-07, + "logits/chosen": 0.12045355886220932, + "logits/rejected": 0.06095130741596222, + "logps/chosen": -62.37218475341797, + "logps/ref_chosen": -61.410560607910156, + "logps/ref_rejected": -78.66004943847656, + "logps/rejected": -79.99700927734375, + "loss": 1.146, + "margin_dpo/margin_mean": 0.37534135580062866, + "margin_dpo/margin_std": 0.7054092884063721, + "step": 80 + }, + { + "KL/chosen_KL_mean": -1.0198345184326172, + "KL/mean": -1.2168505191802979, + "KL/rejected_KL_mean": -1.4138603210449219, + "KL/std": 0.6422163248062134, + "epoch": 0.12244897959183673, + "fcm_dpo/beta": 1.3078601360321045, + "fcm_dpo/delta": -0.12261784076690674, + "fcm_dpo/margin": 0.39402908086776733, + "fcm_dpo/q_t": 0.3908241391181946, + "grad_norm": 330.7373962402344, + "learning_rate": 4.994093197099587e-07, + "logits/chosen": 0.07247719168663025, + "logits/rejected": 0.03978278860449791, + "logps/chosen": -64.82420349121094, + "logps/ref_chosen": -63.80437088012695, + "logps/ref_rejected": -79.3484115600586, + "logps/rejected": -80.76226806640625, + "loss": 1.0932, + "margin_dpo/margin_mean": 0.3940291702747345, + "margin_dpo/margin_std": 0.64724200963974, + "step": 81 + }, + { + "KL/chosen_KL_mean": -0.8772220611572266, + "KL/mean": -1.1485819816589355, + "KL/rejected_KL_mean": -1.419942855834961, + "KL/std": 0.6433102488517761, + "epoch": 0.12396069538926682, + "fcm_dpo/beta": 1.2471857070922852, + "fcm_dpo/delta": -0.2975814938545227, + "fcm_dpo/margin": 0.542713463306427, + "fcm_dpo/q_t": 0.34925180673599243, + "grad_norm": 278.6336975097656, + "learning_rate": 4.993149937871306e-07, + "logits/chosen": 0.05593370646238327, + "logits/rejected": -0.0052494630217552185, + "logps/chosen": -49.69511413574219, + "logps/ref_chosen": -48.817893981933594, + "logps/ref_rejected": -70.31497955322266, + "logps/rejected": -71.73492431640625, + "loss": 0.964, + "margin_dpo/margin_mean": 0.5427138209342957, + "margin_dpo/margin_std": 0.651484489440918, + "step": 82 + }, + { + "KL/chosen_KL_mean": -0.9735927581787109, + "KL/mean": -1.2231380939483643, + "KL/rejected_KL_mean": -1.472686767578125, + "KL/std": 0.6370868682861328, + "epoch": 0.1254724111866969, + "fcm_dpo/beta": 1.1937222480773926, + "fcm_dpo/delta": -0.2080550491809845, + "fcm_dpo/margin": 0.4990951120853424, + "fcm_dpo/q_t": 0.37340906262397766, + "grad_norm": 277.4635925292969, + "learning_rate": 4.992136939879856e-07, + "logits/chosen": 0.1766347587108612, + "logits/rejected": 0.12540043890476227, + "logps/chosen": -58.124366760253906, + "logps/ref_chosen": -57.15077209472656, + "logps/ref_rejected": -75.1710205078125, + "logps/rejected": -76.64370727539062, + "loss": 1.0473, + "margin_dpo/margin_mean": 0.49909481406211853, + "margin_dpo/margin_std": 0.7604484558105469, + "step": 83 + }, + { + "KL/chosen_KL_mean": -1.1488761901855469, + "KL/mean": -1.290728211402893, + "KL/rejected_KL_mean": -1.4325942993164062, + "KL/std": 0.730034589767456, + "epoch": 0.12698412698412698, + "fcm_dpo/beta": 1.1920582056045532, + "fcm_dpo/delta": 0.06391885131597519, + "fcm_dpo/margin": 0.28371578454971313, + "fcm_dpo/q_t": 0.42771369218826294, + "grad_norm": 378.65191650390625, + "learning_rate": 4.991054231460969e-07, + "logits/chosen": 0.11678484082221985, + "logits/rejected": 0.0760355293750763, + "logps/chosen": -65.92617797851562, + "logps/ref_chosen": -64.77729797363281, + "logps/ref_rejected": -84.71949768066406, + "logps/rejected": -86.15209197998047, + "loss": 1.2627, + "margin_dpo/margin_mean": 0.2837153375148773, + "margin_dpo/margin_std": 0.7518688440322876, + "step": 84 + }, + { + "KL/chosen_KL_mean": -0.9664173126220703, + "KL/mean": -1.2492319345474243, + "KL/rejected_KL_mean": -1.53204345703125, + "KL/std": 0.746864914894104, + "epoch": 0.12849584278155707, + "fcm_dpo/beta": 1.1534860134124756, + "fcm_dpo/delta": -0.26930180191993713, + "fcm_dpo/margin": 0.5656286478042603, + "fcm_dpo/q_t": 0.3612124025821686, + "grad_norm": 289.289794921875, + "learning_rate": 4.989901842900325e-07, + "logits/chosen": 0.09598151594400406, + "logits/rejected": 0.05402902513742447, + "logps/chosen": -51.21811294555664, + "logps/ref_chosen": -50.25169372558594, + "logps/ref_rejected": -66.55439758300781, + "logps/rejected": -68.08644104003906, + "loss": 1.0391, + "margin_dpo/margin_mean": 0.5656284093856812, + "margin_dpo/margin_std": 0.8260301351547241, + "step": 85 + }, + { + "KL/chosen_KL_mean": -1.0875301361083984, + "KL/mean": -1.324817419052124, + "KL/rejected_KL_mean": -1.562103271484375, + "KL/std": 0.7085767388343811, + "epoch": 0.13000755857898716, + "fcm_dpo/beta": 1.1074466705322266, + "fcm_dpo/delta": -0.13303929567337036, + "fcm_dpo/margin": 0.47456830739974976, + "fcm_dpo/q_t": 0.3882756233215332, + "grad_norm": 245.15232849121094, + "learning_rate": 4.988679806432711e-07, + "logits/chosen": 0.14248552918434143, + "logits/rejected": 0.12411345541477203, + "logps/chosen": -61.81671142578125, + "logps/ref_chosen": -60.72917938232422, + "logps/ref_rejected": -72.30961608886719, + "logps/rejected": -73.87171936035156, + "loss": 1.0649, + "margin_dpo/margin_mean": 0.47456836700439453, + "margin_dpo/margin_std": 0.7193993330001831, + "step": 86 + }, + { + "KL/chosen_KL_mean": -1.125885009765625, + "KL/mean": -1.3568286895751953, + "KL/rejected_KL_mean": -1.5877647399902344, + "KL/std": 0.8056492805480957, + "epoch": 0.13151927437641722, + "fcm_dpo/beta": 1.076570987701416, + "fcm_dpo/delta": -0.1047540009021759, + "fcm_dpo/margin": 0.46188533306121826, + "fcm_dpo/q_t": 0.3923509120941162, + "grad_norm": 332.5335388183594, + "learning_rate": 4.987388156241114e-07, + "logits/chosen": 0.10630130767822266, + "logits/rejected": 0.04966554418206215, + "logps/chosen": -66.88385009765625, + "logps/ref_chosen": -65.75796508789062, + "logps/ref_rejected": -84.81159973144531, + "logps/rejected": -86.39936065673828, + "loss": 1.1763, + "margin_dpo/margin_mean": 0.4618854522705078, + "margin_dpo/margin_std": 0.9339560270309448, + "step": 87 + }, + { + "KL/chosen_KL_mean": -1.1211109161376953, + "KL/mean": -1.355789303779602, + "KL/rejected_KL_mean": -1.5904655456542969, + "KL/std": 0.7843002676963806, + "epoch": 0.1330309901738473, + "fcm_dpo/beta": 1.0574965476989746, + "fcm_dpo/delta": -0.10266615450382233, + "fcm_dpo/margin": 0.46935614943504333, + "fcm_dpo/q_t": 0.3971374034881592, + "grad_norm": 295.2401428222656, + "learning_rate": 4.986026928455767e-07, + "logits/chosen": 0.14652103185653687, + "logits/rejected": 0.12086239457130432, + "logps/chosen": -63.94513702392578, + "logps/ref_chosen": -62.82402801513672, + "logps/ref_rejected": -74.9607162475586, + "logps/rejected": -76.55117797851562, + "loss": 1.1424, + "margin_dpo/margin_mean": 0.4693562090396881, + "margin_dpo/margin_std": 0.8767245411872864, + "step": 88 + }, + { + "KL/chosen_KL_mean": -0.9972019195556641, + "KL/mean": -1.2437442541122437, + "KL/rejected_KL_mean": -1.4902877807617188, + "KL/std": 0.8000632524490356, + "epoch": 0.1345427059712774, + "fcm_dpo/beta": 1.0466407537460327, + "fcm_dpo/delta": -0.12258999794721603, + "fcm_dpo/margin": 0.49308302998542786, + "fcm_dpo/q_t": 0.38694024085998535, + "grad_norm": 262.8199157714844, + "learning_rate": 4.984596161153135e-07, + "logits/chosen": 0.2086452841758728, + "logits/rejected": 0.12626194953918457, + "logps/chosen": -42.188636779785156, + "logps/ref_chosen": -41.191436767578125, + "logps/ref_rejected": -85.44769287109375, + "logps/rejected": -86.93798828125, + "loss": 1.1202, + "margin_dpo/margin_mean": 0.4930838346481323, + "margin_dpo/margin_std": 0.8570024967193604, + "step": 89 + }, + { + "KL/chosen_KL_mean": -1.1763248443603516, + "KL/mean": -1.383636713027954, + "KL/rejected_KL_mean": -1.5909461975097656, + "KL/std": 0.7591300010681152, + "epoch": 0.1360544217687075, + "fcm_dpo/beta": 1.0314490795135498, + "fcm_dpo/delta": -0.0290251225233078, + "fcm_dpo/margin": 0.4146195352077484, + "fcm_dpo/q_t": 0.4088814854621887, + "grad_norm": 280.2547302246094, + "learning_rate": 4.983095894354857e-07, + "logits/chosen": 0.07813448458909988, + "logits/rejected": 0.02608073130249977, + "logps/chosen": -57.760231018066406, + "logps/ref_chosen": -56.58390808105469, + "logps/ref_rejected": -86.86978149414062, + "logps/rejected": -88.46072387695312, + "loss": 1.1623, + "margin_dpo/margin_mean": 0.4146192967891693, + "margin_dpo/margin_std": 0.8090165853500366, + "step": 90 + }, + { + "KL/chosen_KL_mean": -1.1240406036376953, + "KL/mean": -1.4229687452316284, + "KL/rejected_KL_mean": -1.7218971252441406, + "KL/std": 0.8534637093544006, + "epoch": 0.13756613756613756, + "fcm_dpo/beta": 0.9964852333068848, + "fcm_dpo/delta": -0.207942932844162, + "fcm_dpo/margin": 0.5978548526763916, + "fcm_dpo/q_t": 0.37541231513023376, + "grad_norm": 219.49380493164062, + "learning_rate": 4.98152617002662e-07, + "logits/chosen": 0.10854806005954742, + "logits/rejected": 0.06545954942703247, + "logps/chosen": -53.506385803222656, + "logps/ref_chosen": -52.38234329223633, + "logps/ref_rejected": -72.17642211914062, + "logps/rejected": -73.8983154296875, + "loss": 1.0439, + "margin_dpo/margin_mean": 0.5978548526763916, + "margin_dpo/margin_std": 0.8873600959777832, + "step": 91 + }, + { + "KL/chosen_KL_mean": -1.245452880859375, + "KL/mean": -1.4888556003570557, + "KL/rejected_KL_mean": -1.7322654724121094, + "KL/std": 0.8518181443214417, + "epoch": 0.13907785336356765, + "fcm_dpo/beta": 0.9819341897964478, + "fcm_dpo/delta": -0.08298197388648987, + "fcm_dpo/margin": 0.4868069291114807, + "fcm_dpo/q_t": 0.3972088098526001, + "grad_norm": 227.3011016845703, + "learning_rate": 4.979887032076988e-07, + "logits/chosen": 0.1596505492925644, + "logits/rejected": 0.12032721936702728, + "logps/chosen": -54.254154205322266, + "logps/ref_chosen": -53.00870132446289, + "logps/ref_rejected": -79.77812957763672, + "logps/rejected": -81.51039123535156, + "loss": 1.1362, + "margin_dpo/margin_mean": 0.4868068993091583, + "margin_dpo/margin_std": 0.8662936687469482, + "step": 92 + }, + { + "KL/chosen_KL_mean": -1.30902099609375, + "KL/mean": -1.531526803970337, + "KL/rejected_KL_mean": -1.7540321350097656, + "KL/std": 0.8664084672927856, + "epoch": 0.14058956916099774, + "fcm_dpo/beta": 0.9646643400192261, + "fcm_dpo/delta": -0.030587686225771904, + "fcm_dpo/margin": 0.4450099766254425, + "fcm_dpo/q_t": 0.4068792164325714, + "grad_norm": 209.89547729492188, + "learning_rate": 4.978178526356172e-07, + "logits/chosen": 0.10871215164661407, + "logits/rejected": 0.08188057690858841, + "logps/chosen": -46.21607208251953, + "logps/ref_chosen": -44.90705108642578, + "logps/ref_rejected": -58.7879524230957, + "logps/rejected": -60.54198455810547, + "loss": 1.1758, + "margin_dpo/margin_mean": 0.44500961899757385, + "margin_dpo/margin_std": 0.9137963056564331, + "step": 93 + }, + { + "KL/chosen_KL_mean": -1.0369033813476562, + "KL/mean": -1.324483871459961, + "KL/rejected_KL_mean": -1.612060546875, + "KL/std": 0.8918026685714722, + "epoch": 0.1421012849584278, + "fcm_dpo/beta": 0.9319857358932495, + "fcm_dpo/delta": -0.147476464509964, + "fcm_dpo/margin": 0.5751581192016602, + "fcm_dpo/q_t": 0.38574111461639404, + "grad_norm": 235.57188415527344, + "learning_rate": 4.976400700654751e-07, + "logits/chosen": 0.16680672764778137, + "logits/rejected": 0.1295737624168396, + "logps/chosen": -60.974674224853516, + "logps/ref_chosen": -59.93777084350586, + "logps/ref_rejected": -79.3138427734375, + "logps/rejected": -80.9259033203125, + "loss": 1.1527, + "margin_dpo/margin_mean": 0.5751579999923706, + "margin_dpo/margin_std": 1.1100279092788696, + "step": 94 + }, + { + "KL/chosen_KL_mean": -1.3213310241699219, + "KL/mean": -1.6439839601516724, + "KL/rejected_KL_mean": -1.9666366577148438, + "KL/std": 0.838896632194519, + "epoch": 0.1436130007558579, + "fcm_dpo/beta": 0.9028377532958984, + "fcm_dpo/delta": -0.1956050544977188, + "fcm_dpo/margin": 0.6453028917312622, + "fcm_dpo/q_t": 0.37172389030456543, + "grad_norm": 220.64878845214844, + "learning_rate": 4.974553604702332e-07, + "logits/chosen": 0.07326029241085052, + "logits/rejected": 0.012177206575870514, + "logps/chosen": -61.48982238769531, + "logps/ref_chosen": -60.168487548828125, + "logps/ref_rejected": -90.73665618896484, + "logps/rejected": -92.70329284667969, + "loss": 1.0313, + "margin_dpo/margin_mean": 0.6453026533126831, + "margin_dpo/margin_std": 0.8951278328895569, + "step": 95 + }, + { + "KL/chosen_KL_mean": -1.2811088562011719, + "KL/mean": -1.5985305309295654, + "KL/rejected_KL_mean": -1.91595458984375, + "KL/std": 0.8724809885025024, + "epoch": 0.14512471655328799, + "fcm_dpo/beta": 0.8822429180145264, + "fcm_dpo/delta": -0.16936419904232025, + "fcm_dpo/margin": 0.634851336479187, + "fcm_dpo/q_t": 0.37748467922210693, + "grad_norm": 206.07525634765625, + "learning_rate": 4.972637290166157e-07, + "logits/chosen": 0.1217946782708168, + "logits/rejected": 0.07989707589149475, + "logps/chosen": -61.949886322021484, + "logps/ref_chosen": -60.66877746582031, + "logps/ref_rejected": -88.30673217773438, + "logps/rejected": -90.22268676757812, + "loss": 1.0672, + "margin_dpo/margin_mean": 0.6348508596420288, + "margin_dpo/margin_std": 0.949616551399231, + "step": 96 + }, + { + "KL/chosen_KL_mean": -1.5132274627685547, + "KL/mean": -1.7518854141235352, + "KL/rejected_KL_mean": -1.9905433654785156, + "KL/std": 0.8738954067230225, + "epoch": 0.14663643235071808, + "fcm_dpo/beta": 0.8598639965057373, + "fcm_dpo/delta": -0.012458987534046173, + "fcm_dpo/margin": 0.4773203432559967, + "fcm_dpo/q_t": 0.4190227687358856, + "grad_norm": 245.85345458984375, + "learning_rate": 4.970651810649666e-07, + "logits/chosen": 0.06072793900966644, + "logits/rejected": 0.01822689175605774, + "logps/chosen": -66.5573501586914, + "logps/ref_chosen": -65.04412078857422, + "logps/ref_rejected": -78.42092895507812, + "logps/rejected": -80.41146850585938, + "loss": 1.192, + "margin_dpo/margin_mean": 0.47732043266296387, + "margin_dpo/margin_std": 1.0347280502319336, + "step": 97 + }, + { + "KL/chosen_KL_mean": -1.2721519470214844, + "KL/mean": -1.4518141746520996, + "KL/rejected_KL_mean": -1.6314773559570312, + "KL/std": 0.8410429954528809, + "epoch": 0.14814814814814814, + "fcm_dpo/beta": 0.8739485144615173, + "fcm_dpo/delta": 0.08870487660169601, + "fcm_dpo/margin": 0.3593263030052185, + "fcm_dpo/q_t": 0.43300604820251465, + "grad_norm": 235.42140197753906, + "learning_rate": 4.968597221690985e-07, + "logits/chosen": 0.1716204583644867, + "logits/rejected": 0.14437055587768555, + "logps/chosen": -56.77538299560547, + "logps/ref_chosen": -55.503231048583984, + "logps/ref_rejected": -72.81553649902344, + "logps/rejected": -74.447021484375, + "loss": 1.2292, + "margin_dpo/margin_mean": 0.35932624340057373, + "margin_dpo/margin_std": 0.8575361967086792, + "step": 98 + }, + { + "KL/chosen_KL_mean": -1.261861801147461, + "KL/mean": -1.5936691761016846, + "KL/rejected_KL_mean": -1.92547607421875, + "KL/std": 1.0141127109527588, + "epoch": 0.14965986394557823, + "fcm_dpo/beta": 0.8584821224212646, + "fcm_dpo/delta": -0.1800466775894165, + "fcm_dpo/margin": 0.663617730140686, + "fcm_dpo/q_t": 0.38134729862213135, + "grad_norm": 223.2831268310547, + "learning_rate": 4.966473580761389e-07, + "logits/chosen": 0.15894950926303864, + "logits/rejected": 0.12229165434837341, + "logps/chosen": -59.837501525878906, + "logps/ref_chosen": -58.57563781738281, + "logps/ref_rejected": -78.693603515625, + "logps/rejected": -80.61907958984375, + "loss": 1.0597, + "margin_dpo/margin_mean": 0.6636172533035278, + "margin_dpo/margin_std": 1.0391426086425781, + "step": 99 + }, + { + "KL/chosen_KL_mean": -1.4188766479492188, + "KL/mean": -1.6849563121795654, + "KL/rejected_KL_mean": -1.9510383605957031, + "KL/std": 0.9670500755310059, + "epoch": 0.15117157974300832, + "fcm_dpo/beta": 0.8525861501693726, + "fcm_dpo/delta": -0.05790305882692337, + "fcm_dpo/margin": 0.5321630239486694, + "fcm_dpo/q_t": 0.4141319692134857, + "grad_norm": 253.90597534179688, + "learning_rate": 4.964280947263676e-07, + "logits/chosen": 0.12935097515583038, + "logits/rejected": 0.12235041707754135, + "logps/chosen": -81.00231170654297, + "logps/ref_chosen": -79.58343505859375, + "logps/ref_rejected": -92.152587890625, + "logps/rejected": -94.10362243652344, + "loss": 1.2366, + "margin_dpo/margin_mean": 0.5321632623672485, + "margin_dpo/margin_std": 1.2714905738830566, + "step": 100 + }, + { + "KL/chosen_KL_mean": -1.2464981079101562, + "KL/mean": -1.6342787742614746, + "KL/rejected_KL_mean": -2.0220565795898438, + "KL/std": 0.988823652267456, + "epoch": 0.15268329554043839, + "fcm_dpo/beta": 0.8176276683807373, + "fcm_dpo/delta": -0.2493351250886917, + "fcm_dpo/margin": 0.7755611538887024, + "fcm_dpo/q_t": 0.36288192868232727, + "grad_norm": 168.5565948486328, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": 0.15569889545440674, + "logits/rejected": 0.11886613070964813, + "logps/chosen": -53.57928466796875, + "logps/ref_chosen": -52.332786560058594, + "logps/ref_rejected": -69.55589294433594, + "logps/rejected": -71.57795715332031, + "loss": 0.9804, + "margin_dpo/margin_mean": 0.7755616903305054, + "margin_dpo/margin_std": 0.9270931482315063, + "step": 101 + }, + { + "KL/chosen_KL_mean": -1.4936847686767578, + "KL/mean": -1.764020323753357, + "KL/rejected_KL_mean": -2.03436279296875, + "KL/std": 0.9787428379058838, + "epoch": 0.15419501133786848, + "fcm_dpo/beta": 0.7976725101470947, + "fcm_dpo/delta": -0.03269146382808685, + "fcm_dpo/margin": 0.5406721234321594, + "fcm_dpo/q_t": 0.40772902965545654, + "grad_norm": 206.413818359375, + "learning_rate": 4.959688949822748e-07, + "logits/chosen": 0.0799262598156929, + "logits/rejected": 0.04157480597496033, + "logps/chosen": -66.23716735839844, + "logps/ref_chosen": -64.74348449707031, + "logps/ref_rejected": -69.06132507324219, + "logps/rejected": -71.09568786621094, + "loss": 1.1749, + "margin_dpo/margin_mean": 0.5406715869903564, + "margin_dpo/margin_std": 1.1025452613830566, + "step": 102 + }, + { + "KL/chosen_KL_mean": -1.4423542022705078, + "KL/mean": -1.7633211612701416, + "KL/rejected_KL_mean": -2.08428955078125, + "KL/std": 1.0820667743682861, + "epoch": 0.15570672713529857, + "fcm_dpo/beta": 0.7843052744865417, + "fcm_dpo/delta": -0.10881097614765167, + "fcm_dpo/margin": 0.6419306993484497, + "fcm_dpo/q_t": 0.3946912884712219, + "grad_norm": 215.6351318359375, + "learning_rate": 4.957289714327572e-07, + "logits/chosen": 0.16433322429656982, + "logits/rejected": 0.13270039856433868, + "logps/chosen": -65.27899932861328, + "logps/ref_chosen": -63.83664321899414, + "logps/ref_rejected": -79.32362365722656, + "logps/rejected": -81.40791320800781, + "loss": 1.1226, + "margin_dpo/margin_mean": 0.6419302821159363, + "margin_dpo/margin_std": 1.135719656944275, + "step": 103 + }, + { + "KL/chosen_KL_mean": -1.387624740600586, + "KL/mean": -1.7114492654800415, + "KL/rejected_KL_mean": -2.0352783203125, + "KL/std": 1.1588332653045654, + "epoch": 0.15721844293272866, + "fcm_dpo/beta": 0.7690836787223816, + "fcm_dpo/delta": -0.10314887762069702, + "fcm_dpo/margin": 0.6476492285728455, + "fcm_dpo/q_t": 0.39995285868644714, + "grad_norm": 215.02183532714844, + "learning_rate": 4.954821743156767e-07, + "logits/chosen": 0.17702050507068634, + "logits/rejected": 0.09312883764505386, + "logps/chosen": -62.38683319091797, + "logps/ref_chosen": -60.99920654296875, + "logps/ref_rejected": -98.84645080566406, + "logps/rejected": -100.88172912597656, + "loss": 1.1088, + "margin_dpo/margin_mean": 0.6476488709449768, + "margin_dpo/margin_std": 1.122774600982666, + "step": 104 + }, + { + "KL/chosen_KL_mean": -1.458160400390625, + "KL/mean": -1.7229704856872559, + "KL/rejected_KL_mean": -1.9877815246582031, + "KL/std": 1.0482707023620605, + "epoch": 0.15873015873015872, + "fcm_dpo/beta": 0.7652902007102966, + "fcm_dpo/delta": -0.006055701524019241, + "fcm_dpo/margin": 0.529621958732605, + "fcm_dpo/q_t": 0.4152873754501343, + "grad_norm": 237.42147827148438, + "learning_rate": 4.952285105344791e-07, + "logits/chosen": 0.09127002209424973, + "logits/rejected": 0.04044020175933838, + "logps/chosen": -72.40843200683594, + "logps/ref_chosen": -70.95027160644531, + "logps/ref_rejected": -87.88340759277344, + "logps/rejected": -89.8711929321289, + "loss": 1.1959, + "margin_dpo/margin_mean": 0.5296220779418945, + "margin_dpo/margin_std": 1.151416301727295, + "step": 105 + }, + { + "KL/chosen_KL_mean": -1.3451671600341797, + "KL/mean": -1.6426818370819092, + "KL/rejected_KL_mean": -1.9401969909667969, + "KL/std": 0.9987841844558716, + "epoch": 0.1602418745275888, + "fcm_dpo/beta": 0.7533386945724487, + "fcm_dpo/delta": -0.05080245062708855, + "fcm_dpo/margin": 0.5950255393981934, + "fcm_dpo/q_t": 0.4004518985748291, + "grad_norm": 204.8391571044922, + "learning_rate": 4.949679871846857e-07, + "logits/chosen": 0.16151435673236847, + "logits/rejected": 0.14808031916618347, + "logps/chosen": -63.80449676513672, + "logps/ref_chosen": -62.45933151245117, + "logps/ref_rejected": -67.00595092773438, + "logps/rejected": -68.9461441040039, + "loss": 1.1499, + "margin_dpo/margin_mean": 0.5950250029563904, + "margin_dpo/margin_std": 1.1285878419876099, + "step": 106 + }, + { + "KL/chosen_KL_mean": -1.5086212158203125, + "KL/mean": -1.7033252716064453, + "KL/rejected_KL_mean": -1.8980293273925781, + "KL/std": 1.1155552864074707, + "epoch": 0.1617535903250189, + "fcm_dpo/beta": 0.7590333819389343, + "fcm_dpo/delta": 0.10701896250247955, + "fcm_dpo/margin": 0.3894113004207611, + "fcm_dpo/q_t": 0.4394727349281311, + "grad_norm": 257.46246337890625, + "learning_rate": 4.947006115536947e-07, + "logits/chosen": 0.10400848835706711, + "logits/rejected": 0.0829182118177414, + "logps/chosen": -77.34658813476562, + "logps/ref_chosen": -75.83796691894531, + "logps/ref_rejected": -87.74038696289062, + "logps/rejected": -89.63841247558594, + "loss": 1.3187, + "margin_dpo/margin_mean": 0.3894110918045044, + "margin_dpo/margin_std": 1.2437622547149658, + "step": 107 + }, + { + "KL/chosen_KL_mean": -1.3229484558105469, + "KL/mean": -1.6645491123199463, + "KL/rejected_KL_mean": -2.0061492919921875, + "KL/std": 0.9608972668647766, + "epoch": 0.16326530612244897, + "fcm_dpo/beta": 0.7589484453201294, + "fcm_dpo/delta": -0.1248454749584198, + "fcm_dpo/margin": 0.6832001805305481, + "fcm_dpo/q_t": 0.39446821808815, + "grad_norm": 196.87106323242188, + "learning_rate": 4.944263911205772e-07, + "logits/chosen": 0.0801883414387703, + "logits/rejected": 0.053226158022880554, + "logps/chosen": -69.7161865234375, + "logps/ref_chosen": -68.39323425292969, + "logps/ref_rejected": -83.24267578125, + "logps/rejected": -85.24882507324219, + "loss": 1.1301, + "margin_dpo/margin_mean": 0.6831998229026794, + "margin_dpo/margin_std": 1.2444127798080444, + "step": 108 + }, + { + "KL/chosen_KL_mean": -1.3410778045654297, + "KL/mean": -1.7414236068725586, + "KL/rejected_KL_mean": -2.1417694091796875, + "KL/std": 1.0900359153747559, + "epoch": 0.16477702191987906, + "fcm_dpo/beta": 0.729695200920105, + "fcm_dpo/delta": -0.19558557868003845, + "fcm_dpo/margin": 0.8006910085678101, + "fcm_dpo/q_t": 0.38231924176216125, + "grad_norm": 165.07757568359375, + "learning_rate": 4.941453335558681e-07, + "logits/chosen": 0.12404434382915497, + "logits/rejected": 0.07289332896471024, + "logps/chosen": -56.868560791015625, + "logps/ref_chosen": -55.52748107910156, + "logps/ref_rejected": -83.55218505859375, + "logps/rejected": -85.69395446777344, + "loss": 1.0315, + "margin_dpo/margin_mean": 0.8006904125213623, + "margin_dpo/margin_std": 1.1890592575073242, + "step": 109 + }, + { + "KL/chosen_KL_mean": -1.5522308349609375, + "KL/mean": -1.7875815629959106, + "KL/rejected_KL_mean": -2.022930145263672, + "KL/std": 1.0237528085708618, + "epoch": 0.16628873771730915, + "fcm_dpo/beta": 0.7312818169593811, + "fcm_dpo/delta": 0.05713893473148346, + "fcm_dpo/margin": 0.4706960916519165, + "fcm_dpo/q_t": 0.4263302683830261, + "grad_norm": 223.92262268066406, + "learning_rate": 4.938574467213517e-07, + "logits/chosen": 0.06877541542053223, + "logits/rejected": 0.07652991265058517, + "logps/chosen": -82.71097564697266, + "logps/ref_chosen": -81.15874481201172, + "logps/ref_rejected": -72.56021118164062, + "logps/rejected": -74.58314514160156, + "loss": 1.2187, + "margin_dpo/margin_mean": 0.47069618105888367, + "margin_dpo/margin_std": 1.1055645942687988, + "step": 110 + }, + { + "KL/chosen_KL_mean": -1.7109088897705078, + "KL/mean": -1.9763455390930176, + "KL/rejected_KL_mean": -2.2417869567871094, + "KL/std": 1.0940556526184082, + "epoch": 0.16780045351473924, + "fcm_dpo/beta": 0.7348920106887817, + "fcm_dpo/delta": 0.009718731045722961, + "fcm_dpo/margin": 0.5308776497840881, + "fcm_dpo/q_t": 0.41322413086891174, + "grad_norm": 196.10678100585938, + "learning_rate": 4.935627386698418e-07, + "logits/chosen": 0.1737518310546875, + "logits/rejected": 0.139468714594841, + "logps/chosen": -54.06989288330078, + "logps/ref_chosen": -52.358985900878906, + "logps/ref_rejected": -77.06150817871094, + "logps/rejected": -79.30329132080078, + "loss": 1.2238, + "margin_dpo/margin_mean": 0.5308778285980225, + "margin_dpo/margin_std": 1.2132712602615356, + "step": 111 + }, + { + "KL/chosen_KL_mean": -1.4015159606933594, + "KL/mean": -1.761267066001892, + "KL/rejected_KL_mean": -2.1210174560546875, + "KL/std": 1.0548936128616333, + "epoch": 0.1693121693121693, + "fcm_dpo/beta": 0.7160321474075317, + "fcm_dpo/delta": -0.12210012227296829, + "fcm_dpo/margin": 0.7195063829421997, + "fcm_dpo/q_t": 0.3860316574573517, + "grad_norm": 200.53457641601562, + "learning_rate": 4.932612176449559e-07, + "logits/chosen": 0.08839970827102661, + "logits/rejected": 0.030455876141786575, + "logps/chosen": -64.42158508300781, + "logps/ref_chosen": -63.02006530761719, + "logps/ref_rejected": -111.36941528320312, + "logps/rejected": -113.49043273925781, + "loss": 1.0908, + "margin_dpo/margin_mean": 0.7195055484771729, + "margin_dpo/margin_std": 1.1482014656066895, + "step": 112 + }, + { + "KL/chosen_KL_mean": -1.6727027893066406, + "KL/mean": -1.985128402709961, + "KL/rejected_KL_mean": -2.2975540161132812, + "KL/std": 1.1449403762817383, + "epoch": 0.1708238851095994, + "fcm_dpo/beta": 0.7157949805259705, + "fcm_dpo/delta": -0.05103008449077606, + "fcm_dpo/margin": 0.6248476505279541, + "fcm_dpo/q_t": 0.4022253751754761, + "grad_norm": 205.1897735595703, + "learning_rate": 4.929528920808854e-07, + "logits/chosen": 0.10172772407531738, + "logits/rejected": 0.06672029197216034, + "logps/chosen": -57.48036575317383, + "logps/ref_chosen": -55.80766296386719, + "logps/ref_rejected": -69.84014129638672, + "logps/rejected": -72.1376953125, + "loss": 1.1852, + "margin_dpo/margin_mean": 0.6248472929000854, + "margin_dpo/margin_std": 1.253474235534668, + "step": 113 + }, + { + "KL/chosen_KL_mean": -1.373983383178711, + "KL/mean": -1.8980087041854858, + "KL/rejected_KL_mean": -2.422039031982422, + "KL/std": 1.2073814868927002, + "epoch": 0.17233560090702948, + "fcm_dpo/beta": 0.6685370206832886, + "fcm_dpo/delta": -0.3267349302768707, + "fcm_dpo/margin": 1.0480527877807617, + "fcm_dpo/q_t": 0.3540674149990082, + "grad_norm": 143.26123046875, + "learning_rate": 4.92637770602159e-07, + "logits/chosen": 0.16866153478622437, + "logits/rejected": 0.11124296486377716, + "logps/chosen": -67.70675659179688, + "logps/ref_chosen": -66.33277130126953, + "logps/ref_rejected": -71.61489868164062, + "logps/rejected": -74.03693389892578, + "loss": 0.9664, + "margin_dpo/margin_mean": 1.048052430152893, + "margin_dpo/margin_std": 1.3153923749923706, + "step": 114 + }, + { + "KL/chosen_KL_mean": -1.6200141906738281, + "KL/mean": -1.9981410503387451, + "KL/rejected_KL_mean": -2.3762683868408203, + "KL/std": 1.1273773908615112, + "epoch": 0.17384731670445955, + "fcm_dpo/beta": 0.6535633206367493, + "fcm_dpo/delta": -0.09904934465885162, + "fcm_dpo/margin": 0.7562557458877563, + "fcm_dpo/q_t": 0.39832305908203125, + "grad_norm": 175.138427734375, + "learning_rate": 4.923158620234019e-07, + "logits/chosen": 0.14349928498268127, + "logits/rejected": 0.08649460226297379, + "logps/chosen": -57.36905288696289, + "logps/ref_chosen": -55.74903869628906, + "logps/ref_rejected": -79.59849548339844, + "logps/rejected": -81.97476196289062, + "loss": 1.102, + "margin_dpo/margin_mean": 0.7562559843063354, + "margin_dpo/margin_std": 1.2784497737884521, + "step": 115 + }, + { + "KL/chosen_KL_mean": -1.5154285430908203, + "KL/mean": -1.9526760578155518, + "KL/rejected_KL_mean": -2.3899269104003906, + "KL/std": 1.1451892852783203, + "epoch": 0.17535903250188964, + "fcm_dpo/beta": 0.6347832679748535, + "fcm_dpo/delta": -0.1639837920665741, + "fcm_dpo/margin": 0.8745034337043762, + "fcm_dpo/q_t": 0.37725359201431274, + "grad_norm": 135.62338256835938, + "learning_rate": 4.91987175349089e-07, + "logits/chosen": 0.14225362241268158, + "logits/rejected": 0.07968921959400177, + "logps/chosen": -50.880592346191406, + "logps/ref_chosen": -49.36516571044922, + "logps/ref_rejected": -72.84671020507812, + "logps/rejected": -75.23663330078125, + "loss": 1.0137, + "margin_dpo/margin_mean": 0.8745033740997314, + "margin_dpo/margin_std": 1.0888726711273193, + "step": 116 + }, + { + "KL/chosen_KL_mean": -1.4024600982666016, + "KL/mean": -1.747429609298706, + "KL/rejected_KL_mean": -2.0924034118652344, + "KL/std": 1.0587306022644043, + "epoch": 0.17687074829931973, + "fcm_dpo/beta": 0.6196062564849854, + "fcm_dpo/delta": -0.030679665505886078, + "fcm_dpo/margin": 0.6899378895759583, + "fcm_dpo/q_t": 0.4040665626525879, + "grad_norm": 151.564697265625, + "learning_rate": 4.916517197732933e-07, + "logits/chosen": 0.1525934934616089, + "logits/rejected": 0.1185963824391365, + "logps/chosen": -59.11335754394531, + "logps/ref_chosen": -57.710899353027344, + "logps/ref_rejected": -69.77253723144531, + "logps/rejected": -71.86494445800781, + "loss": 1.1369, + "margin_dpo/margin_mean": 0.6899383068084717, + "margin_dpo/margin_std": 1.2021629810333252, + "step": 117 + }, + { + "KL/chosen_KL_mean": -1.4043560028076172, + "KL/mean": -1.8314993381500244, + "KL/rejected_KL_mean": -2.258636474609375, + "KL/std": 1.092620849609375, + "epoch": 0.17838246409674982, + "fcm_dpo/beta": 0.6096054315567017, + "fcm_dpo/delta": -0.12845522165298462, + "fcm_dpo/margin": 0.8542841672897339, + "fcm_dpo/q_t": 0.38544875383377075, + "grad_norm": 150.0379180908203, + "learning_rate": 4.913095046794281e-07, + "logits/chosen": 0.18141870200634003, + "logits/rejected": 0.1441626399755478, + "logps/chosen": -53.884254455566406, + "logps/ref_chosen": -52.479896545410156, + "logps/ref_rejected": -81.359130859375, + "logps/rejected": -83.61776733398438, + "loss": 1.0589, + "margin_dpo/margin_mean": 0.8542840480804443, + "margin_dpo/margin_std": 1.218125820159912, + "step": 118 + }, + { + "KL/chosen_KL_mean": -1.8466033935546875, + "KL/mean": -2.1639060974121094, + "KL/rejected_KL_mean": -2.481212615966797, + "KL/std": 1.2136081457138062, + "epoch": 0.17989417989417988, + "fcm_dpo/beta": 0.6111596822738647, + "fcm_dpo/delta": 0.012469585984945297, + "fcm_dpo/margin": 0.6346088647842407, + "fcm_dpo/q_t": 0.41613152623176575, + "grad_norm": 163.7132568359375, + "learning_rate": 4.909605396399855e-07, + "logits/chosen": 0.1168576031923294, + "logits/rejected": 0.08203059434890747, + "logps/chosen": -63.20427703857422, + "logps/ref_chosen": -61.35767364501953, + "logps/ref_rejected": -75.71510314941406, + "logps/rejected": -78.1963119506836, + "loss": 1.1991, + "margin_dpo/margin_mean": 0.6346089839935303, + "margin_dpo/margin_std": 1.380927562713623, + "step": 119 + }, + { + "KL/chosen_KL_mean": -1.4735527038574219, + "KL/mean": -1.9682976007461548, + "KL/rejected_KL_mean": -2.463043212890625, + "KL/std": 1.1643463373184204, + "epoch": 0.18140589569160998, + "fcm_dpo/beta": 0.5917296409606934, + "fcm_dpo/delta": -0.19727984070777893, + "fcm_dpo/margin": 0.9894887208938599, + "fcm_dpo/q_t": 0.3721131980419159, + "grad_norm": 135.58323669433594, + "learning_rate": 4.906048344162676e-07, + "logits/chosen": 0.1065509170293808, + "logits/rejected": 0.05253705382347107, + "logps/chosen": -61.38111877441406, + "logps/ref_chosen": -59.907569885253906, + "logps/ref_rejected": -79.6910629272461, + "logps/rejected": -82.15410614013672, + "loss": 0.9956, + "margin_dpo/margin_mean": 0.989488959312439, + "margin_dpo/margin_std": 1.2124404907226562, + "step": 120 + }, + { + "KL/chosen_KL_mean": -1.682455062866211, + "KL/mean": -2.09917950630188, + "KL/rejected_KL_mean": -2.5159072875976562, + "KL/std": 1.2347018718719482, + "epoch": 0.18291761148904007, + "fcm_dpo/beta": 0.577370822429657, + "fcm_dpo/delta": -0.08569201827049255, + "fcm_dpo/margin": 0.8334502577781677, + "fcm_dpo/q_t": 0.39638879895210266, + "grad_norm": 128.85064697265625, + "learning_rate": 4.902423989581143e-07, + "logits/chosen": 0.21048860251903534, + "logits/rejected": 0.133277028799057, + "logps/chosen": -57.34849548339844, + "logps/ref_chosen": -55.66604232788086, + "logps/ref_rejected": -101.56233978271484, + "logps/rejected": -104.0782470703125, + "loss": 1.0754, + "margin_dpo/margin_mean": 0.8334493637084961, + "margin_dpo/margin_std": 1.2757973670959473, + "step": 121 + }, + { + "KL/chosen_KL_mean": -1.7021980285644531, + "KL/mean": -2.229940414428711, + "KL/rejected_KL_mean": -2.7576828002929688, + "KL/std": 1.328824520111084, + "epoch": 0.18442932728647016, + "fcm_dpo/beta": 0.5614632368087769, + "fcm_dpo/delta": -0.20425938069820404, + "fcm_dpo/margin": 1.0554801225662231, + "fcm_dpo/q_t": 0.374777227640152, + "grad_norm": 151.41307067871094, + "learning_rate": 4.898732434036243e-07, + "logits/chosen": 0.11900610476732254, + "logits/rejected": 0.08751888573169708, + "logps/chosen": -65.03657531738281, + "logps/ref_chosen": -63.334373474121094, + "logps/ref_rejected": -73.67523193359375, + "logps/rejected": -76.43292236328125, + "loss": 1.0244, + "margin_dpo/margin_mean": 1.05548095703125, + "margin_dpo/margin_std": 1.4838553667068481, + "step": 122 + }, + { + "KL/chosen_KL_mean": -1.6267662048339844, + "KL/mean": -2.0346689224243164, + "KL/rejected_KL_mean": -2.4425735473632812, + "KL/std": 1.209721326828003, + "epoch": 0.18594104308390022, + "fcm_dpo/beta": 0.5542995929718018, + "fcm_dpo/delta": -0.05618344992399216, + "fcm_dpo/margin": 0.8158098459243774, + "fcm_dpo/q_t": 0.39761269092559814, + "grad_norm": 141.0292205810547, + "learning_rate": 4.894973780788722e-07, + "logits/chosen": 0.158452570438385, + "logits/rejected": 0.11947640776634216, + "logps/chosen": -58.525508880615234, + "logps/ref_chosen": -56.89874267578125, + "logps/ref_rejected": -78.97028350830078, + "logps/rejected": -81.41285705566406, + "loss": 1.1263, + "margin_dpo/margin_mean": 0.8158098459243774, + "margin_dpo/margin_std": 1.3711764812469482, + "step": 123 + }, + { + "KL/chosen_KL_mean": -1.748464584350586, + "KL/mean": -2.3354647159576416, + "KL/rejected_KL_mean": -2.922466278076172, + "KL/std": 1.2677295207977295, + "epoch": 0.1874527588813303, + "fcm_dpo/beta": 0.5262091159820557, + "fcm_dpo/delta": -0.23351916670799255, + "fcm_dpo/margin": 1.1740025281906128, + "fcm_dpo/q_t": 0.3634141683578491, + "grad_norm": 114.40848541259766, + "learning_rate": 4.89114813497619e-07, + "logits/chosen": 0.16980503499507904, + "logits/rejected": 0.11593275517225266, + "logps/chosen": -58.86454772949219, + "logps/ref_chosen": -57.116085052490234, + "logps/ref_rejected": -87.93074035644531, + "logps/rejected": -90.85321044921875, + "loss": 0.9742, + "margin_dpo/margin_mean": 1.1740025281906128, + "margin_dpo/margin_std": 1.339629888534546, + "step": 124 + }, + { + "KL/chosen_KL_mean": -1.9098529815673828, + "KL/mean": -2.3675875663757324, + "KL/rejected_KL_mean": -2.8253173828125, + "KL/std": 1.4183313846588135, + "epoch": 0.1889644746787604, + "fcm_dpo/beta": 0.5146192312240601, + "fcm_dpo/delta": -0.07511819899082184, + "fcm_dpo/margin": 0.9154636859893799, + "fcm_dpo/q_t": 0.39758336544036865, + "grad_norm": 131.99765014648438, + "learning_rate": 4.887255603610184e-07, + "logits/chosen": 0.19287623465061188, + "logits/rejected": 0.13918162882328033, + "logps/chosen": -67.61602783203125, + "logps/ref_chosen": -65.7061767578125, + "logps/ref_rejected": -91.72711944580078, + "logps/rejected": -94.55243682861328, + "loss": 1.0888, + "margin_dpo/margin_mean": 0.9154644012451172, + "margin_dpo/margin_std": 1.4167892932891846, + "step": 125 + }, + { + "KL/chosen_KL_mean": -1.6904373168945312, + "KL/mean": -2.1172661781311035, + "KL/rejected_KL_mean": -2.544099807739258, + "KL/std": 1.5803725719451904, + "epoch": 0.19047619047619047, + "fcm_dpo/beta": 0.5104827880859375, + "fcm_dpo/delta": -0.037569280713796616, + "fcm_dpo/margin": 0.8536651134490967, + "fcm_dpo/q_t": 0.4115992784500122, + "grad_norm": 120.52528381347656, + "learning_rate": 4.883296295573176e-07, + "logits/chosen": 0.007485136389732361, + "logits/rejected": 0.001452924683690071, + "logps/chosen": -69.86652374267578, + "logps/ref_chosen": -68.17608642578125, + "logps/ref_rejected": -65.1175537109375, + "logps/rejected": -67.66165161132812, + "loss": 1.1464, + "margin_dpo/margin_mean": 0.8536652326583862, + "margin_dpo/margin_std": 1.6970319747924805, + "step": 126 + }, + { + "KL/chosen_KL_mean": -1.8688240051269531, + "KL/mean": -2.4150619506835938, + "KL/rejected_KL_mean": -2.961301803588867, + "KL/std": 1.2444980144500732, + "epoch": 0.19198790627362056, + "fcm_dpo/beta": 0.4957229495048523, + "fcm_dpo/delta": -0.15168866515159607, + "fcm_dpo/margin": 1.0924748182296753, + "fcm_dpo/q_t": 0.378802627325058, + "grad_norm": 115.55281829833984, + "learning_rate": 4.87927032161552e-07, + "logits/chosen": 0.1286633312702179, + "logits/rejected": 0.0994386374950409, + "logps/chosen": -63.74905776977539, + "logps/ref_chosen": -61.88023376464844, + "logps/ref_rejected": -68.46012878417969, + "logps/rejected": -71.42142486572266, + "loss": 1.0108, + "margin_dpo/margin_mean": 1.0924749374389648, + "margin_dpo/margin_std": 1.278612494468689, + "step": 127 + }, + { + "KL/chosen_KL_mean": -2.003080368041992, + "KL/mean": -2.453127384185791, + "KL/rejected_KL_mean": -2.903179168701172, + "KL/std": 1.4483566284179688, + "epoch": 0.19349962207105065, + "fcm_dpo/beta": 0.49017786979675293, + "fcm_dpo/delta": -0.04357750713825226, + "fcm_dpo/margin": 0.900090754032135, + "fcm_dpo/q_t": 0.40476569533348083, + "grad_norm": 127.00218963623047, + "learning_rate": 4.875177794352363e-07, + "logits/chosen": 0.16356945037841797, + "logits/rejected": 0.11114731431007385, + "logps/chosen": -68.71206665039062, + "logps/ref_chosen": -66.708984375, + "logps/ref_rejected": -94.97969055175781, + "logps/rejected": -97.88287353515625, + "loss": 1.1399, + "margin_dpo/margin_mean": 0.9000910520553589, + "margin_dpo/margin_std": 1.6488995552062988, + "step": 128 + }, + { + "KL/chosen_KL_mean": -2.2038707733154297, + "KL/mean": -2.5676302909851074, + "KL/rejected_KL_mean": -2.931396484375, + "KL/std": 1.304401159286499, + "epoch": 0.19501133786848074, + "fcm_dpo/beta": 0.49330243468284607, + "fcm_dpo/delta": 0.04265614598989487, + "fcm_dpo/margin": 0.7275242805480957, + "fcm_dpo/q_t": 0.4221858084201813, + "grad_norm": 138.87521362304688, + "learning_rate": 4.871018828260491e-07, + "logits/chosen": 0.13030391931533813, + "logits/rejected": 0.12313122302293777, + "logps/chosen": -67.54269409179688, + "logps/ref_chosen": -65.33882904052734, + "logps/ref_rejected": -68.06109619140625, + "logps/rejected": -70.99249267578125, + "loss": 1.177, + "margin_dpo/margin_mean": 0.7275242805480957, + "margin_dpo/margin_std": 1.4568142890930176, + "step": 129 + }, + { + "KL/chosen_KL_mean": -2.025636672973633, + "KL/mean": -2.517505645751953, + "KL/rejected_KL_mean": -3.0093765258789062, + "KL/std": 1.313607931137085, + "epoch": 0.1965230536659108, + "fcm_dpo/beta": 0.48430347442626953, + "fcm_dpo/delta": -0.08306446671485901, + "fcm_dpo/margin": 0.9837359189987183, + "fcm_dpo/q_t": 0.3932478129863739, + "grad_norm": 130.4871063232422, + "learning_rate": 4.866793539675126e-07, + "logits/chosen": 0.08757339417934418, + "logits/rejected": 0.04383823275566101, + "logps/chosen": -60.68638229370117, + "logps/ref_chosen": -58.660743713378906, + "logps/ref_rejected": -79.24510192871094, + "logps/rejected": -82.25447845458984, + "loss": 1.0553, + "margin_dpo/margin_mean": 0.9837361574172974, + "margin_dpo/margin_std": 1.2436227798461914, + "step": 130 + }, + { + "KL/chosen_KL_mean": -1.933258056640625, + "KL/mean": -2.4867091178894043, + "KL/rejected_KL_mean": -3.0401611328125, + "KL/std": 1.5524529218673706, + "epoch": 0.1980347694633409, + "fcm_dpo/beta": 0.4773693382740021, + "fcm_dpo/delta": -0.13555176556110382, + "fcm_dpo/margin": 1.1069055795669556, + "fcm_dpo/q_t": 0.3849940001964569, + "grad_norm": 114.3026123046875, + "learning_rate": 4.86250204678667e-07, + "logits/chosen": 0.1033594086766243, + "logits/rejected": 0.04820697754621506, + "logps/chosen": -54.447792053222656, + "logps/ref_chosen": -52.51453399658203, + "logps/ref_rejected": -85.18299865722656, + "logps/rejected": -88.22315979003906, + "loss": 1.0762, + "margin_dpo/margin_mean": 1.1069060564041138, + "margin_dpo/margin_std": 1.702505111694336, + "step": 131 + }, + { + "KL/chosen_KL_mean": -2.1760196685791016, + "KL/mean": -2.6583878993988037, + "KL/rejected_KL_mean": -3.140756607055664, + "KL/std": 1.49051034450531, + "epoch": 0.19954648526077098, + "fcm_dpo/beta": 0.4701109528541565, + "fcm_dpo/delta": -0.05607073754072189, + "fcm_dpo/margin": 0.9647371768951416, + "fcm_dpo/q_t": 0.3987612724304199, + "grad_norm": 122.06319427490234, + "learning_rate": 4.858144469637408e-07, + "logits/chosen": 0.21260422468185425, + "logits/rejected": 0.18176573514938354, + "logps/chosen": -67.86115264892578, + "logps/ref_chosen": -65.68513488769531, + "logps/ref_rejected": -69.54120635986328, + "logps/rejected": -72.68196105957031, + "loss": 1.1113, + "margin_dpo/margin_mean": 0.9647375345230103, + "margin_dpo/margin_std": 1.6063919067382812, + "step": 132 + }, + { + "KL/chosen_KL_mean": -2.1477317810058594, + "KL/mean": -2.5903117656707764, + "KL/rejected_KL_mean": -3.032886505126953, + "KL/std": 1.4966492652893066, + "epoch": 0.20105820105820105, + "fcm_dpo/beta": 0.4674025774002075, + "fcm_dpo/delta": -0.014334164559841156, + "fcm_dpo/margin": 0.8851548433303833, + "fcm_dpo/q_t": 0.4093969762325287, + "grad_norm": 125.8607177734375, + "learning_rate": 4.853720930118138e-07, + "logits/chosen": 0.12384993582963943, + "logits/rejected": 0.11437453329563141, + "logps/chosen": -65.745849609375, + "logps/ref_chosen": -63.598114013671875, + "logps/ref_rejected": -73.72798156738281, + "logps/rejected": -76.7608642578125, + "loss": 1.1389, + "margin_dpo/margin_mean": 0.8851544857025146, + "margin_dpo/margin_std": 1.5839178562164307, + "step": 133 + }, + { + "KL/chosen_KL_mean": -2.069742202758789, + "KL/mean": -2.7214608192443848, + "KL/rejected_KL_mean": -3.373180389404297, + "KL/std": 1.6641645431518555, + "epoch": 0.20256991685563114, + "fcm_dpo/beta": 0.45107993483543396, + "fcm_dpo/delta": -0.20145797729492188, + "fcm_dpo/margin": 1.3034403324127197, + "fcm_dpo/q_t": 0.3716809153556824, + "grad_norm": 105.60230255126953, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": 0.21557536721229553, + "logits/rejected": 0.16419634222984314, + "logps/chosen": -55.86431121826172, + "logps/ref_chosen": -53.79457092285156, + "logps/ref_rejected": -74.16741943359375, + "logps/rejected": -77.54060363769531, + "loss": 1.0145, + "margin_dpo/margin_mean": 1.303440809249878, + "margin_dpo/margin_std": 1.7132940292358398, + "step": 134 + }, + { + "KL/chosen_KL_mean": -2.093151092529297, + "KL/mean": -2.524364471435547, + "KL/rejected_KL_mean": -2.9555816650390625, + "KL/std": 1.5630788803100586, + "epoch": 0.20408163265306123, + "fcm_dpo/beta": 0.44986188411712646, + "fcm_dpo/delta": 0.012506252154707909, + "fcm_dpo/margin": 0.8624306321144104, + "fcm_dpo/q_t": 0.4156408905982971, + "grad_norm": 102.62089538574219, + "learning_rate": 4.844676460754862e-07, + "logits/chosen": 0.14737987518310547, + "logits/rejected": 0.11628959327936172, + "logps/chosen": -51.53422546386719, + "logps/ref_chosen": -49.441078186035156, + "logps/ref_rejected": -65.96878051757812, + "logps/rejected": -68.92436218261719, + "loss": 1.1536, + "margin_dpo/margin_mean": 0.8624303340911865, + "margin_dpo/margin_std": 1.610809564590454, + "step": 135 + }, + { + "KL/chosen_KL_mean": -2.437408447265625, + "KL/mean": -2.967642307281494, + "KL/rejected_KL_mean": -3.4978790283203125, + "KL/std": 1.7607862949371338, + "epoch": 0.20559334845049132, + "fcm_dpo/beta": 0.4447017312049866, + "fcm_dpo/delta": -0.07520163059234619, + "fcm_dpo/margin": 1.0604674816131592, + "fcm_dpo/q_t": 0.4090201258659363, + "grad_norm": 136.33038330078125, + "learning_rate": 4.840055783904106e-07, + "logits/chosen": 0.1400383710861206, + "logits/rejected": 0.0758211761713028, + "logps/chosen": -69.19667053222656, + "logps/ref_chosen": -66.75926208496094, + "logps/ref_rejected": -94.61787414550781, + "logps/rejected": -98.11575317382812, + "loss": 1.1821, + "margin_dpo/margin_mean": 1.0604681968688965, + "margin_dpo/margin_std": 2.285677909851074, + "step": 136 + }, + { + "KL/chosen_KL_mean": -2.2930164337158203, + "KL/mean": -2.846302032470703, + "KL/rejected_KL_mean": -3.3995895385742188, + "KL/std": 1.5753281116485596, + "epoch": 0.20710506424792138, + "fcm_dpo/beta": 0.4380854368209839, + "fcm_dpo/delta": -0.08900899440050125, + "fcm_dpo/margin": 1.1065716743469238, + "fcm_dpo/q_t": 0.39505431056022644, + "grad_norm": 101.92052459716797, + "learning_rate": 4.835369650662767e-07, + "logits/chosen": 0.17767782509326935, + "logits/rejected": 0.15075279772281647, + "logps/chosen": -59.076812744140625, + "logps/ref_chosen": -56.78379821777344, + "logps/ref_rejected": -69.89952087402344, + "logps/rejected": -73.29911041259766, + "loss": 1.098, + "margin_dpo/margin_mean": 1.106572151184082, + "margin_dpo/margin_std": 1.8102577924728394, + "step": 137 + }, + { + "KL/chosen_KL_mean": -2.701139450073242, + "KL/mean": -3.1472229957580566, + "KL/rejected_KL_mean": -3.5933074951171875, + "KL/std": 1.609253168106079, + "epoch": 0.20861678004535147, + "fcm_dpo/beta": 0.4359511137008667, + "fcm_dpo/delta": 0.01150511484593153, + "fcm_dpo/margin": 0.8921665549278259, + "fcm_dpo/q_t": 0.4128834009170532, + "grad_norm": 108.2798080444336, + "learning_rate": 4.830618192112065e-07, + "logits/chosen": 0.16457317769527435, + "logits/rejected": 0.13061930239200592, + "logps/chosen": -61.4671516418457, + "logps/ref_chosen": -58.766014099121094, + "logps/ref_rejected": -68.12371826171875, + "logps/rejected": -71.71702575683594, + "loss": 1.1536, + "margin_dpo/margin_mean": 0.8921666145324707, + "margin_dpo/margin_std": 1.644634485244751, + "step": 138 + }, + { + "KL/chosen_KL_mean": -2.6400203704833984, + "KL/mean": -3.2433128356933594, + "KL/rejected_KL_mean": -3.846607208251953, + "KL/std": 1.5791709423065186, + "epoch": 0.21012849584278157, + "fcm_dpo/beta": 0.4313918948173523, + "fcm_dpo/delta": -0.1272524744272232, + "fcm_dpo/margin": 1.2065834999084473, + "fcm_dpo/q_t": 0.38509002327919006, + "grad_norm": 110.69141387939453, + "learning_rate": 4.825801541160509e-07, + "logits/chosen": 0.1212557703256607, + "logits/rejected": 0.09481331706047058, + "logps/chosen": -73.86560821533203, + "logps/ref_chosen": -71.2255859375, + "logps/ref_rejected": -82.1834716796875, + "logps/rejected": -86.03007507324219, + "loss": 1.0402, + "margin_dpo/margin_mean": 1.2065834999084473, + "margin_dpo/margin_std": 1.5707457065582275, + "step": 139 + }, + { + "KL/chosen_KL_mean": -2.432710647583008, + "KL/mean": -3.1976771354675293, + "KL/rejected_KL_mean": -3.9626426696777344, + "KL/std": 2.112020969390869, + "epoch": 0.21164021164021163, + "fcm_dpo/beta": 0.41016870737075806, + "fcm_dpo/delta": -0.2429695725440979, + "fcm_dpo/margin": 1.5299389362335205, + "fcm_dpo/q_t": 0.3645162582397461, + "grad_norm": 113.63577270507812, + "learning_rate": 4.820919832540181e-07, + "logits/chosen": 0.1273362636566162, + "logits/rejected": 0.08577500283718109, + "logps/chosen": -65.71037292480469, + "logps/ref_chosen": -63.27766418457031, + "logps/ref_rejected": -83.30647277832031, + "logps/rejected": -87.26911926269531, + "loss": 1.0491, + "margin_dpo/margin_mean": 1.5299385786056519, + "margin_dpo/margin_std": 2.3173608779907227, + "step": 140 + }, + { + "KL/chosen_KL_mean": -2.585247039794922, + "KL/mean": -3.257610321044922, + "KL/rejected_KL_mean": -3.929973602294922, + "KL/std": 1.727674961090088, + "epoch": 0.21315192743764172, + "fcm_dpo/beta": 0.3947093188762665, + "fcm_dpo/delta": -0.1396196335554123, + "fcm_dpo/margin": 1.344720482826233, + "fcm_dpo/q_t": 0.3882700204849243, + "grad_norm": 102.85530853271484, + "learning_rate": 4.815973202802966e-07, + "logits/chosen": 0.15521973371505737, + "logits/rejected": 0.11595935374498367, + "logps/chosen": -64.35202026367188, + "logps/ref_chosen": -61.76676940917969, + "logps/ref_rejected": -88.60601806640625, + "logps/rejected": -92.53599548339844, + "loss": 1.0768, + "margin_dpo/margin_mean": 1.3447209596633911, + "margin_dpo/margin_std": 2.0976152420043945, + "step": 141 + }, + { + "KL/chosen_KL_mean": -2.634784698486328, + "KL/mean": -3.140658378601074, + "KL/rejected_KL_mean": -3.6465377807617188, + "KL/std": 1.6917712688446045, + "epoch": 0.2146636432350718, + "fcm_dpo/beta": 0.39509785175323486, + "fcm_dpo/delta": 0.0001626778393983841, + "fcm_dpo/margin": 1.0117509365081787, + "fcm_dpo/q_t": 0.4111095666885376, + "grad_norm": 99.67555236816406, + "learning_rate": 4.810961790316729e-07, + "logits/chosen": 0.15274107456207275, + "logits/rejected": 0.12943851947784424, + "logps/chosen": -67.90956115722656, + "logps/ref_chosen": -65.2747802734375, + "logps/ref_rejected": -81.1378173828125, + "logps/rejected": -84.78435516357422, + "loss": 1.1272, + "margin_dpo/margin_mean": 1.01175057888031, + "margin_dpo/margin_std": 1.6819437742233276, + "step": 142 + }, + { + "KL/chosen_KL_mean": -2.6737728118896484, + "KL/mean": -3.2099857330322266, + "KL/rejected_KL_mean": -3.7461929321289062, + "KL/std": 1.7688965797424316, + "epoch": 0.2161753590325019, + "fcm_dpo/beta": 0.391811341047287, + "fcm_dpo/delta": -0.02130448818206787, + "fcm_dpo/margin": 1.0724186897277832, + "fcm_dpo/q_t": 0.40727514028549194, + "grad_norm": 119.30868530273438, + "learning_rate": 4.805885735261454e-07, + "logits/chosen": 0.18711808323860168, + "logits/rejected": 0.17115537822246552, + "logps/chosen": -65.29159545898438, + "logps/ref_chosen": -62.617828369140625, + "logps/ref_rejected": -70.39239501953125, + "logps/rejected": -74.13858032226562, + "loss": 1.1568, + "margin_dpo/margin_mean": 1.0724186897277832, + "margin_dpo/margin_std": 2.059246063232422, + "step": 143 + }, + { + "KL/chosen_KL_mean": -2.8863258361816406, + "KL/mean": -3.4791462421417236, + "KL/rejected_KL_mean": -4.071956634521484, + "KL/std": 1.829078197479248, + "epoch": 0.21768707482993196, + "fcm_dpo/beta": 0.3889148235321045, + "fcm_dpo/delta": -0.06404763460159302, + "fcm_dpo/margin": 1.1856343746185303, + "fcm_dpo/q_t": 0.4013003408908844, + "grad_norm": 105.5179443359375, + "learning_rate": 4.800745179625307e-07, + "logits/chosen": 0.1439233273267746, + "logits/rejected": 0.11750271171331406, + "logps/chosen": -63.68901443481445, + "logps/ref_chosen": -60.80268859863281, + "logps/ref_rejected": -79.07284545898438, + "logps/rejected": -83.1447982788086, + "loss": 1.1181, + "margin_dpo/margin_mean": 1.1856343746185303, + "margin_dpo/margin_std": 2.050663471221924, + "step": 144 + }, + { + "KL/chosen_KL_mean": -2.7375221252441406, + "KL/mean": -3.34666109085083, + "KL/rejected_KL_mean": -3.9557952880859375, + "KL/std": 1.9480266571044922, + "epoch": 0.21919879062736206, + "fcm_dpo/beta": 0.3836725950241089, + "fcm_dpo/delta": -0.07069326192140579, + "fcm_dpo/margin": 1.2182751893997192, + "fcm_dpo/q_t": 0.4001026749610901, + "grad_norm": 115.86017608642578, + "learning_rate": 4.795540267200686e-07, + "logits/chosen": 0.11822667717933655, + "logits/rejected": 0.13500581681728363, + "logps/chosen": -77.34898376464844, + "logps/ref_chosen": -74.61146545410156, + "logps/ref_rejected": -83.24461364746094, + "logps/rejected": -87.20040893554688, + "loss": 1.1347, + "margin_dpo/margin_mean": 1.2182748317718506, + "margin_dpo/margin_std": 2.2440905570983887, + "step": 145 + }, + { + "KL/chosen_KL_mean": -2.5694961547851562, + "KL/mean": -3.215273857116699, + "KL/rejected_KL_mean": -3.8610496520996094, + "KL/std": 1.8966618776321411, + "epoch": 0.22071050642479215, + "fcm_dpo/beta": 0.3777027726173401, + "fcm_dpo/delta": -0.09222418069839478, + "fcm_dpo/margin": 1.2915546894073486, + "fcm_dpo/q_t": 0.39369216561317444, + "grad_norm": 95.42322540283203, + "learning_rate": 4.790271143580173e-07, + "logits/chosen": 0.12121773511171341, + "logits/rejected": 0.1055947095155716, + "logps/chosen": -60.41048049926758, + "logps/ref_chosen": -57.84098434448242, + "logps/ref_rejected": -67.47422790527344, + "logps/rejected": -71.33528137207031, + "loss": 1.0812, + "margin_dpo/margin_mean": 1.2915539741516113, + "margin_dpo/margin_std": 1.969018578529358, + "step": 146 + }, + { + "KL/chosen_KL_mean": -3.021350860595703, + "KL/mean": -3.513603687286377, + "KL/rejected_KL_mean": -4.005867004394531, + "KL/std": 1.8846187591552734, + "epoch": 0.2222222222222222, + "fcm_dpo/beta": 0.3774845004081726, + "fcm_dpo/delta": 0.029318859800696373, + "fcm_dpo/margin": 0.9845160245895386, + "fcm_dpo/q_t": 0.417360782623291, + "grad_norm": 120.66484069824219, + "learning_rate": 4.784937956152489e-07, + "logits/chosen": 0.12826010584831238, + "logits/rejected": 0.08931194245815277, + "logps/chosen": -69.83482360839844, + "logps/ref_chosen": -66.81346893310547, + "logps/ref_rejected": -81.1796875, + "logps/rejected": -85.18556213378906, + "loss": 1.1863, + "margin_dpo/margin_mean": 0.9845158457756042, + "margin_dpo/margin_std": 2.072244644165039, + "step": 147 + }, + { + "KL/chosen_KL_mean": -2.9220848083496094, + "KL/mean": -3.633432388305664, + "KL/rejected_KL_mean": -4.344776153564453, + "KL/std": 1.9022397994995117, + "epoch": 0.2237339380196523, + "fcm_dpo/beta": 0.3716948628425598, + "fcm_dpo/delta": -0.135844424366951, + "fcm_dpo/margin": 1.4226953983306885, + "fcm_dpo/q_t": 0.38612425327301025, + "grad_norm": 80.1585693359375, + "learning_rate": 4.779540854098347e-07, + "logits/chosen": 0.2026696801185608, + "logits/rejected": 0.13863450288772583, + "logps/chosen": -51.609840393066406, + "logps/ref_chosen": -48.6877555847168, + "logps/ref_rejected": -67.50503540039062, + "logps/rejected": -71.84980773925781, + "loss": 1.0722, + "margin_dpo/margin_mean": 1.4226953983306885, + "margin_dpo/margin_std": 2.160921096801758, + "step": 148 + }, + { + "KL/chosen_KL_mean": -3.5425682067871094, + "KL/mean": -4.333852767944336, + "KL/rejected_KL_mean": -5.1251373291015625, + "KL/std": 2.3124797344207764, + "epoch": 0.2252456538170824, + "fcm_dpo/beta": 0.3573852777481079, + "fcm_dpo/delta": -0.1757103055715561, + "fcm_dpo/margin": 1.5825666189193726, + "fcm_dpo/q_t": 0.3796079754829407, + "grad_norm": 85.42340850830078, + "learning_rate": 4.774079988386296e-07, + "logits/chosen": 0.11232784390449524, + "logits/rejected": 0.06763314455747604, + "logps/chosen": -58.68634796142578, + "logps/ref_chosen": -55.143775939941406, + "logps/ref_rejected": -64.79888916015625, + "logps/rejected": -69.92402648925781, + "loss": 1.0345, + "margin_dpo/margin_mean": 1.582566261291504, + "margin_dpo/margin_std": 2.227118968963623, + "step": 149 + }, + { + "KL/chosen_KL_mean": -2.5940914154052734, + "KL/mean": -3.5496535301208496, + "KL/rejected_KL_mean": -4.505218505859375, + "KL/std": 2.0950212478637695, + "epoch": 0.22675736961451248, + "fcm_dpo/beta": 0.3405567407608032, + "fcm_dpo/delta": -0.26843270659446716, + "fcm_dpo/margin": 1.9111292362213135, + "fcm_dpo/q_t": 0.3549882173538208, + "grad_norm": 78.97344207763672, + "learning_rate": 4.768555511768486e-07, + "logits/chosen": 0.14000877737998962, + "logits/rejected": 0.10108865052461624, + "logps/chosen": -70.06483459472656, + "logps/ref_chosen": -67.47074890136719, + "logps/ref_rejected": -89.21170806884766, + "logps/rejected": -93.71692657470703, + "loss": 0.9473, + "margin_dpo/margin_mean": 1.911129355430603, + "margin_dpo/margin_std": 2.0432076454162598, + "step": 150 + }, + { + "KL/chosen_KL_mean": -2.626737594604492, + "KL/mean": -3.621738910675049, + "KL/rejected_KL_mean": -4.616741180419922, + "KL/std": 2.104227066040039, + "epoch": 0.22826908541194255, + "fcm_dpo/beta": 0.3234705328941345, + "fcm_dpo/delta": -0.26033222675323486, + "fcm_dpo/margin": 1.9900016784667969, + "fcm_dpo/q_t": 0.357607901096344, + "grad_norm": 69.46760559082031, + "learning_rate": 4.762967578776406e-07, + "logits/chosen": 0.15124168992042542, + "logits/rejected": 0.10280808806419373, + "logps/chosen": -55.086280822753906, + "logps/ref_chosen": -52.45954132080078, + "logps/ref_rejected": -79.0630111694336, + "logps/rejected": -83.67974853515625, + "loss": 0.9495, + "margin_dpo/margin_mean": 1.9900016784667969, + "margin_dpo/margin_std": 2.164335012435913, + "step": 151 + }, + { + "KL/chosen_KL_mean": -3.3076038360595703, + "KL/mean": -4.1723432540893555, + "KL/rejected_KL_mean": -5.037086486816406, + "KL/std": 2.2166404724121094, + "epoch": 0.22978080120937264, + "fcm_dpo/beta": 0.31334036588668823, + "fcm_dpo/delta": -0.14978964626789093, + "fcm_dpo/margin": 1.729482650756836, + "fcm_dpo/q_t": 0.3832073211669922, + "grad_norm": 76.83300018310547, + "learning_rate": 4.757316345716553e-07, + "logits/chosen": 0.22163252532482147, + "logits/rejected": 0.1748337298631668, + "logps/chosen": -59.86143493652344, + "logps/ref_chosen": -56.5538330078125, + "logps/ref_rejected": -76.55074310302734, + "logps/rejected": -81.58782958984375, + "loss": 1.0511, + "margin_dpo/margin_mean": 1.729482650756836, + "margin_dpo/margin_std": 2.4772982597351074, + "step": 152 + }, + { + "KL/chosen_KL_mean": -3.1301536560058594, + "KL/mean": -3.992166042327881, + "KL/rejected_KL_mean": -4.854175567626953, + "KL/std": 2.3730661869049072, + "epoch": 0.23129251700680273, + "fcm_dpo/beta": 0.30294376611709595, + "fcm_dpo/delta": -0.12941709160804749, + "fcm_dpo/margin": 1.7240203619003296, + "fcm_dpo/q_t": 0.3813447952270508, + "grad_norm": 70.95280456542969, + "learning_rate": 4.751601970666064e-07, + "logits/chosen": 0.12408562004566193, + "logits/rejected": 0.08880177140235901, + "logps/chosen": -71.13704681396484, + "logps/ref_chosen": -68.00689697265625, + "logps/ref_rejected": -74.83482360839844, + "logps/rejected": -79.68899536132812, + "loss": 1.0188, + "margin_dpo/margin_mean": 1.7240206003189087, + "margin_dpo/margin_std": 2.048651933670044, + "step": 153 + }, + { + "KL/chosen_KL_mean": -3.7474632263183594, + "KL/mean": -4.369100570678711, + "KL/rejected_KL_mean": -4.990741729736328, + "KL/std": 2.367417812347412, + "epoch": 0.2328042328042328, + "fcm_dpo/beta": 0.30299603939056396, + "fcm_dpo/delta": 0.0241601150482893, + "fcm_dpo/margin": 1.2432812452316284, + "fcm_dpo/q_t": 0.4153980016708374, + "grad_norm": 75.19510650634766, + "learning_rate": 4.745824613468292e-07, + "logits/chosen": 0.20469820499420166, + "logits/rejected": 0.20125876367092133, + "logps/chosen": -62.970001220703125, + "logps/ref_chosen": -59.222537994384766, + "logps/ref_rejected": -64.19131469726562, + "logps/rejected": -69.18206024169922, + "loss": 1.181, + "margin_dpo/margin_mean": 1.243281364440918, + "margin_dpo/margin_std": 2.5532095432281494, + "step": 154 + }, + { + "KL/chosen_KL_mean": -3.6322250366210938, + "KL/mean": -4.443280220031738, + "KL/rejected_KL_mean": -5.25433349609375, + "KL/std": 2.3358683586120605, + "epoch": 0.23431594860166288, + "fcm_dpo/beta": 0.29752117395401, + "fcm_dpo/delta": -0.08788089454174042, + "fcm_dpo/margin": 1.6221065521240234, + "fcm_dpo/q_t": 0.39678919315338135, + "grad_norm": 78.17323303222656, + "learning_rate": 4.7399844357283393e-07, + "logits/chosen": 0.18790429830551147, + "logits/rejected": 0.16967862844467163, + "logps/chosen": -72.08692169189453, + "logps/ref_chosen": -68.45469665527344, + "logps/ref_rejected": -77.91763305664062, + "logps/rejected": -83.17196655273438, + "loss": 1.1194, + "margin_dpo/margin_mean": 1.6221063137054443, + "margin_dpo/margin_std": 2.804651975631714, + "step": 155 + }, + { + "KL/chosen_KL_mean": -3.6310901641845703, + "KL/mean": -4.6390886306762695, + "KL/rejected_KL_mean": -5.647087097167969, + "KL/std": 2.6043336391448975, + "epoch": 0.23582766439909297, + "fcm_dpo/beta": 0.2896882891654968, + "fcm_dpo/delta": -0.19520190358161926, + "fcm_dpo/margin": 2.0159971714019775, + "fcm_dpo/q_t": 0.37270045280456543, + "grad_norm": 76.12146759033203, + "learning_rate": 4.7340816008085305e-07, + "logits/chosen": 0.14739282429218292, + "logits/rejected": 0.10633807629346848, + "logps/chosen": -70.90068817138672, + "logps/ref_chosen": -67.26959991455078, + "logps/ref_rejected": -86.95914459228516, + "logps/rejected": -92.60623168945312, + "loss": 0.9996, + "margin_dpo/margin_mean": 2.0159971714019775, + "margin_dpo/margin_std": 2.4939284324645996, + "step": 156 + }, + { + "KL/chosen_KL_mean": -3.4813175201416016, + "KL/mean": -4.297900676727295, + "KL/rejected_KL_mean": -5.114484786987305, + "KL/std": 2.685899257659912, + "epoch": 0.23733938019652306, + "fcm_dpo/beta": 0.2811231017112732, + "fcm_dpo/delta": -0.06421680748462677, + "fcm_dpo/margin": 1.6331628561019897, + "fcm_dpo/q_t": 0.39780938625335693, + "grad_norm": 66.9979019165039, + "learning_rate": 4.728116273823847e-07, + "logits/chosen": 0.1526588797569275, + "logits/rejected": 0.13304036855697632, + "logps/chosen": -58.25419235229492, + "logps/ref_chosen": -54.77287292480469, + "logps/ref_rejected": -63.87866973876953, + "logps/rejected": -68.99315643310547, + "loss": 1.0859, + "margin_dpo/margin_mean": 1.6331627368927002, + "margin_dpo/margin_std": 2.385460376739502, + "step": 157 + }, + { + "KL/chosen_KL_mean": -3.7610626220703125, + "KL/mean": -4.613132953643799, + "KL/rejected_KL_mean": -5.465198516845703, + "KL/std": 2.5535340309143066, + "epoch": 0.23885109599395313, + "fcm_dpo/beta": 0.2791249454021454, + "fcm_dpo/delta": -0.07955023646354675, + "fcm_dpo/margin": 1.7041363716125488, + "fcm_dpo/q_t": 0.394135445356369, + "grad_norm": 71.29205322265625, + "learning_rate": 4.7220886216373085e-07, + "logits/chosen": 0.17621684074401855, + "logits/rejected": 0.14411872625350952, + "logps/chosen": -68.68377685546875, + "logps/ref_chosen": -64.92271423339844, + "logps/ref_rejected": -82.23789978027344, + "logps/rejected": -87.70309448242188, + "loss": 1.0743, + "margin_dpo/margin_mean": 1.7041361331939697, + "margin_dpo/margin_std": 2.4774935245513916, + "step": 158 + }, + { + "KL/chosen_KL_mean": -3.9513187408447266, + "KL/mean": -4.878116607666016, + "KL/rejected_KL_mean": -5.804912567138672, + "KL/std": 2.7326745986938477, + "epoch": 0.24036281179138322, + "fcm_dpo/beta": 0.2755487263202667, + "fcm_dpo/delta": -0.11698535084724426, + "fcm_dpo/margin": 1.8535940647125244, + "fcm_dpo/q_t": 0.3866373896598816, + "grad_norm": 79.30245208740234, + "learning_rate": 4.715998812855304e-07, + "logits/chosen": 0.18606290221214294, + "logits/rejected": 0.1528480499982834, + "logps/chosen": -60.998313903808594, + "logps/ref_chosen": -57.046993255615234, + "logps/ref_rejected": -73.32441711425781, + "logps/rejected": -79.12933349609375, + "loss": 1.0857, + "margin_dpo/margin_mean": 1.853594183921814, + "margin_dpo/margin_std": 2.899454116821289, + "step": 159 + }, + { + "KL/chosen_KL_mean": -4.420055389404297, + "KL/mean": -5.301321983337402, + "KL/rejected_KL_mean": -6.182594299316406, + "KL/std": 2.5777411460876465, + "epoch": 0.2418745275888133, + "fcm_dpo/beta": 0.26921501755714417, + "fcm_dpo/delta": -0.07813730090856552, + "fcm_dpo/margin": 1.7625341415405273, + "fcm_dpo/q_t": 0.399819016456604, + "grad_norm": 63.23322677612305, + "learning_rate": 4.7098470178228755e-07, + "logits/chosen": 0.06095326691865921, + "logits/rejected": 0.0238176379352808, + "logps/chosen": -54.226966857910156, + "logps/ref_chosen": -49.806915283203125, + "logps/ref_rejected": -68.3370132446289, + "logps/rejected": -74.51960754394531, + "loss": 1.1081, + "margin_dpo/margin_mean": 1.7625336647033691, + "margin_dpo/margin_std": 2.9826478958129883, + "step": 160 + }, + { + "KL/chosen_KL_mean": -4.423643112182617, + "KL/mean": -5.383838653564453, + "KL/rejected_KL_mean": -6.344034194946289, + "KL/std": 2.7854318618774414, + "epoch": 0.24338624338624337, + "fcm_dpo/beta": 0.2639704644680023, + "fcm_dpo/delta": -0.11248427629470825, + "fcm_dpo/margin": 1.9203957319259644, + "fcm_dpo/q_t": 0.39128196239471436, + "grad_norm": 61.89495086669922, + "learning_rate": 4.703633408618955e-07, + "logits/chosen": 0.165007084608078, + "logits/rejected": 0.13076232373714447, + "logps/chosen": -56.92413330078125, + "logps/ref_chosen": -52.50048828125, + "logps/ref_rejected": -66.04540252685547, + "logps/rejected": -72.38943481445312, + "loss": 1.0779, + "margin_dpo/margin_mean": 1.920395851135254, + "margin_dpo/margin_std": 2.972217559814453, + "step": 161 + }, + { + "KL/chosen_KL_mean": -4.803016662597656, + "KL/mean": -6.140519142150879, + "KL/rejected_KL_mean": -7.47802734375, + "KL/std": 3.0558063983917236, + "epoch": 0.24489795918367346, + "fcm_dpo/beta": 0.24996890127658844, + "fcm_dpo/delta": -0.2889677882194519, + "fcm_dpo/margin": 2.675013542175293, + "fcm_dpo/q_t": 0.3522527813911438, + "grad_norm": 60.043983459472656, + "learning_rate": 4.697358159051549e-07, + "logits/chosen": 0.2536744773387909, + "logits/rejected": 0.2079133689403534, + "logps/chosen": -74.27220916748047, + "logps/ref_chosen": -69.46919250488281, + "logps/ref_rejected": -92.00952911376953, + "logps/rejected": -99.48755645751953, + "loss": 0.9402, + "margin_dpo/margin_mean": 2.6750144958496094, + "margin_dpo/margin_std": 2.9115896224975586, + "step": 162 + }, + { + "KL/chosen_KL_mean": -4.403450012207031, + "KL/mean": -5.708391189575195, + "KL/rejected_KL_mean": -7.013332366943359, + "KL/std": 2.9234018325805664, + "epoch": 0.24640967498110355, + "fcm_dpo/beta": 0.24058547616004944, + "fcm_dpo/delta": -0.24273084104061127, + "fcm_dpo/margin": 2.6098759174346924, + "fcm_dpo/q_t": 0.3646067976951599, + "grad_norm": 57.50065994262695, + "learning_rate": 4.691021444652876e-07, + "logits/chosen": 0.18654045462608337, + "logits/rejected": 0.1429648995399475, + "logps/chosen": -55.01728439331055, + "logps/ref_chosen": -50.613834381103516, + "logps/ref_rejected": -74.62033081054688, + "logps/rejected": -81.63365936279297, + "loss": 1.0005, + "margin_dpo/margin_mean": 2.6098763942718506, + "margin_dpo/margin_std": 3.280284881591797, + "step": 163 + }, + { + "KL/chosen_KL_mean": -4.918027877807617, + "KL/mean": -6.209537029266357, + "KL/rejected_KL_mean": -7.501045227050781, + "KL/std": 3.221226215362549, + "epoch": 0.24792139077853365, + "fcm_dpo/beta": 0.22847194969654083, + "fcm_dpo/delta": -0.20196107029914856, + "fcm_dpo/margin": 2.583019256591797, + "fcm_dpo/q_t": 0.37327778339385986, + "grad_norm": 54.232357025146484, + "learning_rate": 4.6846234426744624e-07, + "logits/chosen": 0.17183548212051392, + "logits/rejected": 0.11392419040203094, + "logps/chosen": -59.76613998413086, + "logps/ref_chosen": -54.848114013671875, + "logps/ref_rejected": -79.0630111694336, + "logps/rejected": -86.56405639648438, + "loss": 1.0215, + "margin_dpo/margin_mean": 2.583019495010376, + "margin_dpo/margin_std": 3.419358015060425, + "step": 164 + }, + { + "KL/chosen_KL_mean": -5.296621322631836, + "KL/mean": -6.522599220275879, + "KL/rejected_KL_mean": -7.7485809326171875, + "KL/std": 3.3008265495300293, + "epoch": 0.2494331065759637, + "fcm_dpo/beta": 0.2219952940940857, + "fcm_dpo/delta": -0.152376189827919, + "fcm_dpo/margin": 2.451958417892456, + "fcm_dpo/q_t": 0.37785401940345764, + "grad_norm": 51.94645309448242, + "learning_rate": 4.678164332082175e-07, + "logits/chosen": 0.2255769670009613, + "logits/rejected": 0.17380759119987488, + "logps/chosen": -56.385833740234375, + "logps/ref_chosen": -51.089210510253906, + "logps/ref_rejected": -71.23370361328125, + "logps/rejected": -78.98228454589844, + "loss": 1.0168, + "margin_dpo/margin_mean": 2.451957941055298, + "margin_dpo/margin_std": 2.9465808868408203, + "step": 165 + }, + { + "KL/chosen_KL_mean": -5.235536575317383, + "KL/mean": -6.11887264251709, + "KL/rejected_KL_mean": -7.0022125244140625, + "KL/std": 2.914708375930786, + "epoch": 0.2509448223733938, + "fcm_dpo/beta": 0.21907421946525574, + "fcm_dpo/delta": 0.013396350666880608, + "fcm_dpo/margin": 1.7666780948638916, + "fcm_dpo/q_t": 0.4149834215641022, + "grad_norm": 61.127227783203125, + "learning_rate": 4.6716442935512214e-07, + "logits/chosen": 0.20381613075733185, + "logits/rejected": 0.12277201563119888, + "logps/chosen": -68.42634582519531, + "logps/ref_chosen": -63.19081115722656, + "logps/ref_rejected": -93.8402099609375, + "logps/rejected": -100.84242248535156, + "loss": 1.1348, + "margin_dpo/margin_mean": 1.7666782140731812, + "margin_dpo/margin_std": 2.991272449493408, + "step": 166 + }, + { + "KL/chosen_KL_mean": -4.86152458190918, + "KL/mean": -6.267144203186035, + "KL/rejected_KL_mean": -7.672763824462891, + "KL/std": 3.211589813232422, + "epoch": 0.25245653817082386, + "fcm_dpo/beta": 0.2118726372718811, + "fcm_dpo/delta": -0.20976994931697845, + "fcm_dpo/margin": 2.81123685836792, + "fcm_dpo/q_t": 0.36726197600364685, + "grad_norm": 46.722129821777344, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": 0.15115973353385925, + "logits/rejected": 0.11766296625137329, + "logps/chosen": -63.785797119140625, + "logps/ref_chosen": -58.92427062988281, + "logps/ref_rejected": -72.97377014160156, + "logps/rejected": -80.64653015136719, + "loss": 0.9841, + "margin_dpo/margin_mean": 2.811237335205078, + "margin_dpo/margin_std": 3.2436766624450684, + "step": 167 + }, + { + "KL/chosen_KL_mean": -6.010761260986328, + "KL/mean": -6.915679931640625, + "KL/rejected_KL_mean": -7.8206024169921875, + "KL/std": 3.365790367126465, + "epoch": 0.25396825396825395, + "fcm_dpo/beta": 0.21115413308143616, + "fcm_dpo/delta": 0.018556442111730576, + "fcm_dpo/margin": 1.809836983680725, + "fcm_dpo/q_t": 0.4141741096973419, + "grad_norm": 59.15978240966797, + "learning_rate": 4.6584221638904767e-07, + "logits/chosen": 0.16713829338550568, + "logits/rejected": 0.13781222701072693, + "logps/chosen": -71.66213989257812, + "logps/ref_chosen": -65.65138244628906, + "logps/ref_rejected": -79.71418762207031, + "logps/rejected": -87.5347900390625, + "loss": 1.1305, + "margin_dpo/margin_mean": 1.8098368644714355, + "margin_dpo/margin_std": 2.919257640838623, + "step": 168 + }, + { + "KL/chosen_KL_mean": -5.380914688110352, + "KL/mean": -6.703707695007324, + "KL/rejected_KL_mean": -8.026496887207031, + "KL/std": 3.6164283752441406, + "epoch": 0.25547996976568405, + "fcm_dpo/beta": 0.20682457089424133, + "fcm_dpo/delta": -0.15560072660446167, + "fcm_dpo/margin": 2.645585060119629, + "fcm_dpo/q_t": 0.3858036398887634, + "grad_norm": 53.96177291870117, + "learning_rate": 4.651720442612075e-07, + "logits/chosen": 0.23225507140159607, + "logits/rejected": 0.20093366503715515, + "logps/chosen": -66.80677795410156, + "logps/ref_chosen": -61.425865173339844, + "logps/ref_rejected": -76.09590148925781, + "logps/rejected": -84.12239074707031, + "loss": 1.051, + "margin_dpo/margin_mean": 2.645585060119629, + "margin_dpo/margin_std": 3.978856086730957, + "step": 169 + }, + { + "KL/chosen_KL_mean": -5.6312408447265625, + "KL/mean": -6.840090751647949, + "KL/rejected_KL_mean": -8.04893684387207, + "KL/std": 3.505796432495117, + "epoch": 0.25699168556311414, + "fcm_dpo/beta": 0.2026190310716629, + "fcm_dpo/delta": -0.09439290314912796, + "fcm_dpo/margin": 2.417698383331299, + "fcm_dpo/q_t": 0.39143288135528564, + "grad_norm": 46.08080291748047, + "learning_rate": 4.6449585330874425e-07, + "logits/chosen": 0.16252544522285461, + "logits/rejected": 0.16037079691886902, + "logps/chosen": -62.28443145751953, + "logps/ref_chosen": -56.65319061279297, + "logps/ref_rejected": -63.45965576171875, + "logps/rejected": -71.50859069824219, + "loss": 1.0905, + "margin_dpo/margin_mean": 2.4176979064941406, + "margin_dpo/margin_std": 3.8021955490112305, + "step": 170 + }, + { + "KL/chosen_KL_mean": -5.986204147338867, + "KL/mean": -7.4430012702941895, + "KL/rejected_KL_mean": -8.899799346923828, + "KL/std": 4.016740798950195, + "epoch": 0.2585034013605442, + "fcm_dpo/beta": 0.19423140585422516, + "fcm_dpo/delta": -0.17897388339042664, + "fcm_dpo/margin": 2.91359806060791, + "fcm_dpo/q_t": 0.3770707845687866, + "grad_norm": 51.48481369018555, + "learning_rate": 4.6381366244617224e-07, + "logits/chosen": 0.23160839080810547, + "logits/rejected": 0.18538227677345276, + "logps/chosen": -69.72096252441406, + "logps/ref_chosen": -63.73476028442383, + "logps/ref_rejected": -78.50328063964844, + "logps/rejected": -87.403076171875, + "loss": 1.0544, + "margin_dpo/margin_mean": 2.9135982990264893, + "margin_dpo/margin_std": 4.2580180168151855, + "step": 171 + }, + { + "KL/chosen_KL_mean": -6.52869987487793, + "KL/mean": -8.056679725646973, + "KL/rejected_KL_mean": -9.58465576171875, + "KL/std": 4.038125038146973, + "epoch": 0.2600151171579743, + "fcm_dpo/beta": 0.1900688111782074, + "fcm_dpo/delta": -0.19167888164520264, + "fcm_dpo/margin": 3.0559561252593994, + "fcm_dpo/q_t": 0.37122875452041626, + "grad_norm": 44.15302276611328, + "learning_rate": 4.631254907558365e-07, + "logits/chosen": 0.23724429309368134, + "logits/rejected": 0.186652272939682, + "logps/chosen": -58.73046112060547, + "logps/ref_chosen": -52.201759338378906, + "logps/ref_rejected": -82.85285949707031, + "logps/rejected": -92.43751525878906, + "loss": 1.0195, + "margin_dpo/margin_mean": 3.055955410003662, + "margin_dpo/margin_std": 3.9646401405334473, + "step": 172 + }, + { + "KL/chosen_KL_mean": -6.710458755493164, + "KL/mean": -8.274261474609375, + "KL/rejected_KL_mean": -9.838062286376953, + "KL/std": 4.230109214782715, + "epoch": 0.2615268329554044, + "fcm_dpo/beta": 0.17975875735282898, + "fcm_dpo/delta": -0.17714735865592957, + "fcm_dpo/margin": 3.1276025772094727, + "fcm_dpo/q_t": 0.38370195031166077, + "grad_norm": 43.27824020385742, + "learning_rate": 4.624313574873786e-07, + "logits/chosen": 0.24271810054779053, + "logits/rejected": 0.1620805412530899, + "logps/chosen": -62.14518356323242, + "logps/ref_chosen": -55.434722900390625, + "logps/ref_rejected": -77.81967163085938, + "logps/rejected": -87.6577377319336, + "loss": 1.0919, + "margin_dpo/margin_mean": 3.1276025772094727, + "margin_dpo/margin_std": 5.053474426269531, + "step": 173 + }, + { + "KL/chosen_KL_mean": -7.342222213745117, + "KL/mean": -9.029733657836914, + "KL/rejected_KL_mean": -10.717247009277344, + "KL/std": 4.268304824829102, + "epoch": 0.26303854875283444, + "fcm_dpo/beta": 0.1746232807636261, + "fcm_dpo/delta": -0.20176327228546143, + "fcm_dpo/margin": 3.375027894973755, + "fcm_dpo/q_t": 0.3743577003479004, + "grad_norm": 46.385498046875, + "learning_rate": 4.61731282057198e-07, + "logits/chosen": 0.232137069106102, + "logits/rejected": 0.17018833756446838, + "logps/chosen": -64.51417541503906, + "logps/ref_chosen": -57.17195129394531, + "logps/ref_rejected": -85.47578430175781, + "logps/rejected": -96.19302368164062, + "loss": 1.0447, + "margin_dpo/margin_mean": 3.375028371810913, + "margin_dpo/margin_std": 4.97684383392334, + "step": 174 + }, + { + "KL/chosen_KL_mean": -7.433553695678711, + "KL/mean": -9.300992965698242, + "KL/rejected_KL_mean": -11.168437957763672, + "KL/std": 4.688220977783203, + "epoch": 0.26455026455026454, + "fcm_dpo/beta": 0.16730672121047974, + "fcm_dpo/delta": -0.23966068029403687, + "fcm_dpo/margin": 3.7348804473876953, + "fcm_dpo/q_t": 0.37021753191947937, + "grad_norm": 44.081607818603516, + "learning_rate": 4.6102528404790965e-07, + "logits/chosen": 0.2679125964641571, + "logits/rejected": 0.23914484679698944, + "logps/chosen": -75.09918212890625, + "logps/ref_chosen": -67.6656265258789, + "logps/ref_rejected": -84.36766815185547, + "logps/rejected": -95.53610229492188, + "loss": 1.0271, + "margin_dpo/margin_mean": 3.7348804473876953, + "margin_dpo/margin_std": 5.312939643859863, + "step": 175 + }, + { + "KL/chosen_KL_mean": -8.312698364257812, + "KL/mean": -9.597501754760742, + "KL/rejected_KL_mean": -10.882305145263672, + "KL/std": 4.97230339050293, + "epoch": 0.2660619803476946, + "fcm_dpo/beta": 0.16365355253219604, + "fcm_dpo/delta": -0.022150814533233643, + "fcm_dpo/margin": 2.569603681564331, + "fcm_dpo/q_t": 0.4120991826057434, + "grad_norm": 52.76869583129883, + "learning_rate": 4.603133832077953e-07, + "logits/chosen": 0.21717190742492676, + "logits/rejected": 0.19207710027694702, + "logps/chosen": -86.17146301269531, + "logps/ref_chosen": -77.8587646484375, + "logps/ref_rejected": -81.08732604980469, + "logps/rejected": -91.96963500976562, + "loss": 1.1776, + "margin_dpo/margin_mean": 2.569603681564331, + "margin_dpo/margin_std": 5.330869197845459, + "step": 176 + }, + { + "KL/chosen_KL_mean": -7.614345550537109, + "KL/mean": -10.125162124633789, + "KL/rejected_KL_mean": -12.635982513427734, + "KL/std": 5.118657112121582, + "epoch": 0.2675736961451247, + "fcm_dpo/beta": 0.15402725338935852, + "fcm_dpo/delta": -0.4072016775608063, + "fcm_dpo/margin": 5.021634101867676, + "fcm_dpo/q_t": 0.33033889532089233, + "grad_norm": 44.16107940673828, + "learning_rate": 4.5959559945025183e-07, + "logits/chosen": 0.31596648693084717, + "logits/rejected": 0.22701580822467804, + "logps/chosen": -62.83474349975586, + "logps/ref_chosen": -55.22039794921875, + "logps/ref_rejected": -92.54973602294922, + "logps/rejected": -105.18571472167969, + "loss": 0.8755, + "margin_dpo/margin_mean": 5.021634101867676, + "margin_dpo/margin_std": 4.907200813293457, + "step": 177 + }, + { + "KL/chosen_KL_mean": -8.229791641235352, + "KL/mean": -9.763175964355469, + "KL/rejected_KL_mean": -11.296558380126953, + "KL/std": 4.665809631347656, + "epoch": 0.2690854119425548, + "fcm_dpo/beta": 0.14871251583099365, + "fcm_dpo/delta": -0.06117831915616989, + "fcm_dpo/margin": 3.0667619705200195, + "fcm_dpo/q_t": 0.3978401720523834, + "grad_norm": 40.795989990234375, + "learning_rate": 4.588719528532341e-07, + "logits/chosen": 0.2093246877193451, + "logits/rejected": 0.1630682349205017, + "logps/chosen": -69.040283203125, + "logps/ref_chosen": -60.81049346923828, + "logps/ref_rejected": -81.12973022460938, + "logps/rejected": -92.42628479003906, + "loss": 1.0854, + "margin_dpo/margin_mean": 3.0667624473571777, + "margin_dpo/margin_std": 4.368191242218018, + "step": 178 + }, + { + "KL/chosen_KL_mean": -9.152389526367188, + "KL/mean": -10.899008750915527, + "KL/rejected_KL_mean": -12.645626068115234, + "KL/std": 4.8825554847717285, + "epoch": 0.2705971277399849, + "fcm_dpo/beta": 0.14718642830848694, + "fcm_dpo/delta": -0.1202310174703598, + "fcm_dpo/margin": 3.4932327270507812, + "fcm_dpo/q_t": 0.38942965865135193, + "grad_norm": 39.2275390625, + "learning_rate": 4.581424636586928e-07, + "logits/chosen": 0.25605344772338867, + "logits/rejected": 0.23996365070343018, + "logps/chosen": -74.82410430908203, + "logps/ref_chosen": -65.67171478271484, + "logps/ref_rejected": -75.32586669921875, + "logps/rejected": -87.97149658203125, + "loss": 1.0858, + "margin_dpo/margin_mean": 3.4932332038879395, + "margin_dpo/margin_std": 5.598725318908691, + "step": 179 + }, + { + "KL/chosen_KL_mean": -7.865631103515625, + "KL/mean": -9.393498420715332, + "KL/rejected_KL_mean": -10.921365737915039, + "KL/std": 5.0374755859375, + "epoch": 0.272108843537415, + "fcm_dpo/beta": 0.14556020498275757, + "fcm_dpo/delta": -0.04694035276770592, + "fcm_dpo/margin": 3.0557327270507812, + "fcm_dpo/q_t": 0.4066722095012665, + "grad_norm": 39.35282897949219, + "learning_rate": 4.5740715227200897e-07, + "logits/chosen": 0.10142640769481659, + "logits/rejected": 0.0828198567032814, + "logps/chosen": -64.5484390258789, + "logps/ref_chosen": -56.68280792236328, + "logps/ref_rejected": -64.94414520263672, + "logps/rejected": -75.86550903320312, + "loss": 1.1508, + "margin_dpo/margin_mean": 3.0557332038879395, + "margin_dpo/margin_std": 5.835036754608154, + "step": 180 + }, + { + "KL/chosen_KL_mean": -7.593027114868164, + "KL/mean": -10.005693435668945, + "KL/rejected_KL_mean": -12.418354034423828, + "KL/std": 5.6760430335998535, + "epoch": 0.273620559334845, + "fcm_dpo/beta": 0.13861241936683655, + "fcm_dpo/delta": -0.2883051633834839, + "fcm_dpo/margin": 4.825331687927246, + "fcm_dpo/q_t": 0.3515399098396301, + "grad_norm": 34.905521392822266, + "learning_rate": 4.566660392614228e-07, + "logits/chosen": 0.29494979977607727, + "logits/rejected": 0.25655397772789, + "logps/chosen": -68.36907958984375, + "logps/ref_chosen": -60.77604675292969, + "logps/ref_rejected": -83.98361206054688, + "logps/rejected": -96.40196990966797, + "loss": 0.9296, + "margin_dpo/margin_mean": 4.825331211090088, + "margin_dpo/margin_std": 5.00650691986084, + "step": 181 + }, + { + "KL/chosen_KL_mean": -8.961589813232422, + "KL/mean": -11.407659530639648, + "KL/rejected_KL_mean": -13.85372543334961, + "KL/std": 5.829415321350098, + "epoch": 0.2751322751322751, + "fcm_dpo/beta": 0.13101884722709656, + "fcm_dpo/delta": -0.25866827368736267, + "fcm_dpo/margin": 4.892138957977295, + "fcm_dpo/q_t": 0.36321139335632324, + "grad_norm": 35.85899353027344, + "learning_rate": 4.5591914535745817e-07, + "logits/chosen": 0.24604353308677673, + "logits/rejected": 0.1717510223388672, + "logps/chosen": -69.21537780761719, + "logps/ref_chosen": -60.2537841796875, + "logps/ref_rejected": -89.7706298828125, + "logps/rejected": -103.62435913085938, + "loss": 1.0077, + "margin_dpo/margin_mean": 4.892138957977295, + "margin_dpo/margin_std": 6.573358535766602, + "step": 182 + }, + { + "KL/chosen_KL_mean": -10.759580612182617, + "KL/mean": -11.767297744750977, + "KL/rejected_KL_mean": -12.775016784667969, + "KL/std": 5.797929763793945, + "epoch": 0.2766439909297052, + "fcm_dpo/beta": 0.13212950527668, + "fcm_dpo/delta": 0.137569397687912, + "fcm_dpo/margin": 2.015444278717041, + "fcm_dpo/q_t": 0.44033879041671753, + "grad_norm": 38.02180862426758, + "learning_rate": 4.551664914523433e-07, + "logits/chosen": 0.2160715013742447, + "logits/rejected": 0.19661223888397217, + "logps/chosen": -72.52099609375, + "logps/ref_chosen": -61.76142120361328, + "logps/ref_rejected": -72.54627990722656, + "logps/rejected": -85.32130432128906, + "loss": 1.2589, + "margin_dpo/margin_mean": 2.015444278717041, + "margin_dpo/margin_std": 5.434455394744873, + "step": 183 + }, + { + "KL/chosen_KL_mean": -8.564130783081055, + "KL/mean": -10.616769790649414, + "KL/rejected_KL_mean": -12.66940689086914, + "KL/std": 5.086275100708008, + "epoch": 0.2781557067271353, + "fcm_dpo/beta": 0.1294369101524353, + "fcm_dpo/delta": -0.1415943056344986, + "fcm_dpo/margin": 4.105273246765137, + "fcm_dpo/q_t": 0.3822307884693146, + "grad_norm": 29.23053550720215, + "learning_rate": 4.544080985994258e-07, + "logits/chosen": 0.3366903066635132, + "logits/rejected": 0.27632272243499756, + "logps/chosen": -55.40485382080078, + "logps/ref_chosen": -46.840721130371094, + "logps/ref_rejected": -69.3609390258789, + "logps/rejected": -82.03034973144531, + "loss": 1.0213, + "margin_dpo/margin_mean": 4.105273246765137, + "margin_dpo/margin_std": 4.934863090515137, + "step": 184 + }, + { + "KL/chosen_KL_mean": -9.769927978515625, + "KL/mean": -11.914457321166992, + "KL/rejected_KL_mean": -14.058982849121094, + "KL/std": 5.9276604652404785, + "epoch": 0.2796674225245654, + "fcm_dpo/beta": 0.1260417103767395, + "fcm_dpo/delta": -0.15020085871219635, + "fcm_dpo/margin": 4.289056777954102, + "fcm_dpo/q_t": 0.3870917856693268, + "grad_norm": 31.231266021728516, + "learning_rate": 4.5364398801258394e-07, + "logits/chosen": 0.25812608003616333, + "logits/rejected": 0.21318121254444122, + "logps/chosen": -62.091068267822266, + "logps/ref_chosen": -52.32114028930664, + "logps/ref_rejected": -68.3885726928711, + "logps/rejected": -82.44755554199219, + "loss": 1.1037, + "margin_dpo/margin_mean": 4.289056777954102, + "margin_dpo/margin_std": 7.260632514953613, + "step": 185 + }, + { + "KL/chosen_KL_mean": -9.387929916381836, + "KL/mean": -11.70936393737793, + "KL/rejected_KL_mean": -14.03079605102539, + "KL/std": 5.995885848999023, + "epoch": 0.2811791383219955, + "fcm_dpo/beta": 0.12234297394752502, + "fcm_dpo/delta": -0.1784307062625885, + "fcm_dpo/margin": 4.642869472503662, + "fcm_dpo/q_t": 0.3811233341693878, + "grad_norm": 37.083396911621094, + "learning_rate": 4.5287418106563354e-07, + "logits/chosen": 0.20448735356330872, + "logits/rejected": 0.16464738547801971, + "logps/chosen": -76.80805969238281, + "logps/ref_chosen": -67.42012786865234, + "logps/ref_rejected": -82.50968933105469, + "logps/rejected": -96.54048156738281, + "loss": 1.0687, + "margin_dpo/margin_mean": 4.64286994934082, + "margin_dpo/margin_std": 7.29622745513916, + "step": 186 + }, + { + "KL/chosen_KL_mean": -10.648483276367188, + "KL/mean": -12.794281005859375, + "KL/rejected_KL_mean": -14.940082550048828, + "KL/std": 6.210048675537109, + "epoch": 0.28269085411942557, + "fcm_dpo/beta": 0.11887460947036743, + "fcm_dpo/delta": -0.11676714569330215, + "fcm_dpo/margin": 4.291601657867432, + "fcm_dpo/q_t": 0.3875843286514282, + "grad_norm": 38.01297378540039, + "learning_rate": 4.520986992917297e-07, + "logits/chosen": 0.24502941966056824, + "logits/rejected": 0.1913175731897354, + "logps/chosen": -86.17398071289062, + "logps/ref_chosen": -75.52549743652344, + "logps/ref_rejected": -94.76289367675781, + "logps/rejected": -109.70297241210938, + "loss": 1.0866, + "margin_dpo/margin_mean": 4.29160213470459, + "margin_dpo/margin_std": 6.724270820617676, + "step": 187 + }, + { + "KL/chosen_KL_mean": -9.933109283447266, + "KL/mean": -12.137847900390625, + "KL/rejected_KL_mean": -14.342586517333984, + "KL/std": 6.472745418548584, + "epoch": 0.2842025699168556, + "fcm_dpo/beta": 0.11644455790519714, + "fcm_dpo/delta": -0.11966116726398468, + "fcm_dpo/margin": 4.409476280212402, + "fcm_dpo/q_t": 0.3881905674934387, + "grad_norm": 36.36927032470703, + "learning_rate": 4.5131756438276466e-07, + "logits/chosen": 0.2666782736778259, + "logits/rejected": 0.22476297616958618, + "logps/chosen": -81.45643615722656, + "logps/ref_chosen": -71.52333068847656, + "logps/ref_rejected": -78.29949951171875, + "logps/rejected": -92.64208984375, + "loss": 1.0861, + "margin_dpo/margin_mean": 4.409476280212402, + "margin_dpo/margin_std": 7.109898567199707, + "step": 188 + }, + { + "KL/chosen_KL_mean": -9.555837631225586, + "KL/mean": -11.801464080810547, + "KL/rejected_KL_mean": -14.04709243774414, + "KL/std": 6.3911943435668945, + "epoch": 0.2857142857142857, + "fcm_dpo/beta": 0.11252661049365997, + "fcm_dpo/delta": -0.11500401049852371, + "fcm_dpo/margin": 4.491254806518555, + "fcm_dpo/q_t": 0.3868621587753296, + "grad_norm": 33.63637161254883, + "learning_rate": 4.5053079818876096e-07, + "logits/chosen": 0.2828933596611023, + "logits/rejected": 0.2940501868724823, + "logps/chosen": -81.73210144042969, + "logps/ref_chosen": -72.17626953125, + "logps/ref_rejected": -75.26313781738281, + "logps/rejected": -89.31022644042969, + "loss": 1.074, + "margin_dpo/margin_mean": 4.491255760192871, + "margin_dpo/margin_std": 6.432233810424805, + "step": 189 + }, + { + "KL/chosen_KL_mean": -9.486690521240234, + "KL/mean": -12.508905410766602, + "KL/rejected_KL_mean": -15.531116485595703, + "KL/std": 6.638813495635986, + "epoch": 0.2872260015117158, + "fcm_dpo/beta": 0.10864052176475525, + "fcm_dpo/delta": -0.2747190594673157, + "fcm_dpo/margin": 6.044425010681152, + "fcm_dpo/q_t": 0.35702377557754517, + "grad_norm": 32.1728401184082, + "learning_rate": 4.4973842271726024e-07, + "logits/chosen": 0.32548272609710693, + "logits/rejected": 0.19106276333332062, + "logps/chosen": -64.1109619140625, + "logps/ref_chosen": -54.624271392822266, + "logps/ref_rejected": -101.47068786621094, + "logps/rejected": -117.00180053710938, + "loss": 0.9636, + "margin_dpo/margin_mean": 6.044425010681152, + "margin_dpo/margin_std": 7.100013256072998, + "step": 190 + }, + { + "KL/chosen_KL_mean": -11.378448486328125, + "KL/mean": -13.6748046875, + "KL/rejected_KL_mean": -15.97116470336914, + "KL/std": 6.738143444061279, + "epoch": 0.2887377173091459, + "fcm_dpo/beta": 0.10574862360954285, + "fcm_dpo/delta": -0.0899963304400444, + "fcm_dpo/margin": 4.592713832855225, + "fcm_dpo/q_t": 0.3922455310821533, + "grad_norm": 34.2210807800293, + "learning_rate": 4.48940460132708e-07, + "logits/chosen": 0.3366280794143677, + "logits/rejected": 0.3104870915412903, + "logps/chosen": -84.31095886230469, + "logps/ref_chosen": -72.93251037597656, + "logps/ref_rejected": -89.95103454589844, + "logps/rejected": -105.92219543457031, + "loss": 1.0981, + "margin_dpo/margin_mean": 4.592713356018066, + "margin_dpo/margin_std": 7.371159553527832, + "step": 191 + }, + { + "KL/chosen_KL_mean": -11.613393783569336, + "KL/mean": -13.261909484863281, + "KL/rejected_KL_mean": -14.910423278808594, + "KL/std": 7.089072227478027, + "epoch": 0.29024943310657597, + "fcm_dpo/beta": 0.1063033863902092, + "fcm_dpo/delta": 0.050992466509342194, + "fcm_dpo/margin": 3.297028064727783, + "fcm_dpo/q_t": 0.42166829109191895, + "grad_norm": 27.335500717163086, + "learning_rate": 4.481369327558329e-07, + "logits/chosen": 0.30276888608932495, + "logits/rejected": 0.2783222496509552, + "logps/chosen": -65.61451721191406, + "logps/ref_chosen": -54.001121520996094, + "logps/ref_rejected": -63.531551361083984, + "logps/rejected": -78.44197082519531, + "loss": 1.1783, + "margin_dpo/margin_mean": 3.297028064727783, + "margin_dpo/margin_std": 6.561872482299805, + "step": 192 + }, + { + "KL/chosen_KL_mean": -10.664052963256836, + "KL/mean": -13.527870178222656, + "KL/rejected_KL_mean": -16.39168930053711, + "KL/std": 6.945682525634766, + "epoch": 0.29176114890400606, + "fcm_dpo/beta": 0.10263273119926453, + "fcm_dpo/delta": -0.20108722150325775, + "fcm_dpo/margin": 5.727632522583008, + "fcm_dpo/q_t": 0.3708486557006836, + "grad_norm": 26.038829803466797, + "learning_rate": 4.47327863063023e-07, + "logits/chosen": 0.2456463873386383, + "logits/rejected": 0.22312812507152557, + "logps/chosen": -67.413330078125, + "logps/ref_chosen": -56.74927520751953, + "logps/ref_rejected": -58.80629348754883, + "logps/rejected": -75.19798278808594, + "loss": 0.9983, + "margin_dpo/margin_mean": 5.727632522583008, + "margin_dpo/margin_std": 7.0640764236450195, + "step": 193 + }, + { + "KL/chosen_KL_mean": -10.503494262695312, + "KL/mean": -12.863252639770508, + "KL/rejected_KL_mean": -15.223018646240234, + "KL/std": 6.697719573974609, + "epoch": 0.29327286470143615, + "fcm_dpo/beta": 0.10034053027629852, + "fcm_dpo/delta": -0.07828196883201599, + "fcm_dpo/margin": 4.719526290893555, + "fcm_dpo/q_t": 0.395353227853775, + "grad_norm": 26.579702377319336, + "learning_rate": 4.4651327368569684e-07, + "logits/chosen": 0.31545290350914, + "logits/rejected": 0.28684914112091064, + "logps/chosen": -67.15293884277344, + "logps/ref_chosen": -56.64944076538086, + "logps/ref_rejected": -69.98954772949219, + "logps/rejected": -85.21256256103516, + "loss": 1.1333, + "margin_dpo/margin_mean": 4.719526767730713, + "margin_dpo/margin_std": 8.421168327331543, + "step": 194 + }, + { + "KL/chosen_KL_mean": -12.053577423095703, + "KL/mean": -14.948898315429688, + "KL/rejected_KL_mean": -17.844223022460938, + "KL/std": 7.79929256439209, + "epoch": 0.2947845804988662, + "fcm_dpo/beta": 0.097338005900383, + "fcm_dpo/delta": -0.175175741314888, + "fcm_dpo/margin": 5.790646553039551, + "fcm_dpo/q_t": 0.3771982192993164, + "grad_norm": 27.508337020874023, + "learning_rate": 4.4569318740967043e-07, + "logits/chosen": 0.2035025656223297, + "logits/rejected": 0.20420676469802856, + "logps/chosen": -82.4633560180664, + "logps/ref_chosen": -70.40977478027344, + "logps/ref_rejected": -74.39448547363281, + "logps/rejected": -92.23870849609375, + "loss": 1.0269, + "margin_dpo/margin_mean": 5.790646553039551, + "margin_dpo/margin_std": 7.719186782836914, + "step": 195 + }, + { + "KL/chosen_KL_mean": -11.237726211547852, + "KL/mean": -13.784488677978516, + "KL/rejected_KL_mean": -16.331253051757812, + "KL/std": 7.774652481079102, + "epoch": 0.2962962962962963, + "fcm_dpo/beta": 0.09619662165641785, + "fcm_dpo/delta": -0.094697006046772, + "fcm_dpo/margin": 5.093526840209961, + "fcm_dpo/q_t": 0.3892650604248047, + "grad_norm": 25.448198318481445, + "learning_rate": 4.448676271745197e-07, + "logits/chosen": 0.3075557351112366, + "logits/rejected": 0.267707496881485, + "logps/chosen": -70.46530151367188, + "logps/ref_chosen": -59.227577209472656, + "logps/ref_rejected": -83.54757690429688, + "logps/rejected": -99.87882995605469, + "loss": 1.0733, + "margin_dpo/margin_mean": 5.093526840209961, + "margin_dpo/margin_std": 7.3492279052734375, + "step": 196 + }, + { + "KL/chosen_KL_mean": -10.425508499145508, + "KL/mean": -13.5262451171875, + "KL/rejected_KL_mean": -16.626976013183594, + "KL/std": 8.094280242919922, + "epoch": 0.29780801209372637, + "fcm_dpo/beta": 0.09323252737522125, + "fcm_dpo/delta": -0.1887865662574768, + "fcm_dpo/margin": 6.201464653015137, + "fcm_dpo/q_t": 0.37647128105163574, + "grad_norm": 25.889739990234375, + "learning_rate": 4.440366160729392e-07, + "logits/chosen": 0.3803238570690155, + "logits/rejected": 0.3330530822277069, + "logps/chosen": -61.95463943481445, + "logps/ref_chosen": -51.52912902832031, + "logps/ref_rejected": -73.70631408691406, + "logps/rejected": -90.33328247070312, + "loss": 1.0836, + "margin_dpo/margin_mean": 6.201464653015137, + "margin_dpo/margin_std": 9.826972007751465, + "step": 197 + }, + { + "KL/chosen_KL_mean": -10.89830207824707, + "KL/mean": -14.291038513183594, + "KL/rejected_KL_mean": -17.683780670166016, + "KL/std": 7.223607063293457, + "epoch": 0.29931972789115646, + "fcm_dpo/beta": 0.08935528993606567, + "fcm_dpo/delta": -0.21908718347549438, + "fcm_dpo/margin": 6.7854743003845215, + "fcm_dpo/q_t": 0.3636322617530823, + "grad_norm": 24.653465270996094, + "learning_rate": 4.432001773500957e-07, + "logits/chosen": 0.34279125928878784, + "logits/rejected": 0.3045985698699951, + "logps/chosen": -70.68098449707031, + "logps/ref_chosen": -59.78268051147461, + "logps/ref_rejected": -72.24533081054688, + "logps/rejected": -89.92910766601562, + "loss": 0.9697, + "margin_dpo/margin_mean": 6.7854743003845215, + "margin_dpo/margin_std": 7.347314357757568, + "step": 198 + }, + { + "KL/chosen_KL_mean": -12.324453353881836, + "KL/mean": -14.889604568481445, + "KL/rejected_KL_mean": -17.454750061035156, + "KL/std": 7.923635005950928, + "epoch": 0.30083144368858655, + "fcm_dpo/beta": 0.08797682076692581, + "fcm_dpo/delta": -0.05551917850971222, + "fcm_dpo/margin": 5.130303859710693, + "fcm_dpo/q_t": 0.40116453170776367, + "grad_norm": 26.78217124938965, + "learning_rate": 4.4235833440297856e-07, + "logits/chosen": 0.28896427154541016, + "logits/rejected": 0.20546600222587585, + "logps/chosen": -68.71122741699219, + "logps/ref_chosen": -56.38677215576172, + "logps/ref_rejected": -74.56779479980469, + "logps/rejected": -92.02255249023438, + "loss": 1.1612, + "margin_dpo/margin_mean": 5.130303382873535, + "margin_dpo/margin_std": 9.651971817016602, + "step": 199 + }, + { + "KL/chosen_KL_mean": -9.989532470703125, + "KL/mean": -13.755962371826172, + "KL/rejected_KL_mean": -17.52239990234375, + "KL/std": 8.623006820678711, + "epoch": 0.30234315948601664, + "fcm_dpo/beta": 0.08316583186388016, + "fcm_dpo/delta": -0.24403418600559235, + "fcm_dpo/margin": 7.532864570617676, + "fcm_dpo/q_t": 0.3652196526527405, + "grad_norm": 24.068204879760742, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": 0.3554863929748535, + "logits/rejected": 0.28665488958358765, + "logps/chosen": -67.81385803222656, + "logps/ref_chosen": -57.82432556152344, + "logps/ref_rejected": -89.28246307373047, + "logps/rejected": -106.80486297607422, + "loss": 1.017, + "margin_dpo/margin_mean": 7.532864093780518, + "margin_dpo/margin_std": 10.306327819824219, + "step": 200 + }, + { + "KL/chosen_KL_mean": -12.684650421142578, + "KL/mean": -15.855332374572754, + "KL/rejected_KL_mean": -19.026016235351562, + "KL/std": 8.930841445922852, + "epoch": 0.30385487528344673, + "fcm_dpo/beta": 0.08092570304870605, + "fcm_dpo/delta": -0.11991845071315765, + "fcm_dpo/margin": 6.341360092163086, + "fcm_dpo/q_t": 0.38912898302078247, + "grad_norm": 25.944599151611328, + "learning_rate": 4.4065853017905953e-07, + "logits/chosen": 0.37400752305984497, + "logits/rejected": 0.3305491805076599, + "logps/chosen": -71.68441009521484, + "logps/ref_chosen": -58.999759674072266, + "logps/ref_rejected": -84.67575073242188, + "logps/rejected": -103.70176696777344, + "loss": 1.0722, + "margin_dpo/margin_mean": 6.341360092163086, + "margin_dpo/margin_std": 9.579675674438477, + "step": 201 + }, + { + "KL/chosen_KL_mean": -11.093196868896484, + "KL/mean": -14.728687286376953, + "KL/rejected_KL_mean": -18.364173889160156, + "KL/std": 8.838526725769043, + "epoch": 0.30536659108087677, + "fcm_dpo/beta": 0.07887430489063263, + "fcm_dpo/delta": -0.18363875150680542, + "fcm_dpo/margin": 7.270977020263672, + "fcm_dpo/q_t": 0.3720179796218872, + "grad_norm": 22.277822494506836, + "learning_rate": 4.3980061644943575e-07, + "logits/chosen": 0.2807554602622986, + "logits/rejected": 0.21378284692764282, + "logps/chosen": -58.75384521484375, + "logps/ref_chosen": -47.660648345947266, + "logps/ref_rejected": -73.63249969482422, + "logps/rejected": -91.99667358398438, + "loss": 1.0233, + "margin_dpo/margin_mean": 7.270977020263672, + "margin_dpo/margin_std": 9.475584030151367, + "step": 202 + }, + { + "KL/chosen_KL_mean": -12.31509780883789, + "KL/mean": -15.686324119567871, + "KL/rejected_KL_mean": -19.057544708251953, + "KL/std": 9.032432556152344, + "epoch": 0.30687830687830686, + "fcm_dpo/beta": 0.07658464461565018, + "fcm_dpo/delta": -0.12252387404441833, + "fcm_dpo/margin": 6.742450714111328, + "fcm_dpo/q_t": 0.38822680711746216, + "grad_norm": 25.437036514282227, + "learning_rate": 4.3893739358856455e-07, + "logits/chosen": 0.34383517503738403, + "logits/rejected": 0.2762662172317505, + "logps/chosen": -74.64063262939453, + "logps/ref_chosen": -62.32553482055664, + "logps/ref_rejected": -99.37226104736328, + "logps/rejected": -118.4298095703125, + "loss": 1.063, + "margin_dpo/margin_mean": 6.742450714111328, + "margin_dpo/margin_std": 9.920048713684082, + "step": 203 + }, + { + "KL/chosen_KL_mean": -12.230262756347656, + "KL/mean": -15.652043342590332, + "KL/rejected_KL_mean": -19.073822021484375, + "KL/std": 9.692943572998047, + "epoch": 0.30839002267573695, + "fcm_dpo/beta": 0.07370705157518387, + "fcm_dpo/delta": -0.11449645459651947, + "fcm_dpo/margin": 6.843562126159668, + "fcm_dpo/q_t": 0.3891705870628357, + "grad_norm": 22.042787551879883, + "learning_rate": 4.380688857426449e-07, + "logits/chosen": 0.2842333912849426, + "logits/rejected": 0.21995952725410461, + "logps/chosen": -62.859580993652344, + "logps/ref_chosen": -50.62931823730469, + "logps/ref_rejected": -66.60475158691406, + "logps/rejected": -85.67857360839844, + "loss": 1.077, + "margin_dpo/margin_mean": 6.843562602996826, + "margin_dpo/margin_std": 9.982553482055664, + "step": 204 + }, + { + "KL/chosen_KL_mean": -13.222759246826172, + "KL/mean": -16.707935333251953, + "KL/rejected_KL_mean": -20.193115234375, + "KL/std": 9.57691478729248, + "epoch": 0.30990173847316704, + "fcm_dpo/beta": 0.07268859446048737, + "fcm_dpo/delta": -0.11301695555448532, + "fcm_dpo/margin": 6.9703569412231445, + "fcm_dpo/q_t": 0.39053136110305786, + "grad_norm": 27.645692825317383, + "learning_rate": 4.3719511720570814e-07, + "logits/chosen": 0.3358631730079651, + "logits/rejected": 0.2786390781402588, + "logps/chosen": -83.57893371582031, + "logps/ref_chosen": -70.3561782836914, + "logps/ref_rejected": -93.39848327636719, + "logps/rejected": -113.59159851074219, + "loss": 1.0963, + "margin_dpo/margin_mean": 6.9703569412231445, + "margin_dpo/margin_std": 11.408649444580078, + "step": 205 + }, + { + "KL/chosen_KL_mean": -13.618179321289062, + "KL/mean": -16.21625518798828, + "KL/rejected_KL_mean": -18.814334869384766, + "KL/std": 9.856538772583008, + "epoch": 0.31141345427059713, + "fcm_dpo/beta": 0.0731147825717926, + "fcm_dpo/delta": 0.020249299705028534, + "fcm_dpo/margin": 5.1961565017700195, + "fcm_dpo/q_t": 0.4198933243751526, + "grad_norm": 24.5611515045166, + "learning_rate": 4.363161124189387e-07, + "logits/chosen": 0.382521390914917, + "logits/rejected": 0.36643290519714355, + "logps/chosen": -81.26365661621094, + "logps/ref_chosen": -67.64547729492188, + "logps/ref_rejected": -79.89584350585938, + "logps/rejected": -98.71017456054688, + "loss": 1.2133, + "margin_dpo/margin_mean": 5.196155548095703, + "margin_dpo/margin_std": 11.782098770141602, + "step": 206 + }, + { + "KL/chosen_KL_mean": -14.933658599853516, + "KL/mean": -18.79200553894043, + "KL/rejected_KL_mean": -22.650352478027344, + "KL/std": 10.600395202636719, + "epoch": 0.3129251700680272, + "fcm_dpo/beta": 0.07093209028244019, + "fcm_dpo/delta": -0.1565505564212799, + "fcm_dpo/margin": 7.716689586639404, + "fcm_dpo/q_t": 0.38311976194381714, + "grad_norm": 21.809812545776367, + "learning_rate": 4.3543189596998986e-07, + "logits/chosen": 0.2879766821861267, + "logits/rejected": 0.2246440052986145, + "logps/chosen": -82.59785461425781, + "logps/ref_chosen": -67.66419219970703, + "logps/ref_rejected": -85.10249328613281, + "logps/rejected": -107.75283813476562, + "loss": 1.0423, + "margin_dpo/margin_mean": 7.716688632965088, + "margin_dpo/margin_std": 10.914000511169434, + "step": 207 + }, + { + "KL/chosen_KL_mean": -12.039043426513672, + "KL/mean": -14.267229080200195, + "KL/rejected_KL_mean": -16.495407104492188, + "KL/std": 9.71860122680664, + "epoch": 0.3144368858654573, + "fcm_dpo/beta": 0.07146717607975006, + "fcm_dpo/delta": 0.08419809490442276, + "fcm_dpo/margin": 4.456361770629883, + "fcm_dpo/q_t": 0.43023359775543213, + "grad_norm": 24.523517608642578, + "learning_rate": 4.3454249259229664e-07, + "logits/chosen": 0.31310391426086426, + "logits/rejected": 0.28862231969833374, + "logps/chosen": -69.770751953125, + "logps/ref_chosen": -57.731712341308594, + "logps/ref_rejected": -74.19276428222656, + "logps/rejected": -90.68817138671875, + "loss": 1.2268, + "margin_dpo/margin_mean": 4.456361770629883, + "margin_dpo/margin_std": 10.545194625854492, + "step": 208 + }, + { + "KL/chosen_KL_mean": -12.184415817260742, + "KL/mean": -16.680660247802734, + "KL/rejected_KL_mean": -21.176910400390625, + "KL/std": 11.24008560180664, + "epoch": 0.31594860166288735, + "fcm_dpo/beta": 0.0693952813744545, + "fcm_dpo/delta": -0.23861436545848846, + "fcm_dpo/margin": 8.9924898147583, + "fcm_dpo/q_t": 0.3661319613456726, + "grad_norm": 24.027284622192383, + "learning_rate": 4.336479271643833e-07, + "logits/chosen": 0.28997889161109924, + "logits/rejected": 0.23683319985866547, + "logps/chosen": -80.7344970703125, + "logps/ref_chosen": -68.55007934570312, + "logps/ref_rejected": -87.90541076660156, + "logps/rejected": -109.08232116699219, + "loss": 1.0303, + "margin_dpo/margin_mean": 8.992490768432617, + "margin_dpo/margin_std": 12.858354568481445, + "step": 209 + }, + { + "KL/chosen_KL_mean": -12.258684158325195, + "KL/mean": -16.604808807373047, + "KL/rejected_KL_mean": -20.950939178466797, + "KL/std": 11.92989730834961, + "epoch": 0.31746031746031744, + "fcm_dpo/beta": 0.06671866774559021, + "fcm_dpo/delta": -0.19063086807727814, + "fcm_dpo/margin": 8.692248344421387, + "fcm_dpo/q_t": 0.37737345695495605, + "grad_norm": 21.236785888671875, + "learning_rate": 4.327482247091679e-07, + "logits/chosen": 0.4041959047317505, + "logits/rejected": 0.3123385012149811, + "logps/chosen": -69.52696228027344, + "logps/ref_chosen": -57.268272399902344, + "logps/ref_rejected": -85.72807312011719, + "logps/rejected": -106.67901611328125, + "loss": 1.0505, + "margin_dpo/margin_mean": 8.69224739074707, + "margin_dpo/margin_std": 12.821832656860352, + "step": 210 + }, + { + "KL/chosen_KL_mean": -10.797460556030273, + "KL/mean": -14.93586540222168, + "KL/rejected_KL_mean": -19.07427978515625, + "KL/std": 11.016217231750488, + "epoch": 0.31897203325774753, + "fcm_dpo/beta": 0.06479281187057495, + "fcm_dpo/delta": -0.1437854766845703, + "fcm_dpo/margin": 8.276815414428711, + "fcm_dpo/q_t": 0.3854495584964752, + "grad_norm": 23.455148696899414, + "learning_rate": 4.3184341039326217e-07, + "logits/chosen": 0.37784039974212646, + "logits/rejected": 0.2924278676509857, + "logps/chosen": -64.43817138671875, + "logps/ref_chosen": -53.640708923339844, + "logps/ref_rejected": -93.0387954711914, + "logps/rejected": -112.11307525634766, + "loss": 1.055, + "margin_dpo/margin_mean": 8.276814460754395, + "margin_dpo/margin_std": 11.989517211914062, + "step": 211 + }, + { + "KL/chosen_KL_mean": -12.629556655883789, + "KL/mean": -17.281192779541016, + "KL/rejected_KL_mean": -21.932838439941406, + "KL/std": 11.690235137939453, + "epoch": 0.3204837490551776, + "fcm_dpo/beta": 0.06214489787817001, + "fcm_dpo/delta": -0.18937504291534424, + "fcm_dpo/margin": 9.303281784057617, + "fcm_dpo/q_t": 0.37309345602989197, + "grad_norm": 19.30466651916504, + "learning_rate": 4.309335095262675e-07, + "logits/chosen": 0.3716275095939636, + "logits/rejected": 0.3046720027923584, + "logps/chosen": -69.99629974365234, + "logps/ref_chosen": -57.36674499511719, + "logps/ref_rejected": -79.89643096923828, + "logps/rejected": -101.82926940917969, + "loss": 1.0357, + "margin_dpo/margin_mean": 9.303279876708984, + "margin_dpo/margin_std": 13.027929306030273, + "step": 212 + }, + { + "KL/chosen_KL_mean": -9.048931121826172, + "KL/mean": -14.40077018737793, + "KL/rejected_KL_mean": -19.752609252929688, + "KL/std": 12.173477172851562, + "epoch": 0.3219954648526077, + "fcm_dpo/beta": 0.05909217149019241, + "fcm_dpo/delta": -0.24991421401500702, + "fcm_dpo/margin": 10.703678131103516, + "fcm_dpo/q_t": 0.3626417815685272, + "grad_norm": 17.545997619628906, + "learning_rate": 4.3001854756006724e-07, + "logits/chosen": 0.39142847061157227, + "logits/rejected": 0.3677615523338318, + "logps/chosen": -74.27005004882812, + "logps/ref_chosen": -65.22111511230469, + "logps/ref_rejected": -80.1810302734375, + "logps/rejected": -99.93363952636719, + "loss": 1.0005, + "margin_dpo/margin_mean": 10.703678131103516, + "margin_dpo/margin_std": 14.004202842712402, + "step": 213 + }, + { + "KL/chosen_KL_mean": -10.82396125793457, + "KL/mean": -15.959489822387695, + "KL/rejected_KL_mean": -21.095014572143555, + "KL/std": 12.298513412475586, + "epoch": 0.3235071806500378, + "fcm_dpo/beta": 0.05724663287401199, + "fcm_dpo/delta": -0.19926320016384125, + "fcm_dpo/margin": 10.271051406860352, + "fcm_dpo/q_t": 0.37353187799453735, + "grad_norm": 21.21104621887207, + "learning_rate": 4.290985500881143e-07, + "logits/chosen": 0.2602912187576294, + "logits/rejected": 0.23779809474945068, + "logps/chosen": -72.11628723144531, + "logps/ref_chosen": -61.292327880859375, + "logps/ref_rejected": -67.69841003417969, + "logps/rejected": -88.79342651367188, + "loss": 1.0253, + "margin_dpo/margin_mean": 10.271051406860352, + "margin_dpo/margin_std": 13.87718391418457, + "step": 214 + }, + { + "KL/chosen_KL_mean": -13.394432067871094, + "KL/mean": -18.717758178710938, + "KL/rejected_KL_mean": -24.041080474853516, + "KL/std": 12.819029808044434, + "epoch": 0.3250188964474679, + "fcm_dpo/beta": 0.055094048380851746, + "fcm_dpo/delta": -0.19782397150993347, + "fcm_dpo/margin": 10.646649360656738, + "fcm_dpo/q_t": 0.37471532821655273, + "grad_norm": 19.239511489868164, + "learning_rate": 4.281735428447157e-07, + "logits/chosen": 0.29680147767066956, + "logits/rejected": 0.19680196046829224, + "logps/chosen": -77.26356506347656, + "logps/ref_chosen": -63.869136810302734, + "logps/ref_rejected": -98.7657241821289, + "logps/rejected": -122.80680847167969, + "loss": 1.0339, + "margin_dpo/margin_mean": 10.646649360656738, + "margin_dpo/margin_std": 14.607285499572754, + "step": 215 + }, + { + "KL/chosen_KL_mean": -12.626960754394531, + "KL/mean": -18.285587310791016, + "KL/rejected_KL_mean": -23.9442138671875, + "KL/std": 13.705946922302246, + "epoch": 0.32653061224489793, + "fcm_dpo/beta": 0.05256032198667526, + "fcm_dpo/delta": -0.20697513222694397, + "fcm_dpo/margin": 11.317251205444336, + "fcm_dpo/q_t": 0.37171292304992676, + "grad_norm": 21.164426803588867, + "learning_rate": 4.2724355170431247e-07, + "logits/chosen": 0.4230896234512329, + "logits/rejected": 0.3399661183357239, + "logps/chosen": -80.45191955566406, + "logps/ref_chosen": -67.824951171875, + "logps/ref_rejected": -96.40231323242188, + "logps/rejected": -120.34652709960938, + "loss": 1.013, + "margin_dpo/margin_mean": 11.31725025177002, + "margin_dpo/margin_std": 15.183269500732422, + "step": 216 + }, + { + "KL/chosen_KL_mean": -14.25343132019043, + "KL/mean": -20.095951080322266, + "KL/rejected_KL_mean": -25.938472747802734, + "KL/std": 13.898407936096191, + "epoch": 0.328042328042328, + "fcm_dpo/beta": 0.05035218223929405, + "fcm_dpo/delta": -0.2004796266555786, + "fcm_dpo/margin": 11.685037612915039, + "fcm_dpo/q_t": 0.37172871828079224, + "grad_norm": 17.534391403198242, + "learning_rate": 4.26308602680756e-07, + "logits/chosen": 0.3500717282295227, + "logits/rejected": 0.24914100766181946, + "logps/chosen": -74.7584228515625, + "logps/ref_chosen": -60.5049934387207, + "logps/ref_rejected": -84.26618194580078, + "logps/rejected": -110.20465087890625, + "loss": 1.0056, + "margin_dpo/margin_mean": 11.685037612915039, + "margin_dpo/margin_std": 14.929855346679688, + "step": 217 + }, + { + "KL/chosen_KL_mean": -14.796497344970703, + "KL/mean": -18.559711456298828, + "KL/rejected_KL_mean": -22.322925567626953, + "KL/std": 13.00196647644043, + "epoch": 0.3295540438397581, + "fcm_dpo/beta": 0.05051659792661667, + "fcm_dpo/delta": 0.01861324906349182, + "fcm_dpo/margin": 7.526429176330566, + "fcm_dpo/q_t": 0.41731739044189453, + "grad_norm": 20.19041633605957, + "learning_rate": 4.253687219265803e-07, + "logits/chosen": 0.2565411925315857, + "logits/rejected": 0.25069519877433777, + "logps/chosen": -85.39080810546875, + "logps/ref_chosen": -70.59431457519531, + "logps/ref_rejected": -73.89038848876953, + "logps/rejected": -96.21331787109375, + "loss": 1.2045, + "margin_dpo/margin_mean": 7.526429176330566, + "margin_dpo/margin_std": 16.088058471679688, + "step": 218 + }, + { + "KL/chosen_KL_mean": -13.562408447265625, + "KL/mean": -17.571258544921875, + "KL/rejected_KL_mean": -21.580114364624023, + "KL/std": 14.031749725341797, + "epoch": 0.3310657596371882, + "fcm_dpo/beta": 0.04987781122326851, + "fcm_dpo/delta": -0.0002720542252063751, + "fcm_dpo/margin": 8.017708778381348, + "fcm_dpo/q_t": 0.41187363862991333, + "grad_norm": 19.11667823791504, + "learning_rate": 4.2442393573227043e-07, + "logits/chosen": 0.32280057668685913, + "logits/rejected": 0.2829885184764862, + "logps/chosen": -74.05335235595703, + "logps/ref_chosen": -60.490943908691406, + "logps/ref_rejected": -75.85001373291016, + "logps/rejected": -97.43013000488281, + "loss": 1.1407, + "margin_dpo/margin_mean": 8.017709732055664, + "margin_dpo/margin_std": 14.044523239135742, + "step": 219 + }, + { + "KL/chosen_KL_mean": -12.115495681762695, + "KL/mean": -16.92727279663086, + "KL/rejected_KL_mean": -21.739051818847656, + "KL/std": 14.268013000488281, + "epoch": 0.3325774754346183, + "fcm_dpo/beta": 0.04924558848142624, + "fcm_dpo/delta": -0.07858522981405258, + "fcm_dpo/margin": 9.623554229736328, + "fcm_dpo/q_t": 0.39898520708084106, + "grad_norm": 16.392789840698242, + "learning_rate": 4.234742705255272e-07, + "logits/chosen": 0.4274730086326599, + "logits/rejected": 0.36710894107818604, + "logps/chosen": -57.1288948059082, + "logps/ref_chosen": -45.013397216796875, + "logps/ref_rejected": -70.49369812011719, + "logps/rejected": -92.23275756835938, + "loss": 1.1145, + "margin_dpo/margin_mean": 9.623554229736328, + "margin_dpo/margin_std": 16.26483154296875, + "step": 220 + }, + { + "KL/chosen_KL_mean": -11.787042617797852, + "KL/mean": -17.021156311035156, + "KL/rejected_KL_mean": -22.255271911621094, + "KL/std": 14.119913101196289, + "epoch": 0.3340891912320484, + "fcm_dpo/beta": 0.04853355139493942, + "fcm_dpo/delta": -0.11374709010124207, + "fcm_dpo/margin": 10.468233108520508, + "fcm_dpo/q_t": 0.39125609397888184, + "grad_norm": 17.957706451416016, + "learning_rate": 4.22519752870528e-07, + "logits/chosen": 0.3971208333969116, + "logits/rejected": 0.3293982148170471, + "logps/chosen": -70.88288116455078, + "logps/ref_chosen": -59.09584045410156, + "logps/ref_rejected": -88.64388275146484, + "logps/rejected": -110.89915466308594, + "loss": 1.0807, + "margin_dpo/margin_mean": 10.468233108520508, + "margin_dpo/margin_std": 16.371990203857422, + "step": 221 + }, + { + "KL/chosen_KL_mean": -12.735214233398438, + "KL/mean": -19.463855743408203, + "KL/rejected_KL_mean": -26.19249725341797, + "KL/std": 15.829582214355469, + "epoch": 0.3356009070294785, + "fcm_dpo/beta": 0.046448659151792526, + "fcm_dpo/delta": -0.24044792354106903, + "fcm_dpo/margin": 13.45728874206543, + "fcm_dpo/q_t": 0.36255699396133423, + "grad_norm": 18.125226974487305, + "learning_rate": 4.2156040946718343e-07, + "logits/chosen": 0.4456883668899536, + "logits/rejected": 0.36282992362976074, + "logps/chosen": -68.73291015625, + "logps/ref_chosen": -55.9976921081543, + "logps/ref_rejected": -111.94727325439453, + "logps/rejected": -138.1397705078125, + "loss": 0.9962, + "margin_dpo/margin_mean": 13.457290649414062, + "margin_dpo/margin_std": 16.974525451660156, + "step": 222 + }, + { + "KL/chosen_KL_mean": -14.5010986328125, + "KL/mean": -20.88688850402832, + "KL/rejected_KL_mean": -27.272682189941406, + "KL/std": 15.840429306030273, + "epoch": 0.3371126228269085, + "fcm_dpo/beta": 0.04450047016143799, + "fcm_dpo/delta": -0.1799127459526062, + "fcm_dpo/margin": 12.771581649780273, + "fcm_dpo/q_t": 0.37460440397262573, + "grad_norm": 16.625930786132812, + "learning_rate": 4.2059626715039065e-07, + "logits/chosen": 0.4114035665988922, + "logits/rejected": 0.357668936252594, + "logps/chosen": -74.39251708984375, + "logps/ref_chosen": -59.891422271728516, + "logps/ref_rejected": -86.28954315185547, + "logps/rejected": -113.56222534179688, + "loss": 1.0008, + "margin_dpo/margin_mean": 12.771581649780273, + "margin_dpo/margin_std": 15.203545570373535, + "step": 223 + }, + { + "KL/chosen_KL_mean": -16.549915313720703, + "KL/mean": -20.222265243530273, + "KL/rejected_KL_mean": -23.894615173339844, + "KL/std": 15.480034828186035, + "epoch": 0.3386243386243386, + "fcm_dpo/beta": 0.04471778869628906, + "fcm_dpo/delta": 0.07405854016542435, + "fcm_dpo/margin": 7.344696998596191, + "fcm_dpo/q_t": 0.42939624190330505, + "grad_norm": 21.553468704223633, + "learning_rate": 4.1962735288928304e-07, + "logits/chosen": 0.463707834482193, + "logits/rejected": 0.44336050748825073, + "logps/chosen": -80.59455108642578, + "logps/ref_chosen": -64.04463195800781, + "logps/ref_rejected": -75.05450439453125, + "logps/rejected": -98.94911193847656, + "loss": 1.1979, + "margin_dpo/margin_mean": 7.344696998596191, + "margin_dpo/margin_std": 15.690269470214844, + "step": 224 + }, + { + "KL/chosen_KL_mean": -15.249135971069336, + "KL/mean": -21.75031280517578, + "KL/rejected_KL_mean": -28.25149154663086, + "KL/std": 17.294654846191406, + "epoch": 0.3401360544217687, + "fcm_dpo/beta": 0.04361790418624878, + "fcm_dpo/delta": -0.17827551066875458, + "fcm_dpo/margin": 13.002354621887207, + "fcm_dpo/q_t": 0.37681907415390015, + "grad_norm": 18.24221420288086, + "learning_rate": 4.186536937864752e-07, + "logits/chosen": 0.4368096590042114, + "logits/rejected": 0.32517528533935547, + "logps/chosen": -81.34495544433594, + "logps/ref_chosen": -66.0958251953125, + "logps/ref_rejected": -97.68675231933594, + "logps/rejected": -125.93824005126953, + "loss": 1.0228, + "margin_dpo/margin_mean": 13.002354621887207, + "margin_dpo/margin_std": 17.1579647064209, + "step": 225 + }, + { + "KL/chosen_KL_mean": -14.37647819519043, + "KL/mean": -19.67775535583496, + "KL/rejected_KL_mean": -24.97903060913086, + "KL/std": 16.739892959594727, + "epoch": 0.3416477702191988, + "fcm_dpo/beta": 0.04320550337433815, + "fcm_dpo/delta": -0.06102943420410156, + "fcm_dpo/margin": 10.602548599243164, + "fcm_dpo/q_t": 0.4007114768028259, + "grad_norm": 15.894902229309082, + "learning_rate": 4.176753170773052e-07, + "logits/chosen": 0.49875736236572266, + "logits/rejected": 0.45056477189064026, + "logps/chosen": -65.79335021972656, + "logps/ref_chosen": -51.4168701171875, + "logps/ref_rejected": -66.30068969726562, + "logps/rejected": -91.27971649169922, + "loss": 1.1381, + "margin_dpo/margin_mean": 10.602548599243164, + "margin_dpo/margin_std": 19.16706085205078, + "step": 226 + }, + { + "KL/chosen_KL_mean": -15.519994735717773, + "KL/mean": -21.301816940307617, + "KL/rejected_KL_mean": -27.083641052246094, + "KL/std": 17.973379135131836, + "epoch": 0.3431594860166289, + "fcm_dpo/beta": 0.042112961411476135, + "fcm_dpo/delta": -0.09216802567243576, + "fcm_dpo/margin": 11.563647270202637, + "fcm_dpo/q_t": 0.3971030116081238, + "grad_norm": 17.23468589782715, + "learning_rate": 4.166922501290729e-07, + "logits/chosen": 0.48868125677108765, + "logits/rejected": 0.4502916932106018, + "logps/chosen": -73.50977325439453, + "logps/ref_chosen": -57.989776611328125, + "logps/ref_rejected": -75.05464172363281, + "logps/rejected": -102.13829040527344, + "loss": 1.117, + "margin_dpo/margin_mean": 11.563648223876953, + "margin_dpo/margin_std": 19.933834075927734, + "step": 227 + }, + { + "KL/chosen_KL_mean": -17.620210647583008, + "KL/mean": -23.31317901611328, + "KL/rejected_KL_mean": -29.006141662597656, + "KL/std": 17.29057502746582, + "epoch": 0.34467120181405897, + "fcm_dpo/beta": 0.0417354553937912, + "fcm_dpo/delta": -0.07891019433736801, + "fcm_dpo/margin": 11.385927200317383, + "fcm_dpo/q_t": 0.3964489698410034, + "grad_norm": 16.280597686767578, + "learning_rate": 4.1570452044027405e-07, + "logits/chosen": 0.46561571955680847, + "logits/rejected": 0.3897852301597595, + "logps/chosen": -73.17958068847656, + "logps/ref_chosen": -55.55936813354492, + "logps/ref_rejected": -77.02364349365234, + "logps/rejected": -106.02978515625, + "loss": 1.0928, + "margin_dpo/margin_mean": 11.385927200317383, + "margin_dpo/margin_std": 17.77695083618164, + "step": 228 + }, + { + "KL/chosen_KL_mean": -13.526348114013672, + "KL/mean": -19.22211265563965, + "KL/rejected_KL_mean": -24.917877197265625, + "KL/std": 16.780139923095703, + "epoch": 0.34618291761148906, + "fcm_dpo/beta": 0.04094117134809494, + "fcm_dpo/delta": -0.06977853924036026, + "fcm_dpo/margin": 11.39152717590332, + "fcm_dpo/q_t": 0.3969324827194214, + "grad_norm": 33.38159942626953, + "learning_rate": 4.147121556398312e-07, + "logits/chosen": 0.5544859170913696, + "logits/rejected": 0.4903777837753296, + "logps/chosen": -64.32101440429688, + "logps/ref_chosen": -50.79466247558594, + "logps/ref_rejected": -78.4474105834961, + "logps/rejected": -103.36528778076172, + "loss": 1.1309, + "margin_dpo/margin_mean": 11.39152717590332, + "margin_dpo/margin_std": 20.368953704833984, + "step": 229 + }, + { + "KL/chosen_KL_mean": -15.6832275390625, + "KL/mean": -22.08908462524414, + "KL/rejected_KL_mean": -28.49494171142578, + "KL/std": 18.338987350463867, + "epoch": 0.3476946334089191, + "fcm_dpo/beta": 0.040487367659807205, + "fcm_dpo/delta": -0.12587293982505798, + "fcm_dpo/margin": 12.811714172363281, + "fcm_dpo/q_t": 0.38656824827194214, + "grad_norm": 16.915529251098633, + "learning_rate": 4.137151834863213e-07, + "logits/chosen": 0.46070948243141174, + "logits/rejected": 0.4588784873485565, + "logps/chosen": -72.4124526977539, + "logps/ref_chosen": -56.729225158691406, + "logps/ref_rejected": -62.99180603027344, + "logps/rejected": -91.48674774169922, + "loss": 1.0587, + "margin_dpo/margin_mean": 12.811714172363281, + "margin_dpo/margin_std": 17.839630126953125, + "step": 230 + }, + { + "KL/chosen_KL_mean": -18.668453216552734, + "KL/mean": -27.829383850097656, + "KL/rejected_KL_mean": -36.990318298339844, + "KL/std": 18.366313934326172, + "epoch": 0.3492063492063492, + "fcm_dpo/beta": 0.037801600992679596, + "fcm_dpo/delta": -0.3168046176433563, + "fcm_dpo/margin": 18.32186508178711, + "fcm_dpo/q_t": 0.34421560168266296, + "grad_norm": 17.75885581970215, + "learning_rate": 4.1271363186719835e-07, + "logits/chosen": 0.3724779784679413, + "logits/rejected": 0.35940617322921753, + "logps/chosen": -91.26554870605469, + "logps/ref_chosen": -72.59709930419922, + "logps/ref_rejected": -86.2322998046875, + "logps/rejected": -123.22261810302734, + "loss": 0.9318, + "margin_dpo/margin_mean": 18.32186508178711, + "margin_dpo/margin_std": 19.380470275878906, + "step": 231 + }, + { + "KL/chosen_KL_mean": -18.254302978515625, + "KL/mean": -24.430221557617188, + "KL/rejected_KL_mean": -30.606136322021484, + "KL/std": 19.692710876464844, + "epoch": 0.3507180650037793, + "fcm_dpo/beta": 0.03702760860323906, + "fcm_dpo/delta": -0.06017792224884033, + "fcm_dpo/margin": 12.351831436157227, + "fcm_dpo/q_t": 0.40328365564346313, + "grad_norm": 17.288766860961914, + "learning_rate": 4.1170752879801436e-07, + "logits/chosen": 0.4320966601371765, + "logits/rejected": 0.4029006063938141, + "logps/chosen": -86.37283325195312, + "logps/ref_chosen": -68.1185302734375, + "logps/ref_rejected": -83.79415893554688, + "logps/rejected": -114.40029907226562, + "loss": 1.1292, + "margin_dpo/margin_mean": 12.351832389831543, + "margin_dpo/margin_std": 22.07088851928711, + "step": 232 + }, + { + "KL/chosen_KL_mean": -21.45623207092285, + "KL/mean": -26.537189483642578, + "KL/rejected_KL_mean": -31.618144989013672, + "KL/std": 19.625057220458984, + "epoch": 0.35222978080120937, + "fcm_dpo/beta": 0.03667011857032776, + "fcm_dpo/delta": 0.026034735143184662, + "fcm_dpo/margin": 10.16191291809082, + "fcm_dpo/q_t": 0.41923320293426514, + "grad_norm": 16.044673919677734, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": 0.47052958607673645, + "logits/rejected": 0.4147252142429352, + "logps/chosen": -76.52638244628906, + "logps/ref_chosen": -55.070152282714844, + "logps/ref_rejected": -66.61845397949219, + "logps/rejected": -98.2365951538086, + "loss": 1.1726, + "margin_dpo/margin_mean": 10.16191291809082, + "margin_dpo/margin_std": 18.987119674682617, + "step": 233 + }, + { + "KL/chosen_KL_mean": -19.42575454711914, + "KL/mean": -24.680850982666016, + "KL/rejected_KL_mean": -29.93594741821289, + "KL/std": 18.938419342041016, + "epoch": 0.35374149659863946, + "fcm_dpo/beta": 0.0376385897397995, + "fcm_dpo/delta": 0.0010745078325271606, + "fcm_dpo/margin": 10.51019287109375, + "fcm_dpo/q_t": 0.41605016589164734, + "grad_norm": 19.10257339477539, + "learning_rate": 4.09681781007452e-07, + "logits/chosen": 0.4139387309551239, + "logits/rejected": 0.4022018313407898, + "logps/chosen": -75.35165405273438, + "logps/ref_chosen": -55.92589569091797, + "logps/ref_rejected": -51.11608123779297, + "logps/rejected": -81.05203247070312, + "loss": 1.1893, + "margin_dpo/margin_mean": 10.51019287109375, + "margin_dpo/margin_std": 20.87730598449707, + "step": 234 + }, + { + "KL/chosen_KL_mean": -15.336202621459961, + "KL/mean": -24.05404281616211, + "KL/rejected_KL_mean": -32.771888732910156, + "KL/std": 20.12569808959961, + "epoch": 0.35525321239606955, + "fcm_dpo/beta": 0.03593028336763382, + "fcm_dpo/delta": -0.24115046858787537, + "fcm_dpo/margin": 17.435686111450195, + "fcm_dpo/q_t": 0.3593088388442993, + "grad_norm": 16.729801177978516, + "learning_rate": 4.08662192950594e-07, + "logits/chosen": 0.5166130661964417, + "logits/rejected": 0.4991014003753662, + "logps/chosen": -79.87593078613281, + "logps/ref_chosen": -64.53972625732422, + "logps/ref_rejected": -77.69151306152344, + "logps/rejected": -110.46339416503906, + "loss": 0.9662, + "margin_dpo/margin_mean": 17.435686111450195, + "margin_dpo/margin_std": 19.512123107910156, + "step": 235 + }, + { + "KL/chosen_KL_mean": -25.423974990844727, + "KL/mean": -32.12845993041992, + "KL/rejected_KL_mean": -38.83293914794922, + "KL/std": 20.33449935913086, + "epoch": 0.35676492819349964, + "fcm_dpo/beta": 0.03493765741586685, + "fcm_dpo/delta": -0.07212943583726883, + "fcm_dpo/margin": 13.408965110778809, + "fcm_dpo/q_t": 0.39774391055107117, + "grad_norm": 16.289026260375977, + "learning_rate": 4.076381667711306e-07, + "logits/chosen": 0.4709205627441406, + "logits/rejected": 0.457000195980072, + "logps/chosen": -96.5787124633789, + "logps/ref_chosen": -71.15473937988281, + "logps/ref_rejected": -84.88541412353516, + "logps/rejected": -123.71835327148438, + "loss": 1.1192, + "margin_dpo/margin_mean": 13.408964157104492, + "margin_dpo/margin_std": 22.929529190063477, + "step": 236 + }, + { + "KL/chosen_KL_mean": -22.029151916503906, + "KL/mean": -29.681533813476562, + "KL/rejected_KL_mean": -37.33391189575195, + "KL/std": 20.521644592285156, + "epoch": 0.35827664399092973, + "fcm_dpo/beta": 0.034239333122968674, + "fcm_dpo/delta": -0.13081705570220947, + "fcm_dpo/margin": 15.304759979248047, + "fcm_dpo/q_t": 0.3851306438446045, + "grad_norm": 18.769914627075195, + "learning_rate": 4.066097311132753e-07, + "logits/chosen": 0.5257338881492615, + "logits/rejected": 0.5131624341011047, + "logps/chosen": -98.1711654663086, + "logps/ref_chosen": -76.14201354980469, + "logps/ref_rejected": -80.88479614257812, + "logps/rejected": -118.21870422363281, + "loss": 1.0769, + "margin_dpo/margin_mean": 15.304759979248047, + "margin_dpo/margin_std": 23.187271118164062, + "step": 237 + }, + { + "KL/chosen_KL_mean": -19.780475616455078, + "KL/mean": -27.63768768310547, + "KL/rejected_KL_mean": -35.49489974975586, + "KL/std": 22.138229370117188, + "epoch": 0.35978835978835977, + "fcm_dpo/beta": 0.033210694789886475, + "fcm_dpo/delta": -0.12943503260612488, + "fcm_dpo/margin": 15.714425086975098, + "fcm_dpo/q_t": 0.3854626417160034, + "grad_norm": 25.83220672607422, + "learning_rate": 4.0557691474458414e-07, + "logits/chosen": 0.4778020977973938, + "logits/rejected": 0.4644400179386139, + "logps/chosen": -88.66532897949219, + "logps/ref_chosen": -68.88484954833984, + "logps/ref_rejected": -75.8946304321289, + "logps/rejected": -111.3895263671875, + "loss": 1.0601, + "margin_dpo/margin_mean": 15.714425086975098, + "margin_dpo/margin_std": 22.588531494140625, + "step": 238 + }, + { + "KL/chosen_KL_mean": -24.469463348388672, + "KL/mean": -32.38288116455078, + "KL/rejected_KL_mean": -40.296295166015625, + "KL/std": 23.43410873413086, + "epoch": 0.36130007558578986, + "fcm_dpo/beta": 0.03276742249727249, + "fcm_dpo/delta": -0.12536010146141052, + "fcm_dpo/margin": 15.826839447021484, + "fcm_dpo/q_t": 0.3873627185821533, + "grad_norm": 17.71445655822754, + "learning_rate": 4.045397465551513e-07, + "logits/chosen": 0.6087825894355774, + "logits/rejected": 0.4834781587123871, + "logps/chosen": -81.24128723144531, + "logps/ref_chosen": -56.771827697753906, + "logps/ref_rejected": -116.23050689697266, + "logps/rejected": -156.52679443359375, + "loss": 1.0763, + "margin_dpo/margin_mean": 15.826841354370117, + "margin_dpo/margin_std": 23.740345001220703, + "step": 239 + }, + { + "KL/chosen_KL_mean": -23.602027893066406, + "KL/mean": -33.94664764404297, + "KL/rejected_KL_mean": -44.29126739501953, + "KL/std": 23.732091903686523, + "epoch": 0.36281179138321995, + "fcm_dpo/beta": 0.03111092746257782, + "fcm_dpo/delta": -0.2601979374885559, + "fcm_dpo/margin": 20.689245223999023, + "fcm_dpo/q_t": 0.3577980101108551, + "grad_norm": 13.626633644104004, + "learning_rate": 4.0349825555680045e-07, + "logits/chosen": 0.5065616369247437, + "logits/rejected": 0.4162125587463379, + "logps/chosen": -76.95613861083984, + "logps/ref_chosen": -53.35411071777344, + "logps/ref_rejected": -80.12019348144531, + "logps/rejected": -124.41146850585938, + "loss": 0.9648, + "margin_dpo/margin_mean": 20.689245223999023, + "margin_dpo/margin_std": 23.720182418823242, + "step": 240 + }, + { + "KL/chosen_KL_mean": -23.950889587402344, + "KL/mean": -30.622859954833984, + "KL/rejected_KL_mean": -37.29482650756836, + "KL/std": 22.645187377929688, + "epoch": 0.36432350718065004, + "fcm_dpo/beta": 0.03072282299399376, + "fcm_dpo/delta": -0.010602390393614769, + "fcm_dpo/margin": 13.343942642211914, + "fcm_dpo/q_t": 0.4106614291667938, + "grad_norm": 17.857189178466797, + "learning_rate": 4.0245247088227377e-07, + "logits/chosen": 0.4656754732131958, + "logits/rejected": 0.43102699518203735, + "logps/chosen": -95.84629821777344, + "logps/ref_chosen": -71.89541625976562, + "logps/ref_rejected": -83.03492736816406, + "logps/rejected": -120.32975769042969, + "loss": 1.142, + "margin_dpo/margin_mean": 13.343942642211914, + "margin_dpo/margin_std": 23.82295799255371, + "step": 241 + }, + { + "KL/chosen_KL_mean": -24.637340545654297, + "KL/mean": -34.245872497558594, + "KL/rejected_KL_mean": -43.854408264160156, + "KL/std": 24.97643280029297, + "epoch": 0.36583522297808013, + "fcm_dpo/beta": 0.029523026198148727, + "fcm_dpo/delta": -0.18067717552185059, + "fcm_dpo/margin": 19.217071533203125, + "fcm_dpo/q_t": 0.3758378028869629, + "grad_norm": 13.497591972351074, + "learning_rate": 4.0140242178441665e-07, + "logits/chosen": 0.46095046401023865, + "logits/rejected": 0.4407215714454651, + "logps/chosen": -82.56477355957031, + "logps/ref_chosen": -57.927433013916016, + "logps/ref_rejected": -67.838623046875, + "logps/rejected": -111.69302368164062, + "loss": 1.03, + "margin_dpo/margin_mean": 19.217071533203125, + "margin_dpo/margin_std": 25.75708770751953, + "step": 242 + }, + { + "KL/chosen_KL_mean": -25.040569305419922, + "KL/mean": -33.38526916503906, + "KL/rejected_KL_mean": -41.72996139526367, + "KL/std": 24.873241424560547, + "epoch": 0.3673469387755102, + "fcm_dpo/beta": 0.0291166752576828, + "fcm_dpo/delta": -0.09033676981925964, + "fcm_dpo/margin": 16.68939208984375, + "fcm_dpo/q_t": 0.39298439025878906, + "grad_norm": 17.021177291870117, + "learning_rate": 4.003481376353596e-07, + "logits/chosen": 0.5095110535621643, + "logits/rejected": 0.5118368864059448, + "logps/chosen": -99.31724548339844, + "logps/ref_chosen": -74.27667236328125, + "logps/ref_rejected": -73.24340057373047, + "logps/rejected": -114.97335815429688, + "loss": 1.0805, + "margin_dpo/margin_mean": 16.68939208984375, + "margin_dpo/margin_std": 25.188854217529297, + "step": 243 + }, + { + "KL/chosen_KL_mean": -25.32716178894043, + "KL/mean": -36.57783126831055, + "KL/rejected_KL_mean": -47.82850646972656, + "KL/std": 24.990251541137695, + "epoch": 0.3688586545729403, + "fcm_dpo/beta": 0.028054017573595047, + "fcm_dpo/delta": -0.24619925022125244, + "fcm_dpo/margin": 22.501346588134766, + "fcm_dpo/q_t": 0.3596031367778778, + "grad_norm": 15.71311092376709, + "learning_rate": 3.9928964792569654e-07, + "logits/chosen": 0.5257828831672668, + "logits/rejected": 0.44374197721481323, + "logps/chosen": -78.69107055664062, + "logps/ref_chosen": -53.36390686035156, + "logps/ref_rejected": -71.10276794433594, + "logps/rejected": -118.9312744140625, + "loss": 0.9604, + "margin_dpo/margin_mean": 22.501344680786133, + "margin_dpo/margin_std": 24.605850219726562, + "step": 244 + }, + { + "KL/chosen_KL_mean": -27.530160903930664, + "KL/mean": -40.439083099365234, + "KL/rejected_KL_mean": -53.34800720214844, + "KL/std": 26.418277740478516, + "epoch": 0.37037037037037035, + "fcm_dpo/beta": 0.02635239064693451, + "fcm_dpo/delta": -0.3012879490852356, + "fcm_dpo/margin": 25.81784439086914, + "fcm_dpo/q_t": 0.3481583893299103, + "grad_norm": 20.990034103393555, + "learning_rate": 3.982269822636601e-07, + "logits/chosen": 0.5564769506454468, + "logits/rejected": 0.5310531854629517, + "logps/chosen": -98.72526550292969, + "logps/ref_chosen": -71.19510650634766, + "logps/ref_rejected": -80.76235961914062, + "logps/rejected": -134.11036682128906, + "loss": 0.9256, + "margin_dpo/margin_mean": 25.817842483520508, + "margin_dpo/margin_std": 26.52771759033203, + "step": 245 + }, + { + "KL/chosen_KL_mean": -33.67304229736328, + "KL/mean": -44.59162521362305, + "KL/rejected_KL_mean": -55.51020431518555, + "KL/std": 27.117572784423828, + "epoch": 0.37188208616780044, + "fcm_dpo/beta": 0.02548816055059433, + "fcm_dpo/delta": -0.1655275523662567, + "fcm_dpo/margin": 21.837169647216797, + "fcm_dpo/q_t": 0.37953728437423706, + "grad_norm": 17.551103591918945, + "learning_rate": 3.971601703742932e-07, + "logits/chosen": 0.5693104863166809, + "logits/rejected": 0.5100914239883423, + "logps/chosen": -105.2940902709961, + "logps/ref_chosen": -71.62104797363281, + "logps/ref_rejected": -94.03392028808594, + "logps/rejected": -149.54412841796875, + "loss": 1.0613, + "margin_dpo/margin_mean": 21.837169647216797, + "margin_dpo/margin_std": 32.29937744140625, + "step": 246 + }, + { + "KL/chosen_KL_mean": -38.533203125, + "KL/mean": -44.415794372558594, + "KL/rejected_KL_mean": -50.29838562011719, + "KL/std": 27.172008514404297, + "epoch": 0.37339380196523053, + "fcm_dpo/beta": 0.025636455044150352, + "fcm_dpo/delta": 0.10047095268964767, + "fcm_dpo/margin": 11.765192985534668, + "fcm_dpo/q_t": 0.4332590401172638, + "grad_norm": 18.015737533569336, + "learning_rate": 3.960892420986177e-07, + "logits/chosen": 0.5601674318313599, + "logits/rejected": 0.5501619577407837, + "logps/chosen": -118.55574798583984, + "logps/ref_chosen": -80.02254486083984, + "logps/ref_rejected": -89.22705841064453, + "logps/rejected": -139.52545166015625, + "loss": 1.2296, + "margin_dpo/margin_mean": 11.765192985534668, + "margin_dpo/margin_std": 27.990760803222656, + "step": 247 + }, + { + "KL/chosen_KL_mean": -32.911590576171875, + "KL/mean": -44.313560485839844, + "KL/rejected_KL_mean": -55.71553421020508, + "KL/std": 30.605388641357422, + "epoch": 0.3749055177626606, + "fcm_dpo/beta": 0.02507779374718666, + "fcm_dpo/delta": -0.1818971335887909, + "fcm_dpo/margin": 22.803951263427734, + "fcm_dpo/q_t": 0.3799017667770386, + "grad_norm": 15.453660011291504, + "learning_rate": 3.9501422739279953e-07, + "logits/chosen": 0.5543103218078613, + "logits/rejected": 0.5999008417129517, + "logps/chosen": -98.28955078125, + "logps/ref_chosen": -65.37796020507812, + "logps/ref_rejected": -61.365787506103516, + "logps/rejected": -117.0813217163086, + "loss": 1.0584, + "margin_dpo/margin_mean": 22.803951263427734, + "margin_dpo/margin_std": 34.07537841796875, + "step": 248 + }, + { + "KL/chosen_KL_mean": -43.59640121459961, + "KL/mean": -48.266441345214844, + "KL/rejected_KL_mean": -52.936485290527344, + "KL/std": 29.317392349243164, + "epoch": 0.3764172335600907, + "fcm_dpo/beta": 0.025514479726552963, + "fcm_dpo/delta": 0.16352643072605133, + "fcm_dpo/margin": 9.340082168579102, + "fcm_dpo/q_t": 0.447033166885376, + "grad_norm": 22.10836410522461, + "learning_rate": 3.9393515632731094e-07, + "logits/chosen": 0.546309232711792, + "logits/rejected": 0.582387387752533, + "logps/chosen": -118.19786071777344, + "logps/ref_chosen": -74.60145568847656, + "logps/ref_rejected": -63.79338455200195, + "logps/rejected": -116.72987365722656, + "loss": 1.3176, + "margin_dpo/margin_mean": 9.340083122253418, + "margin_dpo/margin_std": 31.282310485839844, + "step": 249 + }, + { + "KL/chosen_KL_mean": -37.384254455566406, + "KL/mean": -49.02488708496094, + "KL/rejected_KL_mean": -60.66551971435547, + "KL/std": 30.749019622802734, + "epoch": 0.3779289493575208, + "fcm_dpo/beta": 0.02490631863474846, + "fcm_dpo/delta": -0.19055120646953583, + "fcm_dpo/margin": 23.281265258789062, + "fcm_dpo/q_t": 0.37171927094459534, + "grad_norm": 15.915420532226562, + "learning_rate": 3.9285205908608934e-07, + "logits/chosen": 0.6429681181907654, + "logits/rejected": 0.6006037592887878, + "logps/chosen": -99.32246398925781, + "logps/ref_chosen": -61.938209533691406, + "logps/ref_rejected": -72.21602630615234, + "logps/rejected": -132.8815460205078, + "loss": 1.0288, + "margin_dpo/margin_mean": 23.281265258789062, + "margin_dpo/margin_std": 31.80821418762207, + "step": 250 + }, + { + "KL/chosen_KL_mean": -42.976444244384766, + "KL/mean": -50.41681671142578, + "KL/rejected_KL_mean": -57.85719680786133, + "KL/std": 29.169586181640625, + "epoch": 0.3794406651549509, + "fcm_dpo/beta": 0.024686751887202263, + "fcm_dpo/delta": 0.03389812633395195, + "fcm_dpo/margin": 14.880743980407715, + "fcm_dpo/q_t": 0.4203200936317444, + "grad_norm": 23.5372371673584, + "learning_rate": 3.9176496596569265e-07, + "logits/chosen": 0.6521850824356079, + "logits/rejected": 0.6113159656524658, + "logps/chosen": -109.83338928222656, + "logps/ref_chosen": -66.85694885253906, + "logps/ref_rejected": -84.83396911621094, + "logps/rejected": -142.691162109375, + "loss": 1.2014, + "margin_dpo/margin_mean": 14.880743026733398, + "margin_dpo/margin_std": 32.66436004638672, + "step": 251 + }, + { + "KL/chosen_KL_mean": -39.951820373535156, + "KL/mean": -47.22810363769531, + "KL/rejected_KL_mean": -54.50439453125, + "KL/std": 31.698406219482422, + "epoch": 0.38095238095238093, + "fcm_dpo/beta": 0.025234483182430267, + "fcm_dpo/delta": 0.02930210530757904, + "fcm_dpo/margin": 14.55257511138916, + "fcm_dpo/q_t": 0.42153429985046387, + "grad_norm": 27.598159790039062, + "learning_rate": 3.9067390737445254e-07, + "logits/chosen": 0.5371665954589844, + "logits/rejected": 0.4842592477798462, + "logps/chosen": -96.17575073242188, + "logps/ref_chosen": -56.22393035888672, + "logps/ref_rejected": -77.1136245727539, + "logps/rejected": -131.61801147460938, + "loss": 1.2388, + "margin_dpo/margin_mean": 14.552577018737793, + "margin_dpo/margin_std": 33.91108703613281, + "step": 252 + }, + { + "KL/chosen_KL_mean": -42.02704620361328, + "KL/mean": -50.181373596191406, + "KL/rejected_KL_mean": -58.3357048034668, + "KL/std": 31.003231048583984, + "epoch": 0.382464096749811, + "fcm_dpo/beta": 0.024656172841787338, + "fcm_dpo/delta": -0.004082784056663513, + "fcm_dpo/margin": 16.308658599853516, + "fcm_dpo/q_t": 0.4137699604034424, + "grad_norm": 19.795211791992188, + "learning_rate": 3.8957891383162304e-07, + "logits/chosen": 0.618567705154419, + "logits/rejected": 0.5784329175949097, + "logps/chosen": -94.237060546875, + "logps/ref_chosen": -52.21001434326172, + "logps/ref_rejected": -58.75764846801758, + "logps/rejected": -117.09335327148438, + "loss": 1.1667, + "margin_dpo/margin_mean": 16.308656692504883, + "margin_dpo/margin_std": 30.997234344482422, + "step": 253 + }, + { + "KL/chosen_KL_mean": -43.96406555175781, + "KL/mean": -53.92585754394531, + "KL/rejected_KL_mean": -63.88764190673828, + "KL/std": 32.45115280151367, + "epoch": 0.3839758125472411, + "fcm_dpo/beta": 0.02441282756626606, + "fcm_dpo/delta": -0.09193407744169235, + "fcm_dpo/margin": 19.923580169677734, + "fcm_dpo/q_t": 0.39709532260894775, + "grad_norm": 16.3914737701416, + "learning_rate": 3.884800159665276e-07, + "logits/chosen": 0.5786576271057129, + "logits/rejected": 0.5283867716789246, + "logps/chosen": -109.60038757324219, + "logps/ref_chosen": -65.63632202148438, + "logps/ref_rejected": -82.34425354003906, + "logps/rejected": -146.2318878173828, + "loss": 1.1015, + "margin_dpo/margin_mean": 19.923580169677734, + "margin_dpo/margin_std": 32.648799896240234, + "step": 254 + }, + { + "KL/chosen_KL_mean": -42.81669998168945, + "KL/mean": -53.882850646972656, + "KL/rejected_KL_mean": -64.94898986816406, + "KL/std": 33.449005126953125, + "epoch": 0.3854875283446712, + "fcm_dpo/beta": 0.023891150951385498, + "fcm_dpo/delta": -0.13633979856967926, + "fcm_dpo/margin": 22.132293701171875, + "fcm_dpo/q_t": 0.38834911584854126, + "grad_norm": 22.88119888305664, + "learning_rate": 3.873772445177015e-07, + "logits/chosen": 0.5677551627159119, + "logits/rejected": 0.538213849067688, + "logps/chosen": -110.72779083251953, + "logps/ref_chosen": -67.91108703613281, + "logps/ref_rejected": -83.89114379882812, + "logps/rejected": -148.84014892578125, + "loss": 1.0954, + "margin_dpo/margin_mean": 22.132293701171875, + "margin_dpo/margin_std": 36.88736343383789, + "step": 255 + }, + { + "KL/chosen_KL_mean": -49.34206008911133, + "KL/mean": -59.96579360961914, + "KL/rejected_KL_mean": -70.58953094482422, + "KL/std": 30.069900512695312, + "epoch": 0.3869992441421013, + "fcm_dpo/beta": 0.023339707404375076, + "fcm_dpo/delta": -0.10149209201335907, + "fcm_dpo/margin": 21.247478485107422, + "fcm_dpo/q_t": 0.39474910497665405, + "grad_norm": 19.329296112060547, + "learning_rate": 3.862706303320329e-07, + "logits/chosen": 0.5633618831634521, + "logits/rejected": 0.5043342113494873, + "logps/chosen": -112.842041015625, + "logps/ref_chosen": -63.49998474121094, + "logps/ref_rejected": -90.77104187011719, + "logps/rejected": -161.36056518554688, + "loss": 1.1258, + "margin_dpo/margin_mean": 21.247478485107422, + "margin_dpo/margin_std": 37.96196746826172, + "step": 256 + }, + { + "KL/chosen_KL_mean": -49.642337799072266, + "KL/mean": -61.384517669677734, + "KL/rejected_KL_mean": -73.12670135498047, + "KL/std": 34.05942916870117, + "epoch": 0.3885109599395314, + "fcm_dpo/beta": 0.022757742553949356, + "fcm_dpo/delta": -0.1424705535173416, + "fcm_dpo/margin": 23.484365463256836, + "fcm_dpo/q_t": 0.38810110092163086, + "grad_norm": 17.054840087890625, + "learning_rate": 3.851602043638994e-07, + "logits/chosen": 0.5620399713516235, + "logits/rejected": 0.4995781183242798, + "logps/chosen": -120.24298858642578, + "logps/ref_chosen": -70.60064697265625, + "logps/ref_rejected": -108.58313751220703, + "logps/rejected": -181.7098388671875, + "loss": 1.09, + "margin_dpo/margin_mean": 23.484363555908203, + "margin_dpo/margin_std": 38.58222198486328, + "step": 257 + }, + { + "KL/chosen_KL_mean": -48.516571044921875, + "KL/mean": -59.06629943847656, + "KL/rejected_KL_mean": -69.61604309082031, + "KL/std": 30.164875030517578, + "epoch": 0.3900226757369615, + "fcm_dpo/beta": 0.022407684475183487, + "fcm_dpo/delta": -0.07634987682104111, + "fcm_dpo/margin": 21.099472045898438, + "fcm_dpo/q_t": 0.39316412806510925, + "grad_norm": 16.7133731842041, + "learning_rate": 3.840459976743023e-07, + "logits/chosen": 0.6331174373626709, + "logits/rejected": 0.5832922458648682, + "logps/chosen": -107.77073669433594, + "logps/ref_chosen": -59.25416564941406, + "logps/ref_rejected": -85.58709716796875, + "logps/rejected": -155.203125, + "loss": 1.0633, + "margin_dpo/margin_mean": 21.099475860595703, + "margin_dpo/margin_std": 28.931100845336914, + "step": 258 + }, + { + "KL/chosen_KL_mean": -39.62800216674805, + "KL/mean": -55.97937774658203, + "KL/rejected_KL_mean": -72.33076477050781, + "KL/std": 34.773963928222656, + "epoch": 0.3915343915343915, + "fcm_dpo/beta": 0.02117306925356388, + "fcm_dpo/delta": -0.3162829279899597, + "fcm_dpo/margin": 32.70275115966797, + "fcm_dpo/q_t": 0.3467109799385071, + "grad_norm": 14.047627449035645, + "learning_rate": 3.8292804142999796e-07, + "logits/chosen": 0.5472550392150879, + "logits/rejected": 0.4483594298362732, + "logps/chosen": -105.06288146972656, + "logps/ref_chosen": -65.43487548828125, + "logps/ref_rejected": -95.41731262207031, + "logps/rejected": -167.74807739257812, + "loss": 0.9574, + "margin_dpo/margin_mean": 32.70275115966797, + "margin_dpo/margin_std": 38.047218322753906, + "step": 259 + }, + { + "KL/chosen_KL_mean": -40.95439910888672, + "KL/mean": -53.225013732910156, + "KL/rejected_KL_mean": -65.4956283569336, + "KL/std": 32.90972900390625, + "epoch": 0.3930461073318216, + "fcm_dpo/beta": 0.02049969881772995, + "fcm_dpo/delta": -0.1094924658536911, + "fcm_dpo/margin": 24.54123306274414, + "fcm_dpo/q_t": 0.3917416036128998, + "grad_norm": 16.254623413085938, + "learning_rate": 3.818063669026256e-07, + "logits/chosen": 0.5976210832595825, + "logits/rejected": 0.513933002948761, + "logps/chosen": -90.04397583007812, + "logps/ref_chosen": -49.08958435058594, + "logps/ref_rejected": -79.01708221435547, + "logps/rejected": -144.51271057128906, + "loss": 1.0999, + "margin_dpo/margin_mean": 24.54123306274414, + "margin_dpo/margin_std": 40.160011291503906, + "step": 260 + }, + { + "KL/chosen_KL_mean": -42.14891815185547, + "KL/mean": -52.500186920166016, + "KL/rejected_KL_mean": -62.85145568847656, + "KL/std": 31.833454132080078, + "epoch": 0.3945578231292517, + "fcm_dpo/beta": 0.02039218880236149, + "fcm_dpo/delta": -0.023179400712251663, + "fcm_dpo/margin": 20.702537536621094, + "fcm_dpo/q_t": 0.407938152551651, + "grad_norm": 17.459196090698242, + "learning_rate": 3.806810054678331e-07, + "logits/chosen": 0.47628605365753174, + "logits/rejected": 0.5041095018386841, + "logps/chosen": -113.02131652832031, + "logps/ref_chosen": -70.87239074707031, + "logps/ref_rejected": -65.01522064208984, + "logps/rejected": -127.8666763305664, + "loss": 1.1258, + "margin_dpo/margin_mean": 20.702537536621094, + "margin_dpo/margin_std": 35.16142272949219, + "step": 261 + }, + { + "KL/chosen_KL_mean": -41.45574951171875, + "KL/mean": -53.318363189697266, + "KL/rejected_KL_mean": -65.18097686767578, + "KL/std": 30.827220916748047, + "epoch": 0.3960695389266818, + "fcm_dpo/beta": 0.020171720534563065, + "fcm_dpo/delta": -0.0824466198682785, + "fcm_dpo/margin": 23.725223541259766, + "fcm_dpo/q_t": 0.3938416838645935, + "grad_norm": 14.521263122558594, + "learning_rate": 3.7955198860439887e-07, + "logits/chosen": 0.6390868425369263, + "logits/rejected": 0.5787541270256042, + "logps/chosen": -109.32637786865234, + "logps/ref_chosen": -67.8706283569336, + "logps/ref_rejected": -88.7205810546875, + "logps/rejected": -153.90155029296875, + "loss": 1.0722, + "margin_dpo/margin_mean": 23.725223541259766, + "margin_dpo/margin_std": 34.007354736328125, + "step": 262 + }, + { + "KL/chosen_KL_mean": -40.47084045410156, + "KL/mean": -50.60345458984375, + "KL/rejected_KL_mean": -60.7360725402832, + "KL/std": 32.10907745361328, + "epoch": 0.3975812547241119, + "fcm_dpo/beta": 0.020096808671951294, + "fcm_dpo/delta": -0.007788047194480896, + "fcm_dpo/margin": 20.265230178833008, + "fcm_dpo/q_t": 0.40929681062698364, + "grad_norm": 15.321991920471191, + "learning_rate": 3.784193478933516e-07, + "logits/chosen": 0.5773823261260986, + "logits/rejected": 0.4729832410812378, + "logps/chosen": -95.66542053222656, + "logps/ref_chosen": -55.194583892822266, + "logps/ref_rejected": -80.54048156738281, + "logps/rejected": -141.27655029296875, + "loss": 1.134, + "margin_dpo/margin_mean": 20.265230178833008, + "margin_dpo/margin_std": 34.80146789550781, + "step": 263 + }, + { + "KL/chosen_KL_mean": -41.95197296142578, + "KL/mean": -53.48637390136719, + "KL/rejected_KL_mean": -65.0207748413086, + "KL/std": 33.68387222290039, + "epoch": 0.39909297052154197, + "fcm_dpo/beta": 0.019823521375656128, + "fcm_dpo/delta": -0.06012331694364548, + "fcm_dpo/margin": 23.068805694580078, + "fcm_dpo/q_t": 0.3996187448501587, + "grad_norm": 15.330732345581055, + "learning_rate": 3.7728311501708674e-07, + "logits/chosen": 0.5034226775169373, + "logits/rejected": 0.4578082263469696, + "logps/chosen": -125.1226577758789, + "logps/ref_chosen": -83.17068481445312, + "logps/ref_rejected": -88.33625793457031, + "logps/rejected": -153.35702514648438, + "loss": 1.1033, + "margin_dpo/margin_mean": 23.068805694580078, + "margin_dpo/margin_std": 37.05900573730469, + "step": 264 + }, + { + "KL/chosen_KL_mean": -38.964439392089844, + "KL/mean": -52.6373291015625, + "KL/rejected_KL_mean": -66.31021118164062, + "KL/std": 35.45323944091797, + "epoch": 0.40060468631897206, + "fcm_dpo/beta": 0.01950901746749878, + "fcm_dpo/delta": -0.14103645086288452, + "fcm_dpo/margin": 27.345767974853516, + "fcm_dpo/q_t": 0.38523828983306885, + "grad_norm": 15.546408653259277, + "learning_rate": 3.7614332175848027e-07, + "logits/chosen": 0.678200900554657, + "logits/rejected": 0.6123419404029846, + "logps/chosen": -90.62728881835938, + "logps/ref_chosen": -51.66284942626953, + "logps/ref_rejected": -67.1720962524414, + "logps/rejected": -133.48231506347656, + "loss": 1.0887, + "margin_dpo/margin_mean": 27.34576988220215, + "margin_dpo/margin_std": 43.287872314453125, + "step": 265 + }, + { + "KL/chosen_KL_mean": -37.169830322265625, + "KL/mean": -49.80873107910156, + "KL/rejected_KL_mean": -62.447635650634766, + "KL/std": 36.28041076660156, + "epoch": 0.4021164021164021, + "fcm_dpo/beta": 0.018952175974845886, + "fcm_dpo/delta": -0.08332791179418564, + "fcm_dpo/margin": 25.277809143066406, + "fcm_dpo/q_t": 0.39448273181915283, + "grad_norm": 14.954691886901855, + "learning_rate": 3.75e-07, + "logits/chosen": 0.5855690240859985, + "logits/rejected": 0.5146670341491699, + "logps/chosen": -94.62032318115234, + "logps/ref_chosen": -57.45049285888672, + "logps/ref_rejected": -77.60826110839844, + "logps/rejected": -140.05589294433594, + "loss": 1.0813, + "margin_dpo/margin_mean": 25.277809143066406, + "margin_dpo/margin_std": 38.019737243652344, + "step": 266 + }, + { + "KL/chosen_KL_mean": -32.33792495727539, + "KL/mean": -41.74827575683594, + "KL/rejected_KL_mean": -51.158626556396484, + "KL/std": 32.253929138183594, + "epoch": 0.4036281179138322, + "fcm_dpo/beta": 0.019205017015337944, + "fcm_dpo/delta": 0.03771677613258362, + "fcm_dpo/margin": 18.82069969177246, + "fcm_dpo/q_t": 0.4196903705596924, + "grad_norm": 15.400888442993164, + "learning_rate": 3.738531817228131e-07, + "logits/chosen": 0.6113137006759644, + "logits/rejected": 0.5943053960800171, + "logps/chosen": -87.37327575683594, + "logps/ref_chosen": -55.03535079956055, + "logps/ref_rejected": -66.0953369140625, + "logps/rejected": -117.25395965576172, + "loss": 1.1901, + "margin_dpo/margin_mean": 18.82069969177246, + "margin_dpo/margin_std": 37.8029899597168, + "step": 267 + }, + { + "KL/chosen_KL_mean": -32.36951446533203, + "KL/mean": -44.62208557128906, + "KL/rejected_KL_mean": -56.874656677246094, + "KL/std": 35.25495910644531, + "epoch": 0.4051398337112623, + "fcm_dpo/beta": 0.01891086995601654, + "fcm_dpo/delta": -0.06644882261753082, + "fcm_dpo/margin": 24.505144119262695, + "fcm_dpo/q_t": 0.3967518210411072, + "grad_norm": 13.332362174987793, + "learning_rate": 3.7270289900589204e-07, + "logits/chosen": 0.4891277849674225, + "logits/rejected": 0.47358354926109314, + "logps/chosen": -97.44125366210938, + "logps/ref_chosen": -65.07174682617188, + "logps/ref_rejected": -71.42485809326172, + "logps/rejected": -128.2995147705078, + "loss": 1.0643, + "margin_dpo/margin_mean": 24.505144119262695, + "margin_dpo/margin_std": 32.81787872314453, + "step": 268 + }, + { + "KL/chosen_KL_mean": -37.45969772338867, + "KL/mean": -51.041168212890625, + "KL/rejected_KL_mean": -64.62263488769531, + "KL/std": 36.909271240234375, + "epoch": 0.40665154950869237, + "fcm_dpo/beta": 0.018644198775291443, + "fcm_dpo/delta": -0.11254137754440308, + "fcm_dpo/margin": 27.162946701049805, + "fcm_dpo/q_t": 0.3878785967826843, + "grad_norm": 14.419161796569824, + "learning_rate": 3.7154918402511714e-07, + "logits/chosen": 0.6810588836669922, + "logits/rejected": 0.6340845823287964, + "logps/chosen": -104.59590148925781, + "logps/ref_chosen": -67.1362075805664, + "logps/ref_rejected": -82.55778503417969, + "logps/rejected": -147.180419921875, + "loss": 1.0505, + "margin_dpo/margin_mean": 27.162946701049805, + "margin_dpo/margin_std": 35.8624382019043, + "step": 269 + }, + { + "KL/chosen_KL_mean": -38.30808639526367, + "KL/mean": -48.73957061767578, + "KL/rejected_KL_mean": -59.17105484008789, + "KL/std": 36.70610046386719, + "epoch": 0.40816326530612246, + "fcm_dpo/beta": 0.01835768297314644, + "fcm_dpo/delta": 0.017404425889253616, + "fcm_dpo/margin": 20.86296844482422, + "fcm_dpo/q_t": 0.4143076539039612, + "grad_norm": 14.80038070678711, + "learning_rate": 3.7039206905237656e-07, + "logits/chosen": 0.6649652719497681, + "logits/rejected": 0.5865774750709534, + "logps/chosen": -104.99678039550781, + "logps/ref_chosen": -66.6886978149414, + "logps/ref_rejected": -85.16129302978516, + "logps/rejected": -144.3323516845703, + "loss": 1.1578, + "margin_dpo/margin_mean": 20.86296844482422, + "margin_dpo/margin_std": 38.87541198730469, + "step": 270 + }, + { + "KL/chosen_KL_mean": -36.99169921875, + "KL/mean": -46.25218963623047, + "KL/rejected_KL_mean": -55.51268005371094, + "KL/std": 37.55319595336914, + "epoch": 0.40967498110355255, + "fcm_dpo/beta": 0.018530219793319702, + "fcm_dpo/delta": 0.05868082866072655, + "fcm_dpo/margin": 18.520978927612305, + "fcm_dpo/q_t": 0.43027639389038086, + "grad_norm": 16.319501876831055, + "learning_rate": 3.692315864546635e-07, + "logits/chosen": 0.6354309320449829, + "logits/rejected": 0.5745704174041748, + "logps/chosen": -109.39924621582031, + "logps/ref_chosen": -72.40754699707031, + "logps/ref_rejected": -92.06311798095703, + "logps/rejected": -147.5758056640625, + "loss": 1.2254, + "margin_dpo/margin_mean": 18.520980834960938, + "margin_dpo/margin_std": 44.24129104614258, + "step": 271 + }, + { + "KL/chosen_KL_mean": -29.994138717651367, + "KL/mean": -48.404571533203125, + "KL/rejected_KL_mean": -66.81500244140625, + "KL/std": 36.747406005859375, + "epoch": 0.41118669690098264, + "fcm_dpo/beta": 0.017964601516723633, + "fcm_dpo/delta": -0.2793579697608948, + "fcm_dpo/margin": 36.820865631103516, + "fcm_dpo/q_t": 0.35114264488220215, + "grad_norm": 16.6761531829834, + "learning_rate": 3.6806776869317067e-07, + "logits/chosen": 0.6202855706214905, + "logits/rejected": 0.6367502212524414, + "logps/chosen": -96.59553527832031, + "logps/ref_chosen": -66.60140228271484, + "logps/ref_rejected": -67.74340057373047, + "logps/rejected": -134.55841064453125, + "loss": 0.9196, + "margin_dpo/margin_mean": 36.82086181640625, + "margin_dpo/margin_std": 35.04280090332031, + "step": 272 + }, + { + "KL/chosen_KL_mean": -44.28934097290039, + "KL/mean": -56.83588790893555, + "KL/rejected_KL_mean": -69.38243865966797, + "KL/std": 39.201812744140625, + "epoch": 0.4126984126984127, + "fcm_dpo/beta": 0.017505496740341187, + "fcm_dpo/delta": -0.04126958176493645, + "fcm_dpo/margin": 25.093097686767578, + "fcm_dpo/q_t": 0.4040055572986603, + "grad_norm": 17.27816390991211, + "learning_rate": 3.669006483223828e-07, + "logits/chosen": 0.6422320604324341, + "logits/rejected": 0.5763329267501831, + "logps/chosen": -101.64421844482422, + "logps/ref_chosen": -57.35487747192383, + "logps/ref_rejected": -84.17168426513672, + "logps/rejected": -153.5541229248047, + "loss": 1.165, + "margin_dpo/margin_mean": 25.093097686767578, + "margin_dpo/margin_std": 49.38603973388672, + "step": 273 + }, + { + "KL/chosen_KL_mean": -43.80359649658203, + "KL/mean": -57.957054138183594, + "KL/rejected_KL_mean": -72.11050415039062, + "KL/std": 40.29290771484375, + "epoch": 0.41421012849584277, + "fcm_dpo/beta": 0.0172972921282053, + "fcm_dpo/delta": -0.09414344280958176, + "fcm_dpo/margin": 28.306915283203125, + "fcm_dpo/q_t": 0.39450687170028687, + "grad_norm": 14.509082794189453, + "learning_rate": 3.657302579891656e-07, + "logits/chosen": 0.548915684223175, + "logits/rejected": 0.5313537120819092, + "logps/chosen": -103.4450912475586, + "logps/ref_chosen": -59.64149475097656, + "logps/ref_rejected": -68.29348754882812, + "logps/rejected": -140.40399169921875, + "loss": 1.1079, + "margin_dpo/margin_mean": 28.306915283203125, + "margin_dpo/margin_std": 47.45249557495117, + "step": 274 + }, + { + "KL/chosen_KL_mean": -42.47492980957031, + "KL/mean": -57.8199462890625, + "KL/rejected_KL_mean": -73.16497039794922, + "KL/std": 39.48030090332031, + "epoch": 0.41572184429327286, + "fcm_dpo/beta": 0.016921717673540115, + "fcm_dpo/delta": -0.1256747543811798, + "fcm_dpo/margin": 30.690044403076172, + "fcm_dpo/q_t": 0.38423293828964233, + "grad_norm": 14.718132972717285, + "learning_rate": 3.645566304318526e-07, + "logits/chosen": 0.6262120008468628, + "logits/rejected": 0.544623851776123, + "logps/chosen": -95.7415771484375, + "logps/ref_chosen": -53.26664352416992, + "logps/ref_rejected": -73.84062194824219, + "logps/rejected": -147.00558471679688, + "loss": 1.049, + "margin_dpo/margin_mean": 30.690044403076172, + "margin_dpo/margin_std": 42.01318359375, + "step": 275 + }, + { + "KL/chosen_KL_mean": -40.822547912597656, + "KL/mean": -54.688377380371094, + "KL/rejected_KL_mean": -68.55419921875, + "KL/std": 38.242088317871094, + "epoch": 0.41723356009070295, + "fcm_dpo/beta": 0.016595926135778427, + "fcm_dpo/delta": -0.06314849853515625, + "fcm_dpo/margin": 27.73164939880371, + "fcm_dpo/q_t": 0.3971312642097473, + "grad_norm": 16.296781539916992, + "learning_rate": 3.633797984793294e-07, + "logits/chosen": 0.5706058144569397, + "logits/rejected": 0.538550615310669, + "logps/chosen": -93.84334564208984, + "logps/ref_chosen": -53.02079772949219, + "logps/ref_rejected": -61.56678771972656, + "logps/rejected": -130.12098693847656, + "loss": 1.0897, + "margin_dpo/margin_mean": 27.73164939880371, + "margin_dpo/margin_std": 41.87763214111328, + "step": 276 + }, + { + "KL/chosen_KL_mean": -45.02290344238281, + "KL/mean": -54.18437957763672, + "KL/rejected_KL_mean": -63.345855712890625, + "KL/std": 37.87938690185547, + "epoch": 0.41874527588813304, + "fcm_dpo/beta": 0.01680148020386696, + "fcm_dpo/delta": 0.09492812305688858, + "fcm_dpo/margin": 18.322956085205078, + "fcm_dpo/q_t": 0.4322887063026428, + "grad_norm": 18.618850708007812, + "learning_rate": 3.6219979505011555e-07, + "logits/chosen": 0.6885474920272827, + "logits/rejected": 0.7152642607688904, + "logps/chosen": -116.45588684082031, + "logps/ref_chosen": -71.43299102783203, + "logps/ref_rejected": -67.65852355957031, + "logps/rejected": -131.00437927246094, + "loss": 1.2154, + "margin_dpo/margin_mean": 18.322956085205078, + "margin_dpo/margin_std": 41.572853088378906, + "step": 277 + }, + { + "KL/chosen_KL_mean": -50.65361785888672, + "KL/mean": -64.87567901611328, + "KL/rejected_KL_mean": -79.09774780273438, + "KL/std": 43.28910827636719, + "epoch": 0.42025699168556313, + "fcm_dpo/beta": 0.016770116984844208, + "fcm_dpo/delta": -0.08121868222951889, + "fcm_dpo/margin": 28.44412612915039, + "fcm_dpo/q_t": 0.3952862620353699, + "grad_norm": 23.123279571533203, + "learning_rate": 3.6101665315144353e-07, + "logits/chosen": 0.5624623894691467, + "logits/rejected": 0.5138440132141113, + "logps/chosen": -117.76438903808594, + "logps/ref_chosen": -67.11076354980469, + "logps/ref_rejected": -88.74851989746094, + "logps/rejected": -167.8462677001953, + "loss": 1.1019, + "margin_dpo/margin_mean": 28.44412612915039, + "margin_dpo/margin_std": 45.58794021606445, + "step": 278 + }, + { + "KL/chosen_KL_mean": -36.818626403808594, + "KL/mean": -56.46868896484375, + "KL/rejected_KL_mean": -76.11874389648438, + "KL/std": 42.45269775390625, + "epoch": 0.4217687074829932, + "fcm_dpo/beta": 0.016032833606004715, + "fcm_dpo/delta": -0.24514293670654297, + "fcm_dpo/margin": 39.30011749267578, + "fcm_dpo/q_t": 0.3587535619735718, + "grad_norm": 18.80735206604004, + "learning_rate": 3.5983040587833563e-07, + "logits/chosen": 0.5911531448364258, + "logits/rejected": 0.5552926063537598, + "logps/chosen": -91.31611633300781, + "logps/ref_chosen": -54.49748611450195, + "logps/ref_rejected": -70.42373657226562, + "logps/rejected": -146.54248046875, + "loss": 0.9504, + "margin_dpo/margin_mean": 39.30011749267578, + "margin_dpo/margin_std": 40.607357025146484, + "step": 279 + }, + { + "KL/chosen_KL_mean": -40.82258224487305, + "KL/mean": -61.65607452392578, + "KL/rejected_KL_mean": -82.48957061767578, + "KL/std": 44.68724822998047, + "epoch": 0.42328042328042326, + "fcm_dpo/beta": 0.015228111296892166, + "fcm_dpo/delta": -0.2502999007701874, + "fcm_dpo/margin": 41.6669921875, + "fcm_dpo/q_t": 0.3581681251525879, + "grad_norm": 11.828532218933105, + "learning_rate": 3.586410864126781e-07, + "logits/chosen": 0.6419994235038757, + "logits/rejected": 0.6040011644363403, + "logps/chosen": -101.2553939819336, + "logps/ref_chosen": -60.43281173706055, + "logps/ref_rejected": -78.39051818847656, + "logps/rejected": -160.88009643554688, + "loss": 0.9388, + "margin_dpo/margin_mean": 41.6669921875, + "margin_dpo/margin_std": 41.892921447753906, + "step": 280 + }, + { + "KL/chosen_KL_mean": -45.86112976074219, + "KL/mean": -64.68388366699219, + "KL/rejected_KL_mean": -83.50663757324219, + "KL/std": 44.603004455566406, + "epoch": 0.42479213907785335, + "fcm_dpo/beta": 0.014688611030578613, + "fcm_dpo/delta": -0.1617349535226822, + "fcm_dpo/margin": 37.64551544189453, + "fcm_dpo/q_t": 0.37709325551986694, + "grad_norm": 12.660872459411621, + "learning_rate": 3.574487280222929e-07, + "logits/chosen": 0.5922428965568542, + "logits/rejected": 0.6139553785324097, + "logps/chosen": -106.14321899414062, + "logps/ref_chosen": -60.2820930480957, + "logps/ref_rejected": -62.04009246826172, + "logps/rejected": -145.54673767089844, + "loss": 1.0202, + "margin_dpo/margin_mean": 37.64551544189453, + "margin_dpo/margin_std": 47.75347137451172, + "step": 281 + }, + { + "KL/chosen_KL_mean": -52.35889434814453, + "KL/mean": -68.90093994140625, + "KL/rejected_KL_mean": -85.44297790527344, + "KL/std": 44.76057434082031, + "epoch": 0.42630385487528344, + "fcm_dpo/beta": 0.014499800279736519, + "fcm_dpo/delta": -0.08470294624567032, + "fcm_dpo/margin": 33.084075927734375, + "fcm_dpo/q_t": 0.39539259672164917, + "grad_norm": 18.454259872436523, + "learning_rate": 3.562533640600075e-07, + "logits/chosen": 0.5745443105697632, + "logits/rejected": 0.5282710790634155, + "logps/chosen": -112.98281860351562, + "logps/ref_chosen": -60.623924255371094, + "logps/ref_rejected": -68.67400360107422, + "logps/rejected": -154.11697387695312, + "loss": 1.097, + "margin_dpo/margin_mean": 33.084075927734375, + "margin_dpo/margin_std": 51.07267379760742, + "step": 282 + }, + { + "KL/chosen_KL_mean": -53.396873474121094, + "KL/mean": -68.42160034179688, + "KL/rejected_KL_mean": -83.44633483886719, + "KL/std": 43.950477600097656, + "epoch": 0.42781557067271353, + "fcm_dpo/beta": 0.014289552345871925, + "fcm_dpo/delta": -0.030874045565724373, + "fcm_dpo/margin": 30.049449920654297, + "fcm_dpo/q_t": 0.4046875834465027, + "grad_norm": 15.984442710876465, + "learning_rate": 3.550550279627215e-07, + "logits/chosen": 0.6232196688652039, + "logits/rejected": 0.525344729423523, + "logps/chosen": -121.04461669921875, + "logps/ref_chosen": -67.64775085449219, + "logps/ref_rejected": -99.96835327148438, + "logps/rejected": -183.41468811035156, + "loss": 1.121, + "margin_dpo/margin_mean": 30.049449920654297, + "margin_dpo/margin_std": 49.89061737060547, + "step": 283 + }, + { + "KL/chosen_KL_mean": -51.50770950317383, + "KL/mean": -67.859619140625, + "KL/rejected_KL_mean": -84.21153259277344, + "KL/std": 47.82459259033203, + "epoch": 0.4293272864701436, + "fcm_dpo/beta": 0.01402394287288189, + "fcm_dpo/delta": -0.06209135428071022, + "fcm_dpo/margin": 32.703819274902344, + "fcm_dpo/q_t": 0.3963380455970764, + "grad_norm": 13.340622901916504, + "learning_rate": 3.5385375325047163e-07, + "logits/chosen": 0.697113037109375, + "logits/rejected": 0.6345241069793701, + "logps/chosen": -108.47513580322266, + "logps/ref_chosen": -56.96742630004883, + "logps/ref_rejected": -86.36236572265625, + "logps/rejected": -170.5738983154297, + "loss": 1.0736, + "margin_dpo/margin_mean": 32.703819274902344, + "margin_dpo/margin_std": 45.35961151123047, + "step": 284 + }, + { + "KL/chosen_KL_mean": -64.46051025390625, + "KL/mean": -78.41537475585938, + "KL/rejected_KL_mean": -92.3702392578125, + "KL/std": 42.90559005737305, + "epoch": 0.4308390022675737, + "fcm_dpo/beta": 0.014100934378802776, + "fcm_dpo/delta": 0.006449030712246895, + "fcm_dpo/margin": 27.90973472595215, + "fcm_dpo/q_t": 0.412301242351532, + "grad_norm": 17.84235191345215, + "learning_rate": 3.5264957352549375e-07, + "logits/chosen": 0.686227560043335, + "logits/rejected": 0.6604204177856445, + "logps/chosen": -136.11660766601562, + "logps/ref_chosen": -71.65611267089844, + "logps/ref_rejected": -81.63829803466797, + "logps/rejected": -174.008544921875, + "loss": 1.1348, + "margin_dpo/margin_mean": 27.909732818603516, + "margin_dpo/margin_std": 47.20280456542969, + "step": 285 + }, + { + "KL/chosen_KL_mean": -58.55393600463867, + "KL/mean": -80.9109878540039, + "KL/rejected_KL_mean": -103.26805114746094, + "KL/std": 46.35716247558594, + "epoch": 0.4323507180650038, + "fcm_dpo/beta": 0.013572895899415016, + "fcm_dpo/delta": -0.2208898961544037, + "fcm_dpo/margin": 44.71410369873047, + "fcm_dpo/q_t": 0.3641088902950287, + "grad_norm": 14.018209457397461, + "learning_rate": 3.514425224712835e-07, + "logits/chosen": 0.5620067715644836, + "logits/rejected": 0.4744671583175659, + "logps/chosen": -119.63345336914062, + "logps/ref_chosen": -61.07952117919922, + "logps/ref_rejected": -91.28128051757812, + "logps/rejected": -194.54931640625, + "loss": 0.9634, + "margin_dpo/margin_mean": 44.71410369873047, + "margin_dpo/margin_std": 48.03024673461914, + "step": 286 + }, + { + "KL/chosen_KL_mean": -47.898338317871094, + "KL/mean": -70.18797302246094, + "KL/rejected_KL_mean": -92.47761535644531, + "KL/std": 49.48754119873047, + "epoch": 0.43386243386243384, + "fcm_dpo/beta": 0.01308109425008297, + "fcm_dpo/delta": -0.19431273639202118, + "fcm_dpo/margin": 44.57927703857422, + "fcm_dpo/q_t": 0.3690979480743408, + "grad_norm": 12.084219932556152, + "learning_rate": 3.502326338516534e-07, + "logits/chosen": 0.6664276719093323, + "logits/rejected": 0.6316767930984497, + "logps/chosen": -93.93412780761719, + "logps/ref_chosen": -46.035789489746094, + "logps/ref_rejected": -59.95293426513672, + "logps/rejected": -152.4305419921875, + "loss": 0.9841, + "margin_dpo/margin_mean": 44.57927703857422, + "margin_dpo/margin_std": 50.418670654296875, + "step": 287 + }, + { + "KL/chosen_KL_mean": -63.46906661987305, + "KL/mean": -79.67216491699219, + "KL/rejected_KL_mean": -95.87527465820312, + "KL/std": 46.347686767578125, + "epoch": 0.43537414965986393, + "fcm_dpo/beta": 0.012921486049890518, + "fcm_dpo/delta": -0.019581297412514687, + "fcm_dpo/margin": 32.406211853027344, + "fcm_dpo/q_t": 0.4060286283493042, + "grad_norm": 14.524436950683594, + "learning_rate": 3.490199415097892e-07, + "logits/chosen": 0.5129345655441284, + "logits/rejected": 0.4621993899345398, + "logps/chosen": -128.8599090576172, + "logps/ref_chosen": -65.3908462524414, + "logps/ref_rejected": -88.53607940673828, + "logps/rejected": -184.41134643554688, + "loss": 1.1137, + "margin_dpo/margin_mean": 32.40620803833008, + "margin_dpo/margin_std": 51.43866729736328, + "step": 288 + }, + { + "KL/chosen_KL_mean": -63.998863220214844, + "KL/mean": -78.66741943359375, + "KL/rejected_KL_mean": -93.33598327636719, + "KL/std": 46.610191345214844, + "epoch": 0.436885865457294, + "fcm_dpo/beta": 0.012985285371541977, + "fcm_dpo/delta": 0.019250046461820602, + "fcm_dpo/margin": 29.337120056152344, + "fcm_dpo/q_t": 0.41725146770477295, + "grad_norm": 16.7989444732666, + "learning_rate": 3.4780447936730247e-07, + "logits/chosen": 0.758312463760376, + "logits/rejected": 0.7211253643035889, + "logps/chosen": -118.59249114990234, + "logps/ref_chosen": -54.5936279296875, + "logps/ref_rejected": -67.20855712890625, + "logps/rejected": -160.54452514648438, + "loss": 1.1555, + "margin_dpo/margin_mean": 29.337120056152344, + "margin_dpo/margin_std": 53.40695571899414, + "step": 289 + }, + { + "KL/chosen_KL_mean": -73.34171295166016, + "KL/mean": -91.50950622558594, + "KL/rejected_KL_mean": -109.67730712890625, + "KL/std": 45.4484977722168, + "epoch": 0.4383975812547241, + "fcm_dpo/beta": 0.01275827456265688, + "fcm_dpo/delta": -0.06723187863826752, + "fcm_dpo/margin": 36.33559036254883, + "fcm_dpo/q_t": 0.396625816822052, + "grad_norm": 16.9279727935791, + "learning_rate": 3.465862814232821e-07, + "logits/chosen": 0.7583713531494141, + "logits/rejected": 0.6876074075698853, + "logps/chosen": -134.72628784179688, + "logps/ref_chosen": -61.38457489013672, + "logps/ref_rejected": -91.92778015136719, + "logps/rejected": -201.60507202148438, + "loss": 1.0937, + "margin_dpo/margin_mean": 36.33559036254883, + "margin_dpo/margin_std": 56.050289154052734, + "step": 290 + }, + { + "KL/chosen_KL_mean": -70.3595962524414, + "KL/mean": -89.24057006835938, + "KL/rejected_KL_mean": -108.12155151367188, + "KL/std": 45.235504150390625, + "epoch": 0.4399092970521542, + "fcm_dpo/beta": 0.012722047045826912, + "fcm_dpo/delta": -0.08546677231788635, + "fcm_dpo/margin": 37.76197052001953, + "fcm_dpo/q_t": 0.3933485150337219, + "grad_norm": 16.51990509033203, + "learning_rate": 3.4536538175334343e-07, + "logits/chosen": 0.8289006352424622, + "logits/rejected": 0.7604660987854004, + "logps/chosen": -121.2226333618164, + "logps/ref_chosen": -50.863037109375, + "logps/ref_rejected": -82.20868682861328, + "logps/rejected": -190.33023071289062, + "loss": 1.0671, + "margin_dpo/margin_mean": 37.761966705322266, + "margin_dpo/margin_std": 50.908905029296875, + "step": 291 + }, + { + "KL/chosen_KL_mean": -69.3774642944336, + "KL/mean": -84.8953857421875, + "KL/rejected_KL_mean": -100.41329956054688, + "KL/std": 48.227535247802734, + "epoch": 0.4414210128495843, + "fcm_dpo/beta": 0.012567687779664993, + "fcm_dpo/delta": 0.010354185476899147, + "fcm_dpo/margin": 31.035823822021484, + "fcm_dpo/q_t": 0.4122801125049591, + "grad_norm": 15.849407196044922, + "learning_rate": 3.4414181450867465e-07, + "logits/chosen": 0.6781643033027649, + "logits/rejected": 0.6310281753540039, + "logps/chosen": -133.72634887695312, + "logps/ref_chosen": -64.34888458251953, + "logps/ref_rejected": -72.86434173583984, + "logps/rejected": -173.27764892578125, + "loss": 1.1477, + "margin_dpo/margin_mean": 31.035825729370117, + "margin_dpo/margin_std": 55.531700134277344, + "step": 292 + }, + { + "KL/chosen_KL_mean": -68.59029388427734, + "KL/mean": -91.03254699707031, + "KL/rejected_KL_mean": -113.47480773925781, + "KL/std": 49.78746795654297, + "epoch": 0.4429327286470144, + "fcm_dpo/beta": 0.012334262020885944, + "fcm_dpo/delta": -0.16231675446033478, + "fcm_dpo/margin": 44.88451385498047, + "fcm_dpo/q_t": 0.3771790862083435, + "grad_norm": 11.519031524658203, + "learning_rate": 3.4291561391508185e-07, + "logits/chosen": 0.7449074983596802, + "logits/rejected": 0.661964476108551, + "logps/chosen": -123.45976257324219, + "logps/ref_chosen": -54.869468688964844, + "logps/ref_rejected": -81.858642578125, + "logps/rejected": -195.3334503173828, + "loss": 1.0391, + "margin_dpo/margin_mean": 44.88451385498047, + "margin_dpo/margin_std": 60.664588928222656, + "step": 293 + }, + { + "KL/chosen_KL_mean": -71.3341064453125, + "KL/mean": -86.29549407958984, + "KL/rejected_KL_mean": -101.25688171386719, + "KL/std": 51.49919509887695, + "epoch": 0.4444444444444444, + "fcm_dpo/beta": 0.012190229259431362, + "fcm_dpo/delta": 0.03639962524175644, + "fcm_dpo/margin": 29.922760009765625, + "fcm_dpo/q_t": 0.4193214476108551, + "grad_norm": 12.29130744934082, + "learning_rate": 3.4168681427203153e-07, + "logits/chosen": 0.717154860496521, + "logits/rejected": 0.6727120280265808, + "logps/chosen": -128.0050048828125, + "logps/ref_chosen": -56.670902252197266, + "logps/ref_rejected": -70.32819366455078, + "logps/rejected": -171.58506774902344, + "loss": 1.1385, + "margin_dpo/margin_mean": 29.922760009765625, + "margin_dpo/margin_std": 49.23522186279297, + "step": 294 + }, + { + "KL/chosen_KL_mean": -77.55728149414062, + "KL/mean": -91.69718933105469, + "KL/rejected_KL_mean": -105.83709716796875, + "KL/std": 49.87922668457031, + "epoch": 0.4459561602418745, + "fcm_dpo/beta": 0.012327713891863823, + "fcm_dpo/delta": 0.05323711037635803, + "fcm_dpo/margin": 28.27980613708496, + "fcm_dpo/q_t": 0.42287230491638184, + "grad_norm": 19.666093826293945, + "learning_rate": 3.4045544995169125e-07, + "logits/chosen": 0.7301595211029053, + "logits/rejected": 0.6291660070419312, + "logps/chosen": -127.95817565917969, + "logps/ref_chosen": -50.40088653564453, + "logps/ref_rejected": -83.43521881103516, + "logps/rejected": -189.27230834960938, + "loss": 1.1712, + "margin_dpo/margin_mean": 28.27980613708496, + "margin_dpo/margin_std": 53.96001434326172, + "step": 295 + }, + { + "KL/chosen_KL_mean": -76.57630920410156, + "KL/mean": -94.01647186279297, + "KL/rejected_KL_mean": -111.45663452148438, + "KL/std": 50.745975494384766, + "epoch": 0.4474678760393046, + "fcm_dpo/beta": 0.0122788455337286, + "fcm_dpo/delta": -0.030141662806272507, + "fcm_dpo/margin": 34.88032150268555, + "fcm_dpo/q_t": 0.40467900037765503, + "grad_norm": 12.9859619140625, + "learning_rate": 3.392215553979679e-07, + "logits/chosen": 0.6649519205093384, + "logits/rejected": 0.6230114698410034, + "logps/chosen": -145.72665405273438, + "logps/ref_chosen": -69.15034484863281, + "logps/ref_rejected": -89.60166931152344, + "logps/rejected": -201.0583038330078, + "loss": 1.1129, + "margin_dpo/margin_mean": 34.88031768798828, + "margin_dpo/margin_std": 55.64665222167969, + "step": 296 + }, + { + "KL/chosen_KL_mean": -80.91482543945312, + "KL/mean": -100.35860443115234, + "KL/rejected_KL_mean": -119.80238342285156, + "KL/std": 45.48234176635742, + "epoch": 0.4489795918367347, + "fcm_dpo/beta": 0.012235373258590698, + "fcm_dpo/delta": -0.0797240287065506, + "fcm_dpo/margin": 38.88756561279297, + "fcm_dpo/q_t": 0.392116516828537, + "grad_norm": 13.477424621582031, + "learning_rate": 3.3798516512554485e-07, + "logits/chosen": 0.6948250532150269, + "logits/rejected": 0.638218343257904, + "logps/chosen": -138.93113708496094, + "logps/ref_chosen": -58.01630401611328, + "logps/ref_rejected": -69.95780944824219, + "logps/rejected": -189.76019287109375, + "loss": 1.0511, + "margin_dpo/margin_mean": 38.887569427490234, + "margin_dpo/margin_std": 48.81712341308594, + "step": 297 + }, + { + "KL/chosen_KL_mean": -78.26363372802734, + "KL/mean": -93.04943084716797, + "KL/rejected_KL_mean": -107.83523559570312, + "KL/std": 48.65501403808594, + "epoch": 0.4504913076341648, + "fcm_dpo/beta": 0.012198349460959435, + "fcm_dpo/delta": 0.04073771834373474, + "fcm_dpo/margin": 29.57158088684082, + "fcm_dpo/q_t": 0.4210200309753418, + "grad_norm": 13.820934295654297, + "learning_rate": 3.367463137189156e-07, + "logits/chosen": 0.8141465187072754, + "logits/rejected": 0.7540981769561768, + "logps/chosen": -134.43295288085938, + "logps/ref_chosen": -56.1693115234375, + "logps/ref_rejected": -68.55052185058594, + "logps/rejected": -176.3857421875, + "loss": 1.1832, + "margin_dpo/margin_mean": 29.57158088684082, + "margin_dpo/margin_std": 59.90074920654297, + "step": 298 + }, + { + "KL/chosen_KL_mean": -77.41154479980469, + "KL/mean": -90.41871643066406, + "KL/rejected_KL_mean": -103.42588806152344, + "KL/std": 46.954566955566406, + "epoch": 0.4520030234315949, + "fcm_dpo/beta": 0.012318475171923637, + "fcm_dpo/delta": 0.08214488625526428, + "fcm_dpo/margin": 26.01433753967285, + "fcm_dpo/q_t": 0.42924293875694275, + "grad_norm": 18.75337791442871, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": 0.6139011383056641, + "logits/rejected": 0.586826503276825, + "logps/chosen": -139.72933959960938, + "logps/ref_chosen": -62.31780242919922, + "logps/ref_rejected": -72.60028839111328, + "logps/rejected": -176.02618408203125, + "loss": 1.2215, + "margin_dpo/margin_mean": 26.01433753967285, + "margin_dpo/margin_std": 59.20307159423828, + "step": 299 + }, + { + "KL/chosen_KL_mean": -77.873291015625, + "KL/mean": -94.65007019042969, + "KL/rejected_KL_mean": -111.42684936523438, + "KL/std": 47.93767547607422, + "epoch": 0.45351473922902497, + "fcm_dpo/beta": 0.012403897941112518, + "fcm_dpo/delta": -0.016899462789297104, + "fcm_dpo/margin": 33.553558349609375, + "fcm_dpo/q_t": 0.40718337893486023, + "grad_norm": 13.85849666595459, + "learning_rate": 3.3426136618426043e-07, + "logits/chosen": 0.7300994396209717, + "logits/rejected": 0.6637899875640869, + "logps/chosen": -138.2548828125, + "logps/ref_chosen": -60.38157653808594, + "logps/ref_rejected": -75.45442199707031, + "logps/rejected": -186.8812713623047, + "loss": 1.1346, + "margin_dpo/margin_mean": 33.553558349609375, + "margin_dpo/margin_std": 58.16008377075195, + "step": 300 + }, + { + "KL/chosen_KL_mean": -75.81689453125, + "KL/mean": -90.96798706054688, + "KL/rejected_KL_mean": -106.11907958984375, + "KL/std": 47.327850341796875, + "epoch": 0.455026455026455, + "fcm_dpo/beta": 0.012391261756420135, + "fcm_dpo/delta": 0.025361964479088783, + "fcm_dpo/margin": 30.30219268798828, + "fcm_dpo/q_t": 0.41580748558044434, + "grad_norm": 15.147626876831055, + "learning_rate": 3.3301533956555885e-07, + "logits/chosen": 0.7389202117919922, + "logits/rejected": 0.7118491530418396, + "logps/chosen": -128.66778564453125, + "logps/ref_chosen": -52.85089111328125, + "logps/ref_rejected": -69.97584533691406, + "logps/rejected": -176.0949249267578, + "loss": 1.1545, + "margin_dpo/margin_mean": 30.30219078063965, + "margin_dpo/margin_std": 55.01183319091797, + "step": 301 + }, + { + "KL/chosen_KL_mean": -80.43342590332031, + "KL/mean": -91.39155578613281, + "KL/rejected_KL_mean": -102.34968566894531, + "KL/std": 47.64586639404297, + "epoch": 0.4565381708238851, + "fcm_dpo/beta": 0.012671603821218014, + "fcm_dpo/delta": 0.12586434185504913, + "fcm_dpo/margin": 21.916255950927734, + "fcm_dpo/q_t": 0.43761053681373596, + "grad_norm": 18.956130981445312, + "learning_rate": 3.317669908293554e-07, + "logits/chosen": 0.5832021832466125, + "logits/rejected": 0.5297772288322449, + "logps/chosen": -147.39993286132812, + "logps/ref_chosen": -66.96650695800781, + "logps/ref_rejected": -88.09510803222656, + "logps/rejected": -190.44479370117188, + "loss": 1.2328, + "margin_dpo/margin_mean": 21.916255950927734, + "margin_dpo/margin_std": 52.96202850341797, + "step": 302 + }, + { + "KL/chosen_KL_mean": -69.9097900390625, + "KL/mean": -90.5808334350586, + "KL/rejected_KL_mean": -111.25189208984375, + "KL/std": 48.5565185546875, + "epoch": 0.4580498866213152, + "fcm_dpo/beta": 0.012538776732981205, + "fcm_dpo/delta": -0.12476019561290741, + "fcm_dpo/margin": 41.34212112426758, + "fcm_dpo/q_t": 0.38511401414871216, + "grad_norm": 12.396648406982422, + "learning_rate": 3.3051635489464793e-07, + "logits/chosen": 0.6951050758361816, + "logits/rejected": 0.6301414966583252, + "logps/chosen": -132.03131103515625, + "logps/ref_chosen": -62.12152862548828, + "logps/ref_rejected": -90.31204223632812, + "logps/rejected": -201.56393432617188, + "loss": 1.0655, + "margin_dpo/margin_mean": 41.342124938964844, + "margin_dpo/margin_std": 60.36699295043945, + "step": 303 + }, + { + "KL/chosen_KL_mean": -60.231353759765625, + "KL/mean": -80.84054565429688, + "KL/rejected_KL_mean": -101.44972229003906, + "KL/std": 51.155372619628906, + "epoch": 0.4595616024187453, + "fcm_dpo/beta": 0.0122285857796669, + "fcm_dpo/delta": -0.10976764559745789, + "fcm_dpo/margin": 41.21836853027344, + "fcm_dpo/q_t": 0.38361793756484985, + "grad_norm": 13.756430625915527, + "learning_rate": 3.292634667444117e-07, + "logits/chosen": 0.6984633207321167, + "logits/rejected": 0.6454420685768127, + "logps/chosen": -120.92644500732422, + "logps/ref_chosen": -60.695091247558594, + "logps/ref_rejected": -78.2525405883789, + "logps/rejected": -179.7022705078125, + "loss": 1.0208, + "margin_dpo/margin_mean": 41.21836853027344, + "margin_dpo/margin_std": 46.54623794555664, + "step": 304 + }, + { + "KL/chosen_KL_mean": -70.44589233398438, + "KL/mean": -86.21815490722656, + "KL/rejected_KL_mean": -101.99043273925781, + "KL/std": 48.03346252441406, + "epoch": 0.46107331821617537, + "fcm_dpo/beta": 0.012153957970440388, + "fcm_dpo/delta": 0.016938693821430206, + "fcm_dpo/margin": 31.544536590576172, + "fcm_dpo/q_t": 0.41375792026519775, + "grad_norm": 13.166147232055664, + "learning_rate": 3.280083614246217e-07, + "logits/chosen": 0.6348699331283569, + "logits/rejected": 0.6634021997451782, + "logps/chosen": -143.14503479003906, + "logps/ref_chosen": -72.69914245605469, + "logps/ref_rejected": -65.65670776367188, + "logps/rejected": -167.6471405029297, + "loss": 1.155, + "margin_dpo/margin_mean": 31.544536590576172, + "margin_dpo/margin_std": 57.23649597167969, + "step": 305 + }, + { + "KL/chosen_KL_mean": -66.22614288330078, + "KL/mean": -82.06704711914062, + "KL/rejected_KL_mean": -97.9079360961914, + "KL/std": 46.619834899902344, + "epoch": 0.46258503401360546, + "fcm_dpo/beta": 0.012171389535069466, + "fcm_dpo/delta": 0.014306794852018356, + "fcm_dpo/margin": 31.681785583496094, + "fcm_dpo/q_t": 0.41203343868255615, + "grad_norm": 13.922178268432617, + "learning_rate": 3.267510740432719e-07, + "logits/chosen": 0.7303614616394043, + "logits/rejected": 0.6184151768684387, + "logps/chosen": -120.19667053222656, + "logps/ref_chosen": -53.97052764892578, + "logps/ref_rejected": -71.02423095703125, + "logps/rejected": -168.93215942382812, + "loss": 1.1116, + "margin_dpo/margin_mean": 31.681781768798828, + "margin_dpo/margin_std": 45.20012283325195, + "step": 306 + }, + { + "KL/chosen_KL_mean": -60.205955505371094, + "KL/mean": -68.00119018554688, + "KL/rejected_KL_mean": -75.79643249511719, + "KL/std": 45.12220001220703, + "epoch": 0.46409674981103555, + "fcm_dpo/beta": 0.012687662616372108, + "fcm_dpo/delta": 0.20508906245231628, + "fcm_dpo/margin": 15.590478897094727, + "fcm_dpo/q_t": 0.457867294549942, + "grad_norm": 17.767627716064453, + "learning_rate": 3.2549163976939285e-07, + "logits/chosen": 0.7361500859260559, + "logits/rejected": 0.6878568530082703, + "logps/chosen": -117.61906433105469, + "logps/ref_chosen": -57.413108825683594, + "logps/ref_rejected": -68.68010711669922, + "logps/rejected": -144.47653198242188, + "loss": 1.3222, + "margin_dpo/margin_mean": 15.590479850769043, + "margin_dpo/margin_std": 56.86015701293945, + "step": 307 + }, + { + "KL/chosen_KL_mean": -59.32799530029297, + "KL/mean": -74.51152038574219, + "KL/rejected_KL_mean": -89.69503784179688, + "KL/std": 46.99913787841797, + "epoch": 0.4656084656084656, + "fcm_dpo/beta": 0.012745361775159836, + "fcm_dpo/delta": 0.013256091624498367, + "fcm_dpo/margin": 30.367046356201172, + "fcm_dpo/q_t": 0.41266879439353943, + "grad_norm": 11.760419845581055, + "learning_rate": 3.2423009383206874e-07, + "logits/chosen": 0.6760587096214294, + "logits/rejected": 0.6648536324501038, + "logps/chosen": -125.92678833007812, + "logps/ref_chosen": -66.59879302978516, + "logps/ref_rejected": -74.337158203125, + "logps/rejected": -164.03219604492188, + "loss": 1.1453, + "margin_dpo/margin_mean": 30.367046356201172, + "margin_dpo/margin_std": 53.325408935546875, + "step": 308 + }, + { + "KL/chosen_KL_mean": -68.26148986816406, + "KL/mean": -84.21513366699219, + "KL/rejected_KL_mean": -100.16877746582031, + "KL/std": 44.402587890625, + "epoch": 0.4671201814058957, + "fcm_dpo/beta": 0.012739481404423714, + "fcm_dpo/delta": -0.00707148015499115, + "fcm_dpo/margin": 31.907291412353516, + "fcm_dpo/q_t": 0.407754123210907, + "grad_norm": 12.303749084472656, + "learning_rate": 3.229664715194511e-07, + "logits/chosen": 0.744890570640564, + "logits/rejected": 0.6873106956481934, + "logps/chosen": -133.65623474121094, + "logps/ref_chosen": -65.39474487304688, + "logps/ref_rejected": -75.70930480957031, + "logps/rejected": -175.87808227539062, + "loss": 1.1057, + "margin_dpo/margin_mean": 31.90729331970215, + "margin_dpo/margin_std": 46.92948913574219, + "step": 309 + }, + { + "KL/chosen_KL_mean": -68.58441162109375, + "KL/mean": -78.29966735839844, + "KL/rejected_KL_mean": -88.01490020751953, + "KL/std": 41.900726318359375, + "epoch": 0.46863189720332576, + "fcm_dpo/beta": 0.01305387169122696, + "fcm_dpo/delta": 0.1503462791442871, + "fcm_dpo/margin": 19.430477142333984, + "fcm_dpo/q_t": 0.4454615116119385, + "grad_norm": 14.570418357849121, + "learning_rate": 3.2170080817777257e-07, + "logits/chosen": 0.6618015766143799, + "logits/rejected": 0.6511035561561584, + "logps/chosen": -143.252685546875, + "logps/ref_chosen": -74.66827392578125, + "logps/ref_rejected": -80.5689697265625, + "logps/rejected": -168.5838623046875, + "loss": 1.2538, + "margin_dpo/margin_mean": 19.430479049682617, + "margin_dpo/margin_std": 51.15632629394531, + "step": 310 + }, + { + "KL/chosen_KL_mean": -55.306392669677734, + "KL/mean": -70.36241149902344, + "KL/rejected_KL_mean": -85.41842651367188, + "KL/std": 48.275238037109375, + "epoch": 0.47014361300075586, + "fcm_dpo/beta": 0.013104679062962532, + "fcm_dpo/delta": 0.004920497536659241, + "fcm_dpo/margin": 30.112035751342773, + "fcm_dpo/q_t": 0.41163355112075806, + "grad_norm": 15.74968433380127, + "learning_rate": 3.204331392103574e-07, + "logits/chosen": 0.5947407484054565, + "logits/rejected": 0.46077167987823486, + "logps/chosen": -115.04442596435547, + "logps/ref_chosen": -59.738033294677734, + "logps/ref_rejected": -93.60757446289062, + "logps/rejected": -179.0260009765625, + "loss": 1.1241, + "margin_dpo/margin_mean": 30.112035751342773, + "margin_dpo/margin_std": 47.993797302246094, + "step": 311 + }, + { + "KL/chosen_KL_mean": -57.70798873901367, + "KL/mean": -77.60112762451172, + "KL/rejected_KL_mean": -97.4942626953125, + "KL/std": 46.28860092163086, + "epoch": 0.47165532879818595, + "fcm_dpo/beta": 0.012983372434973717, + "fcm_dpo/delta": -0.12272703647613525, + "fcm_dpo/margin": 39.786277770996094, + "fcm_dpo/q_t": 0.3818385899066925, + "grad_norm": 13.52045726776123, + "learning_rate": 3.1916350007663176e-07, + "logits/chosen": 0.7198551893234253, + "logits/rejected": 0.6226128339767456, + "logps/chosen": -111.52442169189453, + "logps/ref_chosen": -53.816436767578125, + "logps/ref_rejected": -68.6575698852539, + "logps/rejected": -166.15182495117188, + "loss": 1.0173, + "margin_dpo/margin_mean": 39.786277770996094, + "margin_dpo/margin_std": 45.88895797729492, + "step": 312 + }, + { + "KL/chosen_KL_mean": -58.545257568359375, + "KL/mean": -69.0672836303711, + "KL/rejected_KL_mean": -79.58930206298828, + "KL/std": 44.045310974121094, + "epoch": 0.47316704459561604, + "fcm_dpo/beta": 0.013112520799040794, + "fcm_dpo/delta": 0.12759315967559814, + "fcm_dpo/margin": 21.044044494628906, + "fcm_dpo/q_t": 0.44002604484558105, + "grad_norm": 12.471875190734863, + "learning_rate": 3.178919262911314e-07, + "logits/chosen": 0.7591273784637451, + "logits/rejected": 0.738348662853241, + "logps/chosen": -118.50261688232422, + "logps/ref_chosen": -59.957359313964844, + "logps/ref_rejected": -69.31729888916016, + "logps/rejected": -148.90660095214844, + "loss": 1.2263, + "margin_dpo/margin_mean": 21.044044494628906, + "margin_dpo/margin_std": 49.037994384765625, + "step": 313 + }, + { + "KL/chosen_KL_mean": -55.58160400390625, + "KL/mean": -76.64013671875, + "KL/rejected_KL_mean": -97.69866180419922, + "KL/std": 47.22520446777344, + "epoch": 0.47467876039304613, + "fcm_dpo/beta": 0.0128698218613863, + "fcm_dpo/delta": -0.15077346563339233, + "fcm_dpo/margin": 42.11705017089844, + "fcm_dpo/q_t": 0.3795948624610901, + "grad_norm": 12.830547332763672, + "learning_rate": 3.166184534225087e-07, + "logits/chosen": 0.6727685928344727, + "logits/rejected": 0.7047948837280273, + "logps/chosen": -125.84976196289062, + "logps/ref_chosen": -70.26815795898438, + "logps/ref_rejected": -69.23971557617188, + "logps/rejected": -166.93838500976562, + "loss": 1.0173, + "margin_dpo/margin_mean": 42.11705017089844, + "margin_dpo/margin_std": 51.619964599609375, + "step": 314 + }, + { + "KL/chosen_KL_mean": -61.01652145385742, + "KL/mean": -77.28446960449219, + "KL/rejected_KL_mean": -93.55242919921875, + "KL/std": 46.021209716796875, + "epoch": 0.47619047619047616, + "fcm_dpo/beta": 0.012844046577811241, + "fcm_dpo/delta": -0.01923982799053192, + "fcm_dpo/margin": 32.535919189453125, + "fcm_dpo/q_t": 0.40471553802490234, + "grad_norm": 12.773975372314453, + "learning_rate": 3.1534311709253723e-07, + "logits/chosen": 0.6400498151779175, + "logits/rejected": 0.6017441749572754, + "logps/chosen": -128.81121826171875, + "logps/ref_chosen": -67.79469299316406, + "logps/ref_rejected": -74.55148315429688, + "logps/rejected": -168.10391235351562, + "loss": 1.1001, + "margin_dpo/margin_mean": 32.535919189453125, + "margin_dpo/margin_std": 47.00663757324219, + "step": 315 + }, + { + "KL/chosen_KL_mean": -56.92601776123047, + "KL/mean": -77.34451293945312, + "KL/rejected_KL_mean": -97.76300811767578, + "KL/std": 47.411216735839844, + "epoch": 0.47770219198790626, + "fcm_dpo/beta": 0.012653250247240067, + "fcm_dpo/delta": -0.12391935288906097, + "fcm_dpo/margin": 40.83698654174805, + "fcm_dpo/q_t": 0.38212883472442627, + "grad_norm": 13.032649040222168, + "learning_rate": 3.1406595297511564e-07, + "logits/chosen": 0.5666062831878662, + "logits/rejected": 0.4426162838935852, + "logps/chosen": -112.2145004272461, + "logps/ref_chosen": -55.288482666015625, + "logps/ref_rejected": -96.15723419189453, + "logps/rejected": -193.9202423095703, + "loss": 1.0177, + "margin_dpo/margin_mean": 40.83698272705078, + "margin_dpo/margin_std": 44.17725372314453, + "step": 316 + }, + { + "KL/chosen_KL_mean": -54.153907775878906, + "KL/mean": -75.89384460449219, + "KL/rejected_KL_mean": -97.6337890625, + "KL/std": 46.43383026123047, + "epoch": 0.47921390778533635, + "fcm_dpo/beta": 0.01215449906885624, + "fcm_dpo/delta": -0.1363387107849121, + "fcm_dpo/margin": 43.47986602783203, + "fcm_dpo/q_t": 0.3797203302383423, + "grad_norm": 17.4111328125, + "learning_rate": 3.1278699679526975e-07, + "logits/chosen": 0.7092440724372864, + "logits/rejected": 0.6640247106552124, + "logps/chosen": -108.73529052734375, + "logps/ref_chosen": -54.58137512207031, + "logps/ref_rejected": -72.77232360839844, + "logps/rejected": -170.40611267089844, + "loss": 1.0127, + "margin_dpo/margin_mean": 43.47986602783203, + "margin_dpo/margin_std": 50.48588562011719, + "step": 317 + }, + { + "KL/chosen_KL_mean": -62.91188049316406, + "KL/mean": -77.77674865722656, + "KL/rejected_KL_mean": -92.64160919189453, + "KL/std": 49.21617889404297, + "epoch": 0.48072562358276644, + "fcm_dpo/beta": 0.012163069099187851, + "fcm_dpo/delta": 0.03984696790575981, + "fcm_dpo/margin": 29.729740142822266, + "fcm_dpo/q_t": 0.4216643273830414, + "grad_norm": 13.982091903686523, + "learning_rate": 3.1150628432815336e-07, + "logits/chosen": 0.7147679328918457, + "logits/rejected": 0.6473454833030701, + "logps/chosen": -115.80010986328125, + "logps/ref_chosen": -52.88822937011719, + "logps/ref_rejected": -80.63988494873047, + "logps/rejected": -173.281494140625, + "loss": 1.1905, + "margin_dpo/margin_mean": 29.7297420501709, + "margin_dpo/margin_std": 62.43902587890625, + "step": 318 + }, + { + "KL/chosen_KL_mean": -59.110321044921875, + "KL/mean": -80.11236572265625, + "KL/rejected_KL_mean": -101.11442565917969, + "KL/std": 50.182960510253906, + "epoch": 0.48223733938019653, + "fcm_dpo/beta": 0.012012695893645287, + "fcm_dpo/delta": -0.11011025309562683, + "fcm_dpo/margin": 42.00410461425781, + "fcm_dpo/q_t": 0.38811129331588745, + "grad_norm": 14.462267875671387, + "learning_rate": 3.1022385139804707e-07, + "logits/chosen": 0.6171474456787109, + "logits/rejected": 0.6034401655197144, + "logps/chosen": -123.47364807128906, + "logps/ref_chosen": -64.36333465576172, + "logps/ref_rejected": -79.47296142578125, + "logps/rejected": -180.58737182617188, + "loss": 1.0538, + "margin_dpo/margin_mean": 42.00410461425781, + "margin_dpo/margin_std": 57.84809112548828, + "step": 319 + }, + { + "KL/chosen_KL_mean": -57.963951110839844, + "KL/mean": -74.89822387695312, + "KL/rejected_KL_mean": -91.83250427246094, + "KL/std": 50.19336700439453, + "epoch": 0.4837490551776266, + "fcm_dpo/beta": 0.01207122951745987, + "fcm_dpo/delta": -0.012276045978069305, + "fcm_dpo/margin": 33.86854934692383, + "fcm_dpo/q_t": 0.4102700352668762, + "grad_norm": 15.46420955657959, + "learning_rate": 3.0893973387735683e-07, + "logits/chosen": 0.575273334980011, + "logits/rejected": 0.5354819893836975, + "logps/chosen": -107.52269744873047, + "logps/ref_chosen": -49.558746337890625, + "logps/ref_rejected": -71.23444366455078, + "logps/rejected": -163.06695556640625, + "loss": 1.1313, + "margin_dpo/margin_mean": 33.86854934692383, + "margin_dpo/margin_std": 54.04207229614258, + "step": 320 + }, + { + "KL/chosen_KL_mean": -62.31312561035156, + "KL/mean": -81.66812896728516, + "KL/rejected_KL_mean": -101.02313232421875, + "KL/std": 49.185813903808594, + "epoch": 0.4852607709750567, + "fcm_dpo/beta": 0.011711912229657173, + "fcm_dpo/delta": -0.057817112654447556, + "fcm_dpo/margin": 38.71000671386719, + "fcm_dpo/q_t": 0.3971693515777588, + "grad_norm": 19.20660400390625, + "learning_rate": 3.0765396768561004e-07, + "logits/chosen": 0.6728286743164062, + "logits/rejected": 0.6586112976074219, + "logps/chosen": -114.39839172363281, + "logps/ref_chosen": -52.08526611328125, + "logps/ref_rejected": -55.58674621582031, + "logps/rejected": -156.60987854003906, + "loss": 1.0955, + "margin_dpo/margin_mean": 38.71000671386719, + "margin_dpo/margin_std": 57.03093719482422, + "step": 321 + }, + { + "KL/chosen_KL_mean": -73.02706146240234, + "KL/mean": -96.4334716796875, + "KL/rejected_KL_mean": -119.83988952636719, + "KL/std": 53.84697723388672, + "epoch": 0.48677248677248675, + "fcm_dpo/beta": 0.011545605957508087, + "fcm_dpo/delta": -0.14827696979045868, + "fcm_dpo/margin": 46.81283187866211, + "fcm_dpo/q_t": 0.3775472640991211, + "grad_norm": 13.035255432128906, + "learning_rate": 3.063665887884511e-07, + "logits/chosen": 0.7389756441116333, + "logits/rejected": 0.6568834781646729, + "logps/chosen": -120.43116760253906, + "logps/ref_chosen": -47.404109954833984, + "logps/ref_rejected": -73.4260025024414, + "logps/rejected": -193.26589965820312, + "loss": 1.0112, + "margin_dpo/margin_mean": 46.812828063964844, + "margin_dpo/margin_std": 55.330101013183594, + "step": 322 + }, + { + "KL/chosen_KL_mean": -75.228515625, + "KL/mean": -90.48625183105469, + "KL/rejected_KL_mean": -105.7439956665039, + "KL/std": 52.34074020385742, + "epoch": 0.48828420256991684, + "fcm_dpo/beta": 0.011556407436728477, + "fcm_dpo/delta": 0.048632849007844925, + "fcm_dpo/margin": 30.515487670898438, + "fcm_dpo/q_t": 0.42401984333992004, + "grad_norm": 14.605672836303711, + "learning_rate": 3.0507763319663517e-07, + "logits/chosen": 0.6270068883895874, + "logits/rejected": 0.5478232502937317, + "logps/chosen": -145.23480224609375, + "logps/ref_chosen": -70.00630187988281, + "logps/ref_rejected": -86.96690368652344, + "logps/rejected": -192.71090698242188, + "loss": 1.1979, + "margin_dpo/margin_mean": 30.515487670898438, + "margin_dpo/margin_std": 65.71580505371094, + "step": 323 + }, + { + "KL/chosen_KL_mean": -64.62179565429688, + "KL/mean": -86.31397247314453, + "KL/rejected_KL_mean": -108.00615692138672, + "KL/std": 55.89674377441406, + "epoch": 0.4897959183673469, + "fcm_dpo/beta": 0.011341476812958717, + "fcm_dpo/delta": -0.09731088578701019, + "fcm_dpo/margin": 43.384361267089844, + "fcm_dpo/q_t": 0.3883642852306366, + "grad_norm": 18.400531768798828, + "learning_rate": 3.0378713696502097e-07, + "logits/chosen": 0.7231769561767578, + "logits/rejected": 0.6682534217834473, + "logps/chosen": -120.51061248779297, + "logps/ref_chosen": -55.88882064819336, + "logps/ref_rejected": -75.23088073730469, + "logps/rejected": -183.23703002929688, + "loss": 1.0393, + "margin_dpo/margin_mean": 43.384361267089844, + "margin_dpo/margin_std": 53.82892608642578, + "step": 324 + }, + { + "KL/chosen_KL_mean": -81.2076416015625, + "KL/mean": -100.78556823730469, + "KL/rejected_KL_mean": -120.36351013183594, + "KL/std": 53.15364074707031, + "epoch": 0.491307634164777, + "fcm_dpo/beta": 0.011201553046703339, + "fcm_dpo/delta": -0.040966056287288666, + "fcm_dpo/margin": 39.15586853027344, + "fcm_dpo/q_t": 0.4002138674259186, + "grad_norm": 15.490608215332031, + "learning_rate": 3.0249513619156206e-07, + "logits/chosen": 0.6761427521705627, + "logits/rejected": 0.6141278743743896, + "logps/chosen": -145.3546600341797, + "logps/ref_chosen": -64.14701843261719, + "logps/ref_rejected": -79.91143798828125, + "logps/rejected": -200.27493286132812, + "loss": 1.1039, + "margin_dpo/margin_mean": 39.15586471557617, + "margin_dpo/margin_std": 60.82697296142578, + "step": 325 + }, + { + "KL/chosen_KL_mean": -95.6961898803711, + "KL/mean": -105.70980834960938, + "KL/rejected_KL_mean": -115.72342681884766, + "KL/std": 55.04910659790039, + "epoch": 0.4928193499622071, + "fcm_dpo/beta": 0.011495251208543777, + "fcm_dpo/delta": 0.17409512400627136, + "fcm_dpo/margin": 20.02724266052246, + "fcm_dpo/q_t": 0.4493880867958069, + "grad_norm": 14.850746154785156, + "learning_rate": 3.012016670162977e-07, + "logits/chosen": 0.6151560544967651, + "logits/rejected": 0.6213278770446777, + "logps/chosen": -171.22750854492188, + "logps/ref_chosen": -75.53131103515625, + "logps/ref_rejected": -76.5898666381836, + "logps/rejected": -192.31329345703125, + "loss": 1.2868, + "margin_dpo/margin_mean": 20.027244567871094, + "margin_dpo/margin_std": 60.19707107543945, + "step": 326 + }, + { + "KL/chosen_KL_mean": -91.24993896484375, + "KL/mean": -106.86180114746094, + "KL/rejected_KL_mean": -122.47364807128906, + "KL/std": 56.48907470703125, + "epoch": 0.4943310657596372, + "fcm_dpo/beta": 0.011725237593054771, + "fcm_dpo/delta": 0.03484252840280533, + "fcm_dpo/margin": 31.223709106445312, + "fcm_dpo/q_t": 0.4198496341705322, + "grad_norm": 17.629716873168945, + "learning_rate": 2.99906765620341e-07, + "logits/chosen": 0.5963134765625, + "logits/rejected": 0.5632212162017822, + "logps/chosen": -160.58709716796875, + "logps/ref_chosen": -69.33717346191406, + "logps/ref_rejected": -73.37751770019531, + "logps/rejected": -195.85116577148438, + "loss": 1.1785, + "margin_dpo/margin_mean": 31.22370719909668, + "margin_dpo/margin_std": 62.13964080810547, + "step": 327 + }, + { + "KL/chosen_KL_mean": -78.11366271972656, + "KL/mean": -97.55792236328125, + "KL/rejected_KL_mean": -117.00218200683594, + "KL/std": 57.609336853027344, + "epoch": 0.4958427815570673, + "fcm_dpo/beta": 0.011624249629676342, + "fcm_dpo/delta": -0.05448012053966522, + "fcm_dpo/margin": 38.888511657714844, + "fcm_dpo/q_t": 0.39957690238952637, + "grad_norm": 13.897930145263672, + "learning_rate": 2.9861046822486766e-07, + "logits/chosen": 0.6323528289794922, + "logits/rejected": 0.6021204590797424, + "logps/chosen": -139.81988525390625, + "logps/ref_chosen": -61.70623016357422, + "logps/ref_rejected": -83.73808288574219, + "logps/rejected": -200.74026489257812, + "loss": 1.0849, + "margin_dpo/margin_mean": 38.888511657714844, + "margin_dpo/margin_std": 56.55249786376953, + "step": 328 + }, + { + "KL/chosen_KL_mean": -87.73931884765625, + "KL/mean": -107.61006164550781, + "KL/rejected_KL_mean": -127.48081970214844, + "KL/std": 55.76243209838867, + "epoch": 0.4973544973544973, + "fcm_dpo/beta": 0.01148184947669506, + "fcm_dpo/delta": -0.05898508056998253, + "fcm_dpo/margin": 39.74150085449219, + "fcm_dpo/q_t": 0.3992629051208496, + "grad_norm": 18.06145477294922, + "learning_rate": 2.9731281109010253e-07, + "logits/chosen": 0.6991169452667236, + "logits/rejected": 0.6458307504653931, + "logps/chosen": -152.23773193359375, + "logps/ref_chosen": -64.4984130859375, + "logps/ref_rejected": -83.6591796875, + "logps/rejected": -211.13998413085938, + "loss": 1.0842, + "margin_dpo/margin_mean": 39.74150085449219, + "margin_dpo/margin_std": 58.48515701293945, + "step": 329 + }, + { + "KL/chosen_KL_mean": -74.7954330444336, + "KL/mean": -96.87043762207031, + "KL/rejected_KL_mean": -118.9454345703125, + "KL/std": 56.645729064941406, + "epoch": 0.4988662131519274, + "fcm_dpo/beta": 0.01131785660982132, + "fcm_dpo/delta": -0.10486404597759247, + "fcm_dpo/margin": 44.15000534057617, + "fcm_dpo/q_t": 0.3897658586502075, + "grad_norm": 13.613192558288574, + "learning_rate": 2.9601383051430505e-07, + "logits/chosen": 0.7008596062660217, + "logits/rejected": 0.6330828666687012, + "logps/chosen": -129.60006713867188, + "logps/ref_chosen": -54.80464172363281, + "logps/ref_rejected": -75.3194351196289, + "logps/rejected": -194.26486206054688, + "loss": 1.0977, + "margin_dpo/margin_mean": 44.15000534057617, + "margin_dpo/margin_std": 70.77532196044922, + "step": 330 + }, + { + "KL/chosen_KL_mean": -81.3387451171875, + "KL/mean": -108.13944244384766, + "KL/rejected_KL_mean": -134.94012451171875, + "KL/std": 59.597442626953125, + "epoch": 0.5003779289493575, + "fcm_dpo/beta": 0.0109153613448143, + "fcm_dpo/delta": -0.196200892329216, + "fcm_dpo/margin": 53.60139465332031, + "fcm_dpo/q_t": 0.368771493434906, + "grad_norm": 13.300724029541016, + "learning_rate": 2.947135628327544e-07, + "logits/chosen": 0.8084205985069275, + "logits/rejected": 0.7829192876815796, + "logps/chosen": -140.58132934570312, + "logps/ref_chosen": -59.242584228515625, + "logps/ref_rejected": -69.87483215332031, + "logps/rejected": -204.81497192382812, + "loss": 1.0072, + "margin_dpo/margin_mean": 53.60139465332031, + "margin_dpo/margin_std": 66.9174575805664, + "step": 331 + }, + { + "KL/chosen_KL_mean": -82.68574523925781, + "KL/mean": -104.79304504394531, + "KL/rejected_KL_mean": -126.90032958984375, + "KL/std": 56.367454528808594, + "epoch": 0.5018896447467877, + "fcm_dpo/beta": 0.010739924386143684, + "fcm_dpo/delta": -0.07937489449977875, + "fcm_dpo/margin": 44.21459197998047, + "fcm_dpo/q_t": 0.393557608127594, + "grad_norm": 14.145315170288086, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": 0.6289262175559998, + "logits/rejected": 0.5868539810180664, + "logps/chosen": -149.79550170898438, + "logps/ref_chosen": -67.10975646972656, + "logps/ref_rejected": -77.11839294433594, + "logps/rejected": -204.0187225341797, + "loss": 1.0615, + "margin_dpo/margin_mean": 44.21459197998047, + "margin_dpo/margin_std": 57.61412048339844, + "step": 332 + }, + { + "KL/chosen_KL_mean": -87.93297576904297, + "KL/mean": -111.47476196289062, + "KL/rejected_KL_mean": -135.0165557861328, + "KL/std": 56.53441619873047, + "epoch": 0.5034013605442177, + "fcm_dpo/beta": 0.010472394526004791, + "fcm_dpo/delta": -0.09779446572065353, + "fcm_dpo/margin": 47.08356475830078, + "fcm_dpo/q_t": 0.3887941241264343, + "grad_norm": 12.955265998840332, + "learning_rate": 2.921093116725076e-07, + "logits/chosen": 0.6959058046340942, + "logits/rejected": 0.6248373985290527, + "logps/chosen": -146.31411743164062, + "logps/ref_chosen": -58.381134033203125, + "logps/ref_rejected": -85.02839660644531, + "logps/rejected": -220.04495239257812, + "loss": 1.0398, + "margin_dpo/margin_mean": 47.08356475830078, + "margin_dpo/margin_std": 58.59512710571289, + "step": 333 + }, + { + "KL/chosen_KL_mean": -83.53094482421875, + "KL/mean": -100.3437271118164, + "KL/rejected_KL_mean": -117.15650177001953, + "KL/std": 58.58315658569336, + "epoch": 0.5049130763416477, + "fcm_dpo/beta": 0.010494356974959373, + "fcm_dpo/delta": 0.04870123788714409, + "fcm_dpo/margin": 33.62556457519531, + "fcm_dpo/q_t": 0.42179661989212036, + "grad_norm": 13.383609771728516, + "learning_rate": 2.9080540104031484e-07, + "logits/chosen": 0.7296691536903381, + "logits/rejected": 0.685336172580719, + "logps/chosen": -150.42294311523438, + "logps/ref_chosen": -66.89199829101562, + "logps/ref_rejected": -91.83695220947266, + "logps/rejected": -208.9934539794922, + "loss": 1.1831, + "margin_dpo/margin_mean": 33.62556457519531, + "margin_dpo/margin_std": 68.05831146240234, + "step": 334 + }, + { + "KL/chosen_KL_mean": -85.08909606933594, + "KL/mean": -104.37361145019531, + "KL/rejected_KL_mean": -123.65814208984375, + "KL/std": 57.88601303100586, + "epoch": 0.5064247921390779, + "fcm_dpo/beta": 0.010544189251959324, + "fcm_dpo/delta": -0.007449910044670105, + "fcm_dpo/margin": 38.56904220581055, + "fcm_dpo/q_t": 0.41014015674591064, + "grad_norm": 19.125410079956055, + "learning_rate": 2.895003489933375e-07, + "logits/chosen": 0.720893383026123, + "logits/rejected": 0.6842397451400757, + "logps/chosen": -146.60354614257812, + "logps/ref_chosen": -61.51445770263672, + "logps/ref_rejected": -75.68916320800781, + "logps/rejected": -199.3472900390625, + "loss": 1.131, + "margin_dpo/margin_mean": 38.56904220581055, + "margin_dpo/margin_std": 64.91070556640625, + "step": 335 + }, + { + "KL/chosen_KL_mean": -94.51179504394531, + "KL/mean": -114.01209259033203, + "KL/rejected_KL_mean": -133.51239013671875, + "KL/std": 58.32542037963867, + "epoch": 0.5079365079365079, + "fcm_dpo/beta": 0.010401548817753792, + "fcm_dpo/delta": -0.006995633244514465, + "fcm_dpo/margin": 39.00059127807617, + "fcm_dpo/q_t": 0.4102246165275574, + "grad_norm": 12.244279861450195, + "learning_rate": 2.8819419203668675e-07, + "logits/chosen": 0.6266505122184753, + "logits/rejected": 0.6072256565093994, + "logps/chosen": -163.3618621826172, + "logps/ref_chosen": -68.85006713867188, + "logps/ref_rejected": -92.99603271484375, + "logps/rejected": -226.5084228515625, + "loss": 1.1214, + "margin_dpo/margin_mean": 39.00059509277344, + "margin_dpo/margin_std": 61.9708251953125, + "step": 336 + }, + { + "KL/chosen_KL_mean": -97.12661743164062, + "KL/mean": -112.62835693359375, + "KL/rejected_KL_mean": -128.13009643554688, + "KL/std": 57.338382720947266, + "epoch": 0.509448223733938, + "fcm_dpo/beta": 0.010596680454909801, + "fcm_dpo/delta": 0.07382012903690338, + "fcm_dpo/margin": 31.003463745117188, + "fcm_dpo/q_t": 0.4254772961139679, + "grad_norm": 12.672013282775879, + "learning_rate": 2.8688696670638053e-07, + "logits/chosen": 0.5881543159484863, + "logits/rejected": 0.5562857389450073, + "logps/chosen": -170.314453125, + "logps/ref_chosen": -73.18783569335938, + "logps/ref_rejected": -86.89118957519531, + "logps/rejected": -215.0212860107422, + "loss": 1.1771, + "margin_dpo/margin_mean": 31.003459930419922, + "margin_dpo/margin_std": 59.29844284057617, + "step": 337 + }, + { + "KL/chosen_KL_mean": -92.96856689453125, + "KL/mean": -109.99942016601562, + "KL/rejected_KL_mean": -127.03028869628906, + "KL/std": 57.12294006347656, + "epoch": 0.5109599395313681, + "fcm_dpo/beta": 0.010666831396520138, + "fcm_dpo/delta": 0.038046009838581085, + "fcm_dpo/margin": 34.061710357666016, + "fcm_dpo/q_t": 0.41940271854400635, + "grad_norm": 12.995147705078125, + "learning_rate": 2.8557870956832133e-07, + "logits/chosen": 0.6552461385726929, + "logits/rejected": 0.6303938627243042, + "logps/chosen": -156.90817260742188, + "logps/ref_chosen": -63.939613342285156, + "logps/ref_rejected": -75.34243774414062, + "logps/rejected": -202.3727264404297, + "loss": 1.1656, + "margin_dpo/margin_mean": 34.061710357666016, + "margin_dpo/margin_std": 64.4132308959961, + "step": 338 + }, + { + "KL/chosen_KL_mean": -76.82237243652344, + "KL/mean": -95.09283447265625, + "KL/rejected_KL_mean": -113.3633041381836, + "KL/std": 55.23368835449219, + "epoch": 0.5124716553287982, + "fcm_dpo/beta": 0.010703757405281067, + "fcm_dpo/delta": 0.009182396344840527, + "fcm_dpo/margin": 36.54094696044922, + "fcm_dpo/q_t": 0.4115958511829376, + "grad_norm": 12.69047737121582, + "learning_rate": 2.842694572172736e-07, + "logits/chosen": 0.7880140542984009, + "logits/rejected": 0.7043054103851318, + "logps/chosen": -122.37149810791016, + "logps/ref_chosen": -45.54913330078125, + "logps/ref_rejected": -67.0482177734375, + "logps/rejected": -180.41152954101562, + "loss": 1.1228, + "margin_dpo/margin_mean": 36.54094696044922, + "margin_dpo/margin_std": 57.50127029418945, + "step": 339 + }, + { + "KL/chosen_KL_mean": -86.79705047607422, + "KL/mean": -106.69656372070312, + "KL/rejected_KL_mean": -126.59608459472656, + "KL/std": 61.46400451660156, + "epoch": 0.5139833711262283, + "fcm_dpo/beta": 0.010734757408499718, + "fcm_dpo/delta": -0.02878701686859131, + "fcm_dpo/margin": 39.79903030395508, + "fcm_dpo/q_t": 0.4072033762931824, + "grad_norm": 12.821864128112793, + "learning_rate": 2.8295924627584004e-07, + "logits/chosen": 0.6960463523864746, + "logits/rejected": 0.6766098141670227, + "logps/chosen": -140.80270385742188, + "logps/ref_chosen": -54.00564956665039, + "logps/ref_rejected": -61.314430236816406, + "logps/rejected": -187.9105224609375, + "loss": 1.1353, + "margin_dpo/margin_mean": 39.79903030395508, + "margin_dpo/margin_std": 69.87727355957031, + "step": 340 + }, + { + "KL/chosen_KL_mean": -84.44651794433594, + "KL/mean": -106.48483276367188, + "KL/rejected_KL_mean": -128.52316284179688, + "KL/std": 59.11941909790039, + "epoch": 0.5154950869236583, + "fcm_dpo/beta": 0.010431801900267601, + "fcm_dpo/delta": -0.06666833162307739, + "fcm_dpo/margin": 44.07664108276367, + "fcm_dpo/q_t": 0.39677852392196655, + "grad_norm": 13.550869941711426, + "learning_rate": 2.816481133934373e-07, + "logits/chosen": 0.7298463582992554, + "logits/rejected": 0.6815722584724426, + "logps/chosen": -147.84161376953125, + "logps/ref_chosen": -63.39509582519531, + "logps/ref_rejected": -76.20973205566406, + "logps/rejected": -204.73287963867188, + "loss": 1.0912, + "margin_dpo/margin_mean": 44.07664108276367, + "margin_dpo/margin_std": 62.568634033203125, + "step": 341 + }, + { + "KL/chosen_KL_mean": -83.20333099365234, + "KL/mean": -106.3760986328125, + "KL/rejected_KL_mean": -129.5488739013672, + "KL/std": 61.28248596191406, + "epoch": 0.5170068027210885, + "fcm_dpo/beta": 0.010350107215344906, + "fcm_dpo/delta": -0.08443048596382141, + "fcm_dpo/margin": 46.345550537109375, + "fcm_dpo/q_t": 0.39365118741989136, + "grad_norm": 11.774492263793945, + "learning_rate": 2.8033609524527046e-07, + "logits/chosen": 0.7456991672515869, + "logits/rejected": 0.7070844769477844, + "logps/chosen": -136.2511444091797, + "logps/ref_chosen": -53.047813415527344, + "logps/ref_rejected": -68.2854232788086, + "logps/rejected": -197.83428955078125, + "loss": 1.0632, + "margin_dpo/margin_mean": 46.345550537109375, + "margin_dpo/margin_std": 63.2640495300293, + "step": 342 + }, + { + "KL/chosen_KL_mean": -79.01636505126953, + "KL/mean": -93.48959350585938, + "KL/rejected_KL_mean": -107.96282958984375, + "KL/std": 57.215553283691406, + "epoch": 0.5185185185185185, + "fcm_dpo/beta": 0.010420860722661018, + "fcm_dpo/delta": 0.10085594654083252, + "fcm_dpo/margin": 28.94646644592285, + "fcm_dpo/q_t": 0.43098893761634827, + "grad_norm": 12.47730541229248, + "learning_rate": 2.7902322853130753e-07, + "logits/chosen": 0.6005036234855652, + "logits/rejected": 0.5945264101028442, + "logps/chosen": -149.5948944091797, + "logps/ref_chosen": -70.57852935791016, + "logps/ref_rejected": -84.73873901367188, + "logps/rejected": -192.70156860351562, + "loss": 1.1988, + "margin_dpo/margin_mean": 28.946468353271484, + "margin_dpo/margin_std": 57.34397888183594, + "step": 343 + }, + { + "KL/chosen_KL_mean": -86.4252700805664, + "KL/mean": -109.1060791015625, + "KL/rejected_KL_mean": -131.78689575195312, + "KL/std": 58.18048095703125, + "epoch": 0.5200302343159486, + "fcm_dpo/beta": 0.010408244095742702, + "fcm_dpo/delta": -0.07615945488214493, + "fcm_dpo/margin": 45.36162567138672, + "fcm_dpo/q_t": 0.3939076066017151, + "grad_norm": 14.552360534667969, + "learning_rate": 2.7770954997525274e-07, + "logits/chosen": 0.715709388256073, + "logits/rejected": 0.6526391506195068, + "logps/chosen": -142.23626708984375, + "logps/ref_chosen": -55.811004638671875, + "logps/ref_rejected": -84.77637481689453, + "logps/rejected": -216.56326293945312, + "loss": 1.061, + "margin_dpo/margin_mean": 45.36162567138672, + "margin_dpo/margin_std": 60.71054458618164, + "step": 344 + }, + { + "KL/chosen_KL_mean": -67.61167907714844, + "KL/mean": -87.00637817382812, + "KL/rejected_KL_mean": -106.40106201171875, + "KL/std": 51.96855545043945, + "epoch": 0.5215419501133787, + "fcm_dpo/beta": 0.01042355690151453, + "fcm_dpo/delta": -0.004661137238144875, + "fcm_dpo/margin": 38.78938674926758, + "fcm_dpo/q_t": 0.40908119082450867, + "grad_norm": 13.888686180114746, + "learning_rate": 2.7639509632351927e-07, + "logits/chosen": 0.7611949443817139, + "logits/rejected": 0.7161680459976196, + "logps/chosen": -125.39778137207031, + "logps/ref_chosen": -57.78609848022461, + "logps/ref_rejected": -78.91847229003906, + "logps/rejected": -185.3195343017578, + "loss": 1.1181, + "margin_dpo/margin_mean": 38.78938674926758, + "margin_dpo/margin_std": 61.487510681152344, + "step": 345 + }, + { + "KL/chosen_KL_mean": -75.70014953613281, + "KL/mean": -98.07574462890625, + "KL/rejected_KL_mean": -120.45133972167969, + "KL/std": 59.482688903808594, + "epoch": 0.5230536659108088, + "fcm_dpo/beta": 0.010338194668293, + "fcm_dpo/delta": -0.06582339853048325, + "fcm_dpo/margin": 44.75118637084961, + "fcm_dpo/q_t": 0.3961416482925415, + "grad_norm": 14.283039093017578, + "learning_rate": 2.7507990434420123e-07, + "logits/chosen": 0.7346894145011902, + "logits/rejected": 0.6535829305648804, + "logps/chosen": -131.9852752685547, + "logps/ref_chosen": -56.285125732421875, + "logps/ref_rejected": -91.15303039550781, + "logps/rejected": -211.6043701171875, + "loss": 1.084, + "margin_dpo/margin_mean": 44.751190185546875, + "margin_dpo/margin_std": 64.32740020751953, + "step": 346 + }, + { + "KL/chosen_KL_mean": -82.76616668701172, + "KL/mean": -99.7265625, + "KL/rejected_KL_mean": -116.68695068359375, + "KL/std": 58.51282501220703, + "epoch": 0.5245653817082389, + "fcm_dpo/beta": 0.01033782958984375, + "fcm_dpo/delta": 0.05110158771276474, + "fcm_dpo/margin": 33.920780181884766, + "fcm_dpo/q_t": 0.4226782023906708, + "grad_norm": 17.104528427124023, + "learning_rate": 2.737640108260456e-07, + "logits/chosen": 0.8117585182189941, + "logits/rejected": 0.7625047564506531, + "logps/chosen": -136.26571655273438, + "logps/ref_chosen": -53.499542236328125, + "logps/ref_rejected": -72.52565002441406, + "logps/rejected": -189.21261596679688, + "loss": 1.1607, + "margin_dpo/margin_mean": 33.920780181884766, + "margin_dpo/margin_std": 61.820030212402344, + "step": 347 + }, + { + "KL/chosen_KL_mean": -73.69622039794922, + "KL/mean": -95.5750732421875, + "KL/rejected_KL_mean": -117.45392608642578, + "KL/std": 57.17124938964844, + "epoch": 0.5260770975056689, + "fcm_dpo/beta": 0.010252213105559349, + "fcm_dpo/delta": -0.051215190440416336, + "fcm_dpo/margin": 43.75770950317383, + "fcm_dpo/q_t": 0.40280699729919434, + "grad_norm": 12.792646408081055, + "learning_rate": 2.724474525774229e-07, + "logits/chosen": 0.816604495048523, + "logits/rejected": 0.7879601716995239, + "logps/chosen": -124.48307037353516, + "logps/ref_chosen": -50.78684997558594, + "logps/ref_rejected": -68.63732147216797, + "logps/rejected": -186.09124755859375, + "loss": 1.1036, + "margin_dpo/margin_mean": 43.75770568847656, + "margin_dpo/margin_std": 69.70893859863281, + "step": 348 + }, + { + "KL/chosen_KL_mean": -72.48434448242188, + "KL/mean": -94.19596099853516, + "KL/rejected_KL_mean": -115.9075927734375, + "KL/std": 58.99268341064453, + "epoch": 0.527588813303099, + "fcm_dpo/beta": 0.010226529091596603, + "fcm_dpo/delta": -0.046270184218883514, + "fcm_dpo/margin": 43.42324447631836, + "fcm_dpo/q_t": 0.4015776515007019, + "grad_norm": 14.216320037841797, + "learning_rate": 2.711302664252973e-07, + "logits/chosen": 0.7372004985809326, + "logits/rejected": 0.646864652633667, + "logps/chosen": -125.80935668945312, + "logps/ref_chosen": -53.325008392333984, + "logps/ref_rejected": -83.21236419677734, + "logps/rejected": -199.11996459960938, + "loss": 1.09, + "margin_dpo/margin_mean": 43.42324447631836, + "margin_dpo/margin_std": 63.5445556640625, + "step": 349 + }, + { + "KL/chosen_KL_mean": -78.7496109008789, + "KL/mean": -104.87985229492188, + "KL/rejected_KL_mean": -131.0100860595703, + "KL/std": 62.48160171508789, + "epoch": 0.5291005291005291, + "fcm_dpo/beta": 0.009954242035746574, + "fcm_dpo/delta": -0.1270718276500702, + "fcm_dpo/margin": 52.26048278808594, + "fcm_dpo/q_t": 0.38217905163764954, + "grad_norm": 15.84421157836914, + "learning_rate": 2.698124892141971e-07, + "logits/chosen": 0.6647555828094482, + "logits/rejected": 0.5865734219551086, + "logps/chosen": -140.37538146972656, + "logps/ref_chosen": -61.625770568847656, + "logps/ref_rejected": -87.63627624511719, + "logps/rejected": -218.6463623046875, + "loss": 1.0268, + "margin_dpo/margin_mean": 52.26047897338867, + "margin_dpo/margin_std": 64.52607727050781, + "step": 350 + }, + { + "KL/chosen_KL_mean": -74.44353485107422, + "KL/mean": -96.31437683105469, + "KL/rejected_KL_mean": -118.18523406982422, + "KL/std": 56.73322677612305, + "epoch": 0.5306122448979592, + "fcm_dpo/beta": 0.009825417771935463, + "fcm_dpo/delta": -0.03146745264530182, + "fcm_dpo/margin": 43.74169921875, + "fcm_dpo/q_t": 0.4013119041919708, + "grad_norm": 13.427343368530273, + "learning_rate": 2.6849415780518357e-07, + "logits/chosen": 0.6526576280593872, + "logits/rejected": 0.5777317881584167, + "logps/chosen": -130.6998748779297, + "logps/ref_chosen": -56.2563362121582, + "logps/ref_rejected": -79.11589813232422, + "logps/rejected": -197.30113220214844, + "loss": 1.1167, + "margin_dpo/margin_mean": 43.74169921875, + "margin_dpo/margin_std": 70.45606231689453, + "step": 351 + }, + { + "KL/chosen_KL_mean": -72.97793579101562, + "KL/mean": -95.73782348632812, + "KL/rejected_KL_mean": -118.4977035522461, + "KL/std": 58.78670883178711, + "epoch": 0.5321239606953893, + "fcm_dpo/beta": 0.009755909442901611, + "fcm_dpo/delta": -0.04631410166621208, + "fcm_dpo/margin": 45.51976776123047, + "fcm_dpo/q_t": 0.3998575806617737, + "grad_norm": 13.304686546325684, + "learning_rate": 2.6717530907482024e-07, + "logits/chosen": 0.7246947288513184, + "logits/rejected": 0.672113835811615, + "logps/chosen": -136.02987670898438, + "logps/ref_chosen": -63.05195236206055, + "logps/ref_rejected": -85.52035522460938, + "logps/rejected": -204.01806640625, + "loss": 1.0803, + "margin_dpo/margin_mean": 45.51976776123047, + "margin_dpo/margin_std": 63.94942855834961, + "step": 352 + }, + { + "KL/chosen_KL_mean": -70.28274536132812, + "KL/mean": -93.31478881835938, + "KL/rejected_KL_mean": -116.3468017578125, + "KL/std": 58.474891662597656, + "epoch": 0.5336356764928194, + "fcm_dpo/beta": 0.009713239967823029, + "fcm_dpo/delta": -0.04971469193696976, + "fcm_dpo/margin": 46.06404495239258, + "fcm_dpo/q_t": 0.39870840311050415, + "grad_norm": 11.444036483764648, + "learning_rate": 2.658559799141411e-07, + "logits/chosen": 0.7031028270721436, + "logits/rejected": 0.7094468474388123, + "logps/chosen": -139.29193115234375, + "logps/ref_chosen": -69.00918579101562, + "logps/ref_rejected": -72.65840148925781, + "logps/rejected": -189.0052032470703, + "loss": 1.083, + "margin_dpo/margin_mean": 46.064048767089844, + "margin_dpo/margin_std": 65.06970977783203, + "step": 353 + }, + { + "KL/chosen_KL_mean": -74.60330963134766, + "KL/mean": -99.59640502929688, + "KL/rejected_KL_mean": -124.5895004272461, + "KL/std": 57.67888641357422, + "epoch": 0.5351473922902494, + "fcm_dpo/beta": 0.009495913982391357, + "fcm_dpo/delta": -0.07927623391151428, + "fcm_dpo/margin": 49.98619079589844, + "fcm_dpo/q_t": 0.3928118348121643, + "grad_norm": 13.079569816589355, + "learning_rate": 2.6453620722761895e-07, + "logits/chosen": 0.7806761264801025, + "logits/rejected": 0.6481184959411621, + "logps/chosen": -114.39164733886719, + "logps/ref_chosen": -39.78833770751953, + "logps/ref_rejected": -69.56885528564453, + "logps/rejected": -194.15835571289062, + "loss": 1.0723, + "margin_dpo/margin_mean": 49.98619079589844, + "margin_dpo/margin_std": 69.9796142578125, + "step": 354 + }, + { + "KL/chosen_KL_mean": -78.99346923828125, + "KL/mean": -104.57441711425781, + "KL/rejected_KL_mean": -130.1553497314453, + "KL/std": 62.086181640625, + "epoch": 0.5366591080876795, + "fcm_dpo/beta": 0.00941452570259571, + "fcm_dpo/delta": -0.0857347846031189, + "fcm_dpo/margin": 51.16187286376953, + "fcm_dpo/q_t": 0.3914402723312378, + "grad_norm": 15.902485847473145, + "learning_rate": 2.632160279321328e-07, + "logits/chosen": 0.7517096996307373, + "logits/rejected": 0.6187185645103455, + "logps/chosen": -125.24885559082031, + "logps/ref_chosen": -46.25537872314453, + "logps/ref_rejected": -78.20236206054688, + "logps/rejected": -208.3577117919922, + "loss": 1.0716, + "margin_dpo/margin_mean": 51.16187286376953, + "margin_dpo/margin_std": 72.24238586425781, + "step": 355 + }, + { + "KL/chosen_KL_mean": -73.79627990722656, + "KL/mean": -94.85540771484375, + "KL/rejected_KL_mean": -115.91452026367188, + "KL/std": 61.34975051879883, + "epoch": 0.5381708238851096, + "fcm_dpo/beta": 0.00931059941649437, + "fcm_dpo/delta": 0.007776662707328796, + "fcm_dpo/margin": 42.118247985839844, + "fcm_dpo/q_t": 0.4127386808395386, + "grad_norm": 12.62276554107666, + "learning_rate": 2.618954789559356e-07, + "logits/chosen": 0.7203996181488037, + "logits/rejected": 0.64216548204422, + "logps/chosen": -121.70243835449219, + "logps/ref_chosen": -47.906158447265625, + "logps/ref_rejected": -74.29397583007812, + "logps/rejected": -190.20849609375, + "loss": 1.1675, + "margin_dpo/margin_mean": 42.118247985839844, + "margin_dpo/margin_std": 81.22731018066406, + "step": 356 + }, + { + "KL/chosen_KL_mean": -89.2645034790039, + "KL/mean": -108.21392822265625, + "KL/rejected_KL_mean": -127.16334533691406, + "KL/std": 59.75215530395508, + "epoch": 0.5396825396825397, + "fcm_dpo/beta": 0.009327213279902935, + "fcm_dpo/delta": 0.04655788093805313, + "fcm_dpo/margin": 37.898834228515625, + "fcm_dpo/q_t": 0.41907134652137756, + "grad_norm": 12.688922882080078, + "learning_rate": 2.6057459723762076e-07, + "logits/chosen": 0.6992335319519043, + "logits/rejected": 0.6748742461204529, + "logps/chosen": -151.89950561523438, + "logps/ref_chosen": -62.63500213623047, + "logps/ref_rejected": -65.11399841308594, + "logps/rejected": -192.27734375, + "loss": 1.1569, + "margin_dpo/margin_mean": 37.898834228515625, + "margin_dpo/margin_std": 63.26298904418945, + "step": 357 + }, + { + "KL/chosen_KL_mean": -83.01390075683594, + "KL/mean": -109.11406707763672, + "KL/rejected_KL_mean": -135.2142333984375, + "KL/std": 60.27477264404297, + "epoch": 0.5411942554799698, + "fcm_dpo/beta": 0.009361564181745052, + "fcm_dpo/delta": -0.0932769775390625, + "fcm_dpo/margin": 52.2003288269043, + "fcm_dpo/q_t": 0.3902199864387512, + "grad_norm": 15.763985633850098, + "learning_rate": 2.5925341972508954e-07, + "logits/chosen": 0.6433045864105225, + "logits/rejected": 0.6595355272293091, + "logps/chosen": -150.2235107421875, + "logps/ref_chosen": -67.20960998535156, + "logps/ref_rejected": -69.34715270996094, + "logps/rejected": -204.5614013671875, + "loss": 1.0522, + "margin_dpo/margin_mean": 52.20033264160156, + "margin_dpo/margin_std": 68.1099853515625, + "step": 358 + }, + { + "KL/chosen_KL_mean": -94.01116943359375, + "KL/mean": -106.93074035644531, + "KL/rejected_KL_mean": -119.85032653808594, + "KL/std": 60.080284118652344, + "epoch": 0.5427059712773998, + "fcm_dpo/beta": 0.009553344920277596, + "fcm_dpo/delta": 0.15497523546218872, + "fcm_dpo/margin": 25.83917236328125, + "fcm_dpo/q_t": 0.4445451498031616, + "grad_norm": 13.971330642700195, + "learning_rate": 2.579319833745169e-07, + "logits/chosen": 0.6465896368026733, + "logits/rejected": 0.6198573112487793, + "logps/chosen": -156.53695678710938, + "logps/ref_chosen": -62.52578353881836, + "logps/ref_rejected": -76.63114929199219, + "logps/rejected": -196.48147583007812, + "loss": 1.2395, + "margin_dpo/margin_mean": 25.83917236328125, + "margin_dpo/margin_std": 60.63203430175781, + "step": 359 + }, + { + "KL/chosen_KL_mean": -92.86167907714844, + "KL/mean": -114.65642547607422, + "KL/rejected_KL_mean": -136.45118713378906, + "KL/std": 63.91606140136719, + "epoch": 0.54421768707483, + "fcm_dpo/beta": 0.00952062476426363, + "fcm_dpo/delta": -0.01576114259660244, + "fcm_dpo/margin": 43.589508056640625, + "fcm_dpo/q_t": 0.40740275382995605, + "grad_norm": 12.668853759765625, + "learning_rate": 2.5661032514931834e-07, + "logits/chosen": 0.6336207985877991, + "logits/rejected": 0.5403010249137878, + "logps/chosen": -156.34939575195312, + "logps/ref_chosen": -63.48772048950195, + "logps/ref_rejected": -90.6891098022461, + "logps/rejected": -227.14028930664062, + "loss": 1.1058, + "margin_dpo/margin_mean": 43.589508056640625, + "margin_dpo/margin_std": 66.03152465820312, + "step": 360 + }, + { + "KL/chosen_KL_mean": -91.1150894165039, + "KL/mean": -116.79319763183594, + "KL/rejected_KL_mean": -142.4713134765625, + "KL/std": 64.54615783691406, + "epoch": 0.54572940287226, + "fcm_dpo/beta": 0.009435942396521568, + "fcm_dpo/delta": -0.08882280439138412, + "fcm_dpo/margin": 51.35621643066406, + "fcm_dpo/q_t": 0.3894497752189636, + "grad_norm": 12.657031059265137, + "learning_rate": 2.552884820191154e-07, + "logits/chosen": 0.7684783935546875, + "logits/rejected": 0.7189006805419922, + "logps/chosen": -149.0322265625, + "logps/ref_chosen": -57.917144775390625, + "logps/ref_rejected": -72.39089965820312, + "logps/rejected": -214.86221313476562, + "loss": 1.0443, + "margin_dpo/margin_mean": 51.35621643066406, + "margin_dpo/margin_std": 63.71247863769531, + "step": 361 + }, + { + "KL/chosen_KL_mean": -92.40254211425781, + "KL/mean": -118.06507873535156, + "KL/rejected_KL_mean": -143.7276153564453, + "KL/std": 65.0129623413086, + "epoch": 0.54724111866969, + "fcm_dpo/beta": 0.009297511540353298, + "fcm_dpo/delta": -0.08115798979997635, + "fcm_dpo/margin": 51.3250617980957, + "fcm_dpo/q_t": 0.3938571810722351, + "grad_norm": 14.403992652893066, + "learning_rate": 2.53966490958702e-07, + "logits/chosen": 0.7443888783454895, + "logits/rejected": 0.637453556060791, + "logps/chosen": -155.84600830078125, + "logps/ref_chosen": -63.4434700012207, + "logps/ref_rejected": -103.45516967773438, + "logps/rejected": -247.1827850341797, + "loss": 1.0805, + "margin_dpo/margin_mean": 51.32505798339844, + "margin_dpo/margin_std": 75.57747650146484, + "step": 362 + }, + { + "KL/chosen_KL_mean": -98.28819274902344, + "KL/mean": -123.49717712402344, + "KL/rejected_KL_mean": -148.7061767578125, + "KL/std": 60.82764434814453, + "epoch": 0.5487528344671202, + "fcm_dpo/beta": 0.009109338745474815, + "fcm_dpo/delta": -0.06234493851661682, + "fcm_dpo/margin": 50.41798782348633, + "fcm_dpo/q_t": 0.3953966796398163, + "grad_norm": 15.891867637634277, + "learning_rate": 2.526443889470099e-07, + "logits/chosen": 0.7446722984313965, + "logits/rejected": 0.6137604713439941, + "logps/chosen": -146.9400177001953, + "logps/ref_chosen": -48.65182876586914, + "logps/ref_rejected": -88.65904235839844, + "logps/rejected": -237.365234375, + "loss": 1.0679, + "margin_dpo/margin_mean": 50.41798782348633, + "margin_dpo/margin_std": 68.12222290039062, + "step": 363 + }, + { + "KL/chosen_KL_mean": -88.94969177246094, + "KL/mean": -116.70195007324219, + "KL/rejected_KL_mean": -144.45420837402344, + "KL/std": 67.17678833007812, + "epoch": 0.5502645502645502, + "fcm_dpo/beta": 0.008948586881160736, + "fcm_dpo/delta": -0.10194439440965652, + "fcm_dpo/margin": 55.50453186035156, + "fcm_dpo/q_t": 0.38964322209358215, + "grad_norm": 11.782469749450684, + "learning_rate": 2.513222129660744e-07, + "logits/chosen": 0.6123770475387573, + "logits/rejected": 0.5227110981941223, + "logps/chosen": -146.82077026367188, + "logps/ref_chosen": -57.87107467651367, + "logps/ref_rejected": -80.95503234863281, + "logps/rejected": -225.40924072265625, + "loss": 1.0762, + "margin_dpo/margin_mean": 55.50453186035156, + "margin_dpo/margin_std": 82.74433898925781, + "step": 364 + }, + { + "KL/chosen_KL_mean": -79.53478240966797, + "KL/mean": -105.70310974121094, + "KL/rejected_KL_mean": -131.87142944335938, + "KL/std": 68.90129089355469, + "epoch": 0.5517762660619804, + "fcm_dpo/beta": 0.008822593837976456, + "fcm_dpo/delta": -0.06480256468057632, + "fcm_dpo/margin": 52.3366584777832, + "fcm_dpo/q_t": 0.39349961280822754, + "grad_norm": 11.822827339172363, + "learning_rate": 2.5e-07, + "logits/chosen": 0.7287126183509827, + "logits/rejected": 0.7254834175109863, + "logps/chosen": -144.47695922851562, + "logps/ref_chosen": -64.94217681884766, + "logps/ref_rejected": -74.8599853515625, + "logps/rejected": -206.73141479492188, + "loss": 1.0442, + "margin_dpo/margin_mean": 52.3366584777832, + "margin_dpo/margin_std": 60.6138801574707, + "step": 365 + }, + { + "KL/chosen_KL_mean": -80.14967346191406, + "KL/mean": -102.59008026123047, + "KL/rejected_KL_mean": -125.03048706054688, + "KL/std": 61.42876434326172, + "epoch": 0.5532879818594104, + "fcm_dpo/beta": 0.008836669847369194, + "fcm_dpo/delta": 0.003269568085670471, + "fcm_dpo/margin": 44.88079833984375, + "fcm_dpo/q_t": 0.41164153814315796, + "grad_norm": 13.908421516418457, + "learning_rate": 2.486777870339255e-07, + "logits/chosen": 0.6701743602752686, + "logits/rejected": 0.6558536291122437, + "logps/chosen": -135.31565856933594, + "logps/ref_chosen": -55.16598129272461, + "logps/ref_rejected": -65.26121520996094, + "logps/rejected": -190.29168701171875, + "loss": 1.1444, + "margin_dpo/margin_mean": 44.88079833984375, + "margin_dpo/margin_std": 79.36189270019531, + "step": 366 + }, + { + "KL/chosen_KL_mean": -89.8109130859375, + "KL/mean": -112.27853393554688, + "KL/rejected_KL_mean": -134.7461700439453, + "KL/std": 62.997825622558594, + "epoch": 0.5547996976568406, + "fcm_dpo/beta": 0.008791262283921242, + "fcm_dpo/delta": 0.0050530992448329926, + "fcm_dpo/margin": 44.93526077270508, + "fcm_dpo/q_t": 0.4080847501754761, + "grad_norm": 12.340958595275879, + "learning_rate": 2.4735561105299014e-07, + "logits/chosen": 0.7136199474334717, + "logits/rejected": 0.6086731553077698, + "logps/chosen": -145.82138061523438, + "logps/ref_chosen": -56.01046371459961, + "logps/ref_rejected": -77.31010437011719, + "logps/rejected": -212.0562744140625, + "loss": 1.1178, + "margin_dpo/margin_mean": 44.935264587402344, + "margin_dpo/margin_std": 69.36185455322266, + "step": 367 + }, + { + "KL/chosen_KL_mean": -99.1821517944336, + "KL/mean": -120.2548828125, + "KL/rejected_KL_mean": -141.32760620117188, + "KL/std": 63.947425842285156, + "epoch": 0.5563114134542706, + "fcm_dpo/beta": 0.008854447863996029, + "fcm_dpo/delta": 0.02785618230700493, + "fcm_dpo/margin": 42.145442962646484, + "fcm_dpo/q_t": 0.4143589735031128, + "grad_norm": 12.742534637451172, + "learning_rate": 2.46033509041298e-07, + "logits/chosen": 0.5594249963760376, + "logits/rejected": 0.5591206550598145, + "logps/chosen": -174.0114288330078, + "logps/ref_chosen": -74.82927703857422, + "logps/ref_rejected": -76.11680603027344, + "logps/rejected": -217.44439697265625, + "loss": 1.1347, + "margin_dpo/margin_mean": 42.14544677734375, + "margin_dpo/margin_std": 68.57220458984375, + "step": 368 + }, + { + "KL/chosen_KL_mean": -93.78787231445312, + "KL/mean": -112.06151580810547, + "KL/rejected_KL_mean": -130.33514404296875, + "KL/std": 63.32222366333008, + "epoch": 0.5578231292517006, + "fcm_dpo/beta": 0.009002558887004852, + "fcm_dpo/delta": 0.0726294070482254, + "fcm_dpo/margin": 36.547271728515625, + "fcm_dpo/q_t": 0.42570725083351135, + "grad_norm": 13.213028907775879, + "learning_rate": 2.447115179808846e-07, + "logits/chosen": 0.7237619757652283, + "logits/rejected": 0.6711267232894897, + "logps/chosen": -152.1140899658203, + "logps/ref_chosen": -58.32621765136719, + "logps/ref_rejected": -80.92183685302734, + "logps/rejected": -211.25698852539062, + "loss": 1.1729, + "margin_dpo/margin_mean": 36.547271728515625, + "margin_dpo/margin_std": 67.68240356445312, + "step": 369 + }, + { + "KL/chosen_KL_mean": -89.63414001464844, + "KL/mean": -115.5050048828125, + "KL/rejected_KL_mean": -141.37588500976562, + "KL/std": 66.52014923095703, + "epoch": 0.5593348450491308, + "fcm_dpo/beta": 0.008897393941879272, + "fcm_dpo/delta": -0.06340888887643814, + "fcm_dpo/margin": 51.74176025390625, + "fcm_dpo/q_t": 0.39567339420318604, + "grad_norm": 13.42084789276123, + "learning_rate": 2.4338967485068164e-07, + "logits/chosen": 0.8113032579421997, + "logits/rejected": 0.7431550025939941, + "logps/chosen": -142.51785278320312, + "logps/ref_chosen": -52.88372039794922, + "logps/ref_rejected": -79.43692016601562, + "logps/rejected": -220.81280517578125, + "loss": 1.0872, + "margin_dpo/margin_mean": 51.741764068603516, + "margin_dpo/margin_std": 77.3583984375, + "step": 370 + }, + { + "KL/chosen_KL_mean": -89.46411895751953, + "KL/mean": -112.49140930175781, + "KL/rejected_KL_mean": -135.51870727539062, + "KL/std": 65.15025329589844, + "epoch": 0.5608465608465608, + "fcm_dpo/beta": 0.00891664158552885, + "fcm_dpo/delta": -0.011712536215782166, + "fcm_dpo/margin": 46.05458068847656, + "fcm_dpo/q_t": 0.40757519006729126, + "grad_norm": 15.0881986618042, + "learning_rate": 2.420680166254831e-07, + "logits/chosen": 0.8154574632644653, + "logits/rejected": 0.7834243774414062, + "logps/chosen": -138.68832397460938, + "logps/ref_chosen": -49.224212646484375, + "logps/ref_rejected": -63.348472595214844, + "logps/rejected": -198.86717224121094, + "loss": 1.1103, + "margin_dpo/margin_mean": 46.05458068847656, + "margin_dpo/margin_std": 69.55319213867188, + "step": 371 + }, + { + "KL/chosen_KL_mean": -96.4041748046875, + "KL/mean": -110.4830551147461, + "KL/rejected_KL_mean": -124.56193542480469, + "KL/std": 64.20169830322266, + "epoch": 0.562358276643991, + "fcm_dpo/beta": 0.009113049134612083, + "fcm_dpo/delta": 0.1454039216041565, + "fcm_dpo/margin": 28.15774917602539, + "fcm_dpo/q_t": 0.443379282951355, + "grad_norm": 15.851375579833984, + "learning_rate": 2.4074658027491044e-07, + "logits/chosen": 0.7606231570243835, + "logits/rejected": 0.6628408432006836, + "logps/chosen": -148.67373657226562, + "logps/ref_chosen": -52.269554138183594, + "logps/ref_rejected": -72.99522399902344, + "logps/rejected": -197.55715942382812, + "loss": 1.2817, + "margin_dpo/margin_mean": 28.15774917602539, + "margin_dpo/margin_std": 82.369384765625, + "step": 372 + }, + { + "KL/chosen_KL_mean": -105.25238800048828, + "KL/mean": -124.43365478515625, + "KL/rejected_KL_mean": -143.61492919921875, + "KL/std": 64.15481567382812, + "epoch": 0.563869992441421, + "fcm_dpo/beta": 0.009200419299304485, + "fcm_dpo/delta": 0.04866807162761688, + "fcm_dpo/margin": 38.362518310546875, + "fcm_dpo/q_t": 0.422132670879364, + "grad_norm": 14.45687198638916, + "learning_rate": 2.394254027623792e-07, + "logits/chosen": 0.7406034469604492, + "logits/rejected": 0.6692646741867065, + "logps/chosen": -166.36538696289062, + "logps/ref_chosen": -61.112998962402344, + "logps/ref_rejected": -76.24851989746094, + "logps/rejected": -219.86343383789062, + "loss": 1.2032, + "margin_dpo/margin_mean": 38.362518310546875, + "margin_dpo/margin_std": 83.87307739257812, + "step": 373 + }, + { + "KL/chosen_KL_mean": -87.62055206298828, + "KL/mean": -120.09354400634766, + "KL/rejected_KL_mean": -152.56654357910156, + "KL/std": 64.96662139892578, + "epoch": 0.5653817082388511, + "fcm_dpo/beta": 0.00893518328666687, + "fcm_dpo/delta": -0.19185763597488403, + "fcm_dpo/margin": 64.94598388671875, + "fcm_dpo/q_t": 0.36740460991859436, + "grad_norm": 15.2799072265625, + "learning_rate": 2.381045210440644e-07, + "logits/chosen": 0.6385786533355713, + "logits/rejected": 0.6403213739395142, + "logps/chosen": -160.28976440429688, + "logps/ref_chosen": -72.66920471191406, + "logps/ref_rejected": -76.83158874511719, + "logps/rejected": -229.39813232421875, + "loss": 0.9805, + "margin_dpo/margin_mean": 64.94598388671875, + "margin_dpo/margin_std": 71.19766998291016, + "step": 374 + }, + { + "KL/chosen_KL_mean": -86.73007202148438, + "KL/mean": -108.32252502441406, + "KL/rejected_KL_mean": -129.91497802734375, + "KL/std": 67.20675659179688, + "epoch": 0.5668934240362812, + "fcm_dpo/beta": 0.008861735463142395, + "fcm_dpo/delta": 0.01787165366113186, + "fcm_dpo/margin": 43.18490982055664, + "fcm_dpo/q_t": 0.41384047269821167, + "grad_norm": 14.840850830078125, + "learning_rate": 2.3678397206786715e-07, + "logits/chosen": 0.7619016170501709, + "logits/rejected": 0.7026609778404236, + "logps/chosen": -144.4133758544922, + "logps/ref_chosen": -57.68330383300781, + "logps/ref_rejected": -79.34097290039062, + "logps/rejected": -209.25595092773438, + "loss": 1.1436, + "margin_dpo/margin_mean": 43.184906005859375, + "margin_dpo/margin_std": 74.84222412109375, + "step": 375 + }, + { + "KL/chosen_KL_mean": -94.26927185058594, + "KL/mean": -121.34205627441406, + "KL/rejected_KL_mean": -148.4148406982422, + "KL/std": 67.80638122558594, + "epoch": 0.5684051398337112, + "fcm_dpo/beta": 0.008781617507338524, + "fcm_dpo/delta": -0.07936884462833405, + "fcm_dpo/margin": 54.14557647705078, + "fcm_dpo/q_t": 0.3939359188079834, + "grad_norm": 14.06691837310791, + "learning_rate": 2.3546379277238103e-07, + "logits/chosen": 0.7732566595077515, + "logits/rejected": 0.7034204006195068, + "logps/chosen": -145.94332885742188, + "logps/ref_chosen": -51.674072265625, + "logps/ref_rejected": -75.69713592529297, + "logps/rejected": -224.11196899414062, + "loss": 1.078, + "margin_dpo/margin_mean": 54.14557647705078, + "margin_dpo/margin_std": 79.33256530761719, + "step": 376 + }, + { + "KL/chosen_KL_mean": -98.88938903808594, + "KL/mean": -118.69151306152344, + "KL/rejected_KL_mean": -138.49365234375, + "KL/std": 63.79244613647461, + "epoch": 0.5699168556311414, + "fcm_dpo/beta": 0.00886094756424427, + "fcm_dpo/delta": 0.05022633075714111, + "fcm_dpo/margin": 39.60426330566406, + "fcm_dpo/q_t": 0.42025965452194214, + "grad_norm": 14.110357284545898, + "learning_rate": 2.3414402008585886e-07, + "logits/chosen": 0.7264569997787476, + "logits/rejected": 0.7055130004882812, + "logps/chosen": -145.0679168701172, + "logps/ref_chosen": -46.17853546142578, + "logps/ref_rejected": -57.756500244140625, + "logps/rejected": -196.25015258789062, + "loss": 1.1649, + "margin_dpo/margin_mean": 39.60426330566406, + "margin_dpo/margin_std": 72.60446166992188, + "step": 377 + }, + { + "KL/chosen_KL_mean": -94.4163818359375, + "KL/mean": -113.24252319335938, + "KL/rejected_KL_mean": -132.0686798095703, + "KL/std": 66.46366882324219, + "epoch": 0.5714285714285714, + "fcm_dpo/beta": 0.00897032581269741, + "fcm_dpo/delta": 0.0636821836233139, + "fcm_dpo/margin": 37.652305603027344, + "fcm_dpo/q_t": 0.423561155796051, + "grad_norm": 13.255199432373047, + "learning_rate": 2.3282469092517977e-07, + "logits/chosen": 0.7863258719444275, + "logits/rejected": 0.7367914319038391, + "logps/chosen": -153.63525390625, + "logps/ref_chosen": -59.21887969970703, + "logps/ref_rejected": -71.24818420410156, + "logps/rejected": -203.31686401367188, + "loss": 1.1713, + "margin_dpo/margin_mean": 37.652305603027344, + "margin_dpo/margin_std": 69.34526062011719, + "step": 378 + }, + { + "KL/chosen_KL_mean": -89.37190246582031, + "KL/mean": -113.827880859375, + "KL/rejected_KL_mean": -138.28387451171875, + "KL/std": 65.70942687988281, + "epoch": 0.5729402872260015, + "fcm_dpo/beta": 0.008909247815608978, + "fcm_dpo/delta": -0.03739330917596817, + "fcm_dpo/margin": 48.91199493408203, + "fcm_dpo/q_t": 0.40265560150146484, + "grad_norm": 14.700591087341309, + "learning_rate": 2.3150584219481643e-07, + "logits/chosen": 0.7219586372375488, + "logits/rejected": 0.6492105722427368, + "logps/chosen": -165.6884765625, + "logps/ref_chosen": -76.31658935546875, + "logps/ref_rejected": -104.26200103759766, + "logps/rejected": -242.54588317871094, + "loss": 1.0938, + "margin_dpo/margin_mean": 48.91199493408203, + "margin_dpo/margin_std": 72.89404296875, + "step": 379 + }, + { + "KL/chosen_KL_mean": -79.29644012451172, + "KL/mean": -110.41128540039062, + "KL/rejected_KL_mean": -141.52613830566406, + "KL/std": 63.3236083984375, + "epoch": 0.5744520030234316, + "fcm_dpo/beta": 0.008703596889972687, + "fcm_dpo/delta": -0.14961189031600952, + "fcm_dpo/margin": 62.229698181152344, + "fcm_dpo/q_t": 0.3763843774795532, + "grad_norm": 13.61119556427002, + "learning_rate": 2.3018751078580283e-07, + "logits/chosen": 0.7565820217132568, + "logits/rejected": 0.7187171578407288, + "logps/chosen": -140.57960510253906, + "logps/ref_chosen": -61.283164978027344, + "logps/ref_rejected": -72.38892364501953, + "logps/rejected": -213.91505432128906, + "loss": 1.0228, + "margin_dpo/margin_mean": 62.229698181152344, + "margin_dpo/margin_std": 77.56293487548828, + "step": 380 + }, + { + "KL/chosen_KL_mean": -97.59381866455078, + "KL/mean": -109.89251708984375, + "KL/rejected_KL_mean": -122.19123077392578, + "KL/std": 63.37030029296875, + "epoch": 0.5759637188208617, + "fcm_dpo/beta": 0.008918230421841145, + "fcm_dpo/delta": 0.18300682306289673, + "fcm_dpo/margin": 24.597408294677734, + "fcm_dpo/q_t": 0.4514863193035126, + "grad_norm": 12.83474349975586, + "learning_rate": 2.288697335747027e-07, + "logits/chosen": 0.704852819442749, + "logits/rejected": 0.6807745695114136, + "logps/chosen": -155.80780029296875, + "logps/ref_chosen": -58.2139892578125, + "logps/ref_rejected": -60.78669357299805, + "logps/rejected": -182.97792053222656, + "loss": 1.2863, + "margin_dpo/margin_mean": 24.597410202026367, + "margin_dpo/margin_std": 74.75566101074219, + "step": 381 + }, + { + "KL/chosen_KL_mean": -98.49388122558594, + "KL/mean": -119.5999526977539, + "KL/rejected_KL_mean": -140.70602416992188, + "KL/std": 65.70405578613281, + "epoch": 0.5774754346182918, + "fcm_dpo/beta": 0.009017659351229668, + "fcm_dpo/delta": 0.019596684724092484, + "fcm_dpo/margin": 42.21213150024414, + "fcm_dpo/q_t": 0.4132706820964813, + "grad_norm": 13.350465774536133, + "learning_rate": 2.2755254742257706e-07, + "logits/chosen": 0.7516306638717651, + "logits/rejected": 0.6984708309173584, + "logps/chosen": -160.3192138671875, + "logps/ref_chosen": -61.82532501220703, + "logps/ref_rejected": -83.0452880859375, + "logps/rejected": -223.7512969970703, + "loss": 1.1191, + "margin_dpo/margin_mean": 42.212135314941406, + "margin_dpo/margin_std": 62.93214416503906, + "step": 382 + }, + { + "KL/chosen_KL_mean": -95.25296783447266, + "KL/mean": -118.4464111328125, + "KL/rejected_KL_mean": -141.63986206054688, + "KL/std": 65.69584655761719, + "epoch": 0.5789871504157218, + "fcm_dpo/beta": 0.008922882378101349, + "fcm_dpo/delta": -0.015040282160043716, + "fcm_dpo/margin": 46.386905670166016, + "fcm_dpo/q_t": 0.4087279438972473, + "grad_norm": 14.565018653869629, + "learning_rate": 2.2623598917395436e-07, + "logits/chosen": 0.5949560403823853, + "logits/rejected": 0.6251660585403442, + "logps/chosen": -175.81622314453125, + "logps/ref_chosen": -80.56326293945312, + "logps/ref_rejected": -74.62922668457031, + "logps/rejected": -216.26910400390625, + "loss": 1.1348, + "margin_dpo/margin_mean": 46.386905670166016, + "margin_dpo/margin_std": 80.02427673339844, + "step": 383 + }, + { + "KL/chosen_KL_mean": -95.48326110839844, + "KL/mean": -117.88404083251953, + "KL/rejected_KL_mean": -140.28482055664062, + "KL/std": 65.55471801757812, + "epoch": 0.5804988662131519, + "fcm_dpo/beta": 0.00895443931221962, + "fcm_dpo/delta": -0.001223698491230607, + "fcm_dpo/margin": 44.801551818847656, + "fcm_dpo/q_t": 0.4078155755996704, + "grad_norm": 14.621257781982422, + "learning_rate": 2.2492009565579875e-07, + "logits/chosen": 0.733665943145752, + "logits/rejected": 0.6879531145095825, + "logps/chosen": -160.95840454101562, + "logps/ref_chosen": -65.47514343261719, + "logps/ref_rejected": -79.67378234863281, + "logps/rejected": -219.95860290527344, + "loss": 1.113, + "margin_dpo/margin_mean": 44.801551818847656, + "margin_dpo/margin_std": 68.81768798828125, + "step": 384 + }, + { + "KL/chosen_KL_mean": -93.10316467285156, + "KL/mean": -122.23687744140625, + "KL/rejected_KL_mean": -151.37060546875, + "KL/std": 64.7177505493164, + "epoch": 0.582010582010582, + "fcm_dpo/beta": 0.008825909346342087, + "fcm_dpo/delta": -0.12028289586305618, + "fcm_dpo/margin": 58.26744842529297, + "fcm_dpo/q_t": 0.38351568579673767, + "grad_norm": 12.378807067871094, + "learning_rate": 2.2360490367648084e-07, + "logits/chosen": 0.6500192284584045, + "logits/rejected": 0.6126409769058228, + "logps/chosen": -159.15966796875, + "logps/ref_chosen": -66.0565185546875, + "logps/ref_rejected": -86.68023681640625, + "logps/rejected": -238.05084228515625, + "loss": 1.0231, + "margin_dpo/margin_mean": 58.26744842529297, + "margin_dpo/margin_std": 68.7908706665039, + "step": 385 + }, + { + "KL/chosen_KL_mean": -107.4451904296875, + "KL/mean": -125.80526733398438, + "KL/rejected_KL_mean": -144.16531372070312, + "KL/std": 64.58238220214844, + "epoch": 0.5835222978080121, + "fcm_dpo/beta": 0.008834145963191986, + "fcm_dpo/delta": 0.0782126784324646, + "fcm_dpo/margin": 36.72013473510742, + "fcm_dpo/q_t": 0.4248654842376709, + "grad_norm": 14.023968696594238, + "learning_rate": 2.2229045002474724e-07, + "logits/chosen": 0.6312617659568787, + "logits/rejected": 0.5722914934158325, + "logps/chosen": -183.06884765625, + "logps/ref_chosen": -75.6236572265625, + "logps/ref_rejected": -92.62330627441406, + "logps/rejected": -236.78863525390625, + "loss": 1.1725, + "margin_dpo/margin_mean": 36.72013473510742, + "margin_dpo/margin_std": 67.48342895507812, + "step": 386 + }, + { + "KL/chosen_KL_mean": -95.84181213378906, + "KL/mean": -123.29605102539062, + "KL/rejected_KL_mean": -150.75030517578125, + "KL/std": 63.237667083740234, + "epoch": 0.5850340136054422, + "fcm_dpo/beta": 0.008767899125814438, + "fcm_dpo/delta": -0.08556218445301056, + "fcm_dpo/margin": 54.908485412597656, + "fcm_dpo/q_t": 0.39054542779922485, + "grad_norm": 12.948434829711914, + "learning_rate": 2.209767714686924e-07, + "logits/chosen": 0.7226855754852295, + "logits/rejected": 0.618818998336792, + "logps/chosen": -143.06350708007812, + "logps/ref_chosen": -47.22170639038086, + "logps/ref_rejected": -87.338134765625, + "logps/rejected": -238.08843994140625, + "loss": 1.0379, + "margin_dpo/margin_mean": 54.908485412597656, + "margin_dpo/margin_std": 65.7297592163086, + "step": 387 + }, + { + "KL/chosen_KL_mean": -96.84806823730469, + "KL/mean": -114.32777404785156, + "KL/rejected_KL_mean": -131.8074951171875, + "KL/std": 64.9911880493164, + "epoch": 0.5865457294028723, + "fcm_dpo/beta": 0.008810779079794884, + "fcm_dpo/delta": 0.09486287087202072, + "fcm_dpo/margin": 34.95942306518555, + "fcm_dpo/q_t": 0.4320222735404968, + "grad_norm": 12.77639102935791, + "learning_rate": 2.1966390475472954e-07, + "logits/chosen": 0.7181829810142517, + "logits/rejected": 0.7101290225982666, + "logps/chosen": -171.42755126953125, + "logps/ref_chosen": -74.5794677734375, + "logps/ref_rejected": -79.92558288574219, + "logps/rejected": -211.7330780029297, + "loss": 1.2113, + "margin_dpo/margin_mean": 34.95942306518555, + "margin_dpo/margin_std": 76.60311126708984, + "step": 388 + }, + { + "KL/chosen_KL_mean": -93.4125747680664, + "KL/mean": -121.59010314941406, + "KL/rejected_KL_mean": -149.76763916015625, + "KL/std": 66.75245666503906, + "epoch": 0.5880574452003023, + "fcm_dpo/beta": 0.008777271024882793, + "fcm_dpo/delta": -0.09948212653398514, + "fcm_dpo/margin": 56.35505294799805, + "fcm_dpo/q_t": 0.3881889581680298, + "grad_norm": 28.938011169433594, + "learning_rate": 2.1835188660656265e-07, + "logits/chosen": 0.7351957559585571, + "logits/rejected": 0.6990658044815063, + "logps/chosen": -155.0369415283203, + "logps/ref_chosen": -61.624366760253906, + "logps/ref_rejected": -76.50978088378906, + "logps/rejected": -226.2774200439453, + "loss": 1.0491, + "margin_dpo/margin_mean": 56.35505676269531, + "margin_dpo/margin_std": 74.05264282226562, + "step": 389 + }, + { + "KL/chosen_KL_mean": -86.54637145996094, + "KL/mean": -108.1448974609375, + "KL/rejected_KL_mean": -129.74343872070312, + "KL/std": 63.3463134765625, + "epoch": 0.5895691609977324, + "fcm_dpo/beta": 0.008739067241549492, + "fcm_dpo/delta": 0.023366158828139305, + "fcm_dpo/margin": 43.197052001953125, + "fcm_dpo/q_t": 0.4136941432952881, + "grad_norm": 10.710925102233887, + "learning_rate": 2.170407537241599e-07, + "logits/chosen": 0.7972933650016785, + "logits/rejected": 0.7268404960632324, + "logps/chosen": -132.41824340820312, + "logps/ref_chosen": -45.871864318847656, + "logps/ref_rejected": -61.305999755859375, + "logps/rejected": -191.0494384765625, + "loss": 1.1237, + "margin_dpo/margin_mean": 43.197052001953125, + "margin_dpo/margin_std": 66.48844146728516, + "step": 390 + }, + { + "KL/chosen_KL_mean": -93.65834045410156, + "KL/mean": -119.96128845214844, + "KL/rejected_KL_mean": -146.26422119140625, + "KL/std": 64.77439880371094, + "epoch": 0.5910808767951625, + "fcm_dpo/beta": 0.008656751364469528, + "fcm_dpo/delta": -0.05850217863917351, + "fcm_dpo/margin": 52.605873107910156, + "fcm_dpo/q_t": 0.3968993127346039, + "grad_norm": 12.943575859069824, + "learning_rate": 2.1573054278272636e-07, + "logits/chosen": 0.728482723236084, + "logits/rejected": 0.6600795984268188, + "logps/chosen": -151.84535217285156, + "logps/ref_chosen": -58.18701171875, + "logps/ref_rejected": -83.63442993164062, + "logps/rejected": -229.89866638183594, + "loss": 1.0959, + "margin_dpo/margin_mean": 52.605873107910156, + "margin_dpo/margin_std": 80.5517807006836, + "step": 391 + }, + { + "KL/chosen_KL_mean": -80.34383392333984, + "KL/mean": -109.28814697265625, + "KL/rejected_KL_mean": -138.23245239257812, + "KL/std": 66.96438598632812, + "epoch": 0.5925925925925926, + "fcm_dpo/beta": 0.008585932664573193, + "fcm_dpo/delta": -0.10228624939918518, + "fcm_dpo/margin": 57.88862991333008, + "fcm_dpo/q_t": 0.3883962333202362, + "grad_norm": 11.231030464172363, + "learning_rate": 2.1442129043167873e-07, + "logits/chosen": 0.7651580572128296, + "logits/rejected": 0.7059136629104614, + "logps/chosen": -150.08836364746094, + "logps/ref_chosen": -69.7445297241211, + "logps/ref_rejected": -94.05877685546875, + "logps/rejected": -232.29122924804688, + "loss": 1.0572, + "margin_dpo/margin_mean": 57.88862609863281, + "margin_dpo/margin_std": 78.10966491699219, + "step": 392 + }, + { + "KL/chosen_KL_mean": -94.92709350585938, + "KL/mean": -123.78874206542969, + "KL/rejected_KL_mean": -152.650390625, + "KL/std": 66.06975555419922, + "epoch": 0.5941043083900227, + "fcm_dpo/beta": 0.008353622630238533, + "fcm_dpo/delta": -0.08677142858505249, + "fcm_dpo/margin": 57.723289489746094, + "fcm_dpo/q_t": 0.3901318311691284, + "grad_norm": 11.89721965789795, + "learning_rate": 2.131130332936195e-07, + "logits/chosen": 0.7530436515808105, + "logits/rejected": 0.7150501012802124, + "logps/chosen": -147.26199340820312, + "logps/ref_chosen": -52.33489990234375, + "logps/ref_rejected": -74.33809661865234, + "logps/rejected": -226.98849487304688, + "loss": 1.0421, + "margin_dpo/margin_mean": 57.723289489746094, + "margin_dpo/margin_std": 70.54959106445312, + "step": 393 + }, + { + "KL/chosen_KL_mean": -89.56831359863281, + "KL/mean": -114.85767364501953, + "KL/rejected_KL_mean": -140.14703369140625, + "KL/std": 60.31583023071289, + "epoch": 0.5956160241874527, + "fcm_dpo/beta": 0.008336104452610016, + "fcm_dpo/delta": -0.022687222808599472, + "fcm_dpo/margin": 50.57872772216797, + "fcm_dpo/q_t": 0.40135329961776733, + "grad_norm": 12.148786544799805, + "learning_rate": 2.1180580796331323e-07, + "logits/chosen": 0.7515368461608887, + "logits/rejected": 0.7221952676773071, + "logps/chosen": -150.24444580078125, + "logps/ref_chosen": -60.6761360168457, + "logps/ref_rejected": -71.36074829101562, + "logps/rejected": -211.50778198242188, + "loss": 1.0731, + "margin_dpo/margin_mean": 50.57872772216797, + "margin_dpo/margin_std": 61.64701843261719, + "step": 394 + }, + { + "KL/chosen_KL_mean": -95.43479919433594, + "KL/mean": -115.49359130859375, + "KL/rejected_KL_mean": -135.55238342285156, + "KL/std": 62.913185119628906, + "epoch": 0.5971277399848829, + "fcm_dpo/beta": 0.008418946526944637, + "fcm_dpo/delta": 0.06374331563711166, + "fcm_dpo/margin": 40.117591857910156, + "fcm_dpo/q_t": 0.42396989464759827, + "grad_norm": 15.213231086730957, + "learning_rate": 2.104996510066625e-07, + "logits/chosen": 0.7378535270690918, + "logits/rejected": 0.6408002376556396, + "logps/chosen": -146.03912353515625, + "logps/ref_chosen": -50.60432434082031, + "logps/ref_rejected": -77.08731079101562, + "logps/rejected": -212.63970947265625, + "loss": 1.1554, + "margin_dpo/margin_mean": 40.117591857910156, + "margin_dpo/margin_std": 67.76335906982422, + "step": 395 + }, + { + "KL/chosen_KL_mean": -89.58931732177734, + "KL/mean": -113.90975952148438, + "KL/rejected_KL_mean": -138.23020935058594, + "KL/std": 70.15506744384766, + "epoch": 0.5986394557823129, + "fcm_dpo/beta": 0.008337254635989666, + "fcm_dpo/delta": -0.006977491080760956, + "fcm_dpo/margin": 48.64088439941406, + "fcm_dpo/q_t": 0.40703338384628296, + "grad_norm": 11.953834533691406, + "learning_rate": 2.0919459895968517e-07, + "logits/chosen": 0.7067517042160034, + "logits/rejected": 0.6094527244567871, + "logps/chosen": -140.9489288330078, + "logps/ref_chosen": -51.35961151123047, + "logps/ref_rejected": -79.89360046386719, + "logps/rejected": -218.12380981445312, + "loss": 1.091, + "margin_dpo/margin_mean": 48.64088439941406, + "margin_dpo/margin_std": 62.8603515625, + "step": 396 + }, + { + "KL/chosen_KL_mean": -103.04716491699219, + "KL/mean": -116.64557647705078, + "KL/rejected_KL_mean": -130.24398803710938, + "KL/std": 64.92282104492188, + "epoch": 0.600151171579743, + "fcm_dpo/beta": 0.008626183494925499, + "fcm_dpo/delta": 0.16903799772262573, + "fcm_dpo/margin": 27.196826934814453, + "fcm_dpo/q_t": 0.4464913010597229, + "grad_norm": 13.257553100585938, + "learning_rate": 2.078906883274924e-07, + "logits/chosen": 0.6656841039657593, + "logits/rejected": 0.6178501844406128, + "logps/chosen": -169.50338745117188, + "logps/ref_chosen": -66.45622253417969, + "logps/ref_rejected": -85.74736785888672, + "logps/rejected": -215.99136352539062, + "loss": 1.2806, + "margin_dpo/margin_mean": 27.196826934814453, + "margin_dpo/margin_std": 80.3773193359375, + "step": 397 + }, + { + "KL/chosen_KL_mean": -88.44233703613281, + "KL/mean": -118.74835205078125, + "KL/rejected_KL_mean": -149.0543670654297, + "KL/std": 67.16844177246094, + "epoch": 0.6016628873771731, + "fcm_dpo/beta": 0.008501345291733742, + "fcm_dpo/delta": -0.1221761479973793, + "fcm_dpo/margin": 60.61201477050781, + "fcm_dpo/q_t": 0.38299331068992615, + "grad_norm": 11.214990615844727, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": 0.719185471534729, + "logits/rejected": 0.6512874960899353, + "logps/chosen": -137.68658447265625, + "logps/ref_chosen": -49.244239807128906, + "logps/ref_rejected": -75.18949127197266, + "logps/rejected": -224.2438507080078, + "loss": 1.0132, + "margin_dpo/margin_mean": 60.61201477050781, + "margin_dpo/margin_std": 68.35538482666016, + "step": 398 + }, + { + "KL/chosen_KL_mean": -103.34333801269531, + "KL/mean": -136.01007080078125, + "KL/rejected_KL_mean": -168.6768035888672, + "KL/std": 69.25430297851562, + "epoch": 0.6031746031746031, + "fcm_dpo/beta": 0.008262135088443756, + "fcm_dpo/delta": -0.1485450565814972, + "fcm_dpo/margin": 65.33348083496094, + "fcm_dpo/q_t": 0.37789463996887207, + "grad_norm": 13.473124504089355, + "learning_rate": 2.052864371672457e-07, + "logits/chosen": 0.6423044204711914, + "logits/rejected": 0.4999236464500427, + "logps/chosen": -171.65013122558594, + "logps/ref_chosen": -68.30679321289062, + "logps/ref_rejected": -113.2708511352539, + "logps/rejected": -281.9476623535156, + "loss": 1.0073, + "margin_dpo/margin_mean": 65.33348083496094, + "margin_dpo/margin_std": 75.8927001953125, + "step": 399 + }, + { + "KL/chosen_KL_mean": -114.28192901611328, + "KL/mean": -134.40957641601562, + "KL/rejected_KL_mean": -154.5372314453125, + "KL/std": 67.74653625488281, + "epoch": 0.6046863189720333, + "fcm_dpo/beta": 0.008252747356891632, + "fcm_dpo/delta": 0.0695052295923233, + "fcm_dpo/margin": 40.25529098510742, + "fcm_dpo/q_t": 0.424492746591568, + "grad_norm": 18.078105926513672, + "learning_rate": 2.0398616948569493e-07, + "logits/chosen": 0.6950872540473938, + "logits/rejected": 0.636712908744812, + "logps/chosen": -185.90841674804688, + "logps/ref_chosen": -71.62649536132812, + "logps/ref_rejected": -90.98765563964844, + "logps/rejected": -245.52487182617188, + "loss": 1.1625, + "margin_dpo/margin_mean": 40.25529479980469, + "margin_dpo/margin_std": 68.32823181152344, + "step": 400 + }, + { + "KL/chosen_KL_mean": -88.39434814453125, + "KL/mean": -116.34048461914062, + "KL/rejected_KL_mean": -144.28662109375, + "KL/std": 71.89583587646484, + "epoch": 0.6061980347694633, + "fcm_dpo/beta": 0.008242808282375336, + "fcm_dpo/delta": -0.06394520401954651, + "fcm_dpo/margin": 55.89228057861328, + "fcm_dpo/q_t": 0.39509570598602295, + "grad_norm": 10.358372688293457, + "learning_rate": 2.0268718890989752e-07, + "logits/chosen": 0.7502726316452026, + "logits/rejected": 0.6499309539794922, + "logps/chosen": -142.11929321289062, + "logps/ref_chosen": -53.72495651245117, + "logps/ref_rejected": -75.06304931640625, + "logps/rejected": -219.34967041015625, + "loss": 1.0506, + "margin_dpo/margin_mean": 55.89228057861328, + "margin_dpo/margin_std": 67.9534683227539, + "step": 401 + }, + { + "KL/chosen_KL_mean": -95.33905029296875, + "KL/mean": -119.19203186035156, + "KL/rejected_KL_mean": -143.04501342773438, + "KL/std": 64.60128021240234, + "epoch": 0.6077097505668935, + "fcm_dpo/beta": 0.008193889632821083, + "fcm_dpo/delta": 0.008644327521324158, + "fcm_dpo/margin": 47.705970764160156, + "fcm_dpo/q_t": 0.41146695613861084, + "grad_norm": 13.163501739501953, + "learning_rate": 2.013895317751323e-07, + "logits/chosen": 0.7267850637435913, + "logits/rejected": 0.6951309442520142, + "logps/chosen": -157.21298217773438, + "logps/ref_chosen": -61.873931884765625, + "logps/ref_rejected": -66.15198516845703, + "logps/rejected": -209.19700622558594, + "loss": 1.1294, + "margin_dpo/margin_mean": 47.705970764160156, + "margin_dpo/margin_std": 76.21807098388672, + "step": 402 + }, + { + "KL/chosen_KL_mean": -103.14884948730469, + "KL/mean": -131.67453002929688, + "KL/rejected_KL_mean": -160.20021057128906, + "KL/std": 71.38824462890625, + "epoch": 0.6092214663643235, + "fcm_dpo/beta": 0.008181357756257057, + "fcm_dpo/delta": -0.0699668824672699, + "fcm_dpo/margin": 57.05136489868164, + "fcm_dpo/q_t": 0.3950841426849365, + "grad_norm": 11.315382957458496, + "learning_rate": 2.0009323437965898e-07, + "logits/chosen": 0.8247023820877075, + "logits/rejected": 0.740180253982544, + "logps/chosen": -154.47035217285156, + "logps/ref_chosen": -51.321502685546875, + "logps/ref_rejected": -86.54010772705078, + "logps/rejected": -246.74032592773438, + "loss": 1.0712, + "margin_dpo/margin_mean": 57.05136489868164, + "margin_dpo/margin_std": 78.94970703125, + "step": 403 + }, + { + "KL/chosen_KL_mean": -95.03086853027344, + "KL/mean": -124.51020050048828, + "KL/rejected_KL_mean": -153.98953247070312, + "KL/std": 71.967529296875, + "epoch": 0.6107331821617535, + "fcm_dpo/beta": 0.00798763893544674, + "fcm_dpo/delta": -0.07605341076850891, + "fcm_dpo/margin": 58.95866012573242, + "fcm_dpo/q_t": 0.39368003606796265, + "grad_norm": 14.19412899017334, + "learning_rate": 1.9879833298370237e-07, + "logits/chosen": 0.6934102773666382, + "logits/rejected": 0.5980790853500366, + "logps/chosen": -157.29376220703125, + "logps/ref_chosen": -62.26288604736328, + "logps/ref_rejected": -95.19029998779297, + "logps/rejected": -249.17982482910156, + "loss": 1.0654, + "margin_dpo/margin_mean": 58.958656311035156, + "margin_dpo/margin_std": 77.97892761230469, + "step": 404 + }, + { + "KL/chosen_KL_mean": -99.53964233398438, + "KL/mean": -122.60385131835938, + "KL/rejected_KL_mean": -145.66806030273438, + "KL/std": 66.06510925292969, + "epoch": 0.6122448979591837, + "fcm_dpo/beta": 0.007995378226041794, + "fcm_dpo/delta": 0.03158438578248024, + "fcm_dpo/margin": 46.12843322753906, + "fcm_dpo/q_t": 0.4166746735572815, + "grad_norm": 11.438131332397461, + "learning_rate": 1.975048638084379e-07, + "logits/chosen": 0.8034164905548096, + "logits/rejected": 0.7549797296524048, + "logps/chosen": -150.1239776611328, + "logps/ref_chosen": -50.5843391418457, + "logps/ref_rejected": -65.43156433105469, + "logps/rejected": -211.09963989257812, + "loss": 1.1317, + "margin_dpo/margin_mean": 46.12843322753906, + "margin_dpo/margin_std": 70.4107437133789, + "step": 405 + }, + { + "KL/chosen_KL_mean": -97.1700439453125, + "KL/mean": -126.85040283203125, + "KL/rejected_KL_mean": -156.53074645996094, + "KL/std": 70.42720031738281, + "epoch": 0.6137566137566137, + "fcm_dpo/beta": 0.007977031171321869, + "fcm_dpo/delta": -0.07714153826236725, + "fcm_dpo/margin": 59.36070251464844, + "fcm_dpo/q_t": 0.3917776644229889, + "grad_norm": 14.39725112915039, + "learning_rate": 1.9621286303497914e-07, + "logits/chosen": 0.7903730869293213, + "logits/rejected": 0.6238706111907959, + "logps/chosen": -146.1656494140625, + "logps/ref_chosen": -48.99560546875, + "logps/ref_rejected": -92.47774505615234, + "logps/rejected": -249.00848388671875, + "loss": 1.0647, + "margin_dpo/margin_mean": 59.36070251464844, + "margin_dpo/margin_std": 80.22438049316406, + "step": 406 + }, + { + "KL/chosen_KL_mean": -117.63362884521484, + "KL/mean": -140.78936767578125, + "KL/rejected_KL_mean": -163.9451141357422, + "KL/std": 74.67190551757812, + "epoch": 0.6152683295540439, + "fcm_dpo/beta": 0.007999084889888763, + "fcm_dpo/delta": 0.030455123633146286, + "fcm_dpo/margin": 46.31147003173828, + "fcm_dpo/q_t": 0.4163338243961334, + "grad_norm": 14.213495254516602, + "learning_rate": 1.9492236680336483e-07, + "logits/chosen": 0.606561541557312, + "logits/rejected": 0.5306586027145386, + "logps/chosen": -207.03419494628906, + "logps/ref_chosen": -89.40056610107422, + "logps/ref_rejected": -99.28775024414062, + "logps/rejected": -263.23284912109375, + "loss": 1.1412, + "margin_dpo/margin_mean": 46.31147003173828, + "margin_dpo/margin_std": 77.54383850097656, + "step": 407 + }, + { + "KL/chosen_KL_mean": -89.46073913574219, + "KL/mean": -124.67997741699219, + "KL/rejected_KL_mean": -159.8992156982422, + "KL/std": 69.03326416015625, + "epoch": 0.6167800453514739, + "fcm_dpo/beta": 0.00785075593739748, + "fcm_dpo/delta": -0.16175517439842224, + "fcm_dpo/margin": 70.4384765625, + "fcm_dpo/q_t": 0.37296241521835327, + "grad_norm": 10.863518714904785, + "learning_rate": 1.9363341121154895e-07, + "logits/chosen": 0.7440523505210876, + "logits/rejected": 0.6608445048332214, + "logps/chosen": -144.16465759277344, + "logps/ref_chosen": -54.70391845703125, + "logps/ref_rejected": -73.98648834228516, + "logps/rejected": -233.88571166992188, + "loss": 0.9903, + "margin_dpo/margin_mean": 70.4384765625, + "margin_dpo/margin_std": 73.6658935546875, + "step": 408 + }, + { + "KL/chosen_KL_mean": -111.31812286376953, + "KL/mean": -128.97467041015625, + "KL/rejected_KL_mean": -146.6312255859375, + "KL/std": 62.695716857910156, + "epoch": 0.618291761148904, + "fcm_dpo/beta": 0.007871621288359165, + "fcm_dpo/delta": 0.12564970552921295, + "fcm_dpo/margin": 35.31309509277344, + "fcm_dpo/q_t": 0.4370152950286865, + "grad_norm": 13.255595207214355, + "learning_rate": 1.9234603231438994e-07, + "logits/chosen": 0.7345675826072693, + "logits/rejected": 0.743270754814148, + "logps/chosen": -173.43634033203125, + "logps/ref_chosen": -62.11822509765625, + "logps/ref_rejected": -61.933509826660156, + "logps/rejected": -208.56472778320312, + "loss": 1.2051, + "margin_dpo/margin_mean": 35.31309509277344, + "margin_dpo/margin_std": 72.25135803222656, + "step": 409 + }, + { + "KL/chosen_KL_mean": -103.84161376953125, + "KL/mean": -132.0266876220703, + "KL/rejected_KL_mean": -160.21176147460938, + "KL/std": 65.34846496582031, + "epoch": 0.6198034769463341, + "fcm_dpo/beta": 0.007834449410438538, + "fcm_dpo/delta": -0.044498007744550705, + "fcm_dpo/margin": 56.37013626098633, + "fcm_dpo/q_t": 0.3971004784107208, + "grad_norm": 11.596942901611328, + "learning_rate": 1.9106026612264315e-07, + "logits/chosen": 0.7368378639221191, + "logits/rejected": 0.7123606204986572, + "logps/chosen": -165.644287109375, + "logps/ref_chosen": -61.80266189575195, + "logps/ref_rejected": -76.60002136230469, + "logps/rejected": -236.811767578125, + "loss": 1.0501, + "margin_dpo/margin_mean": 56.37013626098633, + "margin_dpo/margin_std": 61.56016540527344, + "step": 410 + }, + { + "KL/chosen_KL_mean": -109.17242431640625, + "KL/mean": -136.98342895507812, + "KL/rejected_KL_mean": -164.79446411132812, + "KL/std": 74.19166564941406, + "epoch": 0.6213151927437641, + "fcm_dpo/beta": 0.007825829088687897, + "fcm_dpo/delta": -0.03687795624136925, + "fcm_dpo/margin": 55.622039794921875, + "fcm_dpo/q_t": 0.40110480785369873, + "grad_norm": 10.238993644714355, + "learning_rate": 1.8977614860195296e-07, + "logits/chosen": 0.7125971913337708, + "logits/rejected": 0.6526937484741211, + "logps/chosen": -163.61781311035156, + "logps/ref_chosen": -54.44539260864258, + "logps/ref_rejected": -74.5650863647461, + "logps/rejected": -239.3595428466797, + "loss": 1.087, + "margin_dpo/margin_mean": 55.622039794921875, + "margin_dpo/margin_std": 79.11244201660156, + "step": 411 + }, + { + "KL/chosen_KL_mean": -115.19978332519531, + "KL/mean": -141.18414306640625, + "KL/rejected_KL_mean": -167.16848754882812, + "KL/std": 65.96856689453125, + "epoch": 0.6228269085411943, + "fcm_dpo/beta": 0.007769760675728321, + "fcm_dpo/delta": -0.004255469888448715, + "fcm_dpo/margin": 51.968719482421875, + "fcm_dpo/q_t": 0.40769216418266296, + "grad_norm": 12.5255708694458, + "learning_rate": 1.8849371567184662e-07, + "logits/chosen": 0.7548251152038574, + "logits/rejected": 0.6842193603515625, + "logps/chosen": -170.4478759765625, + "logps/ref_chosen": -55.248085021972656, + "logps/ref_rejected": -68.96623229980469, + "logps/rejected": -236.1347198486328, + "loss": 1.0951, + "margin_dpo/margin_mean": 51.96871566772461, + "margin_dpo/margin_std": 70.8434829711914, + "step": 412 + }, + { + "KL/chosen_KL_mean": -124.92682647705078, + "KL/mean": -146.77780151367188, + "KL/rejected_KL_mean": -168.62875366210938, + "KL/std": 68.0516357421875, + "epoch": 0.6243386243386243, + "fcm_dpo/beta": 0.007867778651416302, + "fcm_dpo/delta": 0.05814486742019653, + "fcm_dpo/margin": 43.7019157409668, + "fcm_dpo/q_t": 0.4224427342414856, + "grad_norm": 14.508427619934082, + "learning_rate": 1.872130032047302e-07, + "logits/chosen": 0.5644307136535645, + "logits/rejected": 0.5286777019500732, + "logps/chosen": -193.6475830078125, + "logps/ref_chosen": -68.72074890136719, + "logps/ref_rejected": -78.76539611816406, + "logps/rejected": -247.39414978027344, + "loss": 1.1851, + "margin_dpo/margin_mean": 43.70191955566406, + "margin_dpo/margin_std": 87.32107543945312, + "step": 413 + }, + { + "KL/chosen_KL_mean": -109.9261474609375, + "KL/mean": -138.1259765625, + "KL/rejected_KL_mean": -166.32582092285156, + "KL/std": 74.28414916992188, + "epoch": 0.6258503401360545, + "fcm_dpo/beta": 0.00784690398722887, + "fcm_dpo/delta": -0.04451046139001846, + "fcm_dpo/margin": 56.39968490600586, + "fcm_dpo/q_t": 0.39813709259033203, + "grad_norm": 13.054259300231934, + "learning_rate": 1.8593404702488436e-07, + "logits/chosen": 0.7380519509315491, + "logits/rejected": 0.674519419670105, + "logps/chosen": -164.06436157226562, + "logps/ref_chosen": -54.138214111328125, + "logps/ref_rejected": -74.65741729736328, + "logps/rejected": -240.98324584960938, + "loss": 1.0685, + "margin_dpo/margin_mean": 56.399681091308594, + "margin_dpo/margin_std": 72.53543853759766, + "step": 414 + }, + { + "KL/chosen_KL_mean": -110.6787109375, + "KL/mean": -135.13711547851562, + "KL/rejected_KL_mean": -159.5955047607422, + "KL/std": 72.54624938964844, + "epoch": 0.6273620559334845, + "fcm_dpo/beta": 0.007824394851922989, + "fcm_dpo/delta": 0.0178983211517334, + "fcm_dpo/margin": 48.91680145263672, + "fcm_dpo/q_t": 0.4126567244529724, + "grad_norm": 12.129854202270508, + "learning_rate": 1.846568829074628e-07, + "logits/chosen": 0.7589465379714966, + "logits/rejected": 0.7433362603187561, + "logps/chosen": -166.59725952148438, + "logps/ref_chosen": -55.91856002807617, + "logps/ref_rejected": -61.747703552246094, + "logps/rejected": -221.34320068359375, + "loss": 1.1291, + "margin_dpo/margin_mean": 48.91680145263672, + "margin_dpo/margin_std": 78.5286865234375, + "step": 415 + }, + { + "KL/chosen_KL_mean": -116.60771942138672, + "KL/mean": -137.52841186523438, + "KL/rejected_KL_mean": -158.44911193847656, + "KL/std": 74.35231018066406, + "epoch": 0.6288737717309146, + "fcm_dpo/beta": 0.008027415722608566, + "fcm_dpo/delta": 0.06183099001646042, + "fcm_dpo/margin": 41.841392517089844, + "fcm_dpo/q_t": 0.42444467544555664, + "grad_norm": 14.416735649108887, + "learning_rate": 1.8338154657749128e-07, + "logits/chosen": 0.7013071775436401, + "logits/rejected": 0.6499512195587158, + "logps/chosen": -171.330810546875, + "logps/ref_chosen": -54.72308349609375, + "logps/ref_rejected": -69.17388916015625, + "logps/rejected": -227.6230010986328, + "loss": 1.1802, + "margin_dpo/margin_mean": 41.841392517089844, + "margin_dpo/margin_std": 74.81488037109375, + "step": 416 + }, + { + "KL/chosen_KL_mean": -119.80218505859375, + "KL/mean": -148.10443115234375, + "KL/rejected_KL_mean": -176.40670776367188, + "KL/std": 70.20147705078125, + "epoch": 0.6303854875283447, + "fcm_dpo/beta": 0.007916878908872604, + "fcm_dpo/delta": -0.05051497370004654, + "fcm_dpo/margin": 56.604530334472656, + "fcm_dpo/q_t": 0.39684611558914185, + "grad_norm": 12.716817855834961, + "learning_rate": 1.8210807370886849e-07, + "logits/chosen": 0.8077340126037598, + "logits/rejected": 0.7370149493217468, + "logps/chosen": -176.59344482421875, + "logps/ref_chosen": -56.791259765625, + "logps/ref_rejected": -68.7791748046875, + "logps/rejected": -245.18588256835938, + "loss": 1.0905, + "margin_dpo/margin_mean": 56.604530334472656, + "margin_dpo/margin_std": 83.14535522460938, + "step": 417 + }, + { + "KL/chosen_KL_mean": -128.22242736816406, + "KL/mean": -149.95968627929688, + "KL/rejected_KL_mean": -171.69696044921875, + "KL/std": 73.53202056884766, + "epoch": 0.6318972033257747, + "fcm_dpo/beta": 0.007869708351790905, + "fcm_dpo/delta": 0.059024084359407425, + "fcm_dpo/margin": 43.47452163696289, + "fcm_dpo/q_t": 0.42358940839767456, + "grad_norm": 13.894302368164062, + "learning_rate": 1.8083649992336825e-07, + "logits/chosen": 0.7667458057403564, + "logits/rejected": 0.7746227979660034, + "logps/chosen": -197.33041381835938, + "logps/ref_chosen": -69.10798645019531, + "logps/ref_rejected": -75.09132385253906, + "logps/rejected": -246.7882843017578, + "loss": 1.1614, + "margin_dpo/margin_mean": 43.47452163696289, + "margin_dpo/margin_std": 74.61412048339844, + "step": 418 + }, + { + "KL/chosen_KL_mean": -102.58634948730469, + "KL/mean": -134.0736083984375, + "KL/rejected_KL_mean": -165.56088256835938, + "KL/std": 73.33294677734375, + "epoch": 0.6334089191232048, + "fcm_dpo/beta": 0.007823498919606209, + "fcm_dpo/delta": -0.09785882383584976, + "fcm_dpo/margin": 62.974517822265625, + "fcm_dpo/q_t": 0.3892057538032532, + "grad_norm": 12.586299896240234, + "learning_rate": 1.7956686078964255e-07, + "logits/chosen": 0.6310225129127502, + "logits/rejected": 0.5767621994018555, + "logps/chosen": -160.75811767578125, + "logps/ref_chosen": -58.1717643737793, + "logps/ref_rejected": -71.67066955566406, + "logps/rejected": -237.23153686523438, + "loss": 1.0468, + "margin_dpo/margin_mean": 62.97452163696289, + "margin_dpo/margin_std": 81.59907531738281, + "step": 419 + }, + { + "KL/chosen_KL_mean": -129.5718231201172, + "KL/mean": -146.15481567382812, + "KL/rejected_KL_mean": -162.73782348632812, + "KL/std": 73.85808563232422, + "epoch": 0.6349206349206349, + "fcm_dpo/beta": 0.007942350581288338, + "fcm_dpo/delta": 0.14052259922027588, + "fcm_dpo/margin": 33.16600036621094, + "fcm_dpo/q_t": 0.4420320391654968, + "grad_norm": 13.626842498779297, + "learning_rate": 1.782991918222275e-07, + "logits/chosen": 0.7083577513694763, + "logits/rejected": 0.6613192558288574, + "logps/chosen": -186.62533569335938, + "logps/ref_chosen": -57.05351257324219, + "logps/ref_rejected": -62.670982360839844, + "logps/rejected": -225.4088134765625, + "loss": 1.2491, + "margin_dpo/margin_mean": 33.16600036621094, + "margin_dpo/margin_std": 84.56375122070312, + "step": 420 + }, + { + "KL/chosen_KL_mean": -121.5405502319336, + "KL/mean": -145.28880310058594, + "KL/rejected_KL_mean": -169.03704833984375, + "KL/std": 73.51364135742188, + "epoch": 0.636432350718065, + "fcm_dpo/beta": 0.008013014681637287, + "fcm_dpo/delta": 0.0198320671916008, + "fcm_dpo/margin": 47.49650573730469, + "fcm_dpo/q_t": 0.41550660133361816, + "grad_norm": 13.676653861999512, + "learning_rate": 1.7703352848054887e-07, + "logits/chosen": 0.6900692582130432, + "logits/rejected": 0.6295258402824402, + "logps/chosen": -178.86380004882812, + "logps/ref_chosen": -57.32324981689453, + "logps/ref_rejected": -75.33782958984375, + "logps/rejected": -244.3748779296875, + "loss": 1.1733, + "margin_dpo/margin_mean": 47.49650573730469, + "margin_dpo/margin_std": 92.68571472167969, + "step": 421 + }, + { + "KL/chosen_KL_mean": -105.38725280761719, + "KL/mean": -136.9675750732422, + "KL/rejected_KL_mean": -168.5478973388672, + "KL/std": 72.90692138671875, + "epoch": 0.6379440665154951, + "fcm_dpo/beta": 0.007948427461087704, + "fcm_dpo/delta": -0.10728380084037781, + "fcm_dpo/margin": 63.160648345947266, + "fcm_dpo/q_t": 0.38589245080947876, + "grad_norm": 14.917747497558594, + "learning_rate": 1.7576990616793137e-07, + "logits/chosen": 0.735478937625885, + "logits/rejected": 0.7193405628204346, + "logps/chosen": -172.44482421875, + "logps/ref_chosen": -67.05757141113281, + "logps/ref_rejected": -72.12803649902344, + "logps/rejected": -240.67593383789062, + "loss": 1.0318, + "margin_dpo/margin_mean": 63.160648345947266, + "margin_dpo/margin_std": 76.23689270019531, + "step": 422 + }, + { + "KL/chosen_KL_mean": -109.40867614746094, + "KL/mean": -140.88455200195312, + "KL/rejected_KL_mean": -172.3604278564453, + "KL/std": 75.55366516113281, + "epoch": 0.6394557823129252, + "fcm_dpo/beta": 0.007762731984257698, + "fcm_dpo/delta": -0.09350243210792542, + "fcm_dpo/margin": 62.951759338378906, + "fcm_dpo/q_t": 0.3896293640136719, + "grad_norm": 11.912402153015137, + "learning_rate": 1.745083602306071e-07, + "logits/chosen": 0.7740094661712646, + "logits/rejected": 0.700740396976471, + "logps/chosen": -163.47035217285156, + "logps/ref_chosen": -54.06167221069336, + "logps/ref_rejected": -76.64092254638672, + "logps/rejected": -249.00135803222656, + "loss": 1.0444, + "margin_dpo/margin_mean": 62.95175552368164, + "margin_dpo/margin_std": 79.72633361816406, + "step": 423 + }, + { + "KL/chosen_KL_mean": -117.83436584472656, + "KL/mean": -148.37539672851562, + "KL/rejected_KL_mean": -178.91644287109375, + "KL/std": 72.689697265625, + "epoch": 0.6409674981103552, + "fcm_dpo/beta": 0.007616098970174789, + "fcm_dpo/delta": -0.0695209950208664, + "fcm_dpo/margin": 61.08207702636719, + "fcm_dpo/q_t": 0.39452964067459106, + "grad_norm": 16.555137634277344, + "learning_rate": 1.7324892595672804e-07, + "logits/chosen": 0.660463809967041, + "logits/rejected": 0.62034010887146, + "logps/chosen": -171.4432373046875, + "logps/ref_chosen": -53.60887145996094, + "logps/ref_rejected": -79.2139892578125, + "logps/rejected": -258.13043212890625, + "loss": 1.0664, + "margin_dpo/margin_mean": 61.08207702636719, + "margin_dpo/margin_std": 81.5423583984375, + "step": 424 + }, + { + "KL/chosen_KL_mean": -114.80853271484375, + "KL/mean": -139.91513061523438, + "KL/rejected_KL_mean": -165.02171325683594, + "KL/std": 71.188232421875, + "epoch": 0.6424792139077853, + "fcm_dpo/beta": 0.00764683960005641, + "fcm_dpo/delta": 0.01666775718331337, + "fcm_dpo/margin": 50.21317672729492, + "fcm_dpo/q_t": 0.41330617666244507, + "grad_norm": 14.110657691955566, + "learning_rate": 1.7199163857537824e-07, + "logits/chosen": 0.7706844806671143, + "logits/rejected": 0.7416727542877197, + "logps/chosen": -173.22320556640625, + "logps/ref_chosen": -58.41468048095703, + "logps/ref_rejected": -66.59054565429688, + "logps/rejected": -231.6122589111328, + "loss": 1.1279, + "margin_dpo/margin_mean": 50.21317672729492, + "margin_dpo/margin_std": 80.54307556152344, + "step": 425 + }, + { + "KL/chosen_KL_mean": -138.42042541503906, + "KL/mean": -153.50100708007812, + "KL/rejected_KL_mean": -168.58157348632812, + "KL/std": 72.29527282714844, + "epoch": 0.6439909297052154, + "fcm_dpo/beta": 0.007851322181522846, + "fcm_dpo/delta": 0.1672021448612213, + "fcm_dpo/margin": 30.16112518310547, + "fcm_dpo/q_t": 0.4458393454551697, + "grad_norm": 15.771385192871094, + "learning_rate": 1.7073653325558828e-07, + "logits/chosen": 0.657271683216095, + "logits/rejected": 0.6661276817321777, + "logps/chosen": -210.128662109375, + "logps/ref_chosen": -71.70822143554688, + "logps/ref_rejected": -73.57725524902344, + "logps/rejected": -242.1588134765625, + "loss": 1.2797, + "margin_dpo/margin_mean": 30.1611270904541, + "margin_dpo/margin_std": 89.65312957763672, + "step": 426 + }, + { + "KL/chosen_KL_mean": -129.74212646484375, + "KL/mean": -153.55258178710938, + "KL/rejected_KL_mean": -177.36302185058594, + "KL/std": 77.05906677246094, + "epoch": 0.6455026455026455, + "fcm_dpo/beta": 0.007935117930173874, + "fcm_dpo/delta": 0.022978732362389565, + "fcm_dpo/margin": 47.620872497558594, + "fcm_dpo/q_t": 0.41612735390663147, + "grad_norm": 14.509443283081055, + "learning_rate": 1.6948364510535218e-07, + "logits/chosen": 0.7272225618362427, + "logits/rejected": 0.6681046485900879, + "logps/chosen": -188.38490295410156, + "logps/ref_chosen": -58.64276885986328, + "logps/ref_rejected": -86.25437927246094, + "logps/rejected": -263.6174011230469, + "loss": 1.1563, + "margin_dpo/margin_mean": 47.620872497558594, + "margin_dpo/margin_std": 87.77416229248047, + "step": 427 + }, + { + "KL/chosen_KL_mean": -121.83946990966797, + "KL/mean": -150.60853576660156, + "KL/rejected_KL_mean": -179.37759399414062, + "KL/std": 78.71310424804688, + "epoch": 0.6470143613000756, + "fcm_dpo/beta": 0.007923326455056667, + "fcm_dpo/delta": -0.058716095983982086, + "fcm_dpo/margin": 57.53812789916992, + "fcm_dpo/q_t": 0.39860618114471436, + "grad_norm": 12.580979347229004, + "learning_rate": 1.6823300917064458e-07, + "logits/chosen": 0.6929279565811157, + "logits/rejected": 0.6461859941482544, + "logps/chosen": -188.43551635742188, + "logps/ref_chosen": -66.5960464477539, + "logps/ref_rejected": -82.3941650390625, + "logps/rejected": -261.7717590332031, + "loss": 1.0884, + "margin_dpo/margin_mean": 57.538124084472656, + "margin_dpo/margin_std": 85.31922912597656, + "step": 428 + }, + { + "KL/chosen_KL_mean": -126.24545288085938, + "KL/mean": -148.31695556640625, + "KL/rejected_KL_mean": -170.38845825195312, + "KL/std": 72.1160888671875, + "epoch": 0.6485260770975056, + "fcm_dpo/beta": 0.007961621508002281, + "fcm_dpo/delta": 0.04956657066941261, + "fcm_dpo/margin": 44.14300537109375, + "fcm_dpo/q_t": 0.4205780029296875, + "grad_norm": 14.980849266052246, + "learning_rate": 1.669846604344412e-07, + "logits/chosen": 0.6782246828079224, + "logits/rejected": 0.6961945295333862, + "logps/chosen": -183.25515747070312, + "logps/ref_chosen": -57.00970458984375, + "logps/ref_rejected": -59.86549377441406, + "logps/rejected": -230.2539520263672, + "loss": 1.1697, + "margin_dpo/margin_mean": 44.14300537109375, + "margin_dpo/margin_std": 82.05901336669922, + "step": 429 + }, + { + "KL/chosen_KL_mean": -113.78839111328125, + "KL/mean": -149.21728515625, + "KL/rejected_KL_mean": -184.64617919921875, + "KL/std": 74.32037353515625, + "epoch": 0.6500377928949358, + "fcm_dpo/beta": 0.0077576362527906895, + "fcm_dpo/delta": -0.15839435160160065, + "fcm_dpo/margin": 70.8577880859375, + "fcm_dpo/q_t": 0.37446311116218567, + "grad_norm": 12.859457015991211, + "learning_rate": 1.6573863381573954e-07, + "logits/chosen": 0.6114371418952942, + "logits/rejected": 0.6136624813079834, + "logps/chosen": -173.35159301757812, + "logps/ref_chosen": -59.563194274902344, + "logps/ref_rejected": -70.52289581298828, + "logps/rejected": -255.16908264160156, + "loss": 1.0126, + "margin_dpo/margin_mean": 70.85778045654297, + "margin_dpo/margin_std": 84.95809936523438, + "step": 430 + }, + { + "KL/chosen_KL_mean": -110.34442901611328, + "KL/mean": -134.9564971923828, + "KL/rejected_KL_mean": -159.56854248046875, + "KL/std": 74.60652160644531, + "epoch": 0.6515495086923658, + "fcm_dpo/beta": 0.00771299796178937, + "fcm_dpo/delta": 0.021141374483704567, + "fcm_dpo/margin": 49.22412872314453, + "fcm_dpo/q_t": 0.41382941603660583, + "grad_norm": 12.430391311645508, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": 0.7100223302841187, + "logits/rejected": 0.6575336456298828, + "logps/chosen": -160.5447540283203, + "logps/ref_chosen": -50.20032501220703, + "logps/ref_rejected": -77.81680297851562, + "logps/rejected": -237.38534545898438, + "loss": 1.1325, + "margin_dpo/margin_mean": 49.224124908447266, + "margin_dpo/margin_std": 80.1656494140625, + "step": 431 + }, + { + "KL/chosen_KL_mean": -115.60556030273438, + "KL/mean": -142.0184326171875, + "KL/rejected_KL_mean": -168.4313201904297, + "KL/std": 71.81451416015625, + "epoch": 0.6530612244897959, + "fcm_dpo/beta": 0.007714337669312954, + "fcm_dpo/delta": -0.00784326996654272, + "fcm_dpo/margin": 52.82575607299805, + "fcm_dpo/q_t": 0.40753287076950073, + "grad_norm": 12.736228942871094, + "learning_rate": 1.632536862810844e-07, + "logits/chosen": 0.714253306388855, + "logits/rejected": 0.6646943092346191, + "logps/chosen": -177.268310546875, + "logps/ref_chosen": -61.662757873535156, + "logps/ref_rejected": -83.94496154785156, + "logps/rejected": -252.37628173828125, + "loss": 1.1177, + "margin_dpo/margin_mean": 52.82575988769531, + "margin_dpo/margin_std": 84.00270080566406, + "step": 432 + }, + { + "KL/chosen_KL_mean": -114.59454345703125, + "KL/mean": -147.77920532226562, + "KL/rejected_KL_mean": -180.96388244628906, + "KL/std": 71.03900146484375, + "epoch": 0.654572940287226, + "fcm_dpo/beta": 0.007613149471580982, + "fcm_dpo/delta": -0.1107371374964714, + "fcm_dpo/margin": 66.36933135986328, + "fcm_dpo/q_t": 0.3842705488204956, + "grad_norm": 13.615633010864258, + "learning_rate": 1.6201483487445515e-07, + "logits/chosen": 0.8374870419502258, + "logits/rejected": 0.8358460664749146, + "logps/chosen": -178.32373046875, + "logps/ref_chosen": -63.72917938232422, + "logps/ref_rejected": -65.8391342163086, + "logps/rejected": -246.80300903320312, + "loss": 1.0352, + "margin_dpo/margin_mean": 66.36933135986328, + "margin_dpo/margin_std": 81.50929260253906, + "step": 433 + }, + { + "KL/chosen_KL_mean": -96.70651245117188, + "KL/mean": -131.61221313476562, + "KL/rejected_KL_mean": -166.51791381835938, + "KL/std": 76.29074096679688, + "epoch": 0.656084656084656, + "fcm_dpo/beta": 0.007363257929682732, + "fcm_dpo/delta": -0.12183529138565063, + "fcm_dpo/margin": 69.8114013671875, + "fcm_dpo/q_t": 0.3824422359466553, + "grad_norm": 12.417044639587402, + "learning_rate": 1.6077844460203204e-07, + "logits/chosen": 0.8204172849655151, + "logits/rejected": 0.7561012506484985, + "logps/chosen": -144.67984008789062, + "logps/ref_chosen": -47.97331619262695, + "logps/ref_rejected": -72.51132202148438, + "logps/rejected": -239.02923583984375, + "loss": 1.0481, + "margin_dpo/margin_mean": 69.81139373779297, + "margin_dpo/margin_std": 92.68022155761719, + "step": 434 + }, + { + "KL/chosen_KL_mean": -118.66361999511719, + "KL/mean": -146.527587890625, + "KL/rejected_KL_mean": -174.39151000976562, + "KL/std": 73.32241821289062, + "epoch": 0.6575963718820862, + "fcm_dpo/beta": 0.0073859672993421555, + "fcm_dpo/delta": -0.01251184195280075, + "fcm_dpo/margin": 55.7279052734375, + "fcm_dpo/q_t": 0.4067944884300232, + "grad_norm": 13.418008804321289, + "learning_rate": 1.5954455004830878e-07, + "logits/chosen": 0.8510400056838989, + "logits/rejected": 0.8107461333274841, + "logps/chosen": -175.723876953125, + "logps/ref_chosen": -57.06024932861328, + "logps/ref_rejected": -71.69146728515625, + "logps/rejected": -246.08297729492188, + "loss": 1.1112, + "margin_dpo/margin_mean": 55.727901458740234, + "margin_dpo/margin_std": 85.01725006103516, + "step": 435 + }, + { + "KL/chosen_KL_mean": -116.6640853881836, + "KL/mean": -139.56753540039062, + "KL/rejected_KL_mean": -162.47097778320312, + "KL/std": 72.98526000976562, + "epoch": 0.6591080876795162, + "fcm_dpo/beta": 0.007416677661240101, + "fcm_dpo/delta": 0.06236447021365166, + "fcm_dpo/margin": 45.806915283203125, + "fcm_dpo/q_t": 0.4232286810874939, + "grad_norm": 14.484575271606445, + "learning_rate": 1.5831318572796847e-07, + "logits/chosen": 0.7485842108726501, + "logits/rejected": 0.690552830696106, + "logps/chosen": -172.8221435546875, + "logps/ref_chosen": -56.158050537109375, + "logps/ref_rejected": -67.63787841796875, + "logps/rejected": -230.10885620117188, + "loss": 1.1785, + "margin_dpo/margin_mean": 45.80691146850586, + "margin_dpo/margin_std": 89.24258422851562, + "step": 436 + }, + { + "KL/chosen_KL_mean": -123.96295928955078, + "KL/mean": -149.2787322998047, + "KL/rejected_KL_mean": -174.59451293945312, + "KL/std": 75.40545654296875, + "epoch": 0.6606198034769464, + "fcm_dpo/beta": 0.0073969424702227116, + "fcm_dpo/delta": 0.024935893714427948, + "fcm_dpo/margin": 50.63155746459961, + "fcm_dpo/q_t": 0.41527464985847473, + "grad_norm": 16.458776473999023, + "learning_rate": 1.5708438608491815e-07, + "logits/chosen": 0.6856381893157959, + "logits/rejected": 0.5573318004608154, + "logps/chosen": -180.94873046875, + "logps/ref_chosen": -56.98578643798828, + "logps/ref_rejected": -85.61524963378906, + "logps/rejected": -260.20977783203125, + "loss": 1.1665, + "margin_dpo/margin_mean": 50.63155746459961, + "margin_dpo/margin_std": 93.0724868774414, + "step": 437 + }, + { + "KL/chosen_KL_mean": -106.18060302734375, + "KL/mean": -141.49575805664062, + "KL/rejected_KL_mean": -176.81088256835938, + "KL/std": 79.54093933105469, + "epoch": 0.6621315192743764, + "fcm_dpo/beta": 0.007361456751823425, + "fcm_dpo/delta": -0.12632791697978973, + "fcm_dpo/margin": 70.63029479980469, + "fcm_dpo/q_t": 0.38328486680984497, + "grad_norm": 12.317891120910645, + "learning_rate": 1.558581854913253e-07, + "logits/chosen": 0.7801381349563599, + "logits/rejected": 0.7134337425231934, + "logps/chosen": -147.4583740234375, + "logps/ref_chosen": -41.27777862548828, + "logps/ref_rejected": -65.33840942382812, + "logps/rejected": -242.1492919921875, + "loss": 1.023, + "margin_dpo/margin_mean": 70.63029479980469, + "margin_dpo/margin_std": 84.64463806152344, + "step": 438 + }, + { + "KL/chosen_KL_mean": -119.46957397460938, + "KL/mean": -148.65093994140625, + "KL/rejected_KL_mean": -177.83230590820312, + "KL/std": 80.0428237915039, + "epoch": 0.6636432350718064, + "fcm_dpo/beta": 0.0072980402037501335, + "fcm_dpo/delta": -0.027525458484888077, + "fcm_dpo/margin": 58.362728118896484, + "fcm_dpo/q_t": 0.4030565023422241, + "grad_norm": 12.534804344177246, + "learning_rate": 1.5463461824665658e-07, + "logits/chosen": 0.626327633857727, + "logits/rejected": 0.5900809168815613, + "logps/chosen": -200.88723754882812, + "logps/ref_chosen": -81.41764831542969, + "logps/ref_rejected": -94.72309875488281, + "logps/rejected": -272.555419921875, + "loss": 1.0883, + "margin_dpo/margin_mean": 58.362728118896484, + "margin_dpo/margin_std": 80.85906982421875, + "step": 439 + }, + { + "KL/chosen_KL_mean": -106.9936294555664, + "KL/mean": -138.42535400390625, + "KL/rejected_KL_mean": -169.8570556640625, + "KL/std": 74.2397689819336, + "epoch": 0.6651549508692366, + "fcm_dpo/beta": 0.007225216832011938, + "fcm_dpo/delta": -0.05703933909535408, + "fcm_dpo/margin": 62.86344528198242, + "fcm_dpo/q_t": 0.3970058262348175, + "grad_norm": 21.790699005126953, + "learning_rate": 1.534137185767178e-07, + "logits/chosen": 0.7353362441062927, + "logits/rejected": 0.6341665387153625, + "logps/chosen": -149.5318145751953, + "logps/ref_chosen": -42.538185119628906, + "logps/ref_rejected": -69.78813934326172, + "logps/rejected": -239.64520263671875, + "loss": 1.0815, + "margin_dpo/margin_mean": 62.863441467285156, + "margin_dpo/margin_std": 88.48552703857422, + "step": 440 + }, + { + "KL/chosen_KL_mean": -104.67693328857422, + "KL/mean": -137.52198791503906, + "KL/rejected_KL_mean": -170.36705017089844, + "KL/std": 76.4732666015625, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.007063678000122309, + "fcm_dpo/delta": -0.06781375408172607, + "fcm_dpo/margin": 65.69010925292969, + "fcm_dpo/q_t": 0.3922446072101593, + "grad_norm": 13.523235321044922, + "learning_rate": 1.521955206326976e-07, + "logits/chosen": 0.6944692134857178, + "logits/rejected": 0.5998711585998535, + "logps/chosen": -162.27015686035156, + "logps/ref_chosen": -57.593223571777344, + "logps/ref_rejected": -84.82878875732422, + "logps/rejected": -255.19583129882812, + "loss": 1.0366, + "margin_dpo/margin_mean": 65.69010925292969, + "margin_dpo/margin_std": 71.41874694824219, + "step": 441 + }, + { + "KL/chosen_KL_mean": -125.55252075195312, + "KL/mean": -157.33255004882812, + "KL/rejected_KL_mean": -189.11260986328125, + "KL/std": 75.97675323486328, + "epoch": 0.6681783824640968, + "fcm_dpo/beta": 0.0070096999406814575, + "fcm_dpo/delta": -0.04777270928025246, + "fcm_dpo/margin": 63.56007766723633, + "fcm_dpo/q_t": 0.3974016606807709, + "grad_norm": 15.098261833190918, + "learning_rate": 1.5098005849021078e-07, + "logits/chosen": 0.7284977436065674, + "logits/rejected": 0.6738515496253967, + "logps/chosen": -193.01373291015625, + "logps/ref_chosen": -67.46121978759766, + "logps/ref_rejected": -89.0693588256836, + "logps/rejected": -278.18194580078125, + "loss": 1.0661, + "margin_dpo/margin_mean": 63.56007385253906, + "margin_dpo/margin_std": 81.50393676757812, + "step": 442 + }, + { + "KL/chosen_KL_mean": -104.5126953125, + "KL/mean": -144.3497772216797, + "KL/rejected_KL_mean": -184.18685913085938, + "KL/std": 81.44729614257812, + "epoch": 0.6696900982615268, + "fcm_dpo/beta": 0.0068482570350170135, + "fcm_dpo/delta": -0.15395689010620117, + "fcm_dpo/margin": 79.67416381835938, + "fcm_dpo/q_t": 0.3748539090156555, + "grad_norm": 13.461181640625, + "learning_rate": 1.4976736614834662e-07, + "logits/chosen": 0.7253681421279907, + "logits/rejected": 0.6550537943840027, + "logps/chosen": -159.30880737304688, + "logps/ref_chosen": -54.79610061645508, + "logps/ref_rejected": -77.80781555175781, + "logps/rejected": -261.99468994140625, + "loss": 1.0049, + "margin_dpo/margin_mean": 79.67416381835938, + "margin_dpo/margin_std": 91.89553833007812, + "step": 443 + }, + { + "KL/chosen_KL_mean": -131.12448120117188, + "KL/mean": -145.68206787109375, + "KL/rejected_KL_mean": -160.23968505859375, + "KL/std": 78.9239501953125, + "epoch": 0.671201814058957, + "fcm_dpo/beta": 0.006958891171962023, + "fcm_dpo/delta": 0.20196352899074554, + "fcm_dpo/margin": 29.115209579467773, + "fcm_dpo/q_t": 0.45473960041999817, + "grad_norm": 17.20579719543457, + "learning_rate": 1.4855747752871654e-07, + "logits/chosen": 0.737562894821167, + "logits/rejected": 0.6410528421401978, + "logps/chosen": -189.87353515625, + "logps/ref_chosen": -58.749061584472656, + "logps/ref_rejected": -86.87396240234375, + "logps/rejected": -247.1136474609375, + "loss": 1.2886, + "margin_dpo/margin_mean": 29.11520767211914, + "margin_dpo/margin_std": 87.74626159667969, + "step": 444 + }, + { + "KL/chosen_KL_mean": -123.03514099121094, + "KL/mean": -157.38128662109375, + "KL/rejected_KL_mean": -191.7274169921875, + "KL/std": 77.95574951171875, + "epoch": 0.672713529856387, + "fcm_dpo/beta": 0.006988787557929754, + "fcm_dpo/delta": -0.0840548425912857, + "fcm_dpo/margin": 68.69226837158203, + "fcm_dpo/q_t": 0.39024075865745544, + "grad_norm": 15.367069244384766, + "learning_rate": 1.473504264745062e-07, + "logits/chosen": 0.7162766456604004, + "logits/rejected": 0.7037680149078369, + "logps/chosen": -183.95257568359375, + "logps/ref_chosen": -60.91743850708008, + "logps/ref_rejected": -71.5637435913086, + "logps/rejected": -263.2911376953125, + "loss": 1.0475, + "margin_dpo/margin_mean": 68.6922607421875, + "margin_dpo/margin_std": 86.09353637695312, + "step": 445 + }, + { + "KL/chosen_KL_mean": -111.1749267578125, + "KL/mean": -146.70782470703125, + "KL/rejected_KL_mean": -182.24072265625, + "KL/std": 79.02342224121094, + "epoch": 0.674225245653817, + "fcm_dpo/beta": 0.0069199297577142715, + "fcm_dpo/delta": -0.09750378131866455, + "fcm_dpo/margin": 71.06580352783203, + "fcm_dpo/q_t": 0.3867424726486206, + "grad_norm": 11.384917259216309, + "learning_rate": 1.461462467495284e-07, + "logits/chosen": 0.7947849631309509, + "logits/rejected": 0.7108563184738159, + "logps/chosen": -159.97418212890625, + "logps/ref_chosen": -48.79924774169922, + "logps/ref_rejected": -71.8719482421875, + "logps/rejected": -254.1126708984375, + "loss": 1.0243, + "margin_dpo/margin_mean": 71.06580352783203, + "margin_dpo/margin_std": 73.57239532470703, + "step": 446 + }, + { + "KL/chosen_KL_mean": -109.34071350097656, + "KL/mean": -149.2745361328125, + "KL/rejected_KL_mean": -189.2083740234375, + "KL/std": 78.32331848144531, + "epoch": 0.6757369614512472, + "fcm_dpo/beta": 0.006653862539678812, + "fcm_dpo/delta": -0.13958829641342163, + "fcm_dpo/margin": 79.86766052246094, + "fcm_dpo/q_t": 0.37667399644851685, + "grad_norm": 16.19961929321289, + "learning_rate": 1.4494497203727843e-07, + "logits/chosen": 0.677545428276062, + "logits/rejected": 0.5737296342849731, + "logps/chosen": -163.0234375, + "logps/ref_chosen": -53.682716369628906, + "logps/ref_rejected": -88.17315673828125, + "logps/rejected": -277.38153076171875, + "loss": 1.0133, + "margin_dpo/margin_mean": 79.86766052246094, + "margin_dpo/margin_std": 91.65660095214844, + "step": 447 + }, + { + "KL/chosen_KL_mean": -116.87925720214844, + "KL/mean": -148.92613220214844, + "KL/rejected_KL_mean": -180.9730224609375, + "KL/std": 78.32139587402344, + "epoch": 0.6772486772486772, + "fcm_dpo/beta": 0.006604928523302078, + "fcm_dpo/delta": -0.024372804909944534, + "fcm_dpo/margin": 64.09376525878906, + "fcm_dpo/q_t": 0.4019731283187866, + "grad_norm": 11.611702919006348, + "learning_rate": 1.4374663593999256e-07, + "logits/chosen": 0.7389341592788696, + "logits/rejected": 0.6849086880683899, + "logps/chosen": -170.63050842285156, + "logps/ref_chosen": -53.75125503540039, + "logps/ref_rejected": -77.17623901367188, + "logps/rejected": -258.1492614746094, + "loss": 1.0813, + "margin_dpo/margin_mean": 64.09376525878906, + "margin_dpo/margin_std": 84.98629760742188, + "step": 448 + }, + { + "KL/chosen_KL_mean": -138.70216369628906, + "KL/mean": -153.8754425048828, + "KL/rejected_KL_mean": -169.04872131347656, + "KL/std": 79.18728637695312, + "epoch": 0.6787603930461074, + "fcm_dpo/beta": 0.006783435121178627, + "fcm_dpo/delta": 0.19855058193206787, + "fcm_dpo/margin": 30.346546173095703, + "fcm_dpo/q_t": 0.45337286591529846, + "grad_norm": 18.612869262695312, + "learning_rate": 1.4255127197770707e-07, + "logits/chosen": 0.5823932886123657, + "logits/rejected": 0.5809487104415894, + "logps/chosen": -214.529541015625, + "logps/ref_chosen": -75.82737731933594, + "logps/ref_rejected": -82.20687866210938, + "logps/rejected": -251.25558471679688, + "loss": 1.2588, + "margin_dpo/margin_mean": 30.346548080444336, + "margin_dpo/margin_std": 77.39900970458984, + "step": 449 + }, + { + "KL/chosen_KL_mean": -117.12738037109375, + "KL/mean": -141.69601440429688, + "KL/rejected_KL_mean": -166.26466369628906, + "KL/std": 79.31319427490234, + "epoch": 0.6802721088435374, + "fcm_dpo/beta": 0.006913709454238415, + "fcm_dpo/delta": 0.06243385374546051, + "fcm_dpo/margin": 49.137298583984375, + "fcm_dpo/q_t": 0.4247365891933441, + "grad_norm": 13.703746795654297, + "learning_rate": 1.4135891358732205e-07, + "logits/chosen": 0.8435397148132324, + "logits/rejected": 0.7237097024917603, + "logps/chosen": -164.24310302734375, + "logps/ref_chosen": -47.11572265625, + "logps/ref_rejected": -78.7546615600586, + "logps/rejected": -245.0193328857422, + "loss": 1.1683, + "margin_dpo/margin_mean": 49.137298583984375, + "margin_dpo/margin_std": 90.78799438476562, + "step": 450 + }, + { + "KL/chosen_KL_mean": -117.35845184326172, + "KL/mean": -139.92562866210938, + "KL/rejected_KL_mean": -162.49281311035156, + "KL/std": 78.64613342285156, + "epoch": 0.6817838246409675, + "fcm_dpo/beta": 0.007054210640490055, + "fcm_dpo/delta": 0.08403199911117554, + "fcm_dpo/margin": 45.134361267089844, + "fcm_dpo/q_t": 0.4276706576347351, + "grad_norm": 13.453941345214844, + "learning_rate": 1.4016959412166437e-07, + "logits/chosen": 0.7027239799499512, + "logits/rejected": 0.6480429172515869, + "logps/chosen": -180.70889282226562, + "logps/ref_chosen": -63.350440979003906, + "logps/ref_rejected": -76.28530883789062, + "logps/rejected": -238.77810668945312, + "loss": 1.1803, + "margin_dpo/margin_mean": 45.134361267089844, + "margin_dpo/margin_std": 86.11540222167969, + "step": 451 + }, + { + "KL/chosen_KL_mean": -115.05659484863281, + "KL/mean": -141.87234497070312, + "KL/rejected_KL_mean": -168.6881103515625, + "KL/std": 74.24472045898438, + "epoch": 0.6832955404383976, + "fcm_dpo/beta": 0.007099970243871212, + "fcm_dpo/delta": 0.01991662010550499, + "fcm_dpo/margin": 53.63151550292969, + "fcm_dpo/q_t": 0.413438618183136, + "grad_norm": 14.759235382080078, + "learning_rate": 1.3898334684855645e-07, + "logits/chosen": 0.685683012008667, + "logits/rejected": 0.6011617183685303, + "logps/chosen": -170.64242553710938, + "logps/ref_chosen": -55.58583450317383, + "logps/ref_rejected": -77.68738555908203, + "logps/rejected": -246.37548828125, + "loss": 1.1438, + "margin_dpo/margin_mean": 53.63151550292969, + "margin_dpo/margin_std": 92.3481674194336, + "step": 452 + }, + { + "KL/chosen_KL_mean": -111.09956359863281, + "KL/mean": -140.02688598632812, + "KL/rejected_KL_mean": -168.95423889160156, + "KL/std": 77.94131469726562, + "epoch": 0.6848072562358276, + "fcm_dpo/beta": 0.007077625021338463, + "fcm_dpo/delta": -0.009929983876645565, + "fcm_dpo/margin": 57.85468292236328, + "fcm_dpo/q_t": 0.4080149531364441, + "grad_norm": 15.38828182220459, + "learning_rate": 1.3780020494988445e-07, + "logits/chosen": 0.6994049549102783, + "logits/rejected": 0.6696133613586426, + "logps/chosen": -172.8777618408203, + "logps/ref_chosen": -61.778202056884766, + "logps/ref_rejected": -71.51403045654297, + "logps/rejected": -240.46826171875, + "loss": 1.116, + "margin_dpo/margin_mean": 57.85468292236328, + "margin_dpo/margin_std": 91.03846740722656, + "step": 453 + }, + { + "KL/chosen_KL_mean": -104.55786895751953, + "KL/mean": -136.8068084716797, + "KL/rejected_KL_mean": -169.05575561523438, + "KL/std": 79.33438110351562, + "epoch": 0.6863189720332578, + "fcm_dpo/beta": 0.007016636431217194, + "fcm_dpo/delta": -0.05515647307038307, + "fcm_dpo/margin": 64.49788665771484, + "fcm_dpo/q_t": 0.3954145610332489, + "grad_norm": 13.037610054016113, + "learning_rate": 1.366202015206706e-07, + "logits/chosen": 0.7593111991882324, + "logits/rejected": 0.7165501117706299, + "logps/chosen": -156.15301513671875, + "logps/ref_chosen": -51.59515380859375, + "logps/ref_rejected": -63.96732711791992, + "logps/rejected": -233.0230712890625, + "loss": 1.0717, + "margin_dpo/margin_mean": 64.49789428710938, + "margin_dpo/margin_std": 87.31886291503906, + "step": 454 + }, + { + "KL/chosen_KL_mean": -120.95310974121094, + "KL/mean": -151.56578063964844, + "KL/rejected_KL_mean": -182.17843627929688, + "KL/std": 77.6214599609375, + "epoch": 0.6878306878306878, + "fcm_dpo/beta": 0.006965038366615772, + "fcm_dpo/delta": -0.027724087238311768, + "fcm_dpo/margin": 61.225337982177734, + "fcm_dpo/q_t": 0.40344443917274475, + "grad_norm": 12.801860809326172, + "learning_rate": 1.354433695681474e-07, + "logits/chosen": 0.6048588156700134, + "logits/rejected": 0.570784330368042, + "logps/chosen": -191.6048126220703, + "logps/ref_chosen": -70.65170288085938, + "logps/ref_rejected": -77.44276428222656, + "logps/rejected": -259.6212158203125, + "loss": 1.0845, + "margin_dpo/margin_mean": 61.22533416748047, + "margin_dpo/margin_std": 84.29511260986328, + "step": 455 + }, + { + "KL/chosen_KL_mean": -121.42324829101562, + "KL/mean": -147.92445373535156, + "KL/rejected_KL_mean": -174.42567443847656, + "KL/std": 77.14093017578125, + "epoch": 0.6893424036281179, + "fcm_dpo/beta": 0.006989809684455395, + "fcm_dpo/delta": 0.0306635070592165, + "fcm_dpo/margin": 53.002410888671875, + "fcm_dpo/q_t": 0.4164670705795288, + "grad_norm": 15.369559288024902, + "learning_rate": 1.3426974201083439e-07, + "logits/chosen": 0.6348047852516174, + "logits/rejected": 0.5682092905044556, + "logps/chosen": -177.821533203125, + "logps/ref_chosen": -56.398284912109375, + "logps/ref_rejected": -82.61642456054688, + "logps/rejected": -257.0420837402344, + "loss": 1.1384, + "margin_dpo/margin_mean": 53.002410888671875, + "margin_dpo/margin_std": 87.64601135253906, + "step": 456 + }, + { + "KL/chosen_KL_mean": -118.45246887207031, + "KL/mean": -147.9862060546875, + "KL/rejected_KL_mean": -177.5199737548828, + "KL/std": 78.8670883178711, + "epoch": 0.690854119425548, + "fcm_dpo/beta": 0.0069747138768434525, + "fcm_dpo/delta": -0.012739144265651703, + "fcm_dpo/margin": 59.06749725341797, + "fcm_dpo/q_t": 0.40447694063186646, + "grad_norm": 11.998747825622559, + "learning_rate": 1.3309935167761717e-07, + "logits/chosen": 0.808464765548706, + "logits/rejected": 0.7306153774261475, + "logps/chosen": -163.17303466796875, + "logps/ref_chosen": -44.72057342529297, + "logps/ref_rejected": -68.1158676147461, + "logps/rejected": -245.63583374023438, + "loss": 1.0837, + "margin_dpo/margin_mean": 59.06749725341797, + "margin_dpo/margin_std": 76.19584655761719, + "step": 457 + }, + { + "KL/chosen_KL_mean": -114.44857788085938, + "KL/mean": -144.8822479248047, + "KL/rejected_KL_mean": -175.31591796875, + "KL/std": 81.75318908691406, + "epoch": 0.6923658352229781, + "fcm_dpo/beta": 0.0069658150896430016, + "fcm_dpo/delta": -0.02505449205636978, + "fcm_dpo/margin": 60.867340087890625, + "fcm_dpo/q_t": 0.4031820595264435, + "grad_norm": 13.551817893981934, + "learning_rate": 1.3193223130682936e-07, + "logits/chosen": 0.7497356534004211, + "logits/rejected": 0.6261578798294067, + "logps/chosen": -164.4542694091797, + "logps/ref_chosen": -50.00569152832031, + "logps/ref_rejected": -87.50015258789062, + "logps/rejected": -262.8160705566406, + "loss": 1.1009, + "margin_dpo/margin_mean": 60.867340087890625, + "margin_dpo/margin_std": 91.40193176269531, + "step": 458 + }, + { + "KL/chosen_KL_mean": -106.50845336914062, + "KL/mean": -144.07794189453125, + "KL/rejected_KL_mean": -181.64743041992188, + "KL/std": 86.97746276855469, + "epoch": 0.6938775510204082, + "fcm_dpo/beta": 0.006888561882078648, + "fcm_dpo/delta": -0.12443508207798004, + "fcm_dpo/margin": 75.13898468017578, + "fcm_dpo/q_t": 0.38124239444732666, + "grad_norm": 12.490287780761719, + "learning_rate": 1.3076841354533658e-07, + "logits/chosen": 0.7258360981941223, + "logits/rejected": 0.690863847732544, + "logps/chosen": -171.8863983154297, + "logps/ref_chosen": -65.37794494628906, + "logps/ref_rejected": -88.19244384765625, + "logps/rejected": -269.8398742675781, + "loss": 1.0209, + "margin_dpo/margin_mean": 75.13898468017578, + "margin_dpo/margin_std": 84.60592651367188, + "step": 459 + }, + { + "KL/chosen_KL_mean": -120.4828109741211, + "KL/mean": -156.8914794921875, + "KL/rejected_KL_mean": -193.3001708984375, + "KL/std": 80.62806701660156, + "epoch": 0.6953892668178382, + "fcm_dpo/beta": 0.006671781651675701, + "fcm_dpo/delta": -0.09077716618776321, + "fcm_dpo/margin": 72.81735229492188, + "fcm_dpo/q_t": 0.38928472995758057, + "grad_norm": 12.194050788879395, + "learning_rate": 1.2960793094762345e-07, + "logits/chosen": 0.7333745956420898, + "logits/rejected": 0.6062943935394287, + "logps/chosen": -185.0444793701172, + "logps/ref_chosen": -64.5616683959961, + "logps/ref_rejected": -88.67890167236328, + "logps/rejected": -281.97906494140625, + "loss": 1.0364, + "margin_dpo/margin_mean": 72.81735229492188, + "margin_dpo/margin_std": 86.62645721435547, + "step": 460 + }, + { + "KL/chosen_KL_mean": -99.09426879882812, + "KL/mean": -136.2252960205078, + "KL/rejected_KL_mean": -173.35629272460938, + "KL/std": 74.94339752197266, + "epoch": 0.6969009826152683, + "fcm_dpo/beta": 0.006514571141451597, + "fcm_dpo/delta": -0.0906326100230217, + "fcm_dpo/margin": 74.26202392578125, + "fcm_dpo/q_t": 0.3882749676704407, + "grad_norm": 12.746190071105957, + "learning_rate": 1.2845081597488286e-07, + "logits/chosen": 0.8655481338500977, + "logits/rejected": 0.7720522880554199, + "logps/chosen": -148.5721893310547, + "logps/ref_chosen": -49.4779167175293, + "logps/ref_rejected": -72.65262603759766, + "logps/rejected": -246.00892639160156, + "loss": 1.0371, + "margin_dpo/margin_mean": 74.26202392578125, + "margin_dpo/margin_std": 83.59939575195312, + "step": 461 + }, + { + "KL/chosen_KL_mean": -111.19660949707031, + "KL/mean": -149.9906463623047, + "KL/rejected_KL_mean": -188.78466796875, + "KL/std": 79.32613372802734, + "epoch": 0.6984126984126984, + "fcm_dpo/beta": 0.006418838165700436, + "fcm_dpo/delta": -0.10389072448015213, + "fcm_dpo/margin": 77.58805847167969, + "fcm_dpo/q_t": 0.3840404748916626, + "grad_norm": 12.104856491088867, + "learning_rate": 1.27297100994108e-07, + "logits/chosen": 0.7282806634902954, + "logits/rejected": 0.6704307198524475, + "logps/chosen": -171.6917266845703, + "logps/ref_chosen": -60.4951171875, + "logps/ref_rejected": -74.82136535644531, + "logps/rejected": -263.6060485839844, + "loss": 1.0245, + "margin_dpo/margin_mean": 77.58805847167969, + "margin_dpo/margin_std": 87.69475555419922, + "step": 462 + }, + { + "KL/chosen_KL_mean": -130.89987182617188, + "KL/mean": -155.0953369140625, + "KL/rejected_KL_mean": -179.29078674316406, + "KL/std": 75.73387145996094, + "epoch": 0.6999244142101285, + "fcm_dpo/beta": 0.00646938756108284, + "fcm_dpo/delta": 0.08976850658655167, + "fcm_dpo/margin": 48.39093017578125, + "fcm_dpo/q_t": 0.4274500608444214, + "grad_norm": 15.210125923156738, + "learning_rate": 1.2614681827718695e-07, + "logits/chosen": 0.7191846966743469, + "logits/rejected": 0.7206935882568359, + "logps/chosen": -198.58499145507812, + "logps/ref_chosen": -67.68511962890625, + "logps/ref_rejected": -71.32196044921875, + "logps/rejected": -250.6127471923828, + "loss": 1.1651, + "margin_dpo/margin_mean": 48.39093017578125, + "margin_dpo/margin_std": 80.35902404785156, + "step": 463 + }, + { + "KL/chosen_KL_mean": -121.09156799316406, + "KL/mean": -155.86016845703125, + "KL/rejected_KL_mean": -190.62875366210938, + "KL/std": 81.31057739257812, + "epoch": 0.7014361300075586, + "fcm_dpo/beta": 0.006499043665826321, + "fcm_dpo/delta": -0.05448608100414276, + "fcm_dpo/margin": 69.53720092773438, + "fcm_dpo/q_t": 0.39638495445251465, + "grad_norm": 11.573769569396973, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": 0.7025905847549438, + "logits/rejected": 0.6747890710830688, + "logps/chosen": -180.2572021484375, + "logps/ref_chosen": -59.16564178466797, + "logps/ref_rejected": -69.56146240234375, + "logps/rejected": -260.1902160644531, + "loss": 1.0773, + "margin_dpo/margin_mean": 69.53720092773438, + "margin_dpo/margin_std": 94.30844116210938, + "step": 464 + }, + { + "KL/chosen_KL_mean": -127.25249481201172, + "KL/mean": -157.2509765625, + "KL/rejected_KL_mean": -187.2494659423828, + "KL/std": 79.7821044921875, + "epoch": 0.7029478458049887, + "fcm_dpo/beta": 0.006456965580582619, + "fcm_dpo/delta": 0.013073013164103031, + "fcm_dpo/margin": 59.9969596862793, + "fcm_dpo/q_t": 0.41156071424484253, + "grad_norm": 13.067460060119629, + "learning_rate": 1.238566782415197e-07, + "logits/chosen": 0.8062239289283752, + "logits/rejected": 0.7412049770355225, + "logps/chosen": -185.76617431640625, + "logps/ref_chosen": -58.513671875, + "logps/ref_rejected": -84.31745910644531, + "logps/rejected": -271.5669250488281, + "loss": 1.1205, + "margin_dpo/margin_mean": 59.99695587158203, + "margin_dpo/margin_std": 92.42967224121094, + "step": 465 + }, + { + "KL/chosen_KL_mean": -137.55921936035156, + "KL/mean": -156.19586181640625, + "KL/rejected_KL_mean": -174.83248901367188, + "KL/std": 84.31153869628906, + "epoch": 0.7044595616024187, + "fcm_dpo/beta": 0.0066132983192801476, + "fcm_dpo/delta": 0.1576167196035385, + "fcm_dpo/margin": 37.27326583862305, + "fcm_dpo/q_t": 0.4440735876560211, + "grad_norm": 17.67957878112793, + "learning_rate": 1.2271688498291334e-07, + "logits/chosen": 0.7369842529296875, + "logits/rejected": 0.7448593378067017, + "logps/chosen": -210.82504272460938, + "logps/ref_chosen": -73.26580810546875, + "logps/ref_rejected": -74.83621215820312, + "logps/rejected": -249.668701171875, + "loss": 1.2244, + "margin_dpo/margin_mean": 37.27326965332031, + "margin_dpo/margin_std": 81.10662841796875, + "step": 466 + }, + { + "KL/chosen_KL_mean": -123.79527282714844, + "KL/mean": -149.83084106445312, + "KL/rejected_KL_mean": -175.86642456054688, + "KL/std": 81.82411193847656, + "epoch": 0.7059712773998488, + "fcm_dpo/beta": 0.006732389330863953, + "fcm_dpo/delta": 0.05121883749961853, + "fcm_dpo/margin": 52.0711555480957, + "fcm_dpo/q_t": 0.42000895738601685, + "grad_norm": 12.780661582946777, + "learning_rate": 1.2158065210664848e-07, + "logits/chosen": 0.7948445081710815, + "logits/rejected": 0.6336175203323364, + "logps/chosen": -171.374755859375, + "logps/ref_chosen": -47.57947540283203, + "logps/ref_rejected": -78.68522644042969, + "logps/rejected": -254.5516357421875, + "loss": 1.1342, + "margin_dpo/margin_mean": 52.07115936279297, + "margin_dpo/margin_std": 80.35679626464844, + "step": 467 + }, + { + "KL/chosen_KL_mean": -115.81289672851562, + "KL/mean": -152.6653289794922, + "KL/rejected_KL_mean": -189.51779174804688, + "KL/std": 81.38355255126953, + "epoch": 0.7074829931972789, + "fcm_dpo/beta": 0.006664079613983631, + "fcm_dpo/delta": -0.09578202664852142, + "fcm_dpo/margin": 73.70487976074219, + "fcm_dpo/q_t": 0.38689950108528137, + "grad_norm": 15.063311576843262, + "learning_rate": 1.204480113956011e-07, + "logits/chosen": 0.6837078928947449, + "logits/rejected": 0.6759430170059204, + "logps/chosen": -179.74069213867188, + "logps/ref_chosen": -63.92778778076172, + "logps/ref_rejected": -76.51626586914062, + "logps/rejected": -266.0340576171875, + "loss": 1.0428, + "margin_dpo/margin_mean": 73.70487976074219, + "margin_dpo/margin_std": 91.98550415039062, + "step": 468 + }, + { + "KL/chosen_KL_mean": -117.41605377197266, + "KL/mean": -152.40176391601562, + "KL/rejected_KL_mean": -187.38748168945312, + "KL/std": 80.2814712524414, + "epoch": 0.708994708994709, + "fcm_dpo/beta": 0.006520968861877918, + "fcm_dpo/delta": -0.060121070593595505, + "fcm_dpo/margin": 69.971435546875, + "fcm_dpo/q_t": 0.3936420977115631, + "grad_norm": 12.434749603271484, + "learning_rate": 1.1931899453216697e-07, + "logits/chosen": 0.8080254793167114, + "logits/rejected": 0.7981134653091431, + "logps/chosen": -176.4742431640625, + "logps/ref_chosen": -59.05818176269531, + "logps/ref_rejected": -75.67672729492188, + "logps/rejected": -263.064208984375, + "loss": 1.0419, + "margin_dpo/margin_mean": 69.971435546875, + "margin_dpo/margin_std": 75.70211029052734, + "step": 469 + }, + { + "KL/chosen_KL_mean": -114.1371841430664, + "KL/mean": -146.21981811523438, + "KL/rejected_KL_mean": -178.30245971679688, + "KL/std": 78.92847442626953, + "epoch": 0.7105064247921391, + "fcm_dpo/beta": 0.006540331523865461, + "fcm_dpo/delta": -0.020640280097723007, + "fcm_dpo/margin": 64.16529846191406, + "fcm_dpo/q_t": 0.4032108783721924, + "grad_norm": 12.303239822387695, + "learning_rate": 1.1819363309737438e-07, + "logits/chosen": 0.773910641670227, + "logits/rejected": 0.6982936859130859, + "logps/chosen": -162.00462341308594, + "logps/ref_chosen": -47.86743927001953, + "logps/ref_rejected": -65.96859741210938, + "logps/rejected": -244.27105712890625, + "loss": 1.0885, + "margin_dpo/margin_mean": 64.16529846191406, + "margin_dpo/margin_std": 87.60211181640625, + "step": 470 + }, + { + "KL/chosen_KL_mean": -108.57453155517578, + "KL/mean": -146.01129150390625, + "KL/rejected_KL_mean": -183.44805908203125, + "KL/std": 79.43025207519531, + "epoch": 0.7120181405895691, + "fcm_dpo/beta": 0.006422456353902817, + "fcm_dpo/delta": -0.08513358235359192, + "fcm_dpo/margin": 74.87351989746094, + "fcm_dpo/q_t": 0.38841187953948975, + "grad_norm": 11.945752143859863, + "learning_rate": 1.1707195857000215e-07, + "logits/chosen": 0.7363976240158081, + "logits/rejected": 0.6791332364082336, + "logps/chosen": -166.35238647460938, + "logps/ref_chosen": -57.777854919433594, + "logps/ref_rejected": -73.81172180175781, + "logps/rejected": -257.259765625, + "loss": 1.0388, + "margin_dpo/margin_mean": 74.87351989746094, + "margin_dpo/margin_std": 88.61801147460938, + "step": 471 + }, + { + "KL/chosen_KL_mean": -111.45721435546875, + "KL/mean": -140.34298706054688, + "KL/rejected_KL_mean": -169.22872924804688, + "KL/std": 80.24115753173828, + "epoch": 0.7135298563869993, + "fcm_dpo/beta": 0.006452543195337057, + "fcm_dpo/delta": 0.027957899495959282, + "fcm_dpo/margin": 57.77153015136719, + "fcm_dpo/q_t": 0.41539645195007324, + "grad_norm": 12.780068397521973, + "learning_rate": 1.1595400232569768e-07, + "logits/chosen": 0.7212907075881958, + "logits/rejected": 0.6750373840332031, + "logps/chosen": -167.36587524414062, + "logps/ref_chosen": -55.908668518066406, + "logps/ref_rejected": -74.70294189453125, + "logps/rejected": -243.93167114257812, + "loss": 1.149, + "margin_dpo/margin_mean": 57.771522521972656, + "margin_dpo/margin_std": 101.25234985351562, + "step": 472 + }, + { + "KL/chosen_KL_mean": -113.79630279541016, + "KL/mean": -143.42703247070312, + "KL/rejected_KL_mean": -173.05772399902344, + "KL/std": 83.41293334960938, + "epoch": 0.7150415721844293, + "fcm_dpo/beta": 0.006456817500293255, + "fcm_dpo/delta": 0.018051929771900177, + "fcm_dpo/margin": 59.261444091796875, + "fcm_dpo/q_t": 0.4138728678226471, + "grad_norm": 14.013310432434082, + "learning_rate": 1.1483979563610069e-07, + "logits/chosen": 0.837611198425293, + "logits/rejected": 0.7160617113113403, + "logps/chosen": -167.95718383789062, + "logps/ref_chosen": -54.16088104248047, + "logps/ref_rejected": -92.76789855957031, + "logps/rejected": -265.82562255859375, + "loss": 1.1456, + "margin_dpo/margin_mean": 59.261444091796875, + "margin_dpo/margin_std": 103.79183959960938, + "step": 473 + }, + { + "KL/chosen_KL_mean": -118.40287017822266, + "KL/mean": -146.54998779296875, + "KL/rejected_KL_mean": -174.69711303710938, + "KL/std": 81.75066375732422, + "epoch": 0.7165532879818595, + "fcm_dpo/beta": 0.006502171978354454, + "fcm_dpo/delta": 0.03521895781159401, + "fcm_dpo/margin": 56.29423904418945, + "fcm_dpo/q_t": 0.41749250888824463, + "grad_norm": 16.796932220458984, + "learning_rate": 1.1372936966796709e-07, + "logits/chosen": 0.7968182563781738, + "logits/rejected": 0.7175832986831665, + "logps/chosen": -165.0885772705078, + "logps/ref_chosen": -46.685707092285156, + "logps/ref_rejected": -71.44731903076172, + "logps/rejected": -246.14443969726562, + "loss": 1.1515, + "margin_dpo/margin_mean": 56.29423904418945, + "margin_dpo/margin_std": 99.43895721435547, + "step": 474 + }, + { + "KL/chosen_KL_mean": -110.75035095214844, + "KL/mean": -152.51861572265625, + "KL/rejected_KL_mean": -194.28683471679688, + "KL/std": 83.92522430419922, + "epoch": 0.7180650037792895, + "fcm_dpo/beta": 0.006350798532366753, + "fcm_dpo/delta": -0.13872043788433075, + "fcm_dpo/margin": 83.5364990234375, + "fcm_dpo/q_t": 0.37842634320259094, + "grad_norm": 9.905010223388672, + "learning_rate": 1.126227554822985e-07, + "logits/chosen": 0.7357660531997681, + "logits/rejected": 0.684442400932312, + "logps/chosen": -169.23765563964844, + "logps/ref_chosen": -58.4873046875, + "logps/ref_rejected": -87.00187683105469, + "logps/rejected": -281.2887268066406, + "loss": 1.0003, + "margin_dpo/margin_mean": 83.5364990234375, + "margin_dpo/margin_std": 89.66531372070312, + "step": 475 + }, + { + "KL/chosen_KL_mean": -134.1099853515625, + "KL/mean": -162.94261169433594, + "KL/rejected_KL_mean": -191.7752227783203, + "KL/std": 84.37803649902344, + "epoch": 0.7195767195767195, + "fcm_dpo/beta": 0.006389183923602104, + "fcm_dpo/delta": 0.03224918246269226, + "fcm_dpo/margin": 57.665252685546875, + "fcm_dpo/q_t": 0.4159722924232483, + "grad_norm": 12.826769828796387, + "learning_rate": 1.1151998403347243e-07, + "logits/chosen": 0.6389992237091064, + "logits/rejected": 0.6379086971282959, + "logps/chosen": -209.49160766601562, + "logps/ref_chosen": -75.38162231445312, + "logps/ref_rejected": -76.99822235107422, + "logps/rejected": -268.7734375, + "loss": 1.1346, + "margin_dpo/margin_mean": 57.665252685546875, + "margin_dpo/margin_std": 91.91641235351562, + "step": 476 + }, + { + "KL/chosen_KL_mean": -139.5823974609375, + "KL/mean": -166.55809020996094, + "KL/rejected_KL_mean": -193.53378295898438, + "KL/std": 83.39258575439453, + "epoch": 0.7210884353741497, + "fcm_dpo/beta": 0.0064208912663161755, + "fcm_dpo/delta": 0.055533844977617264, + "fcm_dpo/margin": 53.95138931274414, + "fcm_dpo/q_t": 0.4213593006134033, + "grad_norm": 15.228546142578125, + "learning_rate": 1.1042108616837692e-07, + "logits/chosen": 0.7572512626647949, + "logits/rejected": 0.7053213119506836, + "logps/chosen": -200.65579223632812, + "logps/ref_chosen": -61.073387145996094, + "logps/ref_rejected": -81.34375, + "logps/rejected": -274.8775329589844, + "loss": 1.1896, + "margin_dpo/margin_mean": 53.95138931274414, + "margin_dpo/margin_std": 110.94090270996094, + "step": 477 + }, + { + "KL/chosen_KL_mean": -122.52293395996094, + "KL/mean": -146.7134246826172, + "KL/rejected_KL_mean": -170.9039306640625, + "KL/std": 79.75495910644531, + "epoch": 0.7226001511715797, + "fcm_dpo/beta": 0.0065117040649056435, + "fcm_dpo/delta": 0.08781292289495468, + "fcm_dpo/margin": 48.3809814453125, + "fcm_dpo/q_t": 0.42833536863327026, + "grad_norm": 15.279951095581055, + "learning_rate": 1.0932609262554746e-07, + "logits/chosen": 0.68839031457901, + "logits/rejected": 0.698960542678833, + "logps/chosen": -179.69024658203125, + "logps/ref_chosen": -57.16731643676758, + "logps/ref_rejected": -53.30917739868164, + "logps/rejected": -224.21310424804688, + "loss": 1.1919, + "margin_dpo/margin_mean": 48.3809814453125, + "margin_dpo/margin_std": 97.17034912109375, + "step": 478 + }, + { + "KL/chosen_KL_mean": -127.67341613769531, + "KL/mean": -149.62149047851562, + "KL/rejected_KL_mean": -171.56954956054688, + "KL/std": 76.23765563964844, + "epoch": 0.7241118669690099, + "fcm_dpo/beta": 0.006642760708928108, + "fcm_dpo/delta": 0.11179714649915695, + "fcm_dpo/margin": 43.89613342285156, + "fcm_dpo/q_t": 0.43348538875579834, + "grad_norm": 14.566923141479492, + "learning_rate": 1.0823503403430734e-07, + "logits/chosen": 0.7113995552062988, + "logits/rejected": 0.661020815372467, + "logps/chosen": -186.58673095703125, + "logps/ref_chosen": -58.91331481933594, + "logps/ref_rejected": -63.7403450012207, + "logps/rejected": -235.3098907470703, + "loss": 1.2029, + "margin_dpo/margin_mean": 43.89613342285156, + "margin_dpo/margin_std": 90.19206237792969, + "step": 479 + }, + { + "KL/chosen_KL_mean": -127.95087432861328, + "KL/mean": -161.46563720703125, + "KL/rejected_KL_mean": -194.98040771484375, + "KL/std": 79.78359985351562, + "epoch": 0.7256235827664399, + "fcm_dpo/beta": 0.006699123419821262, + "fcm_dpo/delta": -0.05192215368151665, + "fcm_dpo/margin": 67.02953338623047, + "fcm_dpo/q_t": 0.39704233407974243, + "grad_norm": 15.194873809814453, + "learning_rate": 1.0714794091391072e-07, + "logits/chosen": 0.6825853586196899, + "logits/rejected": 0.6737313270568848, + "logps/chosen": -190.75149536132812, + "logps/ref_chosen": -62.80061340332031, + "logps/ref_rejected": -67.58859252929688, + "logps/rejected": -262.5690002441406, + "loss": 1.084, + "margin_dpo/margin_mean": 67.02952575683594, + "margin_dpo/margin_std": 93.52059173583984, + "step": 480 + }, + { + "KL/chosen_KL_mean": -125.33125305175781, + "KL/mean": -154.91146850585938, + "KL/rejected_KL_mean": -184.49171447753906, + "KL/std": 81.761474609375, + "epoch": 0.72713529856387, + "fcm_dpo/beta": 0.006634542718529701, + "fcm_dpo/delta": 0.007773694582283497, + "fcm_dpo/margin": 59.16046142578125, + "fcm_dpo/q_t": 0.41102468967437744, + "grad_norm": 14.069964408874512, + "learning_rate": 1.0606484367268906e-07, + "logits/chosen": 0.6635995507240295, + "logits/rejected": 0.6619523167610168, + "logps/chosen": -190.61773681640625, + "logps/ref_chosen": -65.28649139404297, + "logps/ref_rejected": -70.78668212890625, + "logps/rejected": -255.2783966064453, + "loss": 1.1262, + "margin_dpo/margin_mean": 59.16046142578125, + "margin_dpo/margin_std": 95.72927856445312, + "step": 481 + }, + { + "KL/chosen_KL_mean": -140.9495391845703, + "KL/mean": -169.96041870117188, + "KL/rejected_KL_mean": -198.9713134765625, + "KL/std": 82.55261993408203, + "epoch": 0.7286470143613001, + "fcm_dpo/beta": 0.0066745951771736145, + "fcm_dpo/delta": 0.01310792751610279, + "fcm_dpo/margin": 58.02178192138672, + "fcm_dpo/q_t": 0.41360071301460266, + "grad_norm": 14.600235939025879, + "learning_rate": 1.0498577260720048e-07, + "logits/chosen": 0.6595815420150757, + "logits/rejected": 0.5047454833984375, + "logps/chosen": -201.855712890625, + "logps/ref_chosen": -60.906185150146484, + "logps/ref_rejected": -103.44656372070312, + "logps/rejected": -302.4178771972656, + "loss": 1.1531, + "margin_dpo/margin_mean": 58.02178192138672, + "margin_dpo/margin_std": 106.09500122070312, + "step": 482 + }, + { + "KL/chosen_KL_mean": -120.04824829101562, + "KL/mean": -156.00111389160156, + "KL/rejected_KL_mean": -191.9539794921875, + "KL/std": 79.58131408691406, + "epoch": 0.7301587301587301, + "fcm_dpo/beta": 0.006596090272068977, + "fcm_dpo/delta": -0.07792834937572479, + "fcm_dpo/margin": 71.90570831298828, + "fcm_dpo/q_t": 0.3909834921360016, + "grad_norm": 12.167679786682129, + "learning_rate": 1.0391075790138232e-07, + "logits/chosen": 0.8162363767623901, + "logits/rejected": 0.7045830488204956, + "logps/chosen": -173.24026489257812, + "logps/ref_chosen": -53.192012786865234, + "logps/ref_rejected": -81.83927154541016, + "logps/rejected": -273.7932434082031, + "loss": 1.0491, + "margin_dpo/margin_mean": 71.90570831298828, + "margin_dpo/margin_std": 89.77267456054688, + "step": 483 + }, + { + "KL/chosen_KL_mean": -122.33198547363281, + "KL/mean": -147.74710083007812, + "KL/rejected_KL_mean": -173.16220092773438, + "KL/std": 76.41363525390625, + "epoch": 0.7316704459561603, + "fcm_dpo/beta": 0.006655214354395866, + "fcm_dpo/delta": 0.06308972835540771, + "fcm_dpo/margin": 50.830196380615234, + "fcm_dpo/q_t": 0.42113012075424194, + "grad_norm": 17.223285675048828, + "learning_rate": 1.0283982962570681e-07, + "logits/chosen": 0.7866549491882324, + "logits/rejected": 0.7504280805587769, + "logps/chosen": -180.1014404296875, + "logps/ref_chosen": -57.76945877075195, + "logps/ref_rejected": -71.6829833984375, + "logps/rejected": -244.84518432617188, + "loss": 1.1308, + "margin_dpo/margin_mean": 50.830196380615234, + "margin_dpo/margin_std": 70.53138732910156, + "step": 484 + }, + { + "KL/chosen_KL_mean": -128.30880737304688, + "KL/mean": -154.93826293945312, + "KL/rejected_KL_mean": -181.56771850585938, + "KL/std": 81.91331481933594, + "epoch": 0.7331821617535903, + "fcm_dpo/beta": 0.0066419942304492, + "fcm_dpo/delta": 0.047177840024232864, + "fcm_dpo/margin": 53.25891876220703, + "fcm_dpo/q_t": 0.41883209347724915, + "grad_norm": 14.847817420959473, + "learning_rate": 1.0177301773633992e-07, + "logits/chosen": 0.7805214524269104, + "logits/rejected": 0.7562474012374878, + "logps/chosen": -184.94467163085938, + "logps/ref_chosen": -56.63584899902344, + "logps/ref_rejected": -70.85614013671875, + "logps/rejected": -252.42385864257812, + "loss": 1.1349, + "margin_dpo/margin_mean": 53.258914947509766, + "margin_dpo/margin_std": 79.47978210449219, + "step": 485 + }, + { + "KL/chosen_KL_mean": -146.35873413085938, + "KL/mean": -170.71510314941406, + "KL/rejected_KL_mean": -195.07147216796875, + "KL/std": 87.13774871826172, + "epoch": 0.7346938775510204, + "fcm_dpo/beta": 0.006780010648071766, + "fcm_dpo/delta": 0.07210691273212433, + "fcm_dpo/margin": 48.712730407714844, + "fcm_dpo/q_t": 0.42639386653900146, + "grad_norm": 13.277361869812012, + "learning_rate": 1.007103520743035e-07, + "logits/chosen": 0.7420072555541992, + "logits/rejected": 0.6164480447769165, + "logps/chosen": -202.70574951171875, + "logps/ref_chosen": -56.347023010253906, + "logps/ref_rejected": -85.97221374511719, + "logps/rejected": -281.043701171875, + "loss": 1.1858, + "margin_dpo/margin_mean": 48.712730407714844, + "margin_dpo/margin_std": 98.05647277832031, + "step": 486 + }, + { + "KL/chosen_KL_mean": -130.673095703125, + "KL/mean": -160.9501953125, + "KL/rejected_KL_mean": -191.227294921875, + "KL/std": 78.9404296875, + "epoch": 0.7362055933484505, + "fcm_dpo/beta": 0.006787487305700779, + "fcm_dpo/delta": -0.011505922302603722, + "fcm_dpo/margin": 60.55419921875, + "fcm_dpo/q_t": 0.4070599675178528, + "grad_norm": 16.90460205078125, + "learning_rate": 9.965186236464046e-08, + "logits/chosen": 0.7969303727149963, + "logits/rejected": 0.7338086366653442, + "logps/chosen": -191.29031372070312, + "logps/ref_chosen": -60.617218017578125, + "logps/ref_rejected": -82.50975036621094, + "logps/rejected": -273.7370300292969, + "loss": 1.1044, + "margin_dpo/margin_mean": 60.55419921875, + "margin_dpo/margin_std": 90.86151123046875, + "step": 487 + }, + { + "KL/chosen_KL_mean": -124.23251342773438, + "KL/mean": -156.62188720703125, + "KL/rejected_KL_mean": -189.01123046875, + "KL/std": 79.33134460449219, + "epoch": 0.7377173091458806, + "fcm_dpo/beta": 0.0067710187286138535, + "fcm_dpo/delta": -0.04048357903957367, + "fcm_dpo/margin": 64.77871704101562, + "fcm_dpo/q_t": 0.4005877673625946, + "grad_norm": 17.137691497802734, + "learning_rate": 9.859757821558337e-08, + "logits/chosen": 0.7359976768493652, + "logits/rejected": 0.669157862663269, + "logps/chosen": -187.34156799316406, + "logps/ref_chosen": -63.10905075073242, + "logps/ref_rejected": -82.49348449707031, + "logps/rejected": -271.50469970703125, + "loss": 1.0807, + "margin_dpo/margin_mean": 64.77871704101562, + "margin_dpo/margin_std": 89.09386444091797, + "step": 488 + }, + { + "KL/chosen_KL_mean": -142.72219848632812, + "KL/mean": -163.11502075195312, + "KL/rejected_KL_mean": -183.50782775878906, + "KL/std": 85.70088958740234, + "epoch": 0.7392290249433107, + "fcm_dpo/beta": 0.0068509941920638084, + "fcm_dpo/delta": 0.12417726218700409, + "fcm_dpo/margin": 40.78562927246094, + "fcm_dpo/q_t": 0.4372069239616394, + "grad_norm": 12.837416648864746, + "learning_rate": 9.754752911772615e-08, + "logits/chosen": 0.7282401323318481, + "logits/rejected": 0.6781407594680786, + "logps/chosen": -207.71116638183594, + "logps/ref_chosen": -64.98896026611328, + "logps/ref_rejected": -84.39607238769531, + "logps/rejected": -267.9039001464844, + "loss": 1.2316, + "margin_dpo/margin_mean": 40.78562927246094, + "margin_dpo/margin_std": 97.32069396972656, + "step": 489 + }, + { + "KL/chosen_KL_mean": -127.41735076904297, + "KL/mean": -152.37875366210938, + "KL/rejected_KL_mean": -177.34014892578125, + "KL/std": 86.41929626464844, + "epoch": 0.7407407407407407, + "fcm_dpo/beta": 0.00692601315677166, + "fcm_dpo/delta": 0.055926259607076645, + "fcm_dpo/margin": 49.92280578613281, + "fcm_dpo/q_t": 0.4228578209877014, + "grad_norm": 12.514721870422363, + "learning_rate": 9.650174444319956e-08, + "logits/chosen": 0.8083858489990234, + "logits/rejected": 0.7824463844299316, + "logps/chosen": -189.3260955810547, + "logps/ref_chosen": -61.90874481201172, + "logps/ref_rejected": -70.58566284179688, + "logps/rejected": -247.92581176757812, + "loss": 1.2091, + "margin_dpo/margin_mean": 49.92280578613281, + "margin_dpo/margin_std": 110.34571838378906, + "step": 490 + }, + { + "KL/chosen_KL_mean": -123.9417724609375, + "KL/mean": -152.5548095703125, + "KL/rejected_KL_mean": -181.16781616210938, + "KL/std": 81.78946685791016, + "epoch": 0.7422524565381708, + "fcm_dpo/beta": 0.006939752493053675, + "fcm_dpo/delta": 0.0021833814680576324, + "fcm_dpo/margin": 57.226051330566406, + "fcm_dpo/q_t": 0.409621000289917, + "grad_norm": 13.716757774353027, + "learning_rate": 9.546025344484868e-08, + "logits/chosen": 0.6482164859771729, + "logits/rejected": 0.5923037528991699, + "logps/chosen": -179.41746520996094, + "logps/ref_chosen": -55.47570037841797, + "logps/ref_rejected": -78.70318603515625, + "logps/rejected": -259.8710021972656, + "loss": 1.1124, + "margin_dpo/margin_mean": 57.226051330566406, + "margin_dpo/margin_std": 84.64307403564453, + "step": 491 + }, + { + "KL/chosen_KL_mean": -147.8551025390625, + "KL/mean": -169.573486328125, + "KL/rejected_KL_mean": -191.29185485839844, + "KL/std": 85.98609924316406, + "epoch": 0.7437641723356009, + "fcm_dpo/beta": 0.00711585208773613, + "fcm_dpo/delta": 0.09289870411157608, + "fcm_dpo/margin": 43.43674850463867, + "fcm_dpo/q_t": 0.43024128675460815, + "grad_norm": 16.301864624023438, + "learning_rate": 9.442308525541589e-08, + "logits/chosen": 0.7306917309761047, + "logits/rejected": 0.6532546281814575, + "logps/chosen": -215.1414794921875, + "logps/ref_chosen": -67.28638458251953, + "logps/ref_rejected": -82.78628540039062, + "logps/rejected": -274.0781555175781, + "loss": 1.2146, + "margin_dpo/margin_mean": 43.43675231933594, + "margin_dpo/margin_std": 95.23860931396484, + "step": 492 + }, + { + "KL/chosen_KL_mean": -120.97908020019531, + "KL/mean": -154.72320556640625, + "KL/rejected_KL_mean": -188.4673309326172, + "KL/std": 84.16322326660156, + "epoch": 0.745275888133031, + "fcm_dpo/beta": 0.007075267843902111, + "fcm_dpo/delta": -0.08170966804027557, + "fcm_dpo/margin": 67.48824310302734, + "fcm_dpo/q_t": 0.39180272817611694, + "grad_norm": 13.524187088012695, + "learning_rate": 9.339026888672468e-08, + "logits/chosen": 0.6676149368286133, + "logits/rejected": 0.5849226117134094, + "logps/chosen": -176.90658569335938, + "logps/ref_chosen": -55.92750549316406, + "logps/ref_rejected": -79.12149810791016, + "logps/rejected": -267.5888366699219, + "loss": 1.0693, + "margin_dpo/margin_mean": 67.48824310302734, + "margin_dpo/margin_std": 92.7247085571289, + "step": 493 + }, + { + "KL/chosen_KL_mean": -126.17925262451172, + "KL/mean": -154.39076232910156, + "KL/rejected_KL_mean": -182.602294921875, + "KL/std": 85.36856079101562, + "epoch": 0.7467876039304611, + "fcm_dpo/beta": 0.006994927302002907, + "fcm_dpo/delta": 0.005503425374627113, + "fcm_dpo/margin": 56.423038482666016, + "fcm_dpo/q_t": 0.4114704728126526, + "grad_norm": 16.0062198638916, + "learning_rate": 9.236183322886945e-08, + "logits/chosen": 0.6324170827865601, + "logits/rejected": 0.5757017135620117, + "logps/chosen": -194.13336181640625, + "logps/ref_chosen": -67.95410919189453, + "logps/ref_rejected": -90.50865173339844, + "logps/rejected": -273.1109619140625, + "loss": 1.1543, + "margin_dpo/margin_mean": 56.423038482666016, + "margin_dpo/margin_std": 104.7546615600586, + "step": 494 + }, + { + "KL/chosen_KL_mean": -123.937744140625, + "KL/mean": -147.29507446289062, + "KL/rejected_KL_mean": -170.6524200439453, + "KL/std": 83.14198303222656, + "epoch": 0.7482993197278912, + "fcm_dpo/beta": 0.00710693746805191, + "fcm_dpo/delta": 0.0698116272687912, + "fcm_dpo/margin": 46.71466064453125, + "fcm_dpo/q_t": 0.4281880259513855, + "grad_norm": 17.973764419555664, + "learning_rate": 9.133780704940594e-08, + "logits/chosen": 0.8091416358947754, + "logits/rejected": 0.7367887496948242, + "logps/chosen": -176.56320190429688, + "logps/ref_chosen": -52.62546157836914, + "logps/ref_rejected": -72.06781005859375, + "logps/rejected": -242.72021484375, + "loss": 1.1976, + "margin_dpo/margin_mean": 46.71466064453125, + "margin_dpo/margin_std": 99.67835998535156, + "step": 495 + }, + { + "KL/chosen_KL_mean": -138.32159423828125, + "KL/mean": -166.4814453125, + "KL/rejected_KL_mean": -194.64129638671875, + "KL/std": 85.53994750976562, + "epoch": 0.7498110355253212, + "fcm_dpo/beta": 0.007061575539410114, + "fcm_dpo/delta": 0.0015222802758216858, + "fcm_dpo/margin": 56.319705963134766, + "fcm_dpo/q_t": 0.41440892219543457, + "grad_norm": 14.250332832336426, + "learning_rate": 9.031821899254797e-08, + "logits/chosen": 0.724806547164917, + "logits/rejected": 0.606552004814148, + "logps/chosen": -195.91891479492188, + "logps/ref_chosen": -57.597320556640625, + "logps/ref_rejected": -94.36127471923828, + "logps/rejected": -289.0025634765625, + "loss": 1.1541, + "margin_dpo/margin_mean": 56.319705963134766, + "margin_dpo/margin_std": 104.30931091308594, + "step": 496 + }, + { + "KL/chosen_KL_mean": -136.56988525390625, + "KL/mean": -170.1834716796875, + "KL/rejected_KL_mean": -203.79702758789062, + "KL/std": 85.25959014892578, + "epoch": 0.7513227513227513, + "fcm_dpo/beta": 0.007018047384917736, + "fcm_dpo/delta": -0.07552605867385864, + "fcm_dpo/margin": 67.22714233398438, + "fcm_dpo/q_t": 0.3920058012008667, + "grad_norm": 11.70369815826416, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": 0.7534537315368652, + "logits/rejected": 0.7208569049835205, + "logps/chosen": -209.35983276367188, + "logps/ref_chosen": -72.78994750976562, + "logps/ref_rejected": -89.48483276367188, + "logps/rejected": -293.2818603515625, + "loss": 1.0679, + "margin_dpo/margin_mean": 67.22714233398438, + "margin_dpo/margin_std": 92.52470397949219, + "step": 497 + }, + { + "KL/chosen_KL_mean": -123.65620422363281, + "KL/mean": -155.9206085205078, + "KL/rejected_KL_mean": -188.1850128173828, + "KL/std": 81.950439453125, + "epoch": 0.7528344671201814, + "fcm_dpo/beta": 0.006946182809770107, + "fcm_dpo/delta": -0.05051981657743454, + "fcm_dpo/margin": 64.52880859375, + "fcm_dpo/q_t": 0.3971995711326599, + "grad_norm": 16.324182510375977, + "learning_rate": 8.829247120198563e-08, + "logits/chosen": 0.7444478869438171, + "logits/rejected": 0.7110167145729065, + "logps/chosen": -192.02191162109375, + "logps/ref_chosen": -68.36572265625, + "logps/ref_rejected": -71.28846740722656, + "logps/rejected": -259.4734802246094, + "loss": 1.069, + "margin_dpo/margin_mean": 64.52880096435547, + "margin_dpo/margin_std": 85.4512939453125, + "step": 498 + }, + { + "KL/chosen_KL_mean": -121.36831665039062, + "KL/mean": -152.87835693359375, + "KL/rejected_KL_mean": -184.38839721679688, + "KL/std": 83.8720474243164, + "epoch": 0.7543461829176115, + "fcm_dpo/beta": 0.006907904986292124, + "fcm_dpo/delta": -0.03695734590291977, + "fcm_dpo/margin": 63.02008819580078, + "fcm_dpo/q_t": 0.40249934792518616, + "grad_norm": 14.695865631103516, + "learning_rate": 8.728636813280163e-08, + "logits/chosen": 0.7439185380935669, + "logits/rejected": 0.6724633574485779, + "logps/chosen": -183.2771453857422, + "logps/ref_chosen": -61.90882873535156, + "logps/ref_rejected": -91.9411392211914, + "logps/rejected": -276.32952880859375, + "loss": 1.1306, + "margin_dpo/margin_mean": 63.02008819580078, + "margin_dpo/margin_std": 108.50146484375, + "step": 499 + }, + { + "KL/chosen_KL_mean": -128.1096954345703, + "KL/mean": -157.46087646484375, + "KL/rejected_KL_mean": -186.81202697753906, + "KL/std": 79.26026916503906, + "epoch": 0.7558578987150416, + "fcm_dpo/beta": 0.00685786921530962, + "fcm_dpo/delta": -0.0028558000922203064, + "fcm_dpo/margin": 58.70232391357422, + "fcm_dpo/q_t": 0.40761202573776245, + "grad_norm": 16.214675903320312, + "learning_rate": 8.628481651367875e-08, + "logits/chosen": 0.6643021702766418, + "logits/rejected": 0.6609218120574951, + "logps/chosen": -198.3355255126953, + "logps/ref_chosen": -70.225830078125, + "logps/ref_rejected": -71.72203063964844, + "logps/rejected": -258.5340576171875, + "loss": 1.1489, + "margin_dpo/margin_mean": 58.70232391357422, + "margin_dpo/margin_std": 106.12753295898438, + "step": 500 + }, + { + "KL/chosen_KL_mean": -128.66282653808594, + "KL/mean": -154.97549438476562, + "KL/rejected_KL_mean": -181.28819274902344, + "KL/std": 83.97608947753906, + "epoch": 0.7573696145124716, + "fcm_dpo/beta": 0.006935385055840015, + "fcm_dpo/delta": 0.03603249043226242, + "fcm_dpo/margin": 52.62535858154297, + "fcm_dpo/q_t": 0.41577181220054626, + "grad_norm": 13.582621574401855, + "learning_rate": 8.528784436016878e-08, + "logits/chosen": 0.7301015853881836, + "logits/rejected": 0.7303687334060669, + "logps/chosen": -193.26162719726562, + "logps/ref_chosen": -64.59880828857422, + "logps/ref_rejected": -70.59329223632812, + "logps/rejected": -251.8814697265625, + "loss": 1.1147, + "margin_dpo/margin_mean": 52.6253547668457, + "margin_dpo/margin_std": 71.9767074584961, + "step": 501 + }, + { + "KL/chosen_KL_mean": -126.94548034667969, + "KL/mean": -154.24295043945312, + "KL/rejected_KL_mean": -181.54042053222656, + "KL/std": 85.87179565429688, + "epoch": 0.7588813303099018, + "fcm_dpo/beta": 0.0069640884175896645, + "fcm_dpo/delta": 0.02041056379675865, + "fcm_dpo/margin": 54.59494400024414, + "fcm_dpo/q_t": 0.4131236672401428, + "grad_norm": 14.177937507629395, + "learning_rate": 8.4295479559726e-08, + "logits/chosen": 0.7359659075737, + "logits/rejected": 0.68549644947052, + "logps/chosen": -192.412109375, + "logps/ref_chosen": -65.46662902832031, + "logps/ref_rejected": -90.22233581542969, + "logps/rejected": -271.76275634765625, + "loss": 1.1209, + "margin_dpo/margin_mean": 54.59494400024414, + "margin_dpo/margin_std": 83.03755187988281, + "step": 502 + }, + { + "KL/chosen_KL_mean": -117.69458770751953, + "KL/mean": -147.38555908203125, + "KL/rejected_KL_mean": -177.07650756835938, + "KL/std": 81.83952331542969, + "epoch": 0.7603930461073318, + "fcm_dpo/beta": 0.006930957548320293, + "fcm_dpo/delta": -0.012141318060457706, + "fcm_dpo/margin": 59.381935119628906, + "fcm_dpo/q_t": 0.4062880873680115, + "grad_norm": 12.284950256347656, + "learning_rate": 8.330774987092712e-08, + "logits/chosen": 0.7286855578422546, + "logits/rejected": 0.7282934188842773, + "logps/chosen": -169.52935791015625, + "logps/ref_chosen": -51.83476257324219, + "logps/ref_rejected": -57.62522506713867, + "logps/rejected": -234.70175170898438, + "loss": 1.117, + "margin_dpo/margin_mean": 59.381935119628906, + "margin_dpo/margin_std": 93.7178726196289, + "step": 503 + }, + { + "KL/chosen_KL_mean": -119.32576751708984, + "KL/mean": -157.84042358398438, + "KL/rejected_KL_mean": -196.35508728027344, + "KL/std": 80.00068664550781, + "epoch": 0.7619047619047619, + "fcm_dpo/beta": 0.006835811771452427, + "fcm_dpo/delta": -0.13343694806098938, + "fcm_dpo/margin": 77.02932739257812, + "fcm_dpo/q_t": 0.3784865736961365, + "grad_norm": 18.003236770629883, + "learning_rate": 8.232468292269479e-08, + "logits/chosen": 0.7067838311195374, + "logits/rejected": 0.6834101676940918, + "logps/chosen": -187.97695922851562, + "logps/ref_chosen": -68.65119934082031, + "logps/ref_rejected": -77.91394805908203, + "logps/rejected": -274.26904296875, + "loss": 1.0051, + "margin_dpo/margin_mean": 77.0293197631836, + "margin_dpo/margin_std": 82.89633178710938, + "step": 504 + }, + { + "KL/chosen_KL_mean": -125.51513671875, + "KL/mean": -150.27340698242188, + "KL/rejected_KL_mean": -175.0316925048828, + "KL/std": 87.24829864501953, + "epoch": 0.763416477702192, + "fcm_dpo/beta": 0.006775484420359135, + "fcm_dpo/delta": 0.0662289410829544, + "fcm_dpo/margin": 49.51654815673828, + "fcm_dpo/q_t": 0.4252125322818756, + "grad_norm": 13.68161678314209, + "learning_rate": 8.134630621352483e-08, + "logits/chosen": 0.729232907295227, + "logits/rejected": 0.6899206638336182, + "logps/chosen": -185.51397705078125, + "logps/ref_chosen": -59.99884796142578, + "logps/ref_rejected": -76.88048553466797, + "logps/rejected": -251.91217041015625, + "loss": 1.1936, + "margin_dpo/margin_mean": 49.51654815673828, + "margin_dpo/margin_std": 100.8291244506836, + "step": 505 + }, + { + "KL/chosen_KL_mean": -121.28506469726562, + "KL/mean": -148.55630493164062, + "KL/rejected_KL_mean": -175.82757568359375, + "KL/std": 79.10469818115234, + "epoch": 0.764928193499622, + "fcm_dpo/beta": 0.006854447070509195, + "fcm_dpo/delta": 0.02708333171904087, + "fcm_dpo/margin": 54.54250717163086, + "fcm_dpo/q_t": 0.41415175795555115, + "grad_norm": 15.952156066894531, + "learning_rate": 8.037264711071698e-08, + "logits/chosen": 0.72569739818573, + "logits/rejected": 0.707362711429596, + "logps/chosen": -191.35635375976562, + "logps/ref_chosen": -70.07130432128906, + "logps/ref_rejected": -82.03775024414062, + "logps/rejected": -257.8653259277344, + "loss": 1.158, + "margin_dpo/margin_mean": 54.542503356933594, + "margin_dpo/margin_std": 99.5510025024414, + "step": 506 + }, + { + "KL/chosen_KL_mean": -134.30670166015625, + "KL/mean": -163.48135375976562, + "KL/rejected_KL_mean": -192.656005859375, + "KL/std": 85.48699951171875, + "epoch": 0.7664399092970522, + "fcm_dpo/beta": 0.006860400550067425, + "fcm_dpo/delta": -0.000510435551404953, + "fcm_dpo/margin": 58.34930419921875, + "fcm_dpo/q_t": 0.4124026894569397, + "grad_norm": 13.52925968170166, + "learning_rate": 7.940373284960933e-08, + "logits/chosen": 0.717583954334259, + "logits/rejected": 0.6637930870056152, + "logps/chosen": -206.31373596191406, + "logps/ref_chosen": -72.00703430175781, + "logps/ref_rejected": -93.94987487792969, + "logps/rejected": -286.60589599609375, + "loss": 1.1391, + "margin_dpo/margin_mean": 58.349308013916016, + "margin_dpo/margin_std": 101.02263641357422, + "step": 507 + }, + { + "KL/chosen_KL_mean": -122.50996398925781, + "KL/mean": -154.63719177246094, + "KL/rejected_KL_mean": -186.76441955566406, + "KL/std": 86.9670181274414, + "epoch": 0.7679516250944822, + "fcm_dpo/beta": 0.006876353174448013, + "fcm_dpo/delta": -0.04420164227485657, + "fcm_dpo/margin": 64.25446319580078, + "fcm_dpo/q_t": 0.4004051089286804, + "grad_norm": 15.83190631866455, + "learning_rate": 7.843959053281663e-08, + "logits/chosen": 0.6986652612686157, + "logits/rejected": 0.5631051659584045, + "logps/chosen": -182.72988891601562, + "logps/ref_chosen": -60.21992492675781, + "logps/ref_rejected": -95.9200668334961, + "logps/rejected": -282.6844787597656, + "loss": 1.0958, + "margin_dpo/margin_mean": 64.25446319580078, + "margin_dpo/margin_std": 95.26860809326172, + "step": 508 + }, + { + "KL/chosen_KL_mean": -127.48722839355469, + "KL/mean": -155.79217529296875, + "KL/rejected_KL_mean": -184.09713745117188, + "KL/std": 81.48516845703125, + "epoch": 0.7694633408919124, + "fcm_dpo/beta": 0.006818421185016632, + "fcm_dpo/delta": 0.014477929100394249, + "fcm_dpo/margin": 56.60991287231445, + "fcm_dpo/q_t": 0.41192716360092163, + "grad_norm": 16.497425079345703, + "learning_rate": 7.748024712947204e-08, + "logits/chosen": 0.6621476411819458, + "logits/rejected": 0.639892578125, + "logps/chosen": -193.7574005126953, + "logps/ref_chosen": -66.27017211914062, + "logps/ref_rejected": -71.73065185546875, + "logps/rejected": -255.82778930664062, + "loss": 1.1274, + "margin_dpo/margin_mean": 56.60991287231445, + "margin_dpo/margin_std": 90.43913269042969, + "step": 509 + }, + { + "KL/chosen_KL_mean": -127.44882202148438, + "KL/mean": -160.54481506347656, + "KL/rejected_KL_mean": -193.64077758789062, + "KL/std": 85.18399047851562, + "epoch": 0.7709750566893424, + "fcm_dpo/beta": 0.006756870076060295, + "fcm_dpo/delta": -0.05019165575504303, + "fcm_dpo/margin": 66.19197845458984, + "fcm_dpo/q_t": 0.40029990673065186, + "grad_norm": 14.753068923950195, + "learning_rate": 7.652572947447272e-08, + "logits/chosen": 0.8324567675590515, + "logits/rejected": 0.7259620428085327, + "logps/chosen": -180.9936981201172, + "logps/ref_chosen": -53.54487609863281, + "logps/ref_rejected": -91.36648559570312, + "logps/rejected": -285.00726318359375, + "loss": 1.1059, + "margin_dpo/margin_mean": 66.19198608398438, + "margin_dpo/margin_std": 104.75790405273438, + "step": 510 + }, + { + "KL/chosen_KL_mean": -122.54195404052734, + "KL/mean": -161.23318481445312, + "KL/rejected_KL_mean": -199.9244384765625, + "KL/std": 82.09759521484375, + "epoch": 0.7724867724867724, + "fcm_dpo/beta": 0.00666405912488699, + "fcm_dpo/delta": -0.12181182205677032, + "fcm_dpo/margin": 77.38249206542969, + "fcm_dpo/q_t": 0.3824625611305237, + "grad_norm": 16.800376892089844, + "learning_rate": 7.557606426772961e-08, + "logits/chosen": 0.7611442804336548, + "logits/rejected": 0.6997200846672058, + "logps/chosen": -178.38633728027344, + "logps/ref_chosen": -55.844383239746094, + "logps/ref_rejected": -86.49819946289062, + "logps/rejected": -286.4226379394531, + "loss": 1.0323, + "margin_dpo/margin_mean": 77.38249206542969, + "margin_dpo/margin_std": 95.76220703125, + "step": 511 + }, + { + "KL/chosen_KL_mean": -123.5075454711914, + "KL/mean": -149.7601318359375, + "KL/rejected_KL_mean": -176.01272583007812, + "KL/std": 77.66827392578125, + "epoch": 0.7739984882842026, + "fcm_dpo/beta": 0.006653419695794582, + "fcm_dpo/delta": 0.0525285080075264, + "fcm_dpo/margin": 52.505191802978516, + "fcm_dpo/q_t": 0.42007583379745483, + "grad_norm": 18.692623138427734, + "learning_rate": 7.463127807341966e-08, + "logits/chosen": 0.6151683926582336, + "logits/rejected": 0.6060619354248047, + "logps/chosen": -185.16058349609375, + "logps/ref_chosen": -61.653038024902344, + "logps/ref_rejected": -72.83148193359375, + "logps/rejected": -248.84420776367188, + "loss": 1.168, + "margin_dpo/margin_mean": 52.50518798828125, + "margin_dpo/margin_std": 97.72168731689453, + "step": 512 + }, + { + "KL/chosen_KL_mean": -110.83531188964844, + "KL/mean": -143.1207275390625, + "KL/rejected_KL_mean": -175.4061279296875, + "KL/std": 80.16166687011719, + "epoch": 0.7755102040816326, + "fcm_dpo/beta": 0.00662595871835947, + "fcm_dpo/delta": -0.02946261130273342, + "fcm_dpo/margin": 64.57081604003906, + "fcm_dpo/q_t": 0.40187883377075195, + "grad_norm": 12.440008163452148, + "learning_rate": 7.369139731924401e-08, + "logits/chosen": 0.842035174369812, + "logits/rejected": 0.7855624556541443, + "logps/chosen": -161.68788146972656, + "logps/ref_chosen": -50.85256576538086, + "logps/ref_rejected": -69.21754455566406, + "logps/rejected": -244.62367248535156, + "loss": 1.0747, + "margin_dpo/margin_mean": 64.57081604003906, + "margin_dpo/margin_std": 82.73863220214844, + "step": 513 + }, + { + "KL/chosen_KL_mean": -125.44157409667969, + "KL/mean": -161.4381561279297, + "KL/rejected_KL_mean": -197.43472290039062, + "KL/std": 83.99867248535156, + "epoch": 0.7770219198790628, + "fcm_dpo/beta": 0.006588710471987724, + "fcm_dpo/delta": -0.07804595679044724, + "fcm_dpo/margin": 71.99317169189453, + "fcm_dpo/q_t": 0.391022652387619, + "grad_norm": 14.347710609436035, + "learning_rate": 7.275644829568747e-08, + "logits/chosen": 0.7848989963531494, + "logits/rejected": 0.7469111680984497, + "logps/chosen": -194.82650756835938, + "logps/ref_chosen": -69.38493347167969, + "logps/ref_rejected": -83.32447814941406, + "logps/rejected": -280.75921630859375, + "loss": 1.0615, + "margin_dpo/margin_mean": 71.9931640625, + "margin_dpo/margin_std": 95.76860046386719, + "step": 514 + }, + { + "KL/chosen_KL_mean": -134.65899658203125, + "KL/mean": -163.10086059570312, + "KL/rejected_KL_mean": -191.542724609375, + "KL/std": 82.38731384277344, + "epoch": 0.7785336356764928, + "fcm_dpo/beta": 0.006560333073139191, + "fcm_dpo/delta": 0.027873113751411438, + "fcm_dpo/margin": 56.88371276855469, + "fcm_dpo/q_t": 0.4149225950241089, + "grad_norm": 17.680585861206055, + "learning_rate": 7.182645715528435e-08, + "logits/chosen": 0.7651482820510864, + "logits/rejected": 0.6802647113800049, + "logps/chosen": -188.34603881835938, + "logps/ref_chosen": -53.687034606933594, + "logps/ref_rejected": -83.59614562988281, + "logps/rejected": -275.13885498046875, + "loss": 1.1414, + "margin_dpo/margin_mean": 56.88371276855469, + "margin_dpo/margin_std": 96.1820297241211, + "step": 515 + }, + { + "KL/chosen_KL_mean": -115.13967895507812, + "KL/mean": -142.64561462402344, + "KL/rejected_KL_mean": -170.15155029296875, + "KL/std": 81.29928588867188, + "epoch": 0.780045351473923, + "fcm_dpo/beta": 0.0066216206178069115, + "fcm_dpo/delta": 0.0369550921022892, + "fcm_dpo/margin": 55.011871337890625, + "fcm_dpo/q_t": 0.41581177711486816, + "grad_norm": 17.865516662597656, + "learning_rate": 7.090144991188568e-08, + "logits/chosen": 0.7032291889190674, + "logits/rejected": 0.6626705527305603, + "logps/chosen": -172.04139709472656, + "logps/ref_chosen": -56.9017219543457, + "logps/ref_rejected": -67.83477783203125, + "logps/rejected": -237.986328125, + "loss": 1.1504, + "margin_dpo/margin_mean": 55.011878967285156, + "margin_dpo/margin_std": 95.83123779296875, + "step": 516 + }, + { + "KL/chosen_KL_mean": -138.56973266601562, + "KL/mean": -158.37887573242188, + "KL/rejected_KL_mean": -178.18801879882812, + "KL/std": 82.17344665527344, + "epoch": 0.781557067271353, + "fcm_dpo/beta": 0.006789041217416525, + "fcm_dpo/delta": 0.13350741565227509, + "fcm_dpo/margin": 39.6182861328125, + "fcm_dpo/q_t": 0.4397648572921753, + "grad_norm": 17.148391723632812, + "learning_rate": 6.998145243993284e-08, + "logits/chosen": 0.7460171580314636, + "logits/rejected": 0.7420037984848022, + "logps/chosen": -200.34487915039062, + "logps/ref_chosen": -61.775142669677734, + "logps/ref_rejected": -62.88270950317383, + "logps/rejected": -241.07073974609375, + "loss": 1.2232, + "margin_dpo/margin_mean": 39.6182861328125, + "margin_dpo/margin_std": 88.61754608154297, + "step": 517 + }, + { + "KL/chosen_KL_mean": -116.14431762695312, + "KL/mean": -144.33358764648438, + "KL/rejected_KL_mean": -172.52285766601562, + "KL/std": 81.81043243408203, + "epoch": 0.783068783068783, + "fcm_dpo/beta": 0.006819076836109161, + "fcm_dpo/delta": 0.016169242560863495, + "fcm_dpo/margin": 56.3785400390625, + "fcm_dpo/q_t": 0.41407328844070435, + "grad_norm": 13.720296859741211, + "learning_rate": 6.906649047373245e-08, + "logits/chosen": 0.7310158014297485, + "logits/rejected": 0.6800619959831238, + "logps/chosen": -178.1695556640625, + "logps/ref_chosen": -62.02523422241211, + "logps/ref_rejected": -79.06085205078125, + "logps/rejected": -251.58370971679688, + "loss": 1.1275, + "margin_dpo/margin_mean": 56.3785400390625, + "margin_dpo/margin_std": 90.98041534423828, + "step": 518 + }, + { + "KL/chosen_KL_mean": -143.06752014160156, + "KL/mean": -160.43930053710938, + "KL/rejected_KL_mean": -177.81106567382812, + "KL/std": 84.53593444824219, + "epoch": 0.7845804988662132, + "fcm_dpo/beta": 0.007009794004261494, + "fcm_dpo/delta": 0.15968723595142365, + "fcm_dpo/margin": 34.74354553222656, + "fcm_dpo/q_t": 0.4461938738822937, + "grad_norm": 23.412137985229492, + "learning_rate": 6.815658960673781e-08, + "logits/chosen": 0.769728422164917, + "logits/rejected": 0.711004376411438, + "logps/chosen": -204.67388916015625, + "logps/ref_chosen": -61.60636901855469, + "logps/ref_rejected": -74.50727844238281, + "logps/rejected": -252.31834411621094, + "loss": 1.3111, + "margin_dpo/margin_mean": 34.74354553222656, + "margin_dpo/margin_std": 114.71300506591797, + "step": 519 + }, + { + "KL/chosen_KL_mean": -130.02651977539062, + "KL/mean": -154.45364379882812, + "KL/rejected_KL_mean": -178.88076782226562, + "KL/std": 83.43859100341797, + "epoch": 0.7860922146636432, + "fcm_dpo/beta": 0.007063240744173527, + "fcm_dpo/delta": 0.05651885271072388, + "fcm_dpo/margin": 48.85424041748047, + "fcm_dpo/q_t": 0.42267489433288574, + "grad_norm": 16.14238166809082, + "learning_rate": 6.725177529083209e-08, + "logits/chosen": 0.8081783056259155, + "logits/rejected": 0.7461931109428406, + "logps/chosen": -192.89996337890625, + "logps/ref_chosen": -62.87343215942383, + "logps/ref_rejected": -76.505615234375, + "logps/rejected": -255.38636779785156, + "loss": 1.1634, + "margin_dpo/margin_mean": 48.85424041748047, + "margin_dpo/margin_std": 87.38202667236328, + "step": 520 + }, + { + "KL/chosen_KL_mean": -128.45904541015625, + "KL/mean": -164.06591796875, + "KL/rejected_KL_mean": -199.67279052734375, + "KL/std": 81.88728332519531, + "epoch": 0.7876039304610734, + "fcm_dpo/beta": 0.0070292167365550995, + "fcm_dpo/delta": -0.1057443767786026, + "fcm_dpo/margin": 71.21371459960938, + "fcm_dpo/q_t": 0.38589245080947876, + "grad_norm": 13.517394065856934, + "learning_rate": 6.63520728356167e-08, + "logits/chosen": 0.5954105257987976, + "logits/rejected": 0.5143781304359436, + "logps/chosen": -192.66574096679688, + "logps/ref_chosen": -64.20668029785156, + "logps/ref_rejected": -92.28083038330078, + "logps/rejected": -291.95361328125, + "loss": 1.042, + "margin_dpo/margin_mean": 71.21371459960938, + "margin_dpo/margin_std": 91.10281372070312, + "step": 521 + }, + { + "KL/chosen_KL_mean": -128.47425842285156, + "KL/mean": -152.20489501953125, + "KL/rejected_KL_mean": -175.93553161621094, + "KL/std": 84.94644165039062, + "epoch": 0.7891156462585034, + "fcm_dpo/beta": 0.007029838860034943, + "fcm_dpo/delta": 0.06868653744459152, + "fcm_dpo/margin": 47.46127700805664, + "fcm_dpo/q_t": 0.4257607161998749, + "grad_norm": 16.1845703125, + "learning_rate": 6.545750740770336e-08, + "logits/chosen": 0.6975493431091309, + "logits/rejected": 0.6909911632537842, + "logps/chosen": -186.84397888183594, + "logps/ref_chosen": -58.369720458984375, + "logps/ref_rejected": -68.79248046875, + "logps/rejected": -244.72801208496094, + "loss": 1.217, + "margin_dpo/margin_mean": 47.461273193359375, + "margin_dpo/margin_std": 108.51187896728516, + "step": 522 + }, + { + "KL/chosen_KL_mean": -132.29612731933594, + "KL/mean": -159.51329040527344, + "KL/rejected_KL_mean": -186.73040771484375, + "KL/std": 82.13084411621094, + "epoch": 0.7906273620559335, + "fcm_dpo/beta": 0.007071667816489935, + "fcm_dpo/delta": 0.015576612204313278, + "fcm_dpo/margin": 54.43429183959961, + "fcm_dpo/q_t": 0.41066181659698486, + "grad_norm": 19.963043212890625, + "learning_rate": 6.456810403001012e-08, + "logits/chosen": 0.748306393623352, + "logits/rejected": 0.616180956363678, + "logps/chosen": -198.00936889648438, + "logps/ref_chosen": -65.71324157714844, + "logps/ref_rejected": -91.98896789550781, + "logps/rejected": -278.7193908691406, + "loss": 1.1578, + "margin_dpo/margin_mean": 54.434295654296875, + "margin_dpo/margin_std": 100.6620101928711, + "step": 523 + }, + { + "KL/chosen_KL_mean": -112.74836730957031, + "KL/mean": -141.45703125, + "KL/rejected_KL_mean": -170.1656951904297, + "KL/std": 79.19841766357422, + "epoch": 0.7921390778533636, + "fcm_dpo/beta": 0.007112534716725349, + "fcm_dpo/delta": -0.009007304906845093, + "fcm_dpo/margin": 57.417335510253906, + "fcm_dpo/q_t": 0.4063315987586975, + "grad_norm": 16.227237701416016, + "learning_rate": 6.368388758106134e-08, + "logits/chosen": 0.6562612056732178, + "logits/rejected": 0.6327396631240845, + "logps/chosen": -189.099609375, + "logps/ref_chosen": -76.35124969482422, + "logps/ref_rejected": -89.96072387695312, + "logps/rejected": -260.1264343261719, + "loss": 1.11, + "margin_dpo/margin_mean": 57.417327880859375, + "margin_dpo/margin_std": 86.96146392822266, + "step": 524 + }, + { + "KL/chosen_KL_mean": -126.07438659667969, + "KL/mean": -150.48638916015625, + "KL/rejected_KL_mean": -174.89840698242188, + "KL/std": 80.91346740722656, + "epoch": 0.7936507936507936, + "fcm_dpo/beta": 0.0071428874507546425, + "fcm_dpo/delta": 0.05306369066238403, + "fcm_dpo/margin": 48.82402420043945, + "fcm_dpo/q_t": 0.42222845554351807, + "grad_norm": 18.715373992919922, + "learning_rate": 6.280488279429185e-08, + "logits/chosen": 0.5620905160903931, + "logits/rejected": 0.5573893785476685, + "logps/chosen": -201.5701904296875, + "logps/ref_chosen": -75.49578857421875, + "logps/ref_rejected": -84.04852294921875, + "logps/rejected": -258.9469299316406, + "loss": 1.1757, + "margin_dpo/margin_mean": 48.82402038574219, + "margin_dpo/margin_std": 95.36903381347656, + "step": 525 + }, + { + "KL/chosen_KL_mean": -137.1304168701172, + "KL/mean": -157.73135375976562, + "KL/rejected_KL_mean": -178.33230590820312, + "KL/std": 81.97193908691406, + "epoch": 0.7951625094482238, + "fcm_dpo/beta": 0.007212967611849308, + "fcm_dpo/delta": 0.1054811105132103, + "fcm_dpo/margin": 41.20189666748047, + "fcm_dpo/q_t": 0.43233275413513184, + "grad_norm": 16.225221633911133, + "learning_rate": 6.193111425735515e-08, + "logits/chosen": 0.7405989170074463, + "logits/rejected": 0.6656317710876465, + "logps/chosen": -198.42282104492188, + "logps/ref_chosen": -61.29241943359375, + "logps/ref_rejected": -82.47763061523438, + "logps/rejected": -260.8099365234375, + "loss": 1.2147, + "margin_dpo/margin_mean": 41.201900482177734, + "margin_dpo/margin_std": 88.14698028564453, + "step": 526 + }, + { + "KL/chosen_KL_mean": -145.92190551757812, + "KL/mean": -163.89610290527344, + "KL/rejected_KL_mean": -181.87030029296875, + "KL/std": 82.55867004394531, + "epoch": 0.7966742252456538, + "fcm_dpo/beta": 0.007491440512239933, + "fcm_dpo/delta": 0.13326705992221832, + "fcm_dpo/margin": 35.948394775390625, + "fcm_dpo/q_t": 0.44097664952278137, + "grad_norm": 16.771879196166992, + "learning_rate": 6.106260641143546e-08, + "logits/chosen": 0.8035439252853394, + "logits/rejected": 0.7174798250198364, + "logps/chosen": -207.39453125, + "logps/ref_chosen": -61.472625732421875, + "logps/ref_rejected": -90.52831268310547, + "logps/rejected": -272.39862060546875, + "loss": 1.2597, + "margin_dpo/margin_mean": 35.948394775390625, + "margin_dpo/margin_std": 96.81865692138672, + "step": 527 + }, + { + "KL/chosen_KL_mean": -129.02345275878906, + "KL/mean": -149.21463012695312, + "KL/rejected_KL_mean": -169.40582275390625, + "KL/std": 82.30186462402344, + "epoch": 0.7981859410430839, + "fcm_dpo/beta": 0.007598328869789839, + "fcm_dpo/delta": 0.0962243527173996, + "fcm_dpo/margin": 40.38237762451172, + "fcm_dpo/q_t": 0.43211063742637634, + "grad_norm": 18.41050148010254, + "learning_rate": 6.019938355056422e-08, + "logits/chosen": 0.642044186592102, + "logits/rejected": 0.5571998357772827, + "logps/chosen": -187.81546020507812, + "logps/ref_chosen": -58.792015075683594, + "logps/ref_rejected": -71.82516479492188, + "logps/rejected": -241.23098754882812, + "loss": 1.248, + "margin_dpo/margin_mean": 40.38237762451172, + "margin_dpo/margin_std": 103.26986694335938, + "step": 528 + }, + { + "KL/chosen_KL_mean": -119.10054779052734, + "KL/mean": -162.97311401367188, + "KL/rejected_KL_mean": -206.845703125, + "KL/std": 82.37154388427734, + "epoch": 0.799697656840514, + "fcm_dpo/beta": 0.007344953715801239, + "fcm_dpo/delta": -0.26204729080200195, + "fcm_dpo/margin": 87.74514770507812, + "fcm_dpo/q_t": 0.3532130718231201, + "grad_norm": 16.302799224853516, + "learning_rate": 5.934146982094049e-08, + "logits/chosen": 0.656600832939148, + "logits/rejected": 0.6004124283790588, + "logps/chosen": -174.1715087890625, + "logps/ref_chosen": -55.070960998535156, + "logps/ref_rejected": -75.44007873535156, + "logps/rejected": -282.2857666015625, + "loss": 0.9417, + "margin_dpo/margin_mean": 87.74514770507812, + "margin_dpo/margin_std": 89.89462280273438, + "step": 529 + }, + { + "KL/chosen_KL_mean": -123.7182846069336, + "KL/mean": -149.55343627929688, + "KL/rejected_KL_mean": -175.3886260986328, + "KL/std": 82.55381774902344, + "epoch": 0.8012093726379441, + "fcm_dpo/beta": 0.007294186390936375, + "fcm_dpo/delta": 0.023981863632798195, + "fcm_dpo/margin": 51.670318603515625, + "fcm_dpo/q_t": 0.41621410846710205, + "grad_norm": 19.178972244262695, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": 0.7491636276245117, + "logits/rejected": 0.7024554014205933, + "logps/chosen": -180.46209716796875, + "logps/ref_chosen": -56.743812561035156, + "logps/ref_rejected": -76.6692123413086, + "logps/rejected": -252.05783081054688, + "loss": 1.1437, + "margin_dpo/margin_mean": 51.670318603515625, + "margin_dpo/margin_std": 88.48190307617188, + "step": 530 + }, + { + "KL/chosen_KL_mean": -124.6670913696289, + "KL/mean": -152.11793518066406, + "KL/rejected_KL_mean": -179.56875610351562, + "KL/std": 83.25489044189453, + "epoch": 0.8027210884353742, + "fcm_dpo/beta": 0.007327442988753319, + "fcm_dpo/delta": -0.0027846992015838623, + "fcm_dpo/margin": 54.901676177978516, + "fcm_dpo/q_t": 0.4097214937210083, + "grad_norm": 15.689215660095215, + "learning_rate": 5.7641665597021435e-08, + "logits/chosen": 0.7318727970123291, + "logits/rejected": 0.6541762351989746, + "logps/chosen": -175.78353881835938, + "logps/ref_chosen": -51.116455078125, + "logps/ref_rejected": -79.52884674072266, + "logps/rejected": -259.09759521484375, + "loss": 1.1231, + "margin_dpo/margin_mean": 54.90167236328125, + "margin_dpo/margin_std": 88.21559143066406, + "step": 531 + }, + { + "KL/chosen_KL_mean": -144.3057403564453, + "KL/mean": -174.24826049804688, + "KL/rejected_KL_mean": -204.19082641601562, + "KL/std": 81.07133483886719, + "epoch": 0.8042328042328042, + "fcm_dpo/beta": 0.007251083850860596, + "fcm_dpo/delta": -0.03580852597951889, + "fcm_dpo/margin": 59.88507080078125, + "fcm_dpo/q_t": 0.4021422564983368, + "grad_norm": 16.10748863220215, + "learning_rate": 5.679982264990424e-08, + "logits/chosen": 0.6791332960128784, + "logits/rejected": 0.6243463754653931, + "logps/chosen": -202.585693359375, + "logps/ref_chosen": -58.279945373535156, + "logps/ref_rejected": -78.05426788330078, + "logps/rejected": -282.2450866699219, + "loss": 1.1035, + "margin_dpo/margin_mean": 59.88507080078125, + "margin_dpo/margin_std": 92.61534118652344, + "step": 532 + }, + { + "KL/chosen_KL_mean": -101.09480285644531, + "KL/mean": -131.0142364501953, + "KL/rejected_KL_mean": -160.9336700439453, + "KL/std": 80.21917724609375, + "epoch": 0.8057445200302343, + "fcm_dpo/beta": 0.007215174846351147, + "fcm_dpo/delta": -0.03317198529839516, + "fcm_dpo/margin": 59.83887481689453, + "fcm_dpo/q_t": 0.4015866816043854, + "grad_norm": 15.353870391845703, + "learning_rate": 5.596338392706076e-08, + "logits/chosen": 0.8840050101280212, + "logits/rejected": 0.8084684610366821, + "logps/chosen": -157.5128173828125, + "logps/ref_chosen": -56.41801071166992, + "logps/ref_rejected": -73.89324951171875, + "logps/rejected": -234.82691955566406, + "loss": 1.0938, + "margin_dpo/margin_mean": 59.83887481689453, + "margin_dpo/margin_std": 86.97126007080078, + "step": 533 + }, + { + "KL/chosen_KL_mean": -125.39263153076172, + "KL/mean": -154.99639892578125, + "KL/rejected_KL_mean": -184.60012817382812, + "KL/std": 83.15888977050781, + "epoch": 0.8072562358276644, + "fcm_dpo/beta": 0.007139452733099461, + "fcm_dpo/delta": -0.024127241224050522, + "fcm_dpo/margin": 59.20750427246094, + "fcm_dpo/q_t": 0.4062623977661133, + "grad_norm": 13.482405662536621, + "learning_rate": 5.513237282548033e-08, + "logits/chosen": 0.7019526958465576, + "logits/rejected": 0.6646519899368286, + "logps/chosen": -186.14132690429688, + "logps/ref_chosen": -60.748687744140625, + "logps/ref_rejected": -73.8623046875, + "logps/rejected": -258.4624328613281, + "loss": 1.1279, + "margin_dpo/margin_mean": 59.20750427246094, + "margin_dpo/margin_std": 100.45853424072266, + "step": 534 + }, + { + "KL/chosen_KL_mean": -137.643798828125, + "KL/mean": -162.00775146484375, + "KL/rejected_KL_mean": -186.37168884277344, + "KL/std": 86.37908935546875, + "epoch": 0.8087679516250945, + "fcm_dpo/beta": 0.007220745086669922, + "fcm_dpo/delta": 0.04981802776455879, + "fcm_dpo/margin": 48.727882385253906, + "fcm_dpo/q_t": 0.4213569462299347, + "grad_norm": 17.222349166870117, + "learning_rate": 5.430681259032957e-08, + "logits/chosen": 0.6391861438751221, + "logits/rejected": 0.5727677345275879, + "logps/chosen": -199.28121948242188, + "logps/ref_chosen": -61.637413024902344, + "logps/ref_rejected": -80.93138885498047, + "logps/rejected": -267.3031005859375, + "loss": 1.1729, + "margin_dpo/margin_mean": 48.727882385253906, + "margin_dpo/margin_std": 93.55561828613281, + "step": 535 + }, + { + "KL/chosen_KL_mean": -118.49983978271484, + "KL/mean": -157.49525451660156, + "KL/rejected_KL_mean": -196.49066162109375, + "KL/std": 87.45317077636719, + "epoch": 0.8102796674225246, + "fcm_dpo/beta": 0.007055111229419708, + "fcm_dpo/delta": -0.15915080904960632, + "fcm_dpo/margin": 77.99081420898438, + "fcm_dpo/q_t": 0.37544578313827515, + "grad_norm": 11.248884201049805, + "learning_rate": 5.3486726314303175e-08, + "logits/chosen": 0.7687609195709229, + "logits/rejected": 0.6766946315765381, + "logps/chosen": -170.38882446289062, + "logps/ref_chosen": -51.88897705078125, + "logps/ref_rejected": -73.34864044189453, + "logps/rejected": -269.83929443359375, + "loss": 0.9901, + "margin_dpo/margin_mean": 77.99082946777344, + "margin_dpo/margin_std": 84.41676330566406, + "step": 536 + }, + { + "KL/chosen_KL_mean": -135.27239990234375, + "KL/mean": -164.75808715820312, + "KL/rejected_KL_mean": -194.24378967285156, + "KL/std": 88.089599609375, + "epoch": 0.8117913832199547, + "fcm_dpo/beta": 0.0069625237956643105, + "fcm_dpo/delta": -0.011460717767477036, + "fcm_dpo/margin": 58.971397399902344, + "fcm_dpo/q_t": 0.40821582078933716, + "grad_norm": 14.422006607055664, + "learning_rate": 5.267213693697695e-08, + "logits/chosen": 0.8400592803955078, + "logits/rejected": 0.7391371726989746, + "logps/chosen": -189.52101135253906, + "logps/ref_chosen": -54.248619079589844, + "logps/ref_rejected": -94.94343566894531, + "logps/rejected": -289.1872253417969, + "loss": 1.1192, + "margin_dpo/margin_mean": 58.971397399902344, + "margin_dpo/margin_std": 94.33164978027344, + "step": 537 + }, + { + "KL/chosen_KL_mean": -130.045654296875, + "KL/mean": -163.341064453125, + "KL/rejected_KL_mean": -196.636474609375, + "KL/std": 84.741943359375, + "epoch": 0.8133030990173847, + "fcm_dpo/beta": 0.006943310610949993, + "fcm_dpo/delta": -0.06539023667573929, + "fcm_dpo/margin": 66.5908203125, + "fcm_dpo/q_t": 0.3946533203125, + "grad_norm": 13.937400817871094, + "learning_rate": 5.1863067244167144e-08, + "logits/chosen": 0.7151967883110046, + "logits/rejected": 0.6850166320800781, + "logps/chosen": -200.13919067382812, + "logps/ref_chosen": -70.09353637695312, + "logps/ref_rejected": -79.49833679199219, + "logps/rejected": -276.13482666015625, + "loss": 1.0624, + "margin_dpo/margin_mean": 66.5908203125, + "margin_dpo/margin_std": 87.32984924316406, + "step": 538 + }, + { + "KL/chosen_KL_mean": -140.43576049804688, + "KL/mean": -165.48690795898438, + "KL/rejected_KL_mean": -190.5380401611328, + "KL/std": 83.2811508178711, + "epoch": 0.8148148148148148, + "fcm_dpo/beta": 0.006932198069989681, + "fcm_dpo/delta": 0.05455028638243675, + "fcm_dpo/margin": 50.1022834777832, + "fcm_dpo/q_t": 0.4223223328590393, + "grad_norm": 14.812973022460938, + "learning_rate": 5.105953986729195e-08, + "logits/chosen": 0.6730055809020996, + "logits/rejected": 0.588666558265686, + "logps/chosen": -202.36746215820312, + "logps/ref_chosen": -61.93169403076172, + "logps/ref_rejected": -84.08946228027344, + "logps/rejected": -274.62750244140625, + "loss": 1.1575, + "margin_dpo/margin_mean": 50.10228729248047, + "margin_dpo/margin_std": 88.67442321777344, + "step": 539 + }, + { + "KL/chosen_KL_mean": -127.05425262451172, + "KL/mean": -163.42059326171875, + "KL/rejected_KL_mean": -199.7869110107422, + "KL/std": 91.1855697631836, + "epoch": 0.8163265306122449, + "fcm_dpo/beta": 0.006901265121996403, + "fcm_dpo/delta": -0.10737490653991699, + "fcm_dpo/margin": 72.73268127441406, + "fcm_dpo/q_t": 0.38571393489837646, + "grad_norm": 13.47286605834961, + "learning_rate": 5.026157728273966e-08, + "logits/chosen": 0.7549730539321899, + "logits/rejected": 0.6512420177459717, + "logps/chosen": -189.7584991455078, + "logps/ref_chosen": -62.704254150390625, + "logps/ref_rejected": -95.63597106933594, + "logps/rejected": -295.4228820800781, + "loss": 1.033, + "margin_dpo/margin_mean": 72.73268127441406, + "margin_dpo/margin_std": 87.08503723144531, + "step": 540 + }, + { + "KL/chosen_KL_mean": -124.87004852294922, + "KL/mean": -158.8612060546875, + "KL/rejected_KL_mean": -192.8523712158203, + "KL/std": 85.35519409179688, + "epoch": 0.817838246409675, + "fcm_dpo/beta": 0.006723807659000158, + "fcm_dpo/delta": -0.060890041291713715, + "fcm_dpo/margin": 67.9823226928711, + "fcm_dpo/q_t": 0.3946753144264221, + "grad_norm": 13.895951271057129, + "learning_rate": 4.9469201811239035e-08, + "logits/chosen": 0.7421223521232605, + "logits/rejected": 0.7663363218307495, + "logps/chosen": -187.35089111328125, + "logps/ref_chosen": -62.48084259033203, + "logps/ref_rejected": -57.55541229248047, + "logps/rejected": -250.40777587890625, + "loss": 1.0618, + "margin_dpo/margin_mean": 67.9823226928711, + "margin_dpo/margin_std": 85.90786743164062, + "step": 541 + }, + { + "KL/chosen_KL_mean": -108.08326721191406, + "KL/mean": -145.9094696044922, + "KL/rejected_KL_mean": -183.73565673828125, + "KL/std": 83.32110595703125, + "epoch": 0.8193499622071051, + "fcm_dpo/beta": 0.006626952439546585, + "fcm_dpo/delta": -0.10687525570392609, + "fcm_dpo/margin": 75.65240478515625, + "fcm_dpo/q_t": 0.3857290744781494, + "grad_norm": 13.757902145385742, + "learning_rate": 4.868243561723534e-08, + "logits/chosen": 0.7640467882156372, + "logits/rejected": 0.7123850584030151, + "logps/chosen": -157.53814697265625, + "logps/ref_chosen": -49.454891204833984, + "logps/ref_rejected": -65.33275604248047, + "logps/rejected": -249.06842041015625, + "loss": 1.0517, + "margin_dpo/margin_mean": 75.65240478515625, + "margin_dpo/margin_std": 101.577392578125, + "step": 542 + }, + { + "KL/chosen_KL_mean": -115.50267791748047, + "KL/mean": -150.74710083007812, + "KL/rejected_KL_mean": -185.9915313720703, + "KL/std": 81.07098388671875, + "epoch": 0.8208616780045351, + "fcm_dpo/beta": 0.006557712331414223, + "fcm_dpo/delta": -0.06527210772037506, + "fcm_dpo/margin": 70.48886108398438, + "fcm_dpo/q_t": 0.39285480976104736, + "grad_norm": 11.07712459564209, + "learning_rate": 4.790130070827028e-08, + "logits/chosen": 0.7415127754211426, + "logits/rejected": 0.6479545831680298, + "logps/chosen": -166.60354614257812, + "logps/ref_chosen": -51.100860595703125, + "logps/ref_rejected": -76.06130981445312, + "logps/rejected": -262.0528564453125, + "loss": 1.0594, + "margin_dpo/margin_mean": 70.48886108398438, + "margin_dpo/margin_std": 89.37535095214844, + "step": 543 + }, + { + "KL/chosen_KL_mean": -127.9332504272461, + "KL/mean": -165.9463653564453, + "KL/rejected_KL_mean": -203.95947265625, + "KL/std": 87.00491333007812, + "epoch": 0.8223733938019653, + "fcm_dpo/beta": 0.006429500877857208, + "fcm_dpo/delta": -0.09331250935792923, + "fcm_dpo/margin": 76.02620697021484, + "fcm_dpo/q_t": 0.38860976696014404, + "grad_norm": 14.734076499938965, + "learning_rate": 4.7125818934366454e-08, + "logits/chosen": 0.7422240972518921, + "logits/rejected": 0.6599665880203247, + "logps/chosen": -188.21047973632812, + "logps/ref_chosen": -60.2772331237793, + "logps/ref_rejected": -88.40553283691406, + "logps/rejected": -292.364990234375, + "loss": 1.051, + "margin_dpo/margin_mean": 76.02620697021484, + "margin_dpo/margin_std": 99.28297424316406, + "step": 544 + }, + { + "KL/chosen_KL_mean": -136.6294708251953, + "KL/mean": -160.23928833007812, + "KL/rejected_KL_mean": -183.84912109375, + "KL/std": 84.00276184082031, + "epoch": 0.8238851095993953, + "fcm_dpo/beta": 0.006486566737294197, + "fcm_dpo/delta": 0.09666649997234344, + "fcm_dpo/margin": 47.219642639160156, + "fcm_dpo/q_t": 0.4306223690509796, + "grad_norm": 13.613739967346191, + "learning_rate": 4.635601198741607e-08, + "logits/chosen": 0.6614166498184204, + "logits/rejected": 0.6035963296890259, + "logps/chosen": -198.24472045898438, + "logps/ref_chosen": -61.61524963378906, + "logps/ref_rejected": -78.71266174316406, + "logps/rejected": -262.561767578125, + "loss": 1.1946, + "margin_dpo/margin_mean": 47.219642639160156, + "margin_dpo/margin_std": 95.0772705078125, + "step": 545 + }, + { + "KL/chosen_KL_mean": -129.49948120117188, + "KL/mean": -156.82077026367188, + "KL/rejected_KL_mean": -184.14207458496094, + "KL/std": 81.37245178222656, + "epoch": 0.8253968253968254, + "fcm_dpo/beta": 0.006546933203935623, + "fcm_dpo/delta": 0.04382575675845146, + "fcm_dpo/margin": 54.64259338378906, + "fcm_dpo/q_t": 0.41790369153022766, + "grad_norm": 15.220459938049316, + "learning_rate": 4.559190140057428e-08, + "logits/chosen": 0.813021183013916, + "logits/rejected": 0.8038173913955688, + "logps/chosen": -188.812744140625, + "logps/ref_chosen": -59.313262939453125, + "logps/ref_rejected": -64.73631286621094, + "logps/rejected": -248.87838745117188, + "loss": 1.1598, + "margin_dpo/margin_mean": 54.6425895690918, + "margin_dpo/margin_std": 98.65786743164062, + "step": 546 + }, + { + "KL/chosen_KL_mean": -115.29865264892578, + "KL/mean": -150.95513916015625, + "KL/rejected_KL_mean": -186.6116180419922, + "KL/std": 84.62432098388672, + "epoch": 0.8269085411942555, + "fcm_dpo/beta": 0.006497037131339312, + "fcm_dpo/delta": -0.06663330644369125, + "fcm_dpo/margin": 71.31297302246094, + "fcm_dpo/q_t": 0.3933113217353821, + "grad_norm": 13.908299446105957, + "learning_rate": 4.483350854765672e-08, + "logits/chosen": 0.6590763926506042, + "logits/rejected": 0.5905526876449585, + "logps/chosen": -170.275390625, + "logps/ref_chosen": -54.97674560546875, + "logps/ref_rejected": -75.35922241210938, + "logps/rejected": -261.9708251953125, + "loss": 1.0669, + "margin_dpo/margin_mean": 71.31297302246094, + "margin_dpo/margin_std": 96.31216430664062, + "step": 547 + }, + { + "KL/chosen_KL_mean": -132.67001342773438, + "KL/mean": -156.9127197265625, + "KL/rejected_KL_mean": -181.15542602539062, + "KL/std": 84.14120483398438, + "epoch": 0.8284202569916855, + "fcm_dpo/beta": 0.0065902965143322945, + "fcm_dpo/delta": 0.0826481357216835, + "fcm_dpo/margin": 48.48541259765625, + "fcm_dpo/q_t": 0.42783021926879883, + "grad_norm": 16.948848724365234, + "learning_rate": 4.4080854642541826e-08, + "logits/chosen": 0.6547988653182983, + "logits/rejected": 0.5863006711006165, + "logps/chosen": -195.88067626953125, + "logps/ref_chosen": -63.21067428588867, + "logps/ref_rejected": -81.23347473144531, + "logps/rejected": -262.388916015625, + "loss": 1.1768, + "margin_dpo/margin_mean": 48.48541259765625, + "margin_dpo/margin_std": 90.33981323242188, + "step": 548 + }, + { + "KL/chosen_KL_mean": -128.98789978027344, + "KL/mean": -156.6746826171875, + "KL/rejected_KL_mean": -184.3614501953125, + "KL/std": 84.68203735351562, + "epoch": 0.8299319727891157, + "fcm_dpo/beta": 0.006622135639190674, + "fcm_dpo/delta": 0.03458665311336517, + "fcm_dpo/margin": 55.373558044433594, + "fcm_dpo/q_t": 0.41641145944595337, + "grad_norm": 15.642621994018555, + "learning_rate": 4.333396073857723e-08, + "logits/chosen": 0.8145290017127991, + "logits/rejected": 0.7393472194671631, + "logps/chosen": -193.26141357421875, + "logps/ref_chosen": -64.27351379394531, + "logps/ref_rejected": -92.31663513183594, + "logps/rejected": -276.6781005859375, + "loss": 1.1508, + "margin_dpo/margin_mean": 55.37355422973633, + "margin_dpo/margin_std": 97.06788635253906, + "step": 549 + }, + { + "KL/chosen_KL_mean": -141.47198486328125, + "KL/mean": -159.78488159179688, + "KL/rejected_KL_mean": -178.09779357910156, + "KL/std": 81.69731140136719, + "epoch": 0.8314436885865457, + "fcm_dpo/beta": 0.006767358630895615, + "fcm_dpo/delta": 0.15628939867019653, + "fcm_dpo/margin": 36.62580490112305, + "fcm_dpo/q_t": 0.4437524974346161, + "grad_norm": 17.119239807128906, + "learning_rate": 4.259284772799099e-08, + "logits/chosen": 0.7669326066970825, + "logits/rejected": 0.7338714599609375, + "logps/chosen": -197.70242309570312, + "logps/ref_chosen": -56.230438232421875, + "logps/ref_rejected": -62.59788513183594, + "logps/rejected": -240.6956787109375, + "loss": 1.2428, + "margin_dpo/margin_mean": 36.62580108642578, + "margin_dpo/margin_std": 88.4005126953125, + "step": 550 + }, + { + "KL/chosen_KL_mean": -139.54542541503906, + "KL/mean": -162.89857482910156, + "KL/rejected_KL_mean": -186.251708984375, + "KL/std": 85.5566635131836, + "epoch": 0.8329554043839759, + "fcm_dpo/beta": 0.0069540319964289665, + "fcm_dpo/delta": 0.07730760425329208, + "fcm_dpo/margin": 46.70626449584961, + "fcm_dpo/q_t": 0.4272865653038025, + "grad_norm": 14.740558624267578, + "learning_rate": 4.1857536341307176e-08, + "logits/chosen": 0.728614330291748, + "logits/rejected": 0.6926642656326294, + "logps/chosen": -207.29263305664062, + "logps/ref_chosen": -67.74720764160156, + "logps/ref_rejected": -87.04285430908203, + "logps/rejected": -273.2945556640625, + "loss": 1.1645, + "margin_dpo/margin_mean": 46.706260681152344, + "margin_dpo/margin_std": 83.12886810302734, + "step": 551 + }, + { + "KL/chosen_KL_mean": -129.97451782226562, + "KL/mean": -158.94329833984375, + "KL/rejected_KL_mean": -187.91207885742188, + "KL/std": 85.75680541992188, + "epoch": 0.8344671201814059, + "fcm_dpo/beta": 0.006984234321862459, + "fcm_dpo/delta": -0.005182873457670212, + "fcm_dpo/margin": 57.937564849853516, + "fcm_dpo/q_t": 0.4047027826309204, + "grad_norm": 14.647710800170898, + "learning_rate": 4.112804714676593e-08, + "logits/chosen": 0.7080692052841187, + "logits/rejected": 0.6515108346939087, + "logps/chosen": -192.90078735351562, + "logps/ref_chosen": -62.92625427246094, + "logps/ref_rejected": -82.98365783691406, + "logps/rejected": -270.8957214355469, + "loss": 1.1029, + "margin_dpo/margin_mean": 57.937564849853516, + "margin_dpo/margin_std": 82.95120239257812, + "step": 552 + }, + { + "KL/chosen_KL_mean": -140.59323120117188, + "KL/mean": -167.88583374023438, + "KL/rejected_KL_mean": -195.17840576171875, + "KL/std": 81.97215270996094, + "epoch": 0.8359788359788359, + "fcm_dpo/beta": 0.0069772712886333466, + "fcm_dpo/delta": 0.019887106493115425, + "fcm_dpo/margin": 54.585182189941406, + "fcm_dpo/q_t": 0.415116548538208, + "grad_norm": 17.540098190307617, + "learning_rate": 4.0404400549748144e-08, + "logits/chosen": 0.6930921673774719, + "logits/rejected": 0.5812926292419434, + "logps/chosen": -196.63172912597656, + "logps/ref_chosen": -56.038490295410156, + "logps/ref_rejected": -84.48454284667969, + "logps/rejected": -279.6629638671875, + "loss": 1.1521, + "margin_dpo/margin_mean": 54.585182189941406, + "margin_dpo/margin_std": 99.14485931396484, + "step": 553 + }, + { + "KL/chosen_KL_mean": -126.84615325927734, + "KL/mean": -159.65020751953125, + "KL/rejected_KL_mean": -192.45425415039062, + "KL/std": 82.10330200195312, + "epoch": 0.8374905517762661, + "fcm_dpo/beta": 0.006926331669092178, + "fcm_dpo/delta": -0.05698609724640846, + "fcm_dpo/margin": 65.60808563232422, + "fcm_dpo/q_t": 0.3963993191719055, + "grad_norm": 14.771784782409668, + "learning_rate": 3.968661679220467e-08, + "logits/chosen": 0.7141705751419067, + "logits/rejected": 0.6928715705871582, + "logps/chosen": -191.37673950195312, + "logps/ref_chosen": -64.53059387207031, + "logps/ref_rejected": -71.2155990600586, + "logps/rejected": -263.66986083984375, + "loss": 1.0791, + "margin_dpo/margin_mean": 65.60808563232422, + "margin_dpo/margin_std": 92.8201904296875, + "step": 554 + }, + { + "KL/chosen_KL_mean": -138.47244262695312, + "KL/mean": -166.33506774902344, + "KL/rejected_KL_mean": -194.19769287109375, + "KL/std": 83.23756408691406, + "epoch": 0.8390022675736961, + "fcm_dpo/beta": 0.006969560869038105, + "fcm_dpo/delta": 0.010640434920787811, + "fcm_dpo/margin": 55.725242614746094, + "fcm_dpo/q_t": 0.41070425510406494, + "grad_norm": 15.12730598449707, + "learning_rate": 3.89747159520904e-08, + "logits/chosen": 0.7322822213172913, + "logits/rejected": 0.703670859336853, + "logps/chosen": -205.12435913085938, + "logps/ref_chosen": -66.65191650390625, + "logps/ref_rejected": -68.6667251586914, + "logps/rejected": -262.8644104003906, + "loss": 1.1592, + "margin_dpo/margin_mean": 55.725242614746094, + "margin_dpo/margin_std": 100.45161437988281, + "step": 555 + }, + { + "KL/chosen_KL_mean": -140.06524658203125, + "KL/mean": -164.30087280273438, + "KL/rejected_KL_mean": -188.53648376464844, + "KL/std": 87.56776428222656, + "epoch": 0.8405139833711263, + "fcm_dpo/beta": 0.006966423708945513, + "fcm_dpo/delta": 0.06454331427812576, + "fcm_dpo/margin": 48.47126007080078, + "fcm_dpo/q_t": 0.42569175362586975, + "grad_norm": 13.4514799118042, + "learning_rate": 3.826871794280192e-08, + "logits/chosen": 0.7811669707298279, + "logits/rejected": 0.7258546948432922, + "logps/chosen": -192.89761352539062, + "logps/ref_chosen": -52.832366943359375, + "logps/ref_rejected": -64.49044036865234, + "logps/rejected": -253.02691650390625, + "loss": 1.1975, + "margin_dpo/margin_mean": 48.47126007080078, + "margin_dpo/margin_std": 102.2444076538086, + "step": 556 + }, + { + "KL/chosen_KL_mean": -137.24900817871094, + "KL/mean": -174.63917541503906, + "KL/rejected_KL_mean": -212.02932739257812, + "KL/std": 84.4654541015625, + "epoch": 0.8420256991685563, + "fcm_dpo/beta": 0.00685353297740221, + "fcm_dpo/delta": -0.11947400867938995, + "fcm_dpo/margin": 74.78031158447266, + "fcm_dpo/q_t": 0.3829210102558136, + "grad_norm": 12.452169418334961, + "learning_rate": 3.756864251262143e-08, + "logits/chosen": 0.8675057888031006, + "logits/rejected": 0.7827339172363281, + "logps/chosen": -192.28500366210938, + "logps/ref_chosen": -55.03598403930664, + "logps/ref_rejected": -75.80644989013672, + "logps/rejected": -287.8357849121094, + "loss": 1.017, + "margin_dpo/margin_mean": 74.78031921386719, + "margin_dpo/margin_std": 84.29581451416016, + "step": 557 + }, + { + "KL/chosen_KL_mean": -123.99850463867188, + "KL/mean": -163.4159393310547, + "KL/rejected_KL_mean": -202.83338928222656, + "KL/std": 89.908935546875, + "epoch": 0.8435374149659864, + "fcm_dpo/beta": 0.0066793859004974365, + "fcm_dpo/delta": -0.1344417929649353, + "fcm_dpo/margin": 78.83488464355469, + "fcm_dpo/q_t": 0.38055527210235596, + "grad_norm": 12.364710807800293, + "learning_rate": 3.687450924416341e-08, + "logits/chosen": 0.7823951840400696, + "logits/rejected": 0.7239669561386108, + "logps/chosen": -187.224853515625, + "logps/ref_chosen": -63.226348876953125, + "logps/ref_rejected": -91.46881866455078, + "logps/rejected": -294.30218505859375, + "loss": 1.0155, + "margin_dpo/margin_mean": 78.83488464355469, + "margin_dpo/margin_std": 92.04693603515625, + "step": 558 + }, + { + "KL/chosen_KL_mean": -129.73312377929688, + "KL/mean": -163.43589782714844, + "KL/rejected_KL_mean": -197.13868713378906, + "KL/std": 86.80623626708984, + "epoch": 0.8450491307634165, + "fcm_dpo/beta": 0.00656571751460433, + "fcm_dpo/delta": -0.04597283899784088, + "fcm_dpo/margin": 67.40557861328125, + "fcm_dpo/q_t": 0.4019114673137665, + "grad_norm": 12.485381126403809, + "learning_rate": 3.6186337553827743e-08, + "logits/chosen": 0.713494062423706, + "logits/rejected": 0.640967845916748, + "logps/chosen": -191.2547607421875, + "logps/ref_chosen": -61.521644592285156, + "logps/ref_rejected": -82.83859252929688, + "logps/rejected": -279.977294921875, + "loss": 1.0991, + "margin_dpo/margin_mean": 67.40557861328125, + "margin_dpo/margin_std": 101.77870178222656, + "step": 559 + }, + { + "KL/chosen_KL_mean": -141.1266326904297, + "KL/mean": -170.9273681640625, + "KL/rejected_KL_mean": -200.7281036376953, + "KL/std": 85.53338623046875, + "epoch": 0.8465608465608465, + "fcm_dpo/beta": 0.006634770426899195, + "fcm_dpo/delta": 0.004244182258844376, + "fcm_dpo/margin": 59.60146713256836, + "fcm_dpo/q_t": 0.4095200300216675, + "grad_norm": 14.990598678588867, + "learning_rate": 3.550414669125573e-08, + "logits/chosen": 0.7438457012176514, + "logits/rejected": 0.7018010020256042, + "logps/chosen": -201.76785278320312, + "logps/ref_chosen": -60.64122009277344, + "logps/ref_rejected": -78.75474548339844, + "logps/rejected": -279.48284912109375, + "loss": 1.1056, + "margin_dpo/margin_mean": 59.60146713256836, + "margin_dpo/margin_std": 84.04834747314453, + "step": 560 + }, + { + "KL/chosen_KL_mean": -128.27587890625, + "KL/mean": -158.63803100585938, + "KL/rejected_KL_mean": -189.0001983642578, + "KL/std": 85.89263916015625, + "epoch": 0.8480725623582767, + "fcm_dpo/beta": 0.0065963175147771835, + "fcm_dpo/delta": -0.000604395754635334, + "fcm_dpo/margin": 60.72432327270508, + "fcm_dpo/q_t": 0.40989649295806885, + "grad_norm": 13.60486125946045, + "learning_rate": 3.482795573879241e-08, + "logits/chosen": 0.7049076557159424, + "logits/rejected": 0.6703420877456665, + "logps/chosen": -190.77447509765625, + "logps/ref_chosen": -62.49859619140625, + "logps/ref_rejected": -78.72064208984375, + "logps/rejected": -267.7208251953125, + "loss": 1.1156, + "margin_dpo/margin_mean": 60.724327087402344, + "margin_dpo/margin_std": 93.66244506835938, + "step": 561 + }, + { + "KL/chosen_KL_mean": -135.21826171875, + "KL/mean": -172.11111450195312, + "KL/rejected_KL_mean": -209.00396728515625, + "KL/std": 90.7817611694336, + "epoch": 0.8495842781557067, + "fcm_dpo/beta": 0.006486967206001282, + "fcm_dpo/delta": -0.08362391591072083, + "fcm_dpo/margin": 73.78570556640625, + "fcm_dpo/q_t": 0.3917215168476105, + "grad_norm": 14.613893508911133, + "learning_rate": 3.415778361095226e-08, + "logits/chosen": 0.7305380702018738, + "logits/rejected": 0.6867002248764038, + "logps/chosen": -210.0, + "logps/ref_chosen": -74.78173828125, + "logps/ref_rejected": -92.63499450683594, + "logps/rejected": -301.63897705078125, + "loss": 1.0514, + "margin_dpo/margin_mean": 73.78570556640625, + "margin_dpo/margin_std": 93.51516723632812, + "step": 562 + }, + { + "KL/chosen_KL_mean": -116.42094421386719, + "KL/mean": -149.75637817382812, + "KL/rejected_KL_mean": -183.09182739257812, + "KL/std": 79.57887268066406, + "epoch": 0.8510959939531368, + "fcm_dpo/beta": 0.006465439684689045, + "fcm_dpo/delta": -0.03244488313794136, + "fcm_dpo/margin": 66.67088317871094, + "fcm_dpo/q_t": 0.40143883228302, + "grad_norm": 17.642074584960938, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": 0.8246127367019653, + "logits/rejected": 0.7726570963859558, + "logps/chosen": -166.61944580078125, + "logps/ref_chosen": -50.19850158691406, + "logps/ref_rejected": -66.76687622070312, + "logps/rejected": -249.85870361328125, + "loss": 1.1031, + "margin_dpo/margin_mean": 66.67088317871094, + "margin_dpo/margin_std": 101.82766723632812, + "step": 563 + }, + { + "KL/chosen_KL_mean": -118.33200073242188, + "KL/mean": -159.535400390625, + "KL/rejected_KL_mean": -200.73880004882812, + "KL/std": 85.4729232788086, + "epoch": 0.8526077097505669, + "fcm_dpo/beta": 0.00632768590003252, + "fcm_dpo/delta": -0.12817589938640594, + "fcm_dpo/margin": 82.40682220458984, + "fcm_dpo/q_t": 0.37969234585762024, + "grad_norm": 13.583505630493164, + "learning_rate": 3.283557064487785e-08, + "logits/chosen": 0.7059386968612671, + "logits/rejected": 0.6721379160881042, + "logps/chosen": -174.07284545898438, + "logps/ref_chosen": -55.7408447265625, + "logps/ref_rejected": -74.82323455810547, + "logps/rejected": -275.5620422363281, + "loss": 1.0288, + "margin_dpo/margin_mean": 82.40682220458984, + "margin_dpo/margin_std": 102.0660171508789, + "step": 564 + }, + { + "KL/chosen_KL_mean": -145.9530487060547, + "KL/mean": -173.38087463378906, + "KL/rejected_KL_mean": -200.8087158203125, + "KL/std": 84.87850952148438, + "epoch": 0.854119425547997, + "fcm_dpo/beta": 0.006371333263814449, + "fcm_dpo/delta": 0.051446303725242615, + "fcm_dpo/margin": 54.85563659667969, + "fcm_dpo/q_t": 0.41960498690605164, + "grad_norm": 15.257643699645996, + "learning_rate": 3.218356679178252e-08, + "logits/chosen": 0.7865220308303833, + "logits/rejected": 0.7267142534255981, + "logps/chosen": -204.29043579101562, + "logps/ref_chosen": -58.33738327026367, + "logps/ref_rejected": -78.31776428222656, + "logps/rejected": -279.12646484375, + "loss": 1.1414, + "margin_dpo/margin_mean": 54.85563278198242, + "margin_dpo/margin_std": 86.63621520996094, + "step": 565 + }, + { + "KL/chosen_KL_mean": -133.1039581298828, + "KL/mean": -160.00714111328125, + "KL/rejected_KL_mean": -186.9103240966797, + "KL/std": 87.2846908569336, + "epoch": 0.8556311413454271, + "fcm_dpo/beta": 0.006431379355490208, + "fcm_dpo/delta": 0.05535087734460831, + "fcm_dpo/margin": 53.80636215209961, + "fcm_dpo/q_t": 0.42274099588394165, + "grad_norm": 16.522539138793945, + "learning_rate": 3.1537655732553764e-08, + "logits/chosen": 0.7471953630447388, + "logits/rejected": 0.7294028997421265, + "logps/chosen": -204.32769775390625, + "logps/ref_chosen": -71.22373962402344, + "logps/ref_rejected": -71.11601257324219, + "logps/rejected": -258.0263366699219, + "loss": 1.1891, + "margin_dpo/margin_mean": 53.806365966796875, + "margin_dpo/margin_std": 110.33029174804688, + "step": 566 + }, + { + "KL/chosen_KL_mean": -129.12673950195312, + "KL/mean": -162.45062255859375, + "KL/rejected_KL_mean": -195.7744903564453, + "KL/std": 85.07839965820312, + "epoch": 0.8571428571428571, + "fcm_dpo/beta": 0.006362162530422211, + "fcm_dpo/delta": -0.025937747210264206, + "fcm_dpo/margin": 66.64776611328125, + "fcm_dpo/q_t": 0.40236735343933105, + "grad_norm": 12.060728073120117, + "learning_rate": 3.089785553471233e-08, + "logits/chosen": 0.7700002789497375, + "logits/rejected": 0.6686394214630127, + "logps/chosen": -181.79600524902344, + "logps/ref_chosen": -52.669273376464844, + "logps/ref_rejected": -74.34785461425781, + "logps/rejected": -270.1223449707031, + "loss": 1.0901, + "margin_dpo/margin_mean": 66.64776611328125, + "margin_dpo/margin_std": 91.76837158203125, + "step": 567 + }, + { + "KL/chosen_KL_mean": -118.26817321777344, + "KL/mean": -159.29119873046875, + "KL/rejected_KL_mean": -200.31422424316406, + "KL/std": 87.94129180908203, + "epoch": 0.8586545729402872, + "fcm_dpo/beta": 0.006308428943157196, + "fcm_dpo/delta": -0.12392584979534149, + "fcm_dpo/margin": 82.04606628417969, + "fcm_dpo/q_t": 0.3821907639503479, + "grad_norm": 13.965873718261719, + "learning_rate": 3.026418409484513e-08, + "logits/chosen": 0.7885270714759827, + "logits/rejected": 0.6974135637283325, + "logps/chosen": -170.4461669921875, + "logps/ref_chosen": -52.178001403808594, + "logps/ref_rejected": -85.8277587890625, + "logps/rejected": -286.1419677734375, + "loss": 1.0129, + "margin_dpo/margin_mean": 82.04606628417969, + "margin_dpo/margin_std": 89.78453063964844, + "step": 568 + }, + { + "KL/chosen_KL_mean": -136.9675750732422, + "KL/mean": -157.94886779785156, + "KL/rejected_KL_mean": -178.93014526367188, + "KL/std": 86.604248046875, + "epoch": 0.8601662887377173, + "fcm_dpo/beta": 0.006321226246654987, + "fcm_dpo/delta": 0.13839997351169586, + "fcm_dpo/margin": 41.96255874633789, + "fcm_dpo/q_t": 0.43971410393714905, + "grad_norm": 14.764012336730957, + "learning_rate": 2.963665913810451e-08, + "logits/chosen": 0.6662417650222778, + "logits/rejected": 0.6308369636535645, + "logps/chosen": -199.61683654785156, + "logps/ref_chosen": -62.649261474609375, + "logps/ref_rejected": -75.4298324584961, + "logps/rejected": -254.3599853515625, + "loss": 1.2198, + "margin_dpo/margin_mean": 41.96255874633789, + "margin_dpo/margin_std": 88.51194763183594, + "step": 569 + }, + { + "KL/chosen_KL_mean": -121.72870635986328, + "KL/mean": -163.15060424804688, + "KL/rejected_KL_mean": -204.57249450683594, + "KL/std": 83.75516510009766, + "epoch": 0.8616780045351474, + "fcm_dpo/beta": 0.0063173240050673485, + "fcm_dpo/delta": -0.13001130521297455, + "fcm_dpo/margin": 82.84379577636719, + "fcm_dpo/q_t": 0.3797076940536499, + "grad_norm": 12.93687915802002, + "learning_rate": 2.9015298217712453e-08, + "logits/chosen": 0.7038021087646484, + "logits/rejected": 0.617842435836792, + "logps/chosen": -171.7705078125, + "logps/ref_chosen": -50.04179382324219, + "logps/ref_rejected": -78.27146911621094, + "logps/rejected": -282.8439636230469, + "loss": 1.0115, + "margin_dpo/margin_mean": 82.84379577636719, + "margin_dpo/margin_std": 92.3207015991211, + "step": 570 + }, + { + "KL/chosen_KL_mean": -133.497802734375, + "KL/mean": -157.1429443359375, + "KL/rejected_KL_mean": -180.78807067871094, + "KL/std": 81.17963409423828, + "epoch": 0.8631897203325775, + "fcm_dpo/beta": 0.006368095986545086, + "fcm_dpo/delta": 0.10106943547725677, + "fcm_dpo/margin": 47.29026794433594, + "fcm_dpo/q_t": 0.4300426244735718, + "grad_norm": 13.396882057189941, + "learning_rate": 2.840011871446962e-08, + "logits/chosen": 0.7285163402557373, + "logits/rejected": 0.695633053779602, + "logps/chosen": -187.15460205078125, + "logps/ref_chosen": -53.65681457519531, + "logps/ref_rejected": -66.13298034667969, + "logps/rejected": -246.92105102539062, + "loss": 1.1986, + "margin_dpo/margin_mean": 47.2902717590332, + "margin_dpo/margin_std": 94.57589721679688, + "step": 571 + }, + { + "KL/chosen_KL_mean": -129.37582397460938, + "KL/mean": -157.59616088867188, + "KL/rejected_KL_mean": -185.8165283203125, + "KL/std": 84.47772216796875, + "epoch": 0.8647014361300076, + "fcm_dpo/beta": 0.006431017071008682, + "fcm_dpo/delta": 0.0380452424287796, + "fcm_dpo/margin": 56.440704345703125, + "fcm_dpo/q_t": 0.41607558727264404, + "grad_norm": 14.078507423400879, + "learning_rate": 2.7791137836269158e-08, + "logits/chosen": 0.6906898617744446, + "logits/rejected": 0.7365379333496094, + "logps/chosen": -204.19374084472656, + "logps/ref_chosen": -74.81792449951172, + "logps/ref_rejected": -65.88681030273438, + "logps/rejected": -251.7033233642578, + "loss": 1.1253, + "margin_dpo/margin_mean": 56.440704345703125, + "margin_dpo/margin_std": 83.41856384277344, + "step": 572 + }, + { + "KL/chosen_KL_mean": -143.9004669189453, + "KL/mean": -172.6072235107422, + "KL/rejected_KL_mean": -201.31398010253906, + "KL/std": 89.49702453613281, + "epoch": 0.8662131519274376, + "fcm_dpo/beta": 0.006447994615882635, + "fcm_dpo/delta": 0.030953213572502136, + "fcm_dpo/margin": 57.413509368896484, + "fcm_dpo/q_t": 0.4167448580265045, + "grad_norm": 14.49169921875, + "learning_rate": 2.718837261761528e-08, + "logits/chosen": 0.7309734225273132, + "logits/rejected": 0.6812965273857117, + "logps/chosen": -212.62611389160156, + "logps/ref_chosen": -68.72564697265625, + "logps/ref_rejected": -88.16201782226562, + "logps/rejected": -289.47601318359375, + "loss": 1.16, + "margin_dpo/margin_mean": 57.413509368896484, + "margin_dpo/margin_std": 106.84358978271484, + "step": 573 + }, + { + "KL/chosen_KL_mean": -128.6298065185547, + "KL/mean": -164.86526489257812, + "KL/rejected_KL_mean": -201.1007080078125, + "KL/std": 81.8821792602539, + "epoch": 0.8677248677248677, + "fcm_dpo/beta": 0.006425968836992979, + "fcm_dpo/delta": -0.06904555857181549, + "fcm_dpo/margin": 72.47090911865234, + "fcm_dpo/q_t": 0.3918231129646301, + "grad_norm": 11.799493789672852, + "learning_rate": 2.659183991914696e-08, + "logits/chosen": 0.7962836623191833, + "logits/rejected": 0.7224004864692688, + "logps/chosen": -184.94320678710938, + "logps/ref_chosen": -56.31340026855469, + "logps/ref_rejected": -83.91553497314453, + "logps/rejected": -285.0162353515625, + "loss": 1.0329, + "margin_dpo/margin_mean": 72.47090911865234, + "margin_dpo/margin_std": 76.56863403320312, + "step": 574 + }, + { + "KL/chosen_KL_mean": -131.7310791015625, + "KL/mean": -157.78036499023438, + "KL/rejected_KL_mean": -183.82962036132812, + "KL/std": 86.12600708007812, + "epoch": 0.8692365835222978, + "fcm_dpo/beta": 0.00639111315831542, + "fcm_dpo/delta": 0.06874041259288788, + "fcm_dpo/margin": 52.09855651855469, + "fcm_dpo/q_t": 0.4262588620185852, + "grad_norm": 14.336535453796387, + "learning_rate": 2.600155642716606e-08, + "logits/chosen": 0.7968418598175049, + "logits/rejected": 0.7067606449127197, + "logps/chosen": -196.31520080566406, + "logps/ref_chosen": -64.5841293334961, + "logps/ref_rejected": -93.47034454345703, + "logps/rejected": -277.29998779296875, + "loss": 1.1947, + "margin_dpo/margin_mean": 52.09856414794922, + "margin_dpo/margin_std": 105.56285095214844, + "step": 575 + }, + { + "KL/chosen_KL_mean": -116.5837631225586, + "KL/mean": -153.09439086914062, + "KL/rejected_KL_mean": -189.60504150390625, + "KL/std": 85.46895599365234, + "epoch": 0.8707482993197279, + "fcm_dpo/beta": 0.006358052603900433, + "fcm_dpo/delta": -0.06849232316017151, + "fcm_dpo/margin": 73.02128601074219, + "fcm_dpo/q_t": 0.39405471086502075, + "grad_norm": 13.795515060424805, + "learning_rate": 2.5417538653170754e-08, + "logits/chosen": 0.7595860958099365, + "logits/rejected": 0.6430308818817139, + "logps/chosen": -169.86428833007812, + "logps/ref_chosen": -53.28052520751953, + "logps/ref_rejected": -84.2000503540039, + "logps/rejected": -273.80511474609375, + "loss": 1.0624, + "margin_dpo/margin_mean": 73.02128601074219, + "margin_dpo/margin_std": 95.01397705078125, + "step": 576 + }, + { + "KL/chosen_KL_mean": -130.9795379638672, + "KL/mean": -157.20046997070312, + "KL/rejected_KL_mean": -183.42141723632812, + "KL/std": 84.2352523803711, + "epoch": 0.872260015117158, + "fcm_dpo/beta": 0.0064292605966329575, + "fcm_dpo/delta": 0.06507161259651184, + "fcm_dpo/margin": 52.44186782836914, + "fcm_dpo/q_t": 0.4226120114326477, + "grad_norm": 14.239356994628906, + "learning_rate": 2.4839802933393607e-08, + "logits/chosen": 0.7271114587783813, + "logits/rejected": 0.7114219665527344, + "logps/chosen": -193.30422973632812, + "logps/ref_chosen": -62.32468795776367, + "logps/ref_rejected": -67.300537109375, + "logps/rejected": -250.72193908691406, + "loss": 1.17, + "margin_dpo/margin_mean": 52.441864013671875, + "margin_dpo/margin_std": 95.66375732421875, + "step": 577 + }, + { + "KL/chosen_KL_mean": -125.42353820800781, + "KL/mean": -150.02896118164062, + "KL/rejected_KL_mean": -174.6343994140625, + "KL/std": 85.88644409179688, + "epoch": 0.873771730914588, + "fcm_dpo/beta": 0.006528651341795921, + "fcm_dpo/delta": 0.08139324188232422, + "fcm_dpo/margin": 49.21087646484375, + "fcm_dpo/q_t": 0.42820054292678833, + "grad_norm": 16.858417510986328, + "learning_rate": 2.4268365428344733e-08, + "logits/chosen": 0.8017420172691345, + "logits/rejected": 0.7788885831832886, + "logps/chosen": -182.0791015625, + "logps/ref_chosen": -56.65557861328125, + "logps/ref_rejected": -68.21835327148438, + "logps/rejected": -242.85275268554688, + "loss": 1.1902, + "margin_dpo/margin_mean": 49.21087646484375, + "margin_dpo/margin_std": 99.24673461914062, + "step": 578 + }, + { + "KL/chosen_KL_mean": -131.38169860839844, + "KL/mean": -167.9196014404297, + "KL/rejected_KL_mean": -204.45751953125, + "KL/std": 82.55770111083984, + "epoch": 0.8752834467120182, + "fcm_dpo/beta": 0.006479881703853607, + "fcm_dpo/delta": -0.07728934288024902, + "fcm_dpo/margin": 73.0758056640625, + "fcm_dpo/q_t": 0.3899138271808624, + "grad_norm": 13.836790084838867, + "learning_rate": 2.3703242122359357e-08, + "logits/chosen": 0.6977138519287109, + "logits/rejected": 0.6707988381385803, + "logps/chosen": -188.1913604736328, + "logps/ref_chosen": -56.809661865234375, + "logps/ref_rejected": -68.09613037109375, + "logps/rejected": -272.55364990234375, + "loss": 1.0382, + "margin_dpo/margin_mean": 73.0758056640625, + "margin_dpo/margin_std": 84.57794189453125, + "step": 579 + }, + { + "KL/chosen_KL_mean": -131.20523071289062, + "KL/mean": -160.20413208007812, + "KL/rejected_KL_mean": -189.20303344726562, + "KL/std": 90.06289672851562, + "epoch": 0.8767951625094482, + "fcm_dpo/beta": 0.00650765048339963, + "fcm_dpo/delta": 0.02309592440724373, + "fcm_dpo/margin": 57.997798919677734, + "fcm_dpo/q_t": 0.4151464104652405, + "grad_norm": 13.468850135803223, + "learning_rate": 2.3144448823151392e-08, + "logits/chosen": 0.7023211717605591, + "logits/rejected": 0.6415441632270813, + "logps/chosen": -188.9053497314453, + "logps/ref_chosen": -57.70011520385742, + "logps/ref_rejected": -77.90664672851562, + "logps/rejected": -267.10968017578125, + "loss": 1.1525, + "margin_dpo/margin_mean": 57.997802734375, + "margin_dpo/margin_std": 103.89713287353516, + "step": 580 + }, + { + "KL/chosen_KL_mean": -142.56919860839844, + "KL/mean": -172.34552001953125, + "KL/rejected_KL_mean": -202.12184143066406, + "KL/std": 82.92886352539062, + "epoch": 0.8783068783068783, + "fcm_dpo/beta": 0.006482171826064587, + "fcm_dpo/delta": 0.014322012662887573, + "fcm_dpo/margin": 59.552635192871094, + "fcm_dpo/q_t": 0.4122830033302307, + "grad_norm": 14.482420921325684, + "learning_rate": 2.259200116137039e-08, + "logits/chosen": 0.7224099636077881, + "logits/rejected": 0.6515974998474121, + "logps/chosen": -201.90155029296875, + "logps/ref_chosen": -59.332359313964844, + "logps/ref_rejected": -83.64482116699219, + "logps/rejected": -285.76666259765625, + "loss": 1.1249, + "margin_dpo/margin_mean": 59.55262756347656, + "margin_dpo/margin_std": 93.51412963867188, + "step": 581 + }, + { + "KL/chosen_KL_mean": -127.24820709228516, + "KL/mean": -157.314453125, + "KL/rejected_KL_mean": -187.38070678710938, + "KL/std": 83.08796691894531, + "epoch": 0.8798185941043084, + "fcm_dpo/beta": 0.006529543083161116, + "fcm_dpo/delta": 0.00757815595716238, + "fcm_dpo/margin": 60.132503509521484, + "fcm_dpo/q_t": 0.4105338454246521, + "grad_norm": 11.920287132263184, + "learning_rate": 2.204591459016525e-08, + "logits/chosen": 0.7106418609619141, + "logits/rejected": 0.7402459383010864, + "logps/chosen": -191.41107177734375, + "logps/ref_chosen": -64.16285705566406, + "logps/ref_rejected": -58.632896423339844, + "logps/rejected": -246.01361083984375, + "loss": 1.121, + "margin_dpo/margin_mean": 60.13249969482422, + "margin_dpo/margin_std": 94.03811645507812, + "step": 582 + }, + { + "KL/chosen_KL_mean": -128.94577026367188, + "KL/mean": -162.572998046875, + "KL/rejected_KL_mean": -196.20022583007812, + "KL/std": 88.48190307617188, + "epoch": 0.8813303099017384, + "fcm_dpo/beta": 0.006483516190201044, + "fcm_dpo/delta": -0.03768821805715561, + "fcm_dpo/margin": 67.25444793701172, + "fcm_dpo/q_t": 0.40098488330841064, + "grad_norm": 15.756782531738281, + "learning_rate": 2.1506204384751064e-08, + "logits/chosen": 0.8333076238632202, + "logits/rejected": 0.7157968282699585, + "logps/chosen": -180.81817626953125, + "logps/ref_chosen": -51.87239456176758, + "logps/ref_rejected": -83.86331176757812, + "logps/rejected": -280.06353759765625, + "loss": 1.107, + "margin_dpo/margin_mean": 67.25444030761719, + "margin_dpo/margin_std": 105.29559326171875, + "step": 583 + }, + { + "KL/chosen_KL_mean": -117.70571899414062, + "KL/mean": -148.47128295898438, + "KL/rejected_KL_mean": -179.23681640625, + "KL/std": 83.452880859375, + "epoch": 0.8828420256991686, + "fcm_dpo/beta": 0.006448123138397932, + "fcm_dpo/delta": 0.0030598118901252747, + "fcm_dpo/margin": 61.531105041503906, + "fcm_dpo/q_t": 0.4117351174354553, + "grad_norm": 12.771221160888672, + "learning_rate": 2.09728856419826e-08, + "logits/chosen": 0.8767859935760498, + "logits/rejected": 0.7617365717887878, + "logps/chosen": -164.277099609375, + "logps/ref_chosen": -46.571388244628906, + "logps/ref_rejected": -80.67969512939453, + "logps/rejected": -259.91650390625, + "loss": 1.1436, + "margin_dpo/margin_mean": 61.531105041503906, + "margin_dpo/margin_std": 108.26390838623047, + "step": 584 + }, + { + "KL/chosen_KL_mean": -136.58163452148438, + "KL/mean": -158.10357666015625, + "KL/rejected_KL_mean": -179.6255340576172, + "KL/std": 89.20777893066406, + "epoch": 0.8843537414965986, + "fcm_dpo/beta": 0.006614618003368378, + "fcm_dpo/delta": 0.11793427169322968, + "fcm_dpo/margin": 43.04389953613281, + "fcm_dpo/q_t": 0.4342869818210602, + "grad_norm": 13.342413902282715, + "learning_rate": 2.044597327993153e-08, + "logits/chosen": 0.7053936719894409, + "logits/rejected": 0.6552038788795471, + "logps/chosen": -194.7061767578125, + "logps/ref_chosen": -58.124534606933594, + "logps/ref_rejected": -79.00538635253906, + "logps/rejected": -258.63092041015625, + "loss": 1.2109, + "margin_dpo/margin_mean": 43.04389953613281, + "margin_dpo/margin_std": 91.91615295410156, + "step": 585 + }, + { + "KL/chosen_KL_mean": -128.1500244140625, + "KL/mean": -160.58642578125, + "KL/rejected_KL_mean": -193.02284240722656, + "KL/std": 80.80349731445312, + "epoch": 0.8858654572940288, + "fcm_dpo/beta": 0.006581123918294907, + "fcm_dpo/delta": -0.02841946855187416, + "fcm_dpo/margin": 64.87281799316406, + "fcm_dpo/q_t": 0.40084946155548096, + "grad_norm": 17.90386199951172, + "learning_rate": 1.9925482037469187e-08, + "logits/chosen": 0.7673027515411377, + "logits/rejected": 0.7177405953407288, + "logps/chosen": -182.25164794921875, + "logps/ref_chosen": -54.10163879394531, + "logps/ref_rejected": -63.72113037109375, + "logps/rejected": -256.74395751953125, + "loss": 1.0596, + "margin_dpo/margin_mean": 64.87281799316406, + "margin_dpo/margin_std": 73.49497985839844, + "step": 586 + }, + { + "KL/chosen_KL_mean": -130.09133911132812, + "KL/mean": -161.97439575195312, + "KL/rejected_KL_mean": -193.85745239257812, + "KL/std": 88.5246810913086, + "epoch": 0.8873771730914588, + "fcm_dpo/beta": 0.006571411155164242, + "fcm_dpo/delta": -0.019869251176714897, + "fcm_dpo/margin": 63.76612091064453, + "fcm_dpo/q_t": 0.40272602438926697, + "grad_norm": 14.561052322387695, + "learning_rate": 1.9411426473854687e-08, + "logits/chosen": 0.7784076929092407, + "logits/rejected": 0.7722544074058533, + "logps/chosen": -193.508544921875, + "logps/ref_chosen": -63.41719436645508, + "logps/ref_rejected": -63.47003936767578, + "logps/rejected": -257.3275146484375, + "loss": 1.1402, + "margin_dpo/margin_mean": 63.76612091064453, + "margin_dpo/margin_std": 114.6404800415039, + "step": 587 + }, + { + "KL/chosen_KL_mean": -132.00868225097656, + "KL/mean": -167.29954528808594, + "KL/rejected_KL_mean": -202.5904083251953, + "KL/std": 87.38584899902344, + "epoch": 0.8888888888888888, + "fcm_dpo/beta": 0.006490037776529789, + "fcm_dpo/delta": -0.06114605814218521, + "fcm_dpo/margin": 70.58171844482422, + "fcm_dpo/q_t": 0.3969864845275879, + "grad_norm": 14.922515869140625, + "learning_rate": 1.890382096832699e-08, + "logits/chosen": 0.7928558588027954, + "logits/rejected": 0.7458092570304871, + "logps/chosen": -194.209716796875, + "logps/ref_chosen": -62.20103454589844, + "logps/ref_rejected": -82.10249328613281, + "logps/rejected": -284.6929016113281, + "loss": 1.0833, + "margin_dpo/margin_mean": 70.58171844482422, + "margin_dpo/margin_std": 103.22766876220703, + "step": 588 + }, + { + "KL/chosen_KL_mean": -125.28984069824219, + "KL/mean": -159.66366577148438, + "KL/rejected_KL_mean": -194.03750610351562, + "KL/std": 86.01371002197266, + "epoch": 0.890400604686319, + "fcm_dpo/beta": 0.0064490921795368195, + "fcm_dpo/delta": -0.04534650966525078, + "fcm_dpo/margin": 68.74765014648438, + "fcm_dpo/q_t": 0.39677882194519043, + "grad_norm": 11.68237018585205, + "learning_rate": 1.840267971970344e-08, + "logits/chosen": 0.7326580286026001, + "logits/rejected": 0.7025114297866821, + "logps/chosen": -182.00344848632812, + "logps/ref_chosen": -56.71361541748047, + "logps/ref_rejected": -76.7366943359375, + "logps/rejected": -270.774169921875, + "loss": 1.0496, + "margin_dpo/margin_mean": 68.7476577758789, + "margin_dpo/margin_std": 77.22756958007812, + "step": 589 + }, + { + "KL/chosen_KL_mean": -139.61033630371094, + "KL/mean": -172.72789001464844, + "KL/rejected_KL_mean": -205.84544372558594, + "KL/std": 81.31230163574219, + "epoch": 0.891912320483749, + "fcm_dpo/beta": 0.006417558528482914, + "fcm_dpo/delta": -0.026233412325382233, + "fcm_dpo/margin": 66.235107421875, + "fcm_dpo/q_t": 0.40231773257255554, + "grad_norm": 14.054452896118164, + "learning_rate": 1.7908016745981856e-08, + "logits/chosen": 0.6718609929084778, + "logits/rejected": 0.6410232782363892, + "logps/chosen": -206.1241455078125, + "logps/ref_chosen": -66.5138168334961, + "logps/ref_rejected": -85.70820617675781, + "logps/rejected": -291.55364990234375, + "loss": 1.0852, + "margin_dpo/margin_mean": 66.235107421875, + "margin_dpo/margin_std": 90.15684509277344, + "step": 590 + }, + { + "KL/chosen_KL_mean": -118.54312896728516, + "KL/mean": -158.638916015625, + "KL/rejected_KL_mean": -198.73472595214844, + "KL/std": 91.9343490600586, + "epoch": 0.8934240362811792, + "fcm_dpo/beta": 0.00624478654935956, + "fcm_dpo/delta": -0.10805796831846237, + "fcm_dpo/margin": 80.19159698486328, + "fcm_dpo/q_t": 0.3870072066783905, + "grad_norm": 15.261886596679688, + "learning_rate": 1.7419845883949098e-08, + "logits/chosen": 0.788226842880249, + "logits/rejected": 0.7209928035736084, + "logps/chosen": -179.2403106689453, + "logps/ref_chosen": -60.697181701660156, + "logps/ref_rejected": -86.12278747558594, + "logps/rejected": -284.8575134277344, + "loss": 1.0635, + "margin_dpo/margin_mean": 80.19159698486328, + "margin_dpo/margin_std": 111.43367004394531, + "step": 591 + }, + { + "KL/chosen_KL_mean": -129.67877197265625, + "KL/mean": -159.23971557617188, + "KL/rejected_KL_mean": -188.8006591796875, + "KL/std": 81.70418548583984, + "epoch": 0.8949357520786092, + "fcm_dpo/beta": 0.006255676969885826, + "fcm_dpo/delta": 0.030980493873357773, + "fcm_dpo/margin": 59.121883392333984, + "fcm_dpo/q_t": 0.41672664880752563, + "grad_norm": 15.047146797180176, + "learning_rate": 1.6938180788793556e-08, + "logits/chosen": 0.8125218152999878, + "logits/rejected": 0.6916435956954956, + "logps/chosen": -180.91610717773438, + "logps/ref_chosen": -51.237327575683594, + "logps/ref_rejected": -81.60242462158203, + "logps/rejected": -270.403076171875, + "loss": 1.1297, + "margin_dpo/margin_mean": 59.12188720703125, + "margin_dpo/margin_std": 91.02842712402344, + "step": 592 + }, + { + "KL/chosen_KL_mean": -109.8043441772461, + "KL/mean": -141.08143615722656, + "KL/rejected_KL_mean": -172.35853576660156, + "KL/std": 84.07896423339844, + "epoch": 0.8964474678760394, + "fcm_dpo/beta": 0.006288354285061359, + "fcm_dpo/delta": 0.006823260337114334, + "fcm_dpo/margin": 62.55418395996094, + "fcm_dpo/q_t": 0.4100233316421509, + "grad_norm": 15.268369674682617, + "learning_rate": 1.6463034933723336e-08, + "logits/chosen": 0.7873135805130005, + "logits/rejected": 0.6847280859947205, + "logps/chosen": -151.88433837890625, + "logps/ref_chosen": -42.08000183105469, + "logps/ref_rejected": -68.47499084472656, + "logps/rejected": -240.83352661132812, + "loss": 1.1211, + "margin_dpo/margin_mean": 62.55418395996094, + "margin_dpo/margin_std": 97.87245178222656, + "step": 593 + }, + { + "KL/chosen_KL_mean": -137.69334411621094, + "KL/mean": -166.93182373046875, + "KL/rejected_KL_mean": -196.17031860351562, + "KL/std": 82.94169616699219, + "epoch": 0.8979591836734694, + "fcm_dpo/beta": 0.006330487318336964, + "fcm_dpo/delta": 0.03096877783536911, + "fcm_dpo/margin": 58.47697067260742, + "fcm_dpo/q_t": 0.4138473868370056, + "grad_norm": 13.774144172668457, + "learning_rate": 1.5994421609589385e-08, + "logits/chosen": 0.680525541305542, + "logits/rejected": 0.6660400629043579, + "logps/chosen": -201.35202026367188, + "logps/ref_chosen": -63.658668518066406, + "logps/ref_rejected": -70.35597229003906, + "logps/rejected": -266.52630615234375, + "loss": 1.12, + "margin_dpo/margin_mean": 58.476966857910156, + "margin_dpo/margin_std": 85.59440612792969, + "step": 594 + }, + { + "KL/chosen_KL_mean": -124.25723266601562, + "KL/mean": -162.75222778320312, + "KL/rejected_KL_mean": -201.24722290039062, + "KL/std": 87.26791381835938, + "epoch": 0.8994708994708994, + "fcm_dpo/beta": 0.006278345361351967, + "fcm_dpo/delta": -0.08751600980758667, + "fcm_dpo/margin": 76.98997497558594, + "fcm_dpo/q_t": 0.3899436593055725, + "grad_norm": 11.65858268737793, + "learning_rate": 1.553235392451377e-08, + "logits/chosen": 0.8071564435958862, + "logits/rejected": 0.7116265296936035, + "logps/chosen": -180.47598266601562, + "logps/ref_chosen": -56.21875762939453, + "logps/ref_rejected": -83.95773315429688, + "logps/rejected": -285.2049560546875, + "loss": 1.0656, + "margin_dpo/margin_mean": 76.98997497558594, + "margin_dpo/margin_std": 107.02861785888672, + "step": 595 + }, + { + "KL/chosen_KL_mean": -143.4345703125, + "KL/mean": -161.5347900390625, + "KL/rejected_KL_mean": -179.63502502441406, + "KL/std": 82.03053283691406, + "epoch": 0.9009826152683296, + "fcm_dpo/beta": 0.00638097058981657, + "fcm_dpo/delta": 0.1733156442642212, + "fcm_dpo/margin": 36.20046615600586, + "fcm_dpo/q_t": 0.44862306118011475, + "grad_norm": 12.725683212280273, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": 0.6726224422454834, + "logits/rejected": 0.6943622827529907, + "logps/chosen": -211.91543579101562, + "logps/ref_chosen": -68.48088073730469, + "logps/ref_rejected": -61.732967376708984, + "logps/rejected": -241.36798095703125, + "loss": 1.2479, + "margin_dpo/margin_mean": 36.200462341308594, + "margin_dpo/margin_std": 89.72453308105469, + "step": 596 + }, + { + "KL/chosen_KL_mean": -113.01495361328125, + "KL/mean": -142.4587860107422, + "KL/rejected_KL_mean": -171.9026336669922, + "KL/std": 75.85586547851562, + "epoch": 0.9024943310657596, + "fcm_dpo/beta": 0.0064786700531840324, + "fcm_dpo/delta": 0.019177531823515892, + "fcm_dpo/margin": 58.8876838684082, + "fcm_dpo/q_t": 0.41371387243270874, + "grad_norm": 11.908020973205566, + "learning_rate": 1.4627906988186111e-08, + "logits/chosen": 0.7234436273574829, + "logits/rejected": 0.7067715525627136, + "logps/chosen": -161.87246704101562, + "logps/ref_chosen": -48.85750961303711, + "logps/ref_rejected": -55.068084716796875, + "logps/rejected": -226.97071838378906, + "loss": 1.1269, + "margin_dpo/margin_mean": 58.8876838684082, + "margin_dpo/margin_std": 93.37579345703125, + "step": 597 + }, + { + "KL/chosen_KL_mean": -147.4332733154297, + "KL/mean": -165.73919677734375, + "KL/rejected_KL_mean": -184.04510498046875, + "KL/std": 84.73491668701172, + "epoch": 0.9040060468631897, + "fcm_dpo/beta": 0.006653393618762493, + "fcm_dpo/delta": 0.15970611572265625, + "fcm_dpo/margin": 36.611839294433594, + "fcm_dpo/q_t": 0.44547873735427856, + "grad_norm": 14.159972190856934, + "learning_rate": 1.4185553036259095e-08, + "logits/chosen": 0.7855315208435059, + "logits/rejected": 0.6991105079650879, + "logps/chosen": -206.3204345703125, + "logps/ref_chosen": -58.88715362548828, + "logps/ref_rejected": -81.43145751953125, + "logps/rejected": -265.4765625, + "loss": 1.2481, + "margin_dpo/margin_mean": 36.611839294433594, + "margin_dpo/margin_std": 92.60962677001953, + "step": 598 + }, + { + "KL/chosen_KL_mean": -148.88742065429688, + "KL/mean": -171.52340698242188, + "KL/rejected_KL_mean": -194.159423828125, + "KL/std": 83.06340026855469, + "epoch": 0.9055177626606198, + "fcm_dpo/beta": 0.00680171512067318, + "fcm_dpo/delta": 0.09460268169641495, + "fcm_dpo/margin": 45.272010803222656, + "fcm_dpo/q_t": 0.43099159002304077, + "grad_norm": 16.28043556213379, + "learning_rate": 1.3749795321332885e-08, + "logits/chosen": 0.810226321220398, + "logits/rejected": 0.7629626989364624, + "logps/chosen": -206.49459838867188, + "logps/ref_chosen": -57.60719299316406, + "logps/ref_rejected": -71.80469512939453, + "logps/rejected": -265.964111328125, + "loss": 1.1985, + "margin_dpo/margin_mean": 45.272010803222656, + "margin_dpo/margin_std": 93.96842193603516, + "step": 599 + }, + { + "KL/chosen_KL_mean": -136.0979766845703, + "KL/mean": -162.57826232910156, + "KL/rejected_KL_mean": -189.0585479736328, + "KL/std": 87.23487854003906, + "epoch": 0.9070294784580499, + "fcm_dpo/beta": 0.006898914463818073, + "fcm_dpo/delta": 0.03474727272987366, + "fcm_dpo/margin": 52.960575103759766, + "fcm_dpo/q_t": 0.4179023802280426, + "grad_norm": 15.556018829345703, + "learning_rate": 1.3320646032487393e-08, + "logits/chosen": 0.807528018951416, + "logits/rejected": 0.7501277923583984, + "logps/chosen": -194.540283203125, + "logps/ref_chosen": -58.44231414794922, + "logps/ref_rejected": -83.64639282226562, + "logps/rejected": -272.7049560546875, + "loss": 1.1522, + "margin_dpo/margin_mean": 52.9605712890625, + "margin_dpo/margin_std": 91.92424011230469, + "step": 600 + }, + { + "KL/chosen_KL_mean": -119.36679077148438, + "KL/mean": -155.3492431640625, + "KL/rejected_KL_mean": -191.33169555664062, + "KL/std": 91.31021118164062, + "epoch": 0.90854119425548, + "fcm_dpo/beta": 0.006747450679540634, + "fcm_dpo/delta": -0.09054695814847946, + "fcm_dpo/margin": 71.96490478515625, + "fcm_dpo/q_t": 0.39126312732696533, + "grad_norm": 12.5004301071167, + "learning_rate": 1.2898117173950868e-08, + "logits/chosen": 0.7509094476699829, + "logits/rejected": 0.6697346568107605, + "logps/chosen": -174.96112060546875, + "logps/ref_chosen": -55.59432601928711, + "logps/ref_rejected": -83.68630981445312, + "logps/rejected": -275.01800537109375, + "loss": 1.0727, + "margin_dpo/margin_mean": 71.96490478515625, + "margin_dpo/margin_std": 103.93519592285156, + "step": 601 + }, + { + "KL/chosen_KL_mean": -107.9805679321289, + "KL/mean": -143.06297302246094, + "KL/rejected_KL_mean": -178.14535522460938, + "KL/std": 81.46908569335938, + "epoch": 0.91005291005291, + "fcm_dpo/beta": 0.006687416695058346, + "fcm_dpo/delta": -0.07258869707584381, + "fcm_dpo/margin": 70.16480255126953, + "fcm_dpo/q_t": 0.392657071352005, + "grad_norm": 15.200639724731445, + "learning_rate": 1.2482220564763667e-08, + "logits/chosen": 0.7669535875320435, + "logits/rejected": 0.7359898090362549, + "logps/chosen": -164.3297576904297, + "logps/ref_chosen": -56.349185943603516, + "logps/ref_rejected": -71.9959716796875, + "logps/rejected": -250.14132690429688, + "loss": 1.0545, + "margin_dpo/margin_mean": 70.16481018066406, + "margin_dpo/margin_std": 88.97705841064453, + "step": 602 + }, + { + "KL/chosen_KL_mean": -121.89750671386719, + "KL/mean": -154.79470825195312, + "KL/rejected_KL_mean": -187.69189453125, + "KL/std": 81.63348388671875, + "epoch": 0.9115646258503401, + "fcm_dpo/beta": 0.006597849540412426, + "fcm_dpo/delta": -0.03575563803315163, + "fcm_dpo/margin": 65.79436492919922, + "fcm_dpo/q_t": 0.40124818682670593, + "grad_norm": 15.745288848876953, + "learning_rate": 1.2072967838448051e-08, + "logits/chosen": 0.7055135369300842, + "logits/rejected": 0.6456756591796875, + "logps/chosen": -175.06588745117188, + "logps/ref_chosen": -53.16838836669922, + "logps/ref_rejected": -73.8604736328125, + "logps/rejected": -261.5523681640625, + "loss": 1.0937, + "margin_dpo/margin_mean": 65.79436492919922, + "margin_dpo/margin_std": 96.13505554199219, + "step": 603 + }, + { + "KL/chosen_KL_mean": -117.92399597167969, + "KL/mean": -147.4550018310547, + "KL/rejected_KL_mean": -176.9860076904297, + "KL/std": 78.23771667480469, + "epoch": 0.9130763416477702, + "fcm_dpo/beta": 0.006615637801587582, + "fcm_dpo/delta": 0.00958221685141325, + "fcm_dpo/margin": 59.06201934814453, + "fcm_dpo/q_t": 0.4119419753551483, + "grad_norm": 15.086995124816895, + "learning_rate": 1.1670370442682459e-08, + "logits/chosen": 0.7008262872695923, + "logits/rejected": 0.7061389684677124, + "logps/chosen": -190.57342529296875, + "logps/ref_chosen": -72.64942169189453, + "logps/ref_rejected": -69.8792724609375, + "logps/rejected": -246.8652801513672, + "loss": 1.132, + "margin_dpo/margin_mean": 59.06201934814453, + "margin_dpo/margin_std": 98.15986633300781, + "step": 604 + }, + { + "KL/chosen_KL_mean": -136.82138061523438, + "KL/mean": -165.74734497070312, + "KL/rejected_KL_mean": -194.67333984375, + "KL/std": 81.17981719970703, + "epoch": 0.9145880574452003, + "fcm_dpo/beta": 0.006611844524741173, + "fcm_dpo/delta": 0.0181251373142004, + "fcm_dpo/margin": 57.851959228515625, + "fcm_dpo/q_t": 0.41242408752441406, + "grad_norm": 14.910988807678223, + "learning_rate": 1.1274439638981532e-08, + "logits/chosen": 0.7845560312271118, + "logits/rejected": 0.7260922193527222, + "logps/chosen": -198.43421936035156, + "logps/ref_chosen": -61.61284637451172, + "logps/ref_rejected": -79.34398651123047, + "logps/rejected": -274.017333984375, + "loss": 1.1413, + "margin_dpo/margin_mean": 57.851959228515625, + "margin_dpo/margin_std": 98.74382019042969, + "step": 605 + }, + { + "KL/chosen_KL_mean": -119.74642944335938, + "KL/mean": -154.47683715820312, + "KL/rejected_KL_mean": -189.207275390625, + "KL/std": 82.59077453613281, + "epoch": 0.9160997732426304, + "fcm_dpo/beta": 0.00659895408898592, + "fcm_dpo/delta": -0.061252620071172714, + "fcm_dpo/margin": 69.46084594726562, + "fcm_dpo/q_t": 0.39719897508621216, + "grad_norm": 16.641035079956055, + "learning_rate": 1.0885186502381016e-08, + "logits/chosen": 0.7329899072647095, + "logits/rejected": 0.662407636642456, + "logps/chosen": -174.21066284179688, + "logps/ref_chosen": -54.46424102783203, + "logps/ref_rejected": -79.62708282470703, + "logps/rejected": -268.8343505859375, + "loss": 1.0787, + "margin_dpo/margin_mean": 69.46084594726562, + "margin_dpo/margin_std": 98.36412048339844, + "step": 606 + }, + { + "KL/chosen_KL_mean": -134.2374725341797, + "KL/mean": -165.53192138671875, + "KL/rejected_KL_mean": -196.82638549804688, + "KL/std": 83.826416015625, + "epoch": 0.9176114890400605, + "fcm_dpo/beta": 0.006492065265774727, + "fcm_dpo/delta": -0.007983453571796417, + "fcm_dpo/margin": 62.58891296386719, + "fcm_dpo/q_t": 0.407237708568573, + "grad_norm": 14.207694053649902, + "learning_rate": 1.0502621921127774e-08, + "logits/chosen": 0.6957761645317078, + "logits/rejected": 0.6701861023902893, + "logps/chosen": -197.09832763671875, + "logps/ref_chosen": -62.86086654663086, + "logps/ref_rejected": -72.5501937866211, + "logps/rejected": -269.3765869140625, + "loss": 1.1179, + "margin_dpo/margin_mean": 62.58891296386719, + "margin_dpo/margin_std": 95.68101501464844, + "step": 607 + }, + { + "KL/chosen_KL_mean": -135.67677307128906, + "KL/mean": -166.17120361328125, + "KL/rejected_KL_mean": -196.66561889648438, + "KL/std": 85.26414489746094, + "epoch": 0.9191232048374905, + "fcm_dpo/beta": 0.006548555102199316, + "fcm_dpo/delta": 0.0006168922409415245, + "fcm_dpo/margin": 60.98883819580078, + "fcm_dpo/q_t": 0.4089788794517517, + "grad_norm": 14.347258567810059, + "learning_rate": 1.0126756596375685e-08, + "logits/chosen": 0.7177504301071167, + "logits/rejected": 0.6351562738418579, + "logps/chosen": -198.8574981689453, + "logps/ref_chosen": -63.18071746826172, + "logps/ref_rejected": -99.15888214111328, + "logps/rejected": -295.82452392578125, + "loss": 1.1048, + "margin_dpo/margin_mean": 60.98883819580078, + "margin_dpo/margin_std": 88.06265258789062, + "step": 608 + }, + { + "KL/chosen_KL_mean": -120.43633270263672, + "KL/mean": -155.95809936523438, + "KL/rejected_KL_mean": -191.4798583984375, + "KL/std": 82.86610412597656, + "epoch": 0.9206349206349206, + "fcm_dpo/beta": 0.006455970928072929, + "fcm_dpo/delta": -0.062065087258815765, + "fcm_dpo/margin": 71.04353332519531, + "fcm_dpo/q_t": 0.3932182192802429, + "grad_norm": 12.620007514953613, + "learning_rate": 9.757601041885694e-09, + "logits/chosen": 0.7929271459579468, + "logits/rejected": 0.7561884522438049, + "logps/chosen": -169.05955505371094, + "logps/ref_chosen": -48.62322235107422, + "logps/ref_rejected": -68.28271484375, + "logps/rejected": -259.7625732421875, + "loss": 1.0536, + "margin_dpo/margin_mean": 71.04353332519531, + "margin_dpo/margin_std": 84.36087036132812, + "step": 609 + }, + { + "KL/chosen_KL_mean": -129.75857543945312, + "KL/mean": -164.24969482421875, + "KL/rejected_KL_mean": -198.74082946777344, + "KL/std": 86.79969024658203, + "epoch": 0.9221466364323507, + "fcm_dpo/beta": 0.0063972556963562965, + "fcm_dpo/delta": -0.04361305385828018, + "fcm_dpo/margin": 68.98225402832031, + "fcm_dpo/q_t": 0.40069544315338135, + "grad_norm": 14.097638130187988, + "learning_rate": 9.395165583732379e-09, + "logits/chosen": 0.699439525604248, + "logits/rejected": 0.696520209312439, + "logps/chosen": -202.4237060546875, + "logps/ref_chosen": -72.66513061523438, + "logps/ref_rejected": -87.15310668945312, + "logps/rejected": -285.8939208984375, + "loss": 1.0843, + "margin_dpo/margin_mean": 68.98225402832031, + "margin_dpo/margin_std": 97.63822937011719, + "step": 610 + }, + { + "KL/chosen_KL_mean": -122.30868530273438, + "KL/mean": -149.33563232421875, + "KL/rejected_KL_mean": -176.3625946044922, + "KL/std": 80.62203979492188, + "epoch": 0.9236583522297808, + "fcm_dpo/beta": 0.006440991070121527, + "fcm_dpo/delta": 0.053715676069259644, + "fcm_dpo/margin": 54.05390167236328, + "fcm_dpo/q_t": 0.41938698291778564, + "grad_norm": 14.460498809814453, + "learning_rate": 9.03946036001449e-09, + "logits/chosen": 0.7778803110122681, + "logits/rejected": 0.7268798351287842, + "logps/chosen": -170.6172637939453, + "logps/ref_chosen": -48.30857849121094, + "logps/ref_rejected": -70.6141128540039, + "logps/rejected": -246.97671508789062, + "loss": 1.1383, + "margin_dpo/margin_mean": 54.05390167236328, + "margin_dpo/margin_std": 83.94294738769531, + "step": 611 + }, + { + "KL/chosen_KL_mean": -132.1772003173828, + "KL/mean": -170.843994140625, + "KL/rejected_KL_mean": -209.51080322265625, + "KL/std": 84.51614379882812, + "epoch": 0.9251700680272109, + "fcm_dpo/beta": 0.006384260021150112, + "fcm_dpo/delta": -0.09853628277778625, + "fcm_dpo/margin": 77.33361053466797, + "fcm_dpo/q_t": 0.3858988583087921, + "grad_norm": 12.366726875305176, + "learning_rate": 8.690495320571839e-09, + "logits/chosen": 0.6507830619812012, + "logits/rejected": 0.5809448957443237, + "logps/chosen": -193.40875244140625, + "logps/ref_chosen": -61.23155975341797, + "logps/ref_rejected": -94.37979888916016, + "logps/rejected": -303.8905944824219, + "loss": 1.0366, + "margin_dpo/margin_mean": 77.33360290527344, + "margin_dpo/margin_std": 94.39988708496094, + "step": 612 + }, + { + "KL/chosen_KL_mean": -112.42122650146484, + "KL/mean": -149.91241455078125, + "KL/rejected_KL_mean": -187.40362548828125, + "KL/std": 86.88549041748047, + "epoch": 0.926681783824641, + "fcm_dpo/beta": 0.006296813488006592, + "fcm_dpo/delta": -0.07568641006946564, + "fcm_dpo/margin": 74.98237609863281, + "fcm_dpo/q_t": 0.3908718228340149, + "grad_norm": 11.499692916870117, + "learning_rate": 8.348280226706722e-09, + "logits/chosen": 0.7143006324768066, + "logits/rejected": 0.7107223272323608, + "logps/chosen": -166.40432739257812, + "logps/ref_chosen": -53.98310852050781, + "logps/ref_rejected": -58.32208251953125, + "logps/rejected": -245.7257080078125, + "loss": 1.0477, + "margin_dpo/margin_mean": 74.98237609863281, + "margin_dpo/margin_std": 91.57196044921875, + "step": 613 + }, + { + "KL/chosen_KL_mean": -132.50953674316406, + "KL/mean": -166.3426971435547, + "KL/rejected_KL_mean": -200.17584228515625, + "KL/std": 77.45066833496094, + "epoch": 0.9281934996220711, + "fcm_dpo/beta": 0.0062566944397985935, + "fcm_dpo/delta": -0.02466324158012867, + "fcm_dpo/margin": 67.66629028320312, + "fcm_dpo/q_t": 0.40125787258148193, + "grad_norm": 14.503677368164062, + "learning_rate": 8.012824650910937e-09, + "logits/chosen": 0.7738802433013916, + "logits/rejected": 0.7663772106170654, + "logps/chosen": -192.75257873535156, + "logps/ref_chosen": -60.24303436279297, + "logps/ref_rejected": -72.26258850097656, + "logps/rejected": -272.43841552734375, + "loss": 1.0817, + "margin_dpo/margin_mean": 67.66629028320312, + "margin_dpo/margin_std": 88.4701156616211, + "step": 614 + }, + { + "KL/chosen_KL_mean": -138.55084228515625, + "KL/mean": -171.46139526367188, + "KL/rejected_KL_mean": -204.37196350097656, + "KL/std": 94.35890197753906, + "epoch": 0.9297052154195011, + "fcm_dpo/beta": 0.006181590724736452, + "fcm_dpo/delta": -0.007811099290847778, + "fcm_dpo/margin": 65.82112121582031, + "fcm_dpo/q_t": 0.4073028564453125, + "grad_norm": 13.95335865020752, + "learning_rate": 7.684137976598088e-09, + "logits/chosen": 0.6753830909729004, + "logits/rejected": 0.6221166849136353, + "logps/chosen": -210.6455078125, + "logps/ref_chosen": -72.09467315673828, + "logps/ref_rejected": -104.02980041503906, + "logps/rejected": -308.4017639160156, + "loss": 1.1256, + "margin_dpo/margin_mean": 65.82112121582031, + "margin_dpo/margin_std": 106.96760559082031, + "step": 615 + }, + { + "KL/chosen_KL_mean": -130.0317840576172, + "KL/mean": -160.46640014648438, + "KL/rejected_KL_mean": -190.9010009765625, + "KL/std": 87.95735168457031, + "epoch": 0.9312169312169312, + "fcm_dpo/beta": 0.006221260875463486, + "fcm_dpo/delta": 0.02212386578321457, + "fcm_dpo/margin": 60.869224548339844, + "fcm_dpo/q_t": 0.41363078355789185, + "grad_norm": 12.034595489501953, + "learning_rate": 7.36222939784098e-09, + "logits/chosen": 0.7884162068367004, + "logits/rejected": 0.7024547457695007, + "logps/chosen": -188.5625, + "logps/ref_chosen": -58.530723571777344, + "logps/ref_rejected": -75.48025512695312, + "logps/rejected": -266.3812561035156, + "loss": 1.1211, + "margin_dpo/margin_mean": 60.869224548339844, + "margin_dpo/margin_std": 92.23234558105469, + "step": 616 + }, + { + "KL/chosen_KL_mean": -144.70751953125, + "KL/mean": -168.7492218017578, + "KL/rejected_KL_mean": -192.79092407226562, + "KL/std": 83.13668823242188, + "epoch": 0.9327286470143613, + "fcm_dpo/beta": 0.006288778968155384, + "fcm_dpo/delta": 0.10031160712242126, + "fcm_dpo/margin": 48.083404541015625, + "fcm_dpo/q_t": 0.430725634098053, + "grad_norm": 15.752001762390137, + "learning_rate": 7.047107919114586e-09, + "logits/chosen": 0.7634665966033936, + "logits/rejected": 0.7092599868774414, + "logps/chosen": -202.31619262695312, + "logps/ref_chosen": -57.608673095703125, + "logps/ref_rejected": -81.22109985351562, + "logps/rejected": -274.01202392578125, + "loss": 1.1847, + "margin_dpo/margin_mean": 48.083404541015625, + "margin_dpo/margin_std": 87.39576721191406, + "step": 617 + }, + { + "KL/chosen_KL_mean": -124.86834716796875, + "KL/mean": -155.3878936767578, + "KL/rejected_KL_mean": -185.90744018554688, + "KL/std": 89.1173095703125, + "epoch": 0.9342403628117913, + "fcm_dpo/beta": 0.006361059378832579, + "fcm_dpo/delta": 0.012086881324648857, + "fcm_dpo/margin": 61.039093017578125, + "fcm_dpo/q_t": 0.41178715229034424, + "grad_norm": 16.19603729248047, + "learning_rate": 6.738782355044048e-09, + "logits/chosen": 0.7317014932632446, + "logits/rejected": 0.6204960346221924, + "logps/chosen": -181.5642852783203, + "logps/ref_chosen": -56.69594192504883, + "logps/ref_rejected": -85.92362976074219, + "logps/rejected": -271.8310546875, + "loss": 1.1099, + "margin_dpo/margin_mean": 61.039093017578125, + "margin_dpo/margin_std": 88.63739013671875, + "step": 618 + }, + { + "KL/chosen_KL_mean": -123.36766815185547, + "KL/mean": -156.4314422607422, + "KL/rejected_KL_mean": -189.4952392578125, + "KL/std": 86.46694946289062, + "epoch": 0.9357520786092215, + "fcm_dpo/beta": 0.0063554528169333935, + "fcm_dpo/delta": -0.021202079951763153, + "fcm_dpo/margin": 66.1275634765625, + "fcm_dpo/q_t": 0.4031738042831421, + "grad_norm": 13.8381929397583, + "learning_rate": 6.437261330158206e-09, + "logits/chosen": 0.8231503963470459, + "logits/rejected": 0.7432982921600342, + "logps/chosen": -177.42608642578125, + "logps/ref_chosen": -54.05841827392578, + "logps/ref_rejected": -83.55493927001953, + "logps/rejected": -273.0501708984375, + "loss": 1.0951, + "margin_dpo/margin_mean": 66.1275634765625, + "margin_dpo/margin_std": 95.22664642333984, + "step": 619 + }, + { + "KL/chosen_KL_mean": -121.48786926269531, + "KL/mean": -149.679443359375, + "KL/rejected_KL_mean": -177.87103271484375, + "KL/std": 78.28753662109375, + "epoch": 0.9372637944066515, + "fcm_dpo/beta": 0.006452272646129131, + "fcm_dpo/delta": 0.03521667420864105, + "fcm_dpo/margin": 56.383155822753906, + "fcm_dpo/q_t": 0.4167051613330841, + "grad_norm": 14.916093826293945, + "learning_rate": 6.142553278648238e-09, + "logits/chosen": 0.7305669188499451, + "logits/rejected": 0.7315757274627686, + "logps/chosen": -184.8575897216797, + "logps/ref_chosen": -63.36971664428711, + "logps/ref_rejected": -65.68269348144531, + "logps/rejected": -243.5537109375, + "loss": 1.14, + "margin_dpo/margin_mean": 56.38315200805664, + "margin_dpo/margin_std": 87.23406982421875, + "step": 620 + }, + { + "KL/chosen_KL_mean": -125.88284301757812, + "KL/mean": -152.0810546875, + "KL/rejected_KL_mean": -178.27926635742188, + "KL/std": 81.47933959960938, + "epoch": 0.9387755102040817, + "fcm_dpo/beta": 0.006477131508290768, + "fcm_dpo/delta": 0.06243708357214928, + "fcm_dpo/margin": 52.39640426635742, + "fcm_dpo/q_t": 0.4232029914855957, + "grad_norm": 16.247522354125977, + "learning_rate": 5.854666444131934e-09, + "logits/chosen": 0.7784920334815979, + "logits/rejected": 0.6623125672340393, + "logps/chosen": -178.20407104492188, + "logps/ref_chosen": -52.321224212646484, + "logps/ref_rejected": -88.09001159667969, + "logps/rejected": -266.3692626953125, + "loss": 1.1679, + "margin_dpo/margin_mean": 52.39640808105469, + "margin_dpo/margin_std": 96.42157745361328, + "step": 621 + }, + { + "KL/chosen_KL_mean": -135.88552856445312, + "KL/mean": -165.81393432617188, + "KL/rejected_KL_mean": -195.74232482910156, + "KL/std": 86.95231628417969, + "epoch": 0.9402872260015117, + "fcm_dpo/beta": 0.006505551747977734, + "fcm_dpo/delta": 0.010898975655436516, + "fcm_dpo/margin": 59.85680389404297, + "fcm_dpo/q_t": 0.41105908155441284, + "grad_norm": 15.815767288208008, + "learning_rate": 5.573608879422875e-09, + "logits/chosen": 0.7089018821716309, + "logits/rejected": 0.6645277142524719, + "logps/chosen": -195.7509765625, + "logps/ref_chosen": -59.86545944213867, + "logps/ref_rejected": -81.86668395996094, + "logps/rejected": -277.6090087890625, + "loss": 1.1207, + "margin_dpo/margin_mean": 59.85680389404297, + "margin_dpo/margin_std": 92.72256469726562, + "step": 622 + }, + { + "KL/chosen_KL_mean": -144.07167053222656, + "KL/mean": -175.5872344970703, + "KL/rejected_KL_mean": -207.102783203125, + "KL/std": 89.51210021972656, + "epoch": 0.9417989417989417, + "fcm_dpo/beta": 0.006464548408985138, + "fcm_dpo/delta": -0.008051551878452301, + "fcm_dpo/margin": 63.0311164855957, + "fcm_dpo/q_t": 0.40689247846603394, + "grad_norm": 13.286748886108398, + "learning_rate": 5.299388446305342e-09, + "logits/chosen": 0.7314743399620056, + "logits/rejected": 0.6700347661972046, + "logps/chosen": -211.44012451171875, + "logps/ref_chosen": -67.36846160888672, + "logps/ref_rejected": -82.02733612060547, + "logps/rejected": -289.130126953125, + "loss": 1.1063, + "margin_dpo/margin_mean": 63.03112030029297, + "margin_dpo/margin_std": 93.41072082519531, + "step": 623 + }, + { + "KL/chosen_KL_mean": -121.00255584716797, + "KL/mean": -156.09371948242188, + "KL/rejected_KL_mean": -191.18487548828125, + "KL/std": 88.24122619628906, + "epoch": 0.9433106575963719, + "fcm_dpo/beta": 0.006418424658477306, + "fcm_dpo/delta": -0.05306580290198326, + "fcm_dpo/margin": 70.18232727050781, + "fcm_dpo/q_t": 0.39883553981781006, + "grad_norm": 16.014657974243164, + "learning_rate": 5.03201281531429e-09, + "logits/chosen": 0.7183233499526978, + "logits/rejected": 0.6201412677764893, + "logps/chosen": -172.02911376953125, + "logps/ref_chosen": -51.02655029296875, + "logps/ref_rejected": -76.49203491210938, + "logps/rejected": -267.6769104003906, + "loss": 1.0823, + "margin_dpo/margin_mean": 70.18232727050781, + "margin_dpo/margin_std": 100.0351333618164, + "step": 624 + }, + { + "KL/chosen_KL_mean": -124.61507415771484, + "KL/mean": -151.14584350585938, + "KL/rejected_KL_mean": -177.6766357421875, + "KL/std": 84.18926239013672, + "epoch": 0.9448223733938019, + "fcm_dpo/beta": 0.0064721377566456795, + "fcm_dpo/delta": 0.05858816206455231, + "fcm_dpo/margin": 53.061546325683594, + "fcm_dpo/q_t": 0.4235745668411255, + "grad_norm": 14.729056358337402, + "learning_rate": 4.7714894655209174e-09, + "logits/chosen": 0.8177133202552795, + "logits/rejected": 0.7256171107292175, + "logps/chosen": -178.82269287109375, + "logps/ref_chosen": -54.20761489868164, + "logps/ref_rejected": -84.93669128417969, + "logps/rejected": -262.61334228515625, + "loss": 1.1742, + "margin_dpo/margin_mean": 53.061546325683594, + "margin_dpo/margin_std": 102.11767578125, + "step": 625 + }, + { + "KL/chosen_KL_mean": -118.29896545410156, + "KL/mean": -153.6566619873047, + "KL/rejected_KL_mean": -189.01434326171875, + "KL/std": 87.5848388671875, + "epoch": 0.9463340891912321, + "fcm_dpo/beta": 0.006424276158213615, + "fcm_dpo/delta": -0.05705726146697998, + "fcm_dpo/margin": 70.71540069580078, + "fcm_dpo/q_t": 0.3995022177696228, + "grad_norm": 13.807735443115234, + "learning_rate": 4.517825684323323e-09, + "logits/chosen": 0.8485870361328125, + "logits/rejected": 0.7073635458946228, + "logps/chosen": -163.36099243164062, + "logps/ref_chosen": -45.06201934814453, + "logps/ref_rejected": -89.66368103027344, + "logps/rejected": -278.67803955078125, + "loss": 1.0926, + "margin_dpo/margin_mean": 70.71539306640625, + "margin_dpo/margin_std": 107.69230651855469, + "step": 626 + }, + { + "KL/chosen_KL_mean": -133.79067993164062, + "KL/mean": -171.92047119140625, + "KL/rejected_KL_mean": -210.05026245117188, + "KL/std": 89.4090347290039, + "epoch": 0.9478458049886621, + "fcm_dpo/beta": 0.006317706312984228, + "fcm_dpo/delta": -0.08633463829755783, + "fcm_dpo/margin": 76.25958251953125, + "fcm_dpo/q_t": 0.38901180028915405, + "grad_norm": 14.114999771118164, + "learning_rate": 4.271028567242818e-09, + "logits/chosen": 0.6920174360275269, + "logits/rejected": 0.5662950277328491, + "logps/chosen": -192.58172607421875, + "logps/ref_chosen": -58.791053771972656, + "logps/ref_rejected": -94.90802001953125, + "logps/rejected": -304.9582824707031, + "loss": 1.0548, + "margin_dpo/margin_mean": 76.25958251953125, + "margin_dpo/margin_std": 99.61177062988281, + "step": 627 + }, + { + "KL/chosen_KL_mean": -119.68309020996094, + "KL/mean": -158.0455780029297, + "KL/rejected_KL_mean": -196.40806579589844, + "KL/std": 86.85939025878906, + "epoch": 0.9493575207860923, + "fcm_dpo/beta": 0.006282473914325237, + "fcm_dpo/delta": -0.08701348304748535, + "fcm_dpo/margin": 76.7249755859375, + "fcm_dpo/q_t": 0.388563334941864, + "grad_norm": 18.08017349243164, + "learning_rate": 4.0311050177251895e-09, + "logits/chosen": 0.734749436378479, + "logits/rejected": 0.6959051489830017, + "logps/chosen": -172.48666381835938, + "logps/ref_chosen": -52.80357360839844, + "logps/ref_rejected": -76.49468994140625, + "logps/rejected": -272.90277099609375, + "loss": 1.0688, + "margin_dpo/margin_mean": 76.7249755859375, + "margin_dpo/margin_std": 100.08718872070312, + "step": 628 + }, + { + "KL/chosen_KL_mean": -129.71226501464844, + "KL/mean": -157.51455688476562, + "KL/rejected_KL_mean": -185.31686401367188, + "KL/std": 83.63412475585938, + "epoch": 0.9508692365835223, + "fcm_dpo/beta": 0.006266086827963591, + "fcm_dpo/delta": 0.0531187504529953, + "fcm_dpo/margin": 55.60459518432617, + "fcm_dpo/q_t": 0.4206107258796692, + "grad_norm": 11.870978355407715, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": 0.7496498823165894, + "logits/rejected": 0.7434634566307068, + "logps/chosen": -200.42974853515625, + "logps/ref_chosen": -70.71749877929688, + "logps/ref_rejected": -78.96273803710938, + "logps/rejected": -264.27960205078125, + "loss": 1.1358, + "margin_dpo/margin_mean": 55.604591369628906, + "margin_dpo/margin_std": 84.9172134399414, + "step": 629 + }, + { + "KL/chosen_KL_mean": -122.6483154296875, + "KL/mean": -159.9776611328125, + "KL/rejected_KL_mean": -197.30702209472656, + "KL/std": 89.26514434814453, + "epoch": 0.9523809523809523, + "fcm_dpo/beta": 0.006218547932803631, + "fcm_dpo/delta": -0.0673595741391182, + "fcm_dpo/margin": 74.65872192382812, + "fcm_dpo/q_t": 0.3939681053161621, + "grad_norm": 10.882546424865723, + "learning_rate": 3.5719052736323806e-09, + "logits/chosen": 0.7177826166152954, + "logits/rejected": 0.671391487121582, + "logps/chosen": -178.8497314453125, + "logps/ref_chosen": -56.201412200927734, + "logps/ref_rejected": -74.69807434082031, + "logps/rejected": -272.0050964355469, + "loss": 1.0578, + "margin_dpo/margin_mean": 74.65872192382812, + "margin_dpo/margin_std": 95.45807647705078, + "step": 630 + }, + { + "KL/chosen_KL_mean": -117.01614379882812, + "KL/mean": -158.29913330078125, + "KL/rejected_KL_mean": -199.58212280273438, + "KL/std": 91.83998107910156, + "epoch": 0.9538926681783825, + "fcm_dpo/beta": 0.0060370988212525845, + "fcm_dpo/delta": -0.1057269275188446, + "fcm_dpo/margin": 82.56596374511719, + "fcm_dpo/q_t": 0.38690662384033203, + "grad_norm": 13.035811424255371, + "learning_rate": 3.352641923861144e-09, + "logits/chosen": 0.8515388369560242, + "logits/rejected": 0.7325578331947327, + "logps/chosen": -175.8367462158203, + "logps/ref_chosen": -58.82059860229492, + "logps/ref_rejected": -96.51437377929688, + "logps/rejected": -296.09649658203125, + "loss": 1.0445, + "margin_dpo/margin_mean": 82.56597137451172, + "margin_dpo/margin_std": 103.42387390136719, + "step": 631 + }, + { + "KL/chosen_KL_mean": -119.71453857421875, + "KL/mean": -160.3410186767578, + "KL/rejected_KL_mean": -200.96749877929688, + "KL/std": 84.33313751220703, + "epoch": 0.9554043839758125, + "fcm_dpo/beta": 0.0059857675805687904, + "fcm_dpo/delta": -0.09067746251821518, + "fcm_dpo/margin": 81.25294494628906, + "fcm_dpo/q_t": 0.3871810734272003, + "grad_norm": 12.395666122436523, + "learning_rate": 3.140277830901428e-09, + "logits/chosen": 0.7373045682907104, + "logits/rejected": 0.7179843187332153, + "logps/chosen": -178.50059509277344, + "logps/ref_chosen": -58.786048889160156, + "logps/ref_rejected": -67.21923828125, + "logps/rejected": -268.1867370605469, + "loss": 1.0324, + "margin_dpo/margin_mean": 81.25294494628906, + "margin_dpo/margin_std": 93.44187927246094, + "step": 632 + }, + { + "KL/chosen_KL_mean": -112.61619567871094, + "KL/mean": -143.66635131835938, + "KL/rejected_KL_mean": -174.71653747558594, + "KL/std": 80.20565032958984, + "epoch": 0.9569160997732427, + "fcm_dpo/beta": 0.005983942188322544, + "fcm_dpo/delta": 0.029293209314346313, + "fcm_dpo/margin": 62.1003303527832, + "fcm_dpo/q_t": 0.41520097851753235, + "grad_norm": 12.656014442443848, + "learning_rate": 2.9348189350335007e-09, + "logits/chosen": 0.7250348329544067, + "logits/rejected": 0.6618514060974121, + "logps/chosen": -164.74639892578125, + "logps/ref_chosen": -52.13019561767578, + "logps/ref_rejected": -67.23016357421875, + "logps/rejected": -241.94668579101562, + "loss": 1.1224, + "margin_dpo/margin_mean": 62.1003303527832, + "margin_dpo/margin_std": 92.83853149414062, + "step": 633 + }, + { + "KL/chosen_KL_mean": -141.18545532226562, + "KL/mean": -153.7686767578125, + "KL/rejected_KL_mean": -166.3519287109375, + "KL/std": 83.00604248046875, + "epoch": 0.9584278155706727, + "fcm_dpo/beta": 0.006214272230863571, + "fcm_dpo/delta": 0.24698540568351746, + "fcm_dpo/margin": 25.166465759277344, + "fcm_dpo/q_t": 0.46551811695098877, + "grad_norm": 15.885687828063965, + "learning_rate": 2.736270983384276e-09, + "logits/chosen": 0.8349080085754395, + "logits/rejected": 0.8475719690322876, + "logps/chosen": -202.16525268554688, + "logps/ref_chosen": -60.97979736328125, + "logps/ref_rejected": -58.50825119018555, + "logps/rejected": -224.86016845703125, + "loss": 1.3209, + "margin_dpo/margin_mean": 25.166465759277344, + "margin_dpo/margin_std": 95.85585021972656, + "step": 634 + }, + { + "KL/chosen_KL_mean": -152.8594970703125, + "KL/mean": -177.40724182128906, + "KL/rejected_KL_mean": -201.95498657226562, + "KL/std": 83.1712646484375, + "epoch": 0.9599395313681028, + "fcm_dpo/beta": 0.006378865800797939, + "fcm_dpo/delta": 0.08934434503316879, + "fcm_dpo/margin": 49.09549331665039, + "fcm_dpo/q_t": 0.42987093329429626, + "grad_norm": 14.362222671508789, + "learning_rate": 2.5446395297668287e-09, + "logits/chosen": 0.6214234828948975, + "logits/rejected": 0.5565516948699951, + "logps/chosen": -218.83251953125, + "logps/ref_chosen": -65.9730224609375, + "logps/ref_rejected": -85.61317443847656, + "logps/rejected": -287.5681457519531, + "loss": 1.2007, + "margin_dpo/margin_mean": 49.095489501953125, + "margin_dpo/margin_std": 103.11039733886719, + "step": 635 + }, + { + "KL/chosen_KL_mean": -120.62947082519531, + "KL/mean": -154.38397216796875, + "KL/rejected_KL_mean": -188.1384735107422, + "KL/std": 84.78205871582031, + "epoch": 0.9614512471655329, + "fcm_dpo/beta": 0.006347968243062496, + "fcm_dpo/delta": -0.030059967190027237, + "fcm_dpo/margin": 67.50898742675781, + "fcm_dpo/q_t": 0.4006476104259491, + "grad_norm": 11.630902290344238, + "learning_rate": 2.359929934524829e-09, + "logits/chosen": 0.7302621006965637, + "logits/rejected": 0.6309200525283813, + "logps/chosen": -169.76963806152344, + "logps/ref_chosen": -49.140167236328125, + "logps/ref_rejected": -81.26971435546875, + "logps/rejected": -269.4081726074219, + "loss": 1.08, + "margin_dpo/margin_mean": 67.50898742675781, + "margin_dpo/margin_std": 89.56611633300781, + "step": 636 + }, + { + "KL/chosen_KL_mean": -157.49581909179688, + "KL/mean": -182.8763427734375, + "KL/rejected_KL_mean": -208.25686645507812, + "KL/std": 80.46435546875, + "epoch": 0.9629629629629629, + "fcm_dpo/beta": 0.006448294967412949, + "fcm_dpo/delta": 0.07466259598731995, + "fcm_dpo/margin": 50.76104736328125, + "fcm_dpo/q_t": 0.42705270648002625, + "grad_norm": 15.661286354064941, + "learning_rate": 2.1821473643827137e-09, + "logits/chosen": 0.7203917503356934, + "logits/rejected": 0.648820161819458, + "logps/chosen": -231.19241333007812, + "logps/ref_chosen": -73.69658660888672, + "logps/ref_rejected": -83.01487731933594, + "logps/rejected": -291.2717590332031, + "loss": 1.1893, + "margin_dpo/margin_mean": 50.76104736328125, + "margin_dpo/margin_std": 103.28202819824219, + "step": 637 + }, + { + "KL/chosen_KL_mean": -138.66412353515625, + "KL/mean": -169.12185668945312, + "KL/rejected_KL_mean": -199.57955932617188, + "KL/std": 83.98857116699219, + "epoch": 0.9644746787603931, + "fcm_dpo/beta": 0.006465583108365536, + "fcm_dpo/delta": 0.0063573624938726425, + "fcm_dpo/margin": 60.91543960571289, + "fcm_dpo/q_t": 0.4109139144420624, + "grad_norm": 14.0470609664917, + "learning_rate": 2.0112967923011646e-09, + "logits/chosen": 0.728748083114624, + "logits/rejected": 0.6802102327346802, + "logps/chosen": -201.44570922851562, + "logps/ref_chosen": -62.78158187866211, + "logps/ref_rejected": -85.40478515625, + "logps/rejected": -284.9843444824219, + "loss": 1.1145, + "margin_dpo/margin_mean": 60.91543960571289, + "margin_dpo/margin_std": 92.47264862060547, + "step": 638 + }, + { + "KL/chosen_KL_mean": -124.04972076416016, + "KL/mean": -159.39202880859375, + "KL/rejected_KL_mean": -194.73435974121094, + "KL/std": 89.03472900390625, + "epoch": 0.9659863945578231, + "fcm_dpo/beta": 0.006414017640054226, + "fcm_dpo/delta": -0.055869266390800476, + "fcm_dpo/margin": 70.68461608886719, + "fcm_dpo/q_t": 0.39816033840179443, + "grad_norm": 14.064215660095215, + "learning_rate": 1.847382997337943e-09, + "logits/chosen": 0.7247157096862793, + "logits/rejected": 0.6204769611358643, + "logps/chosen": -177.81631469726562, + "logps/ref_chosen": -53.76658630371094, + "logps/ref_rejected": -72.30009460449219, + "logps/rejected": -267.0344543457031, + "loss": 1.0755, + "margin_dpo/margin_mean": 70.68461608886719, + "margin_dpo/margin_std": 98.21234893798828, + "step": 639 + }, + { + "KL/chosen_KL_mean": -128.9110870361328, + "KL/mean": -161.59844970703125, + "KL/rejected_KL_mean": -194.2858123779297, + "KL/std": 86.63613891601562, + "epoch": 0.9674981103552532, + "fcm_dpo/beta": 0.00635608471930027, + "fcm_dpo/delta": -0.01637229695916176, + "fcm_dpo/margin": 65.37472534179688, + "fcm_dpo/q_t": 0.40459680557250977, + "grad_norm": 13.13107967376709, + "learning_rate": 1.690410564514244e-09, + "logits/chosen": 0.7759475708007812, + "logits/rejected": 0.7121703624725342, + "logps/chosen": -180.328857421875, + "logps/ref_chosen": -51.41777801513672, + "logps/ref_rejected": -77.27879333496094, + "logps/rejected": -271.5646057128906, + "loss": 1.099, + "margin_dpo/margin_mean": 65.37472534179688, + "margin_dpo/margin_std": 94.90809631347656, + "step": 640 + }, + { + "KL/chosen_KL_mean": -136.29029846191406, + "KL/mean": -165.4718017578125, + "KL/rejected_KL_mean": -194.65328979492188, + "KL/std": 82.64945983886719, + "epoch": 0.9690098261526833, + "fcm_dpo/beta": 0.006381419487297535, + "fcm_dpo/delta": 0.028604013845324516, + "fcm_dpo/margin": 58.363006591796875, + "fcm_dpo/q_t": 0.41331246495246887, + "grad_norm": 13.78744125366211, + "learning_rate": 1.5403838846864692e-09, + "logits/chosen": 0.711683452129364, + "logits/rejected": 0.6908361911773682, + "logps/chosen": -207.344970703125, + "logps/ref_chosen": -71.0546646118164, + "logps/ref_rejected": -82.2440185546875, + "logps/rejected": -276.8973083496094, + "loss": 1.109, + "margin_dpo/margin_mean": 58.363006591796875, + "margin_dpo/margin_std": 79.44768524169922, + "step": 641 + }, + { + "KL/chosen_KL_mean": -146.58837890625, + "KL/mean": -167.76840209960938, + "KL/rejected_KL_mean": -188.9484100341797, + "KL/std": 79.4186782836914, + "epoch": 0.9705215419501134, + "fcm_dpo/beta": 0.00648085493594408, + "fcm_dpo/delta": 0.12887108325958252, + "fcm_dpo/margin": 42.36002731323242, + "fcm_dpo/q_t": 0.43818730115890503, + "grad_norm": 16.881349563598633, + "learning_rate": 1.3973071544233218e-09, + "logits/chosen": 0.7018405795097351, + "logits/rejected": 0.7198396921157837, + "logps/chosen": -215.51766967773438, + "logps/ref_chosen": -68.92927551269531, + "logps/ref_rejected": -70.85682678222656, + "logps/rejected": -259.80523681640625, + "loss": 1.23, + "margin_dpo/margin_mean": 42.36002731323242, + "margin_dpo/margin_std": 95.72892761230469, + "step": 642 + }, + { + "KL/chosen_KL_mean": -130.13673400878906, + "KL/mean": -163.25579833984375, + "KL/rejected_KL_mean": -196.3748779296875, + "KL/std": 84.13433837890625, + "epoch": 0.9720332577475435, + "fcm_dpo/beta": 0.0065269810147583485, + "fcm_dpo/delta": -0.03390323370695114, + "fcm_dpo/margin": 66.23814392089844, + "fcm_dpo/q_t": 0.40193212032318115, + "grad_norm": 20.258853912353516, + "learning_rate": 1.261184375888541e-09, + "logits/chosen": 0.6894593238830566, + "logits/rejected": 0.5981128215789795, + "logps/chosen": -195.44577026367188, + "logps/ref_chosen": -65.30903625488281, + "logps/ref_rejected": -83.61613464355469, + "logps/rejected": -279.99102783203125, + "loss": 1.0996, + "margin_dpo/margin_mean": 66.23814392089844, + "margin_dpo/margin_std": 99.40937805175781, + "step": 643 + }, + { + "KL/chosen_KL_mean": -121.13558959960938, + "KL/mean": -146.59364318847656, + "KL/rejected_KL_mean": -172.0517120361328, + "KL/std": 79.79216003417969, + "epoch": 0.9735449735449735, + "fcm_dpo/beta": 0.006630894262343645, + "fcm_dpo/delta": 0.06329631060361862, + "fcm_dpo/margin": 50.91610336303711, + "fcm_dpo/q_t": 0.4230857491493225, + "grad_norm": 13.730220794677734, + "learning_rate": 1.1320193567288527e-09, + "logits/chosen": 0.8315029144287109, + "logits/rejected": 0.7990458011627197, + "logps/chosen": -172.13818359375, + "logps/ref_chosen": -51.002601623535156, + "logps/ref_rejected": -64.46372985839844, + "logps/rejected": -236.51544189453125, + "loss": 1.1862, + "margin_dpo/margin_mean": 50.916099548339844, + "margin_dpo/margin_std": 100.9294204711914, + "step": 644 + }, + { + "KL/chosen_KL_mean": -130.87391662597656, + "KL/mean": -161.52090454101562, + "KL/rejected_KL_mean": -192.16786193847656, + "KL/std": 83.88922119140625, + "epoch": 0.9750566893424036, + "fcm_dpo/beta": 0.006598624866455793, + "fcm_dpo/delta": -0.004665223415941, + "fcm_dpo/margin": 61.29396057128906, + "fcm_dpo/q_t": 0.406668096780777, + "grad_norm": 14.988168716430664, + "learning_rate": 1.0098157099674987e-09, + "logits/chosen": 0.6676120758056641, + "logits/rejected": 0.6508312225341797, + "logps/chosen": -191.8373260498047, + "logps/ref_chosen": -60.963409423828125, + "logps/ref_rejected": -69.73353576660156, + "logps/rejected": -261.9013977050781, + "loss": 1.0955, + "margin_dpo/margin_mean": 61.29396057128906, + "margin_dpo/margin_std": 84.65945434570312, + "step": 645 + }, + { + "KL/chosen_KL_mean": -144.24465942382812, + "KL/mean": -170.72772216796875, + "KL/rejected_KL_mean": -197.21078491210938, + "KL/std": 90.64221954345703, + "epoch": 0.9765684051398337, + "fcm_dpo/beta": 0.006654680240899324, + "fcm_dpo/delta": 0.0492386557161808, + "fcm_dpo/margin": 52.96611022949219, + "fcm_dpo/q_t": 0.4206312298774719, + "grad_norm": 13.949078559875488, + "learning_rate": 8.945768539031783e-10, + "logits/chosen": 0.7426609992980957, + "logits/rejected": 0.6854439377784729, + "logps/chosen": -206.53472900390625, + "logps/ref_chosen": -62.290069580078125, + "logps/ref_rejected": -85.54812622070312, + "logps/rejected": -282.7589111328125, + "loss": 1.165, + "margin_dpo/margin_mean": 52.96611022949219, + "margin_dpo/margin_std": 98.530029296875, + "step": 646 + }, + { + "KL/chosen_KL_mean": -140.38922119140625, + "KL/mean": -183.11294555664062, + "KL/rejected_KL_mean": -225.836669921875, + "KL/std": 90.22059631347656, + "epoch": 0.9780801209372638, + "fcm_dpo/beta": 0.006521148607134819, + "fcm_dpo/delta": -0.16621947288513184, + "fcm_dpo/margin": 85.44746398925781, + "fcm_dpo/q_t": 0.37209028005599976, + "grad_norm": 13.996685028076172, + "learning_rate": 7.863060120144316e-10, + "logits/chosen": 0.7676196098327637, + "logits/rejected": 0.668016254901886, + "logps/chosen": -207.90509033203125, + "logps/ref_chosen": -67.515869140625, + "logps/ref_rejected": -101.50871276855469, + "logps/rejected": -327.34539794921875, + "loss": 0.9843, + "margin_dpo/margin_mean": 85.44746398925781, + "margin_dpo/margin_std": 89.50350952148438, + "step": 647 + }, + { + "KL/chosen_KL_mean": -139.56105041503906, + "KL/mean": -169.70150756835938, + "KL/rejected_KL_mean": -199.8419189453125, + "KL/std": 80.48648071289062, + "epoch": 0.9795918367346939, + "fcm_dpo/beta": 0.006439392454922199, + "fcm_dpo/delta": 0.012094831094145775, + "fcm_dpo/margin": 60.28087615966797, + "fcm_dpo/q_t": 0.4116312563419342, + "grad_norm": 14.129268646240234, + "learning_rate": 6.850062128694045e-10, + "logits/chosen": 0.7082226276397705, + "logits/rejected": 0.6414953470230103, + "logps/chosen": -204.156982421875, + "logps/ref_chosen": -64.59593963623047, + "logps/ref_rejected": -83.384033203125, + "logps/rejected": -283.2259521484375, + "loss": 1.1374, + "margin_dpo/margin_mean": 60.28087615966797, + "margin_dpo/margin_std": 101.12339782714844, + "step": 648 + }, + { + "KL/chosen_KL_mean": -131.25221252441406, + "KL/mean": -162.04953002929688, + "KL/rejected_KL_mean": -192.84683227539062, + "KL/std": 83.36114501953125, + "epoch": 0.981103552532124, + "fcm_dpo/beta": 0.0064369370229542255, + "fcm_dpo/delta": 0.0031705833971500397, + "fcm_dpo/margin": 61.59461212158203, + "fcm_dpo/q_t": 0.40989232063293457, + "grad_norm": 18.868831634521484, + "learning_rate": 5.906802900412788e-10, + "logits/chosen": 0.7710450887680054, + "logits/rejected": 0.7096936702728271, + "logps/chosen": -180.56185913085938, + "logps/ref_chosen": -49.30964660644531, + "logps/ref_rejected": -73.73710632324219, + "logps/rejected": -266.5839538574219, + "loss": 1.1323, + "margin_dpo/margin_mean": 61.594608306884766, + "margin_dpo/margin_std": 101.82295227050781, + "step": 649 + }, + { + "KL/chosen_KL_mean": -124.59217834472656, + "KL/mean": -155.55430603027344, + "KL/rejected_KL_mean": -186.5164337158203, + "KL/std": 83.22000122070312, + "epoch": 0.982615268329554, + "fcm_dpo/beta": 0.006457272917032242, + "fcm_dpo/delta": 6.21061772108078e-05, + "fcm_dpo/margin": 61.924259185791016, + "fcm_dpo/q_t": 0.4095534086227417, + "grad_norm": 13.497651100158691, + "learning_rate": 5.033308820289184e-10, + "logits/chosen": 0.840035080909729, + "logits/rejected": 0.769923210144043, + "logps/chosen": -179.6554412841797, + "logps/ref_chosen": -55.06325912475586, + "logps/ref_rejected": -77.39610290527344, + "logps/rejected": -263.91253662109375, + "loss": 1.1324, + "margin_dpo/margin_mean": 61.924259185791016, + "margin_dpo/margin_std": 103.21102905273438, + "step": 650 + }, + { + "KL/chosen_KL_mean": -125.5594253540039, + "KL/mean": -152.31875610351562, + "KL/rejected_KL_mean": -179.0780792236328, + "KL/std": 84.67516326904297, + "epoch": 0.9841269841269841, + "fcm_dpo/beta": 0.006532649975270033, + "fcm_dpo/delta": 0.052041828632354736, + "fcm_dpo/margin": 53.518653869628906, + "fcm_dpo/q_t": 0.4217528998851776, + "grad_norm": 13.061285018920898, + "learning_rate": 4.2296043218295606e-10, + "logits/chosen": 0.7976604104042053, + "logits/rejected": 0.7193989753723145, + "logps/chosen": -179.62457275390625, + "logps/ref_chosen": -54.065162658691406, + "logps/ref_rejected": -77.79080200195312, + "logps/rejected": -256.868896484375, + "loss": 1.1507, + "margin_dpo/margin_mean": 53.518653869628906, + "margin_dpo/margin_std": 91.1530990600586, + "step": 651 + }, + { + "KL/chosen_KL_mean": -147.7435760498047, + "KL/mean": -173.71324157714844, + "KL/rejected_KL_mean": -199.68289184570312, + "KL/std": 87.17472076416016, + "epoch": 0.9856386999244142, + "fcm_dpo/beta": 0.0065635861828923225, + "fcm_dpo/delta": 0.06094294786453247, + "fcm_dpo/margin": 51.93932342529297, + "fcm_dpo/q_t": 0.42427873611450195, + "grad_norm": 15.52949333190918, + "learning_rate": 3.4957118863768176e-10, + "logits/chosen": 0.7149733304977417, + "logits/rejected": 0.6630445718765259, + "logps/chosen": -211.38388061523438, + "logps/ref_chosen": -63.64030456542969, + "logps/ref_rejected": -78.86882019042969, + "logps/rejected": -278.55169677734375, + "loss": 1.1832, + "margin_dpo/margin_mean": 51.939327239990234, + "margin_dpo/margin_std": 102.12509155273438, + "step": 652 + }, + { + "KL/chosen_KL_mean": -132.9952392578125, + "KL/mean": -164.79788208007812, + "KL/rejected_KL_mean": -196.60049438476562, + "KL/std": 83.29473114013672, + "epoch": 0.9871504157218443, + "fcm_dpo/beta": 0.006613045930862427, + "fcm_dpo/delta": -0.021599669009447098, + "fcm_dpo/margin": 63.60527801513672, + "fcm_dpo/q_t": 0.4043147563934326, + "grad_norm": 14.842588424682617, + "learning_rate": 2.831652042480093e-10, + "logits/chosen": 0.7406543493270874, + "logits/rejected": 0.6956373453140259, + "logps/chosen": -194.66360473632812, + "logps/ref_chosen": -61.668373107910156, + "logps/ref_rejected": -73.83012390136719, + "logps/rejected": -270.4306335449219, + "loss": 1.1116, + "margin_dpo/margin_mean": 63.60527801513672, + "margin_dpo/margin_std": 99.63491821289062, + "step": 653 + }, + { + "KL/chosen_KL_mean": -137.654052734375, + "KL/mean": -163.44244384765625, + "KL/rejected_KL_mean": -189.2308349609375, + "KL/std": 92.28964233398438, + "epoch": 0.9886621315192744, + "fcm_dpo/beta": 0.006587079726159573, + "fcm_dpo/delta": 0.061099112033843994, + "fcm_dpo/margin": 51.5767822265625, + "fcm_dpo/q_t": 0.42364975810050964, + "grad_norm": 13.761890411376953, + "learning_rate": 2.2374433653205016e-10, + "logits/chosen": 0.7290184497833252, + "logits/rejected": 0.6216862201690674, + "logps/chosen": -195.22232055664062, + "logps/ref_chosen": -57.568267822265625, + "logps/ref_rejected": -87.74789428710938, + "logps/rejected": -276.9787292480469, + "loss": 1.1653, + "margin_dpo/margin_mean": 51.57678985595703, + "margin_dpo/margin_std": 88.0830307006836, + "step": 654 + }, + { + "KL/chosen_KL_mean": -107.77822875976562, + "KL/mean": -150.01248168945312, + "KL/rejected_KL_mean": -192.24676513671875, + "KL/std": 85.24494171142578, + "epoch": 0.9901738473167044, + "fcm_dpo/beta": 0.006456909701228142, + "fcm_dpo/delta": -0.15699413418769836, + "fcm_dpo/margin": 84.46852111816406, + "fcm_dpo/q_t": 0.3735496699810028, + "grad_norm": 12.970213890075684, + "learning_rate": 1.7131024761923852e-10, + "logits/chosen": 0.7261030673980713, + "logits/rejected": 0.6327559947967529, + "logps/chosen": -159.92538452148438, + "logps/ref_chosen": -52.14714813232422, + "logps/ref_rejected": -80.85014343261719, + "logps/rejected": -273.0968933105469, + "loss": 0.9833, + "margin_dpo/margin_mean": 84.46852111816406, + "margin_dpo/margin_std": 81.12989044189453, + "step": 655 + }, + { + "KL/chosen_KL_mean": -138.53622436523438, + "KL/mean": -171.3187255859375, + "KL/rejected_KL_mean": -204.10121154785156, + "KL/std": 85.09347534179688, + "epoch": 0.9916855631141346, + "fcm_dpo/beta": 0.0064401887357234955, + "fcm_dpo/delta": -0.02324036881327629, + "fcm_dpo/margin": 65.56498718261719, + "fcm_dpo/q_t": 0.40316781401634216, + "grad_norm": 10.821041107177734, + "learning_rate": 1.2586440420372934e-10, + "logits/chosen": 0.6753008365631104, + "logits/rejected": 0.6247135400772095, + "logps/chosen": -211.79293823242188, + "logps/ref_chosen": -73.25672912597656, + "logps/ref_rejected": -85.35127258300781, + "logps/rejected": -289.4524841308594, + "loss": 1.0942, + "margin_dpo/margin_mean": 65.56498718261719, + "margin_dpo/margin_std": 94.33009338378906, + "step": 656 + }, + { + "KL/chosen_KL_mean": -126.80171966552734, + "KL/mean": -163.933349609375, + "KL/rejected_KL_mean": -201.06500244140625, + "KL/std": 94.10905456542969, + "epoch": 0.9931972789115646, + "fcm_dpo/beta": 0.006361294537782669, + "fcm_dpo/delta": -0.0760110542178154, + "fcm_dpo/margin": 74.26329803466797, + "fcm_dpo/q_t": 0.39246702194213867, + "grad_norm": 11.380026817321777, + "learning_rate": 8.740807750345913e-11, + "logits/chosen": 0.8557542562484741, + "logits/rejected": 0.7586342096328735, + "logps/chosen": -176.52511596679688, + "logps/ref_chosen": -49.72339630126953, + "logps/ref_rejected": -75.1568603515625, + "logps/rejected": -276.22186279296875, + "loss": 1.0715, + "margin_dpo/margin_mean": 74.26329040527344, + "margin_dpo/margin_std": 104.2190933227539, + "step": 657 + }, + { + "KL/chosen_KL_mean": -133.42100524902344, + "KL/mean": -162.8004150390625, + "KL/rejected_KL_mean": -192.17982482910156, + "KL/std": 91.8773193359375, + "epoch": 0.9947089947089947, + "fcm_dpo/beta": 0.006399291567504406, + "fcm_dpo/delta": 0.024006865918636322, + "fcm_dpo/margin": 58.758811950683594, + "fcm_dpo/q_t": 0.4153459072113037, + "grad_norm": 12.453079223632812, + "learning_rate": 5.594234322453539e-11, + "logits/chosen": 0.7587237358093262, + "logits/rejected": 0.7135840654373169, + "logps/chosen": -196.46734619140625, + "logps/ref_chosen": -63.04634094238281, + "logps/ref_rejected": -83.44963073730469, + "logps/rejected": -275.62945556640625, + "loss": 1.1623, + "margin_dpo/margin_mean": 58.758811950683594, + "margin_dpo/margin_std": 109.14431762695312, + "step": 658 + }, + { + "KL/chosen_KL_mean": -138.11993408203125, + "KL/mean": -162.07423400878906, + "KL/rejected_KL_mean": -186.02853393554688, + "KL/std": 86.01627349853516, + "epoch": 0.9962207105064248, + "fcm_dpo/beta": 0.006414198316633701, + "fcm_dpo/delta": 0.09530773013830185, + "fcm_dpo/margin": 47.90861129760742, + "fcm_dpo/q_t": 0.43013453483581543, + "grad_norm": 17.540708541870117, + "learning_rate": 3.146808153123293e-11, + "logits/chosen": 0.8054675459861755, + "logits/rejected": 0.7398273944854736, + "logps/chosen": -193.20013427734375, + "logps/ref_chosen": -55.0802001953125, + "logps/ref_rejected": -71.91049194335938, + "logps/rejected": -257.93902587890625, + "loss": 1.2052, + "margin_dpo/margin_mean": 47.90861129760742, + "margin_dpo/margin_std": 99.61247253417969, + "step": 659 + }, + { + "KL/chosen_KL_mean": -130.65850830078125, + "KL/mean": -166.74920654296875, + "KL/rejected_KL_mean": -202.83990478515625, + "KL/std": 88.41796875, + "epoch": 0.9977324263038548, + "fcm_dpo/beta": 0.006431464105844498, + "fcm_dpo/delta": -0.06731559336185455, + "fcm_dpo/margin": 72.1814193725586, + "fcm_dpo/q_t": 0.3944994807243347, + "grad_norm": 13.725423812866211, + "learning_rate": 1.3985977021235829e-11, + "logits/chosen": 0.8140730857849121, + "logits/rejected": 0.7385152578353882, + "logps/chosen": -185.18441772460938, + "logps/ref_chosen": -54.525917053222656, + "logps/ref_rejected": -81.23604583740234, + "logps/rejected": -284.075927734375, + "loss": 1.0573, + "margin_dpo/margin_mean": 72.1814193725586, + "margin_dpo/margin_std": 93.41641235351562, + "step": 660 + }, + { + "KL/chosen_KL_mean": -149.49850463867188, + "KL/mean": -173.75375366210938, + "KL/rejected_KL_mean": -198.00900268554688, + "KL/std": 92.14109802246094, + "epoch": 0.999244142101285, + "fcm_dpo/beta": 0.0065015703439712524, + "fcm_dpo/delta": 0.08707258105278015, + "fcm_dpo/margin": 48.51050567626953, + "fcm_dpo/q_t": 0.4291042387485504, + "grad_norm": 14.858864784240723, + "learning_rate": 3.4965187065971735e-12, + "logits/chosen": 0.6810092329978943, + "logits/rejected": 0.5987504720687866, + "logps/chosen": -209.87115478515625, + "logps/ref_chosen": -60.37263870239258, + "logps/ref_rejected": -77.42874145507812, + "logps/rejected": -275.437744140625, + "loss": 1.2089, + "margin_dpo/margin_mean": 48.51050567626953, + "margin_dpo/margin_std": 106.95087432861328, + "step": 661 + }, + { + "epoch": 0.999244142101285, + "step": 661, + "total_flos": 0.0, + "train_loss": 1.132411924234497, + "train_runtime": 1654.1278, + "train_samples_per_second": 25.594, + "train_steps_per_second": 0.4 + } + ], + "logging_steps": 1, + "max_steps": 661, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}