commit 58012c8d13711124df491e61cb6521e42a3101d6 Author: ModelHub XC Date: Mon Jun 1 07:43:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.4 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..08ed21d --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-harmless-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.4 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.4 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-harmless-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-harmless-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..d5bec70 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.999244142101285, + "total_flos": 0.0, + "train_loss": 1.1812428684169693, + "train_runtime": 1648.6054, + "train_samples": 42336, + "train_samples_per_second": 25.68, + "train_steps_per_second": 0.401 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..a2f30e6 --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24a5cf6b5f9f0092d27055cded005ca28d02580ad1ed0565ec726145f287ef73 +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..580229b --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d463389bd6e2dff979ab4ca03d4356f3c13cb07def26673bd01ce80e5c003d3f +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..92e77f5 --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b5f3fdb3f30e1e51a08b22ee7261b954b07e0611f01e1e21ee1190706314ce9 +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..839fca0 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c79023762ec518ac43b65e8ecc13e25027e4aaee95bfe264dbb934f6ad58756d +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..67acf39 --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5eca7bfe8b3886d3b48079f210bc418a1fce88ea84e57605d3491f9886be2c3c +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..7e2b6a9 --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:082d14a9328ee375c37e21a7c6b7403e77a530598a91facf6a35873e31253e97 +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..a14661b --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:350a8a363ee5659fa934a79fd1f143165b2dbdbb8f3130745f89c724b313f53b +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..d5bec70 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.999244142101285, + "total_flos": 0.0, + "train_loss": 1.1812428684169693, + "train_runtime": 1648.6054, + "train_samples": 42336, + "train_samples_per_second": 25.68, + "train_steps_per_second": 0.401 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..a344793 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15246 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.999244142101285, + "eval_steps": 200, + "global_step": 661, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.02867889404296875, + "KL/mean": 0.029354453086853027, + "KL/rejected_KL_mean": 0.030029296875, + "KL/std": 0.2071000635623932, + "epoch": 0.0015117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0013532638549804688, + "fcm_dpo/q_t": 0.5000336766242981, + "grad_norm": 28.219078063964844, + "learning_rate": 0.0, + "logits/chosen": 0.13337239623069763, + "logits/rejected": 0.12492949515581131, + "logps/chosen": -64.5841293334961, + "logps/ref_chosen": -64.61280822753906, + "logps/ref_rejected": -64.17195129394531, + "logps/rejected": -64.14192199707031, + "loss": 1.3866, + "margin_dpo/margin_mean": -0.0013527870178222656, + "margin_dpo/margin_std": 0.2561596930027008, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.00289154052734375, + "KL/mean": -0.021616414189338684, + "KL/rejected_KL_mean": -0.04033660888671875, + "KL/std": 0.19624735414981842, + "epoch": 0.0030234315948601664, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.037450045347213745, + "fcm_dpo/q_t": 0.4990639090538025, + "grad_norm": 27.82866096496582, + "learning_rate": 7.462686567164179e-09, + "logits/chosen": 0.09414851665496826, + "logits/rejected": 0.07363267242908478, + "logps/chosen": -56.101890563964844, + "logps/ref_chosen": -56.0989990234375, + "logps/ref_rejected": -66.59971618652344, + "logps/rejected": -66.64006042480469, + "loss": 1.3828, + "margin_dpo/margin_mean": 0.03744968771934509, + "margin_dpo/margin_std": 0.27811938524246216, + "step": 2 + }, + { + "KL/chosen_KL_mean": 0.02931976318359375, + "KL/mean": 0.034121572971343994, + "KL/rejected_KL_mean": 0.038921356201171875, + "KL/std": 0.24494421482086182, + "epoch": 0.0045351473922902496, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.009602487087249756, + "fcm_dpo/q_t": 0.500239908695221, + "grad_norm": 31.2530460357666, + "learning_rate": 1.4925373134328357e-08, + "logits/chosen": 0.09943141043186188, + "logits/rejected": 0.06147802621126175, + "logps/chosen": -65.42794036865234, + "logps/ref_chosen": -65.45726013183594, + "logps/ref_rejected": -90.82853698730469, + "logps/rejected": -90.78961181640625, + "loss": 1.3875, + "margin_dpo/margin_mean": -0.009602636098861694, + "margin_dpo/margin_std": 0.31281691789627075, + "step": 3 + }, + { + "KL/chosen_KL_mean": 0.007144927978515625, + "KL/mean": 0.002002894878387451, + "KL/rejected_KL_mean": -0.003143310546875, + "KL/std": 0.24436025321483612, + "epoch": 0.006046863189720333, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.010289788246154785, + "fcm_dpo/q_t": 0.4997430443763733, + "grad_norm": 34.29277420043945, + "learning_rate": 2.2388059701492534e-08, + "logits/chosen": 0.11271625757217407, + "logits/rejected": 0.09644747525453568, + "logps/chosen": -76.85304260253906, + "logps/ref_chosen": -76.86018371582031, + "logps/ref_rejected": -79.91523742675781, + "logps/rejected": -79.91838073730469, + "loss": 1.3855, + "margin_dpo/margin_mean": 0.010288774967193604, + "margin_dpo/margin_std": 0.3010460138320923, + "step": 4 + }, + { + "KL/chosen_KL_mean": -0.011060714721679688, + "KL/mean": -0.0001738518476486206, + "KL/rejected_KL_mean": 0.010715484619140625, + "KL/std": 0.2380131483078003, + "epoch": 0.007558578987150416, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02177181839942932, + "fcm_dpo/q_t": 0.5005446076393127, + "grad_norm": 29.53087615966797, + "learning_rate": 2.9850746268656714e-08, + "logits/chosen": 0.08209095895290375, + "logits/rejected": 0.04320637136697769, + "logps/chosen": -62.982398986816406, + "logps/ref_chosen": -62.97134017944336, + "logps/ref_rejected": -79.9192123413086, + "logps/rejected": -79.90849304199219, + "loss": 1.3888, + "margin_dpo/margin_mean": -0.021771669387817383, + "margin_dpo/margin_std": 0.3412056565284729, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.06389617919921875, + "KL/mean": -0.022043123841285706, + "KL/rejected_KL_mean": 0.019815444946289062, + "KL/std": 0.22818465530872345, + "epoch": 0.009070294784580499, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.08370622992515564, + "fcm_dpo/q_t": 0.5020915269851685, + "grad_norm": 29.720598220825195, + "learning_rate": 3.731343283582089e-08, + "logits/chosen": 0.12795253098011017, + "logits/rejected": 0.08949151635169983, + "logps/chosen": -51.37126159667969, + "logps/ref_chosen": -51.30736541748047, + "logps/ref_rejected": -82.77239227294922, + "logps/rejected": -82.75257873535156, + "loss": 1.3949, + "margin_dpo/margin_mean": -0.08370649814605713, + "margin_dpo/margin_std": 0.3061661124229431, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.000690460205078125, + "KL/mean": 0.004148155450820923, + "KL/rejected_KL_mean": 0.0076045989990234375, + "KL/std": 0.2287873923778534, + "epoch": 0.010582010582010581, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.006914794445037842, + "fcm_dpo/q_t": 0.5001726150512695, + "grad_norm": 27.180267333984375, + "learning_rate": 4.477611940298507e-08, + "logits/chosen": 0.01639726385474205, + "logits/rejected": -0.02727954089641571, + "logps/chosen": -51.45872116088867, + "logps/ref_chosen": -51.45941162109375, + "logps/ref_rejected": -66.3828125, + "logps/rejected": -66.37520599365234, + "loss": 1.3872, + "margin_dpo/margin_mean": -0.00691559910774231, + "margin_dpo/margin_std": 0.29766663908958435, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.012800216674804688, + "KL/mean": 0.0012196600437164307, + "KL/rejected_KL_mean": -0.0103607177734375, + "KL/std": 0.21195456385612488, + "epoch": 0.012093726379440665, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.02315998077392578, + "fcm_dpo/q_t": 0.4994213283061981, + "grad_norm": 28.690420150756836, + "learning_rate": 5.223880597014925e-08, + "logits/chosen": 0.06877216696739197, + "logits/rejected": 0.046678461134433746, + "logps/chosen": -62.18474578857422, + "logps/ref_chosen": -62.197547912597656, + "logps/ref_rejected": -74.66180419921875, + "logps/rejected": -74.67216491699219, + "loss": 1.3842, + "margin_dpo/margin_mean": 0.023159563541412354, + "margin_dpo/margin_std": 0.29333364963531494, + "step": 8 + }, + { + "KL/chosen_KL_mean": -0.0251922607421875, + "KL/mean": -0.03252735733985901, + "KL/rejected_KL_mean": -0.03986358642578125, + "KL/std": 0.22694696485996246, + "epoch": 0.013605442176870748, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.014674007892608643, + "fcm_dpo/q_t": 0.49963313341140747, + "grad_norm": 31.518144607543945, + "learning_rate": 5.970149253731343e-08, + "logits/chosen": 0.177996426820755, + "logits/rejected": 0.1178831085562706, + "logps/chosen": -55.65491485595703, + "logps/ref_chosen": -55.629722595214844, + "logps/ref_rejected": -86.21221923828125, + "logps/rejected": -86.25209045410156, + "loss": 1.3851, + "margin_dpo/margin_mean": 0.014674067497253418, + "margin_dpo/margin_std": 0.3014518618583679, + "step": 9 + }, + { + "KL/chosen_KL_mean": 0.011089324951171875, + "KL/mean": 0.04612685739994049, + "KL/rejected_KL_mean": 0.08116531372070312, + "KL/std": 0.2460353970527649, + "epoch": 0.015117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.07007792592048645, + "fcm_dpo/q_t": 0.5017515420913696, + "grad_norm": 29.845827102661133, + "learning_rate": 6.71641791044776e-08, + "logits/chosen": 0.13188131153583527, + "logits/rejected": 0.10091814398765564, + "logps/chosen": -62.67951202392578, + "logps/ref_chosen": -62.69060134887695, + "logps/ref_rejected": -90.610107421875, + "logps/rejected": -90.52894592285156, + "loss": 1.3937, + "margin_dpo/margin_mean": -0.0700773298740387, + "margin_dpo/margin_std": 0.3673255443572998, + "step": 10 + }, + { + "KL/chosen_KL_mean": -0.030055999755859375, + "KL/mean": -0.00986570119857788, + "KL/rejected_KL_mean": 0.01031494140625, + "KL/std": 0.18542543053627014, + "epoch": 0.016628873771730914, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.040368854999542236, + "fcm_dpo/q_t": 0.5010088086128235, + "grad_norm": 29.479524612426758, + "learning_rate": 7.462686567164178e-08, + "logits/chosen": 0.09895411133766174, + "logits/rejected": 0.09225950390100479, + "logps/chosen": -65.79717254638672, + "logps/ref_chosen": -65.76712036132812, + "logps/ref_rejected": -72.4764633178711, + "logps/rejected": -72.46614837646484, + "loss": 1.3905, + "margin_dpo/margin_mean": -0.04036879539489746, + "margin_dpo/margin_std": 0.2673962116241455, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.02222442626953125, + "KL/mean": -0.01459282636642456, + "KL/rejected_KL_mean": -0.0069599151611328125, + "KL/std": 0.2098838984966278, + "epoch": 0.018140589569160998, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.015265405178070068, + "fcm_dpo/q_t": 0.5003814697265625, + "grad_norm": 28.26340103149414, + "learning_rate": 8.208955223880596e-08, + "logits/chosen": 0.022189727053046227, + "logits/rejected": 0.00604275893419981, + "logps/chosen": -60.727115631103516, + "logps/ref_chosen": -60.704891204833984, + "logps/ref_rejected": -69.41564178466797, + "logps/rejected": -69.42259979248047, + "loss": 1.388, + "margin_dpo/margin_mean": -0.01526564359664917, + "margin_dpo/margin_std": 0.2689279317855835, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.0106048583984375, + "KL/mean": 0.01754908263683319, + "KL/rejected_KL_mean": 0.02449798583984375, + "KL/std": 0.22062669694423676, + "epoch": 0.019652305366591082, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.013890773057937622, + "fcm_dpo/q_t": 0.5003470778465271, + "grad_norm": 29.100948333740234, + "learning_rate": 8.955223880597014e-08, + "logits/chosen": 0.11829005181789398, + "logits/rejected": 0.05488961189985275, + "logps/chosen": -49.898651123046875, + "logps/ref_chosen": -49.90925598144531, + "logps/ref_rejected": -92.37818145751953, + "logps/rejected": -92.35368347167969, + "loss": 1.3879, + "margin_dpo/margin_mean": -0.013890862464904785, + "margin_dpo/margin_std": 0.30010896921157837, + "step": 13 + }, + { + "KL/chosen_KL_mean": 0.018400192260742188, + "KL/mean": 0.0002371072769165039, + "KL/rejected_KL_mean": -0.01792144775390625, + "KL/std": 0.2105257362127304, + "epoch": 0.021164021164021163, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03632456064224243, + "fcm_dpo/q_t": 0.49909305572509766, + "grad_norm": 29.442873001098633, + "learning_rate": 9.701492537313432e-08, + "logits/chosen": 0.10424400866031647, + "logits/rejected": 0.08598647266626358, + "logps/chosen": -60.60039520263672, + "logps/ref_chosen": -60.61879348754883, + "logps/ref_rejected": -71.79306030273438, + "logps/rejected": -71.81098175048828, + "loss": 1.3829, + "margin_dpo/margin_mean": 0.03632441163063049, + "margin_dpo/margin_std": 0.3239067792892456, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.05114936828613281, + "KL/mean": -0.06186847388744354, + "KL/rejected_KL_mean": -0.07258224487304688, + "KL/std": 0.2505655288696289, + "epoch": 0.022675736961451247, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.021430760622024536, + "fcm_dpo/q_t": 0.4994640648365021, + "grad_norm": 33.124202728271484, + "learning_rate": 1.044776119402985e-07, + "logits/chosen": 0.09332643449306488, + "logits/rejected": 0.0486547127366066, + "logps/chosen": -63.520687103271484, + "logps/ref_chosen": -63.46953582763672, + "logps/ref_rejected": -88.88951110839844, + "logps/rejected": -88.96209716796875, + "loss": 1.3844, + "margin_dpo/margin_mean": 0.021430641412734985, + "margin_dpo/margin_std": 0.33308592438697815, + "step": 15 + }, + { + "KL/chosen_KL_mean": -0.020135879516601562, + "KL/mean": -0.004521623253822327, + "KL/rejected_KL_mean": 0.011089324951171875, + "KL/std": 0.18780693411827087, + "epoch": 0.02418745275888133, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.031230241060256958, + "fcm_dpo/q_t": 0.5007807016372681, + "grad_norm": 26.916568756103516, + "learning_rate": 1.1194029850746268e-07, + "logits/chosen": 0.12158643454313278, + "logits/rejected": 0.08412902057170868, + "logps/chosen": -46.55243682861328, + "logps/ref_chosen": -46.53229904174805, + "logps/ref_rejected": -74.27533721923828, + "logps/rejected": -74.26425170898438, + "loss": 1.3896, + "margin_dpo/margin_mean": -0.03123021125793457, + "margin_dpo/margin_std": 0.24277648329734802, + "step": 16 + }, + { + "KL/chosen_KL_mean": -0.0008640289306640625, + "KL/mean": -0.015623345971107483, + "KL/rejected_KL_mean": -0.030376434326171875, + "KL/std": 0.20342105627059937, + "epoch": 0.025699168556311415, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.029514700174331665, + "fcm_dpo/q_t": 0.4992625117301941, + "grad_norm": 32.90407180786133, + "learning_rate": 1.1940298507462686e-07, + "logits/chosen": 0.0347953625023365, + "logits/rejected": 0.016835104674100876, + "logps/chosen": -64.07869720458984, + "logps/ref_chosen": -64.07783508300781, + "logps/ref_rejected": -86.40876770019531, + "logps/rejected": -86.43914794921875, + "loss": 1.3836, + "margin_dpo/margin_mean": 0.02951410412788391, + "margin_dpo/margin_std": 0.29045775532722473, + "step": 17 + }, + { + "KL/chosen_KL_mean": 0.0038318634033203125, + "KL/mean": -0.004959017038345337, + "KL/rejected_KL_mean": -0.013753890991210938, + "KL/std": 0.20491787791252136, + "epoch": 0.027210884353741496, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0175778865814209, + "fcm_dpo/q_t": 0.49956053495407104, + "grad_norm": 27.838245391845703, + "learning_rate": 1.2686567164179106e-07, + "logits/chosen": 0.10396843403577805, + "logits/rejected": 0.05753494054079056, + "logps/chosen": -44.870506286621094, + "logps/ref_chosen": -44.87433624267578, + "logps/ref_rejected": -70.97604370117188, + "logps/rejected": -70.98979949951172, + "loss": 1.3847, + "margin_dpo/margin_mean": 0.017578035593032837, + "margin_dpo/margin_std": 0.2776282727718353, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.013826370239257812, + "KL/mean": -0.0132150799036026, + "KL/rejected_KL_mean": -0.040256500244140625, + "KL/std": 0.23131096363067627, + "epoch": 0.02872260015117158, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.054083555936813354, + "fcm_dpo/q_t": 0.49864810705184937, + "grad_norm": 30.866737365722656, + "learning_rate": 1.343283582089552e-07, + "logits/chosen": 0.06870782375335693, + "logits/rejected": 0.05521427094936371, + "logps/chosen": -68.14598083496094, + "logps/ref_chosen": -68.1598129272461, + "logps/ref_rejected": -81.17138671875, + "logps/rejected": -81.2116470336914, + "loss": 1.3811, + "margin_dpo/margin_mean": 0.054083943367004395, + "margin_dpo/margin_std": 0.29337140917778015, + "step": 19 + }, + { + "KL/chosen_KL_mean": 0.0003376007080078125, + "KL/mean": -0.0011960119009017944, + "KL/rejected_KL_mean": -0.002727508544921875, + "KL/std": 0.2168964147567749, + "epoch": 0.030234315948601664, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.003067702054977417, + "fcm_dpo/q_t": 0.49992379546165466, + "grad_norm": 29.303083419799805, + "learning_rate": 1.4179104477611938e-07, + "logits/chosen": 0.13337117433547974, + "logits/rejected": 0.11041676253080368, + "logps/chosen": -53.67822265625, + "logps/ref_chosen": -53.67856216430664, + "logps/ref_rejected": -74.16911315917969, + "logps/rejected": -74.17184448242188, + "loss": 1.3862, + "margin_dpo/margin_mean": 0.0030680596828460693, + "margin_dpo/margin_std": 0.31527629494667053, + "step": 20 + }, + { + "KL/chosen_KL_mean": 0.017999649047851562, + "KL/mean": 0.021966353058815002, + "KL/rejected_KL_mean": 0.02593231201171875, + "KL/std": 0.21482843160629272, + "epoch": 0.031746031746031744, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.007932394742965698, + "fcm_dpo/q_t": 0.5001983642578125, + "grad_norm": 29.316267013549805, + "learning_rate": 1.4925373134328355e-07, + "logits/chosen": 0.11854798346757889, + "logits/rejected": 0.09246804565191269, + "logps/chosen": -64.68355560302734, + "logps/ref_chosen": -64.70155334472656, + "logps/ref_rejected": -81.02095031738281, + "logps/rejected": -80.99501037597656, + "loss": 1.3873, + "margin_dpo/margin_mean": -0.007933080196380615, + "margin_dpo/margin_std": 0.27441009879112244, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.013017654418945312, + "KL/mean": -0.01622338593006134, + "KL/rejected_KL_mean": -0.045467376708984375, + "KL/std": 0.2124525010585785, + "epoch": 0.03325774754346183, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05848535895347595, + "fcm_dpo/q_t": 0.4985382556915283, + "grad_norm": 28.89164161682129, + "learning_rate": 1.5671641791044775e-07, + "logits/chosen": 0.029597830027341843, + "logits/rejected": 0.007340491749346256, + "logps/chosen": -58.022972106933594, + "logps/ref_chosen": -58.03599166870117, + "logps/ref_rejected": -80.72721862792969, + "logps/rejected": -80.77268981933594, + "loss": 1.3806, + "margin_dpo/margin_mean": 0.05848526954650879, + "margin_dpo/margin_std": 0.2741105556488037, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.018955230712890625, + "KL/mean": 0.005454123020172119, + "KL/rejected_KL_mean": -0.00804901123046875, + "KL/std": 0.2213050127029419, + "epoch": 0.03476946334089191, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.02700936794281006, + "fcm_dpo/q_t": 0.4993247985839844, + "grad_norm": 32.89314270019531, + "learning_rate": 1.6417910447761193e-07, + "logits/chosen": 0.14521625638008118, + "logits/rejected": 0.11956644058227539, + "logps/chosen": -66.33712768554688, + "logps/ref_chosen": -66.35608673095703, + "logps/ref_rejected": -93.02769470214844, + "logps/rejected": -93.03573608398438, + "loss": 1.3839, + "margin_dpo/margin_mean": 0.02700933814048767, + "margin_dpo/margin_std": 0.33931535482406616, + "step": 23 + }, + { + "KL/chosen_KL_mean": -0.038440704345703125, + "KL/mean": -0.027752220630645752, + "KL/rejected_KL_mean": -0.017061233520507812, + "KL/std": 0.20728710293769836, + "epoch": 0.036281179138321996, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.021373331546783447, + "fcm_dpo/q_t": 0.500534176826477, + "grad_norm": 26.315832138061523, + "learning_rate": 1.716417910447761e-07, + "logits/chosen": 0.11648119986057281, + "logits/rejected": 0.08412452787160873, + "logps/chosen": -54.49967956542969, + "logps/ref_chosen": -54.461238861083984, + "logps/ref_rejected": -68.33817291259766, + "logps/rejected": -68.35523986816406, + "loss": 1.3886, + "margin_dpo/margin_mean": -0.021373480558395386, + "margin_dpo/margin_std": 0.26126983761787415, + "step": 24 + }, + { + "KL/chosen_KL_mean": -0.0121917724609375, + "KL/mean": -0.02050521969795227, + "KL/rejected_KL_mean": -0.02881622314453125, + "KL/std": 0.21590664982795715, + "epoch": 0.03779289493575208, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.016626060009002686, + "fcm_dpo/q_t": 0.4995846748352051, + "grad_norm": 29.75655746459961, + "learning_rate": 1.7910447761194027e-07, + "logits/chosen": 0.11330267786979675, + "logits/rejected": 0.061182692646980286, + "logps/chosen": -60.016395568847656, + "logps/ref_chosen": -60.00420379638672, + "logps/ref_rejected": -90.47376251220703, + "logps/rejected": -90.50257873535156, + "loss": 1.385, + "margin_dpo/margin_mean": 0.016626060009002686, + "margin_dpo/margin_std": 0.3512018322944641, + "step": 25 + }, + { + "KL/chosen_KL_mean": -0.010725021362304688, + "KL/mean": -0.02841213345527649, + "KL/rejected_KL_mean": -0.046092987060546875, + "KL/std": 0.19508026540279388, + "epoch": 0.039304610733182165, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03537088632583618, + "fcm_dpo/q_t": 0.4991161823272705, + "grad_norm": 29.64750099182129, + "learning_rate": 1.8656716417910447e-07, + "logits/chosen": 0.12426209449768066, + "logits/rejected": 0.10544230788946152, + "logps/chosen": -56.82987976074219, + "logps/ref_chosen": -56.81915283203125, + "logps/ref_rejected": -77.84333038330078, + "logps/rejected": -77.88941955566406, + "loss": 1.383, + "margin_dpo/margin_mean": 0.03537055850028992, + "margin_dpo/margin_std": 0.29276707768440247, + "step": 26 + }, + { + "KL/chosen_KL_mean": -0.0206756591796875, + "KL/mean": -0.012468293309211731, + "KL/rejected_KL_mean": -0.0042629241943359375, + "KL/std": 0.2252947986125946, + "epoch": 0.04081632653061224, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.016415029764175415, + "fcm_dpo/q_t": 0.5004103183746338, + "grad_norm": 29.00526237487793, + "learning_rate": 1.9402985074626865e-07, + "logits/chosen": 0.09494505822658539, + "logits/rejected": 0.0703374445438385, + "logps/chosen": -62.897701263427734, + "logps/ref_chosen": -62.87702560424805, + "logps/ref_rejected": -71.34437561035156, + "logps/rejected": -71.34864044189453, + "loss": 1.3881, + "margin_dpo/margin_mean": -0.016414940357208252, + "margin_dpo/margin_std": 0.2852821946144104, + "step": 27 + }, + { + "KL/chosen_KL_mean": -0.023799896240234375, + "KL/mean": -0.015411525964736938, + "KL/rejected_KL_mean": -0.007022857666015625, + "KL/std": 0.2348794937133789, + "epoch": 0.042328042328042326, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.016774117946624756, + "fcm_dpo/q_t": 0.5004186034202576, + "grad_norm": 27.660003662109375, + "learning_rate": 2.0149253731343282e-07, + "logits/chosen": 0.047501303255558014, + "logits/rejected": 0.03891991078853607, + "logps/chosen": -59.857177734375, + "logps/ref_chosen": -59.8333740234375, + "logps/ref_rejected": -70.39804077148438, + "logps/rejected": -70.40505981445312, + "loss": 1.3882, + "margin_dpo/margin_mean": -0.01677393913269043, + "margin_dpo/margin_std": 0.30225175619125366, + "step": 28 + }, + { + "KL/chosen_KL_mean": -0.011280059814453125, + "KL/mean": -0.03545750677585602, + "KL/rejected_KL_mean": -0.059627532958984375, + "KL/std": 0.2457764446735382, + "epoch": 0.04383975812547241, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04835239052772522, + "fcm_dpo/q_t": 0.4987908601760864, + "grad_norm": 32.58844757080078, + "learning_rate": 2.08955223880597e-07, + "logits/chosen": 0.15888172388076782, + "logits/rejected": 0.14050382375717163, + "logps/chosen": -74.13148498535156, + "logps/ref_chosen": -74.12020111083984, + "logps/ref_rejected": -83.33099365234375, + "logps/rejected": -83.39061737060547, + "loss": 1.3818, + "margin_dpo/margin_mean": 0.04835203289985657, + "margin_dpo/margin_std": 0.355918824672699, + "step": 29 + }, + { + "KL/chosen_KL_mean": -0.04425048828125, + "KL/mean": -0.014838695526123047, + "KL/rejected_KL_mean": 0.0145721435546875, + "KL/std": 0.23300054669380188, + "epoch": 0.045351473922902494, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.05882430076599121, + "fcm_dpo/q_t": 0.5014699697494507, + "grad_norm": 30.512819290161133, + "learning_rate": 2.1641791044776117e-07, + "logits/chosen": 0.13098673522472382, + "logits/rejected": 0.07602270692586899, + "logps/chosen": -50.79553985595703, + "logps/ref_chosen": -50.75128936767578, + "logps/ref_rejected": -89.29063415527344, + "logps/rejected": -89.27606201171875, + "loss": 1.3924, + "margin_dpo/margin_mean": -0.058824241161346436, + "margin_dpo/margin_std": 0.31058943271636963, + "step": 30 + }, + { + "KL/chosen_KL_mean": -0.023962020874023438, + "KL/mean": -0.048681557178497314, + "KL/rejected_KL_mean": -0.07340240478515625, + "KL/std": 0.23842594027519226, + "epoch": 0.04686318972033258, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.049435973167419434, + "fcm_dpo/q_t": 0.498765230178833, + "grad_norm": 34.26254653930664, + "learning_rate": 2.2388059701492537e-07, + "logits/chosen": 0.124081090092659, + "logits/rejected": 0.07678476721048355, + "logps/chosen": -65.3607177734375, + "logps/ref_chosen": -65.33675384521484, + "logps/ref_rejected": -100.76666259765625, + "logps/rejected": -100.8400650024414, + "loss": 1.3817, + "margin_dpo/margin_mean": 0.049436479806900024, + "margin_dpo/margin_std": 0.35601481795310974, + "step": 31 + }, + { + "KL/chosen_KL_mean": -0.029903411865234375, + "KL/mean": -0.018042847514152527, + "KL/rejected_KL_mean": -0.006183624267578125, + "KL/std": 0.268637478351593, + "epoch": 0.04837490551776266, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.023714810609817505, + "fcm_dpo/q_t": 0.5005922913551331, + "grad_norm": 30.1397762298584, + "learning_rate": 2.3134328358208954e-07, + "logits/chosen": 0.0673169419169426, + "logits/rejected": 0.05956514924764633, + "logps/chosen": -67.2132339477539, + "logps/ref_chosen": -67.18333435058594, + "logps/ref_rejected": -82.80763244628906, + "logps/rejected": -82.8138198852539, + "loss": 1.389, + "margin_dpo/margin_mean": -0.023714929819107056, + "margin_dpo/margin_std": 0.356290727853775, + "step": 32 + }, + { + "KL/chosen_KL_mean": -0.008056640625, + "KL/mean": -0.037323206663131714, + "KL/rejected_KL_mean": -0.06658554077148438, + "KL/std": 0.24775487184524536, + "epoch": 0.049886621315192746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0585293173789978, + "fcm_dpo/q_t": 0.4985373020172119, + "grad_norm": 30.928356170654297, + "learning_rate": 2.388059701492537e-07, + "logits/chosen": 0.020615022629499435, + "logits/rejected": -0.004851259291172028, + "logps/chosen": -64.04753875732422, + "logps/ref_chosen": -64.03948211669922, + "logps/ref_rejected": -75.68357849121094, + "logps/rejected": -75.75016784667969, + "loss": 1.3807, + "margin_dpo/margin_mean": 0.058529555797576904, + "margin_dpo/margin_std": 0.32526156306266785, + "step": 33 + }, + { + "KL/chosen_KL_mean": -0.04400825500488281, + "KL/mean": -0.07188649475574493, + "KL/rejected_KL_mean": -0.09976577758789062, + "KL/std": 0.22814133763313293, + "epoch": 0.05139833711262283, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.055761903524398804, + "fcm_dpo/q_t": 0.49860724806785583, + "grad_norm": 28.403575897216797, + "learning_rate": 2.4626865671641786e-07, + "logits/chosen": 0.09220936894416809, + "logits/rejected": 0.06258954852819443, + "logps/chosen": -53.70830535888672, + "logps/ref_chosen": -53.6642951965332, + "logps/ref_rejected": -65.77989959716797, + "logps/rejected": -65.87966918945312, + "loss": 1.381, + "margin_dpo/margin_mean": 0.0557616651058197, + "margin_dpo/margin_std": 0.34898078441619873, + "step": 34 + }, + { + "KL/chosen_KL_mean": -0.08493995666503906, + "KL/mean": -0.09149569272994995, + "KL/rejected_KL_mean": -0.098052978515625, + "KL/std": 0.239829421043396, + "epoch": 0.05291005291005291, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.013110756874084473, + "fcm_dpo/q_t": 0.49967312812805176, + "grad_norm": 27.82366180419922, + "learning_rate": 2.537313432835821e-07, + "logits/chosen": 0.07824540138244629, + "logits/rejected": 0.055066656321287155, + "logps/chosen": -61.101802825927734, + "logps/ref_chosen": -61.01686096191406, + "logps/ref_rejected": -72.78598022460938, + "logps/rejected": -72.884033203125, + "loss": 1.3853, + "margin_dpo/margin_mean": 0.013110697269439697, + "margin_dpo/margin_std": 0.33318257331848145, + "step": 35 + }, + { + "KL/chosen_KL_mean": -0.10409927368164062, + "KL/mean": -0.07765229046344757, + "KL/rejected_KL_mean": -0.051204681396484375, + "KL/std": 0.22821326553821564, + "epoch": 0.05442176870748299, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.05289342999458313, + "fcm_dpo/q_t": 0.5013221502304077, + "grad_norm": 28.933420181274414, + "learning_rate": 2.611940298507462e-07, + "logits/chosen": 0.09718886762857437, + "logits/rejected": 0.04460256174206734, + "logps/chosen": -50.64146041870117, + "logps/ref_chosen": -50.53736114501953, + "logps/ref_rejected": -78.11678314208984, + "logps/rejected": -78.16798400878906, + "loss": 1.3918, + "margin_dpo/margin_mean": -0.052893489599227905, + "margin_dpo/margin_std": 0.3156048059463501, + "step": 36 + }, + { + "KL/chosen_KL_mean": -0.05438232421875, + "KL/mean": -0.10749967396259308, + "KL/rejected_KL_mean": -0.16062164306640625, + "KL/std": 0.2757084369659424, + "epoch": 0.055933484504913075, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.10623124241828918, + "fcm_dpo/q_t": 0.4973453879356384, + "grad_norm": 37.78874969482422, + "learning_rate": 2.686567164179104e-07, + "logits/chosen": 0.07935284823179245, + "logits/rejected": 0.0024427399039268494, + "logps/chosen": -59.60832977294922, + "logps/ref_chosen": -59.55394744873047, + "logps/ref_rejected": -108.27702331542969, + "logps/rejected": -108.43765258789062, + "loss": 1.376, + "margin_dpo/margin_mean": 0.10623079538345337, + "margin_dpo/margin_std": 0.36104801297187805, + "step": 37 + }, + { + "KL/chosen_KL_mean": -0.11186981201171875, + "KL/mean": -0.10196225345134735, + "KL/rejected_KL_mean": -0.0920562744140625, + "KL/std": 0.2679488956928253, + "epoch": 0.05744520030234316, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.019819647073745728, + "fcm_dpo/q_t": 0.5004957318305969, + "grad_norm": 29.816259384155273, + "learning_rate": 2.761194029850746e-07, + "logits/chosen": 0.051365844905376434, + "logits/rejected": 0.03763524442911148, + "logps/chosen": -65.90022277832031, + "logps/ref_chosen": -65.78836059570312, + "logps/ref_rejected": -76.1619873046875, + "logps/rejected": -76.25404357910156, + "loss": 1.3886, + "margin_dpo/margin_mean": -0.019819557666778564, + "margin_dpo/margin_std": 0.36583802103996277, + "step": 38 + }, + { + "KL/chosen_KL_mean": -0.03894615173339844, + "KL/mean": -0.09722255170345306, + "KL/rejected_KL_mean": -0.15549468994140625, + "KL/std": 0.2549492120742798, + "epoch": 0.05895691609977324, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.11654523015022278, + "fcm_dpo/q_t": 0.49708667397499084, + "grad_norm": 29.102598190307617, + "learning_rate": 2.8358208955223876e-07, + "logits/chosen": 0.1645927131175995, + "logits/rejected": 0.13760069012641907, + "logps/chosen": -57.215755462646484, + "logps/ref_chosen": -57.17681121826172, + "logps/ref_rejected": -79.486328125, + "logps/rejected": -79.64183044433594, + "loss": 1.375, + "margin_dpo/margin_mean": 0.11654546856880188, + "margin_dpo/margin_std": 0.3799525499343872, + "step": 39 + }, + { + "KL/chosen_KL_mean": -0.10531044006347656, + "KL/mean": -0.11316066980361938, + "KL/rejected_KL_mean": -0.1210174560546875, + "KL/std": 0.266708105802536, + "epoch": 0.06046863189720333, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0157049298286438, + "fcm_dpo/q_t": 0.4996066093444824, + "grad_norm": 31.49516487121582, + "learning_rate": 2.9104477611940296e-07, + "logits/chosen": 0.09835559129714966, + "logits/rejected": 0.04947054386138916, + "logps/chosen": -61.439476013183594, + "logps/ref_chosen": -61.33416748046875, + "logps/ref_rejected": -79.10697174072266, + "logps/rejected": -79.22798919677734, + "loss": 1.3851, + "margin_dpo/margin_mean": 0.01570478081703186, + "margin_dpo/margin_std": 0.36846187710762024, + "step": 40 + }, + { + "KL/chosen_KL_mean": -0.09158706665039062, + "KL/mean": -0.13693425059318542, + "KL/rejected_KL_mean": -0.18227386474609375, + "KL/std": 0.2849547564983368, + "epoch": 0.06198034769463341, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.09068530797958374, + "fcm_dpo/q_t": 0.49773430824279785, + "grad_norm": 30.41977310180664, + "learning_rate": 2.985074626865671e-07, + "logits/chosen": 0.04748855158686638, + "logits/rejected": 0.02758950926363468, + "logps/chosen": -67.63832092285156, + "logps/ref_chosen": -67.5467300415039, + "logps/ref_rejected": -83.87788391113281, + "logps/rejected": -84.06016540527344, + "loss": 1.3777, + "margin_dpo/margin_mean": 0.09068578481674194, + "margin_dpo/margin_std": 0.40315473079681396, + "step": 41 + }, + { + "KL/chosen_KL_mean": -0.1395435333251953, + "KL/mean": -0.13160492479801178, + "KL/rejected_KL_mean": -0.12366485595703125, + "KL/std": 0.3056698143482208, + "epoch": 0.06349206349206349, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.01587921380996704, + "fcm_dpo/q_t": 0.5003963112831116, + "grad_norm": 29.38926124572754, + "learning_rate": 3.059701492537313e-07, + "logits/chosen": 0.031875789165496826, + "logits/rejected": 0.010956110432744026, + "logps/chosen": -61.40440368652344, + "logps/ref_chosen": -61.26485824584961, + "logps/ref_rejected": -76.3629150390625, + "logps/rejected": -76.48657989501953, + "loss": 1.3882, + "margin_dpo/margin_mean": -0.01587921380996704, + "margin_dpo/margin_std": 0.37683185935020447, + "step": 42 + }, + { + "KL/chosen_KL_mean": -0.14949798583984375, + "KL/mean": -0.16279058158397675, + "KL/rejected_KL_mean": -0.17608070373535156, + "KL/std": 0.32842856645584106, + "epoch": 0.06500377928949358, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.026586264371871948, + "fcm_dpo/q_t": 0.4993341565132141, + "grad_norm": 34.7089729309082, + "learning_rate": 3.134328358208955e-07, + "logits/chosen": 0.09826282411813736, + "logits/rejected": 0.08700218051671982, + "logps/chosen": -71.95852661132812, + "logps/ref_chosen": -71.80902862548828, + "logps/ref_rejected": -81.12464141845703, + "logps/rejected": -81.30072021484375, + "loss": 1.3841, + "margin_dpo/margin_mean": 0.026585698127746582, + "margin_dpo/margin_std": 0.4369037449359894, + "step": 43 + }, + { + "KL/chosen_KL_mean": -0.1836719512939453, + "KL/mean": -0.18749667704105377, + "KL/rejected_KL_mean": -0.19132232666015625, + "KL/std": 0.3246310353279114, + "epoch": 0.06651549508692366, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0076510608196258545, + "fcm_dpo/q_t": 0.49981001019477844, + "grad_norm": 32.50434875488281, + "learning_rate": 3.2089552238805965e-07, + "logits/chosen": 0.04866597801446915, + "logits/rejected": 0.018206467851996422, + "logps/chosen": -66.73410034179688, + "logps/ref_chosen": -66.55043029785156, + "logps/ref_rejected": -85.06198120117188, + "logps/rejected": -85.25330352783203, + "loss": 1.3861, + "margin_dpo/margin_mean": 0.007650971412658691, + "margin_dpo/margin_std": 0.4642822742462158, + "step": 44 + }, + { + "KL/chosen_KL_mean": -0.14289283752441406, + "KL/mean": -0.18554267287254333, + "KL/rejected_KL_mean": -0.22819900512695312, + "KL/std": 0.2894941568374634, + "epoch": 0.06802721088435375, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08530868589878082, + "fcm_dpo/q_t": 0.49786853790283203, + "grad_norm": 31.843597412109375, + "learning_rate": 3.2835820895522385e-07, + "logits/chosen": 0.12124882638454437, + "logits/rejected": 0.06842705607414246, + "logps/chosen": -62.386749267578125, + "logps/ref_chosen": -62.24385452270508, + "logps/ref_rejected": -92.96665954589844, + "logps/rejected": -93.19486236572266, + "loss": 1.3782, + "margin_dpo/margin_mean": 0.0853097140789032, + "margin_dpo/margin_std": 0.4236711859703064, + "step": 45 + }, + { + "KL/chosen_KL_mean": -0.11667251586914062, + "KL/mean": -0.1959172636270523, + "KL/rejected_KL_mean": -0.2751655578613281, + "KL/std": 0.3037590980529785, + "epoch": 0.06953892668178382, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.15848949551582336, + "fcm_dpo/q_t": 0.49603962898254395, + "grad_norm": 31.1035099029541, + "learning_rate": 3.3582089552238805e-07, + "logits/chosen": 0.12088718265295029, + "logits/rejected": 0.07512138783931732, + "logps/chosen": -61.615577697753906, + "logps/ref_chosen": -61.498905181884766, + "logps/ref_rejected": -78.91172790527344, + "logps/rejected": -79.1868896484375, + "loss": 1.3709, + "margin_dpo/margin_mean": 0.15848910808563232, + "margin_dpo/margin_std": 0.39531779289245605, + "step": 46 + }, + { + "KL/chosen_KL_mean": -0.12522506713867188, + "KL/mean": -0.21782416105270386, + "KL/rejected_KL_mean": -0.3104248046875, + "KL/std": 0.27646124362945557, + "epoch": 0.0710506424792139, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.18519717454910278, + "fcm_dpo/q_t": 0.49537193775177, + "grad_norm": 28.432392120361328, + "learning_rate": 3.432835820895522e-07, + "logits/chosen": 0.05430842190980911, + "logits/rejected": 0.01028415560722351, + "logps/chosen": -51.70357131958008, + "logps/ref_chosen": -51.578346252441406, + "logps/ref_rejected": -68.2215576171875, + "logps/rejected": -68.531982421875, + "loss": 1.3682, + "margin_dpo/margin_mean": 0.18519741296768188, + "margin_dpo/margin_std": 0.378314733505249, + "step": 47 + }, + { + "KL/chosen_KL_mean": -0.2560863494873047, + "KL/mean": -0.2473316341638565, + "KL/rejected_KL_mean": -0.23857879638671875, + "KL/std": 0.33754950761795044, + "epoch": 0.07256235827664399, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.017511337995529175, + "fcm_dpo/q_t": 0.5004381537437439, + "grad_norm": 26.973928451538086, + "learning_rate": 3.507462686567164e-07, + "logits/chosen": 0.15267328917980194, + "logits/rejected": 0.12278998643159866, + "logps/chosen": -52.04973602294922, + "logps/ref_chosen": -51.79365158081055, + "logps/ref_rejected": -64.22503662109375, + "logps/rejected": -64.46361541748047, + "loss": 1.3886, + "margin_dpo/margin_mean": -0.017511069774627686, + "margin_dpo/margin_std": 0.47600388526916504, + "step": 48 + }, + { + "KL/chosen_KL_mean": -0.1961212158203125, + "KL/mean": -0.2512929439544678, + "KL/rejected_KL_mean": -0.30646514892578125, + "KL/std": 0.3455272912979126, + "epoch": 0.07407407407407407, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.11034372448921204, + "fcm_dpo/q_t": 0.49724340438842773, + "grad_norm": 27.25345230102539, + "learning_rate": 3.5820895522388055e-07, + "logits/chosen": 0.013772407546639442, + "logits/rejected": -0.007179133594036102, + "logps/chosen": -58.33072280883789, + "logps/ref_chosen": -58.13460159301758, + "logps/ref_rejected": -64.63206481933594, + "logps/rejected": -64.93853759765625, + "loss": 1.3758, + "margin_dpo/margin_mean": 0.11034420132637024, + "margin_dpo/margin_std": 0.468539834022522, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.3067760467529297, + "KL/mean": -0.3461652398109436, + "KL/rejected_KL_mean": -0.3855552673339844, + "KL/std": 0.3353579044342041, + "epoch": 0.07558578987150416, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.07877588272094727, + "fcm_dpo/q_t": 0.4980308413505554, + "grad_norm": 27.978439331054688, + "learning_rate": 3.6567164179104475e-07, + "logits/chosen": 0.10822086036205292, + "logits/rejected": 0.07860735058784485, + "logps/chosen": -53.16321563720703, + "logps/ref_chosen": -52.85643768310547, + "logps/ref_rejected": -72.17460632324219, + "logps/rejected": -72.5601577758789, + "loss": 1.3788, + "margin_dpo/margin_mean": 0.07877624034881592, + "margin_dpo/margin_std": 0.39916181564331055, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.2651844024658203, + "KL/mean": -0.37544912099838257, + "KL/rejected_KL_mean": -0.4857177734375, + "KL/std": 0.3876107335090637, + "epoch": 0.07709750566893424, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.22053340077400208, + "fcm_dpo/q_t": 0.4944919943809509, + "grad_norm": 31.027009963989258, + "learning_rate": 3.7313432835820895e-07, + "logits/chosen": 0.10386451333761215, + "logits/rejected": 0.0755525678396225, + "logps/chosen": -63.921627044677734, + "logps/ref_chosen": -63.65644073486328, + "logps/ref_rejected": -86.13229370117188, + "logps/rejected": -86.61801147460938, + "loss": 1.365, + "margin_dpo/margin_mean": 0.22053375840187073, + "margin_dpo/margin_std": 0.5061776041984558, + "step": 51 + }, + { + "KL/chosen_KL_mean": -0.3102092742919922, + "KL/mean": -0.3927253186702728, + "KL/rejected_KL_mean": -0.47524261474609375, + "KL/std": 0.3908153176307678, + "epoch": 0.07860922146636433, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.16503168642520905, + "fcm_dpo/q_t": 0.49587881565093994, + "grad_norm": 32.90274429321289, + "learning_rate": 3.805970149253731e-07, + "logits/chosen": 0.10359272360801697, + "logits/rejected": 0.05269782245159149, + "logps/chosen": -68.15042114257812, + "logps/ref_chosen": -67.8402099609375, + "logps/ref_rejected": -96.97090911865234, + "logps/rejected": -97.44615173339844, + "loss": 1.3705, + "margin_dpo/margin_mean": 0.16503138840198517, + "margin_dpo/margin_std": 0.5094035863876343, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.35711097717285156, + "KL/mean": -0.4368184208869934, + "KL/rejected_KL_mean": -0.5165271759033203, + "KL/std": 0.3795577883720398, + "epoch": 0.0801209372637944, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.15941789746284485, + "fcm_dpo/q_t": 0.49601683020591736, + "grad_norm": 27.472475051879883, + "learning_rate": 3.880597014925373e-07, + "logits/chosen": 0.08907319605350494, + "logits/rejected": 0.07837893813848495, + "logps/chosen": -57.235252380371094, + "logps/ref_chosen": -56.87813949584961, + "logps/ref_rejected": -60.75569152832031, + "logps/rejected": -61.272220611572266, + "loss": 1.371, + "margin_dpo/margin_mean": 0.1594180166721344, + "margin_dpo/margin_std": 0.46473363041877747, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.36806488037109375, + "KL/mean": -0.4340946674346924, + "KL/rejected_KL_mean": -0.5001239776611328, + "KL/std": 0.42446714639663696, + "epoch": 0.08163265306122448, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.13206136226654053, + "fcm_dpo/q_t": 0.49670255184173584, + "grad_norm": 26.566865921020508, + "learning_rate": 3.9552238805970144e-07, + "logits/chosen": 0.0764525830745697, + "logits/rejected": 0.060887373983860016, + "logps/chosen": -47.634986877441406, + "logps/ref_chosen": -47.26692199707031, + "logps/ref_rejected": -62.19426727294922, + "logps/rejected": -62.69438934326172, + "loss": 1.3739, + "margin_dpo/margin_mean": 0.13206100463867188, + "margin_dpo/margin_std": 0.5414613485336304, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.369964599609375, + "KL/mean": -0.5161944031715393, + "KL/rejected_KL_mean": -0.6624259948730469, + "KL/std": 0.48815223574638367, + "epoch": 0.08314436885865457, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2924594283103943, + "fcm_dpo/q_t": 0.49270111322402954, + "grad_norm": 31.41529083251953, + "learning_rate": 4.0298507462686564e-07, + "logits/chosen": 0.03972642123699188, + "logits/rejected": -0.036940090358257294, + "logps/chosen": -50.6961555480957, + "logps/ref_chosen": -50.32619094848633, + "logps/ref_rejected": -92.44389343261719, + "logps/rejected": -93.10631561279297, + "loss": 1.3584, + "margin_dpo/margin_mean": 0.29245901107788086, + "margin_dpo/margin_std": 0.6678496599197388, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.38695526123046875, + "KL/mean": -0.4743455946445465, + "KL/rejected_KL_mean": -0.5617294311523438, + "KL/std": 0.5216856002807617, + "epoch": 0.08465608465608465, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.17477595806121826, + "fcm_dpo/q_t": 0.49562889337539673, + "grad_norm": 27.613101959228516, + "learning_rate": 4.1044776119402984e-07, + "logits/chosen": 0.13359126448631287, + "logits/rejected": 0.11104871332645416, + "logps/chosen": -57.153926849365234, + "logps/ref_chosen": -56.766971588134766, + "logps/ref_rejected": -66.30504608154297, + "logps/rejected": -66.86677551269531, + "loss": 1.37, + "margin_dpo/margin_mean": 0.17477604746818542, + "margin_dpo/margin_std": 0.6422098875045776, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.5014972686767578, + "KL/mean": -0.6661262512207031, + "KL/rejected_KL_mean": -0.83074951171875, + "KL/std": 0.6199729442596436, + "epoch": 0.08616780045351474, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.32925212383270264, + "fcm_dpo/q_t": 0.4917835295200348, + "grad_norm": 30.380630493164062, + "learning_rate": 4.17910447761194e-07, + "logits/chosen": 0.09156804531812668, + "logits/rejected": 0.027995580807328224, + "logps/chosen": -58.26924133300781, + "logps/ref_chosen": -57.76774597167969, + "logps/ref_rejected": -82.75698852539062, + "logps/rejected": -83.58773803710938, + "loss": 1.3549, + "margin_dpo/margin_mean": 0.3292523920536041, + "margin_dpo/margin_std": 0.6963714361190796, + "step": 57 + }, + { + "KL/chosen_KL_mean": -0.5995521545410156, + "KL/mean": -0.7129923105239868, + "KL/rejected_KL_mean": -0.8264312744140625, + "KL/std": 0.6734466552734375, + "epoch": 0.08767951625094482, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.22688013315200806, + "fcm_dpo/q_t": 0.49438679218292236, + "grad_norm": 30.21674346923828, + "learning_rate": 4.253731343283582e-07, + "logits/chosen": 0.06498520076274872, + "logits/rejected": 0.04932039976119995, + "logps/chosen": -73.36363983154297, + "logps/ref_chosen": -72.76408386230469, + "logps/ref_rejected": -84.49275207519531, + "logps/rejected": -85.31918334960938, + "loss": 1.3665, + "margin_dpo/margin_mean": 0.22687986493110657, + "margin_dpo/margin_std": 0.9897109270095825, + "step": 58 + }, + { + "KL/chosen_KL_mean": -0.5455207824707031, + "KL/mean": -0.7040902972221375, + "KL/rejected_KL_mean": -0.8626651763916016, + "KL/std": 0.6501806974411011, + "epoch": 0.08919123204837491, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3171398937702179, + "fcm_dpo/q_t": 0.49210113286972046, + "grad_norm": 26.566312789916992, + "learning_rate": 4.3283582089552234e-07, + "logits/chosen": 0.10520491003990173, + "logits/rejected": 0.0415894091129303, + "logps/chosen": -50.366294860839844, + "logps/ref_chosen": -49.820777893066406, + "logps/ref_rejected": -77.14368438720703, + "logps/rejected": -78.00634765625, + "loss": 1.3567, + "margin_dpo/margin_mean": 0.31714025139808655, + "margin_dpo/margin_std": 0.8397043943405151, + "step": 59 + }, + { + "KL/chosen_KL_mean": -0.6186370849609375, + "KL/mean": -0.6712080240249634, + "KL/rejected_KL_mean": -0.7237796783447266, + "KL/std": 0.5914332866668701, + "epoch": 0.09070294784580499, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.10513994097709656, + "fcm_dpo/q_t": 0.4973700940608978, + "grad_norm": 29.50324821472168, + "learning_rate": 4.4029850746268654e-07, + "logits/chosen": 0.09925903379917145, + "logits/rejected": 0.09783255308866501, + "logps/chosen": -63.84341049194336, + "logps/ref_chosen": -63.22477340698242, + "logps/ref_rejected": -61.360477447509766, + "logps/rejected": -62.084259033203125, + "loss": 1.3773, + "margin_dpo/margin_mean": 0.10513973236083984, + "margin_dpo/margin_std": 0.7537751197814941, + "step": 60 + }, + { + "KL/chosen_KL_mean": -0.7250652313232422, + "KL/mean": -0.7618651390075684, + "KL/rejected_KL_mean": -0.7986640930175781, + "KL/std": 0.730823278427124, + "epoch": 0.09221466364323508, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.07359689474105835, + "fcm_dpo/q_t": 0.4981638789176941, + "grad_norm": 28.105127334594727, + "learning_rate": 4.4776119402985074e-07, + "logits/chosen": 0.13474620878696442, + "logits/rejected": 0.10221856087446213, + "logps/chosen": -49.74186325073242, + "logps/ref_chosen": -49.01679992675781, + "logps/ref_rejected": -74.90817260742188, + "logps/rejected": -75.70683288574219, + "loss": 1.3817, + "margin_dpo/margin_mean": 0.07359689474105835, + "margin_dpo/margin_std": 1.0419827699661255, + "step": 61 + }, + { + "KL/chosen_KL_mean": -0.7413101196289062, + "KL/mean": -0.8587090969085693, + "KL/rejected_KL_mean": -0.9761085510253906, + "KL/std": 0.6620085835456848, + "epoch": 0.09372637944066516, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.23479923605918884, + "fcm_dpo/q_t": 0.49413466453552246, + "grad_norm": 28.633806228637695, + "learning_rate": 4.552238805970149e-07, + "logits/chosen": 0.08576183021068573, + "logits/rejected": 0.04774682596325874, + "logps/chosen": -63.49317932128906, + "logps/ref_chosen": -62.751869201660156, + "logps/ref_rejected": -78.93360900878906, + "logps/rejected": -79.90971374511719, + "loss": 1.3652, + "margin_dpo/margin_mean": 0.23479923605918884, + "margin_dpo/margin_std": 0.927647054195404, + "step": 62 + }, + { + "KL/chosen_KL_mean": -0.5629634857177734, + "KL/mean": -0.8584684133529663, + "KL/rejected_KL_mean": -1.1539764404296875, + "KL/std": 0.6872652173042297, + "epoch": 0.09523809523809523, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5910177230834961, + "fcm_dpo/q_t": 0.4852627217769623, + "grad_norm": 31.272174835205078, + "learning_rate": 4.626865671641791e-07, + "logits/chosen": 0.18413883447647095, + "logits/rejected": 0.15941709280014038, + "logps/chosen": -61.07821273803711, + "logps/ref_chosen": -60.51525115966797, + "logps/ref_rejected": -85.11021423339844, + "logps/rejected": -86.26419067382812, + "loss": 1.3298, + "margin_dpo/margin_mean": 0.591017484664917, + "margin_dpo/margin_std": 0.8419014811515808, + "step": 63 + }, + { + "KL/chosen_KL_mean": -0.8456764221191406, + "KL/mean": -0.8847048282623291, + "KL/rejected_KL_mean": -0.923736572265625, + "KL/std": 0.7426958084106445, + "epoch": 0.09674981103552532, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0780559778213501, + "fcm_dpo/q_t": 0.4980408251285553, + "grad_norm": 26.20693016052246, + "learning_rate": 4.701492537313433e-07, + "logits/chosen": 0.09699878096580505, + "logits/rejected": 0.07188250124454498, + "logps/chosen": -52.052520751953125, + "logps/ref_chosen": -51.20684814453125, + "logps/ref_rejected": -66.93081665039062, + "logps/rejected": -67.85455322265625, + "loss": 1.3807, + "margin_dpo/margin_mean": 0.07805639505386353, + "margin_dpo/margin_std": 0.9259697794914246, + "step": 64 + }, + { + "KL/chosen_KL_mean": -0.8305568695068359, + "KL/mean": -1.0773502588272095, + "KL/rejected_KL_mean": -1.3241462707519531, + "KL/std": 0.9229645729064941, + "epoch": 0.0982615268329554, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.49359071254730225, + "fcm_dpo/q_t": 0.48773229122161865, + "grad_norm": 30.839191436767578, + "learning_rate": 4.776119402985074e-07, + "logits/chosen": 0.1971920132637024, + "logits/rejected": 0.16757403314113617, + "logps/chosen": -68.11924743652344, + "logps/ref_chosen": -67.2886962890625, + "logps/ref_rejected": -74.44281005859375, + "logps/rejected": -75.76695251464844, + "loss": 1.3412, + "margin_dpo/margin_mean": 0.49358999729156494, + "margin_dpo/margin_std": 1.2063360214233398, + "step": 65 + }, + { + "KL/chosen_KL_mean": -0.9015731811523438, + "KL/mean": -1.0950579643249512, + "KL/rejected_KL_mean": -1.2885398864746094, + "KL/std": 0.8506461381912231, + "epoch": 0.09977324263038549, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3869714140892029, + "fcm_dpo/q_t": 0.4903630018234253, + "grad_norm": 29.211280822753906, + "learning_rate": 4.850746268656717e-07, + "logits/chosen": 0.07336064428091049, + "logits/rejected": 0.05018358305096626, + "logps/chosen": -71.64498901367188, + "logps/ref_chosen": -70.743408203125, + "logps/ref_rejected": -77.26499938964844, + "logps/rejected": -78.55354309082031, + "loss": 1.3513, + "margin_dpo/margin_mean": 0.3869715631008148, + "margin_dpo/margin_std": 1.139390230178833, + "step": 66 + }, + { + "KL/chosen_KL_mean": -0.7808132171630859, + "KL/mean": -0.936085045337677, + "KL/rejected_KL_mean": -1.0913543701171875, + "KL/std": 0.7684589624404907, + "epoch": 0.10128495842781557, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3105343282222748, + "fcm_dpo/q_t": 0.49226200580596924, + "grad_norm": 29.08330726623535, + "learning_rate": 4.925373134328357e-07, + "logits/chosen": 0.11251701414585114, + "logits/rejected": 0.05569060891866684, + "logps/chosen": -61.38341522216797, + "logps/ref_chosen": -60.60260009765625, + "logps/ref_rejected": -75.22235870361328, + "logps/rejected": -76.31371307373047, + "loss": 1.3578, + "margin_dpo/margin_mean": 0.3105342984199524, + "margin_dpo/margin_std": 0.95625901222229, + "step": 67 + }, + { + "KL/chosen_KL_mean": -1.2724533081054688, + "KL/mean": -1.4205259084701538, + "KL/rejected_KL_mean": -1.5685997009277344, + "KL/std": 0.9099494218826294, + "epoch": 0.10279667422524566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.29615065455436707, + "fcm_dpo/q_t": 0.4926130473613739, + "grad_norm": 31.419614791870117, + "learning_rate": 5e-07, + "logits/chosen": 0.04703470319509506, + "logits/rejected": 0.01785670965909958, + "logps/chosen": -78.80081939697266, + "logps/ref_chosen": -77.52836608886719, + "logps/ref_rejected": -93.17778015136719, + "logps/rejected": -94.74637603759766, + "loss": 1.3604, + "margin_dpo/margin_mean": 0.2961508333683014, + "margin_dpo/margin_std": 1.1492292881011963, + "step": 68 + }, + { + "KL/chosen_KL_mean": -1.152212142944336, + "KL/mean": -1.4678752422332764, + "KL/rejected_KL_mean": -1.7835311889648438, + "KL/std": 0.9967954158782959, + "epoch": 0.10430839002267574, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6313230991363525, + "fcm_dpo/q_t": 0.48431456089019775, + "grad_norm": 30.98222541809082, + "learning_rate": 4.999965034812934e-07, + "logits/chosen": 0.08557026088237762, + "logits/rejected": 0.04331623762845993, + "logps/chosen": -67.09526062011719, + "logps/ref_chosen": -65.94305419921875, + "logps/ref_rejected": -89.7735595703125, + "logps/rejected": -91.55709075927734, + "loss": 1.3278, + "margin_dpo/margin_mean": 0.6313223838806152, + "margin_dpo/margin_std": 1.183309555053711, + "step": 69 + }, + { + "KL/chosen_KL_mean": -1.2136993408203125, + "KL/mean": -1.3960089683532715, + "KL/rejected_KL_mean": -1.5783195495605469, + "KL/std": 0.8884181976318359, + "epoch": 0.10582010582010581, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3646145164966583, + "fcm_dpo/q_t": 0.4909105896949768, + "grad_norm": 28.629304885864258, + "learning_rate": 4.999860140229787e-07, + "logits/chosen": 0.1712493747472763, + "logits/rejected": 0.1474042534828186, + "logps/chosen": -63.17161178588867, + "logps/ref_chosen": -61.95791244506836, + "logps/ref_rejected": -75.80945587158203, + "logps/rejected": -77.38777923583984, + "loss": 1.3539, + "margin_dpo/margin_mean": 0.3646150529384613, + "margin_dpo/margin_std": 1.2180376052856445, + "step": 70 + }, + { + "KL/chosen_KL_mean": -1.4519882202148438, + "KL/mean": -1.560695767402649, + "KL/rejected_KL_mean": -1.6694049835205078, + "KL/std": 1.0225567817687988, + "epoch": 0.1073318216175359, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.21741512417793274, + "fcm_dpo/q_t": 0.4945552945137024, + "grad_norm": 28.560457229614258, + "learning_rate": 4.999685319184688e-07, + "logits/chosen": 0.10548342764377594, + "logits/rejected": 0.08939765393733978, + "logps/chosen": -64.79956817626953, + "logps/ref_chosen": -63.34757995605469, + "logps/ref_rejected": -67.49658203125, + "logps/rejected": -69.16598510742188, + "loss": 1.37, + "margin_dpo/margin_mean": 0.2174154818058014, + "margin_dpo/margin_std": 1.438173770904541, + "step": 71 + }, + { + "KL/chosen_KL_mean": -1.1966476440429688, + "KL/mean": -1.5650217533111572, + "KL/rejected_KL_mean": -1.9333915710449219, + "KL/std": 1.066718339920044, + "epoch": 0.10884353741496598, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7367445826530457, + "fcm_dpo/q_t": 0.4816743731498718, + "grad_norm": 30.220441818237305, + "learning_rate": 4.999440576567755e-07, + "logits/chosen": 0.1182754710316658, + "logits/rejected": 0.05511663109064102, + "logps/chosen": -57.055946350097656, + "logps/ref_chosen": -55.85929870605469, + "logps/ref_rejected": -68.45423889160156, + "logps/rejected": -70.38763427734375, + "loss": 1.3184, + "margin_dpo/margin_mean": 0.7367447018623352, + "margin_dpo/margin_std": 1.2988572120666504, + "step": 72 + }, + { + "KL/chosen_KL_mean": -1.8454322814941406, + "KL/mean": -1.8656189441680908, + "KL/rejected_KL_mean": -1.8858089447021484, + "KL/std": 1.2100117206573486, + "epoch": 0.11035525321239607, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.040384888648986816, + "fcm_dpo/q_t": 0.49911391735076904, + "grad_norm": 32.26237106323242, + "learning_rate": 4.999125919224965e-07, + "logits/chosen": 0.09246772527694702, + "logits/rejected": 0.07867833971977234, + "logps/chosen": -70.98423767089844, + "logps/ref_chosen": -69.13880920410156, + "logps/ref_rejected": -79.04586791992188, + "logps/rejected": -80.93167877197266, + "loss": 1.3884, + "margin_dpo/margin_mean": 0.04038524627685547, + "margin_dpo/margin_std": 1.5114188194274902, + "step": 73 + }, + { + "KL/chosen_KL_mean": -1.2555103302001953, + "KL/mean": -1.619905710220337, + "KL/rejected_KL_mean": -1.9843025207519531, + "KL/std": 1.012344479560852, + "epoch": 0.11186696900982615, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7287911176681519, + "fcm_dpo/q_t": 0.48189249634742737, + "grad_norm": 27.792308807373047, + "learning_rate": 4.998741355957963e-07, + "logits/chosen": 0.14601758122444153, + "logits/rejected": 0.09223125129938126, + "logps/chosen": -51.17924499511719, + "logps/ref_chosen": -49.923736572265625, + "logps/ref_rejected": -81.73213958740234, + "logps/rejected": -83.71644592285156, + "loss": 1.3192, + "margin_dpo/margin_mean": 0.7287905812263489, + "margin_dpo/margin_std": 1.300370216369629, + "step": 74 + }, + { + "KL/chosen_KL_mean": -1.3832435607910156, + "KL/mean": -1.7584812641143799, + "KL/rejected_KL_mean": -2.1337146759033203, + "KL/std": 1.0733695030212402, + "epoch": 0.11337868480725624, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7504755258560181, + "fcm_dpo/q_t": 0.48133933544158936, + "grad_norm": 26.247146606445312, + "learning_rate": 4.998286897523808e-07, + "logits/chosen": 0.11124646663665771, + "logits/rejected": 0.07929684221744537, + "logps/chosen": -47.451995849609375, + "logps/ref_chosen": -46.06875228881836, + "logps/ref_rejected": -66.1181411743164, + "logps/rejected": -68.25186157226562, + "loss": 1.3169, + "margin_dpo/margin_mean": 0.7504750490188599, + "margin_dpo/margin_std": 1.2960433959960938, + "step": 75 + }, + { + "KL/chosen_KL_mean": -1.6049728393554688, + "KL/mean": -1.7576977014541626, + "KL/rejected_KL_mean": -1.9104232788085938, + "KL/std": 1.2057301998138428, + "epoch": 0.11489040060468632, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.30545294284820557, + "fcm_dpo/q_t": 0.49238139390945435, + "grad_norm": 29.37653160095215, + "learning_rate": 4.997762556634679e-07, + "logits/chosen": 0.1120544895529747, + "logits/rejected": 0.06925463676452637, + "logps/chosen": -55.667724609375, + "logps/ref_chosen": -54.06275177001953, + "logps/ref_rejected": -74.87464141845703, + "logps/rejected": -76.78506469726562, + "loss": 1.3618, + "margin_dpo/margin_mean": 0.305452823638916, + "margin_dpo/margin_std": 1.5330884456634521, + "step": 76 + }, + { + "KL/chosen_KL_mean": -1.575113296508789, + "KL/mean": -1.9794244766235352, + "KL/rejected_KL_mean": -2.3837318420410156, + "KL/std": 1.2247073650360107, + "epoch": 0.1164021164021164, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.808618426322937, + "fcm_dpo/q_t": 0.47992613911628723, + "grad_norm": 29.050642013549805, + "learning_rate": 4.99716834795752e-07, + "logits/chosen": 0.16106051206588745, + "logits/rejected": 0.11938437819480896, + "logps/chosen": -54.651206970214844, + "logps/ref_chosen": -53.07609176635742, + "logps/ref_rejected": -74.45601654052734, + "logps/rejected": -76.83975219726562, + "loss": 1.3124, + "margin_dpo/margin_mean": 0.8086180686950684, + "margin_dpo/margin_std": 1.4267933368682861, + "step": 77 + }, + { + "KL/chosen_KL_mean": -1.8742866516113281, + "KL/mean": -2.0521607398986816, + "KL/rejected_KL_mean": -2.2300338745117188, + "KL/std": 1.3071465492248535, + "epoch": 0.11791383219954649, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3557405471801758, + "fcm_dpo/q_t": 0.4911442995071411, + "grad_norm": 29.603862762451172, + "learning_rate": 4.996504288113623e-07, + "logits/chosen": 0.09281802177429199, + "logits/rejected": 0.07283802330493927, + "logps/chosen": -69.5997085571289, + "logps/ref_chosen": -67.72541809082031, + "logps/ref_rejected": -79.03926849365234, + "logps/rejected": -81.26930236816406, + "loss": 1.3563, + "margin_dpo/margin_mean": 0.35574063658714294, + "margin_dpo/margin_std": 1.4488558769226074, + "step": 78 + }, + { + "KL/chosen_KL_mean": -1.71051025390625, + "KL/mean": -2.288090467453003, + "KL/rejected_KL_mean": -2.8656692504882812, + "KL/std": 1.5266368389129639, + "epoch": 0.11942554799697656, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1551591157913208, + "fcm_dpo/q_t": 0.4716222286224365, + "grad_norm": 30.841747283935547, + "learning_rate": 4.995770395678171e-07, + "logits/chosen": 0.1539367437362671, + "logits/rejected": 0.09523071348667145, + "logps/chosen": -53.87115478515625, + "logps/ref_chosen": -52.16064453125, + "logps/ref_rejected": -83.31062316894531, + "logps/rejected": -86.17628479003906, + "loss": 1.2838, + "margin_dpo/margin_mean": 1.1551592350006104, + "margin_dpo/margin_std": 1.974243402481079, + "step": 79 + }, + { + "KL/chosen_KL_mean": -2.048025131225586, + "KL/mean": -2.3210818767547607, + "KL/rejected_KL_mean": -2.594135284423828, + "KL/std": 1.3664963245391846, + "epoch": 0.12093726379440665, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5461106300354004, + "fcm_dpo/q_t": 0.486517071723938, + "grad_norm": 28.41274070739746, + "learning_rate": 4.994966691179711e-07, + "logits/chosen": 0.16208377480506897, + "logits/rejected": 0.10084976255893707, + "logps/chosen": -63.458587646484375, + "logps/ref_chosen": -61.410560607910156, + "logps/ref_rejected": -78.66004943847656, + "logps/rejected": -81.25418090820312, + "loss": 1.3421, + "margin_dpo/margin_mean": 0.546110987663269, + "margin_dpo/margin_std": 1.9074268341064453, + "step": 80 + }, + { + "KL/chosen_KL_mean": -2.1474876403808594, + "KL/mean": -2.5709149837493896, + "KL/rejected_KL_mean": -2.9943389892578125, + "KL/std": 1.4404411315917969, + "epoch": 0.12244897959183673, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8468587398529053, + "fcm_dpo/q_t": 0.47913920879364014, + "grad_norm": 29.03059196472168, + "learning_rate": 4.994093197099587e-07, + "logits/chosen": 0.1493208110332489, + "logits/rejected": 0.11450086534023285, + "logps/chosen": -65.95185852050781, + "logps/ref_chosen": -63.80437088012695, + "logps/ref_rejected": -79.3484115600586, + "logps/rejected": -82.3427505493164, + "loss": 1.3115, + "margin_dpo/margin_mean": 0.8468592166900635, + "margin_dpo/margin_std": 1.740824580192566, + "step": 81 + }, + { + "KL/chosen_KL_mean": -1.7643756866455078, + "KL/mean": -2.288670539855957, + "KL/rejected_KL_mean": -2.812967300415039, + "KL/std": 1.3698699474334717, + "epoch": 0.12396069538926682, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0485919713974, + "fcm_dpo/q_t": 0.47396597266197205, + "grad_norm": 26.609189987182617, + "learning_rate": 4.993149937871306e-07, + "logits/chosen": 0.12430812418460846, + "logits/rejected": 0.06034460663795471, + "logps/chosen": -50.58226776123047, + "logps/ref_chosen": -48.817893981933594, + "logps/ref_rejected": -70.31497955322266, + "logps/rejected": -73.12794494628906, + "loss": 1.2906, + "margin_dpo/margin_mean": 1.0485917329788208, + "margin_dpo/margin_std": 1.5989954471588135, + "step": 82 + }, + { + "KL/chosen_KL_mean": -2.0936527252197266, + "KL/mean": -2.651911497116089, + "KL/rejected_KL_mean": -3.210174560546875, + "KL/std": 1.4586286544799805, + "epoch": 0.1254724111866969, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1165223121643066, + "fcm_dpo/q_t": 0.47243359684944153, + "grad_norm": 29.370174407958984, + "learning_rate": 4.992136939879856e-07, + "logits/chosen": 0.203558087348938, + "logits/rejected": 0.15304410457611084, + "logps/chosen": -59.24442672729492, + "logps/ref_chosen": -57.15077209472656, + "logps/ref_rejected": -75.1710205078125, + "logps/rejected": -78.38119506835938, + "loss": 1.286, + "margin_dpo/margin_mean": 1.1165223121643066, + "margin_dpo/margin_std": 1.8079650402069092, + "step": 83 + }, + { + "KL/chosen_KL_mean": -2.5359344482421875, + "KL/mean": -2.9698383808135986, + "KL/rejected_KL_mean": -3.40374755859375, + "KL/std": 1.8658125400543213, + "epoch": 0.12698412698412698, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8678133487701416, + "fcm_dpo/q_t": 0.47854459285736084, + "grad_norm": 30.701486587524414, + "learning_rate": 4.991054231460969e-07, + "logits/chosen": 0.1644321233034134, + "logits/rejected": 0.12267635762691498, + "logps/chosen": -67.313232421875, + "logps/ref_chosen": -64.77729797363281, + "logps/ref_rejected": -84.71949768066406, + "logps/rejected": -88.12324523925781, + "loss": 1.3107, + "margin_dpo/margin_mean": 0.8678131103515625, + "margin_dpo/margin_std": 1.9344353675842285, + "step": 84 + }, + { + "KL/chosen_KL_mean": -2.246265411376953, + "KL/mean": -2.887638568878174, + "KL/rejected_KL_mean": -3.5290069580078125, + "KL/std": 1.7709710597991943, + "epoch": 0.12849584278155707, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2827439308166504, + "fcm_dpo/q_t": 0.46833422780036926, + "grad_norm": 26.880765914916992, + "learning_rate": 4.989901842900325e-07, + "logits/chosen": 0.15874062478542328, + "logits/rejected": 0.11513642966747284, + "logps/chosen": -52.497962951660156, + "logps/ref_chosen": -50.25169372558594, + "logps/ref_rejected": -66.55439758300781, + "logps/rejected": -70.08340454101562, + "loss": 1.2723, + "margin_dpo/margin_mean": 1.2827439308166504, + "margin_dpo/margin_std": 1.971299409866333, + "step": 85 + }, + { + "KL/chosen_KL_mean": -2.840503692626953, + "KL/mean": -3.3124403953552246, + "KL/rejected_KL_mean": -3.784381866455078, + "KL/std": 1.908517837524414, + "epoch": 0.13000755857898716, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.9438769817352295, + "fcm_dpo/q_t": 0.4766838848590851, + "grad_norm": 27.22740364074707, + "learning_rate": 4.988679806432711e-07, + "logits/chosen": 0.20484548807144165, + "logits/rejected": 0.18582268059253693, + "logps/chosen": -63.56968307495117, + "logps/ref_chosen": -60.72917938232422, + "logps/ref_rejected": -72.30961608886719, + "logps/rejected": -76.093994140625, + "loss": 1.305, + "margin_dpo/margin_mean": 0.9438771605491638, + "margin_dpo/margin_std": 2.0875279903411865, + "step": 86 + }, + { + "KL/chosen_KL_mean": -2.993976593017578, + "KL/mean": -3.515448808670044, + "KL/rejected_KL_mean": -4.036918640136719, + "KL/std": 2.058159589767456, + "epoch": 0.13151927437641722, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0429438352584839, + "fcm_dpo/q_t": 0.4742722809314728, + "grad_norm": 30.60996437072754, + "learning_rate": 4.987388156241114e-07, + "logits/chosen": 0.1737605780363083, + "logits/rejected": 0.1141999289393425, + "logps/chosen": -68.75194549560547, + "logps/ref_chosen": -65.75796508789062, + "logps/ref_rejected": -84.81159973144531, + "logps/rejected": -88.8485107421875, + "loss": 1.3, + "margin_dpo/margin_mean": 1.0429439544677734, + "margin_dpo/margin_std": 2.4612228870391846, + "step": 87 + }, + { + "KL/chosen_KL_mean": -3.053285598754883, + "KL/mean": -3.4472389221191406, + "KL/rejected_KL_mean": -3.8411865234375, + "KL/std": 2.2897181510925293, + "epoch": 0.1330309901738473, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7879018783569336, + "fcm_dpo/q_t": 0.48072606325149536, + "grad_norm": 30.390886306762695, + "learning_rate": 4.986026928455767e-07, + "logits/chosen": 0.22705325484275818, + "logits/rejected": 0.20026172697544098, + "logps/chosen": -65.87731170654297, + "logps/ref_chosen": -62.82402801513672, + "logps/ref_rejected": -74.9607162475586, + "logps/rejected": -78.8019027709961, + "loss": 1.3306, + "margin_dpo/margin_mean": 0.7879022359848022, + "margin_dpo/margin_std": 2.8485002517700195, + "step": 88 + }, + { + "KL/chosen_KL_mean": -2.558351516723633, + "KL/mean": -3.2431600093841553, + "KL/rejected_KL_mean": -3.927967071533203, + "KL/std": 2.133073329925537, + "epoch": 0.1345427059712774, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3696104288101196, + "fcm_dpo/q_t": 0.46659523248672485, + "grad_norm": 29.33757209777832, + "learning_rate": 4.984596161153135e-07, + "logits/chosen": 0.24068006873130798, + "logits/rejected": 0.15901678800582886, + "logps/chosen": -43.749786376953125, + "logps/ref_chosen": -41.191436767578125, + "logps/ref_rejected": -85.44769287109375, + "logps/rejected": -89.37566375732422, + "loss": 1.2704, + "margin_dpo/margin_mean": 1.369611382484436, + "margin_dpo/margin_std": 2.50301194190979, + "step": 89 + }, + { + "KL/chosen_KL_mean": -3.100625991821289, + "KL/mean": -3.7633066177368164, + "KL/rejected_KL_mean": -4.425987243652344, + "KL/std": 2.4566781520843506, + "epoch": 0.1360544217687075, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3253576755523682, + "fcm_dpo/q_t": 0.4674978256225586, + "grad_norm": 29.031574249267578, + "learning_rate": 4.983095894354857e-07, + "logits/chosen": 0.14617952704429626, + "logits/rejected": 0.09175971895456314, + "logps/chosen": -59.684532165527344, + "logps/ref_chosen": -56.58390808105469, + "logps/ref_rejected": -86.86978149414062, + "logps/rejected": -91.29576873779297, + "loss": 1.2746, + "margin_dpo/margin_mean": 1.3253577947616577, + "margin_dpo/margin_std": 2.5310139656066895, + "step": 90 + }, + { + "KL/chosen_KL_mean": -3.18841552734375, + "KL/mean": -3.7420177459716797, + "KL/rejected_KL_mean": -4.295619964599609, + "KL/std": 2.2847001552581787, + "epoch": 0.13756613756613756, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1072009801864624, + "fcm_dpo/q_t": 0.473149836063385, + "grad_norm": 25.827465057373047, + "learning_rate": 4.98152617002662e-07, + "logits/chosen": 0.15624286234378815, + "logits/rejected": 0.11258872598409653, + "logps/chosen": -55.57075881958008, + "logps/ref_chosen": -52.38234329223633, + "logps/ref_rejected": -72.17642211914062, + "logps/rejected": -76.4720458984375, + "loss": 1.2981, + "margin_dpo/margin_mean": 1.1072008609771729, + "margin_dpo/margin_std": 2.7962405681610107, + "step": 91 + }, + { + "KL/chosen_KL_mean": -3.2339859008789062, + "KL/mean": -4.029601097106934, + "KL/rejected_KL_mean": -4.825218200683594, + "KL/std": 2.677659511566162, + "epoch": 0.13907785336356765, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.591225266456604, + "fcm_dpo/q_t": 0.46159979701042175, + "grad_norm": 26.94463348388672, + "learning_rate": 4.979887032076988e-07, + "logits/chosen": 0.19445021450519562, + "logits/rejected": 0.15572763979434967, + "logps/chosen": -56.2426872253418, + "logps/ref_chosen": -53.00870132446289, + "logps/ref_rejected": -79.77812957763672, + "logps/rejected": -84.60334777832031, + "loss": 1.2577, + "margin_dpo/margin_mean": 1.591225504875183, + "margin_dpo/margin_std": 3.1309449672698975, + "step": 92 + }, + { + "KL/chosen_KL_mean": -3.5741615295410156, + "KL/mean": -4.070510387420654, + "KL/rejected_KL_mean": -4.566856384277344, + "KL/std": 2.6192855834960938, + "epoch": 0.14058956916099774, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.992700457572937, + "fcm_dpo/q_t": 0.47632110118865967, + "grad_norm": 24.588478088378906, + "learning_rate": 4.978178526356172e-07, + "logits/chosen": 0.19541963934898376, + "logits/rejected": 0.16661858558654785, + "logps/chosen": -48.48121643066406, + "logps/ref_chosen": -44.90705108642578, + "logps/ref_rejected": -58.7879524230957, + "logps/rejected": -63.35480880737305, + "loss": 1.315, + "margin_dpo/margin_mean": 0.9927000999450684, + "margin_dpo/margin_std": 3.234583854675293, + "step": 93 + }, + { + "KL/chosen_KL_mean": -3.2676773071289062, + "KL/mean": -4.234769821166992, + "KL/rejected_KL_mean": -5.201866149902344, + "KL/std": 3.246453285217285, + "epoch": 0.1421012849584278, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.9341883659362793, + "fcm_dpo/q_t": 0.4535469710826874, + "grad_norm": 27.196422576904297, + "learning_rate": 4.976400700654751e-07, + "logits/chosen": 0.2802616059780121, + "logits/rejected": 0.23843827843666077, + "logps/chosen": -63.205448150634766, + "logps/ref_chosen": -59.93777084350586, + "logps/ref_rejected": -79.3138427734375, + "logps/rejected": -84.51570129394531, + "loss": 1.2407, + "margin_dpo/margin_mean": 1.9341883659362793, + "margin_dpo/margin_std": 3.9628539085388184, + "step": 94 + }, + { + "KL/chosen_KL_mean": -4.273235321044922, + "KL/mean": -5.078390121459961, + "KL/rejected_KL_mean": -5.883544921875, + "KL/std": 2.7410950660705566, + "epoch": 0.1436130007558579, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.6103031635284424, + "fcm_dpo/q_t": 0.4611051082611084, + "grad_norm": 29.146900177001953, + "learning_rate": 4.974553604702332e-07, + "logits/chosen": 0.1459863930940628, + "logits/rejected": 0.08234073221683502, + "logps/chosen": -64.44172668457031, + "logps/ref_chosen": -60.168487548828125, + "logps/ref_rejected": -90.73665618896484, + "logps/rejected": -96.62020111083984, + "loss": 1.2611, + "margin_dpo/margin_mean": 1.6103029251098633, + "margin_dpo/margin_std": 3.4583120346069336, + "step": 95 + }, + { + "KL/chosen_KL_mean": -4.176933288574219, + "KL/mean": -5.115758895874023, + "KL/rejected_KL_mean": -6.054584503173828, + "KL/std": 3.4817161560058594, + "epoch": 0.14512471655328799, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.877647876739502, + "fcm_dpo/q_t": 0.456631064414978, + "grad_norm": 27.309680938720703, + "learning_rate": 4.972637290166157e-07, + "logits/chosen": 0.20910996198654175, + "logits/rejected": 0.16543054580688477, + "logps/chosen": -64.84571075439453, + "logps/ref_chosen": -60.66877746582031, + "logps/ref_rejected": -88.30673217773438, + "logps/rejected": -94.36131286621094, + "loss": 1.2584, + "margin_dpo/margin_mean": 1.877647876739502, + "margin_dpo/margin_std": 4.564449787139893, + "step": 96 + }, + { + "KL/chosen_KL_mean": -5.000583648681641, + "KL/mean": -5.523819923400879, + "KL/rejected_KL_mean": -6.047061920166016, + "KL/std": 3.636831760406494, + "epoch": 0.14663643235071808, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0464853048324585, + "fcm_dpo/q_t": 0.4750252962112427, + "grad_norm": 36.2060546875, + "learning_rate": 4.970651810649666e-07, + "logits/chosen": 0.1086052805185318, + "logits/rejected": 0.06623318046331406, + "logps/chosen": -70.04470825195312, + "logps/ref_chosen": -65.04412078857422, + "logps/ref_rejected": -78.42092895507812, + "logps/rejected": -84.46798706054688, + "loss": 1.3438, + "margin_dpo/margin_mean": 1.0464853048324585, + "margin_dpo/margin_std": 4.956970691680908, + "step": 97 + }, + { + "KL/chosen_KL_mean": -4.634487152099609, + "KL/mean": -4.977179527282715, + "KL/rejected_KL_mean": -5.319873809814453, + "KL/std": 3.2767844200134277, + "epoch": 0.14814814814814814, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6853851079940796, + "fcm_dpo/q_t": 0.48287999629974365, + "grad_norm": 30.321731567382812, + "learning_rate": 4.968597221690985e-07, + "logits/chosen": 0.2463841438293457, + "logits/rejected": 0.21862992644309998, + "logps/chosen": -60.137718200683594, + "logps/ref_chosen": -55.503231048583984, + "logps/ref_rejected": -72.81553649902344, + "logps/rejected": -78.13541412353516, + "loss": 1.3458, + "margin_dpo/margin_mean": 0.6853851079940796, + "margin_dpo/margin_std": 3.264176368713379, + "step": 98 + }, + { + "KL/chosen_KL_mean": -4.686042785644531, + "KL/mean": -5.184030532836914, + "KL/rejected_KL_mean": -5.6820220947265625, + "KL/std": 3.4981861114501953, + "epoch": 0.14965986394557823, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.9959783554077148, + "fcm_dpo/q_t": 0.4764708876609802, + "grad_norm": 32.71485900878906, + "learning_rate": 4.966473580761389e-07, + "logits/chosen": 0.2241283655166626, + "logits/rejected": 0.18563303351402283, + "logps/chosen": -63.261680603027344, + "logps/ref_chosen": -58.57563781738281, + "logps/ref_rejected": -78.693603515625, + "logps/rejected": -84.37562561035156, + "loss": 1.3406, + "margin_dpo/margin_mean": 0.9959783554077148, + "margin_dpo/margin_std": 4.575559139251709, + "step": 99 + }, + { + "KL/chosen_KL_mean": -5.299495697021484, + "KL/mean": -6.027351379394531, + "KL/rejected_KL_mean": -6.755210876464844, + "KL/std": 4.143482208251953, + "epoch": 0.15117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.4557148218154907, + "fcm_dpo/q_t": 0.4673404097557068, + "grad_norm": 33.2791862487793, + "learning_rate": 4.964280947263676e-07, + "logits/chosen": 0.22981172800064087, + "logits/rejected": 0.22199641168117523, + "logps/chosen": -84.8829345703125, + "logps/ref_chosen": -79.58343505859375, + "logps/ref_rejected": -92.152587890625, + "logps/rejected": -98.90779113769531, + "loss": 1.3324, + "margin_dpo/margin_mean": 1.4557149410247803, + "margin_dpo/margin_std": 5.845252513885498, + "step": 100 + }, + { + "KL/chosen_KL_mean": -4.274101257324219, + "KL/mean": -5.311479568481445, + "KL/rejected_KL_mean": -6.348859786987305, + "KL/std": 3.6799697875976562, + "epoch": 0.15268329554043839, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.0747623443603516, + "fcm_dpo/q_t": 0.44993698596954346, + "grad_norm": 25.746591567993164, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": 0.2311178743839264, + "logits/rejected": 0.19193893671035767, + "logps/chosen": -56.60688781738281, + "logps/ref_chosen": -52.332786560058594, + "logps/ref_rejected": -69.55589294433594, + "logps/rejected": -75.90475463867188, + "loss": 1.2324, + "margin_dpo/margin_mean": 2.0747623443603516, + "margin_dpo/margin_std": 4.121917724609375, + "step": 101 + }, + { + "KL/chosen_KL_mean": -5.331483840942383, + "KL/mean": -5.742671012878418, + "KL/rejected_KL_mean": -6.153861999511719, + "KL/std": 3.7148096561431885, + "epoch": 0.15419501133786848, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8223755359649658, + "fcm_dpo/q_t": 0.4832330346107483, + "grad_norm": 32.66339874267578, + "learning_rate": 4.959688949822748e-07, + "logits/chosen": 0.18088501691818237, + "logits/rejected": 0.14139467477798462, + "logps/chosen": -70.07496643066406, + "logps/ref_chosen": -64.74348449707031, + "logps/ref_rejected": -69.06132507324219, + "logps/rejected": -75.21519470214844, + "loss": 1.376, + "margin_dpo/margin_mean": 0.8223754167556763, + "margin_dpo/margin_std": 5.470452308654785, + "step": 102 + }, + { + "KL/chosen_KL_mean": -4.802188873291016, + "KL/mean": -5.606751441955566, + "KL/rejected_KL_mean": -6.41131591796875, + "KL/std": 3.937784194946289, + "epoch": 0.15570672713529857, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.6091289520263672, + "fcm_dpo/q_t": 0.4614095985889435, + "grad_norm": 29.439403533935547, + "learning_rate": 4.957289714327572e-07, + "logits/chosen": 0.2667997479438782, + "logits/rejected": 0.23269619047641754, + "logps/chosen": -68.63883209228516, + "logps/ref_chosen": -63.83664321899414, + "logps/ref_rejected": -79.32362365722656, + "logps/rejected": -85.73493957519531, + "loss": 1.2651, + "margin_dpo/margin_mean": 1.6091279983520508, + "margin_dpo/margin_std": 3.6503872871398926, + "step": 103 + }, + { + "KL/chosen_KL_mean": -4.722269058227539, + "KL/mean": -5.856806755065918, + "KL/rejected_KL_mean": -6.991344451904297, + "KL/std": 4.956378936767578, + "epoch": 0.15721844293272866, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.269068956375122, + "fcm_dpo/q_t": 0.4513574540615082, + "grad_norm": 30.835031509399414, + "learning_rate": 4.954821743156767e-07, + "logits/chosen": 0.24004673957824707, + "logits/rejected": 0.15426743030548096, + "logps/chosen": -65.72148132324219, + "logps/ref_chosen": -60.99920654296875, + "logps/ref_rejected": -98.84645080566406, + "logps/rejected": -105.83779907226562, + "loss": 1.2645, + "margin_dpo/margin_mean": 2.2690696716308594, + "margin_dpo/margin_std": 6.295289993286133, + "step": 104 + }, + { + "KL/chosen_KL_mean": -5.31561279296875, + "KL/mean": -6.242671966552734, + "KL/rejected_KL_mean": -7.169731140136719, + "KL/std": 4.008975028991699, + "epoch": 0.15873015873015872, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.854116678237915, + "fcm_dpo/q_t": 0.45764368772506714, + "grad_norm": 30.735721588134766, + "learning_rate": 4.952285105344791e-07, + "logits/chosen": 0.21237248182296753, + "logits/rejected": 0.15671700239181519, + "logps/chosen": -76.26588439941406, + "logps/ref_chosen": -70.95027160644531, + "logps/ref_rejected": -87.88340759277344, + "logps/rejected": -95.05314636230469, + "loss": 1.2691, + "margin_dpo/margin_mean": 1.8541167974472046, + "margin_dpo/margin_std": 4.961075782775879, + "step": 105 + }, + { + "KL/chosen_KL_mean": -5.028408050537109, + "KL/mean": -5.82650089263916, + "KL/rejected_KL_mean": -6.624591827392578, + "KL/std": 3.8884782791137695, + "epoch": 0.1602418745275888, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5961841344833374, + "fcm_dpo/q_t": 0.4617699980735779, + "grad_norm": 30.808412551879883, + "learning_rate": 4.949679871846857e-07, + "logits/chosen": 0.24007391929626465, + "logits/rejected": 0.22604778409004211, + "logps/chosen": -67.48773956298828, + "logps/ref_chosen": -62.45933151245117, + "logps/ref_rejected": -67.00595092773438, + "logps/rejected": -73.63053894042969, + "loss": 1.2995, + "margin_dpo/margin_mean": 1.5961840152740479, + "margin_dpo/margin_std": 5.234855651855469, + "step": 106 + }, + { + "KL/chosen_KL_mean": -6.487876892089844, + "KL/mean": -6.899287223815918, + "KL/rejected_KL_mean": -7.310695648193359, + "KL/std": 4.909206390380859, + "epoch": 0.1617535903250189, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8228192329406738, + "fcm_dpo/q_t": 0.4787726402282715, + "grad_norm": 41.80364227294922, + "learning_rate": 4.947006115536947e-07, + "logits/chosen": 0.15545688569545746, + "logits/rejected": 0.13251692056655884, + "logps/chosen": -82.32584381103516, + "logps/ref_chosen": -75.83796691894531, + "logps/ref_rejected": -87.74038696289062, + "logps/rejected": -95.05107879638672, + "loss": 1.3979, + "margin_dpo/margin_mean": 0.8228189945220947, + "margin_dpo/margin_std": 6.183783531188965, + "step": 107 + }, + { + "KL/chosen_KL_mean": -5.504695892333984, + "KL/mean": -6.444911956787109, + "KL/rejected_KL_mean": -7.385128021240234, + "KL/std": 4.147845268249512, + "epoch": 0.16326530612244897, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8804302215576172, + "fcm_dpo/q_t": 0.45709145069122314, + "grad_norm": 28.610286712646484, + "learning_rate": 4.944263911205772e-07, + "logits/chosen": 0.19874165952205658, + "logits/rejected": 0.16665475070476532, + "logps/chosen": -73.89793395996094, + "logps/ref_chosen": -68.39323425292969, + "logps/ref_rejected": -83.24267578125, + "logps/rejected": -90.62779998779297, + "loss": 1.2779, + "margin_dpo/margin_mean": 1.8804303407669067, + "margin_dpo/margin_std": 5.413944244384766, + "step": 108 + }, + { + "KL/chosen_KL_mean": -4.886608123779297, + "KL/mean": -6.389529228210449, + "KL/rejected_KL_mean": -7.892448425292969, + "KL/std": 5.053153991699219, + "epoch": 0.16477702191987906, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.0058376789093018, + "fcm_dpo/q_t": 0.4337880611419678, + "grad_norm": 28.171001434326172, + "learning_rate": 4.941453335558681e-07, + "logits/chosen": 0.17483273148536682, + "logits/rejected": 0.12170157581567764, + "logps/chosen": -60.41408920288086, + "logps/ref_chosen": -55.52748107910156, + "logps/ref_rejected": -83.55218505859375, + "logps/rejected": -91.44464111328125, + "loss": 1.1749, + "margin_dpo/margin_mean": 3.0058372020721436, + "margin_dpo/margin_std": 5.493310928344727, + "step": 109 + }, + { + "KL/chosen_KL_mean": -6.248790740966797, + "KL/mean": -6.400554656982422, + "KL/rejected_KL_mean": -6.552318572998047, + "KL/std": 4.0687785148620605, + "epoch": 0.16628873771730915, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3035222589969635, + "fcm_dpo/q_t": 0.49050843715667725, + "grad_norm": 44.2472038269043, + "learning_rate": 4.938574467213517e-07, + "logits/chosen": 0.18585237860679626, + "logits/rejected": 0.1952962875366211, + "logps/chosen": -87.40753173828125, + "logps/ref_chosen": -81.15874481201172, + "logps/ref_rejected": -72.56021118164062, + "logps/rejected": -79.11253356933594, + "loss": 1.4264, + "margin_dpo/margin_mean": 0.3035214841365814, + "margin_dpo/margin_std": 5.468291282653809, + "step": 110 + }, + { + "KL/chosen_KL_mean": -5.70982551574707, + "KL/mean": -6.685848236083984, + "KL/rejected_KL_mean": -7.661876678466797, + "KL/std": 4.2794389724731445, + "epoch": 0.16780045351473924, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.952048659324646, + "fcm_dpo/q_t": 0.4560878276824951, + "grad_norm": 25.860748291015625, + "learning_rate": 4.935627386698418e-07, + "logits/chosen": 0.29305583238601685, + "logits/rejected": 0.2550733983516693, + "logps/chosen": -58.068809509277344, + "logps/ref_chosen": -52.358985900878906, + "logps/ref_rejected": -77.06150817871094, + "logps/rejected": -84.72338104248047, + "loss": 1.266, + "margin_dpo/margin_mean": 1.9520485401153564, + "margin_dpo/margin_std": 5.244046688079834, + "step": 111 + }, + { + "KL/chosen_KL_mean": -5.34930419921875, + "KL/mean": -6.553627014160156, + "KL/rejected_KL_mean": -7.7579498291015625, + "KL/std": 4.263571739196777, + "epoch": 0.1693121693121693, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.4086484909057617, + "fcm_dpo/q_t": 0.4429079294204712, + "grad_norm": 32.236202239990234, + "learning_rate": 4.932612176449559e-07, + "logits/chosen": 0.1755533367395401, + "logits/rejected": 0.11200804263353348, + "logps/chosen": -68.36936950683594, + "logps/ref_chosen": -63.02006530761719, + "logps/ref_rejected": -111.36941528320312, + "logps/rejected": -119.12736511230469, + "loss": 1.2155, + "margin_dpo/margin_mean": 2.4086484909057617, + "margin_dpo/margin_std": 4.813718795776367, + "step": 112 + }, + { + "KL/chosen_KL_mean": -5.344881057739258, + "KL/mean": -6.047405242919922, + "KL/rejected_KL_mean": -6.749931335449219, + "KL/std": 3.577111005783081, + "epoch": 0.1708238851095994, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.4050495624542236, + "fcm_dpo/q_t": 0.4661615192890167, + "grad_norm": 35.494136810302734, + "learning_rate": 4.929528920808854e-07, + "logits/chosen": 0.20536774396896362, + "logits/rejected": 0.16611523926258087, + "logps/chosen": -61.15254211425781, + "logps/ref_chosen": -55.80766296386719, + "logps/ref_rejected": -69.84014129638672, + "logps/rejected": -76.59007263183594, + "loss": 1.3032, + "margin_dpo/margin_mean": 1.405049204826355, + "margin_dpo/margin_std": 4.589272499084473, + "step": 113 + }, + { + "KL/chosen_KL_mean": -5.111001968383789, + "KL/mean": -6.265921115875244, + "KL/rejected_KL_mean": -7.420845031738281, + "KL/std": 3.935863494873047, + "epoch": 0.17233560090702948, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.309840202331543, + "fcm_dpo/q_t": 0.4466482996940613, + "grad_norm": 29.656740188598633, + "learning_rate": 4.92637770602159e-07, + "logits/chosen": 0.25359949469566345, + "logits/rejected": 0.19307023286819458, + "logps/chosen": -71.44377136230469, + "logps/ref_chosen": -66.33277130126953, + "logps/ref_rejected": -71.61489868164062, + "logps/rejected": -79.03573608398438, + "loss": 1.2227, + "margin_dpo/margin_mean": 2.309840202331543, + "margin_dpo/margin_std": 4.768823623657227, + "step": 114 + }, + { + "KL/chosen_KL_mean": -5.491331100463867, + "KL/mean": -6.419828414916992, + "KL/rejected_KL_mean": -7.348325729370117, + "KL/std": 3.607484817504883, + "epoch": 0.17384731670445955, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8569921255111694, + "fcm_dpo/q_t": 0.4564257860183716, + "grad_norm": 28.554027557373047, + "learning_rate": 4.923158620234019e-07, + "logits/chosen": 0.23709729313850403, + "logits/rejected": 0.1739954650402069, + "logps/chosen": -61.24037170410156, + "logps/ref_chosen": -55.74903869628906, + "logps/ref_rejected": -79.59849548339844, + "logps/rejected": -86.94682312011719, + "loss": 1.2544, + "margin_dpo/margin_mean": 1.8569918870925903, + "margin_dpo/margin_std": 4.30812931060791, + "step": 115 + }, + { + "KL/chosen_KL_mean": -5.2065277099609375, + "KL/mean": -6.613717079162598, + "KL/rejected_KL_mean": -8.02090835571289, + "KL/std": 3.7173666954040527, + "epoch": 0.17535903250188964, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.814383029937744, + "fcm_dpo/q_t": 0.432966411113739, + "grad_norm": 26.021554946899414, + "learning_rate": 4.91987175349089e-07, + "logits/chosen": 0.20693862438201904, + "logits/rejected": 0.14074885845184326, + "logps/chosen": -54.571693420410156, + "logps/ref_chosen": -49.36516571044922, + "logps/ref_rejected": -72.84671020507812, + "logps/rejected": -80.86761474609375, + "loss": 1.1659, + "margin_dpo/margin_mean": 2.8143835067749023, + "margin_dpo/margin_std": 4.077053070068359, + "step": 116 + }, + { + "KL/chosen_KL_mean": -5.159467697143555, + "KL/mean": -6.121030807495117, + "KL/rejected_KL_mean": -7.0825958251953125, + "KL/std": 3.491952896118164, + "epoch": 0.17687074829931973, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.9231154918670654, + "fcm_dpo/q_t": 0.45408564805984497, + "grad_norm": 28.094541549682617, + "learning_rate": 4.916517197732933e-07, + "logits/chosen": 0.21251803636550903, + "logits/rejected": 0.17677460610866547, + "logps/chosen": -62.870365142822266, + "logps/ref_chosen": -57.710899353027344, + "logps/ref_rejected": -69.77253723144531, + "logps/rejected": -76.85513305664062, + "loss": 1.2545, + "margin_dpo/margin_mean": 1.9231159687042236, + "margin_dpo/margin_std": 4.358719825744629, + "step": 117 + }, + { + "KL/chosen_KL_mean": -5.038366317749023, + "KL/mean": -6.310278415679932, + "KL/rejected_KL_mean": -7.582187652587891, + "KL/std": 3.8248229026794434, + "epoch": 0.17838246409674982, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.543827533721924, + "fcm_dpo/q_t": 0.4391190707683563, + "grad_norm": 28.034107208251953, + "learning_rate": 4.913095046794281e-07, + "logits/chosen": 0.294602632522583, + "logits/rejected": 0.25325727462768555, + "logps/chosen": -57.51826477050781, + "logps/ref_chosen": -52.479896545410156, + "logps/ref_rejected": -81.359130859375, + "logps/rejected": -88.94131469726562, + "loss": 1.1856, + "margin_dpo/margin_mean": 2.5438270568847656, + "margin_dpo/margin_std": 3.8979780673980713, + "step": 118 + }, + { + "KL/chosen_KL_mean": -6.389644622802734, + "KL/mean": -7.3072357177734375, + "KL/rejected_KL_mean": -8.22482681274414, + "KL/std": 4.4486799240112305, + "epoch": 0.17989417989417988, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8351812362670898, + "fcm_dpo/q_t": 0.4581907093524933, + "grad_norm": 29.41862678527832, + "learning_rate": 4.909605396399855e-07, + "logits/chosen": 0.215057373046875, + "logits/rejected": 0.17584341764450073, + "logps/chosen": -67.747314453125, + "logps/ref_chosen": -61.35767364501953, + "logps/ref_rejected": -75.71510314941406, + "logps/rejected": -83.93992614746094, + "loss": 1.283, + "margin_dpo/margin_mean": 1.8351819515228271, + "margin_dpo/margin_std": 5.5315446853637695, + "step": 119 + }, + { + "KL/chosen_KL_mean": -5.267726898193359, + "KL/mean": -6.668299674987793, + "KL/rejected_KL_mean": -8.06887435913086, + "KL/std": 3.7570960521698, + "epoch": 0.18140589569160998, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.801149845123291, + "fcm_dpo/q_t": 0.4335363209247589, + "grad_norm": 28.7716007232666, + "learning_rate": 4.906048344162676e-07, + "logits/chosen": 0.21911869943141937, + "logits/rejected": 0.15837198495864868, + "logps/chosen": -65.17529296875, + "logps/ref_chosen": -59.907569885253906, + "logps/ref_rejected": -79.6910629272461, + "logps/rejected": -87.75993347167969, + "loss": 1.1622, + "margin_dpo/margin_mean": 2.80115008354187, + "margin_dpo/margin_std": 3.8388729095458984, + "step": 120 + }, + { + "KL/chosen_KL_mean": -5.821605682373047, + "KL/mean": -7.307372570037842, + "KL/rejected_KL_mean": -8.793140411376953, + "KL/std": 4.403526306152344, + "epoch": 0.18291761148904007, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.9715352058410645, + "fcm_dpo/q_t": 0.43289828300476074, + "grad_norm": 26.99895477294922, + "learning_rate": 4.902423989581143e-07, + "logits/chosen": 0.3170771598815918, + "logits/rejected": 0.2303023487329483, + "logps/chosen": -61.487648010253906, + "logps/ref_chosen": -55.66604232788086, + "logps/ref_rejected": -101.56233978271484, + "logps/rejected": -110.35548400878906, + "loss": 1.1654, + "margin_dpo/margin_mean": 2.9715352058410645, + "margin_dpo/margin_std": 4.898141860961914, + "step": 121 + }, + { + "KL/chosen_KL_mean": -6.339637756347656, + "KL/mean": -7.675061225891113, + "KL/rejected_KL_mean": -9.010486602783203, + "KL/std": 4.725244998931885, + "epoch": 0.18442932728647016, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.6708438396453857, + "fcm_dpo/q_t": 0.44164925813674927, + "grad_norm": 30.728546142578125, + "learning_rate": 4.898732434036243e-07, + "logits/chosen": 0.24164050817489624, + "logits/rejected": 0.20562344789505005, + "logps/chosen": -69.67401123046875, + "logps/ref_chosen": -63.334373474121094, + "logps/ref_rejected": -73.67523193359375, + "logps/rejected": -82.68572235107422, + "loss": 1.2112, + "margin_dpo/margin_mean": 2.670844078063965, + "margin_dpo/margin_std": 5.738737106323242, + "step": 122 + }, + { + "KL/chosen_KL_mean": -6.028675079345703, + "KL/mean": -7.320953845977783, + "KL/rejected_KL_mean": -8.613231658935547, + "KL/std": 4.104480266571045, + "epoch": 0.18594104308390022, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.5845589637756348, + "fcm_dpo/q_t": 0.4395887851715088, + "grad_norm": 26.99787139892578, + "learning_rate": 4.894973780788722e-07, + "logits/chosen": 0.25603556632995605, + "logits/rejected": 0.214134082198143, + "logps/chosen": -62.92741775512695, + "logps/ref_chosen": -56.89874267578125, + "logps/ref_rejected": -78.97028350830078, + "logps/rejected": -87.58351135253906, + "loss": 1.1913, + "margin_dpo/margin_mean": 2.5845589637756348, + "margin_dpo/margin_std": 4.423001289367676, + "step": 123 + }, + { + "KL/chosen_KL_mean": -6.372430801391602, + "KL/mean": -8.195686340332031, + "KL/rejected_KL_mean": -10.018943786621094, + "KL/std": 4.532326698303223, + "epoch": 0.1874527588813303, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.6465113162994385, + "fcm_dpo/q_t": 0.41543495655059814, + "grad_norm": 27.253131866455078, + "learning_rate": 4.89114813497619e-07, + "logits/chosen": 0.2840479016304016, + "logits/rejected": 0.22104603052139282, + "logps/chosen": -63.48851776123047, + "logps/ref_chosen": -57.116085052490234, + "logps/ref_rejected": -87.93074035644531, + "logps/rejected": -97.94969177246094, + "loss": 1.1103, + "margin_dpo/margin_mean": 3.6465108394622803, + "margin_dpo/margin_std": 4.869706153869629, + "step": 124 + }, + { + "KL/chosen_KL_mean": -6.931028366088867, + "KL/mean": -8.51264762878418, + "KL/rejected_KL_mean": -10.094261169433594, + "KL/std": 4.857443332672119, + "epoch": 0.1889644746787604, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.163234233856201, + "fcm_dpo/q_t": 0.4276365041732788, + "grad_norm": 29.05904769897461, + "learning_rate": 4.887255603610184e-07, + "logits/chosen": 0.30631697177886963, + "logits/rejected": 0.24442438781261444, + "logps/chosen": -72.63720703125, + "logps/ref_chosen": -65.7061767578125, + "logps/ref_rejected": -91.72711944580078, + "logps/rejected": -101.82138061523438, + "loss": 1.1561, + "margin_dpo/margin_mean": 3.163234233856201, + "margin_dpo/margin_std": 5.063149452209473, + "step": 125 + }, + { + "KL/chosen_KL_mean": -6.585702896118164, + "KL/mean": -7.441903114318848, + "KL/rejected_KL_mean": -8.298105239868164, + "KL/std": 4.394684314727783, + "epoch": 0.19047619047619047, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.7124006748199463, + "fcm_dpo/q_t": 0.45973747968673706, + "grad_norm": 28.884706497192383, + "learning_rate": 4.883296295573176e-07, + "logits/chosen": 0.1231585294008255, + "logits/rejected": 0.11754154413938522, + "logps/chosen": -74.76179504394531, + "logps/ref_chosen": -68.17608642578125, + "logps/ref_rejected": -65.1175537109375, + "logps/rejected": -73.41565704345703, + "loss": 1.2761, + "margin_dpo/margin_mean": 1.7124009132385254, + "margin_dpo/margin_std": 4.733203887939453, + "step": 126 + }, + { + "KL/chosen_KL_mean": -6.761753082275391, + "KL/mean": -8.410690307617188, + "KL/rejected_KL_mean": -10.059627532958984, + "KL/std": 3.970813512802124, + "epoch": 0.19198790627362056, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.297870635986328, + "fcm_dpo/q_t": 0.4221423864364624, + "grad_norm": 26.59172821044922, + "learning_rate": 4.87927032161552e-07, + "logits/chosen": 0.23097413778305054, + "logits/rejected": 0.2001955211162567, + "logps/chosen": -68.64198303222656, + "logps/ref_chosen": -61.88023376464844, + "logps/ref_rejected": -68.46012878417969, + "logps/rejected": -78.5197525024414, + "loss": 1.1298, + "margin_dpo/margin_mean": 3.29787015914917, + "margin_dpo/margin_std": 4.276577949523926, + "step": 127 + }, + { + "KL/chosen_KL_mean": -7.305440902709961, + "KL/mean": -9.139411926269531, + "KL/rejected_KL_mean": -10.973388671875, + "KL/std": 5.392538070678711, + "epoch": 0.19349962207105065, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.667938709259033, + "fcm_dpo/q_t": 0.4183761477470398, + "grad_norm": 27.74724006652832, + "learning_rate": 4.875177794352363e-07, + "logits/chosen": 0.25080034136772156, + "logits/rejected": 0.19237586855888367, + "logps/chosen": -74.01441955566406, + "logps/ref_chosen": -66.708984375, + "logps/ref_rejected": -94.97969055175781, + "logps/rejected": -105.95307922363281, + "loss": 1.1497, + "margin_dpo/margin_mean": 3.667938232421875, + "margin_dpo/margin_std": 6.488053321838379, + "step": 128 + }, + { + "KL/chosen_KL_mean": -8.281728744506836, + "KL/mean": -9.327621459960938, + "KL/rejected_KL_mean": -10.373516082763672, + "KL/std": 4.979771137237549, + "epoch": 0.19501133786848074, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.0917892456054688, + "fcm_dpo/q_t": 0.451565682888031, + "grad_norm": 34.47330093383789, + "learning_rate": 4.871018828260491e-07, + "logits/chosen": 0.2508179247379303, + "logits/rejected": 0.24487638473510742, + "logps/chosen": -73.62055969238281, + "logps/ref_chosen": -65.33882904052734, + "logps/ref_rejected": -68.06109619140625, + "logps/rejected": -78.43460845947266, + "loss": 1.2727, + "margin_dpo/margin_mean": 2.091789722442627, + "margin_dpo/margin_std": 5.9948272705078125, + "step": 129 + }, + { + "KL/chosen_KL_mean": -7.84025764465332, + "KL/mean": -9.535599708557129, + "KL/rejected_KL_mean": -11.230945587158203, + "KL/std": 4.948360443115234, + "epoch": 0.1965230536659108, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.390688419342041, + "fcm_dpo/q_t": 0.4209347069263458, + "grad_norm": 30.479267120361328, + "learning_rate": 4.866793539675126e-07, + "logits/chosen": 0.2105114459991455, + "logits/rejected": 0.15794897079467773, + "logps/chosen": -66.50100708007812, + "logps/ref_chosen": -58.660743713378906, + "logps/ref_rejected": -79.24510192871094, + "logps/rejected": -90.47604370117188, + "loss": 1.1345, + "margin_dpo/margin_mean": 3.390688180923462, + "margin_dpo/margin_std": 4.888091087341309, + "step": 130 + }, + { + "KL/chosen_KL_mean": -7.463371276855469, + "KL/mean": -9.659799575805664, + "KL/rejected_KL_mean": -11.856231689453125, + "KL/std": 5.699565410614014, + "epoch": 0.1980347694633409, + "fcm_dpo/beta": 0.09919185191392899, + "fcm_dpo/delta": -0.04057179018855095, + "fcm_dpo/margin": 4.392857551574707, + "fcm_dpo/q_t": 0.4031718075275421, + "grad_norm": 25.621009826660156, + "learning_rate": 4.86250204678667e-07, + "logits/chosen": 0.20928484201431274, + "logits/rejected": 0.1438637375831604, + "logps/chosen": -59.9779052734375, + "logps/ref_chosen": -52.51453399658203, + "logps/ref_rejected": -85.18299865722656, + "logps/rejected": -97.03923034667969, + "loss": 1.0916, + "margin_dpo/margin_mean": 4.392857551574707, + "margin_dpo/margin_std": 6.408463478088379, + "step": 131 + }, + { + "KL/chosen_KL_mean": -8.304485321044922, + "KL/mean": -10.10638427734375, + "KL/rejected_KL_mean": -11.908285140991211, + "KL/std": 5.412184715270996, + "epoch": 0.19954648526077098, + "fcm_dpo/beta": 0.09919185191392899, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.603799343109131, + "fcm_dpo/q_t": 0.41539686918258667, + "grad_norm": 27.230810165405273, + "learning_rate": 4.858144469637408e-07, + "logits/chosen": 0.31601205468177795, + "logits/rejected": 0.2823875844478607, + "logps/chosen": -73.98961639404297, + "logps/ref_chosen": -65.68513488769531, + "logps/ref_rejected": -69.54120635986328, + "logps/rejected": -81.44949340820312, + "loss": 1.125, + "margin_dpo/margin_mean": 3.603799343109131, + "margin_dpo/margin_std": 5.2057013511657715, + "step": 132 + }, + { + "KL/chosen_KL_mean": -8.788467407226562, + "KL/mean": -10.21713638305664, + "KL/rejected_KL_mean": -11.645797729492188, + "KL/std": 5.376225471496582, + "epoch": 0.20105820105820105, + "fcm_dpo/beta": 0.09919185191392899, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.857332944869995, + "fcm_dpo/q_t": 0.4344499409198761, + "grad_norm": 31.633960723876953, + "learning_rate": 4.853720930118138e-07, + "logits/chosen": 0.19638074934482574, + "logits/rejected": 0.18663090467453003, + "logps/chosen": -72.38658142089844, + "logps/ref_chosen": -63.598114013671875, + "logps/ref_rejected": -73.72798156738281, + "logps/rejected": -85.373779296875, + "loss": 1.2047, + "margin_dpo/margin_mean": 2.8573334217071533, + "margin_dpo/margin_std": 5.904394149780273, + "step": 133 + }, + { + "KL/chosen_KL_mean": -8.176952362060547, + "KL/mean": -10.648435592651367, + "KL/rejected_KL_mean": -13.119918823242188, + "KL/std": 6.183529376983643, + "epoch": 0.20256991685563114, + "fcm_dpo/beta": 0.09795012325048447, + "fcm_dpo/delta": -0.0884043499827385, + "fcm_dpo/margin": 4.94296407699585, + "fcm_dpo/q_t": 0.3913155496120453, + "grad_norm": 24.997631072998047, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": 0.3347264528274536, + "logits/rejected": 0.27579599618911743, + "logps/chosen": -61.97152328491211, + "logps/ref_chosen": -53.79457092285156, + "logps/ref_rejected": -74.16741943359375, + "logps/rejected": -87.28733825683594, + "loss": 1.0465, + "margin_dpo/margin_mean": 4.942964553833008, + "margin_dpo/margin_std": 6.3145341873168945, + "step": 134 + }, + { + "KL/chosen_KL_mean": -8.215858459472656, + "KL/mean": -9.784494400024414, + "KL/rejected_KL_mean": -11.353134155273438, + "KL/std": 5.727636814117432, + "epoch": 0.20408163265306123, + "fcm_dpo/beta": 0.09745346754789352, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.1372694969177246, + "fcm_dpo/q_t": 0.42929476499557495, + "grad_norm": 24.82374382019043, + "learning_rate": 4.844676460754862e-07, + "logits/chosen": 0.24679645895957947, + "logits/rejected": 0.21434196829795837, + "logps/chosen": -57.65693664550781, + "logps/ref_chosen": -49.441078186035156, + "logps/ref_rejected": -65.96878051757812, + "logps/rejected": -77.32191467285156, + "loss": 1.1811, + "margin_dpo/margin_mean": 3.1372690200805664, + "margin_dpo/margin_std": 5.890544891357422, + "step": 135 + }, + { + "KL/chosen_KL_mean": -10.235748291015625, + "KL/mean": -12.271726608276367, + "KL/rejected_KL_mean": -14.30771255493164, + "KL/std": 6.477873802185059, + "epoch": 0.20559334845049132, + "fcm_dpo/beta": 0.0964834988117218, + "fcm_dpo/delta": -0.05001489818096161, + "fcm_dpo/margin": 4.071964740753174, + "fcm_dpo/q_t": 0.41472935676574707, + "grad_norm": 30.083513259887695, + "learning_rate": 4.840055783904106e-07, + "logits/chosen": 0.23875057697296143, + "logits/rejected": 0.16482864320278168, + "logps/chosen": -76.99501037597656, + "logps/ref_chosen": -66.75926208496094, + "logps/ref_rejected": -94.61787414550781, + "logps/rejected": -108.92558288574219, + "loss": 1.1717, + "margin_dpo/margin_mean": 4.071965217590332, + "margin_dpo/margin_std": 8.082297325134277, + "step": 136 + }, + { + "KL/chosen_KL_mean": -9.556255340576172, + "KL/mean": -11.618932723999023, + "KL/rejected_KL_mean": -13.68161392211914, + "KL/std": 6.124992847442627, + "epoch": 0.20710506424792138, + "fcm_dpo/beta": 0.09615467488765717, + "fcm_dpo/delta": -0.034197039902210236, + "fcm_dpo/margin": 4.125356674194336, + "fcm_dpo/q_t": 0.41115665435791016, + "grad_norm": 25.18506622314453, + "learning_rate": 4.835369650662767e-07, + "logits/chosen": 0.2673400044441223, + "logits/rejected": 0.23783737421035767, + "logps/chosen": -66.34005737304688, + "logps/ref_chosen": -56.78379821777344, + "logps/ref_rejected": -69.89952087402344, + "logps/rejected": -83.58113098144531, + "loss": 1.1175, + "margin_dpo/margin_mean": 4.125357151031494, + "margin_dpo/margin_std": 6.468832015991211, + "step": 137 + }, + { + "KL/chosen_KL_mean": -11.306020736694336, + "KL/mean": -13.123923301696777, + "KL/rejected_KL_mean": -14.941829681396484, + "KL/std": 6.329860687255859, + "epoch": 0.20861678004535147, + "fcm_dpo/beta": 0.09582585841417313, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.6358046531677246, + "fcm_dpo/q_t": 0.42061278223991394, + "grad_norm": 26.653167724609375, + "learning_rate": 4.830618192112065e-07, + "logits/chosen": 0.2642987370491028, + "logits/rejected": 0.22775018215179443, + "logps/chosen": -70.07203674316406, + "logps/ref_chosen": -58.766014099121094, + "logps/ref_rejected": -68.12371826171875, + "logps/rejected": -83.0655517578125, + "loss": 1.181, + "margin_dpo/margin_mean": 3.6358046531677246, + "margin_dpo/margin_std": 7.232297897338867, + "step": 138 + }, + { + "KL/chosen_KL_mean": -12.002849578857422, + "KL/mean": -14.108619689941406, + "KL/rejected_KL_mean": -16.214393615722656, + "KL/std": 6.1752753257751465, + "epoch": 0.21012849584278157, + "fcm_dpo/beta": 0.09582585841417313, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.211538791656494, + "fcm_dpo/q_t": 0.40779027342796326, + "grad_norm": 29.489917755126953, + "learning_rate": 4.825801541160509e-07, + "logits/chosen": 0.23293596506118774, + "logits/rejected": 0.20248082280158997, + "logps/chosen": -83.22843933105469, + "logps/ref_chosen": -71.2255859375, + "logps/ref_rejected": -82.1834716796875, + "logps/rejected": -98.39786529541016, + "loss": 1.1095, + "margin_dpo/margin_mean": 4.211538314819336, + "margin_dpo/margin_std": 6.356287956237793, + "step": 139 + }, + { + "KL/chosen_KL_mean": -10.922605514526367, + "KL/mean": -13.90744400024414, + "KL/rejected_KL_mean": -16.892284393310547, + "KL/std": 8.277734756469727, + "epoch": 0.21164021164021163, + "fcm_dpo/beta": 0.09324315190315247, + "fcm_dpo/delta": -0.16643883287906647, + "fcm_dpo/margin": 5.9696760177612305, + "fcm_dpo/q_t": 0.37920305132865906, + "grad_norm": 29.492624282836914, + "learning_rate": 4.820919832540181e-07, + "logits/chosen": 0.2372860610485077, + "logits/rejected": 0.1888073980808258, + "logps/chosen": -74.20027160644531, + "logps/ref_chosen": -63.27766418457031, + "logps/ref_rejected": -83.30647277832031, + "logps/rejected": -100.19876098632812, + "loss": 1.074, + "margin_dpo/margin_mean": 5.969675540924072, + "margin_dpo/margin_std": 9.342610359191895, + "step": 140 + }, + { + "KL/chosen_KL_mean": -12.200645446777344, + "KL/mean": -14.866783142089844, + "KL/rejected_KL_mean": -17.53292465209961, + "KL/std": 6.956808090209961, + "epoch": 0.21315192743764172, + "fcm_dpo/beta": 0.09056098014116287, + "fcm_dpo/delta": -0.11610686033964157, + "fcm_dpo/margin": 5.332277774810791, + "fcm_dpo/q_t": 0.3958725929260254, + "grad_norm": 27.12528419494629, + "learning_rate": 4.815973202802966e-07, + "logits/chosen": 0.2885650098323822, + "logits/rejected": 0.2414112687110901, + "logps/chosen": -73.96741485595703, + "logps/ref_chosen": -61.76676940917969, + "logps/ref_rejected": -88.60601806640625, + "logps/rejected": -106.13894653320312, + "loss": 1.1056, + "margin_dpo/margin_mean": 5.332278251647949, + "margin_dpo/margin_std": 8.744904518127441, + "step": 141 + }, + { + "KL/chosen_KL_mean": -11.824771881103516, + "KL/mean": -14.046581268310547, + "KL/rejected_KL_mean": -16.268394470214844, + "KL/std": 6.713468551635742, + "epoch": 0.2146636432350718, + "fcm_dpo/beta": 0.09056098014116287, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.443621635437012, + "fcm_dpo/q_t": 0.40913692116737366, + "grad_norm": 25.67524528503418, + "learning_rate": 4.810961790316729e-07, + "logits/chosen": 0.28370508551597595, + "logits/rejected": 0.2581597864627838, + "logps/chosen": -77.09954833984375, + "logps/ref_chosen": -65.2747802734375, + "logps/ref_rejected": -81.1378173828125, + "logps/rejected": -97.40621185302734, + "loss": 1.1139, + "margin_dpo/margin_mean": 4.443620681762695, + "margin_dpo/margin_std": 6.891501426696777, + "step": 142 + }, + { + "KL/chosen_KL_mean": -12.741323471069336, + "KL/mean": -14.630672454833984, + "KL/rejected_KL_mean": -16.520015716552734, + "KL/std": 8.059417724609375, + "epoch": 0.2161753590325019, + "fcm_dpo/beta": 0.09056098014116287, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.7786927223205566, + "fcm_dpo/q_t": 0.4216327667236328, + "grad_norm": 35.36532974243164, + "learning_rate": 4.805885735261454e-07, + "logits/chosen": 0.3095983862876892, + "logits/rejected": 0.2914605736732483, + "logps/chosen": -75.35914611816406, + "logps/ref_chosen": -62.617828369140625, + "logps/ref_rejected": -70.39239501953125, + "logps/rejected": -86.91240692138672, + "loss": 1.2409, + "margin_dpo/margin_mean": 3.7786929607391357, + "margin_dpo/margin_std": 9.52112102508545, + "step": 143 + }, + { + "KL/chosen_KL_mean": -12.927085876464844, + "KL/mean": -15.399724960327148, + "KL/rejected_KL_mean": -17.872360229492188, + "KL/std": 7.987780570983887, + "epoch": 0.21768707482993196, + "fcm_dpo/beta": 0.08943817764520645, + "fcm_dpo/delta": -0.06237911060452461, + "fcm_dpo/margin": 4.945269584655762, + "fcm_dpo/q_t": 0.4067836105823517, + "grad_norm": 27.427934646606445, + "learning_rate": 4.800745179625307e-07, + "logits/chosen": 0.283732146024704, + "logits/rejected": 0.2507033348083496, + "logps/chosen": -73.72977447509766, + "logps/ref_chosen": -60.80268859863281, + "logps/ref_rejected": -79.07284545898438, + "logps/rejected": -96.94520568847656, + "loss": 1.1433, + "margin_dpo/margin_mean": 4.945269584655762, + "margin_dpo/margin_std": 9.206413269042969, + "step": 144 + }, + { + "KL/chosen_KL_mean": -12.03497314453125, + "KL/mean": -14.69092845916748, + "KL/rejected_KL_mean": -17.346881866455078, + "KL/std": 8.228681564331055, + "epoch": 0.21919879062736206, + "fcm_dpo/beta": 0.08845508843660355, + "fcm_dpo/delta": -0.0552634559571743, + "fcm_dpo/margin": 5.31191349029541, + "fcm_dpo/q_t": 0.3977242112159729, + "grad_norm": 30.822948455810547, + "learning_rate": 4.795540267200686e-07, + "logits/chosen": 0.24015925824642181, + "logits/rejected": 0.256152868270874, + "logps/chosen": -86.64643859863281, + "logps/ref_chosen": -74.61146545410156, + "logps/ref_rejected": -83.24461364746094, + "logps/rejected": -100.59149169921875, + "loss": 1.1399, + "margin_dpo/margin_mean": 5.31191349029541, + "margin_dpo/margin_std": 9.92963981628418, + "step": 145 + }, + { + "KL/chosen_KL_mean": -11.776222229003906, + "KL/mean": -14.419098854064941, + "KL/rejected_KL_mean": -17.06197738647461, + "KL/std": 8.018354415893555, + "epoch": 0.22071050642479215, + "fcm_dpo/beta": 0.08789414912462234, + "fcm_dpo/delta": -0.06382076442241669, + "fcm_dpo/margin": 5.285751819610596, + "fcm_dpo/q_t": 0.39964884519577026, + "grad_norm": 26.835702896118164, + "learning_rate": 4.790271143580173e-07, + "logits/chosen": 0.23064787685871124, + "logits/rejected": 0.21661049127578735, + "logps/chosen": -69.61720275878906, + "logps/ref_chosen": -57.84098434448242, + "logps/ref_rejected": -67.47422790527344, + "logps/rejected": -84.53620910644531, + "loss": 1.113, + "margin_dpo/margin_mean": 5.2857513427734375, + "margin_dpo/margin_std": 8.87518310546875, + "step": 146 + }, + { + "KL/chosen_KL_mean": -13.491147994995117, + "KL/mean": -15.663849830627441, + "KL/rejected_KL_mean": -17.836559295654297, + "KL/std": 8.625712394714355, + "epoch": 0.2222222222222222, + "fcm_dpo/beta": 0.08733320981264114, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.3454060554504395, + "fcm_dpo/q_t": 0.4166657328605652, + "grad_norm": 32.175376892089844, + "learning_rate": 4.784937956152489e-07, + "logits/chosen": 0.278936505317688, + "logits/rejected": 0.2334621697664261, + "logps/chosen": -80.30461883544922, + "logps/ref_chosen": -66.81346893310547, + "logps/ref_rejected": -81.1796875, + "logps/rejected": -99.01625061035156, + "loss": 1.1904, + "margin_dpo/margin_mean": 4.345406532287598, + "margin_dpo/margin_std": 9.280193328857422, + "step": 147 + }, + { + "KL/chosen_KL_mean": -12.208412170410156, + "KL/mean": -15.023258209228516, + "KL/rejected_KL_mean": -17.83810043334961, + "KL/std": 8.074333190917969, + "epoch": 0.2237339380196523, + "fcm_dpo/beta": 0.08660124242305756, + "fcm_dpo/delta": -0.08452388644218445, + "fcm_dpo/margin": 5.629689693450928, + "fcm_dpo/q_t": 0.3923068046569824, + "grad_norm": 22.291276931762695, + "learning_rate": 4.779540854098347e-07, + "logits/chosen": 0.3799922466278076, + "logits/rejected": 0.30557140707969666, + "logps/chosen": -60.89616775512695, + "logps/ref_chosen": -48.6877555847168, + "logps/ref_rejected": -67.50503540039062, + "logps/rejected": -85.3431396484375, + "loss": 1.0956, + "margin_dpo/margin_mean": 5.629690170288086, + "margin_dpo/margin_std": 8.904212951660156, + "step": 148 + }, + { + "KL/chosen_KL_mean": -14.778066635131836, + "KL/mean": -17.901935577392578, + "KL/rejected_KL_mean": -21.025802612304688, + "KL/std": 8.484848022460938, + "epoch": 0.2252456538170824, + "fcm_dpo/beta": 0.0842820554971695, + "fcm_dpo/delta": -0.13348132371902466, + "fcm_dpo/margin": 6.247742176055908, + "fcm_dpo/q_t": 0.3821312189102173, + "grad_norm": 23.382434844970703, + "learning_rate": 4.774079988386296e-07, + "logits/chosen": 0.24903056025505066, + "logits/rejected": 0.20170432329177856, + "logps/chosen": -69.92184448242188, + "logps/ref_chosen": -55.143775939941406, + "logps/ref_rejected": -64.79888916015625, + "logps/rejected": -85.82469177246094, + "loss": 1.0324, + "margin_dpo/margin_mean": 6.24774169921875, + "margin_dpo/margin_std": 8.07992935180664, + "step": 149 + }, + { + "KL/chosen_KL_mean": -12.613779067993164, + "KL/mean": -16.744375228881836, + "KL/rejected_KL_mean": -20.87497329711914, + "KL/std": 9.419195175170898, + "epoch": 0.22675736961451248, + "fcm_dpo/beta": 0.08011375367641449, + "fcm_dpo/delta": -0.28073906898498535, + "fcm_dpo/margin": 8.261198997497559, + "fcm_dpo/q_t": 0.35848724842071533, + "grad_norm": 22.24016761779785, + "learning_rate": 4.768555511768486e-07, + "logits/chosen": 0.24488872289657593, + "logits/rejected": 0.20085716247558594, + "logps/chosen": -80.08452606201172, + "logps/ref_chosen": -67.47074890136719, + "logps/ref_rejected": -89.21170806884766, + "logps/rejected": -110.08668518066406, + "loss": 0.9712, + "margin_dpo/margin_mean": 8.261198043823242, + "margin_dpo/margin_std": 9.845287322998047, + "step": 150 + }, + { + "KL/chosen_KL_mean": -10.584939956665039, + "KL/mean": -15.298471450805664, + "KL/rejected_KL_mean": -20.012008666992188, + "KL/std": 10.123547554016113, + "epoch": 0.22826908541194255, + "fcm_dpo/beta": 0.07523253560066223, + "fcm_dpo/delta": -0.33289921283721924, + "fcm_dpo/margin": 9.427069664001465, + "fcm_dpo/q_t": 0.3464317321777344, + "grad_norm": 20.026641845703125, + "learning_rate": 4.762967578776406e-07, + "logits/chosen": 0.27385711669921875, + "logits/rejected": 0.21827185153961182, + "logps/chosen": -63.04448318481445, + "logps/ref_chosen": -52.45954132080078, + "logps/ref_rejected": -79.0630111694336, + "logps/rejected": -99.07501983642578, + "loss": 0.9213, + "margin_dpo/margin_mean": 9.427068710327148, + "margin_dpo/margin_std": 10.230758666992188, + "step": 151 + }, + { + "KL/chosen_KL_mean": -13.667095184326172, + "KL/mean": -16.71866226196289, + "KL/rejected_KL_mean": -19.770233154296875, + "KL/std": 9.496614456176758, + "epoch": 0.22978080120937264, + "fcm_dpo/beta": 0.0733184814453125, + "fcm_dpo/delta": -0.08626732975244522, + "fcm_dpo/margin": 6.103139877319336, + "fcm_dpo/q_t": 0.40311574935913086, + "grad_norm": 22.40052604675293, + "learning_rate": 4.757316345716553e-07, + "logits/chosen": 0.343588650226593, + "logits/rejected": 0.288934588432312, + "logps/chosen": -70.22093200683594, + "logps/ref_chosen": -56.5538330078125, + "logps/ref_rejected": -76.55074310302734, + "logps/rejected": -96.32097625732422, + "loss": 1.1362, + "margin_dpo/margin_mean": 6.103139877319336, + "margin_dpo/margin_std": 10.882282257080078, + "step": 152 + }, + { + "KL/chosen_KL_mean": -12.759742736816406, + "KL/mean": -16.359935760498047, + "KL/rejected_KL_mean": -19.960128784179688, + "KL/std": 9.789167404174805, + "epoch": 0.23129251700680273, + "fcm_dpo/beta": 0.07141783833503723, + "fcm_dpo/delta": -0.12039655447006226, + "fcm_dpo/margin": 7.200382232666016, + "fcm_dpo/q_t": 0.3846072554588318, + "grad_norm": 20.95531463623047, + "learning_rate": 4.751601970666064e-07, + "logits/chosen": 0.2524608373641968, + "logits/rejected": 0.21613654494285583, + "logps/chosen": -80.76663208007812, + "logps/ref_chosen": -68.00689697265625, + "logps/ref_rejected": -74.83482360839844, + "logps/rejected": -94.79495239257812, + "loss": 1.0378, + "margin_dpo/margin_mean": 7.200382709503174, + "margin_dpo/margin_std": 9.319150924682617, + "step": 153 + }, + { + "KL/chosen_KL_mean": -13.830446243286133, + "KL/mean": -16.472694396972656, + "KL/rejected_KL_mean": -19.114944458007812, + "KL/std": 10.336735725402832, + "epoch": 0.2328042328042328, + "fcm_dpo/beta": 0.07095666974782944, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 5.2844977378845215, + "fcm_dpo/q_t": 0.4158506393432617, + "grad_norm": 22.322782516479492, + "learning_rate": 4.745824613468292e-07, + "logits/chosen": 0.3161347806453705, + "logits/rejected": 0.31378883123397827, + "logps/chosen": -73.05298614501953, + "logps/ref_chosen": -59.222537994384766, + "logps/ref_rejected": -64.19131469726562, + "logps/rejected": -83.30625915527344, + "loss": 1.1798, + "margin_dpo/margin_mean": 5.28449821472168, + "margin_dpo/margin_std": 10.676871299743652, + "step": 154 + }, + { + "KL/chosen_KL_mean": -14.819591522216797, + "KL/mean": -18.539260864257812, + "KL/rejected_KL_mean": -22.258926391601562, + "KL/std": 10.206151962280273, + "epoch": 0.23431594860166288, + "fcm_dpo/beta": 0.06935182213783264, + "fcm_dpo/delta": -0.11438489705324173, + "fcm_dpo/margin": 7.439325332641602, + "fcm_dpo/q_t": 0.3903166353702545, + "grad_norm": 21.931781768798828, + "learning_rate": 4.7399844357283393e-07, + "logits/chosen": 0.32706189155578613, + "logits/rejected": 0.30682075023651123, + "logps/chosen": -83.2742919921875, + "logps/ref_chosen": -68.45469665527344, + "logps/ref_rejected": -77.91763305664062, + "logps/rejected": -100.17655944824219, + "loss": 1.1136, + "margin_dpo/margin_mean": 7.439325332641602, + "margin_dpo/margin_std": 12.90648365020752, + "step": 155 + }, + { + "KL/chosen_KL_mean": -14.782560348510742, + "KL/mean": -19.166622161865234, + "KL/rejected_KL_mean": -23.55068588256836, + "KL/std": 11.518760681152344, + "epoch": 0.23582766439909297, + "fcm_dpo/beta": 0.06735508888959885, + "fcm_dpo/delta": -0.20239472389221191, + "fcm_dpo/margin": 8.76812744140625, + "fcm_dpo/q_t": 0.37083950638771057, + "grad_norm": 20.634889602661133, + "learning_rate": 4.7340816008085305e-07, + "logits/chosen": 0.2739195227622986, + "logits/rejected": 0.22890859842300415, + "logps/chosen": -82.05215454101562, + "logps/ref_chosen": -67.26959991455078, + "logps/ref_rejected": -86.95914459228516, + "logps/rejected": -110.50982666015625, + "loss": 0.9973, + "margin_dpo/margin_mean": 8.76812744140625, + "margin_dpo/margin_std": 10.767464637756348, + "step": 156 + }, + { + "KL/chosen_KL_mean": -12.96525764465332, + "KL/mean": -16.47336196899414, + "KL/rejected_KL_mean": -19.981468200683594, + "KL/std": 10.980501174926758, + "epoch": 0.23733938019652306, + "fcm_dpo/beta": 0.06471969187259674, + "fcm_dpo/delta": -0.14324051141738892, + "fcm_dpo/margin": 7.016205310821533, + "fcm_dpo/q_t": 0.3986649513244629, + "grad_norm": 20.302101135253906, + "learning_rate": 4.728116273823847e-07, + "logits/chosen": 0.2908157408237457, + "logits/rejected": 0.27248579263687134, + "logps/chosen": -67.73812866210938, + "logps/ref_chosen": -54.77287292480469, + "logps/ref_rejected": -63.87866973876953, + "logps/rejected": -83.86013793945312, + "loss": 1.1058, + "margin_dpo/margin_mean": 7.016204833984375, + "margin_dpo/margin_std": 10.806330680847168, + "step": 157 + }, + { + "KL/chosen_KL_mean": -14.528236389160156, + "KL/mean": -18.347126007080078, + "KL/rejected_KL_mean": -22.166015625, + "KL/std": 11.439427375793457, + "epoch": 0.23885109599395313, + "fcm_dpo/beta": 0.06432721018791199, + "fcm_dpo/delta": -0.061014823615550995, + "fcm_dpo/margin": 7.637775897979736, + "fcm_dpo/q_t": 0.39471274614334106, + "grad_norm": 20.84183692932129, + "learning_rate": 4.7220886216373085e-07, + "logits/chosen": 0.32839739322662354, + "logits/rejected": 0.28976303339004517, + "logps/chosen": -79.45095825195312, + "logps/ref_chosen": -64.92271423339844, + "logps/ref_rejected": -82.23789978027344, + "logps/rejected": -104.40391540527344, + "loss": 1.0947, + "margin_dpo/margin_mean": 7.637775897979736, + "margin_dpo/margin_std": 12.304495811462402, + "step": 158 + }, + { + "KL/chosen_KL_mean": -14.803840637207031, + "KL/mean": -19.1191463470459, + "KL/rejected_KL_mean": -23.4344482421875, + "KL/std": 12.163007736206055, + "epoch": 0.24036281179138322, + "fcm_dpo/beta": 0.06311751902103424, + "fcm_dpo/delta": -0.1294807642698288, + "fcm_dpo/margin": 8.630603790283203, + "fcm_dpo/q_t": 0.3808811604976654, + "grad_norm": 21.571117401123047, + "learning_rate": 4.715998812855304e-07, + "logits/chosen": 0.357462614774704, + "logits/rejected": 0.316484272480011, + "logps/chosen": -71.850830078125, + "logps/ref_chosen": -57.046993255615234, + "logps/ref_rejected": -73.32441711425781, + "logps/rejected": -96.75886535644531, + "loss": 1.0613, + "margin_dpo/margin_mean": 8.630603790283203, + "margin_dpo/margin_std": 12.803619384765625, + "step": 159 + }, + { + "KL/chosen_KL_mean": -16.574190139770508, + "KL/mean": -20.34947967529297, + "KL/rejected_KL_mean": -24.124771118164062, + "KL/std": 11.359709739685059, + "epoch": 0.2418745275888133, + "fcm_dpo/beta": 0.0620175302028656, + "fcm_dpo/delta": -0.04559735208749771, + "fcm_dpo/margin": 7.550580978393555, + "fcm_dpo/q_t": 0.4014434516429901, + "grad_norm": 20.139118194580078, + "learning_rate": 4.7098470178228755e-07, + "logits/chosen": 0.18542692065238953, + "logits/rejected": 0.14045989513397217, + "logps/chosen": -66.381103515625, + "logps/ref_chosen": -49.806915283203125, + "logps/ref_rejected": -68.3370132446289, + "logps/rejected": -92.46178436279297, + "loss": 1.1185, + "margin_dpo/margin_mean": 7.550581932067871, + "margin_dpo/margin_std": 13.154045104980469, + "step": 160 + }, + { + "KL/chosen_KL_mean": -16.436386108398438, + "KL/mean": -20.611623764038086, + "KL/rejected_KL_mean": -24.786865234375, + "KL/std": 11.288849830627441, + "epoch": 0.24338624338624337, + "fcm_dpo/beta": 0.060755591839551926, + "fcm_dpo/delta": -0.11299815028905869, + "fcm_dpo/margin": 8.35047721862793, + "fcm_dpo/q_t": 0.3882335424423218, + "grad_norm": 18.839397430419922, + "learning_rate": 4.703633408618955e-07, + "logits/chosen": 0.3484821915626526, + "logits/rejected": 0.30886244773864746, + "logps/chosen": -68.93687438964844, + "logps/ref_chosen": -52.50048828125, + "logps/ref_rejected": -66.04540252685547, + "logps/rejected": -90.83226776123047, + "loss": 1.0766, + "margin_dpo/margin_mean": 8.35047721862793, + "margin_dpo/margin_std": 12.587860107421875, + "step": 161 + }, + { + "KL/chosen_KL_mean": -17.52142333984375, + "KL/mean": -22.971660614013672, + "KL/rejected_KL_mean": -28.421897888183594, + "KL/std": 12.801250457763672, + "epoch": 0.24489795918367346, + "fcm_dpo/beta": 0.05810220539569855, + "fcm_dpo/delta": -0.2494203895330429, + "fcm_dpo/margin": 10.90047836303711, + "fcm_dpo/q_t": 0.3596557080745697, + "grad_norm": 20.55975341796875, + "learning_rate": 4.697358159051549e-07, + "logits/chosen": 0.37461674213409424, + "logits/rejected": 0.32385122776031494, + "logps/chosen": -86.99061584472656, + "logps/ref_chosen": -69.46919250488281, + "logps/ref_rejected": -92.00952911376953, + "logps/rejected": -120.43142700195312, + "loss": 0.983, + "margin_dpo/margin_mean": 10.900479316711426, + "margin_dpo/margin_std": 13.380390167236328, + "step": 162 + }, + { + "KL/chosen_KL_mean": -14.830795288085938, + "KL/mean": -20.422958374023438, + "KL/rejected_KL_mean": -26.015121459960938, + "KL/std": 12.709386825561523, + "epoch": 0.24640967498110355, + "fcm_dpo/beta": 0.05606578290462494, + "fcm_dpo/delta": -0.24128547310829163, + "fcm_dpo/margin": 11.184324264526367, + "fcm_dpo/q_t": 0.36260414123535156, + "grad_norm": 18.12739372253418, + "learning_rate": 4.691021444652876e-07, + "logits/chosen": 0.31139200925827026, + "logits/rejected": 0.2624328136444092, + "logps/chosen": -65.44462585449219, + "logps/ref_chosen": -50.613834381103516, + "logps/ref_rejected": -74.62033081054688, + "logps/rejected": -100.63545227050781, + "loss": 0.9951, + "margin_dpo/margin_mean": 11.184325218200684, + "margin_dpo/margin_std": 13.240589141845703, + "step": 163 + }, + { + "KL/chosen_KL_mean": -16.169986724853516, + "KL/mean": -21.591407775878906, + "KL/rejected_KL_mean": -27.012828826904297, + "KL/std": 13.454334259033203, + "epoch": 0.24792139077853365, + "fcm_dpo/beta": 0.05384572595357895, + "fcm_dpo/delta": -0.16112825274467468, + "fcm_dpo/margin": 10.842841148376465, + "fcm_dpo/q_t": 0.37319236993789673, + "grad_norm": 16.985376358032227, + "learning_rate": 4.6846234426744624e-07, + "logits/chosen": 0.32287657260894775, + "logits/rejected": 0.256435751914978, + "logps/chosen": -71.01809692382812, + "logps/ref_chosen": -54.848114013671875, + "logps/ref_rejected": -79.0630111694336, + "logps/rejected": -106.07583618164062, + "loss": 1.0313, + "margin_dpo/margin_mean": 10.842840194702148, + "margin_dpo/margin_std": 14.509929656982422, + "step": 164 + }, + { + "KL/chosen_KL_mean": -18.375152587890625, + "KL/mean": -23.237693786621094, + "KL/rejected_KL_mean": -28.100234985351562, + "KL/std": 13.708030700683594, + "epoch": 0.2494331065759637, + "fcm_dpo/beta": 0.05240638554096222, + "fcm_dpo/delta": -0.10911506414413452, + "fcm_dpo/margin": 9.725078582763672, + "fcm_dpo/q_t": 0.3871835172176361, + "grad_norm": 17.69078254699707, + "learning_rate": 4.678164332082175e-07, + "logits/chosen": 0.3671991229057312, + "logits/rejected": 0.31109094619750977, + "logps/chosen": -69.46436309814453, + "logps/ref_chosen": -51.089210510253906, + "logps/ref_rejected": -71.23370361328125, + "logps/rejected": -99.33393859863281, + "loss": 1.0871, + "margin_dpo/margin_mean": 9.725078582763672, + "margin_dpo/margin_std": 14.881806373596191, + "step": 165 + }, + { + "KL/chosen_KL_mean": -17.435197830200195, + "KL/mean": -21.304927825927734, + "KL/rejected_KL_mean": -25.174659729003906, + "KL/std": 13.740734100341797, + "epoch": 0.2509448223733938, + "fcm_dpo/beta": 0.051834575831890106, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.7394609451293945, + "fcm_dpo/q_t": 0.41240036487579346, + "grad_norm": 20.236228942871094, + "learning_rate": 4.6716442935512214e-07, + "logits/chosen": 0.3307892084121704, + "logits/rejected": 0.2354990839958191, + "logps/chosen": -80.62600708007812, + "logps/ref_chosen": -63.19081115722656, + "logps/ref_rejected": -93.8402099609375, + "logps/rejected": -119.0148696899414, + "loss": 1.1315, + "margin_dpo/margin_mean": 7.739460468292236, + "margin_dpo/margin_std": 13.147438049316406, + "step": 166 + }, + { + "KL/chosen_KL_mean": -14.410505294799805, + "KL/mean": -20.310304641723633, + "KL/rejected_KL_mean": -26.210105895996094, + "KL/std": 13.815751075744629, + "epoch": 0.25245653817082386, + "fcm_dpo/beta": 0.050010696053504944, + "fcm_dpo/delta": -0.20380280911922455, + "fcm_dpo/margin": 11.79959487915039, + "fcm_dpo/q_t": 0.3688342571258545, + "grad_norm": 18.09819793701172, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": 0.2875286936759949, + "logits/rejected": 0.2525779902935028, + "logps/chosen": -73.33477783203125, + "logps/ref_chosen": -58.92427062988281, + "logps/ref_rejected": -72.97377014160156, + "logps/rejected": -99.18387603759766, + "loss": 0.985, + "margin_dpo/margin_mean": 11.79959487915039, + "margin_dpo/margin_std": 13.582077026367188, + "step": 167 + }, + { + "KL/chosen_KL_mean": -17.27781105041504, + "KL/mean": -22.058712005615234, + "KL/rejected_KL_mean": -26.839614868164062, + "KL/std": 15.021439552307129, + "epoch": 0.25396825396825395, + "fcm_dpo/beta": 0.049220357090234756, + "fcm_dpo/delta": -0.05494771525263786, + "fcm_dpo/margin": 9.561796188354492, + "fcm_dpo/q_t": 0.39759543538093567, + "grad_norm": 18.367740631103516, + "learning_rate": 4.6584221638904767e-07, + "logits/chosen": 0.32051700353622437, + "logits/rejected": 0.282333642244339, + "logps/chosen": -82.92919158935547, + "logps/ref_chosen": -65.65138244628906, + "logps/ref_rejected": -79.71418762207031, + "logps/rejected": -106.55380249023438, + "loss": 1.0793, + "margin_dpo/margin_mean": 9.561796188354492, + "margin_dpo/margin_std": 14.055609703063965, + "step": 168 + }, + { + "KL/chosen_KL_mean": -13.753213882446289, + "KL/mean": -19.046161651611328, + "KL/rejected_KL_mean": -24.339111328125, + "KL/std": 14.64225959777832, + "epoch": 0.25547996976568405, + "fcm_dpo/beta": 0.0483536496758461, + "fcm_dpo/delta": -0.11793233454227448, + "fcm_dpo/margin": 10.585901260375977, + "fcm_dpo/q_t": 0.39021724462509155, + "grad_norm": 20.023157119750977, + "learning_rate": 4.651720442612075e-07, + "logits/chosen": 0.39085686206817627, + "logits/rejected": 0.35531675815582275, + "logps/chosen": -75.1790771484375, + "logps/ref_chosen": -61.425865173339844, + "logps/ref_rejected": -76.09590148925781, + "logps/rejected": -100.43501281738281, + "loss": 1.0937, + "margin_dpo/margin_mean": 10.585901260375977, + "margin_dpo/margin_std": 17.442779541015625, + "step": 169 + }, + { + "KL/chosen_KL_mean": -12.770061492919922, + "KL/mean": -18.207551956176758, + "KL/rejected_KL_mean": -23.645044326782227, + "KL/std": 14.344905853271484, + "epoch": 0.25699168556311414, + "fcm_dpo/beta": 0.04731673002243042, + "fcm_dpo/delta": -0.12062279880046844, + "fcm_dpo/margin": 10.874979972839355, + "fcm_dpo/q_t": 0.38753047585487366, + "grad_norm": 16.221012115478516, + "learning_rate": 4.6449585330874425e-07, + "logits/chosen": 0.33115026354789734, + "logits/rejected": 0.3292052447795868, + "logps/chosen": -69.42324829101562, + "logps/ref_chosen": -56.65319061279297, + "logps/ref_rejected": -63.45965576171875, + "logps/rejected": -87.10469818115234, + "loss": 1.1045, + "margin_dpo/margin_mean": 10.874979019165039, + "margin_dpo/margin_std": 17.857650756835938, + "step": 170 + }, + { + "KL/chosen_KL_mean": -15.31631088256836, + "KL/mean": -21.237224578857422, + "KL/rejected_KL_mean": -27.15814208984375, + "KL/std": 16.512523651123047, + "epoch": 0.2585034013605442, + "fcm_dpo/beta": 0.045385200530290604, + "fcm_dpo/delta": -0.1670508235692978, + "fcm_dpo/margin": 11.841828346252441, + "fcm_dpo/q_t": 0.38251635432243347, + "grad_norm": 16.580425262451172, + "learning_rate": 4.6381366244617224e-07, + "logits/chosen": 0.401705265045166, + "logits/rejected": 0.34923315048217773, + "logps/chosen": -79.05107116699219, + "logps/ref_chosen": -63.73476028442383, + "logps/ref_rejected": -78.50328063964844, + "logps/rejected": -105.66142272949219, + "loss": 1.0877, + "margin_dpo/margin_mean": 11.841829299926758, + "margin_dpo/margin_std": 18.744915008544922, + "step": 171 + }, + { + "KL/chosen_KL_mean": -17.131195068359375, + "KL/mean": -22.653350830078125, + "KL/rejected_KL_mean": -28.175498962402344, + "KL/std": 16.811195373535156, + "epoch": 0.2600151171579743, + "fcm_dpo/beta": 0.044791221618652344, + "fcm_dpo/delta": -0.13261796534061432, + "fcm_dpo/margin": 11.044305801391602, + "fcm_dpo/q_t": 0.3933247923851013, + "grad_norm": 17.82714080810547, + "learning_rate": 4.631254907558365e-07, + "logits/chosen": 0.4093649089336395, + "logits/rejected": 0.34693312644958496, + "logps/chosen": -69.33295440673828, + "logps/ref_chosen": -52.201759338378906, + "logps/ref_rejected": -82.85285949707031, + "logps/rejected": -111.02836608886719, + "loss": 1.0927, + "margin_dpo/margin_mean": 11.044305801391602, + "margin_dpo/margin_std": 17.3895263671875, + "step": 172 + }, + { + "KL/chosen_KL_mean": -15.58431625366211, + "KL/mean": -21.632266998291016, + "KL/rejected_KL_mean": -27.680221557617188, + "KL/std": 17.105926513671875, + "epoch": 0.2615268329554044, + "fcm_dpo/beta": 0.04276077076792717, + "fcm_dpo/delta": -0.16520676016807556, + "fcm_dpo/margin": 12.095901489257812, + "fcm_dpo/q_t": 0.3909044563770294, + "grad_norm": 17.054214477539062, + "learning_rate": 4.624313574873786e-07, + "logits/chosen": 0.3971649408340454, + "logits/rejected": 0.30516812205314636, + "logps/chosen": -71.01904296875, + "logps/ref_chosen": -55.434722900390625, + "logps/ref_rejected": -77.81967163085938, + "logps/rejected": -105.49989318847656, + "loss": 1.1222, + "margin_dpo/margin_mean": 12.095902442932129, + "margin_dpo/margin_std": 21.07677459716797, + "step": 173 + }, + { + "KL/chosen_KL_mean": -17.717504501342773, + "KL/mean": -24.340293884277344, + "KL/rejected_KL_mean": -30.963085174560547, + "KL/std": 17.349761962890625, + "epoch": 0.26303854875283444, + "fcm_dpo/beta": 0.04165055602788925, + "fcm_dpo/delta": -0.16104662418365479, + "fcm_dpo/margin": 13.245588302612305, + "fcm_dpo/q_t": 0.3786090612411499, + "grad_norm": 17.300262451171875, + "learning_rate": 4.61731282057198e-07, + "logits/chosen": 0.374323308467865, + "logits/rejected": 0.3048890233039856, + "logps/chosen": -74.88945770263672, + "logps/ref_chosen": -57.17195129394531, + "logps/ref_rejected": -85.47578430175781, + "logps/rejected": -116.4388656616211, + "loss": 1.0454, + "margin_dpo/margin_mean": 13.245588302612305, + "margin_dpo/margin_std": 18.74484634399414, + "step": 174 + }, + { + "KL/chosen_KL_mean": -15.173547744750977, + "KL/mean": -22.60834503173828, + "KL/rejected_KL_mean": -30.043148040771484, + "KL/std": 17.65919303894043, + "epoch": 0.26455026455026454, + "fcm_dpo/beta": 0.04021874815225601, + "fcm_dpo/delta": -0.21028748154640198, + "fcm_dpo/margin": 14.869599342346191, + "fcm_dpo/q_t": 0.37366974353790283, + "grad_norm": 16.226505279541016, + "learning_rate": 4.6102528404790965e-07, + "logits/chosen": 0.3995314836502075, + "logits/rejected": 0.3691650629043579, + "logps/chosen": -82.83917236328125, + "logps/ref_chosen": -67.6656265258789, + "logps/ref_rejected": -84.36766815185547, + "logps/rejected": -114.41081237792969, + "loss": 1.0296, + "margin_dpo/margin_mean": 14.869599342346191, + "margin_dpo/margin_std": 20.803794860839844, + "step": 175 + }, + { + "KL/chosen_KL_mean": -19.654399871826172, + "KL/mean": -24.40457534790039, + "KL/rejected_KL_mean": -29.154754638671875, + "KL/std": 18.916370391845703, + "epoch": 0.2660619803476946, + "fcm_dpo/beta": 0.039426859468221664, + "fcm_dpo/delta": -0.03453513979911804, + "fcm_dpo/margin": 9.500359535217285, + "fcm_dpo/q_t": 0.41981422901153564, + "grad_norm": 20.208744049072266, + "learning_rate": 4.603133832077953e-07, + "logits/chosen": 0.3505083918571472, + "logits/rejected": 0.3238475024700165, + "logps/chosen": -97.5131607055664, + "logps/ref_chosen": -77.8587646484375, + "logps/ref_rejected": -81.08732604980469, + "logps/rejected": -110.24208068847656, + "loss": 1.2111, + "margin_dpo/margin_mean": 9.500359535217285, + "margin_dpo/margin_std": 21.4652099609375, + "step": 176 + }, + { + "KL/chosen_KL_mean": -14.536983489990234, + "KL/mean": -23.876373291015625, + "KL/rejected_KL_mean": -33.21575927734375, + "KL/std": 19.081396102905273, + "epoch": 0.2675736961451247, + "fcm_dpo/beta": 0.03744693845510483, + "fcm_dpo/delta": -0.3234514594078064, + "fcm_dpo/margin": 18.67877197265625, + "fcm_dpo/q_t": 0.34551703929901123, + "grad_norm": 19.835054397583008, + "learning_rate": 4.5959559945025183e-07, + "logits/chosen": 0.4804215729236603, + "logits/rejected": 0.38235175609588623, + "logps/chosen": -69.75738525390625, + "logps/ref_chosen": -55.22039794921875, + "logps/ref_rejected": -92.54973602294922, + "logps/rejected": -125.76549530029297, + "loss": 0.9197, + "margin_dpo/margin_mean": 18.67877197265625, + "margin_dpo/margin_std": 19.456634521484375, + "step": 177 + }, + { + "KL/chosen_KL_mean": -17.895334243774414, + "KL/mean": -23.562305450439453, + "KL/rejected_KL_mean": -29.22927474975586, + "KL/std": 18.05139923095703, + "epoch": 0.2690854119425548, + "fcm_dpo/beta": 0.036295369267463684, + "fcm_dpo/delta": -0.09033413976430893, + "fcm_dpo/margin": 11.333934783935547, + "fcm_dpo/q_t": 0.40735214948654175, + "grad_norm": 19.553966522216797, + "learning_rate": 4.588719528532341e-07, + "logits/chosen": 0.3673199415206909, + "logits/rejected": 0.3156449496746063, + "logps/chosen": -78.70582580566406, + "logps/ref_chosen": -60.81049346923828, + "logps/ref_rejected": -81.12973022460938, + "logps/rejected": -110.35900115966797, + "loss": 1.1214, + "margin_dpo/margin_mean": 11.333934783935547, + "margin_dpo/margin_std": 18.04642105102539, + "step": 178 + }, + { + "KL/chosen_KL_mean": -18.201736450195312, + "KL/mean": -25.386974334716797, + "KL/rejected_KL_mean": -32.57221221923828, + "KL/std": 19.182567596435547, + "epoch": 0.2705971277399849, + "fcm_dpo/beta": 0.0356178879737854, + "fcm_dpo/delta": -0.11806750297546387, + "fcm_dpo/margin": 14.370477676391602, + "fcm_dpo/q_t": 0.3880265951156616, + "grad_norm": 17.179960250854492, + "learning_rate": 4.581424636586928e-07, + "logits/chosen": 0.4421459436416626, + "logits/rejected": 0.4255564212799072, + "logps/chosen": -83.87345123291016, + "logps/ref_chosen": -65.67171478271484, + "logps/ref_rejected": -75.32586669921875, + "logps/rejected": -107.89807891845703, + "loss": 1.0816, + "margin_dpo/margin_mean": 14.370477676391602, + "margin_dpo/margin_std": 22.50054931640625, + "step": 179 + }, + { + "KL/chosen_KL_mean": -13.901803970336914, + "KL/mean": -19.616230010986328, + "KL/rejected_KL_mean": -25.330659866333008, + "KL/std": 18.365304946899414, + "epoch": 0.272108843537415, + "fcm_dpo/beta": 0.0354483462870121, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 11.428857803344727, + "fcm_dpo/q_t": 0.41211187839508057, + "grad_norm": 14.897982597351074, + "learning_rate": 4.5740715227200897e-07, + "logits/chosen": 0.2380199283361435, + "logits/rejected": 0.21933087706565857, + "logps/chosen": -70.58460998535156, + "logps/ref_chosen": -56.68280792236328, + "logps/ref_rejected": -64.94414520263672, + "logps/rejected": -90.27481079101562, + "loss": 1.1499, + "margin_dpo/margin_mean": 11.428857803344727, + "margin_dpo/margin_std": 21.246871948242188, + "step": 180 + }, + { + "KL/chosen_KL_mean": -12.012491226196289, + "KL/mean": -20.349918365478516, + "KL/rejected_KL_mean": -28.68734359741211, + "KL/std": 19.755210876464844, + "epoch": 0.273620559334845, + "fcm_dpo/beta": 0.03441469371318817, + "fcm_dpo/delta": -0.1849377453327179, + "fcm_dpo/margin": 16.674854278564453, + "fcm_dpo/q_t": 0.37329137325286865, + "grad_norm": 17.271093368530273, + "learning_rate": 4.566660392614228e-07, + "logits/chosen": 0.4154462218284607, + "logits/rejected": 0.37344780564308167, + "logps/chosen": -72.78854370117188, + "logps/ref_chosen": -60.77604675292969, + "logps/ref_rejected": -83.98361206054688, + "logps/rejected": -112.67095947265625, + "loss": 0.9922, + "margin_dpo/margin_mean": 16.674854278564453, + "margin_dpo/margin_std": 19.586292266845703, + "step": 181 + }, + { + "KL/chosen_KL_mean": -18.33358383178711, + "KL/mean": -26.865509033203125, + "KL/rejected_KL_mean": -35.397430419921875, + "KL/std": 21.36233139038086, + "epoch": 0.2751322751322751, + "fcm_dpo/beta": 0.03329051285982132, + "fcm_dpo/delta": -0.17808839678764343, + "fcm_dpo/margin": 17.06385040283203, + "fcm_dpo/q_t": 0.37668097019195557, + "grad_norm": 18.594280242919922, + "learning_rate": 4.5591914535745817e-07, + "logits/chosen": 0.3790324926376343, + "logits/rejected": 0.29383039474487305, + "logps/chosen": -78.58737182617188, + "logps/ref_chosen": -60.2537841796875, + "logps/ref_rejected": -89.7706298828125, + "logps/rejected": -125.16806030273438, + "loss": 1.0409, + "margin_dpo/margin_mean": 17.06385040283203, + "margin_dpo/margin_std": 24.047889709472656, + "step": 182 + }, + { + "KL/chosen_KL_mean": -22.590940475463867, + "KL/mean": -26.173816680908203, + "KL/rejected_KL_mean": -29.75668716430664, + "KL/std": 20.493114471435547, + "epoch": 0.2766439909297052, + "fcm_dpo/beta": 0.032965824007987976, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.1657514572143555, + "fcm_dpo/q_t": 0.44580399990081787, + "grad_norm": 16.00063705444336, + "learning_rate": 4.551664914523433e-07, + "logits/chosen": 0.3670530319213867, + "logits/rejected": 0.3447875380516052, + "logps/chosen": -84.35235595703125, + "logps/ref_chosen": -61.76142120361328, + "logps/ref_rejected": -72.54627990722656, + "logps/rejected": -102.30297088623047, + "loss": 1.2746, + "margin_dpo/margin_mean": 7.1657514572143555, + "margin_dpo/margin_std": 20.836429595947266, + "step": 183 + }, + { + "KL/chosen_KL_mean": -13.242368698120117, + "KL/mean": -19.70210838317871, + "KL/rejected_KL_mean": -26.161846160888672, + "KL/std": 17.709320068359375, + "epoch": 0.2781557067271353, + "fcm_dpo/beta": 0.03258639574050903, + "fcm_dpo/delta": -0.0578826479613781, + "fcm_dpo/margin": 12.919473648071289, + "fcm_dpo/q_t": 0.40624159574508667, + "grad_norm": 16.270227432250977, + "learning_rate": 4.544080985994258e-07, + "logits/chosen": 0.4893711805343628, + "logits/rejected": 0.42196178436279297, + "logps/chosen": -60.083091735839844, + "logps/ref_chosen": -46.840721130371094, + "logps/ref_rejected": -69.3609390258789, + "logps/rejected": -95.52278137207031, + "loss": 1.097, + "margin_dpo/margin_mean": 12.919473648071289, + "margin_dpo/margin_std": 18.708168029785156, + "step": 184 + }, + { + "KL/chosen_KL_mean": -17.824491500854492, + "KL/mean": -25.49761199951172, + "KL/rejected_KL_mean": -33.17073440551758, + "KL/std": 21.816608428955078, + "epoch": 0.2796674225245654, + "fcm_dpo/beta": 0.03175631910562515, + "fcm_dpo/delta": -0.12901580333709717, + "fcm_dpo/margin": 15.346242904663086, + "fcm_dpo/q_t": 0.3941470980644226, + "grad_norm": 14.84249496459961, + "learning_rate": 4.5364398801258394e-07, + "logits/chosen": 0.4229778051376343, + "logits/rejected": 0.37561944127082825, + "logps/chosen": -70.1456298828125, + "logps/ref_chosen": -52.32114028930664, + "logps/ref_rejected": -68.3885726928711, + "logps/rejected": -101.55931091308594, + "loss": 1.1206, + "margin_dpo/margin_mean": 15.346242904663086, + "margin_dpo/margin_std": 26.30462646484375, + "step": 185 + }, + { + "KL/chosen_KL_mean": -15.705266952514648, + "KL/mean": -24.041990280151367, + "KL/rejected_KL_mean": -32.37870788574219, + "KL/std": 21.698299407958984, + "epoch": 0.2811791383219955, + "fcm_dpo/beta": 0.0311187282204628, + "fcm_dpo/delta": -0.1256175935268402, + "fcm_dpo/margin": 16.673439025878906, + "fcm_dpo/q_t": 0.3876885771751404, + "grad_norm": 18.64262580871582, + "learning_rate": 4.5287418106563354e-07, + "logits/chosen": 0.3782210350036621, + "logits/rejected": 0.33319205045700073, + "logps/chosen": -83.12539672851562, + "logps/ref_chosen": -67.42012786865234, + "logps/ref_rejected": -82.50968933105469, + "logps/rejected": -114.88839721679688, + "loss": 1.0813, + "margin_dpo/margin_mean": 16.673439025878906, + "margin_dpo/margin_std": 26.293109893798828, + "step": 186 + }, + { + "KL/chosen_KL_mean": -20.900047302246094, + "KL/mean": -28.237571716308594, + "KL/rejected_KL_mean": -35.575096130371094, + "KL/std": 23.087440490722656, + "epoch": 0.28269085411942557, + "fcm_dpo/beta": 0.030645418912172318, + "fcm_dpo/delta": -0.05242550000548363, + "fcm_dpo/margin": 14.67505168914795, + "fcm_dpo/q_t": 0.4052157998085022, + "grad_norm": 17.817066192626953, + "learning_rate": 4.520986992917297e-07, + "logits/chosen": 0.40862882137298584, + "logits/rejected": 0.35285601019859314, + "logps/chosen": -96.425537109375, + "logps/ref_chosen": -75.52549743652344, + "logps/ref_rejected": -94.76289367675781, + "logps/rejected": -130.33798217773438, + "loss": 1.141, + "margin_dpo/margin_mean": 14.675050735473633, + "margin_dpo/margin_std": 27.1290283203125, + "step": 187 + }, + { + "KL/chosen_KL_mean": -18.666240692138672, + "KL/mean": -27.357372283935547, + "KL/rejected_KL_mean": -36.048500061035156, + "KL/std": 23.889400482177734, + "epoch": 0.2842025699168556, + "fcm_dpo/beta": 0.030069969594478607, + "fcm_dpo/delta": -0.1294178068637848, + "fcm_dpo/margin": 17.38225555419922, + "fcm_dpo/q_t": 0.38837963342666626, + "grad_norm": 14.66856575012207, + "learning_rate": 4.5131756438276466e-07, + "logits/chosen": 0.4121420383453369, + "logits/rejected": 0.36901307106018066, + "logps/chosen": -90.1895751953125, + "logps/ref_chosen": -71.52333068847656, + "logps/ref_rejected": -78.29949951171875, + "logps/rejected": -114.3479995727539, + "loss": 1.0634, + "margin_dpo/margin_mean": 17.38225555419922, + "margin_dpo/margin_std": 25.864315032958984, + "step": 188 + }, + { + "KL/chosen_KL_mean": -18.839712142944336, + "KL/mean": -26.519317626953125, + "KL/rejected_KL_mean": -34.19892501831055, + "KL/std": 23.881752014160156, + "epoch": 0.2857142857142857, + "fcm_dpo/beta": 0.029213856905698776, + "fcm_dpo/delta": -0.10978260636329651, + "fcm_dpo/margin": 15.359216690063477, + "fcm_dpo/q_t": 0.40099185705184937, + "grad_norm": 16.19655418395996, + "learning_rate": 4.5053079818876096e-07, + "logits/chosen": 0.4219725728034973, + "logits/rejected": 0.43759363889694214, + "logps/chosen": -91.01597595214844, + "logps/ref_chosen": -72.17626953125, + "logps/ref_rejected": -75.26313781738281, + "logps/rejected": -109.4620590209961, + "loss": 1.11, + "margin_dpo/margin_mean": 15.35921859741211, + "margin_dpo/margin_std": 24.676734924316406, + "step": 189 + }, + { + "KL/chosen_KL_mean": -18.256019592285156, + "KL/mean": -29.198854446411133, + "KL/rejected_KL_mean": -40.141685485839844, + "KL/std": 24.935558319091797, + "epoch": 0.2872260015117158, + "fcm_dpo/beta": 0.0281969103962183, + "fcm_dpo/delta": -0.231668621301651, + "fcm_dpo/margin": 21.885665893554688, + "fcm_dpo/q_t": 0.36639824509620667, + "grad_norm": 20.939247131347656, + "learning_rate": 4.4973842271726024e-07, + "logits/chosen": 0.5043848752975464, + "logits/rejected": 0.3508925437927246, + "logps/chosen": -72.88029479980469, + "logps/ref_chosen": -54.624271392822266, + "logps/ref_rejected": -101.47068786621094, + "logps/rejected": -141.61236572265625, + "loss": 0.9898, + "margin_dpo/margin_mean": 21.885665893554688, + "margin_dpo/margin_std": 27.162105560302734, + "step": 190 + }, + { + "KL/chosen_KL_mean": -26.937339782714844, + "KL/mean": -35.45838165283203, + "KL/rejected_KL_mean": -43.97942352294922, + "KL/std": 26.476070404052734, + "epoch": 0.2887377173091459, + "fcm_dpo/beta": 0.027564112097024918, + "fcm_dpo/delta": -0.07326999306678772, + "fcm_dpo/margin": 17.042081832885742, + "fcm_dpo/q_t": 0.3958955407142639, + "grad_norm": 17.457185745239258, + "learning_rate": 4.48940460132708e-07, + "logits/chosen": 0.46865350008010864, + "logits/rejected": 0.43871521949768066, + "logps/chosen": -99.86985778808594, + "logps/ref_chosen": -72.93251037597656, + "logps/ref_rejected": -89.95103454589844, + "logps/rejected": -133.93045043945312, + "loss": 1.1102, + "margin_dpo/margin_mean": 17.042081832885742, + "margin_dpo/margin_std": 28.385475158691406, + "step": 191 + }, + { + "KL/chosen_KL_mean": -30.011014938354492, + "KL/mean": -35.43492889404297, + "KL/rejected_KL_mean": -40.858848571777344, + "KL/std": 27.704442977905273, + "epoch": 0.29024943310657597, + "fcm_dpo/beta": 0.02748541347682476, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 10.847834587097168, + "fcm_dpo/q_t": 0.4331100285053253, + "grad_norm": 16.07075309753418, + "learning_rate": 4.481369327558329e-07, + "logits/chosen": 0.5033490657806396, + "logits/rejected": 0.47497987747192383, + "logps/chosen": -84.01213073730469, + "logps/ref_chosen": -54.001121520996094, + "logps/ref_rejected": -63.531551361083984, + "logps/rejected": -104.39039611816406, + "loss": 1.2127, + "margin_dpo/margin_mean": 10.847833633422852, + "margin_dpo/margin_std": 24.105567932128906, + "step": 192 + }, + { + "KL/chosen_KL_mean": -24.1334285736084, + "KL/mean": -34.30812454223633, + "KL/rejected_KL_mean": -44.48281478881836, + "KL/std": 25.59041404724121, + "epoch": 0.29176114890400606, + "fcm_dpo/beta": 0.026737425476312637, + "fcm_dpo/delta": -0.15358255803585052, + "fcm_dpo/margin": 20.349384307861328, + "fcm_dpo/q_t": 0.37987715005874634, + "grad_norm": 13.550483703613281, + "learning_rate": 4.47327863063023e-07, + "logits/chosen": 0.44205987453460693, + "logits/rejected": 0.4210042357444763, + "logps/chosen": -80.88270568847656, + "logps/ref_chosen": -56.74927520751953, + "logps/ref_rejected": -58.80629348754883, + "logps/rejected": -103.28910827636719, + "loss": 1.0276, + "margin_dpo/margin_mean": 20.349384307861328, + "margin_dpo/margin_std": 26.462234497070312, + "step": 193 + }, + { + "KL/chosen_KL_mean": -26.698780059814453, + "KL/mean": -35.048187255859375, + "KL/rejected_KL_mean": -43.39760208129883, + "KL/std": 27.187034606933594, + "epoch": 0.29327286470143615, + "fcm_dpo/beta": 0.026069460436701775, + "fcm_dpo/delta": -0.11087217181921005, + "fcm_dpo/margin": 16.698816299438477, + "fcm_dpo/q_t": 0.4047401547431946, + "grad_norm": 18.267810821533203, + "learning_rate": 4.4651327368569684e-07, + "logits/chosen": 0.5409113168716431, + "logits/rejected": 0.5088620185852051, + "logps/chosen": -83.34822082519531, + "logps/ref_chosen": -56.64944076538086, + "logps/ref_rejected": -69.98954772949219, + "logps/rejected": -113.38714599609375, + "loss": 1.1575, + "margin_dpo/margin_mean": 16.69881820678711, + "margin_dpo/margin_std": 31.43659019470215, + "step": 194 + }, + { + "KL/chosen_KL_mean": -31.998184204101562, + "KL/mean": -42.90058898925781, + "KL/rejected_KL_mean": -53.80299758911133, + "KL/std": 29.968097686767578, + "epoch": 0.2947845804988662, + "fcm_dpo/beta": 0.025347769260406494, + "fcm_dpo/delta": -0.16259239614009857, + "fcm_dpo/margin": 21.80481719970703, + "fcm_dpo/q_t": 0.3793768882751465, + "grad_norm": 17.709516525268555, + "learning_rate": 4.4569318740967043e-07, + "logits/chosen": 0.3753807246685028, + "logits/rejected": 0.3845909535884857, + "logps/chosen": -102.407958984375, + "logps/ref_chosen": -70.40977478027344, + "logps/ref_rejected": -74.39448547363281, + "logps/rejected": -128.19747924804688, + "loss": 1.0462, + "margin_dpo/margin_mean": 21.80481719970703, + "margin_dpo/margin_std": 30.805763244628906, + "step": 195 + }, + { + "KL/chosen_KL_mean": -31.96177101135254, + "KL/mean": -40.567466735839844, + "KL/rejected_KL_mean": -49.173160552978516, + "KL/std": 30.731754302978516, + "epoch": 0.2962962962962963, + "fcm_dpo/beta": 0.02507277950644493, + "fcm_dpo/delta": -0.06484292447566986, + "fcm_dpo/margin": 17.211387634277344, + "fcm_dpo/q_t": 0.40247493982315063, + "grad_norm": 18.032867431640625, + "learning_rate": 4.448676271745197e-07, + "logits/chosen": 0.4948139786720276, + "logits/rejected": 0.4499012231826782, + "logps/chosen": -91.18934631347656, + "logps/ref_chosen": -59.227577209472656, + "logps/ref_rejected": -83.54757690429688, + "logps/rejected": -132.72073364257812, + "loss": 1.1033, + "margin_dpo/margin_mean": 17.211387634277344, + "margin_dpo/margin_std": 26.30760955810547, + "step": 196 + }, + { + "KL/chosen_KL_mean": -28.26848793029785, + "KL/mean": -38.98504638671875, + "KL/rejected_KL_mean": -49.701595306396484, + "KL/std": 29.063312530517578, + "epoch": 0.29780801209372637, + "fcm_dpo/beta": 0.024582477286458015, + "fcm_dpo/delta": -0.13332447409629822, + "fcm_dpo/margin": 21.433109283447266, + "fcm_dpo/q_t": 0.38597390055656433, + "grad_norm": 17.628982543945312, + "learning_rate": 4.440366160729392e-07, + "logits/chosen": 0.6032913327217102, + "logits/rejected": 0.5473294258117676, + "logps/chosen": -79.79762268066406, + "logps/ref_chosen": -51.52912902832031, + "logps/ref_rejected": -73.70631408691406, + "logps/rejected": -123.40790557861328, + "loss": 1.1096, + "margin_dpo/margin_mean": 21.433109283447266, + "margin_dpo/margin_std": 36.053382873535156, + "step": 197 + }, + { + "KL/chosen_KL_mean": -31.24224090576172, + "KL/mean": -42.64985656738281, + "KL/rejected_KL_mean": -54.0574836730957, + "KL/std": 29.603408813476562, + "epoch": 0.29931972789115646, + "fcm_dpo/beta": 0.023822344839572906, + "fcm_dpo/delta": -0.15149368345737457, + "fcm_dpo/margin": 22.815235137939453, + "fcm_dpo/q_t": 0.3782616853713989, + "grad_norm": 16.272396087646484, + "learning_rate": 4.432001773500957e-07, + "logits/chosen": 0.572478711605072, + "logits/rejected": 0.5288437604904175, + "logps/chosen": -91.02491760253906, + "logps/ref_chosen": -59.78268051147461, + "logps/ref_rejected": -72.24533081054688, + "logps/rejected": -126.30281066894531, + "loss": 1.0302, + "margin_dpo/margin_mean": 22.815235137939453, + "margin_dpo/margin_std": 29.531604766845703, + "step": 198 + }, + { + "KL/chosen_KL_mean": -37.891090393066406, + "KL/mean": -46.90549087524414, + "KL/rejected_KL_mean": -55.91989517211914, + "KL/std": 30.329345703125, + "epoch": 0.30083144368858655, + "fcm_dpo/beta": 0.023282885551452637, + "fcm_dpo/delta": -0.10651887953281403, + "fcm_dpo/margin": 18.028810501098633, + "fcm_dpo/q_t": 0.40665411949157715, + "grad_norm": 15.3820219039917, + "learning_rate": 4.4235833440297856e-07, + "logits/chosen": 0.49368178844451904, + "logits/rejected": 0.40359988808631897, + "logps/chosen": -94.27786254882812, + "logps/ref_chosen": -56.38677215576172, + "logps/ref_rejected": -74.56779479980469, + "logps/rejected": -130.48770141601562, + "loss": 1.1625, + "margin_dpo/margin_mean": 18.028810501098633, + "margin_dpo/margin_std": 33.92662048339844, + "step": 199 + }, + { + "KL/chosen_KL_mean": -30.862167358398438, + "KL/mean": -43.695106506347656, + "KL/rejected_KL_mean": -56.528053283691406, + "KL/std": 31.77981185913086, + "epoch": 0.30234315948601664, + "fcm_dpo/beta": 0.0223165862262249, + "fcm_dpo/delta": -0.184445321559906, + "fcm_dpo/margin": 25.66588592529297, + "fcm_dpo/q_t": 0.37537479400634766, + "grad_norm": 14.177912712097168, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": 0.5794877409934998, + "logits/rejected": 0.49834418296813965, + "logps/chosen": -88.68649291992188, + "logps/ref_chosen": -57.82432556152344, + "logps/ref_rejected": -89.28246307373047, + "logps/rejected": -145.81051635742188, + "loss": 1.0502, + "margin_dpo/margin_mean": 25.66588592529297, + "margin_dpo/margin_std": 37.56428527832031, + "step": 200 + }, + { + "KL/chosen_KL_mean": -42.7253303527832, + "KL/mean": -52.768768310546875, + "KL/rejected_KL_mean": -62.81220626831055, + "KL/std": 33.54126739501953, + "epoch": 0.30385487528344673, + "fcm_dpo/beta": 0.02184602990746498, + "fcm_dpo/delta": -0.0805087760090828, + "fcm_dpo/margin": 20.086875915527344, + "fcm_dpo/q_t": 0.403793066740036, + "grad_norm": 16.322202682495117, + "learning_rate": 4.4065853017905953e-07, + "logits/chosen": 0.5901821851730347, + "logits/rejected": 0.5399671196937561, + "logps/chosen": -101.72509002685547, + "logps/ref_chosen": -58.999759674072266, + "logps/ref_rejected": -84.67575073242188, + "logps/rejected": -147.48794555664062, + "loss": 1.1266, + "margin_dpo/margin_mean": 20.086875915527344, + "margin_dpo/margin_std": 34.55057907104492, + "step": 201 + }, + { + "KL/chosen_KL_mean": -36.28567886352539, + "KL/mean": -49.248268127441406, + "KL/rejected_KL_mean": -62.21084976196289, + "KL/std": 31.714256286621094, + "epoch": 0.30536659108087677, + "fcm_dpo/beta": 0.021372711285948753, + "fcm_dpo/delta": -0.16286848485469818, + "fcm_dpo/margin": 25.925172805786133, + "fcm_dpo/q_t": 0.3748985528945923, + "grad_norm": 14.946732521057129, + "learning_rate": 4.3980061644943575e-07, + "logits/chosen": 0.4884384274482727, + "logits/rejected": 0.41029107570648193, + "logps/chosen": -83.94632720947266, + "logps/ref_chosen": -47.660648345947266, + "logps/ref_rejected": -73.63249969482422, + "logps/rejected": -135.84335327148438, + "loss": 1.0348, + "margin_dpo/margin_mean": 25.925174713134766, + "margin_dpo/margin_std": 34.66651916503906, + "step": 202 + }, + { + "KL/chosen_KL_mean": -44.81160354614258, + "KL/mean": -56.35142517089844, + "KL/rejected_KL_mean": -67.89125061035156, + "KL/std": 32.933311462402344, + "epoch": 0.30687830687830686, + "fcm_dpo/beta": 0.02090853452682495, + "fcm_dpo/delta": -0.08666198700666428, + "fcm_dpo/margin": 23.07965660095215, + "fcm_dpo/q_t": 0.3943900465965271, + "grad_norm": 17.459609985351562, + "learning_rate": 4.3893739358856455e-07, + "logits/chosen": 0.5723869800567627, + "logits/rejected": 0.48567742109298706, + "logps/chosen": -107.13713836669922, + "logps/ref_chosen": -62.32553482055664, + "logps/ref_rejected": -99.37226104736328, + "logps/rejected": -167.26351928710938, + "loss": 1.0892, + "margin_dpo/margin_mean": 23.07965660095215, + "margin_dpo/margin_std": 36.10200881958008, + "step": 203 + }, + { + "KL/chosen_KL_mean": -40.54402160644531, + "KL/mean": -52.845306396484375, + "KL/rejected_KL_mean": -65.14659118652344, + "KL/std": 33.49970626831055, + "epoch": 0.30839002267573695, + "fcm_dpo/beta": 0.02015521563589573, + "fcm_dpo/delta": -0.15325036644935608, + "fcm_dpo/margin": 24.60256576538086, + "fcm_dpo/q_t": 0.3900430500507355, + "grad_norm": 16.921817779541016, + "learning_rate": 4.380688857426449e-07, + "logits/chosen": 0.5573503971099854, + "logits/rejected": 0.48204463720321655, + "logps/chosen": -91.17333984375, + "logps/ref_chosen": -50.62931823730469, + "logps/ref_rejected": -66.60475158691406, + "logps/rejected": -131.7513427734375, + "loss": 1.0682, + "margin_dpo/margin_mean": 24.60256576538086, + "margin_dpo/margin_std": 34.54716491699219, + "step": 204 + }, + { + "KL/chosen_KL_mean": -47.66603088378906, + "KL/mean": -59.343788146972656, + "KL/rejected_KL_mean": -71.02154541015625, + "KL/std": 35.73222732543945, + "epoch": 0.30990173847316704, + "fcm_dpo/beta": 0.0199609212577343, + "fcm_dpo/delta": -0.048433154821395874, + "fcm_dpo/margin": 23.355518341064453, + "fcm_dpo/q_t": 0.39880335330963135, + "grad_norm": 18.173583984375, + "learning_rate": 4.3719511720570814e-07, + "logits/chosen": 0.5487678050994873, + "logits/rejected": 0.4762420058250427, + "logps/chosen": -118.02220916748047, + "logps/ref_chosen": -70.3561782836914, + "logps/ref_rejected": -93.39848327636719, + "logps/rejected": -164.42002868652344, + "loss": 1.1161, + "margin_dpo/margin_mean": 23.355518341064453, + "margin_dpo/margin_std": 39.755157470703125, + "step": 205 + }, + { + "KL/chosen_KL_mean": -49.99560546875, + "KL/mean": -58.32989501953125, + "KL/rejected_KL_mean": -66.6641845703125, + "KL/std": 34.640377044677734, + "epoch": 0.31141345427059713, + "fcm_dpo/beta": 0.0199609212577343, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 16.668582916259766, + "fcm_dpo/q_t": 0.42973265051841736, + "grad_norm": 16.76526641845703, + "learning_rate": 4.363161124189387e-07, + "logits/chosen": 0.6296049356460571, + "logits/rejected": 0.6118504405021667, + "logps/chosen": -117.64108276367188, + "logps/ref_chosen": -67.64547729492188, + "logps/ref_rejected": -79.89584350585938, + "logps/rejected": -146.56002807617188, + "loss": 1.2418, + "margin_dpo/margin_mean": 16.6685848236084, + "margin_dpo/margin_std": 41.699256896972656, + "step": 206 + }, + { + "KL/chosen_KL_mean": -58.37126922607422, + "KL/mean": -69.7347412109375, + "KL/rejected_KL_mean": -81.09820556640625, + "KL/std": 35.99120330810547, + "epoch": 0.3129251700680272, + "fcm_dpo/beta": 0.0199609212577343, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 22.726932525634766, + "fcm_dpo/q_t": 0.4033956825733185, + "grad_norm": 14.681884765625, + "learning_rate": 4.3543189596998986e-07, + "logits/chosen": 0.5348644852638245, + "logits/rejected": 0.46388787031173706, + "logps/chosen": -126.03546142578125, + "logps/ref_chosen": -67.66419219970703, + "logps/ref_rejected": -85.10249328613281, + "logps/rejected": -166.20068359375, + "loss": 1.1111, + "margin_dpo/margin_mean": 22.726932525634766, + "margin_dpo/margin_std": 37.863197326660156, + "step": 207 + }, + { + "KL/chosen_KL_mean": -49.66230773925781, + "KL/mean": -55.65148162841797, + "KL/rejected_KL_mean": -61.64065170288086, + "KL/std": 34.22406005859375, + "epoch": 0.3144368858654573, + "fcm_dpo/beta": 0.0199609212577343, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 11.978349685668945, + "fcm_dpo/q_t": 0.4482603669166565, + "grad_norm": 21.4254207611084, + "learning_rate": 4.3454249259229664e-07, + "logits/chosen": 0.576629638671875, + "logits/rejected": 0.5478168725967407, + "logps/chosen": -107.3940200805664, + "logps/ref_chosen": -57.731712341308594, + "logps/ref_rejected": -74.19276428222656, + "logps/rejected": -135.8334197998047, + "loss": 1.3075, + "margin_dpo/margin_mean": 11.978348731994629, + "margin_dpo/margin_std": 39.34952926635742, + "step": 208 + }, + { + "KL/chosen_KL_mean": -49.522850036621094, + "KL/mean": -64.13494873046875, + "KL/rejected_KL_mean": -78.7470703125, + "KL/std": 38.795494079589844, + "epoch": 0.31594860166288735, + "fcm_dpo/beta": 0.019417954608798027, + "fcm_dpo/delta": -0.1777927577495575, + "fcm_dpo/margin": 29.224210739135742, + "fcm_dpo/q_t": 0.37642014026641846, + "grad_norm": 16.401376724243164, + "learning_rate": 4.336479271643833e-07, + "logits/chosen": 0.4981778860092163, + "logits/rejected": 0.44314074516296387, + "logps/chosen": -118.07292938232422, + "logps/ref_chosen": -68.55007934570312, + "logps/ref_rejected": -87.90541076660156, + "logps/rejected": -166.65248107910156, + "loss": 1.0706, + "margin_dpo/margin_mean": 29.224210739135742, + "margin_dpo/margin_std": 45.77735137939453, + "step": 209 + }, + { + "KL/chosen_KL_mean": -49.30180358886719, + "KL/mean": -63.557167053222656, + "KL/rejected_KL_mean": -77.81253051757812, + "KL/std": 40.000423431396484, + "epoch": 0.31746031746031744, + "fcm_dpo/beta": 0.01899714395403862, + "fcm_dpo/delta": -0.14027619361877441, + "fcm_dpo/margin": 28.510725021362305, + "fcm_dpo/q_t": 0.38312628865242004, + "grad_norm": 16.62250518798828, + "learning_rate": 4.327482247091679e-07, + "logits/chosen": 0.5970274209976196, + "logits/rejected": 0.4921113848686218, + "logps/chosen": -106.57007598876953, + "logps/ref_chosen": -57.268272399902344, + "logps/ref_rejected": -85.72807312011719, + "logps/rejected": -163.54061889648438, + "loss": 1.0505, + "margin_dpo/margin_mean": 28.510725021362305, + "margin_dpo/margin_std": 40.20121765136719, + "step": 210 + }, + { + "KL/chosen_KL_mean": -45.32893371582031, + "KL/mean": -59.812034606933594, + "KL/rejected_KL_mean": -74.2951431274414, + "KL/std": 36.853477478027344, + "epoch": 0.31897203325774753, + "fcm_dpo/beta": 0.018486086279153824, + "fcm_dpo/delta": -0.13231848180294037, + "fcm_dpo/margin": 28.96620750427246, + "fcm_dpo/q_t": 0.38412922620773315, + "grad_norm": 17.82309341430664, + "learning_rate": 4.3184341039326217e-07, + "logits/chosen": 0.6039122343063354, + "logits/rejected": 0.4997963309288025, + "logps/chosen": -98.96964263916016, + "logps/ref_chosen": -53.640708923339844, + "logps/ref_rejected": -93.0387954711914, + "logps/rejected": -167.3339385986328, + "loss": 1.0304, + "margin_dpo/margin_mean": 28.966205596923828, + "margin_dpo/margin_std": 37.75336456298828, + "step": 211 + }, + { + "KL/chosen_KL_mean": -50.42332458496094, + "KL/mean": -64.77632141113281, + "KL/rejected_KL_mean": -79.12932586669922, + "KL/std": 39.548606872558594, + "epoch": 0.3204837490551776, + "fcm_dpo/beta": 0.01791585050523281, + "fcm_dpo/delta": -0.12050309777259827, + "fcm_dpo/margin": 28.705997467041016, + "fcm_dpo/q_t": 0.38503497838974, + "grad_norm": 12.132513999938965, + "learning_rate": 4.309335095262675e-07, + "logits/chosen": 0.6209944486618042, + "logits/rejected": 0.5416966676712036, + "logps/chosen": -107.79006958007812, + "logps/ref_chosen": -57.36674499511719, + "logps/ref_rejected": -79.89643096923828, + "logps/rejected": -159.0257568359375, + "loss": 1.0681, + "margin_dpo/margin_mean": 28.705997467041016, + "margin_dpo/margin_std": 42.09522247314453, + "step": 212 + }, + { + "KL/chosen_KL_mean": -40.895442962646484, + "KL/mean": -57.22308349609375, + "KL/rejected_KL_mean": -73.55073547363281, + "KL/std": 40.150550842285156, + "epoch": 0.3219954648526077, + "fcm_dpo/beta": 0.017257830128073692, + "fcm_dpo/delta": -0.17485719919204712, + "fcm_dpo/margin": 32.65528106689453, + "fcm_dpo/q_t": 0.3760501742362976, + "grad_norm": 13.000662803649902, + "learning_rate": 4.3001854756006724e-07, + "logits/chosen": 0.5805448293685913, + "logits/rejected": 0.5558933019638062, + "logps/chosen": -106.11656188964844, + "logps/ref_chosen": -65.22111511230469, + "logps/ref_rejected": -80.1810302734375, + "logps/rejected": -153.73175048828125, + "loss": 1.0314, + "margin_dpo/margin_mean": 32.65528106689453, + "margin_dpo/margin_std": 44.30155944824219, + "step": 213 + }, + { + "KL/chosen_KL_mean": -47.93132781982422, + "KL/mean": -63.84461212158203, + "KL/rejected_KL_mean": -79.75790405273438, + "KL/std": 40.14862823486328, + "epoch": 0.3235071806500378, + "fcm_dpo/beta": 0.016959507018327713, + "fcm_dpo/delta": -0.13894835114479065, + "fcm_dpo/margin": 31.826576232910156, + "fcm_dpo/q_t": 0.3835788369178772, + "grad_norm": 16.028963088989258, + "learning_rate": 4.290985500881143e-07, + "logits/chosen": 0.4599738121032715, + "logits/rejected": 0.44451236724853516, + "logps/chosen": -109.2236557006836, + "logps/ref_chosen": -61.292327880859375, + "logps/ref_rejected": -67.69841003417969, + "logps/rejected": -147.45631408691406, + "loss": 1.0611, + "margin_dpo/margin_mean": 31.82657814025879, + "margin_dpo/margin_std": 46.38356018066406, + "step": 214 + }, + { + "KL/chosen_KL_mean": -53.18816375732422, + "KL/mean": -69.61984252929688, + "KL/rejected_KL_mean": -86.05149841308594, + "KL/std": 41.73292541503906, + "epoch": 0.3250188964474679, + "fcm_dpo/beta": 0.016454260796308517, + "fcm_dpo/delta": -0.16386979818344116, + "fcm_dpo/margin": 32.86333465576172, + "fcm_dpo/q_t": 0.38295409083366394, + "grad_norm": 14.382888793945312, + "learning_rate": 4.281735428447157e-07, + "logits/chosen": 0.4620933532714844, + "logits/rejected": 0.34720370173454285, + "logps/chosen": -117.05729675292969, + "logps/ref_chosen": -63.869136810302734, + "logps/ref_rejected": -98.7657241821289, + "logps/rejected": -184.81723022460938, + "loss": 1.0662, + "margin_dpo/margin_mean": 32.863338470458984, + "margin_dpo/margin_std": 48.01820373535156, + "step": 215 + }, + { + "KL/chosen_KL_mean": -44.4687385559082, + "KL/mean": -62.103668212890625, + "KL/rejected_KL_mean": -79.73858642578125, + "KL/std": 40.250755310058594, + "epoch": 0.32653061224489793, + "fcm_dpo/beta": 0.015767354518175125, + "fcm_dpo/delta": -0.16560883820056915, + "fcm_dpo/margin": 35.269840240478516, + "fcm_dpo/q_t": 0.3732219636440277, + "grad_norm": 25.336891174316406, + "learning_rate": 4.2724355170431247e-07, + "logits/chosen": 0.6101344227790833, + "logits/rejected": 0.5182064771652222, + "logps/chosen": -112.29369354248047, + "logps/ref_chosen": -67.824951171875, + "logps/ref_rejected": -96.40231323242188, + "logps/rejected": -176.14089965820312, + "loss": 0.9984, + "margin_dpo/margin_mean": 35.26984405517578, + "margin_dpo/margin_std": 40.447853088378906, + "step": 216 + }, + { + "KL/chosen_KL_mean": -48.50625228881836, + "KL/mean": -66.75289916992188, + "KL/rejected_KL_mean": -84.99954986572266, + "KL/std": 42.09430694580078, + "epoch": 0.328042328042328, + "fcm_dpo/beta": 0.015221597626805305, + "fcm_dpo/delta": -0.16550882160663605, + "fcm_dpo/margin": 36.49330139160156, + "fcm_dpo/q_t": 0.37577375769615173, + "grad_norm": 12.231183052062988, + "learning_rate": 4.26308602680756e-07, + "logits/chosen": 0.5095263719558716, + "logits/rejected": 0.40860888361930847, + "logps/chosen": -109.01124572753906, + "logps/ref_chosen": -60.5049934387207, + "logps/ref_rejected": -84.26618194580078, + "logps/rejected": -169.26573181152344, + "loss": 1.0063, + "margin_dpo/margin_mean": 36.49330139160156, + "margin_dpo/margin_std": 43.78234100341797, + "step": 217 + }, + { + "KL/chosen_KL_mean": -53.025917053222656, + "KL/mean": -63.77705001831055, + "KL/rejected_KL_mean": -74.52819061279297, + "KL/std": 41.17570495605469, + "epoch": 0.3295540438397581, + "fcm_dpo/beta": 0.015110185369849205, + "fcm_dpo/delta": -0.024738004431128502, + "fcm_dpo/margin": 21.50227165222168, + "fcm_dpo/q_t": 0.42662978172302246, + "grad_norm": 13.907445907592773, + "learning_rate": 4.253687219265803e-07, + "logits/chosen": 0.4400632381439209, + "logits/rejected": 0.4424203336238861, + "logps/chosen": -123.62023162841797, + "logps/ref_chosen": -70.59431457519531, + "logps/ref_rejected": -73.89038848876953, + "logps/rejected": -148.4185791015625, + "loss": 1.2201, + "margin_dpo/margin_mean": 21.502273559570312, + "margin_dpo/margin_std": 48.850921630859375, + "step": 218 + }, + { + "KL/chosen_KL_mean": -49.29433822631836, + "KL/mean": -61.04737854003906, + "KL/rejected_KL_mean": -72.80043029785156, + "KL/std": 39.78154754638672, + "epoch": 0.3310657596371882, + "fcm_dpo/beta": 0.015072805806994438, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 23.506093978881836, + "fcm_dpo/q_t": 0.4195045232772827, + "grad_norm": 12.386738777160645, + "learning_rate": 4.2442393573227043e-07, + "logits/chosen": 0.509136974811554, + "logits/rejected": 0.46663516759872437, + "logps/chosen": -109.7852783203125, + "logps/ref_chosen": -60.490943908691406, + "logps/ref_rejected": -75.85001373291016, + "logps/rejected": -148.6504364013672, + "loss": 1.1503, + "margin_dpo/margin_mean": 23.506093978881836, + "margin_dpo/margin_std": 40.30815887451172, + "step": 219 + }, + { + "KL/chosen_KL_mean": -41.13189697265625, + "KL/mean": -55.28445053100586, + "KL/rejected_KL_mean": -69.43699645996094, + "KL/std": 41.99296951293945, + "epoch": 0.3325774754346183, + "fcm_dpo/beta": 0.014830692671239376, + "fcm_dpo/delta": -0.08096643537282944, + "fcm_dpo/margin": 28.305110931396484, + "fcm_dpo/q_t": 0.4073430895805359, + "grad_norm": 13.636482238769531, + "learning_rate": 4.234742705255272e-07, + "logits/chosen": 0.6164438128471375, + "logits/rejected": 0.5471941232681274, + "logps/chosen": -86.14529418945312, + "logps/ref_chosen": -45.013397216796875, + "logps/ref_rejected": -70.49369812011719, + "logps/rejected": -139.93069458007812, + "loss": 1.134, + "margin_dpo/margin_mean": 28.305110931396484, + "margin_dpo/margin_std": 48.46826934814453, + "step": 220 + }, + { + "KL/chosen_KL_mean": -40.829010009765625, + "KL/mean": -56.581878662109375, + "KL/rejected_KL_mean": -72.33475494384766, + "KL/std": 41.920936584472656, + "epoch": 0.3340891912320484, + "fcm_dpo/beta": 0.014756301417946815, + "fcm_dpo/delta": -0.050413843244314194, + "fcm_dpo/margin": 31.505746841430664, + "fcm_dpo/q_t": 0.39809131622314453, + "grad_norm": 15.061563491821289, + "learning_rate": 4.22519752870528e-07, + "logits/chosen": 0.5553480386734009, + "logits/rejected": 0.47446322441101074, + "logps/chosen": -99.92485046386719, + "logps/ref_chosen": -59.09584045410156, + "logps/ref_rejected": -88.64388275146484, + "logps/rejected": -160.9786376953125, + "loss": 1.0837, + "margin_dpo/margin_mean": 31.50574493408203, + "margin_dpo/margin_std": 47.23353576660156, + "step": 221 + }, + { + "KL/chosen_KL_mean": -40.73999786376953, + "KL/mean": -62.21807098388672, + "KL/rejected_KL_mean": -83.6961441040039, + "KL/std": 47.69040298461914, + "epoch": 0.3356009070294785, + "fcm_dpo/beta": 0.014200271107256413, + "fcm_dpo/delta": -0.22367945313453674, + "fcm_dpo/margin": 42.956146240234375, + "fcm_dpo/q_t": 0.3654256761074066, + "grad_norm": 12.030962944030762, + "learning_rate": 4.2156040946718343e-07, + "logits/chosen": 0.6517788171768188, + "logits/rejected": 0.5524156093597412, + "logps/chosen": -96.73768615722656, + "logps/ref_chosen": -55.9976921081543, + "logps/ref_rejected": -111.94727325439453, + "logps/rejected": -195.64341735839844, + "loss": 0.9826, + "margin_dpo/margin_mean": 42.956146240234375, + "margin_dpo/margin_std": 50.614097595214844, + "step": 222 + }, + { + "KL/chosen_KL_mean": -48.65684509277344, + "KL/mean": -64.96463012695312, + "KL/rejected_KL_mean": -81.27240753173828, + "KL/std": 45.18286895751953, + "epoch": 0.3371126228269085, + "fcm_dpo/beta": 0.013911095447838306, + "fcm_dpo/delta": -0.04596729949116707, + "fcm_dpo/margin": 32.615562438964844, + "fcm_dpo/q_t": 0.39882034063339233, + "grad_norm": 15.105962753295898, + "learning_rate": 4.2059626715039065e-07, + "logits/chosen": 0.6108689308166504, + "logits/rejected": 0.5520835518836975, + "logps/chosen": -108.54826354980469, + "logps/ref_chosen": -59.891422271728516, + "logps/ref_rejected": -86.28954315185547, + "logps/rejected": -167.56195068359375, + "loss": 1.0778, + "margin_dpo/margin_mean": 32.615562438964844, + "margin_dpo/margin_std": 46.08930206298828, + "step": 223 + }, + { + "KL/chosen_KL_mean": -56.427490234375, + "KL/mean": -65.35730743408203, + "KL/rejected_KL_mean": -74.287109375, + "KL/std": 45.022212982177734, + "epoch": 0.3386243386243386, + "fcm_dpo/beta": 0.013911095447838306, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 17.859630584716797, + "fcm_dpo/q_t": 0.44438493251800537, + "grad_norm": 16.007753372192383, + "learning_rate": 4.1962735288928304e-07, + "logits/chosen": 0.6295701265335083, + "logits/rejected": 0.6061124205589294, + "logps/chosen": -120.47212219238281, + "logps/ref_chosen": -64.04463195800781, + "logps/ref_rejected": -75.05450439453125, + "logps/rejected": -149.34161376953125, + "loss": 1.2579, + "margin_dpo/margin_mean": 17.859630584716797, + "margin_dpo/margin_std": 48.06422424316406, + "step": 224 + }, + { + "KL/chosen_KL_mean": -52.12030029296875, + "KL/mean": -70.82891845703125, + "KL/rejected_KL_mean": -89.53753662109375, + "KL/std": 50.072288513183594, + "epoch": 0.3401360544217687, + "fcm_dpo/beta": 0.013590985909104347, + "fcm_dpo/delta": -0.11639999598264694, + "fcm_dpo/margin": 37.4172477722168, + "fcm_dpo/q_t": 0.38528722524642944, + "grad_norm": 12.09056282043457, + "learning_rate": 4.186536937864752e-07, + "logits/chosen": 0.626471996307373, + "logits/rejected": 0.5087643265724182, + "logps/chosen": -118.21612548828125, + "logps/ref_chosen": -66.0958251953125, + "logps/ref_rejected": -97.68675231933594, + "logps/rejected": -187.2242889404297, + "loss": 1.0305, + "margin_dpo/margin_mean": 37.4172477722168, + "margin_dpo/margin_std": 45.303226470947266, + "step": 225 + }, + { + "KL/chosen_KL_mean": -49.675743103027344, + "KL/mean": -64.10902404785156, + "KL/rejected_KL_mean": -78.54232025146484, + "KL/std": 44.898719787597656, + "epoch": 0.3416477702191988, + "fcm_dpo/beta": 0.013537578284740448, + "fcm_dpo/delta": -0.039451513439416885, + "fcm_dpo/margin": 28.8665771484375, + "fcm_dpo/q_t": 0.4117968678474426, + "grad_norm": 13.084617614746094, + "learning_rate": 4.176753170773052e-07, + "logits/chosen": 0.6425325274467468, + "logits/rejected": 0.5956849455833435, + "logps/chosen": -101.09261322021484, + "logps/ref_chosen": -51.4168701171875, + "logps/ref_rejected": -66.30068969726562, + "logps/rejected": -144.84300231933594, + "loss": 1.1617, + "margin_dpo/margin_mean": 28.8665771484375, + "margin_dpo/margin_std": 54.40742874145508, + "step": 226 + }, + { + "KL/chosen_KL_mean": -53.59423828125, + "KL/mean": -68.63531494140625, + "KL/rejected_KL_mean": -83.6763916015625, + "KL/std": 49.33631896972656, + "epoch": 0.3431594860166289, + "fcm_dpo/beta": 0.013484170660376549, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 30.082157135009766, + "fcm_dpo/q_t": 0.41013020277023315, + "grad_norm": 14.161370277404785, + "learning_rate": 4.166922501290729e-07, + "logits/chosen": 0.6694810390472412, + "logits/rejected": 0.631727933883667, + "logps/chosen": -111.58401489257812, + "logps/ref_chosen": -57.989776611328125, + "logps/ref_rejected": -75.05464172363281, + "logps/rejected": -158.73104858398438, + "loss": 1.1473, + "margin_dpo/margin_mean": 30.082157135009766, + "margin_dpo/margin_std": 54.82373809814453, + "step": 227 + }, + { + "KL/chosen_KL_mean": -56.00676727294922, + "KL/mean": -73.64747619628906, + "KL/rejected_KL_mean": -91.2881851196289, + "KL/std": 46.921348571777344, + "epoch": 0.34467120181405897, + "fcm_dpo/beta": 0.013376947492361069, + "fcm_dpo/delta": -0.08015699684619904, + "fcm_dpo/margin": 35.28141403198242, + "fcm_dpo/q_t": 0.39347511529922485, + "grad_norm": 14.644017219543457, + "learning_rate": 4.1570452044027405e-07, + "logits/chosen": 0.6866810321807861, + "logits/rejected": 0.6021173000335693, + "logps/chosen": -111.56613159179688, + "logps/ref_chosen": -55.55936813354492, + "logps/ref_rejected": -77.02364349365234, + "logps/rejected": -168.31182861328125, + "loss": 1.0706, + "margin_dpo/margin_mean": 35.28141784667969, + "margin_dpo/margin_std": 49.03618621826172, + "step": 228 + }, + { + "KL/chosen_KL_mean": -48.973968505859375, + "KL/mean": -63.88433074951172, + "KL/rejected_KL_mean": -78.79469299316406, + "KL/std": 46.169944763183594, + "epoch": 0.34618291761148906, + "fcm_dpo/beta": 0.013269724324345589, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 29.820728302001953, + "fcm_dpo/q_t": 0.4127005338668823, + "grad_norm": 18.285600662231445, + "learning_rate": 4.147121556398312e-07, + "logits/chosen": 0.7295674085617065, + "logits/rejected": 0.6509650945663452, + "logps/chosen": -99.76863098144531, + "logps/ref_chosen": -50.79466247558594, + "logps/ref_rejected": -78.4474105834961, + "logps/rejected": -157.24209594726562, + "loss": 1.1571, + "margin_dpo/margin_mean": 29.820728302001953, + "margin_dpo/margin_std": 56.27452087402344, + "step": 229 + }, + { + "KL/chosen_KL_mean": -52.722755432128906, + "KL/mean": -69.46250915527344, + "KL/rejected_KL_mean": -86.2022705078125, + "KL/std": 50.38365173339844, + "epoch": 0.3476946334089191, + "fcm_dpo/beta": 0.013133594766259193, + "fcm_dpo/delta": -0.10365402698516846, + "fcm_dpo/margin": 33.47950744628906, + "fcm_dpo/q_t": 0.4013306498527527, + "grad_norm": 13.488200187683105, + "learning_rate": 4.137151834863213e-07, + "logits/chosen": 0.6259239912033081, + "logits/rejected": 0.6276857852935791, + "logps/chosen": -109.45198059082031, + "logps/ref_chosen": -56.729225158691406, + "logps/ref_rejected": -62.99180603027344, + "logps/rejected": -149.19407653808594, + "loss": 1.1021, + "margin_dpo/margin_mean": 33.47951126098633, + "margin_dpo/margin_std": 50.97336959838867, + "step": 230 + }, + { + "KL/chosen_KL_mean": -58.87866973876953, + "KL/mean": -84.7044677734375, + "KL/rejected_KL_mean": -110.5302734375, + "KL/std": 49.37828063964844, + "epoch": 0.3492063492063492, + "fcm_dpo/beta": 0.012457750737667084, + "fcm_dpo/delta": -0.2614459991455078, + "fcm_dpo/margin": 51.651607513427734, + "fcm_dpo/q_t": 0.35266393423080444, + "grad_norm": 13.210762977600098, + "learning_rate": 4.1271363186719835e-07, + "logits/chosen": 0.5706331729888916, + "logits/rejected": 0.5676780939102173, + "logps/chosen": -131.47576904296875, + "logps/ref_chosen": -72.59709930419922, + "logps/ref_rejected": -86.2322998046875, + "logps/rejected": -196.7625732421875, + "loss": 0.9433, + "margin_dpo/margin_mean": 51.651607513427734, + "margin_dpo/margin_std": 52.80973815917969, + "step": 231 + }, + { + "KL/chosen_KL_mean": -60.621498107910156, + "KL/mean": -77.64936828613281, + "KL/rejected_KL_mean": -94.67724609375, + "KL/std": 54.37424850463867, + "epoch": 0.3507180650037793, + "fcm_dpo/beta": 0.012335299514234066, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 34.05574035644531, + "fcm_dpo/q_t": 0.40971839427948, + "grad_norm": 13.251051902770996, + "learning_rate": 4.1170752879801436e-07, + "logits/chosen": 0.586661696434021, + "logits/rejected": 0.5547990202903748, + "logps/chosen": -128.74002075195312, + "logps/ref_chosen": -68.1185302734375, + "logps/ref_rejected": -83.79415893554688, + "logps/rejected": -178.47140502929688, + "loss": 1.1428, + "margin_dpo/margin_mean": 34.05574417114258, + "margin_dpo/margin_std": 61.997886657714844, + "step": 232 + }, + { + "KL/chosen_KL_mean": -69.0023193359375, + "KL/mean": -81.63502502441406, + "KL/rejected_KL_mean": -94.26773071289062, + "KL/std": 49.81992721557617, + "epoch": 0.35222978080120937, + "fcm_dpo/beta": 0.012211372144520283, + "fcm_dpo/delta": -0.05048698931932449, + "fcm_dpo/margin": 25.26541519165039, + "fcm_dpo/q_t": 0.43137213587760925, + "grad_norm": 14.052996635437012, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": 0.6705083847045898, + "logits/rejected": 0.6091481447219849, + "logps/chosen": -124.07246398925781, + "logps/ref_chosen": -55.070152282714844, + "logps/ref_rejected": -66.61845397949219, + "logps/rejected": -160.8861846923828, + "loss": 1.2025, + "margin_dpo/margin_mean": 25.26541519165039, + "margin_dpo/margin_std": 51.56385803222656, + "step": 233 + }, + { + "KL/chosen_KL_mean": -61.680076599121094, + "KL/mean": -74.02011108398438, + "KL/rejected_KL_mean": -86.36014556884766, + "KL/std": 48.443763732910156, + "epoch": 0.35374149659863946, + "fcm_dpo/beta": 0.012122506275773048, + "fcm_dpo/delta": -0.07330780476331711, + "fcm_dpo/margin": 24.68006134033203, + "fcm_dpo/q_t": 0.4330037236213684, + "grad_norm": 19.079448699951172, + "learning_rate": 4.09681781007452e-07, + "logits/chosen": 0.581956148147583, + "logits/rejected": 0.5792969465255737, + "logps/chosen": -117.60597229003906, + "logps/ref_chosen": -55.92589569091797, + "logps/ref_rejected": -51.11608123779297, + "logps/rejected": -137.47622680664062, + "loss": 1.2224, + "margin_dpo/margin_mean": 24.680063247680664, + "margin_dpo/margin_std": 52.967437744140625, + "step": 234 + }, + { + "KL/chosen_KL_mean": -56.598243713378906, + "KL/mean": -78.36715698242188, + "KL/rejected_KL_mean": -100.13606262207031, + "KL/std": 53.2874641418457, + "epoch": 0.35525321239606955, + "fcm_dpo/beta": 0.011834532022476196, + "fcm_dpo/delta": -0.12138506025075912, + "fcm_dpo/margin": 43.537818908691406, + "fcm_dpo/q_t": 0.38227230310440063, + "grad_norm": 15.617653846740723, + "learning_rate": 4.08662192950594e-07, + "logits/chosen": 0.6357525587081909, + "logits/rejected": 0.6238740682601929, + "logps/chosen": -121.13796997070312, + "logps/ref_chosen": -64.53972625732422, + "logps/ref_rejected": -77.69151306152344, + "logps/rejected": -177.82757568359375, + "loss": 1.0189, + "margin_dpo/margin_mean": 43.537818908691406, + "margin_dpo/margin_std": 50.59941864013672, + "step": 235 + }, + { + "KL/chosen_KL_mean": -81.9928970336914, + "KL/mean": -98.34184265136719, + "KL/rejected_KL_mean": -114.69078063964844, + "KL/std": 51.35175323486328, + "epoch": 0.35676492819349964, + "fcm_dpo/beta": 0.01174501795321703, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 32.69788360595703, + "fcm_dpo/q_t": 0.41291993856430054, + "grad_norm": 12.297943115234375, + "learning_rate": 4.076381667711306e-07, + "logits/chosen": 0.6182401180267334, + "logits/rejected": 0.60513836145401, + "logps/chosen": -153.14764404296875, + "logps/ref_chosen": -71.15473937988281, + "logps/ref_rejected": -84.88541412353516, + "logps/rejected": -199.57620239257812, + "loss": 1.1524, + "margin_dpo/margin_mean": 32.69788360595703, + "margin_dpo/margin_std": 58.897560119628906, + "step": 236 + }, + { + "KL/chosen_KL_mean": -69.26779174804688, + "KL/mean": -89.37890625, + "KL/rejected_KL_mean": -109.49002075195312, + "KL/std": 51.652854919433594, + "epoch": 0.35827664399092973, + "fcm_dpo/beta": 0.011675616726279259, + "fcm_dpo/delta": -0.05944126471877098, + "fcm_dpo/margin": 40.22222137451172, + "fcm_dpo/q_t": 0.39383262395858765, + "grad_norm": 16.05605697631836, + "learning_rate": 4.066097311132753e-07, + "logits/chosen": 0.6465665102005005, + "logits/rejected": 0.6383150815963745, + "logps/chosen": -145.40980529785156, + "logps/ref_chosen": -76.14201354980469, + "logps/ref_rejected": -80.88479614257812, + "logps/rejected": -190.37481689453125, + "loss": 1.0856, + "margin_dpo/margin_mean": 40.22222137451172, + "margin_dpo/margin_std": 59.51573181152344, + "step": 237 + }, + { + "KL/chosen_KL_mean": -62.64219665527344, + "KL/mean": -81.81443786621094, + "KL/rejected_KL_mean": -100.98668670654297, + "KL/std": 53.077049255371094, + "epoch": 0.35978835978835977, + "fcm_dpo/beta": 0.011450215242803097, + "fcm_dpo/delta": -0.0676615908741951, + "fcm_dpo/margin": 38.34449005126953, + "fcm_dpo/q_t": 0.4004179835319519, + "grad_norm": 16.575794219970703, + "learning_rate": 4.0557691474458414e-07, + "logits/chosen": 0.6054675579071045, + "logits/rejected": 0.5951860547065735, + "logps/chosen": -131.52703857421875, + "logps/ref_chosen": -68.88484954833984, + "logps/ref_rejected": -75.8946304321289, + "logps/rejected": -176.88131713867188, + "loss": 1.0933, + "margin_dpo/margin_mean": 38.34449005126953, + "margin_dpo/margin_std": 55.98069763183594, + "step": 238 + }, + { + "KL/chosen_KL_mean": -75.8132553100586, + "KL/mean": -95.68144989013672, + "KL/rejected_KL_mean": -115.54964447021484, + "KL/std": 57.8829231262207, + "epoch": 0.36130007558578986, + "fcm_dpo/beta": 0.011365259066224098, + "fcm_dpo/delta": -0.07475205510854721, + "fcm_dpo/margin": 39.73638153076172, + "fcm_dpo/q_t": 0.39894479513168335, + "grad_norm": 18.736330032348633, + "learning_rate": 4.045397465551513e-07, + "logits/chosen": 0.761664867401123, + "logits/rejected": 0.6149318814277649, + "logps/chosen": -132.5850830078125, + "logps/ref_chosen": -56.771827697753906, + "logps/ref_rejected": -116.23050689697266, + "logps/rejected": -231.7801513671875, + "loss": 1.1029, + "margin_dpo/margin_mean": 39.73638153076172, + "margin_dpo/margin_std": 61.89457702636719, + "step": 239 + }, + { + "KL/chosen_KL_mean": -73.51741027832031, + "KL/mean": -98.4951171875, + "KL/rejected_KL_mean": -123.47282409667969, + "KL/std": 58.34437561035156, + "epoch": 0.36281179138321995, + "fcm_dpo/beta": 0.011012222617864609, + "fcm_dpo/delta": -0.15893766283988953, + "fcm_dpo/margin": 49.955413818359375, + "fcm_dpo/q_t": 0.3772236406803131, + "grad_norm": 12.825221061706543, + "learning_rate": 4.0349825555680045e-07, + "logits/chosen": 0.6706682443618774, + "logits/rejected": 0.5711009502410889, + "logps/chosen": -126.87152099609375, + "logps/ref_chosen": -53.35411071777344, + "logps/ref_rejected": -80.12019348144531, + "logps/rejected": -203.593017578125, + "loss": 1.017, + "margin_dpo/margin_mean": 49.955413818359375, + "margin_dpo/margin_std": 62.52080154418945, + "step": 240 + }, + { + "KL/chosen_KL_mean": -69.59007263183594, + "KL/mean": -86.24327087402344, + "KL/rejected_KL_mean": -102.89646911621094, + "KL/std": 55.32313537597656, + "epoch": 0.36432350718065004, + "fcm_dpo/beta": 0.010927368886768818, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 33.3064079284668, + "fcm_dpo/q_t": 0.41905292868614197, + "grad_norm": 17.20731544494629, + "learning_rate": 4.0245247088227377e-07, + "logits/chosen": 0.6281282901763916, + "logits/rejected": 0.591729998588562, + "logps/chosen": -141.4854736328125, + "logps/ref_chosen": -71.89541625976562, + "logps/ref_rejected": -83.03492736816406, + "logps/rejected": -185.931396484375, + "loss": 1.1626, + "margin_dpo/margin_mean": 33.3064079284668, + "margin_dpo/margin_std": 62.025535583496094, + "step": 241 + }, + { + "KL/chosen_KL_mean": -71.19725036621094, + "KL/mean": -91.92439270019531, + "KL/rejected_KL_mean": -112.65153503417969, + "KL/std": 57.12744140625, + "epoch": 0.36583522297808013, + "fcm_dpo/beta": 0.0107469093054533, + "fcm_dpo/delta": -0.08326168358325958, + "fcm_dpo/margin": 41.45427703857422, + "fcm_dpo/q_t": 0.4003368616104126, + "grad_norm": 12.280982971191406, + "learning_rate": 4.0140242178441665e-07, + "logits/chosen": 0.6384247541427612, + "logits/rejected": 0.6186807155609131, + "logps/chosen": -129.1246795654297, + "logps/ref_chosen": -57.927433013916016, + "logps/ref_rejected": -67.838623046875, + "logps/rejected": -180.49014282226562, + "loss": 1.0906, + "margin_dpo/margin_mean": 41.45427703857422, + "margin_dpo/margin_std": 61.07661056518555, + "step": 242 + }, + { + "KL/chosen_KL_mean": -70.12564086914062, + "KL/mean": -89.19436645507812, + "KL/rejected_KL_mean": -108.26309204101562, + "KL/std": 56.41188430786133, + "epoch": 0.3673469387755102, + "fcm_dpo/beta": 0.0107469093054533, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 38.137451171875, + "fcm_dpo/q_t": 0.40642350912094116, + "grad_norm": 16.068456649780273, + "learning_rate": 4.003481376353596e-07, + "logits/chosen": 0.6034530401229858, + "logits/rejected": 0.6172385215759277, + "logps/chosen": -144.40231323242188, + "logps/ref_chosen": -74.27667236328125, + "logps/ref_rejected": -73.24340057373047, + "logps/rejected": -181.50650024414062, + "loss": 1.1101, + "margin_dpo/margin_mean": 38.137451171875, + "margin_dpo/margin_std": 58.296730041503906, + "step": 243 + }, + { + "KL/chosen_KL_mean": -64.46190643310547, + "KL/mean": -89.28399658203125, + "KL/rejected_KL_mean": -114.10609436035156, + "KL/std": 56.00457000732422, + "epoch": 0.3688586545729403, + "fcm_dpo/beta": 0.010579448193311691, + "fcm_dpo/delta": -0.13194507360458374, + "fcm_dpo/margin": 49.64418029785156, + "fcm_dpo/q_t": 0.3805602192878723, + "grad_norm": 12.418740272521973, + "learning_rate": 3.9928964792569654e-07, + "logits/chosen": 0.6857927441596985, + "logits/rejected": 0.5967106819152832, + "logps/chosen": -117.82581329345703, + "logps/ref_chosen": -53.36390686035156, + "logps/ref_rejected": -71.10276794433594, + "logps/rejected": -185.2088623046875, + "loss": 1.0158, + "margin_dpo/margin_mean": 49.64418029785156, + "margin_dpo/margin_std": 57.57759094238281, + "step": 244 + }, + { + "KL/chosen_KL_mean": -71.72976684570312, + "KL/mean": -100.52909851074219, + "KL/rejected_KL_mean": -129.32843017578125, + "KL/std": 58.295223236083984, + "epoch": 0.37037037037037035, + "fcm_dpo/beta": 0.010150259360671043, + "fcm_dpo/delta": -0.19647106528282166, + "fcm_dpo/margin": 57.598663330078125, + "fcm_dpo/q_t": 0.3648463785648346, + "grad_norm": 13.76476001739502, + "learning_rate": 3.982269822636601e-07, + "logits/chosen": 0.7182765007019043, + "logits/rejected": 0.6913450956344604, + "logps/chosen": -142.92486572265625, + "logps/ref_chosen": -71.19510650634766, + "logps/ref_rejected": -80.76235961914062, + "logps/rejected": -210.09078979492188, + "loss": 0.9685, + "margin_dpo/margin_mean": 57.59866714477539, + "margin_dpo/margin_std": 59.257904052734375, + "step": 245 + }, + { + "KL/chosen_KL_mean": -85.19328308105469, + "KL/mean": -106.13648986816406, + "KL/rejected_KL_mean": -127.07968139648438, + "KL/std": 57.94656753540039, + "epoch": 0.37188208616780044, + "fcm_dpo/beta": 0.010008112527430058, + "fcm_dpo/delta": -0.05554331839084625, + "fcm_dpo/margin": 41.88639831542969, + "fcm_dpo/q_t": 0.4062495231628418, + "grad_norm": 12.851189613342285, + "learning_rate": 3.971601703742932e-07, + "logits/chosen": 0.7253904342651367, + "logits/rejected": 0.6600919961929321, + "logps/chosen": -156.8143310546875, + "logps/ref_chosen": -71.62104797363281, + "logps/ref_rejected": -94.03392028808594, + "logps/rejected": -221.11361694335938, + "loss": 1.1361, + "margin_dpo/margin_mean": 41.88640213012695, + "margin_dpo/margin_std": 73.20356750488281, + "step": 246 + }, + { + "KL/chosen_KL_mean": -87.87586212158203, + "KL/mean": -101.99429321289062, + "KL/rejected_KL_mean": -116.11273193359375, + "KL/std": 58.836448669433594, + "epoch": 0.37339380196523053, + "fcm_dpo/beta": 0.009952524676918983, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 28.236865997314453, + "fcm_dpo/q_t": 0.43647515773773193, + "grad_norm": 15.817756652832031, + "learning_rate": 3.960892420986177e-07, + "logits/chosen": 0.657268762588501, + "logits/rejected": 0.6478973627090454, + "logps/chosen": -167.89840698242188, + "logps/ref_chosen": -80.02254486083984, + "logps/ref_rejected": -89.22705841064453, + "logps/rejected": -205.33978271484375, + "loss": 1.2344, + "margin_dpo/margin_mean": 28.236865997314453, + "margin_dpo/margin_std": 67.63511657714844, + "step": 247 + }, + { + "KL/chosen_KL_mean": -73.84143829345703, + "KL/mean": -96.81367492675781, + "KL/rejected_KL_mean": -119.78591918945312, + "KL/std": 61.485862731933594, + "epoch": 0.3749055177626606, + "fcm_dpo/beta": 0.00990164466202259, + "fcm_dpo/delta": -0.0513855516910553, + "fcm_dpo/margin": 45.94447326660156, + "fcm_dpo/q_t": 0.3986445665359497, + "grad_norm": 16.848495483398438, + "learning_rate": 3.9501422739279953e-07, + "logits/chosen": 0.656964898109436, + "logits/rejected": 0.714156985282898, + "logps/chosen": -139.21939086914062, + "logps/ref_chosen": -65.37796020507812, + "logps/ref_rejected": -61.365787506103516, + "logps/rejected": -181.15170288085938, + "loss": 1.0949, + "margin_dpo/margin_mean": 45.94447326660156, + "margin_dpo/margin_std": 70.1961898803711, + "step": 248 + }, + { + "KL/chosen_KL_mean": -93.00102233886719, + "KL/mean": -101.233154296875, + "KL/rejected_KL_mean": -109.46527099609375, + "KL/std": 57.36132049560547, + "epoch": 0.3764172335600907, + "fcm_dpo/beta": 0.009850765578448772, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 16.464248657226562, + "fcm_dpo/q_t": 0.46179455518722534, + "grad_norm": 16.948257446289062, + "learning_rate": 3.9393515632731094e-07, + "logits/chosen": 0.658106803894043, + "logits/rejected": 0.7000288963317871, + "logps/chosen": -167.60247802734375, + "logps/ref_chosen": -74.60145568847656, + "logps/ref_rejected": -63.79338455200195, + "logps/rejected": -173.2586669921875, + "loss": 1.3486, + "margin_dpo/margin_mean": 16.464248657226562, + "margin_dpo/margin_std": 69.59953308105469, + "step": 249 + }, + { + "KL/chosen_KL_mean": -81.48941040039062, + "KL/mean": -105.01419830322266, + "KL/rejected_KL_mean": -128.53900146484375, + "KL/std": 64.33500671386719, + "epoch": 0.3779289493575208, + "fcm_dpo/beta": 0.009752372279763222, + "fcm_dpo/delta": -0.10089530050754547, + "fcm_dpo/margin": 47.049591064453125, + "fcm_dpo/q_t": 0.3958304524421692, + "grad_norm": 14.456826210021973, + "learning_rate": 3.9285205908608934e-07, + "logits/chosen": 0.7627512216567993, + "logits/rejected": 0.7224296927452087, + "logps/chosen": -143.4276123046875, + "logps/ref_chosen": -61.938209533691406, + "logps/ref_rejected": -72.21602630615234, + "logps/rejected": -200.75502014160156, + "loss": 1.0794, + "margin_dpo/margin_mean": 47.04959487915039, + "margin_dpo/margin_std": 66.09559631347656, + "step": 250 + }, + { + "KL/chosen_KL_mean": -87.73074340820312, + "KL/mean": -103.30741882324219, + "KL/rejected_KL_mean": -118.88409423828125, + "KL/std": 56.92936706542969, + "epoch": 0.3794406651549509, + "fcm_dpo/beta": 0.009653978981077671, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 31.153358459472656, + "fcm_dpo/q_t": 0.43081212043762207, + "grad_norm": 22.772136688232422, + "learning_rate": 3.9176496596569265e-07, + "logits/chosen": 0.7472172975540161, + "logits/rejected": 0.7043584585189819, + "logps/chosen": -154.58767700195312, + "logps/ref_chosen": -66.85694885253906, + "logps/ref_rejected": -84.83396911621094, + "logps/rejected": -203.71807861328125, + "loss": 1.2011, + "margin_dpo/margin_mean": 31.153358459472656, + "margin_dpo/margin_std": 64.73333740234375, + "step": 251 + }, + { + "KL/chosen_KL_mean": -82.26924133300781, + "KL/mean": -94.54788208007812, + "KL/rejected_KL_mean": -106.82652282714844, + "KL/std": 59.7686653137207, + "epoch": 0.38095238095238093, + "fcm_dpo/beta": 0.009653978981077671, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 24.557275772094727, + "fcm_dpo/q_t": 0.4452328681945801, + "grad_norm": 19.71927833557129, + "learning_rate": 3.9067390737445254e-07, + "logits/chosen": 0.6775693893432617, + "logits/rejected": 0.6212589144706726, + "logps/chosen": -138.4931640625, + "logps/ref_chosen": -56.22393035888672, + "logps/ref_rejected": -77.1136245727539, + "logps/rejected": -183.94015502929688, + "loss": 1.2756, + "margin_dpo/margin_mean": 24.557275772094727, + "margin_dpo/margin_std": 68.90715026855469, + "step": 252 + }, + { + "KL/chosen_KL_mean": -80.07911682128906, + "KL/mean": -97.68270874023438, + "KL/rejected_KL_mean": -115.28628540039062, + "KL/std": 57.1429443359375, + "epoch": 0.382464096749811, + "fcm_dpo/beta": 0.009601515717804432, + "fcm_dpo/delta": -0.027246130630373955, + "fcm_dpo/margin": 35.207183837890625, + "fcm_dpo/q_t": 0.42204049229621887, + "grad_norm": 14.635730743408203, + "learning_rate": 3.8957891383162304e-07, + "logits/chosen": 0.7735403776168823, + "logits/rejected": 0.7356526851654053, + "logps/chosen": -132.28912353515625, + "logps/ref_chosen": -52.21001434326172, + "logps/ref_rejected": -58.75764846801758, + "logps/rejected": -174.0439453125, + "loss": 1.1572, + "margin_dpo/margin_mean": 35.207183837890625, + "margin_dpo/margin_std": 59.34178924560547, + "step": 253 + }, + { + "KL/chosen_KL_mean": -88.13926696777344, + "KL/mean": -106.71817016601562, + "KL/rejected_KL_mean": -125.29707336425781, + "KL/std": 62.50201416015625, + "epoch": 0.3839758125472411, + "fcm_dpo/beta": 0.009601515717804432, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 37.157798767089844, + "fcm_dpo/q_t": 0.41981834173202515, + "grad_norm": 13.173120498657227, + "learning_rate": 3.884800159665276e-07, + "logits/chosen": 0.6648178100585938, + "logits/rejected": 0.6116340160369873, + "logps/chosen": -153.77560424804688, + "logps/ref_chosen": -65.63632202148438, + "logps/ref_rejected": -82.34425354003906, + "logps/rejected": -207.64132690429688, + "loss": 1.1467, + "margin_dpo/margin_mean": 37.157798767089844, + "margin_dpo/margin_std": 62.577247619628906, + "step": 254 + }, + { + "KL/chosen_KL_mean": -77.47877502441406, + "KL/mean": -100.01842498779297, + "KL/rejected_KL_mean": -122.55805969238281, + "KL/std": 61.58515167236328, + "epoch": 0.3854875283446712, + "fcm_dpo/beta": 0.009527172893285751, + "fcm_dpo/delta": -0.03886480629444122, + "fcm_dpo/margin": 45.079288482666016, + "fcm_dpo/q_t": 0.4024481773376465, + "grad_norm": 15.716689109802246, + "learning_rate": 3.873772445177015e-07, + "logits/chosen": 0.6487714052200317, + "logits/rejected": 0.6173020005226135, + "logps/chosen": -145.38986206054688, + "logps/ref_chosen": -67.91108703613281, + "logps/ref_rejected": -83.89114379882812, + "logps/rejected": -206.44921875, + "loss": 1.0994, + "margin_dpo/margin_mean": 45.07928466796875, + "margin_dpo/margin_std": 67.75491333007812, + "step": 255 + }, + { + "KL/chosen_KL_mean": -94.20085144042969, + "KL/mean": -115.31936645507812, + "KL/rejected_KL_mean": -136.43788146972656, + "KL/std": 59.03213119506836, + "epoch": 0.3869992441421013, + "fcm_dpo/beta": 0.009447208605706692, + "fcm_dpo/delta": -0.042143430560827255, + "fcm_dpo/margin": 42.237030029296875, + "fcm_dpo/q_t": 0.4108726680278778, + "grad_norm": 14.595268249511719, + "learning_rate": 3.862706303320329e-07, + "logits/chosen": 0.676605761051178, + "logits/rejected": 0.6072751879692078, + "logps/chosen": -157.70083618164062, + "logps/ref_chosen": -63.49998474121094, + "logps/ref_rejected": -90.77104187011719, + "logps/rejected": -227.20892333984375, + "loss": 1.1483, + "margin_dpo/margin_mean": 42.237030029296875, + "margin_dpo/margin_std": 76.56491088867188, + "step": 256 + }, + { + "KL/chosen_KL_mean": -89.88215637207031, + "KL/mean": -116.78739166259766, + "KL/rejected_KL_mean": -143.692626953125, + "KL/std": 65.77367401123047, + "epoch": 0.3885109599395314, + "fcm_dpo/beta": 0.009294034913182259, + "fcm_dpo/delta": -0.08173276484012604, + "fcm_dpo/margin": 53.810462951660156, + "fcm_dpo/q_t": 0.38952571153640747, + "grad_norm": 11.852346420288086, + "learning_rate": 3.851602043638994e-07, + "logits/chosen": 0.672225296497345, + "logits/rejected": 0.5996080636978149, + "logps/chosen": -160.48281860351562, + "logps/ref_chosen": -70.60064697265625, + "logps/ref_rejected": -108.58313751220703, + "logps/rejected": -252.2757568359375, + "loss": 1.0582, + "margin_dpo/margin_mean": 53.810462951660156, + "margin_dpo/margin_std": 75.61648559570312, + "step": 257 + }, + { + "KL/chosen_KL_mean": -90.14016723632812, + "KL/mean": -110.83567810058594, + "KL/rejected_KL_mean": -131.53118896484375, + "KL/std": 57.87552261352539, + "epoch": 0.3900226757369615, + "fcm_dpo/beta": 0.009294034913182259, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 41.391021728515625, + "fcm_dpo/q_t": 0.41010379791259766, + "grad_norm": 13.612093925476074, + "learning_rate": 3.840459976743023e-07, + "logits/chosen": 0.7125341892242432, + "logits/rejected": 0.6583952307701111, + "logps/chosen": -149.3943328857422, + "logps/ref_chosen": -59.25416564941406, + "logps/ref_rejected": -85.58709716796875, + "logps/rejected": -217.1182861328125, + "loss": 1.1076, + "margin_dpo/margin_mean": 41.391021728515625, + "margin_dpo/margin_std": 58.5115966796875, + "step": 258 + }, + { + "KL/chosen_KL_mean": -79.61314392089844, + "KL/mean": -109.55941772460938, + "KL/rejected_KL_mean": -139.50570678710938, + "KL/std": 64.32805633544922, + "epoch": 0.3915343915343915, + "fcm_dpo/beta": 0.009043391793966293, + "fcm_dpo/delta": -0.15097783505916595, + "fcm_dpo/margin": 59.89255905151367, + "fcm_dpo/q_t": 0.3776322901248932, + "grad_norm": 13.222972869873047, + "learning_rate": 3.8292804142999796e-07, + "logits/chosen": 0.6543328166007996, + "logits/rejected": 0.5453038215637207, + "logps/chosen": -145.0480194091797, + "logps/ref_chosen": -65.43487548828125, + "logps/ref_rejected": -95.41731262207031, + "logps/rejected": -234.92300415039062, + "loss": 1.0328, + "margin_dpo/margin_mean": 59.89255905151367, + "margin_dpo/margin_std": 77.88516235351562, + "step": 259 + }, + { + "KL/chosen_KL_mean": -78.07093811035156, + "KL/mean": -102.1653823852539, + "KL/rejected_KL_mean": -126.25981903076172, + "KL/std": 58.916969299316406, + "epoch": 0.3930461073318216, + "fcm_dpo/beta": 0.008919179439544678, + "fcm_dpo/delta": -0.05486639216542244, + "fcm_dpo/margin": 48.18886947631836, + "fcm_dpo/q_t": 0.4026397168636322, + "grad_norm": 19.17668914794922, + "learning_rate": 3.818063669026256e-07, + "logits/chosen": 0.6947526931762695, + "logits/rejected": 0.5949761271476746, + "logps/chosen": -127.1605224609375, + "logps/ref_chosen": -49.08958435058594, + "logps/ref_rejected": -79.01708221435547, + "logps/rejected": -205.2769012451172, + "loss": 1.1036, + "margin_dpo/margin_mean": 48.188865661621094, + "margin_dpo/margin_std": 73.38361358642578, + "step": 260 + }, + { + "KL/chosen_KL_mean": -78.03056335449219, + "KL/mean": -98.04486846923828, + "KL/rejected_KL_mean": -118.0591812133789, + "KL/std": 57.562782287597656, + "epoch": 0.3945578231292517, + "fcm_dpo/beta": 0.008919179439544678, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 40.02862548828125, + "fcm_dpo/q_t": 0.4189288318157196, + "grad_norm": 16.82101058959961, + "learning_rate": 3.806810054678331e-07, + "logits/chosen": 0.5842745304107666, + "logits/rejected": 0.6270730495452881, + "logps/chosen": -148.9029541015625, + "logps/ref_chosen": -70.87239074707031, + "logps/ref_rejected": -65.01522064208984, + "logps/rejected": -183.07440185546875, + "loss": 1.1465, + "margin_dpo/margin_mean": 40.02862548828125, + "margin_dpo/margin_std": 67.19842529296875, + "step": 261 + }, + { + "KL/chosen_KL_mean": -85.85145568847656, + "KL/mean": -104.21426391601562, + "KL/rejected_KL_mean": -122.57707214355469, + "KL/std": 58.654212951660156, + "epoch": 0.3960695389266818, + "fcm_dpo/beta": 0.008919179439544678, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 36.725616455078125, + "fcm_dpo/q_t": 0.4247691035270691, + "grad_norm": 16.46879005432129, + "learning_rate": 3.7955198860439887e-07, + "logits/chosen": 0.710350751876831, + "logits/rejected": 0.6420841217041016, + "logps/chosen": -153.72207641601562, + "logps/ref_chosen": -67.8706283569336, + "logps/ref_rejected": -88.7205810546875, + "logps/rejected": -211.2976531982422, + "loss": 1.16, + "margin_dpo/margin_mean": 36.725616455078125, + "margin_dpo/margin_std": 62.674476623535156, + "step": 262 + }, + { + "KL/chosen_KL_mean": -79.6908187866211, + "KL/mean": -100.17828369140625, + "KL/rejected_KL_mean": -120.66574096679688, + "KL/std": 59.07019805908203, + "epoch": 0.3975812547241119, + "fcm_dpo/beta": 0.008919179439544678, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 40.97491455078125, + "fcm_dpo/q_t": 0.41486692428588867, + "grad_norm": 15.303149223327637, + "learning_rate": 3.784193478933516e-07, + "logits/chosen": 0.6577154397964478, + "logits/rejected": 0.5451149940490723, + "logps/chosen": -134.88540649414062, + "logps/ref_chosen": -55.194583892822266, + "logps/ref_rejected": -80.54048156738281, + "logps/rejected": -201.2062225341797, + "loss": 1.1287, + "margin_dpo/margin_mean": 40.97491455078125, + "margin_dpo/margin_std": 62.79846954345703, + "step": 263 + }, + { + "KL/chosen_KL_mean": -84.52911376953125, + "KL/mean": -106.55697631835938, + "KL/rejected_KL_mean": -128.5848388671875, + "KL/std": 65.0322494506836, + "epoch": 0.39909297052154197, + "fcm_dpo/beta": 0.008919179439544678, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 44.05571746826172, + "fcm_dpo/q_t": 0.41048991680145264, + "grad_norm": 13.230753898620605, + "learning_rate": 3.7728311501708674e-07, + "logits/chosen": 0.5887368321418762, + "logits/rejected": 0.5512345433235168, + "logps/chosen": -167.69979858398438, + "logps/ref_chosen": -83.17068481445312, + "logps/ref_rejected": -88.33625793457031, + "logps/rejected": -216.9210968017578, + "loss": 1.1262, + "margin_dpo/margin_mean": 44.055721282958984, + "margin_dpo/margin_std": 70.82036590576172, + "step": 264 + }, + { + "KL/chosen_KL_mean": -80.12649536132812, + "KL/mean": -102.91234588623047, + "KL/rejected_KL_mean": -125.69821166992188, + "KL/std": 62.13268280029297, + "epoch": 0.40060468631897206, + "fcm_dpo/beta": 0.008919179439544678, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 45.57171630859375, + "fcm_dpo/q_t": 0.4080004394054413, + "grad_norm": 11.275876998901367, + "learning_rate": 3.7614332175848027e-07, + "logits/chosen": 0.7446998357772827, + "logits/rejected": 0.6785590052604675, + "logps/chosen": -131.78933715820312, + "logps/ref_chosen": -51.66284942626953, + "logps/ref_rejected": -67.1720962524414, + "logps/rejected": -192.87030029296875, + "loss": 1.1342, + "margin_dpo/margin_mean": 45.57171630859375, + "margin_dpo/margin_std": 77.82930755615234, + "step": 265 + }, + { + "KL/chosen_KL_mean": -77.90547180175781, + "KL/mean": -99.6781005859375, + "KL/rejected_KL_mean": -121.45071411132812, + "KL/std": 62.47583770751953, + "epoch": 0.4021164021164021, + "fcm_dpo/beta": 0.008919179439544678, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 43.54524612426758, + "fcm_dpo/q_t": 0.41210731863975525, + "grad_norm": 19.963960647583008, + "learning_rate": 3.75e-07, + "logits/chosen": 0.6973221302032471, + "logits/rejected": 0.6187717914581299, + "logps/chosen": -135.35597229003906, + "logps/ref_chosen": -57.45049285888672, + "logps/ref_rejected": -77.60826110839844, + "logps/rejected": -199.05897521972656, + "loss": 1.1361, + "margin_dpo/margin_mean": 43.54524612426758, + "margin_dpo/margin_std": 73.66079711914062, + "step": 266 + }, + { + "KL/chosen_KL_mean": -69.62604522705078, + "KL/mean": -87.49578857421875, + "KL/rejected_KL_mean": -105.36552429199219, + "KL/std": 60.37127685546875, + "epoch": 0.4036281179138322, + "fcm_dpo/beta": 0.00886848010122776, + "fcm_dpo/delta": -0.05716828629374504, + "fcm_dpo/margin": 35.739471435546875, + "fcm_dpo/q_t": 0.42751142382621765, + "grad_norm": 17.76947021484375, + "learning_rate": 3.738531817228131e-07, + "logits/chosen": 0.7317204475402832, + "logits/rejected": 0.712775468826294, + "logps/chosen": -124.66139221191406, + "logps/ref_chosen": -55.03535079956055, + "logps/ref_rejected": -66.0953369140625, + "logps/rejected": -171.4608612060547, + "loss": 1.2038, + "margin_dpo/margin_mean": 35.739471435546875, + "margin_dpo/margin_std": 72.42359924316406, + "step": 267 + }, + { + "KL/chosen_KL_mean": -69.93362426757812, + "KL/mean": -91.20822143554688, + "KL/rejected_KL_mean": -112.48282623291016, + "KL/std": 63.411277770996094, + "epoch": 0.4051398337112623, + "fcm_dpo/beta": 0.008803716860711575, + "fcm_dpo/delta": -0.015976281836628914, + "fcm_dpo/margin": 42.549198150634766, + "fcm_dpo/q_t": 0.414595365524292, + "grad_norm": 16.74564552307129, + "learning_rate": 3.7270289900589204e-07, + "logits/chosen": 0.5760444402694702, + "logits/rejected": 0.5645999312400818, + "logps/chosen": -135.00537109375, + "logps/ref_chosen": -65.07174682617188, + "logps/ref_rejected": -71.42485809326172, + "logps/rejected": -183.90768432617188, + "loss": 1.1182, + "margin_dpo/margin_mean": 42.549198150634766, + "margin_dpo/margin_std": 62.1130485534668, + "step": 268 + }, + { + "KL/chosen_KL_mean": -81.66681671142578, + "KL/mean": -105.6706314086914, + "KL/rejected_KL_mean": -129.6744384765625, + "KL/std": 67.39564514160156, + "epoch": 0.40665154950869237, + "fcm_dpo/beta": 0.008736472576856613, + "fcm_dpo/delta": -0.06087144464254379, + "fcm_dpo/margin": 48.00763702392578, + "fcm_dpo/q_t": 0.4043356776237488, + "grad_norm": 13.045381546020508, + "learning_rate": 3.7154918402511714e-07, + "logits/chosen": 0.7855197191238403, + "logits/rejected": 0.7380328178405762, + "logps/chosen": -148.8030242919922, + "logps/ref_chosen": -67.1362075805664, + "logps/ref_rejected": -82.55778503417969, + "logps/rejected": -212.23223876953125, + "loss": 1.0931, + "margin_dpo/margin_mean": 48.00763702392578, + "margin_dpo/margin_std": 67.01888275146484, + "step": 269 + }, + { + "KL/chosen_KL_mean": -79.60834503173828, + "KL/mean": -100.95454406738281, + "KL/rejected_KL_mean": -122.3007583618164, + "KL/std": 65.51744079589844, + "epoch": 0.40816326530612246, + "fcm_dpo/beta": 0.008683293126523495, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 42.692413330078125, + "fcm_dpo/q_t": 0.414852499961853, + "grad_norm": 14.931079864501953, + "learning_rate": 3.7039206905237656e-07, + "logits/chosen": 0.7250645756721497, + "logits/rejected": 0.6401588320732117, + "logps/chosen": -146.2970428466797, + "logps/ref_chosen": -66.6886978149414, + "logps/ref_rejected": -85.16129302978516, + "logps/rejected": -207.46205139160156, + "loss": 1.1352, + "margin_dpo/margin_mean": 42.692413330078125, + "margin_dpo/margin_std": 69.67828369140625, + "step": 270 + }, + { + "KL/chosen_KL_mean": -81.57511138916016, + "KL/mean": -97.54861450195312, + "KL/rejected_KL_mean": -113.5221176147461, + "KL/std": 70.79106140136719, + "epoch": 0.40967498110355255, + "fcm_dpo/beta": 0.008683293126523495, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 31.947006225585938, + "fcm_dpo/q_t": 0.44066426157951355, + "grad_norm": 20.921737670898438, + "learning_rate": 3.692315864546635e-07, + "logits/chosen": 0.6996304988861084, + "logits/rejected": 0.6347111463546753, + "logps/chosen": -153.982666015625, + "logps/ref_chosen": -72.40754699707031, + "logps/ref_rejected": -92.06311798095703, + "logps/rejected": -205.58523559570312, + "loss": 1.2502, + "margin_dpo/margin_mean": 31.94700813293457, + "margin_dpo/margin_std": 83.21861267089844, + "step": 271 + }, + { + "KL/chosen_KL_mean": -67.67726135253906, + "KL/mean": -98.39691925048828, + "KL/rejected_KL_mean": -129.1165771484375, + "KL/std": 62.144187927246094, + "epoch": 0.41118669690098264, + "fcm_dpo/beta": 0.008535921573638916, + "fcm_dpo/delta": -0.13113445043563843, + "fcm_dpo/margin": 61.4393310546875, + "fcm_dpo/q_t": 0.37742963433265686, + "grad_norm": 12.506288528442383, + "learning_rate": 3.6806776869317067e-07, + "logits/chosen": 0.742793083190918, + "logits/rejected": 0.7653173208236694, + "logps/chosen": -134.27865600585938, + "logps/ref_chosen": -66.60140228271484, + "logps/ref_rejected": -67.74340057373047, + "logps/rejected": -196.8599853515625, + "loss": 0.9911, + "margin_dpo/margin_mean": 61.4393310546875, + "margin_dpo/margin_std": 59.98362731933594, + "step": 272 + }, + { + "KL/chosen_KL_mean": -88.57992553710938, + "KL/mean": -110.85275268554688, + "KL/rejected_KL_mean": -133.12557983398438, + "KL/std": 67.68528747558594, + "epoch": 0.4126984126984127, + "fcm_dpo/beta": 0.008458517491817474, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 44.54566192626953, + "fcm_dpo/q_t": 0.41627636551856995, + "grad_norm": 20.072294235229492, + "learning_rate": 3.669006483223828e-07, + "logits/chosen": 0.7203218936920166, + "logits/rejected": 0.6446723937988281, + "logps/chosen": -145.934814453125, + "logps/ref_chosen": -57.35487747192383, + "logps/ref_rejected": -84.17168426513672, + "logps/rejected": -217.29727172851562, + "loss": 1.1725, + "margin_dpo/margin_mean": 44.545658111572266, + "margin_dpo/margin_std": 86.9574203491211, + "step": 273 + }, + { + "KL/chosen_KL_mean": -85.86259460449219, + "KL/mean": -110.53076171875, + "KL/rejected_KL_mean": -135.1989288330078, + "KL/std": 66.234619140625, + "epoch": 0.41421012849584277, + "fcm_dpo/beta": 0.00843963585793972, + "fcm_dpo/delta": -0.02237236686050892, + "fcm_dpo/margin": 49.33633804321289, + "fcm_dpo/q_t": 0.4048211872577667, + "grad_norm": 14.553647994995117, + "learning_rate": 3.657302579891656e-07, + "logits/chosen": 0.5977407693862915, + "logits/rejected": 0.581523060798645, + "logps/chosen": -145.50408935546875, + "logps/ref_chosen": -59.64149475097656, + "logps/ref_rejected": -68.29348754882812, + "logps/rejected": -203.49240112304688, + "loss": 1.1142, + "margin_dpo/margin_mean": 49.33633804321289, + "margin_dpo/margin_std": 77.90882110595703, + "step": 274 + }, + { + "KL/chosen_KL_mean": -85.02987670898438, + "KL/mean": -108.27204895019531, + "KL/rejected_KL_mean": -131.51422119140625, + "KL/std": 63.815208435058594, + "epoch": 0.41572184429327286, + "fcm_dpo/beta": 0.008404111489653587, + "fcm_dpo/delta": -0.019802536815404892, + "fcm_dpo/margin": 46.48435974121094, + "fcm_dpo/q_t": 0.41008520126342773, + "grad_norm": 21.007122039794922, + "learning_rate": 3.645566304318526e-07, + "logits/chosen": 0.6761637926101685, + "logits/rejected": 0.5931113958358765, + "logps/chosen": -138.2965087890625, + "logps/ref_chosen": -53.26664352416992, + "logps/ref_rejected": -73.84062194824219, + "logps/rejected": -205.3548583984375, + "loss": 1.1075, + "margin_dpo/margin_mean": 46.4843635559082, + "margin_dpo/margin_std": 66.59793090820312, + "step": 275 + }, + { + "KL/chosen_KL_mean": -79.12834167480469, + "KL/mean": -102.84222412109375, + "KL/rejected_KL_mean": -126.55609893798828, + "KL/std": 59.876068115234375, + "epoch": 0.41723356009070295, + "fcm_dpo/beta": 0.008387469686567783, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 47.427764892578125, + "fcm_dpo/q_t": 0.4078884720802307, + "grad_norm": 16.673648834228516, + "learning_rate": 3.633797984793294e-07, + "logits/chosen": 0.6499893069267273, + "logits/rejected": 0.6180636882781982, + "logps/chosen": -132.14913940429688, + "logps/ref_chosen": -53.02079772949219, + "logps/ref_rejected": -61.56678771972656, + "logps/rejected": -188.12289428710938, + "loss": 1.1034, + "margin_dpo/margin_mean": 47.427764892578125, + "margin_dpo/margin_std": 67.45655822753906, + "step": 276 + }, + { + "KL/chosen_KL_mean": -92.67013549804688, + "KL/mean": -104.78817749023438, + "KL/rejected_KL_mean": -116.90620422363281, + "KL/std": 67.29153442382812, + "epoch": 0.41874527588813304, + "fcm_dpo/beta": 0.008387469686567783, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 24.23607635498047, + "fcm_dpo/q_t": 0.4531954526901245, + "grad_norm": 19.32560157775879, + "learning_rate": 3.6219979505011555e-07, + "logits/chosen": 0.7294178009033203, + "logits/rejected": 0.7562111616134644, + "logps/chosen": -164.10311889648438, + "logps/ref_chosen": -71.43299102783203, + "logps/ref_rejected": -67.65852355957031, + "logps/rejected": -184.56472778320312, + "loss": 1.2928, + "margin_dpo/margin_mean": 24.23607635498047, + "margin_dpo/margin_std": 76.65855407714844, + "step": 277 + }, + { + "KL/chosen_KL_mean": -98.54222106933594, + "KL/mean": -121.556884765625, + "KL/rejected_KL_mean": -144.57154846191406, + "KL/std": 69.13101196289062, + "epoch": 0.42025699168556313, + "fcm_dpo/beta": 0.00834178552031517, + "fcm_dpo/delta": -0.05476619303226471, + "fcm_dpo/margin": 46.029335021972656, + "fcm_dpo/q_t": 0.41048651933670044, + "grad_norm": 19.770599365234375, + "learning_rate": 3.6101665315144353e-07, + "logits/chosen": 0.6561414003372192, + "logits/rejected": 0.5932800769805908, + "logps/chosen": -165.65298461914062, + "logps/ref_chosen": -67.11076354980469, + "logps/ref_rejected": -88.74851989746094, + "logps/rejected": -233.320068359375, + "loss": 1.1265, + "margin_dpo/margin_mean": 46.029335021972656, + "margin_dpo/margin_std": 71.37066650390625, + "step": 278 + }, + { + "KL/chosen_KL_mean": -77.33357238769531, + "KL/mean": -108.97832489013672, + "KL/rejected_KL_mean": -140.62307739257812, + "KL/std": 66.38153076171875, + "epoch": 0.4217687074829932, + "fcm_dpo/beta": 0.008170300163328648, + "fcm_dpo/delta": -0.12329418212175369, + "fcm_dpo/margin": 63.28950500488281, + "fcm_dpo/q_t": 0.3793160319328308, + "grad_norm": 18.47074317932129, + "learning_rate": 3.5983040587833563e-07, + "logits/chosen": 0.684441089630127, + "logits/rejected": 0.6455780267715454, + "logps/chosen": -131.8310546875, + "logps/ref_chosen": -54.49748611450195, + "logps/ref_rejected": -70.42373657226562, + "logps/rejected": -211.04681396484375, + "loss": 1.0028, + "margin_dpo/margin_mean": 63.28950500488281, + "margin_dpo/margin_std": 64.86318969726562, + "step": 279 + }, + { + "KL/chosen_KL_mean": -84.39732360839844, + "KL/mean": -117.23088836669922, + "KL/rejected_KL_mean": -150.064453125, + "KL/std": 69.86494445800781, + "epoch": 0.42328042328042326, + "fcm_dpo/beta": 0.007942087948322296, + "fcm_dpo/delta": -0.1282101720571518, + "fcm_dpo/margin": 65.66712951660156, + "fcm_dpo/q_t": 0.3792378902435303, + "grad_norm": 13.688976287841797, + "learning_rate": 3.586410864126781e-07, + "logits/chosen": 0.7099773287773132, + "logits/rejected": 0.6641367673873901, + "logps/chosen": -144.83013916015625, + "logps/ref_chosen": -60.43281173706055, + "logps/ref_rejected": -78.39051818847656, + "logps/rejected": -228.45497131347656, + "loss": 0.9951, + "margin_dpo/margin_mean": 65.66712951660156, + "margin_dpo/margin_std": 66.24174499511719, + "step": 280 + }, + { + "KL/chosen_KL_mean": -87.24712371826172, + "KL/mean": -113.78001403808594, + "KL/rejected_KL_mean": -140.31289672851562, + "KL/std": 68.34732818603516, + "epoch": 0.42479213907785335, + "fcm_dpo/beta": 0.007860523648560047, + "fcm_dpo/delta": -0.03638084977865219, + "fcm_dpo/margin": 53.0657844543457, + "fcm_dpo/q_t": 0.4040064811706543, + "grad_norm": 15.016488075256348, + "learning_rate": 3.574487280222929e-07, + "logits/chosen": 0.678945779800415, + "logits/rejected": 0.7068662643432617, + "logps/chosen": -147.52920532226562, + "logps/ref_chosen": -60.2820930480957, + "logps/ref_rejected": -62.04009246826172, + "logps/rejected": -202.35299682617188, + "loss": 1.0984, + "margin_dpo/margin_mean": 53.0657844543457, + "margin_dpo/margin_std": 76.41177368164062, + "step": 281 + }, + { + "KL/chosen_KL_mean": -98.00955200195312, + "KL/mean": -122.0562973022461, + "KL/rejected_KL_mean": -146.10305786132812, + "KL/std": 69.0963134765625, + "epoch": 0.42630385487528344, + "fcm_dpo/beta": 0.007776046171784401, + "fcm_dpo/delta": -0.07186315953731537, + "fcm_dpo/margin": 48.0934944152832, + "fcm_dpo/q_t": 0.41511061787605286, + "grad_norm": 18.2562313079834, + "learning_rate": 3.562533640600075e-07, + "logits/chosen": 0.615809440612793, + "logits/rejected": 0.5732920169830322, + "logps/chosen": -158.63348388671875, + "logps/ref_chosen": -60.623924255371094, + "logps/ref_rejected": -68.67400360107422, + "logps/rejected": -214.7770538330078, + "loss": 1.144, + "margin_dpo/margin_mean": 48.0934944152832, + "margin_dpo/margin_std": 79.40950775146484, + "step": 282 + }, + { + "KL/chosen_KL_mean": -95.66755676269531, + "KL/mean": -121.100830078125, + "KL/rejected_KL_mean": -146.53411865234375, + "KL/std": 72.49221801757812, + "epoch": 0.42781557067271353, + "fcm_dpo/beta": 0.007720166351646185, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 50.866546630859375, + "fcm_dpo/q_t": 0.410746693611145, + "grad_norm": 13.532066345214844, + "learning_rate": 3.550550279627215e-07, + "logits/chosen": 0.6475073099136353, + "logits/rejected": 0.5460326671600342, + "logps/chosen": -163.3153076171875, + "logps/ref_chosen": -67.64775085449219, + "logps/ref_rejected": -99.96835327148438, + "logps/rejected": -246.50247192382812, + "loss": 1.125, + "margin_dpo/margin_mean": 50.866546630859375, + "margin_dpo/margin_std": 81.53480529785156, + "step": 283 + }, + { + "KL/chosen_KL_mean": -92.60566711425781, + "KL/mean": -117.02914428710938, + "KL/rejected_KL_mean": -141.45263671875, + "KL/std": 71.92315673828125, + "epoch": 0.4293272864701436, + "fcm_dpo/beta": 0.007683092262595892, + "fcm_dpo/delta": -0.024069122970104218, + "fcm_dpo/margin": 48.846961975097656, + "fcm_dpo/q_t": 0.4131779670715332, + "grad_norm": 11.925021171569824, + "learning_rate": 3.5385375325047163e-07, + "logits/chosen": 0.7429874539375305, + "logits/rejected": 0.6754733324050903, + "logps/chosen": -149.57308959960938, + "logps/ref_chosen": -56.96742630004883, + "logps/ref_rejected": -86.36236572265625, + "logps/rejected": -227.81500244140625, + "loss": 1.1139, + "margin_dpo/margin_mean": 48.846961975097656, + "margin_dpo/margin_std": 69.17724609375, + "step": 284 + }, + { + "KL/chosen_KL_mean": -108.53469848632812, + "KL/mean": -128.46315002441406, + "KL/rejected_KL_mean": -148.39158630371094, + "KL/std": 65.19612121582031, + "epoch": 0.4308390022675737, + "fcm_dpo/beta": 0.007683092262595892, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 39.85688781738281, + "fcm_dpo/q_t": 0.4297737181186676, + "grad_norm": 16.518638610839844, + "learning_rate": 3.5264957352549375e-07, + "logits/chosen": 0.7107409238815308, + "logits/rejected": 0.6831285953521729, + "logps/chosen": -180.19081115722656, + "logps/ref_chosen": -71.65611267089844, + "logps/ref_rejected": -81.63829803466797, + "logps/rejected": -230.02987670898438, + "loss": 1.1817, + "margin_dpo/margin_mean": 39.85688400268555, + "margin_dpo/margin_std": 73.79049682617188, + "step": 285 + }, + { + "KL/chosen_KL_mean": -103.9982681274414, + "KL/mean": -136.58309936523438, + "KL/rejected_KL_mean": -169.1679229736328, + "KL/std": 70.7746810913086, + "epoch": 0.4323507180650038, + "fcm_dpo/beta": 0.007573241367936134, + "fcm_dpo/delta": -0.0983816385269165, + "fcm_dpo/margin": 65.16966247558594, + "fcm_dpo/q_t": 0.3875123858451843, + "grad_norm": 14.638473510742188, + "learning_rate": 3.514425224712835e-07, + "logits/chosen": 0.635071873664856, + "logits/rejected": 0.5436767339706421, + "logps/chosen": -165.07778930664062, + "logps/ref_chosen": -61.07952117919922, + "logps/ref_rejected": -91.28128051757812, + "logps/rejected": -260.44921875, + "loss": 1.0267, + "margin_dpo/margin_mean": 65.16966247558594, + "margin_dpo/margin_std": 75.55731964111328, + "step": 286 + }, + { + "KL/chosen_KL_mean": -82.97677612304688, + "KL/mean": -115.36837768554688, + "KL/rejected_KL_mean": -147.7599639892578, + "KL/std": 70.15164184570312, + "epoch": 0.43386243386243384, + "fcm_dpo/beta": 0.007446384988725185, + "fcm_dpo/delta": -0.08650145679712296, + "fcm_dpo/margin": 64.78318786621094, + "fcm_dpo/q_t": 0.3885365128517151, + "grad_norm": 10.52198314666748, + "learning_rate": 3.502326338516534e-07, + "logits/chosen": 0.7257726192474365, + "logits/rejected": 0.6863799095153809, + "logps/chosen": -129.0125732421875, + "logps/ref_chosen": -46.035789489746094, + "logps/ref_rejected": -59.95293426513672, + "logps/rejected": -207.712890625, + "loss": 1.0308, + "margin_dpo/margin_mean": 64.78318786621094, + "margin_dpo/margin_std": 73.37957763671875, + "step": 287 + }, + { + "KL/chosen_KL_mean": -107.4090805053711, + "KL/mean": -132.0665740966797, + "KL/rejected_KL_mean": -156.72406005859375, + "KL/std": 69.27061462402344, + "epoch": 0.43537414965986393, + "fcm_dpo/beta": 0.0074041858315467834, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 49.314971923828125, + "fcm_dpo/q_t": 0.41598302125930786, + "grad_norm": 12.507437705993652, + "learning_rate": 3.490199415097892e-07, + "logits/chosen": 0.5906128883361816, + "logits/rejected": 0.5325485467910767, + "logps/chosen": -172.7999267578125, + "logps/ref_chosen": -65.3908462524414, + "logps/ref_rejected": -88.53607940673828, + "logps/rejected": -245.2601318359375, + "loss": 1.1293, + "margin_dpo/margin_mean": 49.314971923828125, + "margin_dpo/margin_std": 75.95675659179688, + "step": 288 + }, + { + "KL/chosen_KL_mean": -108.69920349121094, + "KL/mean": -132.19676208496094, + "KL/rejected_KL_mean": -155.69430541992188, + "KL/std": 69.22113800048828, + "epoch": 0.436885865457294, + "fcm_dpo/beta": 0.0074041858315467834, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 46.99510192871094, + "fcm_dpo/q_t": 0.4210740029811859, + "grad_norm": 11.996780395507812, + "learning_rate": 3.4780447936730247e-07, + "logits/chosen": 0.7544581294059753, + "logits/rejected": 0.7194222807884216, + "logps/chosen": -163.2928466796875, + "logps/ref_chosen": -54.5936279296875, + "logps/ref_rejected": -67.20855712890625, + "logps/rejected": -222.90286254882812, + "loss": 1.1505, + "margin_dpo/margin_mean": 46.99510192871094, + "margin_dpo/margin_std": 79.12466430664062, + "step": 289 + }, + { + "KL/chosen_KL_mean": -122.24220275878906, + "KL/mean": -147.22738647460938, + "KL/rejected_KL_mean": -172.21255493164062, + "KL/std": 67.15545654296875, + "epoch": 0.4383975812547241, + "fcm_dpo/beta": 0.007365147117525339, + "fcm_dpo/delta": -0.02643224596977234, + "fcm_dpo/margin": 49.970359802246094, + "fcm_dpo/q_t": 0.41537922620773315, + "grad_norm": 15.518068313598633, + "learning_rate": 3.465862814232821e-07, + "logits/chosen": 0.7712986469268799, + "logits/rejected": 0.6948202252388, + "logps/chosen": -183.62677001953125, + "logps/ref_chosen": -61.38457489013672, + "logps/ref_rejected": -91.92778015136719, + "logps/rejected": -264.14031982421875, + "loss": 1.1284, + "margin_dpo/margin_mean": 49.97035598754883, + "margin_dpo/margin_std": 76.6805648803711, + "step": 290 + }, + { + "KL/chosen_KL_mean": -111.89765167236328, + "KL/mean": -143.10159301757812, + "KL/rejected_KL_mean": -174.3055419921875, + "KL/std": 69.66302490234375, + "epoch": 0.4399092970521542, + "fcm_dpo/beta": 0.007289406843483448, + "fcm_dpo/delta": -0.10390862822532654, + "fcm_dpo/margin": 62.407894134521484, + "fcm_dpo/q_t": 0.39798957109451294, + "grad_norm": 12.921642303466797, + "learning_rate": 3.4536538175334343e-07, + "logits/chosen": 0.8233385682106018, + "logits/rejected": 0.7497340440750122, + "logps/chosen": -162.76068115234375, + "logps/ref_chosen": -50.863037109375, + "logps/ref_rejected": -82.20868682861328, + "logps/rejected": -256.51422119140625, + "loss": 1.0685, + "margin_dpo/margin_mean": 62.40789031982422, + "margin_dpo/margin_std": 82.35305786132812, + "step": 291 + }, + { + "KL/chosen_KL_mean": -113.44772338867188, + "KL/mean": -134.71461486816406, + "KL/rejected_KL_mean": -155.9815216064453, + "KL/std": 73.57888793945312, + "epoch": 0.4414210128495843, + "fcm_dpo/beta": 0.00721366610378027, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 42.53379821777344, + "fcm_dpo/q_t": 0.4306395351886749, + "grad_norm": 15.100018501281738, + "learning_rate": 3.4414181450867465e-07, + "logits/chosen": 0.720421552658081, + "logits/rejected": 0.6704583764076233, + "logps/chosen": -177.79660034179688, + "logps/ref_chosen": -64.34888458251953, + "logps/ref_rejected": -72.86434173583984, + "logps/rejected": -228.84585571289062, + "loss": 1.2032, + "margin_dpo/margin_mean": 42.53379821777344, + "margin_dpo/margin_std": 90.44749450683594, + "step": 292 + }, + { + "KL/chosen_KL_mean": -105.96971130371094, + "KL/mean": -140.18777465820312, + "KL/rejected_KL_mean": -174.40582275390625, + "KL/std": 74.28352355957031, + "epoch": 0.4429327286470144, + "fcm_dpo/beta": 0.007149941753596067, + "fcm_dpo/delta": -0.08912809938192368, + "fcm_dpo/margin": 68.43610382080078, + "fcm_dpo/q_t": 0.38950157165527344, + "grad_norm": 10.191673278808594, + "learning_rate": 3.4291561391508185e-07, + "logits/chosen": 0.822028398513794, + "logits/rejected": 0.7336381673812866, + "logps/chosen": -160.83917236328125, + "logps/ref_chosen": -54.869468688964844, + "logps/ref_rejected": -81.858642578125, + "logps/rejected": -256.26446533203125, + "loss": 1.0537, + "margin_dpo/margin_mean": 68.43610382080078, + "margin_dpo/margin_std": 89.89541625976562, + "step": 293 + }, + { + "KL/chosen_KL_mean": -113.77799987792969, + "KL/mean": -136.8584442138672, + "KL/rejected_KL_mean": -159.93887329101562, + "KL/std": 76.31533813476562, + "epoch": 0.4444444444444444, + "fcm_dpo/beta": 0.007086217403411865, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 46.1608772277832, + "fcm_dpo/q_t": 0.4244413375854492, + "grad_norm": 11.190646171569824, + "learning_rate": 3.4168681427203153e-07, + "logits/chosen": 0.7264614105224609, + "logits/rejected": 0.6801228523254395, + "logps/chosen": -170.44891357421875, + "logps/ref_chosen": -56.670902252197266, + "logps/ref_rejected": -70.32819366455078, + "logps/rejected": -230.26707458496094, + "loss": 1.1415, + "margin_dpo/margin_mean": 46.16087341308594, + "margin_dpo/margin_std": 68.03617095947266, + "step": 294 + }, + { + "KL/chosen_KL_mean": -117.91046142578125, + "KL/mean": -139.72238159179688, + "KL/rejected_KL_mean": -161.53427124023438, + "KL/std": 71.26055908203125, + "epoch": 0.4459561602418745, + "fcm_dpo/beta": 0.007086217403411865, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 43.623809814453125, + "fcm_dpo/q_t": 0.4286356568336487, + "grad_norm": 16.999357223510742, + "learning_rate": 3.4045544995169125e-07, + "logits/chosen": 0.7363671660423279, + "logits/rejected": 0.6302077174186707, + "logps/chosen": -168.3113555908203, + "logps/ref_chosen": -50.40088653564453, + "logps/ref_rejected": -83.43521881103516, + "logps/rejected": -244.96949768066406, + "loss": 1.1704, + "margin_dpo/margin_mean": 43.623809814453125, + "margin_dpo/margin_std": 76.41813659667969, + "step": 295 + }, + { + "KL/chosen_KL_mean": -121.54795837402344, + "KL/mean": -148.98709106445312, + "KL/rejected_KL_mean": -176.4262237548828, + "KL/std": 78.2703857421875, + "epoch": 0.4474678760393046, + "fcm_dpo/beta": 0.007052321918308735, + "fcm_dpo/delta": -0.023973939940333366, + "fcm_dpo/margin": 54.87827682495117, + "fcm_dpo/q_t": 0.4116702377796173, + "grad_norm": 12.077364921569824, + "learning_rate": 3.392215553979679e-07, + "logits/chosen": 0.7017396688461304, + "logits/rejected": 0.6508808135986328, + "logps/chosen": -190.69830322265625, + "logps/ref_chosen": -69.15034484863281, + "logps/ref_rejected": -89.60166931152344, + "logps/rejected": -266.02789306640625, + "loss": 1.1304, + "margin_dpo/margin_mean": 54.878273010253906, + "margin_dpo/margin_std": 89.46514892578125, + "step": 296 + }, + { + "KL/chosen_KL_mean": -124.2834243774414, + "KL/mean": -151.585693359375, + "KL/rejected_KL_mean": -178.887939453125, + "KL/std": 68.32637023925781, + "epoch": 0.4489795918367347, + "fcm_dpo/beta": 0.007041130214929581, + "fcm_dpo/delta": -0.01589428447186947, + "fcm_dpo/margin": 54.60452651977539, + "fcm_dpo/q_t": 0.41009995341300964, + "grad_norm": 12.764442443847656, + "learning_rate": 3.3798516512554485e-07, + "logits/chosen": 0.7037564516067505, + "logits/rejected": 0.6517189741134644, + "logps/chosen": -182.2997283935547, + "logps/ref_chosen": -58.01630401611328, + "logps/ref_rejected": -69.95780944824219, + "logps/rejected": -248.84576416015625, + "loss": 1.1013, + "margin_dpo/margin_mean": 54.60452651977539, + "margin_dpo/margin_std": 72.62581634521484, + "step": 297 + }, + { + "KL/chosen_KL_mean": -121.14657592773438, + "KL/mean": -141.3595733642578, + "KL/rejected_KL_mean": -161.57257080078125, + "KL/std": 72.325927734375, + "epoch": 0.4504913076341648, + "fcm_dpo/beta": 0.007029938977211714, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 40.426002502441406, + "fcm_dpo/q_t": 0.4347230792045593, + "grad_norm": 11.478727340698242, + "learning_rate": 3.367463137189156e-07, + "logits/chosen": 0.8037296533584595, + "logits/rejected": 0.7485308647155762, + "logps/chosen": -177.31588745117188, + "logps/ref_chosen": -56.1693115234375, + "logps/ref_rejected": -68.55052185058594, + "logps/rejected": -230.1230926513672, + "loss": 1.2023, + "margin_dpo/margin_mean": 40.426002502441406, + "margin_dpo/margin_std": 82.11355590820312, + "step": 298 + }, + { + "KL/chosen_KL_mean": -127.74474334716797, + "KL/mean": -144.35882568359375, + "KL/rejected_KL_mean": -160.972900390625, + "KL/std": 71.02047729492188, + "epoch": 0.4520030234315949, + "fcm_dpo/beta": 0.007029938977211714, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 33.2281494140625, + "fcm_dpo/q_t": 0.44710543751716614, + "grad_norm": 19.625972747802734, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": 0.5966833829879761, + "logits/rejected": 0.5654438734054565, + "logps/chosen": -190.0625457763672, + "logps/ref_chosen": -62.31780242919922, + "logps/ref_rejected": -72.60028839111328, + "logps/rejected": -233.57318115234375, + "loss": 1.2661, + "margin_dpo/margin_mean": 33.228145599365234, + "margin_dpo/margin_std": 90.59477996826172, + "step": 299 + }, + { + "KL/chosen_KL_mean": -126.5094223022461, + "KL/mean": -148.61163330078125, + "KL/rejected_KL_mean": -170.7138214111328, + "KL/std": 72.69538879394531, + "epoch": 0.45351473922902497, + "fcm_dpo/beta": 0.007029938977211714, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 44.20439147949219, + "fcm_dpo/q_t": 0.428915411233902, + "grad_norm": 13.045540809631348, + "learning_rate": 3.3426136618426043e-07, + "logits/chosen": 0.7217717170715332, + "logits/rejected": 0.6547795534133911, + "logps/chosen": -186.89100646972656, + "logps/ref_chosen": -60.38157653808594, + "logps/ref_rejected": -75.45442199707031, + "logps/rejected": -246.16824340820312, + "loss": 1.185, + "margin_dpo/margin_mean": 44.20439147949219, + "margin_dpo/margin_std": 84.41621398925781, + "step": 300 + }, + { + "KL/chosen_KL_mean": -126.29975891113281, + "KL/mean": -145.92193603515625, + "KL/rejected_KL_mean": -165.54409790039062, + "KL/std": 71.30606079101562, + "epoch": 0.455026455026455, + "fcm_dpo/beta": 0.007029938977211714, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 39.244346618652344, + "fcm_dpo/q_t": 0.4370271563529968, + "grad_norm": 12.925996780395508, + "learning_rate": 3.3301533956555885e-07, + "logits/chosen": 0.7587097883224487, + "logits/rejected": 0.7291955351829529, + "logps/chosen": -179.15066528320312, + "logps/ref_chosen": -52.85089111328125, + "logps/ref_rejected": -69.97584533691406, + "logps/rejected": -235.51995849609375, + "loss": 1.2112, + "margin_dpo/margin_mean": 39.244346618652344, + "margin_dpo/margin_std": 83.38739013671875, + "step": 301 + }, + { + "KL/chosen_KL_mean": -133.72024536132812, + "KL/mean": -148.84934997558594, + "KL/rejected_KL_mean": -163.9784698486328, + "KL/std": 75.41315460205078, + "epoch": 0.4565381708238851, + "fcm_dpo/beta": 0.007029938977211714, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 30.258220672607422, + "fcm_dpo/q_t": 0.45121005177497864, + "grad_norm": 13.880743980407715, + "learning_rate": 3.317669908293554e-07, + "logits/chosen": 0.5911428928375244, + "logits/rejected": 0.5351113677024841, + "logps/chosen": -200.686767578125, + "logps/ref_chosen": -66.96650695800781, + "logps/ref_rejected": -88.09510803222656, + "logps/rejected": -252.07357788085938, + "loss": 1.2674, + "margin_dpo/margin_mean": 30.258224487304688, + "margin_dpo/margin_std": 83.85945129394531, + "step": 302 + }, + { + "KL/chosen_KL_mean": -114.63668823242188, + "KL/mean": -145.92343139648438, + "KL/rejected_KL_mean": -177.21014404296875, + "KL/std": 77.03577423095703, + "epoch": 0.4580498866213152, + "fcm_dpo/beta": 0.007015762850642204, + "fcm_dpo/delta": -0.020205684006214142, + "fcm_dpo/margin": 62.5734748840332, + "fcm_dpo/q_t": 0.40118786692619324, + "grad_norm": 10.651202201843262, + "learning_rate": 3.3051635489464793e-07, + "logits/chosen": 0.6889491081237793, + "logits/rejected": 0.6172829270362854, + "logps/chosen": -176.75820922851562, + "logps/ref_chosen": -62.12152862548828, + "logps/ref_rejected": -90.31204223632812, + "logps/rejected": -267.5221862792969, + "loss": 1.1054, + "margin_dpo/margin_mean": 62.57347106933594, + "margin_dpo/margin_std": 97.65402221679688, + "step": 303 + }, + { + "KL/chosen_KL_mean": -105.85643005371094, + "KL/mean": -137.82760620117188, + "KL/rejected_KL_mean": -169.7987518310547, + "KL/std": 80.725830078125, + "epoch": 0.4595616024187453, + "fcm_dpo/beta": 0.006954976357519627, + "fcm_dpo/delta": -0.033397287130355835, + "fcm_dpo/margin": 63.942317962646484, + "fcm_dpo/q_t": 0.3968748450279236, + "grad_norm": 11.252235412597656, + "learning_rate": 3.292634667444117e-07, + "logits/chosen": 0.7025733590126038, + "logits/rejected": 0.6516116261482239, + "logps/chosen": -166.551513671875, + "logps/ref_chosen": -60.695091247558594, + "logps/ref_rejected": -78.2525405883789, + "logps/rejected": -248.05130004882812, + "loss": 1.055, + "margin_dpo/margin_mean": 63.942317962646484, + "margin_dpo/margin_std": 75.22644805908203, + "step": 304 + }, + { + "KL/chosen_KL_mean": -121.06056213378906, + "KL/mean": -141.9627685546875, + "KL/rejected_KL_mean": -162.86495971679688, + "KL/std": 72.43345642089844, + "epoch": 0.46107331821617537, + "fcm_dpo/beta": 0.006954976357519627, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 41.804405212402344, + "fcm_dpo/q_t": 0.43231460452079773, + "grad_norm": 14.43248176574707, + "learning_rate": 3.280083614246217e-07, + "logits/chosen": 0.6767128705978394, + "logits/rejected": 0.7134938836097717, + "logps/chosen": -193.75970458984375, + "logps/ref_chosen": -72.69914245605469, + "logps/ref_rejected": -65.65670776367188, + "logps/rejected": -228.52166748046875, + "loss": 1.2077, + "margin_dpo/margin_mean": 41.804405212402344, + "margin_dpo/margin_std": 87.34265899658203, + "step": 305 + }, + { + "KL/chosen_KL_mean": -112.66262817382812, + "KL/mean": -136.23704528808594, + "KL/rejected_KL_mean": -159.81146240234375, + "KL/std": 70.7812728881836, + "epoch": 0.46258503401360546, + "fcm_dpo/beta": 0.006920297164469957, + "fcm_dpo/delta": -0.024993740022182465, + "fcm_dpo/margin": 47.14885330200195, + "fcm_dpo/q_t": 0.42409563064575195, + "grad_norm": 11.079917907714844, + "learning_rate": 3.267510740432719e-07, + "logits/chosen": 0.7989003658294678, + "logits/rejected": 0.6930177211761475, + "logps/chosen": -166.63314819335938, + "logps/ref_chosen": -53.97052764892578, + "logps/ref_rejected": -71.02423095703125, + "logps/rejected": -230.835693359375, + "loss": 1.1494, + "margin_dpo/margin_mean": 47.14885330200195, + "margin_dpo/margin_std": 72.32952117919922, + "step": 306 + }, + { + "KL/chosen_KL_mean": -109.00520324707031, + "KL/mean": -120.71246337890625, + "KL/rejected_KL_mean": -132.4197235107422, + "KL/std": 71.84052276611328, + "epoch": 0.46409674981103555, + "fcm_dpo/beta": 0.006920297164469957, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 23.41451644897461, + "fcm_dpo/q_t": 0.4631454348564148, + "grad_norm": 17.977399826049805, + "learning_rate": 3.2549163976939285e-07, + "logits/chosen": 0.7276726961135864, + "logits/rejected": 0.6820414066314697, + "logps/chosen": -166.41830444335938, + "logps/ref_chosen": -57.413108825683594, + "logps/ref_rejected": -68.68010711669922, + "logps/rejected": -201.09982299804688, + "loss": 1.326, + "margin_dpo/margin_mean": 23.41451644897461, + "margin_dpo/margin_std": 90.96375274658203, + "step": 307 + }, + { + "KL/chosen_KL_mean": -110.5123291015625, + "KL/mean": -133.32980346679688, + "KL/rejected_KL_mean": -156.14727783203125, + "KL/std": 74.56875610351562, + "epoch": 0.4656084656084656, + "fcm_dpo/beta": 0.006920297164469957, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 45.63495635986328, + "fcm_dpo/q_t": 0.42745688557624817, + "grad_norm": 10.50518798828125, + "learning_rate": 3.2423009383206874e-07, + "logits/chosen": 0.6881338953971863, + "logits/rejected": 0.6755229234695435, + "logps/chosen": -177.11111450195312, + "logps/ref_chosen": -66.59879302978516, + "logps/ref_rejected": -74.337158203125, + "logps/rejected": -230.48443603515625, + "loss": 1.1838, + "margin_dpo/margin_mean": 45.63495635986328, + "margin_dpo/margin_std": 87.10369873046875, + "step": 308 + }, + { + "KL/chosen_KL_mean": -123.3994140625, + "KL/mean": -147.40701293945312, + "KL/rejected_KL_mean": -171.41461181640625, + "KL/std": 72.49065399169922, + "epoch": 0.4671201814058957, + "fcm_dpo/beta": 0.006920297164469957, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 48.01519012451172, + "fcm_dpo/q_t": 0.424016535282135, + "grad_norm": 11.415412902832031, + "learning_rate": 3.229664715194511e-07, + "logits/chosen": 0.7687605619430542, + "logits/rejected": 0.7142656445503235, + "logps/chosen": -188.79415893554688, + "logps/ref_chosen": -65.39474487304688, + "logps/ref_rejected": -75.70930480957031, + "logps/rejected": -247.12391662597656, + "loss": 1.1677, + "margin_dpo/margin_mean": 48.01519012451172, + "margin_dpo/margin_std": 85.55278015136719, + "step": 309 + }, + { + "KL/chosen_KL_mean": -130.60060119628906, + "KL/mean": -144.18211364746094, + "KL/rejected_KL_mean": -157.76361083984375, + "KL/std": 66.98379516601562, + "epoch": 0.46863189720332576, + "fcm_dpo/beta": 0.006920297164469957, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 27.163013458251953, + "fcm_dpo/q_t": 0.4576417803764343, + "grad_norm": 12.827436447143555, + "learning_rate": 3.2170080817777257e-07, + "logits/chosen": 0.7497519254684448, + "logits/rejected": 0.7340184450149536, + "logps/chosen": -205.26889038085938, + "logps/ref_chosen": -74.66827392578125, + "logps/ref_rejected": -80.5689697265625, + "logps/rejected": -238.33258056640625, + "loss": 1.2808, + "margin_dpo/margin_mean": 27.163015365600586, + "margin_dpo/margin_std": 79.86006164550781, + "step": 310 + }, + { + "KL/chosen_KL_mean": -102.02139282226562, + "KL/mean": -128.64895629882812, + "KL/rejected_KL_mean": -155.27651977539062, + "KL/std": 84.87614440917969, + "epoch": 0.47014361300075586, + "fcm_dpo/beta": 0.006893772166222334, + "fcm_dpo/delta": -0.019201569259166718, + "fcm_dpo/margin": 53.25511169433594, + "fcm_dpo/q_t": 0.4192228317260742, + "grad_norm": 11.669011116027832, + "learning_rate": 3.204331392103574e-07, + "logits/chosen": 0.6330962777137756, + "logits/rejected": 0.49181753396987915, + "logps/chosen": -161.75942993164062, + "logps/ref_chosen": -59.738033294677734, + "logps/ref_rejected": -93.60757446289062, + "logps/rejected": -248.88409423828125, + "loss": 1.1386, + "margin_dpo/margin_mean": 53.25511169433594, + "margin_dpo/margin_std": 89.1590576171875, + "step": 311 + }, + { + "KL/chosen_KL_mean": -105.78706359863281, + "KL/mean": -135.12808227539062, + "KL/rejected_KL_mean": -164.46908569335938, + "KL/std": 74.64009094238281, + "epoch": 0.47165532879818595, + "fcm_dpo/beta": 0.006878282874822617, + "fcm_dpo/delta": -0.02251959592103958, + "fcm_dpo/margin": 58.682029724121094, + "fcm_dpo/q_t": 0.40607959032058716, + "grad_norm": 12.168621063232422, + "learning_rate": 3.1916350007663176e-07, + "logits/chosen": 0.7204309701919556, + "logits/rejected": 0.6345335245132446, + "logps/chosen": -159.60350036621094, + "logps/ref_chosen": -53.816436767578125, + "logps/ref_rejected": -68.6575698852539, + "logps/rejected": -233.12664794921875, + "loss": 1.0835, + "margin_dpo/margin_mean": 58.682029724121094, + "margin_dpo/margin_std": 73.37113952636719, + "step": 312 + }, + { + "KL/chosen_KL_mean": -112.03815460205078, + "KL/mean": -127.57806396484375, + "KL/rejected_KL_mean": -143.11798095703125, + "KL/std": 73.82379150390625, + "epoch": 0.47316704459561604, + "fcm_dpo/beta": 0.006862793117761612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 31.079830169677734, + "fcm_dpo/q_t": 0.45053255558013916, + "grad_norm": 13.539533615112305, + "learning_rate": 3.178919262911314e-07, + "logits/chosen": 0.7814310789108276, + "logits/rejected": 0.7609319686889648, + "logps/chosen": -171.99551391601562, + "logps/ref_chosen": -59.957359313964844, + "logps/ref_rejected": -69.31729888916016, + "logps/rejected": -212.43527221679688, + "loss": 1.2604, + "margin_dpo/margin_mean": 31.07982635498047, + "margin_dpo/margin_std": 82.09465026855469, + "step": 313 + }, + { + "KL/chosen_KL_mean": -102.45433807373047, + "KL/mean": -132.56947326660156, + "KL/rejected_KL_mean": -162.68460083007812, + "KL/std": 69.83338928222656, + "epoch": 0.47467876039304613, + "fcm_dpo/beta": 0.006821155082434416, + "fcm_dpo/delta": -0.030428625643253326, + "fcm_dpo/margin": 60.230255126953125, + "fcm_dpo/q_t": 0.4053559899330139, + "grad_norm": 12.13412094116211, + "learning_rate": 3.166184534225087e-07, + "logits/chosen": 0.7317984104156494, + "logits/rejected": 0.7632255554199219, + "logps/chosen": -172.72250366210938, + "logps/ref_chosen": -70.26815795898438, + "logps/ref_rejected": -69.23971557617188, + "logps/rejected": -231.92431640625, + "loss": 1.082, + "margin_dpo/margin_mean": 60.23025131225586, + "margin_dpo/margin_std": 75.92375183105469, + "step": 314 + }, + { + "KL/chosen_KL_mean": -108.26924896240234, + "KL/mean": -130.59422302246094, + "KL/rejected_KL_mean": -152.91920471191406, + "KL/std": 71.26750183105469, + "epoch": 0.47619047619047616, + "fcm_dpo/beta": 0.006821155082434416, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 44.64996337890625, + "fcm_dpo/q_t": 0.428594708442688, + "grad_norm": 11.18031120300293, + "learning_rate": 3.1534311709253723e-07, + "logits/chosen": 0.6409457921981812, + "logits/rejected": 0.6099203824996948, + "logps/chosen": -176.06393432617188, + "logps/ref_chosen": -67.79469299316406, + "logps/ref_rejected": -74.55148315429688, + "logps/rejected": -227.47068786621094, + "loss": 1.1727, + "margin_dpo/margin_mean": 44.64996337890625, + "margin_dpo/margin_std": 77.41693115234375, + "step": 315 + }, + { + "KL/chosen_KL_mean": -101.49205017089844, + "KL/mean": -135.0659942626953, + "KL/rejected_KL_mean": -168.63992309570312, + "KL/std": 76.8759765625, + "epoch": 0.47770219198790626, + "fcm_dpo/beta": 0.006735288072377443, + "fcm_dpo/delta": -0.12749513983726501, + "fcm_dpo/margin": 67.14787292480469, + "fcm_dpo/q_t": 0.39548853039741516, + "grad_norm": 11.79714298248291, + "learning_rate": 3.1406595297511564e-07, + "logits/chosen": 0.617354691028595, + "logits/rejected": 0.480925977230072, + "logps/chosen": -156.78053283691406, + "logps/ref_chosen": -55.288482666015625, + "logps/ref_rejected": -96.15723419189453, + "logps/rejected": -264.79718017578125, + "loss": 1.0558, + "margin_dpo/margin_mean": 67.14787292480469, + "margin_dpo/margin_std": 76.20933532714844, + "step": 316 + }, + { + "KL/chosen_KL_mean": -91.71333312988281, + "KL/mean": -122.70863342285156, + "KL/rejected_KL_mean": -153.70391845703125, + "KL/std": 68.813720703125, + "epoch": 0.47921390778533635, + "fcm_dpo/beta": 0.006600792054086924, + "fcm_dpo/delta": -0.03670065850019455, + "fcm_dpo/margin": 61.99058532714844, + "fcm_dpo/q_t": 0.4032962918281555, + "grad_norm": 11.325054168701172, + "learning_rate": 3.1278699679526975e-07, + "logits/chosen": 0.7354048490524292, + "logits/rejected": 0.6916720271110535, + "logps/chosen": -146.29470825195312, + "logps/ref_chosen": -54.58137512207031, + "logps/ref_rejected": -72.77232360839844, + "logps/rejected": -226.47625732421875, + "loss": 1.0793, + "margin_dpo/margin_mean": 61.99058532714844, + "margin_dpo/margin_std": 76.21528625488281, + "step": 317 + }, + { + "KL/chosen_KL_mean": -103.02191162109375, + "KL/mean": -125.77468872070312, + "KL/rejected_KL_mean": -148.5274658203125, + "KL/std": 74.2537841796875, + "epoch": 0.48072562358276644, + "fcm_dpo/beta": 0.006600792054086924, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 45.50556182861328, + "fcm_dpo/q_t": 0.4332531690597534, + "grad_norm": 11.361948013305664, + "learning_rate": 3.1150628432815336e-07, + "logits/chosen": 0.7207176685333252, + "logits/rejected": 0.6528332233428955, + "logps/chosen": -155.91014099121094, + "logps/ref_chosen": -52.88822937011719, + "logps/ref_rejected": -80.63988494873047, + "logps/rejected": -229.1673583984375, + "loss": 1.2132, + "margin_dpo/margin_mean": 45.50556182861328, + "margin_dpo/margin_std": 101.61717224121094, + "step": 318 + }, + { + "KL/chosen_KL_mean": -101.49418640136719, + "KL/mean": -133.5969696044922, + "KL/rejected_KL_mean": -165.69973754882812, + "KL/std": 78.28202056884766, + "epoch": 0.48223733938019653, + "fcm_dpo/beta": 0.006600792054086924, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 64.20555877685547, + "fcm_dpo/q_t": 0.40408360958099365, + "grad_norm": 10.197602272033691, + "learning_rate": 3.1022385139804707e-07, + "logits/chosen": 0.6491215229034424, + "logits/rejected": 0.6349847316741943, + "logps/chosen": -165.85751342773438, + "logps/ref_chosen": -64.36333465576172, + "logps/ref_rejected": -79.47296142578125, + "logps/rejected": -245.17269897460938, + "loss": 1.0897, + "margin_dpo/margin_mean": 64.20555877685547, + "margin_dpo/margin_std": 90.96015930175781, + "step": 319 + }, + { + "KL/chosen_KL_mean": -91.69598388671875, + "KL/mean": -118.007568359375, + "KL/rejected_KL_mean": -144.3191680908203, + "KL/std": 79.62506103515625, + "epoch": 0.4837490551776266, + "fcm_dpo/beta": 0.0065584946423769, + "fcm_dpo/delta": -0.0644935593008995, + "fcm_dpo/margin": 52.6231803894043, + "fcm_dpo/q_t": 0.4222339987754822, + "grad_norm": 11.063246726989746, + "learning_rate": 3.0893973387735683e-07, + "logits/chosen": 0.6024397611618042, + "logits/rejected": 0.5657603144645691, + "logps/chosen": -141.25473022460938, + "logps/ref_chosen": -49.558746337890625, + "logps/ref_rejected": -71.23444366455078, + "logps/rejected": -215.55360412597656, + "loss": 1.1494, + "margin_dpo/margin_mean": 52.6231803894043, + "margin_dpo/margin_std": 84.94557189941406, + "step": 320 + }, + { + "KL/chosen_KL_mean": -95.55155944824219, + "KL/mean": -123.66300201416016, + "KL/rejected_KL_mean": -151.77444458007812, + "KL/std": 67.8209228515625, + "epoch": 0.4852607709750567, + "fcm_dpo/beta": 0.006455008406192064, + "fcm_dpo/delta": -0.04717318341135979, + "fcm_dpo/margin": 56.22286605834961, + "fcm_dpo/q_t": 0.4145912230014801, + "grad_norm": 15.530342102050781, + "learning_rate": 3.0765396768561004e-07, + "logits/chosen": 0.6982482671737671, + "logits/rejected": 0.6813958883285522, + "logps/chosen": -147.6368408203125, + "logps/ref_chosen": -52.08526611328125, + "logps/ref_rejected": -55.58674621582031, + "logps/rejected": -207.36117553710938, + "loss": 1.1252, + "margin_dpo/margin_mean": 56.222869873046875, + "margin_dpo/margin_std": 80.80537414550781, + "step": 321 + }, + { + "KL/chosen_KL_mean": -109.41558837890625, + "KL/mean": -143.95343017578125, + "KL/rejected_KL_mean": -178.49130249023438, + "KL/std": 77.21576690673828, + "epoch": 0.48677248677248675, + "fcm_dpo/beta": 0.006419917568564415, + "fcm_dpo/delta": -0.05466010794043541, + "fcm_dpo/margin": 69.07571411132812, + "fcm_dpo/q_t": 0.3954285979270935, + "grad_norm": 9.539613723754883, + "learning_rate": 3.063665887884511e-07, + "logits/chosen": 0.7315066456794739, + "logits/rejected": 0.6500815749168396, + "logps/chosen": -156.8197021484375, + "logps/ref_chosen": -47.404109954833984, + "logps/ref_rejected": -73.4260025024414, + "logps/rejected": -251.91729736328125, + "loss": 1.0462, + "margin_dpo/margin_mean": 69.07571411132812, + "margin_dpo/margin_std": 74.04450988769531, + "step": 322 + }, + { + "KL/chosen_KL_mean": -110.42620849609375, + "KL/mean": -132.9568634033203, + "KL/rejected_KL_mean": -155.48751831054688, + "KL/std": 73.69956970214844, + "epoch": 0.48828420256991684, + "fcm_dpo/beta": 0.006384826265275478, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 45.061317443847656, + "fcm_dpo/q_t": 0.43496036529541016, + "grad_norm": 12.915155410766602, + "learning_rate": 3.0507763319663517e-07, + "logits/chosen": 0.6116001605987549, + "logits/rejected": 0.5418746471405029, + "logps/chosen": -180.4324951171875, + "logps/ref_chosen": -70.00630187988281, + "logps/ref_rejected": -86.96690368652344, + "logps/rejected": -242.4544219970703, + "loss": 1.2044, + "margin_dpo/margin_mean": 45.061317443847656, + "margin_dpo/margin_std": 94.1461181640625, + "step": 323 + }, + { + "KL/chosen_KL_mean": -98.17274475097656, + "KL/mean": -127.4241943359375, + "KL/rejected_KL_mean": -156.67564392089844, + "KL/std": 83.87608337402344, + "epoch": 0.4897959183673469, + "fcm_dpo/beta": 0.006350713782012463, + "fcm_dpo/delta": -0.02678520791232586, + "fcm_dpo/margin": 58.502906799316406, + "fcm_dpo/q_t": 0.41296952962875366, + "grad_norm": 14.295312881469727, + "learning_rate": 3.0378713696502097e-07, + "logits/chosen": 0.7219698429107666, + "logits/rejected": 0.665217399597168, + "logps/chosen": -154.06155395507812, + "logps/ref_chosen": -55.88882064819336, + "logps/ref_rejected": -75.23088073730469, + "logps/rejected": -231.90652465820312, + "loss": 1.1068, + "margin_dpo/margin_mean": 58.502906799316406, + "margin_dpo/margin_std": 76.33798217773438, + "step": 324 + }, + { + "KL/chosen_KL_mean": -118.92147827148438, + "KL/mean": -148.49241638183594, + "KL/rejected_KL_mean": -178.0633544921875, + "KL/std": 82.98800659179688, + "epoch": 0.491307634164777, + "fcm_dpo/beta": 0.006350713782012463, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 59.141883850097656, + "fcm_dpo/q_t": 0.41542884707450867, + "grad_norm": 10.648490905761719, + "learning_rate": 3.0249513619156206e-07, + "logits/chosen": 0.6718249320983887, + "logits/rejected": 0.6107696294784546, + "logps/chosen": -183.06849670410156, + "logps/ref_chosen": -64.14701843261719, + "logps/ref_rejected": -79.91143798828125, + "logps/rejected": -257.97479248046875, + "loss": 1.1377, + "margin_dpo/margin_mean": 59.141883850097656, + "margin_dpo/margin_std": 100.32028198242188, + "step": 325 + }, + { + "KL/chosen_KL_mean": -140.14584350585938, + "KL/mean": -151.37425231933594, + "KL/rejected_KL_mean": -162.6026611328125, + "KL/std": 75.11209869384766, + "epoch": 0.4928193499622071, + "fcm_dpo/beta": 0.006350713782012463, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 22.45681381225586, + "fcm_dpo/q_t": 0.46714678406715393, + "grad_norm": 11.448811531066895, + "learning_rate": 3.012016670162977e-07, + "logits/chosen": 0.6767762899398804, + "logits/rejected": 0.6858989596366882, + "logps/chosen": -215.67715454101562, + "logps/ref_chosen": -75.53131103515625, + "logps/ref_rejected": -76.5898666381836, + "logps/rejected": -239.19252014160156, + "loss": 1.3198, + "margin_dpo/margin_mean": 22.45681381225586, + "margin_dpo/margin_std": 85.329833984375, + "step": 326 + }, + { + "KL/chosen_KL_mean": -123.46060180664062, + "KL/mean": -146.76063537597656, + "KL/rejected_KL_mean": -170.06063842773438, + "KL/std": 80.32379913330078, + "epoch": 0.4943310657596372, + "fcm_dpo/beta": 0.006350713782012463, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 46.600040435791016, + "fcm_dpo/q_t": 0.4317762851715088, + "grad_norm": 11.476740837097168, + "learning_rate": 2.99906765620341e-07, + "logits/chosen": 0.6259051561355591, + "logits/rejected": 0.6025689840316772, + "logps/chosen": -192.7977752685547, + "logps/ref_chosen": -69.33717346191406, + "logps/ref_rejected": -73.37751770019531, + "logps/rejected": -243.4381561279297, + "loss": 1.1923, + "margin_dpo/margin_mean": 46.60003662109375, + "margin_dpo/margin_std": 91.36933135986328, + "step": 327 + }, + { + "KL/chosen_KL_mean": -108.52227783203125, + "KL/mean": -137.43905639648438, + "KL/rejected_KL_mean": -166.35581970214844, + "KL/std": 80.34719848632812, + "epoch": 0.4958427815570673, + "fcm_dpo/beta": 0.006350713782012463, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.833534240722656, + "fcm_dpo/q_t": 0.4146382510662079, + "grad_norm": 10.102066993713379, + "learning_rate": 2.9861046822486766e-07, + "logits/chosen": 0.5871948003768921, + "logits/rejected": 0.5468528270721436, + "logps/chosen": -170.228515625, + "logps/ref_chosen": -61.70623016357422, + "logps/ref_rejected": -83.73808288574219, + "logps/rejected": -250.09390258789062, + "loss": 1.1094, + "margin_dpo/margin_mean": 57.833534240722656, + "margin_dpo/margin_std": 76.56809997558594, + "step": 328 + }, + { + "KL/chosen_KL_mean": -120.30358123779297, + "KL/mean": -149.27682495117188, + "KL/rejected_KL_mean": -178.25009155273438, + "KL/std": 86.12091064453125, + "epoch": 0.4973544973544973, + "fcm_dpo/beta": 0.006350713782012463, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.946502685546875, + "fcm_dpo/q_t": 0.41649869084358215, + "grad_norm": 12.654784202575684, + "learning_rate": 2.9731281109010253e-07, + "logits/chosen": 0.729865550994873, + "logits/rejected": 0.6701647043228149, + "logps/chosen": -184.802001953125, + "logps/ref_chosen": -64.4984130859375, + "logps/ref_rejected": -83.6591796875, + "logps/rejected": -261.9092712402344, + "loss": 1.1267, + "margin_dpo/margin_mean": 57.94650650024414, + "margin_dpo/margin_std": 89.17796325683594, + "step": 329 + }, + { + "KL/chosen_KL_mean": -103.599609375, + "KL/mean": -132.35592651367188, + "KL/rejected_KL_mean": -161.11224365234375, + "KL/std": 77.50123596191406, + "epoch": 0.4988662131519274, + "fcm_dpo/beta": 0.006339719519019127, + "fcm_dpo/delta": -0.01734209805727005, + "fcm_dpo/margin": 57.51264190673828, + "fcm_dpo/q_t": 0.416418194770813, + "grad_norm": 11.395146369934082, + "learning_rate": 2.9601383051430505e-07, + "logits/chosen": 0.7071419954299927, + "logits/rejected": 0.6409908533096313, + "logps/chosen": -158.4042510986328, + "logps/ref_chosen": -54.80464172363281, + "logps/ref_rejected": -75.3194351196289, + "logps/rejected": -236.43167114257812, + "loss": 1.1494, + "margin_dpo/margin_mean": 57.51264572143555, + "margin_dpo/margin_std": 98.8066635131836, + "step": 330 + }, + { + "KL/chosen_KL_mean": -112.81170654296875, + "KL/mean": -148.69761657714844, + "KL/rejected_KL_mean": -184.58352661132812, + "KL/std": 81.27599334716797, + "epoch": 0.5003779289493575, + "fcm_dpo/beta": 0.006279571447521448, + "fcm_dpo/delta": -0.053083788603544235, + "fcm_dpo/margin": 71.77181243896484, + "fcm_dpo/q_t": 0.39528989791870117, + "grad_norm": 11.644200325012207, + "learning_rate": 2.947135628327544e-07, + "logits/chosen": 0.7728036642074585, + "logits/rejected": 0.7491520643234253, + "logps/chosen": -172.05429077148438, + "logps/ref_chosen": -59.242584228515625, + "logps/ref_rejected": -69.87483215332031, + "logps/rejected": -254.45835876464844, + "loss": 1.0565, + "margin_dpo/margin_mean": 71.77180480957031, + "margin_dpo/margin_std": 87.65985107421875, + "step": 331 + }, + { + "KL/chosen_KL_mean": -117.23881530761719, + "KL/mean": -147.25274658203125, + "KL/rejected_KL_mean": -177.26669311523438, + "KL/std": 79.46280670166016, + "epoch": 0.5018896447467877, + "fcm_dpo/beta": 0.00623284000903368, + "fcm_dpo/delta": -0.04660751298069954, + "fcm_dpo/margin": 60.02788543701172, + "fcm_dpo/q_t": 0.41367125511169434, + "grad_norm": 10.473052978515625, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": 0.6207424402236938, + "logits/rejected": 0.5803670883178711, + "logps/chosen": -184.34857177734375, + "logps/ref_chosen": -67.10975646972656, + "logps/ref_rejected": -77.11839294433594, + "logps/rejected": -254.3850860595703, + "loss": 1.1111, + "margin_dpo/margin_mean": 60.02788162231445, + "margin_dpo/margin_std": 81.86831665039062, + "step": 332 + }, + { + "KL/chosen_KL_mean": -125.28157043457031, + "KL/mean": -159.71221923828125, + "KL/rejected_KL_mean": -194.1428680419922, + "KL/std": 81.67819213867188, + "epoch": 0.5034013605442177, + "fcm_dpo/beta": 0.006189778447151184, + "fcm_dpo/delta": -0.022637760266661644, + "fcm_dpo/margin": 68.86128234863281, + "fcm_dpo/q_t": 0.4018930196762085, + "grad_norm": 9.133350372314453, + "learning_rate": 2.921093116725076e-07, + "logits/chosen": 0.712902307510376, + "logits/rejected": 0.6342633962631226, + "logps/chosen": -183.66270446777344, + "logps/ref_chosen": -58.381134033203125, + "logps/ref_rejected": -85.02839660644531, + "logps/rejected": -279.1712646484375, + "loss": 1.0697, + "margin_dpo/margin_mean": 68.86128234863281, + "margin_dpo/margin_std": 85.3521728515625, + "step": 333 + }, + { + "KL/chosen_KL_mean": -119.81755828857422, + "KL/mean": -144.76309204101562, + "KL/rejected_KL_mean": -169.7086181640625, + "KL/std": 80.81599426269531, + "epoch": 0.5049130763416477, + "fcm_dpo/beta": 0.006175766233354807, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 49.89105987548828, + "fcm_dpo/q_t": 0.4292786717414856, + "grad_norm": 12.244359016418457, + "learning_rate": 2.9080540104031484e-07, + "logits/chosen": 0.7428555488586426, + "logits/rejected": 0.6925406455993652, + "logps/chosen": -186.70956420898438, + "logps/ref_chosen": -66.89199829101562, + "logps/ref_rejected": -91.83695220947266, + "logps/rejected": -261.5455627441406, + "loss": 1.1793, + "margin_dpo/margin_mean": 49.89106369018555, + "margin_dpo/margin_std": 92.7274169921875, + "step": 334 + }, + { + "KL/chosen_KL_mean": -120.25526428222656, + "KL/mean": -147.18856811523438, + "KL/rejected_KL_mean": -174.12188720703125, + "KL/std": 80.76295471191406, + "epoch": 0.5064247921390779, + "fcm_dpo/beta": 0.006175766233354807, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 53.86663055419922, + "fcm_dpo/q_t": 0.4242987036705017, + "grad_norm": 14.082091331481934, + "learning_rate": 2.895003489933375e-07, + "logits/chosen": 0.7074752449989319, + "logits/rejected": 0.6703487634658813, + "logps/chosen": -181.76971435546875, + "logps/ref_chosen": -61.51445770263672, + "logps/ref_rejected": -75.68916320800781, + "logps/rejected": -249.81105041503906, + "loss": 1.1612, + "margin_dpo/margin_mean": 53.86663055419922, + "margin_dpo/margin_std": 93.7869873046875, + "step": 335 + }, + { + "KL/chosen_KL_mean": -135.12388610839844, + "KL/mean": -164.06655883789062, + "KL/rejected_KL_mean": -193.00921630859375, + "KL/std": 87.16038513183594, + "epoch": 0.5079365079365079, + "fcm_dpo/beta": 0.006122264079749584, + "fcm_dpo/delta": -0.043504975736141205, + "fcm_dpo/margin": 57.88533401489258, + "fcm_dpo/q_t": 0.4214656949043274, + "grad_norm": 11.543611526489258, + "learning_rate": 2.8819419203668675e-07, + "logits/chosen": 0.6747065186500549, + "logits/rejected": 0.6493879556655884, + "logps/chosen": -203.9739532470703, + "logps/ref_chosen": -68.85006713867188, + "logps/ref_rejected": -92.99603271484375, + "logps/rejected": -286.0052490234375, + "loss": 1.1491, + "margin_dpo/margin_mean": 57.88533401489258, + "margin_dpo/margin_std": 97.946044921875, + "step": 336 + }, + { + "KL/chosen_KL_mean": -143.09075927734375, + "KL/mean": -166.30104064941406, + "KL/rejected_KL_mean": -189.51132202148438, + "KL/std": 86.70985412597656, + "epoch": 0.509448223733938, + "fcm_dpo/beta": 0.006122264079749584, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 46.420562744140625, + "fcm_dpo/q_t": 0.4345945119857788, + "grad_norm": 10.347283363342285, + "learning_rate": 2.8688696670638053e-07, + "logits/chosen": 0.596039891242981, + "logits/rejected": 0.568123459815979, + "logps/chosen": -216.27859497070312, + "logps/ref_chosen": -73.18783569335938, + "logps/ref_rejected": -86.89118957519531, + "logps/rejected": -276.40252685546875, + "loss": 1.1964, + "margin_dpo/margin_mean": 46.420562744140625, + "margin_dpo/margin_std": 91.3853759765625, + "step": 337 + }, + { + "KL/chosen_KL_mean": -139.84075927734375, + "KL/mean": -164.1468505859375, + "KL/rejected_KL_mean": -188.4529266357422, + "KL/std": 78.78346252441406, + "epoch": 0.5109599395313681, + "fcm_dpo/beta": 0.006122264079749584, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 48.612159729003906, + "fcm_dpo/q_t": 0.4303521513938904, + "grad_norm": 11.543200492858887, + "learning_rate": 2.8557870956832133e-07, + "logits/chosen": 0.6580469608306885, + "logits/rejected": 0.6333959102630615, + "logps/chosen": -203.78038024902344, + "logps/ref_chosen": -63.939613342285156, + "logps/ref_rejected": -75.34243774414062, + "logps/rejected": -263.79534912109375, + "loss": 1.1781, + "margin_dpo/margin_mean": 48.612159729003906, + "margin_dpo/margin_std": 86.77371215820312, + "step": 338 + }, + { + "KL/chosen_KL_mean": -114.18165588378906, + "KL/mean": -143.21810913085938, + "KL/rejected_KL_mean": -172.2545928955078, + "KL/std": 81.62635803222656, + "epoch": 0.5124716553287982, + "fcm_dpo/beta": 0.006122264079749584, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 58.07294464111328, + "fcm_dpo/q_t": 0.41747236251831055, + "grad_norm": 12.970744132995605, + "learning_rate": 2.842694572172736e-07, + "logits/chosen": 0.8276938199996948, + "logits/rejected": 0.7435949444770813, + "logps/chosen": -159.73077392578125, + "logps/ref_chosen": -45.54913330078125, + "logps/ref_rejected": -67.0482177734375, + "logps/rejected": -239.3028106689453, + "loss": 1.1311, + "margin_dpo/margin_mean": 58.07294464111328, + "margin_dpo/margin_std": 86.94050598144531, + "step": 339 + }, + { + "KL/chosen_KL_mean": -130.46449279785156, + "KL/mean": -157.4834747314453, + "KL/rejected_KL_mean": -184.50247192382812, + "KL/std": 86.1899642944336, + "epoch": 0.5139833711262283, + "fcm_dpo/beta": 0.006122264079749584, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 54.03797149658203, + "fcm_dpo/q_t": 0.42464643716812134, + "grad_norm": 11.059535026550293, + "learning_rate": 2.8295924627584004e-07, + "logits/chosen": 0.7042380571365356, + "logits/rejected": 0.6846690773963928, + "logps/chosen": -184.4701385498047, + "logps/ref_chosen": -54.00564956665039, + "logps/ref_rejected": -61.314430236816406, + "logps/rejected": -245.81689453125, + "loss": 1.1738, + "margin_dpo/margin_mean": 54.03797149658203, + "margin_dpo/margin_std": 100.92936706542969, + "step": 340 + }, + { + "KL/chosen_KL_mean": -133.33224487304688, + "KL/mean": -164.9116668701172, + "KL/rejected_KL_mean": -196.4910888671875, + "KL/std": 87.80421447753906, + "epoch": 0.5154950869236583, + "fcm_dpo/beta": 0.006004424300044775, + "fcm_dpo/delta": -0.0971769466996193, + "fcm_dpo/margin": 63.15885925292969, + "fcm_dpo/q_t": 0.4124714434146881, + "grad_norm": 12.540910720825195, + "learning_rate": 2.816481133934373e-07, + "logits/chosen": 0.724452555179596, + "logits/rejected": 0.6779919266700745, + "logps/chosen": -196.72732543945312, + "logps/ref_chosen": -63.39509582519531, + "logps/ref_rejected": -76.20973205566406, + "logps/rejected": -272.7008056640625, + "loss": 1.1267, + "margin_dpo/margin_mean": 63.158851623535156, + "margin_dpo/margin_std": 93.57270812988281, + "step": 341 + }, + { + "KL/chosen_KL_mean": -130.59140014648438, + "KL/mean": -160.70944213867188, + "KL/rejected_KL_mean": -190.82748413085938, + "KL/std": 92.62532043457031, + "epoch": 0.5170068027210885, + "fcm_dpo/beta": 0.006004424300044775, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 60.236087799072266, + "fcm_dpo/q_t": 0.41768157482147217, + "grad_norm": 9.833003044128418, + "learning_rate": 2.8033609524527046e-07, + "logits/chosen": 0.7610205411911011, + "logits/rejected": 0.7204849123954773, + "logps/chosen": -183.6392059326172, + "logps/ref_chosen": -53.047813415527344, + "logps/ref_rejected": -68.2854232788086, + "logps/rejected": -259.1129150390625, + "loss": 1.1352, + "margin_dpo/margin_mean": 60.236083984375, + "margin_dpo/margin_std": 95.39666748046875, + "step": 342 + }, + { + "KL/chosen_KL_mean": -125.97956085205078, + "KL/mean": -149.21401977539062, + "KL/rejected_KL_mean": -172.448486328125, + "KL/std": 85.8572006225586, + "epoch": 0.5185185185185185, + "fcm_dpo/beta": 0.006004424300044775, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 46.46892166137695, + "fcm_dpo/q_t": 0.4357026219367981, + "grad_norm": 9.35943603515625, + "learning_rate": 2.7902322853130753e-07, + "logits/chosen": 0.5916262865066528, + "logits/rejected": 0.5824925899505615, + "logps/chosen": -196.55809020996094, + "logps/ref_chosen": -70.57852935791016, + "logps/ref_rejected": -84.73873901367188, + "logps/rejected": -257.1872253417969, + "loss": 1.1976, + "margin_dpo/margin_mean": 46.46892547607422, + "margin_dpo/margin_std": 89.41838073730469, + "step": 343 + }, + { + "KL/chosen_KL_mean": -138.49673461914062, + "KL/mean": -171.74130249023438, + "KL/rejected_KL_mean": -204.98583984375, + "KL/std": 87.63685607910156, + "epoch": 0.5200302343159486, + "fcm_dpo/beta": 0.006004424300044775, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.48912048339844, + "fcm_dpo/q_t": 0.4096273183822632, + "grad_norm": 11.909037590026855, + "learning_rate": 2.7770954997525274e-07, + "logits/chosen": 0.7216250896453857, + "logits/rejected": 0.6522601842880249, + "logps/chosen": -194.3077392578125, + "logps/ref_chosen": -55.811004638671875, + "logps/ref_rejected": -84.77637481689453, + "logps/rejected": -289.76220703125, + "loss": 1.1151, + "margin_dpo/margin_mean": 66.48912048339844, + "margin_dpo/margin_std": 102.74324798583984, + "step": 344 + }, + { + "KL/chosen_KL_mean": -114.75037384033203, + "KL/mean": -144.64398193359375, + "KL/rejected_KL_mean": -174.53756713867188, + "KL/std": 80.88868713378906, + "epoch": 0.5215419501133787, + "fcm_dpo/beta": 0.006004424300044775, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 59.787208557128906, + "fcm_dpo/q_t": 0.4173477292060852, + "grad_norm": 11.631264686584473, + "learning_rate": 2.7639509632351927e-07, + "logits/chosen": 0.7710617780685425, + "logits/rejected": 0.7289673686027527, + "logps/chosen": -172.53646850585938, + "logps/ref_chosen": -57.78609848022461, + "logps/ref_rejected": -78.91847229003906, + "logps/rejected": -253.4560546875, + "loss": 1.1366, + "margin_dpo/margin_mean": 59.787208557128906, + "margin_dpo/margin_std": 95.0912094116211, + "step": 345 + }, + { + "KL/chosen_KL_mean": -120.35940551757812, + "KL/mean": -156.9630126953125, + "KL/rejected_KL_mean": -193.56661987304688, + "KL/std": 94.83656311035156, + "epoch": 0.5230536659108088, + "fcm_dpo/beta": 0.005976714193820953, + "fcm_dpo/delta": -0.04636390507221222, + "fcm_dpo/margin": 73.20722198486328, + "fcm_dpo/q_t": 0.3988511264324188, + "grad_norm": 11.821297645568848, + "learning_rate": 2.7507990434420123e-07, + "logits/chosen": 0.7447773218154907, + "logits/rejected": 0.6635010242462158, + "logps/chosen": -176.64453125, + "logps/ref_chosen": -56.285125732421875, + "logps/ref_rejected": -91.15303039550781, + "logps/rejected": -284.71966552734375, + "loss": 1.0786, + "margin_dpo/margin_mean": 73.20722961425781, + "margin_dpo/margin_std": 97.56934356689453, + "step": 346 + }, + { + "KL/chosen_KL_mean": -133.1365966796875, + "KL/mean": -161.14346313476562, + "KL/rejected_KL_mean": -189.15032958984375, + "KL/std": 87.99459838867188, + "epoch": 0.5245653817082389, + "fcm_dpo/beta": 0.005949004087597132, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 56.01373291015625, + "fcm_dpo/q_t": 0.42295533418655396, + "grad_norm": 10.870003700256348, + "learning_rate": 2.737640108260456e-07, + "logits/chosen": 0.8249450922012329, + "logits/rejected": 0.7794303894042969, + "logps/chosen": -186.63613891601562, + "logps/ref_chosen": -53.499542236328125, + "logps/ref_rejected": -72.52565002441406, + "logps/rejected": -261.6759948730469, + "loss": 1.146, + "margin_dpo/margin_mean": 56.01373291015625, + "margin_dpo/margin_std": 88.20159149169922, + "step": 347 + }, + { + "KL/chosen_KL_mean": -120.46863555908203, + "KL/mean": -151.01657104492188, + "KL/rejected_KL_mean": -181.5644989013672, + "KL/std": 82.77992248535156, + "epoch": 0.5260770975056689, + "fcm_dpo/beta": 0.005949004087597132, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.095863342285156, + "fcm_dpo/q_t": 0.4175592064857483, + "grad_norm": 12.275740623474121, + "learning_rate": 2.724474525774229e-07, + "logits/chosen": 0.7986717224121094, + "logits/rejected": 0.7687997817993164, + "logps/chosen": -171.2554931640625, + "logps/ref_chosen": -50.78684997558594, + "logps/ref_rejected": -68.63732147216797, + "logps/rejected": -250.20181274414062, + "loss": 1.1344, + "margin_dpo/margin_mean": 61.095863342285156, + "margin_dpo/margin_std": 97.12753295898438, + "step": 348 + }, + { + "KL/chosen_KL_mean": -122.85971069335938, + "KL/mean": -155.31509399414062, + "KL/rejected_KL_mean": -187.77047729492188, + "KL/std": 93.95545196533203, + "epoch": 0.527588813303099, + "fcm_dpo/beta": 0.005926626734435558, + "fcm_dpo/delta": -0.03775714710354805, + "fcm_dpo/margin": 64.9107666015625, + "fcm_dpo/q_t": 0.41291543841362, + "grad_norm": 10.789410591125488, + "learning_rate": 2.711302664252973e-07, + "logits/chosen": 0.8072630167007446, + "logits/rejected": 0.7103030681610107, + "logps/chosen": -176.18472290039062, + "logps/ref_chosen": -53.325008392333984, + "logps/ref_rejected": -83.21236419677734, + "logps/rejected": -270.98284912109375, + "loss": 1.1198, + "margin_dpo/margin_mean": 64.9107666015625, + "margin_dpo/margin_std": 98.733154296875, + "step": 349 + }, + { + "KL/chosen_KL_mean": -125.94990539550781, + "KL/mean": -169.85733032226562, + "KL/rejected_KL_mean": -213.7647705078125, + "KL/std": 99.48991394042969, + "epoch": 0.5291005291005291, + "fcm_dpo/beta": 0.005794004071503878, + "fcm_dpo/delta": -0.11490027606487274, + "fcm_dpo/margin": 87.81488037109375, + "fcm_dpo/q_t": 0.3824026882648468, + "grad_norm": 13.445771217346191, + "learning_rate": 2.698124892141971e-07, + "logits/chosen": 0.7362804412841797, + "logits/rejected": 0.6689400672912598, + "logps/chosen": -187.57566833496094, + "logps/ref_chosen": -61.625770568847656, + "logps/ref_rejected": -87.63627624511719, + "logps/rejected": -301.4010314941406, + "loss": 1.0084, + "margin_dpo/margin_mean": 87.81488037109375, + "margin_dpo/margin_std": 93.23359680175781, + "step": 350 + }, + { + "KL/chosen_KL_mean": -121.92890930175781, + "KL/mean": -152.43846130371094, + "KL/rejected_KL_mean": -182.94802856445312, + "KL/std": 83.26048278808594, + "epoch": 0.5306122448979592, + "fcm_dpo/beta": 0.0057701170444488525, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.019126892089844, + "fcm_dpo/q_t": 0.41859257221221924, + "grad_norm": 11.20499324798584, + "learning_rate": 2.6849415780518357e-07, + "logits/chosen": 0.690535306930542, + "logits/rejected": 0.6209006905555725, + "logps/chosen": -178.18524169921875, + "logps/ref_chosen": -56.2563362121582, + "logps/ref_rejected": -79.11589813232422, + "logps/rejected": -262.06390380859375, + "loss": 1.1553, + "margin_dpo/margin_mean": 61.019126892089844, + "margin_dpo/margin_std": 106.53633880615234, + "step": 351 + }, + { + "KL/chosen_KL_mean": -126.20106506347656, + "KL/mean": -160.4630889892578, + "KL/rejected_KL_mean": -194.72511291503906, + "KL/std": 94.95686340332031, + "epoch": 0.5321239606953893, + "fcm_dpo/beta": 0.005735332146286964, + "fcm_dpo/delta": -0.03023368865251541, + "fcm_dpo/margin": 68.52404022216797, + "fcm_dpo/q_t": 0.4111440181732178, + "grad_norm": 10.544001579284668, + "learning_rate": 2.6717530907482024e-07, + "logits/chosen": 0.763157069683075, + "logits/rejected": 0.7131781578063965, + "logps/chosen": -189.25302124023438, + "logps/ref_chosen": -63.05195236206055, + "logps/ref_rejected": -85.52035522460938, + "logps/rejected": -280.2454833984375, + "loss": 1.1116, + "margin_dpo/margin_mean": 68.52404022216797, + "margin_dpo/margin_std": 102.7152099609375, + "step": 352 + }, + { + "KL/chosen_KL_mean": -124.33663940429688, + "KL/mean": -155.11465454101562, + "KL/rejected_KL_mean": -185.89266967773438, + "KL/std": 85.38116455078125, + "epoch": 0.5336356764928194, + "fcm_dpo/beta": 0.005735332146286964, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.55604934692383, + "fcm_dpo/q_t": 0.41740575432777405, + "grad_norm": 11.076839447021484, + "learning_rate": 2.658559799141411e-07, + "logits/chosen": 0.7371421456336975, + "logits/rejected": 0.7465870976448059, + "logps/chosen": -193.3458251953125, + "logps/ref_chosen": -69.00918579101562, + "logps/ref_rejected": -72.65840148925781, + "logps/rejected": -258.55108642578125, + "loss": 1.1249, + "margin_dpo/margin_mean": 61.556053161621094, + "margin_dpo/margin_std": 87.54644012451172, + "step": 353 + }, + { + "KL/chosen_KL_mean": -121.45633697509766, + "KL/mean": -158.8951873779297, + "KL/rejected_KL_mean": -196.33404541015625, + "KL/std": 80.35975646972656, + "epoch": 0.5351473922902494, + "fcm_dpo/beta": 0.005694113671779633, + "fcm_dpo/delta": -0.036063361912965775, + "fcm_dpo/margin": 74.8777084350586, + "fcm_dpo/q_t": 0.40101897716522217, + "grad_norm": 14.158194541931152, + "learning_rate": 2.6453620722761895e-07, + "logits/chosen": 0.8556749820709229, + "logits/rejected": 0.7295167446136475, + "logps/chosen": -161.2446746826172, + "logps/ref_chosen": -39.78833770751953, + "logps/ref_rejected": -69.56885528564453, + "logps/rejected": -265.90289306640625, + "loss": 1.0787, + "margin_dpo/margin_mean": 74.87770080566406, + "margin_dpo/margin_std": 97.76545715332031, + "step": 354 + }, + { + "KL/chosen_KL_mean": -127.52540588378906, + "KL/mean": -169.996826171875, + "KL/rejected_KL_mean": -212.46826171875, + "KL/std": 97.32823181152344, + "epoch": 0.5366591080876795, + "fcm_dpo/beta": 0.005657237023115158, + "fcm_dpo/delta": -0.06518547236919403, + "fcm_dpo/margin": 84.94284057617188, + "fcm_dpo/q_t": 0.39050179719924927, + "grad_norm": 12.261187553405762, + "learning_rate": 2.632160279321328e-07, + "logits/chosen": 0.778215765953064, + "logits/rejected": 0.65799880027771, + "logps/chosen": -173.78079223632812, + "logps/ref_chosen": -46.25537872314453, + "logps/ref_rejected": -78.20236206054688, + "logps/rejected": -290.67059326171875, + "loss": 1.0487, + "margin_dpo/margin_mean": 84.94284057617188, + "margin_dpo/margin_std": 106.72587585449219, + "step": 355 + }, + { + "KL/chosen_KL_mean": -122.94532775878906, + "KL/mean": -155.99130249023438, + "KL/rejected_KL_mean": -189.03729248046875, + "KL/std": 89.70805358886719, + "epoch": 0.5381708238851096, + "fcm_dpo/beta": 0.005620360840111971, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.09196472167969, + "fcm_dpo/q_t": 0.4156641364097595, + "grad_norm": 10.317228317260742, + "learning_rate": 2.618954789559356e-07, + "logits/chosen": 0.7800395488739014, + "logits/rejected": 0.7022900581359863, + "logps/chosen": -170.8514862060547, + "logps/ref_chosen": -47.906158447265625, + "logps/ref_rejected": -74.29397583007812, + "logps/rejected": -263.3312683105469, + "loss": 1.1421, + "margin_dpo/margin_mean": 66.09196472167969, + "margin_dpo/margin_std": 111.29681396484375, + "step": 356 + }, + { + "KL/chosen_KL_mean": -145.76248168945312, + "KL/mean": -171.95567321777344, + "KL/rejected_KL_mean": -198.1488800048828, + "KL/std": 91.05963134765625, + "epoch": 0.5396825396825397, + "fcm_dpo/beta": 0.005574680399149656, + "fcm_dpo/delta": -0.04080449417233467, + "fcm_dpo/margin": 52.38639831542969, + "fcm_dpo/q_t": 0.4316297769546509, + "grad_norm": 11.417281150817871, + "learning_rate": 2.6057459723762076e-07, + "logits/chosen": 0.778508186340332, + "logits/rejected": 0.7620171308517456, + "logps/chosen": -208.39747619628906, + "logps/ref_chosen": -62.63500213623047, + "logps/ref_rejected": -65.11399841308594, + "logps/rejected": -263.26287841796875, + "loss": 1.1986, + "margin_dpo/margin_mean": 52.38639831542969, + "margin_dpo/margin_std": 100.94227600097656, + "step": 357 + }, + { + "KL/chosen_KL_mean": -145.61180114746094, + "KL/mean": -179.46226501464844, + "KL/rejected_KL_mean": -213.31271362304688, + "KL/std": 97.12767028808594, + "epoch": 0.5411942554799698, + "fcm_dpo/beta": 0.005523651838302612, + "fcm_dpo/delta": -0.0923849493265152, + "fcm_dpo/margin": 67.7009048461914, + "fcm_dpo/q_t": 0.415596067905426, + "grad_norm": 16.69784164428711, + "learning_rate": 2.5925341972508954e-07, + "logits/chosen": 0.673900306224823, + "logits/rejected": 0.6875008940696716, + "logps/chosen": -212.8214111328125, + "logps/ref_chosen": -67.20960998535156, + "logps/ref_rejected": -69.34715270996094, + "logps/rejected": -282.65985107421875, + "loss": 1.1401, + "margin_dpo/margin_mean": 67.70091247558594, + "margin_dpo/margin_std": 108.23062896728516, + "step": 358 + }, + { + "KL/chosen_KL_mean": -161.5914306640625, + "KL/mean": -180.28622436523438, + "KL/rejected_KL_mean": -198.98104858398438, + "KL/std": 95.16166687011719, + "epoch": 0.5427059712773998, + "fcm_dpo/beta": 0.005472622811794281, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 37.389610290527344, + "fcm_dpo/q_t": 0.4529466927051544, + "grad_norm": 13.044179916381836, + "learning_rate": 2.579319833745169e-07, + "logits/chosen": 0.74629145860672, + "logits/rejected": 0.71300208568573, + "logps/chosen": -224.11721801757812, + "logps/ref_chosen": -62.52578353881836, + "logps/ref_rejected": -76.63114929199219, + "logps/rejected": -275.6121826171875, + "loss": 1.276, + "margin_dpo/margin_mean": 37.389610290527344, + "margin_dpo/margin_std": 105.70441436767578, + "step": 359 + }, + { + "KL/chosen_KL_mean": -153.78167724609375, + "KL/mean": -187.89193725585938, + "KL/rejected_KL_mean": -222.002197265625, + "KL/std": 100.96775817871094, + "epoch": 0.54421768707483, + "fcm_dpo/beta": 0.005472622811794281, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 68.22052001953125, + "fcm_dpo/q_t": 0.41469377279281616, + "grad_norm": 12.181764602661133, + "learning_rate": 2.5661032514931834e-07, + "logits/chosen": 0.7016806602478027, + "logits/rejected": 0.618013858795166, + "logps/chosen": -217.26937866210938, + "logps/ref_chosen": -63.48772048950195, + "logps/ref_rejected": -90.6891098022461, + "logps/rejected": -312.6912841796875, + "loss": 1.1184, + "margin_dpo/margin_mean": 68.22052001953125, + "margin_dpo/margin_std": 100.18309020996094, + "step": 360 + }, + { + "KL/chosen_KL_mean": -144.5960693359375, + "KL/mean": -181.52798461914062, + "KL/rejected_KL_mean": -218.45993041992188, + "KL/std": 93.3544921875, + "epoch": 0.54572940287226, + "fcm_dpo/beta": 0.0054628076031804085, + "fcm_dpo/delta": -0.017967604100704193, + "fcm_dpo/margin": 73.8638687133789, + "fcm_dpo/q_t": 0.4064033031463623, + "grad_norm": 10.888641357421875, + "learning_rate": 2.552884820191154e-07, + "logits/chosen": 0.8262885808944702, + "logits/rejected": 0.786286473274231, + "logps/chosen": -202.51321411132812, + "logps/ref_chosen": -57.917144775390625, + "logps/ref_rejected": -72.39089965820312, + "logps/rejected": -290.850830078125, + "loss": 1.0897, + "margin_dpo/margin_mean": 73.8638687133789, + "margin_dpo/margin_std": 96.90747833251953, + "step": 361 + }, + { + "KL/chosen_KL_mean": -146.7781982421875, + "KL/mean": -189.61180114746094, + "KL/rejected_KL_mean": -232.44540405273438, + "KL/std": 110.26019287109375, + "epoch": 0.54724111866969, + "fcm_dpo/beta": 0.005410117097198963, + "fcm_dpo/delta": -0.07925108075141907, + "fcm_dpo/margin": 85.66720581054688, + "fcm_dpo/q_t": 0.3988887071609497, + "grad_norm": 12.250260353088379, + "learning_rate": 2.53966490958702e-07, + "logits/chosen": 0.8019309043884277, + "logits/rejected": 0.7083509564399719, + "logps/chosen": -210.2216796875, + "logps/ref_chosen": -63.4434700012207, + "logps/ref_rejected": -103.45516967773438, + "logps/rejected": -335.90057373046875, + "loss": 1.0825, + "margin_dpo/margin_mean": 85.66720581054688, + "margin_dpo/margin_std": 129.54876708984375, + "step": 362 + }, + { + "KL/chosen_KL_mean": -149.93971252441406, + "KL/mean": -188.626220703125, + "KL/rejected_KL_mean": -227.31271362304688, + "KL/std": 94.37256622314453, + "epoch": 0.5487528344671202, + "fcm_dpo/beta": 0.005367242265492678, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 77.3730239868164, + "fcm_dpo/q_t": 0.40640437602996826, + "grad_norm": 12.255396842956543, + "learning_rate": 2.526443889470099e-07, + "logits/chosen": 0.8176679611206055, + "logits/rejected": 0.6928176879882812, + "logps/chosen": -198.59153747558594, + "logps/ref_chosen": -48.65182876586914, + "logps/ref_rejected": -88.65904235839844, + "logps/rejected": -315.9717712402344, + "loss": 1.0943, + "margin_dpo/margin_mean": 77.3730239868164, + "margin_dpo/margin_std": 110.87226867675781, + "step": 363 + }, + { + "KL/chosen_KL_mean": -132.52880859375, + "KL/mean": -176.2803497314453, + "KL/rejected_KL_mean": -220.03187561035156, + "KL/std": 104.11674499511719, + "epoch": 0.5502645502645502, + "fcm_dpo/beta": 0.005317722912877798, + "fcm_dpo/delta": -0.0463450662791729, + "fcm_dpo/margin": 87.50308227539062, + "fcm_dpo/q_t": 0.3970663845539093, + "grad_norm": 10.543391227722168, + "learning_rate": 2.513222129660744e-07, + "logits/chosen": 0.7445929050445557, + "logits/rejected": 0.6653020977973938, + "logps/chosen": -190.39987182617188, + "logps/ref_chosen": -57.87107467651367, + "logps/ref_rejected": -80.95503234863281, + "logps/rejected": -300.9869079589844, + "loss": 1.082, + "margin_dpo/margin_mean": 87.50308227539062, + "margin_dpo/margin_std": 129.79031372070312, + "step": 364 + }, + { + "KL/chosen_KL_mean": -126.51123046875, + "KL/mean": -165.5495147705078, + "KL/rejected_KL_mean": -204.58779907226562, + "KL/std": 111.03411865234375, + "epoch": 0.5517762660619804, + "fcm_dpo/beta": 0.005268987733870745, + "fcm_dpo/delta": -0.04603463038802147, + "fcm_dpo/margin": 78.07655334472656, + "fcm_dpo/q_t": 0.40557482838630676, + "grad_norm": 10.389727592468262, + "learning_rate": 2.5e-07, + "logits/chosen": 0.7515792846679688, + "logits/rejected": 0.7503241300582886, + "logps/chosen": -191.45339965820312, + "logps/ref_chosen": -64.94217681884766, + "logps/ref_rejected": -74.8599853515625, + "logps/rejected": -279.4477844238281, + "loss": 1.0758, + "margin_dpo/margin_mean": 78.07655334472656, + "margin_dpo/margin_std": 97.56391906738281, + "step": 365 + }, + { + "KL/chosen_KL_mean": -125.47860717773438, + "KL/mean": -157.73670959472656, + "KL/rejected_KL_mean": -189.99481201171875, + "KL/std": 97.81672668457031, + "epoch": 0.5532879818594104, + "fcm_dpo/beta": 0.005268987733870745, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 64.5162124633789, + "fcm_dpo/q_t": 0.4233565330505371, + "grad_norm": 14.451668739318848, + "learning_rate": 2.486777870339255e-07, + "logits/chosen": 0.7044786810874939, + "logits/rejected": 0.7007114887237549, + "logps/chosen": -180.64459228515625, + "logps/ref_chosen": -55.16598129272461, + "logps/ref_rejected": -65.26121520996094, + "logps/rejected": -255.25601196289062, + "loss": 1.1763, + "margin_dpo/margin_mean": 64.5162124633789, + "margin_dpo/margin_std": 124.49813842773438, + "step": 366 + }, + { + "KL/chosen_KL_mean": -144.489990234375, + "KL/mean": -176.97052001953125, + "KL/rejected_KL_mean": -209.45106506347656, + "KL/std": 96.73851776123047, + "epoch": 0.5547996976568406, + "fcm_dpo/beta": 0.005268987733870745, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 64.96107482910156, + "fcm_dpo/q_t": 0.4195973873138428, + "grad_norm": 12.064628601074219, + "learning_rate": 2.4735561105299014e-07, + "logits/chosen": 0.7793039083480835, + "logits/rejected": 0.6833162307739258, + "logps/chosen": -200.50045776367188, + "logps/ref_chosen": -56.01046371459961, + "logps/ref_rejected": -77.31010437011719, + "logps/rejected": -286.76116943359375, + "loss": 1.1418, + "margin_dpo/margin_mean": 64.96107482910156, + "margin_dpo/margin_std": 101.67163848876953, + "step": 367 + }, + { + "KL/chosen_KL_mean": -155.61111450195312, + "KL/mean": -184.02906799316406, + "KL/rejected_KL_mean": -212.447021484375, + "KL/std": 94.99668884277344, + "epoch": 0.5563114134542706, + "fcm_dpo/beta": 0.005268987733870745, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 56.83592224121094, + "fcm_dpo/q_t": 0.43033045530319214, + "grad_norm": 14.06124496459961, + "learning_rate": 2.46033509041298e-07, + "logits/chosen": 0.6116068959236145, + "logits/rejected": 0.6141465306282043, + "logps/chosen": -230.44039916992188, + "logps/ref_chosen": -74.82927703857422, + "logps/ref_rejected": -76.11680603027344, + "logps/rejected": -288.5638427734375, + "loss": 1.1803, + "margin_dpo/margin_mean": 56.83592224121094, + "margin_dpo/margin_std": 102.63706970214844, + "step": 368 + }, + { + "KL/chosen_KL_mean": -142.20578002929688, + "KL/mean": -169.76429748535156, + "KL/rejected_KL_mean": -197.32281494140625, + "KL/std": 95.49687194824219, + "epoch": 0.5578231292517006, + "fcm_dpo/beta": 0.005268987733870745, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 55.11702346801758, + "fcm_dpo/q_t": 0.43302732706069946, + "grad_norm": 11.773307800292969, + "learning_rate": 2.447115179808846e-07, + "logits/chosen": 0.7620031833648682, + "logits/rejected": 0.7175683379173279, + "logps/chosen": -200.53199768066406, + "logps/ref_chosen": -58.32621765136719, + "logps/ref_rejected": -80.92183685302734, + "logps/rejected": -278.2446594238281, + "loss": 1.1847, + "margin_dpo/margin_mean": 55.11701965332031, + "margin_dpo/margin_std": 100.87909698486328, + "step": 369 + }, + { + "KL/chosen_KL_mean": -135.1417694091797, + "KL/mean": -175.1524200439453, + "KL/rejected_KL_mean": -215.16307067871094, + "KL/std": 99.09980010986328, + "epoch": 0.5593348450491308, + "fcm_dpo/beta": 0.00522764865309, + "fcm_dpo/delta": -0.03938331827521324, + "fcm_dpo/margin": 80.02131652832031, + "fcm_dpo/q_t": 0.40278932452201843, + "grad_norm": 12.446624755859375, + "learning_rate": 2.4338967485068164e-07, + "logits/chosen": 0.8438280820846558, + "logits/rejected": 0.7817610502243042, + "logps/chosen": -188.02548217773438, + "logps/ref_chosen": -52.88372039794922, + "logps/ref_rejected": -79.43692016601562, + "logps/rejected": -294.5999755859375, + "loss": 1.0979, + "margin_dpo/margin_mean": 80.02131652832031, + "margin_dpo/margin_std": 115.83177185058594, + "step": 370 + }, + { + "KL/chosen_KL_mean": -131.9373016357422, + "KL/mean": -166.3912811279297, + "KL/rejected_KL_mean": -200.8452606201172, + "KL/std": 92.92374420166016, + "epoch": 0.5608465608465608, + "fcm_dpo/beta": 0.005213094875216484, + "fcm_dpo/delta": -0.027917178347706795, + "fcm_dpo/margin": 68.90794372558594, + "fcm_dpo/q_t": 0.41668015718460083, + "grad_norm": 15.715535163879395, + "learning_rate": 2.420680166254831e-07, + "logits/chosen": 0.8548682928085327, + "logits/rejected": 0.8279607892036438, + "logps/chosen": -181.16151428222656, + "logps/ref_chosen": -49.224212646484375, + "logps/ref_rejected": -63.348472595214844, + "logps/rejected": -264.1937255859375, + "loss": 1.1289, + "margin_dpo/margin_mean": 68.90794372558594, + "margin_dpo/margin_std": 102.37510681152344, + "step": 371 + }, + { + "KL/chosen_KL_mean": -140.001708984375, + "KL/mean": -161.2264404296875, + "KL/rejected_KL_mean": -182.451171875, + "KL/std": 93.77758026123047, + "epoch": 0.562358276643991, + "fcm_dpo/beta": 0.005198541563004255, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 42.44948959350586, + "fcm_dpo/q_t": 0.4500351548194885, + "grad_norm": 12.793850898742676, + "learning_rate": 2.4074658027491044e-07, + "logits/chosen": 0.7945922613143921, + "logits/rejected": 0.7056927680969238, + "logps/chosen": -192.271240234375, + "logps/ref_chosen": -52.269554138183594, + "logps/ref_rejected": -72.99522399902344, + "logps/rejected": -255.4464111328125, + "loss": 1.2767, + "margin_dpo/margin_mean": 42.44948959350586, + "margin_dpo/margin_std": 121.45559692382812, + "step": 372 + }, + { + "KL/chosen_KL_mean": -161.46475219726562, + "KL/mean": -188.8463134765625, + "KL/rejected_KL_mean": -216.2279052734375, + "KL/std": 101.70494079589844, + "epoch": 0.563869992441421, + "fcm_dpo/beta": 0.005198541563004255, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 54.76313781738281, + "fcm_dpo/q_t": 0.43512850999832153, + "grad_norm": 11.343805313110352, + "learning_rate": 2.394254027623792e-07, + "logits/chosen": 0.7745212316513062, + "logits/rejected": 0.7118136882781982, + "logps/chosen": -222.57774353027344, + "logps/ref_chosen": -61.112998962402344, + "logps/ref_rejected": -76.24851989746094, + "logps/rejected": -292.4764099121094, + "loss": 1.2211, + "margin_dpo/margin_mean": 54.76313781738281, + "margin_dpo/margin_std": 124.361572265625, + "step": 373 + }, + { + "KL/chosen_KL_mean": -137.03231811523438, + "KL/mean": -180.58938598632812, + "KL/rejected_KL_mean": -224.1464385986328, + "KL/std": 104.06997680664062, + "epoch": 0.5653817082388511, + "fcm_dpo/beta": 0.005156625993549824, + "fcm_dpo/delta": -0.051555268466472626, + "fcm_dpo/margin": 87.11412048339844, + "fcm_dpo/q_t": 0.39748048782348633, + "grad_norm": 14.498461723327637, + "learning_rate": 2.381045210440644e-07, + "logits/chosen": 0.7031779289245605, + "logits/rejected": 0.7090506553649902, + "logps/chosen": -209.7015380859375, + "logps/ref_chosen": -72.66920471191406, + "logps/ref_rejected": -76.83158874511719, + "logps/rejected": -300.97802734375, + "loss": 1.0878, + "margin_dpo/margin_mean": 87.11412048339844, + "margin_dpo/margin_std": 128.58489990234375, + "step": 374 + }, + { + "KL/chosen_KL_mean": -133.54660034179688, + "KL/mean": -166.88844299316406, + "KL/rejected_KL_mean": -200.23028564453125, + "KL/std": 113.39869689941406, + "epoch": 0.5668934240362812, + "fcm_dpo/beta": 0.00514521449804306, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.68367004394531, + "fcm_dpo/q_t": 0.42260217666625977, + "grad_norm": 15.06658935546875, + "learning_rate": 2.3678397206786715e-07, + "logits/chosen": 0.7743487358093262, + "logits/rejected": 0.7246922850608826, + "logps/chosen": -191.22991943359375, + "logps/ref_chosen": -57.68330383300781, + "logps/ref_rejected": -79.34097290039062, + "logps/rejected": -279.5712585449219, + "loss": 1.1673, + "margin_dpo/margin_mean": 66.68367004394531, + "margin_dpo/margin_std": 123.28602600097656, + "step": 375 + }, + { + "KL/chosen_KL_mean": -140.264892578125, + "KL/mean": -181.48529052734375, + "KL/rejected_KL_mean": -222.70571899414062, + "KL/std": 114.37265014648438, + "epoch": 0.5684051398337112, + "fcm_dpo/beta": 0.005105261690914631, + "fcm_dpo/delta": -0.03897655010223389, + "fcm_dpo/margin": 82.44081115722656, + "fcm_dpo/q_t": 0.407854825258255, + "grad_norm": 12.342073440551758, + "learning_rate": 2.3546379277238103e-07, + "logits/chosen": 0.8240178823471069, + "logits/rejected": 0.7641883492469788, + "logps/chosen": -191.93896484375, + "logps/ref_chosen": -51.674072265625, + "logps/ref_rejected": -75.69713592529297, + "logps/rejected": -298.40283203125, + "loss": 1.1138, + "margin_dpo/margin_mean": 82.44081115722656, + "margin_dpo/margin_std": 133.641357421875, + "step": 376 + }, + { + "KL/chosen_KL_mean": -143.00613403320312, + "KL/mean": -170.45001220703125, + "KL/rejected_KL_mean": -197.89390563964844, + "KL/std": 89.45880126953125, + "epoch": 0.5699168556311414, + "fcm_dpo/beta": 0.005105261690914631, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 54.88779067993164, + "fcm_dpo/q_t": 0.43406882882118225, + "grad_norm": 12.48110294342041, + "learning_rate": 2.3414402008585886e-07, + "logits/chosen": 0.7774357795715332, + "logits/rejected": 0.7551122903823853, + "logps/chosen": -189.18466186523438, + "logps/ref_chosen": -46.17853546142578, + "logps/ref_rejected": -57.756500244140625, + "logps/rejected": -255.65040588378906, + "loss": 1.1898, + "margin_dpo/margin_mean": 54.887786865234375, + "margin_dpo/margin_std": 100.91114807128906, + "step": 377 + }, + { + "KL/chosen_KL_mean": -139.11705017089844, + "KL/mean": -167.0404052734375, + "KL/rejected_KL_mean": -194.96377563476562, + "KL/std": 105.56818389892578, + "epoch": 0.5714285714285714, + "fcm_dpo/beta": 0.005105261690914631, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 55.846744537353516, + "fcm_dpo/q_t": 0.43387657403945923, + "grad_norm": 9.497807502746582, + "learning_rate": 2.3282469092517977e-07, + "logits/chosen": 0.817177414894104, + "logits/rejected": 0.7751537561416626, + "logps/chosen": -198.33592224121094, + "logps/ref_chosen": -59.21887969970703, + "logps/ref_rejected": -71.24818420410156, + "logps/rejected": -266.21197509765625, + "loss": 1.1906, + "margin_dpo/margin_mean": 55.84674072265625, + "margin_dpo/margin_std": 102.57722473144531, + "step": 378 + }, + { + "KL/chosen_KL_mean": -144.64332580566406, + "KL/mean": -184.85391235351562, + "KL/rejected_KL_mean": -225.06451416015625, + "KL/std": 111.22518157958984, + "epoch": 0.5729402872260015, + "fcm_dpo/beta": 0.005105261690914631, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.42117309570312, + "fcm_dpo/q_t": 0.40791893005371094, + "grad_norm": 11.734951972961426, + "learning_rate": 2.3150584219481643e-07, + "logits/chosen": 0.7828426361083984, + "logits/rejected": 0.7149155139923096, + "logps/chosen": -220.95989990234375, + "logps/ref_chosen": -76.31658935546875, + "logps/ref_rejected": -104.26200103759766, + "logps/rejected": -329.3265075683594, + "loss": 1.1054, + "margin_dpo/margin_mean": 80.42117309570312, + "margin_dpo/margin_std": 121.62455749511719, + "step": 379 + }, + { + "KL/chosen_KL_mean": -119.44280242919922, + "KL/mean": -162.35813903808594, + "KL/rejected_KL_mean": -205.2735137939453, + "KL/std": 97.02486419677734, + "epoch": 0.5744520030234316, + "fcm_dpo/beta": 0.005090559832751751, + "fcm_dpo/delta": -0.028881238773465157, + "fcm_dpo/margin": 85.83067321777344, + "fcm_dpo/q_t": 0.39921677112579346, + "grad_norm": 12.701099395751953, + "learning_rate": 2.3018751078580283e-07, + "logits/chosen": 0.7823134660720825, + "logits/rejected": 0.7494254112243652, + "logps/chosen": -180.72596740722656, + "logps/ref_chosen": -61.283164978027344, + "logps/ref_rejected": -72.38892364501953, + "logps/rejected": -277.66241455078125, + "loss": 1.0761, + "margin_dpo/margin_mean": 85.83068084716797, + "margin_dpo/margin_std": 114.44374084472656, + "step": 380 + }, + { + "KL/chosen_KL_mean": -146.58526611328125, + "KL/mean": -163.72341918945312, + "KL/rejected_KL_mean": -180.861572265625, + "KL/std": 102.95851135253906, + "epoch": 0.5759637188208617, + "fcm_dpo/beta": 0.005075857508927584, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 34.27632141113281, + "fcm_dpo/q_t": 0.4604954719543457, + "grad_norm": 10.866552352905273, + "learning_rate": 2.288697335747027e-07, + "logits/chosen": 0.7479392290115356, + "logits/rejected": 0.7347009181976318, + "logps/chosen": -204.79925537109375, + "logps/ref_chosen": -58.2139892578125, + "logps/ref_rejected": -60.78669357299805, + "logps/rejected": -241.6482696533203, + "loss": 1.3144, + "margin_dpo/margin_mean": 34.27631759643555, + "margin_dpo/margin_std": 124.18953704833984, + "step": 381 + }, + { + "KL/chosen_KL_mean": -147.87542724609375, + "KL/mean": -179.83062744140625, + "KL/rejected_KL_mean": -211.78582763671875, + "KL/std": 103.192138671875, + "epoch": 0.5774754346182918, + "fcm_dpo/beta": 0.005075857508927584, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 63.91041564941406, + "fcm_dpo/q_t": 0.4254421591758728, + "grad_norm": 12.20398998260498, + "learning_rate": 2.2755254742257706e-07, + "logits/chosen": 0.7671145796775818, + "logits/rejected": 0.7206964492797852, + "logps/chosen": -209.70074462890625, + "logps/ref_chosen": -61.82532501220703, + "logps/ref_rejected": -83.0452880859375, + "logps/rejected": -294.83111572265625, + "loss": 1.1512, + "margin_dpo/margin_mean": 63.91041564941406, + "margin_dpo/margin_std": 100.8181381225586, + "step": 382 + }, + { + "KL/chosen_KL_mean": -153.29888916015625, + "KL/mean": -179.7510986328125, + "KL/rejected_KL_mean": -206.20333862304688, + "KL/std": 102.09304809570312, + "epoch": 0.5789871504157218, + "fcm_dpo/beta": 0.005075857508927584, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 52.904457092285156, + "fcm_dpo/q_t": 0.4390997290611267, + "grad_norm": 13.56281566619873, + "learning_rate": 2.2623598917395436e-07, + "logits/chosen": 0.6700069904327393, + "logits/rejected": 0.7050174474716187, + "logps/chosen": -233.86215209960938, + "logps/ref_chosen": -80.56326293945312, + "logps/ref_rejected": -74.62922668457031, + "logps/rejected": -280.83258056640625, + "loss": 1.2353, + "margin_dpo/margin_mean": 52.90446472167969, + "margin_dpo/margin_std": 127.63932800292969, + "step": 383 + }, + { + "KL/chosen_KL_mean": -138.74447631835938, + "KL/mean": -171.95449829101562, + "KL/rejected_KL_mean": -205.16452026367188, + "KL/std": 96.96101379394531, + "epoch": 0.5804988662131519, + "fcm_dpo/beta": 0.005075857508927584, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.4200439453125, + "fcm_dpo/q_t": 0.420096755027771, + "grad_norm": 13.154545783996582, + "learning_rate": 2.2492009565579875e-07, + "logits/chosen": 0.7832565307617188, + "logits/rejected": 0.7525622844696045, + "logps/chosen": -204.21963500976562, + "logps/ref_chosen": -65.47514343261719, + "logps/ref_rejected": -79.67378234863281, + "logps/rejected": -284.83831787109375, + "loss": 1.1424, + "margin_dpo/margin_mean": 66.4200439453125, + "margin_dpo/margin_std": 103.07756042480469, + "step": 384 + }, + { + "KL/chosen_KL_mean": -141.26541137695312, + "KL/mean": -183.92938232421875, + "KL/rejected_KL_mean": -226.5933380126953, + "KL/std": 103.0797119140625, + "epoch": 0.582010582010582, + "fcm_dpo/beta": 0.005049902945756912, + "fcm_dpo/delta": -0.051395926624536514, + "fcm_dpo/margin": 85.32792663574219, + "fcm_dpo/q_t": 0.40050801634788513, + "grad_norm": 9.880163192749023, + "learning_rate": 2.2360490367648084e-07, + "logits/chosen": 0.7219782471656799, + "logits/rejected": 0.6834163069725037, + "logps/chosen": -207.32192993164062, + "logps/ref_chosen": -66.0565185546875, + "logps/ref_rejected": -86.68023681640625, + "logps/rejected": -313.2735595703125, + "loss": 1.0692, + "margin_dpo/margin_mean": 85.32792663574219, + "margin_dpo/margin_std": 105.10133361816406, + "step": 385 + }, + { + "KL/chosen_KL_mean": -165.14895629882812, + "KL/mean": -193.8024139404297, + "KL/rejected_KL_mean": -222.45584106445312, + "KL/std": 109.71835327148438, + "epoch": 0.5835222978080121, + "fcm_dpo/beta": 0.005023948848247528, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.306884765625, + "fcm_dpo/q_t": 0.4323265552520752, + "grad_norm": 12.290263175964355, + "learning_rate": 2.2229045002474724e-07, + "logits/chosen": 0.7119057178497314, + "logits/rejected": 0.664527177810669, + "logps/chosen": -240.7726287841797, + "logps/ref_chosen": -75.6236572265625, + "logps/ref_rejected": -92.62330627441406, + "logps/rejected": -315.07916259765625, + "loss": 1.1928, + "margin_dpo/margin_mean": 57.306884765625, + "margin_dpo/margin_std": 110.36276245117188, + "step": 386 + }, + { + "KL/chosen_KL_mean": -143.9866485595703, + "KL/mean": -184.1669921875, + "KL/rejected_KL_mean": -224.3473663330078, + "KL/std": 99.10783386230469, + "epoch": 0.5850340136054422, + "fcm_dpo/beta": 0.005023948848247528, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.36071014404297, + "fcm_dpo/q_t": 0.4071784019470215, + "grad_norm": 12.432520866394043, + "learning_rate": 2.209767714686924e-07, + "logits/chosen": 0.787395715713501, + "logits/rejected": 0.6864792704582214, + "logps/chosen": -191.20835876464844, + "logps/ref_chosen": -47.22170639038086, + "logps/ref_rejected": -87.338134765625, + "logps/rejected": -311.68548583984375, + "loss": 1.0866, + "margin_dpo/margin_mean": 80.3607177734375, + "margin_dpo/margin_std": 103.79031372070312, + "step": 387 + }, + { + "KL/chosen_KL_mean": -151.07595825195312, + "KL/mean": -175.32736206054688, + "KL/rejected_KL_mean": -199.5787811279297, + "KL/std": 105.66417694091797, + "epoch": 0.5865457294028723, + "fcm_dpo/beta": 0.005023948848247528, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 48.50281524658203, + "fcm_dpo/q_t": 0.4453752636909485, + "grad_norm": 12.267721176147461, + "learning_rate": 2.1966390475472954e-07, + "logits/chosen": 0.7400516271591187, + "logits/rejected": 0.7385213375091553, + "logps/chosen": -225.65542602539062, + "logps/ref_chosen": -74.5794677734375, + "logps/ref_rejected": -79.92558288574219, + "logps/rejected": -279.5043640136719, + "loss": 1.2521, + "margin_dpo/margin_mean": 48.5028190612793, + "margin_dpo/margin_std": 125.29054260253906, + "step": 388 + }, + { + "KL/chosen_KL_mean": -140.3387451171875, + "KL/mean": -180.39773559570312, + "KL/rejected_KL_mean": -220.45672607421875, + "KL/std": 98.77024841308594, + "epoch": 0.5880574452003023, + "fcm_dpo/beta": 0.005023948848247528, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.11798095703125, + "fcm_dpo/q_t": 0.40777429938316345, + "grad_norm": 20.287439346313477, + "learning_rate": 2.1835188660656265e-07, + "logits/chosen": 0.7831531763076782, + "logits/rejected": 0.7533621788024902, + "logps/chosen": -201.96310424804688, + "logps/ref_chosen": -61.624366760253906, + "logps/ref_rejected": -76.50978088378906, + "logps/rejected": -296.96649169921875, + "loss": 1.0962, + "margin_dpo/margin_mean": 80.11798095703125, + "margin_dpo/margin_std": 110.59423828125, + "step": 389 + }, + { + "KL/chosen_KL_mean": -124.74932098388672, + "KL/mean": -155.60525512695312, + "KL/rejected_KL_mean": -186.46119689941406, + "KL/std": 94.935791015625, + "epoch": 0.5895691609977324, + "fcm_dpo/beta": 0.005023948848247528, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.71186828613281, + "fcm_dpo/q_t": 0.42818349599838257, + "grad_norm": 11.866067886352539, + "learning_rate": 2.170407537241599e-07, + "logits/chosen": 0.8283019065856934, + "logits/rejected": 0.7651022672653198, + "logps/chosen": -170.62118530273438, + "logps/ref_chosen": -45.871864318847656, + "logps/ref_rejected": -61.305999755859375, + "logps/rejected": -247.76719665527344, + "loss": 1.1634, + "margin_dpo/margin_mean": 61.71186828613281, + "margin_dpo/margin_std": 102.22355651855469, + "step": 390 + }, + { + "KL/chosen_KL_mean": -140.8014678955078, + "KL/mean": -179.24481201171875, + "KL/rejected_KL_mean": -217.68814086914062, + "KL/std": 99.41610717773438, + "epoch": 0.5910808767951625, + "fcm_dpo/beta": 0.005023948848247528, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.8866958618164, + "fcm_dpo/q_t": 0.41153138875961304, + "grad_norm": 11.932952880859375, + "learning_rate": 2.1573054278272636e-07, + "logits/chosen": 0.7904618978500366, + "logits/rejected": 0.7304722666740417, + "logps/chosen": -198.98846435546875, + "logps/ref_chosen": -58.18701171875, + "logps/ref_rejected": -83.63442993164062, + "logps/rejected": -301.32257080078125, + "loss": 1.1213, + "margin_dpo/margin_mean": 76.8866958618164, + "margin_dpo/margin_std": 118.32135772705078, + "step": 391 + }, + { + "KL/chosen_KL_mean": -131.2482147216797, + "KL/mean": -170.39854431152344, + "KL/rejected_KL_mean": -209.5488739013672, + "KL/std": 109.20217895507812, + "epoch": 0.5925925925925926, + "fcm_dpo/beta": 0.004997436888515949, + "fcm_dpo/delta": -0.053051162511110306, + "fcm_dpo/margin": 78.3006591796875, + "fcm_dpo/q_t": 0.41144829988479614, + "grad_norm": 11.570497512817383, + "learning_rate": 2.1442129043167873e-07, + "logits/chosen": 0.8129167556762695, + "logits/rejected": 0.7619023323059082, + "logps/chosen": -200.99273681640625, + "logps/ref_chosen": -69.7445297241211, + "logps/ref_rejected": -94.05877685546875, + "logps/rejected": -303.607666015625, + "loss": 1.1248, + "margin_dpo/margin_mean": 78.3006591796875, + "margin_dpo/margin_std": 125.41221618652344, + "step": 392 + }, + { + "KL/chosen_KL_mean": -146.885498046875, + "KL/mean": -188.21755981445312, + "KL/rejected_KL_mean": -229.54962158203125, + "KL/std": 106.22329711914062, + "epoch": 0.5941043083900227, + "fcm_dpo/beta": 0.004906511399894953, + "fcm_dpo/delta": -0.06521414965391159, + "fcm_dpo/margin": 82.66412353515625, + "fcm_dpo/q_t": 0.4055265784263611, + "grad_norm": 9.672101020812988, + "learning_rate": 2.131130332936195e-07, + "logits/chosen": 0.8200071454048157, + "logits/rejected": 0.7858361601829529, + "logps/chosen": -199.22039794921875, + "logps/ref_chosen": -52.33489990234375, + "logps/ref_rejected": -74.33809661865234, + "logps/rejected": -303.8877258300781, + "loss": 1.0866, + "margin_dpo/margin_mean": 82.66412353515625, + "margin_dpo/margin_std": 104.7359619140625, + "step": 393 + }, + { + "KL/chosen_KL_mean": -140.14215087890625, + "KL/mean": -176.29550170898438, + "KL/rejected_KL_mean": -212.44886779785156, + "KL/std": 98.92924499511719, + "epoch": 0.5956160241874527, + "fcm_dpo/beta": 0.004906511399894953, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 72.30670166015625, + "fcm_dpo/q_t": 0.41654276847839355, + "grad_norm": 11.078004837036133, + "learning_rate": 2.1180580796331323e-07, + "logits/chosen": 0.8135772943496704, + "logits/rejected": 0.7879060506820679, + "logps/chosen": -200.81829833984375, + "logps/ref_chosen": -60.6761360168457, + "logps/ref_rejected": -71.36074829101562, + "logps/rejected": -283.80963134765625, + "loss": 1.1207, + "margin_dpo/margin_mean": 72.30670166015625, + "margin_dpo/margin_std": 98.23634338378906, + "step": 394 + }, + { + "KL/chosen_KL_mean": -141.8544158935547, + "KL/mean": -177.10147094726562, + "KL/rejected_KL_mean": -212.34854125976562, + "KL/std": 103.47579193115234, + "epoch": 0.5971277399848829, + "fcm_dpo/beta": 0.004906511399894953, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 70.49411010742188, + "fcm_dpo/q_t": 0.4212576150894165, + "grad_norm": 12.0609769821167, + "learning_rate": 2.104996510066625e-07, + "logits/chosen": 0.8366742134094238, + "logits/rejected": 0.7517580389976501, + "logps/chosen": -192.458740234375, + "logps/ref_chosen": -50.60432434082031, + "logps/ref_rejected": -77.08731079101562, + "logps/rejected": -289.43585205078125, + "loss": 1.1396, + "margin_dpo/margin_mean": 70.4941177368164, + "margin_dpo/margin_std": 111.34257507324219, + "step": 395 + }, + { + "KL/chosen_KL_mean": -137.52777099609375, + "KL/mean": -172.69790649414062, + "KL/rejected_KL_mean": -207.8680419921875, + "KL/std": 110.33338928222656, + "epoch": 0.5986394557823129, + "fcm_dpo/beta": 0.004860267508774996, + "fcm_dpo/delta": -0.04734856262803078, + "fcm_dpo/margin": 70.34027862548828, + "fcm_dpo/q_t": 0.420335054397583, + "grad_norm": 9.277693748474121, + "learning_rate": 2.0919459895968517e-07, + "logits/chosen": 0.8274677991867065, + "logits/rejected": 0.7451637387275696, + "logps/chosen": -188.8873748779297, + "logps/ref_chosen": -51.35961151123047, + "logps/ref_rejected": -79.89360046386719, + "logps/rejected": -287.76165771484375, + "loss": 1.1321, + "margin_dpo/margin_mean": 70.34027862548828, + "margin_dpo/margin_std": 98.32467651367188, + "step": 396 + }, + { + "KL/chosen_KL_mean": -158.9486846923828, + "KL/mean": -179.14877319335938, + "KL/rejected_KL_mean": -199.3488311767578, + "KL/std": 103.03988647460938, + "epoch": 0.600151171579743, + "fcm_dpo/beta": 0.004860267508774996, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 40.400169372558594, + "fcm_dpo/q_t": 0.45476654171943665, + "grad_norm": 10.52593994140625, + "learning_rate": 2.078906883274924e-07, + "logits/chosen": 0.7486104369163513, + "logits/rejected": 0.7011691331863403, + "logps/chosen": -225.4049072265625, + "logps/ref_chosen": -66.45622253417969, + "logps/ref_rejected": -85.74736785888672, + "logps/rejected": -285.09619140625, + "loss": 1.2768, + "margin_dpo/margin_mean": 40.400169372558594, + "margin_dpo/margin_std": 115.52510833740234, + "step": 397 + }, + { + "KL/chosen_KL_mean": -131.5105438232422, + "KL/mean": -179.10305786132812, + "KL/rejected_KL_mean": -226.69557189941406, + "KL/std": 106.75947570800781, + "epoch": 0.6016628873771731, + "fcm_dpo/beta": 0.004792422521859407, + "fcm_dpo/delta": -0.07028739899396896, + "fcm_dpo/margin": 95.18501281738281, + "fcm_dpo/q_t": 0.396945595741272, + "grad_norm": 9.588266372680664, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": 0.8046476244926453, + "logits/rejected": 0.7484967708587646, + "logps/chosen": -180.75479125976562, + "logps/ref_chosen": -49.244239807128906, + "logps/ref_rejected": -75.18949127197266, + "logps/rejected": -301.88507080078125, + "loss": 1.0527, + "margin_dpo/margin_mean": 95.18502044677734, + "margin_dpo/margin_std": 117.26836395263672, + "step": 398 + }, + { + "KL/chosen_KL_mean": -157.90802001953125, + "KL/mean": -212.11856079101562, + "KL/rejected_KL_mean": -266.3291015625, + "KL/std": 123.38093566894531, + "epoch": 0.6031746031746031, + "fcm_dpo/beta": 0.004697504919022322, + "fcm_dpo/delta": -0.10002284497022629, + "fcm_dpo/margin": 108.42111206054688, + "fcm_dpo/q_t": 0.387840211391449, + "grad_norm": 10.69479751586914, + "learning_rate": 2.052864371672457e-07, + "logits/chosen": 0.750247597694397, + "logits/rejected": 0.6243202090263367, + "logps/chosen": -226.21481323242188, + "logps/ref_chosen": -68.30679321289062, + "logps/ref_rejected": -113.2708511352539, + "logps/rejected": -379.5999755859375, + "loss": 1.036, + "margin_dpo/margin_mean": 108.42111206054688, + "margin_dpo/margin_std": 141.37326049804688, + "step": 399 + }, + { + "KL/chosen_KL_mean": -174.25674438476562, + "KL/mean": -206.14834594726562, + "KL/rejected_KL_mean": -238.03994750976562, + "KL/std": 111.63092041015625, + "epoch": 0.6046863189720333, + "fcm_dpo/beta": 0.004666075576096773, + "fcm_dpo/delta": -0.03356543183326721, + "fcm_dpo/margin": 63.78321075439453, + "fcm_dpo/q_t": 0.4290245771408081, + "grad_norm": 14.130722045898438, + "learning_rate": 2.0398616948569493e-07, + "logits/chosen": 0.7988376617431641, + "logits/rejected": 0.74619060754776, + "logps/chosen": -245.88323974609375, + "logps/ref_chosen": -71.62649536132812, + "logps/ref_rejected": -90.98765563964844, + "logps/rejected": -329.027587890625, + "loss": 1.1741, + "margin_dpo/margin_mean": 63.78321075439453, + "margin_dpo/margin_std": 103.69830322265625, + "step": 400 + }, + { + "KL/chosen_KL_mean": -130.93389892578125, + "KL/mean": -174.3590545654297, + "KL/rejected_KL_mean": -217.78421020507812, + "KL/std": 116.89300537109375, + "epoch": 0.6061980347694633, + "fcm_dpo/beta": 0.004630550276488066, + "fcm_dpo/delta": -0.03821335360407829, + "fcm_dpo/margin": 86.85031127929688, + "fcm_dpo/q_t": 0.4078829884529114, + "grad_norm": 7.8426055908203125, + "learning_rate": 2.0268718890989752e-07, + "logits/chosen": 0.8150224685668945, + "logits/rejected": 0.7409332990646362, + "logps/chosen": -184.6588592529297, + "logps/ref_chosen": -53.72495651245117, + "logps/ref_rejected": -75.06304931640625, + "logps/rejected": -292.8472595214844, + "loss": 1.0919, + "margin_dpo/margin_mean": 86.85031127929688, + "margin_dpo/margin_std": 116.00386047363281, + "step": 401 + }, + { + "KL/chosen_KL_mean": -145.6488037109375, + "KL/mean": -178.26016235351562, + "KL/rejected_KL_mean": -210.87152099609375, + "KL/std": 97.66880798339844, + "epoch": 0.6077097505668935, + "fcm_dpo/beta": 0.004630550276488066, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 65.22270965576172, + "fcm_dpo/q_t": 0.4299997091293335, + "grad_norm": 11.507543563842773, + "learning_rate": 2.013895317751323e-07, + "logits/chosen": 0.7916992902755737, + "logits/rejected": 0.785792350769043, + "logps/chosen": -207.52273559570312, + "logps/ref_chosen": -61.873931884765625, + "logps/ref_rejected": -66.15198516845703, + "logps/rejected": -277.02349853515625, + "loss": 1.1841, + "margin_dpo/margin_mean": 65.22270965576172, + "margin_dpo/margin_std": 120.50914001464844, + "step": 402 + }, + { + "KL/chosen_KL_mean": -151.57379150390625, + "KL/mean": -194.6015167236328, + "KL/rejected_KL_mean": -237.6292724609375, + "KL/std": 114.89691162109375, + "epoch": 0.6092214663643235, + "fcm_dpo/beta": 0.004630550276488066, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 86.05549621582031, + "fcm_dpo/q_t": 0.409629762172699, + "grad_norm": 8.590554237365723, + "learning_rate": 2.0009323437965898e-07, + "logits/chosen": 0.9000512957572937, + "logits/rejected": 0.8217411041259766, + "logps/chosen": -202.89527893066406, + "logps/ref_chosen": -51.321502685546875, + "logps/ref_rejected": -86.54010772705078, + "logps/rejected": -324.16937255859375, + "loss": 1.0996, + "margin_dpo/margin_mean": 86.05550384521484, + "margin_dpo/margin_std": 118.92399597167969, + "step": 403 + }, + { + "KL/chosen_KL_mean": -145.5100860595703, + "KL/mean": -190.7388153076172, + "KL/rejected_KL_mean": -235.96755981445312, + "KL/std": 120.02464294433594, + "epoch": 0.6107331821617535, + "fcm_dpo/beta": 0.004570495802909136, + "fcm_dpo/delta": -0.06526995450258255, + "fcm_dpo/margin": 90.45745849609375, + "fcm_dpo/q_t": 0.4066828787326813, + "grad_norm": 12.769746780395508, + "learning_rate": 1.9879833298370237e-07, + "logits/chosen": 0.7784748077392578, + "logits/rejected": 0.6955345869064331, + "logps/chosen": -207.77297973632812, + "logps/ref_chosen": -62.26288604736328, + "logps/ref_rejected": -95.19029998779297, + "logps/rejected": -331.1578369140625, + "loss": 1.105, + "margin_dpo/margin_mean": 90.45745849609375, + "margin_dpo/margin_std": 134.48211669921875, + "step": 404 + }, + { + "KL/chosen_KL_mean": -147.45518493652344, + "KL/mean": -180.65243530273438, + "KL/rejected_KL_mean": -213.84967041015625, + "KL/std": 103.51240539550781, + "epoch": 0.6122448979591837, + "fcm_dpo/beta": 0.004570495802909136, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.39448547363281, + "fcm_dpo/q_t": 0.42996299266815186, + "grad_norm": 9.430486679077148, + "learning_rate": 1.975048638084379e-07, + "logits/chosen": 0.8736664056777954, + "logits/rejected": 0.8355212211608887, + "logps/chosen": -198.03952026367188, + "logps/ref_chosen": -50.5843391418457, + "logps/ref_rejected": -65.43156433105469, + "logps/rejected": -279.2812194824219, + "loss": 1.1645, + "margin_dpo/margin_mean": 66.39448547363281, + "margin_dpo/margin_std": 103.71170043945312, + "step": 405 + }, + { + "KL/chosen_KL_mean": -136.18429565429688, + "KL/mean": -185.00283813476562, + "KL/rejected_KL_mean": -233.82138061523438, + "KL/std": 112.89027404785156, + "epoch": 0.6137566137566137, + "fcm_dpo/beta": 0.00454317219555378, + "fcm_dpo/delta": -0.06014251708984375, + "fcm_dpo/margin": 97.63706970214844, + "fcm_dpo/q_t": 0.397684782743454, + "grad_norm": 9.497514724731445, + "learning_rate": 1.9621286303497914e-07, + "logits/chosen": 0.8508515357971191, + "logits/rejected": 0.71308434009552, + "logps/chosen": -185.17990112304688, + "logps/ref_chosen": -48.99560546875, + "logps/ref_rejected": -92.47774505615234, + "logps/rejected": -326.29913330078125, + "loss": 1.0675, + "margin_dpo/margin_mean": 97.63706970214844, + "margin_dpo/margin_std": 124.28600311279297, + "step": 406 + }, + { + "KL/chosen_KL_mean": -183.27198791503906, + "KL/mean": -220.62786865234375, + "KL/rejected_KL_mean": -257.98370361328125, + "KL/std": 134.5868377685547, + "epoch": 0.6152683295540439, + "fcm_dpo/beta": 0.004515849053859711, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 74.71171569824219, + "fcm_dpo/q_t": 0.4221992492675781, + "grad_norm": 11.74845027923584, + "learning_rate": 1.9492236680336483e-07, + "logits/chosen": 0.7110066413879395, + "logits/rejected": 0.6630129814147949, + "logps/chosen": -272.67254638671875, + "logps/ref_chosen": -89.40056610107422, + "logps/ref_rejected": -99.28775024414062, + "logps/rejected": -357.271484375, + "loss": 1.1663, + "margin_dpo/margin_mean": 74.71171569824219, + "margin_dpo/margin_std": 135.02276611328125, + "step": 407 + }, + { + "KL/chosen_KL_mean": -133.894775390625, + "KL/mean": -185.35739135742188, + "KL/rejected_KL_mean": -236.8200225830078, + "KL/std": 104.712646484375, + "epoch": 0.6167800453514739, + "fcm_dpo/beta": 0.004481324460357428, + "fcm_dpo/delta": -0.07704263180494308, + "fcm_dpo/margin": 102.92523956298828, + "fcm_dpo/q_t": 0.3919832408428192, + "grad_norm": 9.033692359924316, + "learning_rate": 1.9363341121154895e-07, + "logits/chosen": 0.8178368806838989, + "logits/rejected": 0.7567524909973145, + "logps/chosen": -188.59869384765625, + "logps/ref_chosen": -54.70391845703125, + "logps/ref_rejected": -73.98648834228516, + "logps/rejected": -310.8065185546875, + "loss": 1.0345, + "margin_dpo/margin_mean": 102.92523956298828, + "margin_dpo/margin_std": 108.17862701416016, + "step": 408 + }, + { + "KL/chosen_KL_mean": -166.08877563476562, + "KL/mean": -189.05276489257812, + "KL/rejected_KL_mean": -212.01673889160156, + "KL/std": 89.52006530761719, + "epoch": 0.618291761148904, + "fcm_dpo/beta": 0.0044467998668551445, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 45.9279670715332, + "fcm_dpo/q_t": 0.45155104994773865, + "grad_norm": 10.081195831298828, + "learning_rate": 1.9234603231438994e-07, + "logits/chosen": 0.8174892663955688, + "logits/rejected": 0.8276025652885437, + "logps/chosen": -228.20700073242188, + "logps/ref_chosen": -62.11822509765625, + "logps/ref_rejected": -61.933509826660156, + "logps/rejected": -273.95025634765625, + "loss": 1.2391, + "margin_dpo/margin_mean": 45.92797088623047, + "margin_dpo/margin_std": 98.16754150390625, + "step": 409 + }, + { + "KL/chosen_KL_mean": -158.0648956298828, + "KL/mean": -199.19512939453125, + "KL/rejected_KL_mean": -240.3253631591797, + "KL/std": 106.66167449951172, + "epoch": 0.6198034769463341, + "fcm_dpo/beta": 0.0044024912640452385, + "fcm_dpo/delta": -0.05007057636976242, + "fcm_dpo/margin": 82.26046752929688, + "fcm_dpo/q_t": 0.4151955246925354, + "grad_norm": 10.520129203796387, + "learning_rate": 1.9106026612264315e-07, + "logits/chosen": 0.8182001113891602, + "logits/rejected": 0.8017312288284302, + "logps/chosen": -219.8675537109375, + "logps/ref_chosen": -61.80266189575195, + "logps/ref_rejected": -76.60002136230469, + "logps/rejected": -316.9253845214844, + "loss": 1.114, + "margin_dpo/margin_mean": 82.26046752929688, + "margin_dpo/margin_std": 108.58341979980469, + "step": 410 + }, + { + "KL/chosen_KL_mean": -163.16366577148438, + "KL/mean": -204.993896484375, + "KL/rejected_KL_mean": -246.82412719726562, + "KL/std": 111.47209930419922, + "epoch": 0.6213151927437641, + "fcm_dpo/beta": 0.0044024912640452385, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 83.66046142578125, + "fcm_dpo/q_t": 0.414519339799881, + "grad_norm": 8.745308876037598, + "learning_rate": 1.8977614860195296e-07, + "logits/chosen": 0.8693751096725464, + "logits/rejected": 0.8266406655311584, + "logps/chosen": -217.6090545654297, + "logps/ref_chosen": -54.44539260864258, + "logps/ref_rejected": -74.5650863647461, + "logps/rejected": -321.38922119140625, + "loss": 1.1133, + "margin_dpo/margin_mean": 83.66046142578125, + "margin_dpo/margin_std": 116.60626220703125, + "step": 411 + }, + { + "KL/chosen_KL_mean": -167.1968231201172, + "KL/mean": -203.74581909179688, + "KL/rejected_KL_mean": -240.29483032226562, + "KL/std": 93.24005126953125, + "epoch": 0.6228269085411943, + "fcm_dpo/beta": 0.0044024912640452385, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 73.09800720214844, + "fcm_dpo/q_t": 0.42384031414985657, + "grad_norm": 9.76529598236084, + "learning_rate": 1.8849371567184662e-07, + "logits/chosen": 0.8335592746734619, + "logits/rejected": 0.7884241342544556, + "logps/chosen": -222.44491577148438, + "logps/ref_chosen": -55.248085021972656, + "logps/ref_rejected": -68.96623229980469, + "logps/rejected": -309.26104736328125, + "loss": 1.1367, + "margin_dpo/margin_mean": 73.09800720214844, + "margin_dpo/margin_std": 99.23159790039062, + "step": 412 + }, + { + "KL/chosen_KL_mean": -190.23448181152344, + "KL/mean": -225.3627471923828, + "KL/rejected_KL_mean": -260.49102783203125, + "KL/std": 115.87466430664062, + "epoch": 0.6243386243386243, + "fcm_dpo/beta": 0.0044024912640452385, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 70.25653076171875, + "fcm_dpo/q_t": 0.42987310886383057, + "grad_norm": 12.271631240844727, + "learning_rate": 1.872130032047302e-07, + "logits/chosen": 0.6928975582122803, + "logits/rejected": 0.6748654842376709, + "logps/chosen": -258.9552307128906, + "logps/ref_chosen": -68.72074890136719, + "logps/ref_rejected": -78.76539611816406, + "logps/rejected": -339.25640869140625, + "loss": 1.1877, + "margin_dpo/margin_mean": 70.25653839111328, + "margin_dpo/margin_std": 137.60073852539062, + "step": 413 + }, + { + "KL/chosen_KL_mean": -167.34481811523438, + "KL/mean": -208.9085235595703, + "KL/rejected_KL_mean": -250.47225952148438, + "KL/std": 124.45516967773438, + "epoch": 0.6258503401360545, + "fcm_dpo/beta": 0.0044024912640452385, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 83.12744140625, + "fcm_dpo/q_t": 0.4144652783870697, + "grad_norm": 9.069655418395996, + "learning_rate": 1.8593404702488436e-07, + "logits/chosen": 0.8730368614196777, + "logits/rejected": 0.8275924921035767, + "logps/chosen": -221.4830322265625, + "logps/ref_chosen": -54.138214111328125, + "logps/ref_rejected": -74.65741729736328, + "logps/rejected": -325.1296691894531, + "loss": 1.116, + "margin_dpo/margin_mean": 83.12744140625, + "margin_dpo/margin_std": 114.78118896484375, + "step": 414 + }, + { + "KL/chosen_KL_mean": -165.86544799804688, + "KL/mean": -197.71322631835938, + "KL/rejected_KL_mean": -229.56100463867188, + "KL/std": 110.82231903076172, + "epoch": 0.6273620559334845, + "fcm_dpo/beta": 0.0044024912640452385, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 63.69554901123047, + "fcm_dpo/q_t": 0.4349156320095062, + "grad_norm": 9.737541198730469, + "learning_rate": 1.846568829074628e-07, + "logits/chosen": 0.8533459901809692, + "logits/rejected": 0.8423031568527222, + "logps/chosen": -221.7840118408203, + "logps/ref_chosen": -55.91856002807617, + "logps/ref_rejected": -61.747703552246094, + "logps/rejected": -291.3087158203125, + "loss": 1.1901, + "margin_dpo/margin_mean": 63.6955451965332, + "margin_dpo/margin_std": 119.05924987792969, + "step": 415 + }, + { + "KL/chosen_KL_mean": -175.8607177734375, + "KL/mean": -207.20225524902344, + "KL/rejected_KL_mean": -238.54379272460938, + "KL/std": 120.81781005859375, + "epoch": 0.6288737717309146, + "fcm_dpo/beta": 0.004381248727440834, + "fcm_dpo/delta": -0.04848504811525345, + "fcm_dpo/margin": 62.68305587768555, + "fcm_dpo/q_t": 0.4369519352912903, + "grad_norm": 10.474597930908203, + "learning_rate": 1.8338154657749128e-07, + "logits/chosen": 0.8692808151245117, + "logits/rejected": 0.8307493329048157, + "logps/chosen": -230.58380126953125, + "logps/ref_chosen": -54.72308349609375, + "logps/ref_rejected": -69.17388916015625, + "logps/rejected": -307.7176818847656, + "loss": 1.1993, + "margin_dpo/margin_mean": 62.68305206298828, + "margin_dpo/margin_std": 111.89498901367188, + "step": 416 + }, + { + "KL/chosen_KL_mean": -185.3282470703125, + "KL/mean": -222.47647094726562, + "KL/rejected_KL_mean": -259.6246643066406, + "KL/std": 111.57913208007812, + "epoch": 0.6303854875283447, + "fcm_dpo/beta": 0.004360006656497717, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 74.29644012451172, + "fcm_dpo/q_t": 0.4229382574558258, + "grad_norm": 10.666007995605469, + "learning_rate": 1.8210807370886849e-07, + "logits/chosen": 0.9554958343505859, + "logits/rejected": 0.9064658880233765, + "logps/chosen": -242.1195068359375, + "logps/ref_chosen": -56.791259765625, + "logps/ref_rejected": -68.7791748046875, + "logps/rejected": -328.40386962890625, + "loss": 1.172, + "margin_dpo/margin_mean": 74.29644012451172, + "margin_dpo/margin_std": 136.17581176757812, + "step": 417 + }, + { + "KL/chosen_KL_mean": -198.63465881347656, + "KL/mean": -230.21011352539062, + "KL/rejected_KL_mean": -261.78558349609375, + "KL/std": 129.6304473876953, + "epoch": 0.6318972033257747, + "fcm_dpo/beta": 0.004330269526690245, + "fcm_dpo/delta": -0.03421874716877937, + "fcm_dpo/margin": 63.15090560913086, + "fcm_dpo/q_t": 0.437247097492218, + "grad_norm": 11.617671966552734, + "learning_rate": 1.8083649992336825e-07, + "logits/chosen": 0.8838375210762024, + "logits/rejected": 0.8873622417449951, + "logps/chosen": -267.7426452636719, + "logps/ref_chosen": -69.10798645019531, + "logps/ref_rejected": -75.09132385253906, + "logps/rejected": -336.87689208984375, + "loss": 1.2179, + "margin_dpo/margin_mean": 63.150909423828125, + "margin_dpo/margin_std": 133.4095458984375, + "step": 418 + }, + { + "KL/chosen_KL_mean": -160.0953369140625, + "KL/mean": -207.4534912109375, + "KL/rejected_KL_mean": -254.81167602539062, + "KL/std": 113.61080932617188, + "epoch": 0.6334089191232048, + "fcm_dpo/beta": 0.004330269526690245, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.71630859375, + "fcm_dpo/q_t": 0.4075653553009033, + "grad_norm": 10.965876579284668, + "learning_rate": 1.7956686078964255e-07, + "logits/chosen": 0.7953141927719116, + "logits/rejected": 0.7639617919921875, + "logps/chosen": -218.26710510253906, + "logps/ref_chosen": -58.1717643737793, + "logps/ref_rejected": -71.67066955566406, + "logps/rejected": -326.4823303222656, + "loss": 1.0914, + "margin_dpo/margin_mean": 94.71630859375, + "margin_dpo/margin_std": 131.05889892578125, + "step": 419 + }, + { + "KL/chosen_KL_mean": -193.0113525390625, + "KL/mean": -215.9877166748047, + "KL/rejected_KL_mean": -238.96408081054688, + "KL/std": 115.93824768066406, + "epoch": 0.6349206349206349, + "fcm_dpo/beta": 0.004330269526690245, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 45.952728271484375, + "fcm_dpo/q_t": 0.45506352186203003, + "grad_norm": 11.78740119934082, + "learning_rate": 1.782991918222275e-07, + "logits/chosen": 0.8286752700805664, + "logits/rejected": 0.7962324619293213, + "logps/chosen": -250.0648651123047, + "logps/ref_chosen": -57.05351257324219, + "logps/ref_rejected": -62.670982360839844, + "logps/rejected": -301.63507080078125, + "loss": 1.28, + "margin_dpo/margin_mean": 45.952728271484375, + "margin_dpo/margin_std": 135.70907592773438, + "step": 420 + }, + { + "KL/chosen_KL_mean": -182.4674835205078, + "KL/mean": -216.71278381347656, + "KL/rejected_KL_mean": -250.95806884765625, + "KL/std": 116.42398834228516, + "epoch": 0.636432350718065, + "fcm_dpo/beta": 0.004330269526690245, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 68.49060821533203, + "fcm_dpo/q_t": 0.43248650431632996, + "grad_norm": 11.323017120361328, + "learning_rate": 1.7703352848054887e-07, + "logits/chosen": 0.8234415054321289, + "logits/rejected": 0.7761733531951904, + "logps/chosen": -239.7907257080078, + "logps/ref_chosen": -57.32324981689453, + "logps/ref_rejected": -75.33782958984375, + "logps/rejected": -326.2958984375, + "loss": 1.2224, + "margin_dpo/margin_mean": 68.49061584472656, + "margin_dpo/margin_std": 157.9302978515625, + "step": 421 + }, + { + "KL/chosen_KL_mean": -161.13418579101562, + "KL/mean": -204.26010131835938, + "KL/rejected_KL_mean": -247.38600158691406, + "KL/std": 114.96134948730469, + "epoch": 0.6379440665154951, + "fcm_dpo/beta": 0.004321150481700897, + "fcm_dpo/delta": -0.021103385835886, + "fcm_dpo/margin": 86.2518310546875, + "fcm_dpo/q_t": 0.41321080923080444, + "grad_norm": 11.155305862426758, + "learning_rate": 1.7576990616793137e-07, + "logits/chosen": 0.829655647277832, + "logits/rejected": 0.8312000036239624, + "logps/chosen": -228.1917724609375, + "logps/ref_chosen": -67.05757141113281, + "logps/ref_rejected": -72.12803649902344, + "logps/rejected": -319.5140380859375, + "loss": 1.1045, + "margin_dpo/margin_mean": 86.2518310546875, + "margin_dpo/margin_std": 112.21836853027344, + "step": 422 + }, + { + "KL/chosen_KL_mean": -158.37295532226562, + "KL/mean": -207.0155029296875, + "KL/rejected_KL_mean": -255.65805053710938, + "KL/std": 117.40293884277344, + "epoch": 0.6394557823129252, + "fcm_dpo/beta": 0.004282133188098669, + "fcm_dpo/delta": -0.0347890630364418, + "fcm_dpo/margin": 97.28510284423828, + "fcm_dpo/q_t": 0.4046144485473633, + "grad_norm": 10.667545318603516, + "learning_rate": 1.745083602306071e-07, + "logits/chosen": 0.8965709209442139, + "logits/rejected": 0.8465409874916077, + "logps/chosen": -212.43463134765625, + "logps/ref_chosen": -54.06167221069336, + "logps/ref_rejected": -76.64092254638672, + "logps/rejected": -332.2989807128906, + "loss": 1.081, + "margin_dpo/margin_mean": 97.28510284423828, + "margin_dpo/margin_std": 126.60137939453125, + "step": 423 + }, + { + "KL/chosen_KL_mean": -172.59568786621094, + "KL/mean": -219.48318481445312, + "KL/rejected_KL_mean": -266.37066650390625, + "KL/std": 107.72590637207031, + "epoch": 0.6409674981103552, + "fcm_dpo/beta": 0.004237360320985317, + "fcm_dpo/delta": -0.05255415290594101, + "fcm_dpo/margin": 93.7750015258789, + "fcm_dpo/q_t": 0.40855535864830017, + "grad_norm": 13.741423606872559, + "learning_rate": 1.7324892595672804e-07, + "logits/chosen": 0.7810956239700317, + "logits/rejected": 0.7487587928771973, + "logps/chosen": -226.20455932617188, + "logps/ref_chosen": -53.60887145996094, + "logps/ref_rejected": -79.2139892578125, + "logps/rejected": -345.58465576171875, + "loss": 1.0937, + "margin_dpo/margin_mean": 93.7750015258789, + "margin_dpo/margin_std": 124.07484436035156, + "step": 424 + }, + { + "KL/chosen_KL_mean": -170.74240112304688, + "KL/mean": -205.09945678710938, + "KL/rejected_KL_mean": -239.45651245117188, + "KL/std": 103.23504638671875, + "epoch": 0.6424792139077853, + "fcm_dpo/beta": 0.004237360320985317, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 68.714111328125, + "fcm_dpo/q_t": 0.43254685401916504, + "grad_norm": 9.86276912689209, + "learning_rate": 1.7199163857537824e-07, + "logits/chosen": 0.8737081289291382, + "logits/rejected": 0.8555968403816223, + "logps/chosen": -229.15707397460938, + "logps/ref_chosen": -58.41468048095703, + "logps/ref_rejected": -66.59054565429688, + "logps/rejected": -306.04705810546875, + "loss": 1.1778, + "margin_dpo/margin_mean": 68.714111328125, + "margin_dpo/margin_std": 119.1408462524414, + "step": 425 + }, + { + "KL/chosen_KL_mean": -205.45645141601562, + "KL/mean": -226.54766845703125, + "KL/rejected_KL_mean": -247.63888549804688, + "KL/std": 116.4287109375, + "epoch": 0.6439909297052154, + "fcm_dpo/beta": 0.004237360320985317, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 42.18241882324219, + "fcm_dpo/q_t": 0.4587920904159546, + "grad_norm": 13.61494255065918, + "learning_rate": 1.7073653325558828e-07, + "logits/chosen": 0.7965115308761597, + "logits/rejected": 0.7977316379547119, + "logps/chosen": -277.1646728515625, + "logps/ref_chosen": -71.70822143554688, + "logps/ref_rejected": -73.57725524902344, + "logps/rejected": -321.21612548828125, + "loss": 1.3067, + "margin_dpo/margin_mean": 42.18241882324219, + "margin_dpo/margin_std": 147.598876953125, + "step": 426 + }, + { + "KL/chosen_KL_mean": -193.80029296875, + "KL/mean": -232.99258422851562, + "KL/rejected_KL_mean": -272.1848449707031, + "KL/std": 128.01194763183594, + "epoch": 0.6455026455026455, + "fcm_dpo/beta": 0.004237360320985317, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 78.384521484375, + "fcm_dpo/q_t": 0.42532289028167725, + "grad_norm": 11.07363224029541, + "learning_rate": 1.6948364510535218e-07, + "logits/chosen": 0.841190755367279, + "logits/rejected": 0.7872822880744934, + "logps/chosen": -252.4430694580078, + "logps/ref_chosen": -58.64276885986328, + "logps/ref_rejected": -86.25437927246094, + "logps/rejected": -358.439208984375, + "loss": 1.1607, + "margin_dpo/margin_mean": 78.38452911376953, + "margin_dpo/margin_std": 138.72189331054688, + "step": 427 + }, + { + "KL/chosen_KL_mean": -189.307861328125, + "KL/mean": -228.06832885742188, + "KL/rejected_KL_mean": -266.82879638671875, + "KL/std": 130.09091186523438, + "epoch": 0.6470143613000756, + "fcm_dpo/beta": 0.004237360320985317, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 77.52091217041016, + "fcm_dpo/q_t": 0.4252082407474518, + "grad_norm": 9.706740379333496, + "learning_rate": 1.6823300917064458e-07, + "logits/chosen": 0.7988708019256592, + "logits/rejected": 0.7730382084846497, + "logps/chosen": -255.90391540527344, + "logps/ref_chosen": -66.5960464477539, + "logps/ref_rejected": -82.3941650390625, + "logps/rejected": -349.22296142578125, + "loss": 1.1705, + "margin_dpo/margin_mean": 77.52090454101562, + "margin_dpo/margin_std": 139.55557250976562, + "step": 428 + }, + { + "KL/chosen_KL_mean": -191.9099884033203, + "KL/mean": -220.21624755859375, + "KL/rejected_KL_mean": -248.52249145507812, + "KL/std": 103.39564514160156, + "epoch": 0.6485260770975056, + "fcm_dpo/beta": 0.004237360320985317, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 56.61249542236328, + "fcm_dpo/q_t": 0.44227659702301025, + "grad_norm": 10.24012279510498, + "learning_rate": 1.669846604344412e-07, + "logits/chosen": 0.8254667520523071, + "logits/rejected": 0.8455654382705688, + "logps/chosen": -248.91969299316406, + "logps/ref_chosen": -57.00970458984375, + "logps/ref_rejected": -59.86549377441406, + "logps/rejected": -308.38800048828125, + "loss": 1.2237, + "margin_dpo/margin_mean": 56.61249542236328, + "margin_dpo/margin_std": 118.35189819335938, + "step": 429 + }, + { + "KL/chosen_KL_mean": -168.93666076660156, + "KL/mean": -217.52066040039062, + "KL/rejected_KL_mean": -266.10467529296875, + "KL/std": 104.36176300048828, + "epoch": 0.6500377928949358, + "fcm_dpo/beta": 0.004237360320985317, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 97.16799926757812, + "fcm_dpo/q_t": 0.402678519487381, + "grad_norm": 11.644641876220703, + "learning_rate": 1.6573863381573954e-07, + "logits/chosen": 0.7442194223403931, + "logits/rejected": 0.7441260814666748, + "logps/chosen": -228.49984741210938, + "logps/ref_chosen": -59.563194274902344, + "logps/ref_rejected": -70.52289581298828, + "logps/rejected": -336.6275634765625, + "loss": 1.0823, + "margin_dpo/margin_mean": 97.16799926757812, + "margin_dpo/margin_std": 123.15974426269531, + "step": 430 + }, + { + "KL/chosen_KL_mean": -166.54751586914062, + "KL/mean": -204.51138305664062, + "KL/rejected_KL_mean": -242.4752197265625, + "KL/std": 112.0499267578125, + "epoch": 0.6515495086923658, + "fcm_dpo/beta": 0.004237360320985317, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 75.92768859863281, + "fcm_dpo/q_t": 0.4258786141872406, + "grad_norm": 10.839800834655762, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": 0.8408886790275574, + "logits/rejected": 0.7930970191955566, + "logps/chosen": -216.7478485107422, + "logps/ref_chosen": -50.20032501220703, + "logps/ref_rejected": -77.81680297851562, + "logps/rejected": -320.2920227050781, + "loss": 1.1613, + "margin_dpo/margin_mean": 75.92768096923828, + "margin_dpo/margin_std": 128.07762145996094, + "step": 431 + }, + { + "KL/chosen_KL_mean": -177.85556030273438, + "KL/mean": -216.429931640625, + "KL/rejected_KL_mean": -255.00430297851562, + "KL/std": 112.36427307128906, + "epoch": 0.6530612244897959, + "fcm_dpo/beta": 0.004237360320985317, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 77.14874267578125, + "fcm_dpo/q_t": 0.42348039150238037, + "grad_norm": 11.070759773254395, + "learning_rate": 1.632536862810844e-07, + "logits/chosen": 0.8612329959869385, + "logits/rejected": 0.8160426616668701, + "logps/chosen": -239.51832580566406, + "logps/ref_chosen": -61.662757873535156, + "logps/ref_rejected": -83.94496154785156, + "logps/rejected": -338.94927978515625, + "loss": 1.1634, + "margin_dpo/margin_mean": 77.14874267578125, + "margin_dpo/margin_std": 134.41839599609375, + "step": 432 + }, + { + "KL/chosen_KL_mean": -174.31484985351562, + "KL/mean": -217.500732421875, + "KL/rejected_KL_mean": -260.6866455078125, + "KL/std": 102.90592956542969, + "epoch": 0.654572940287226, + "fcm_dpo/beta": 0.004237360320985317, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 86.37177276611328, + "fcm_dpo/q_t": 0.4140329957008362, + "grad_norm": 11.327740669250488, + "learning_rate": 1.6201483487445515e-07, + "logits/chosen": 0.8867149353027344, + "logits/rejected": 0.8934628963470459, + "logps/chosen": -238.04403686523438, + "logps/ref_chosen": -63.72917938232422, + "logps/ref_rejected": -65.8391342163086, + "logps/rejected": -326.5257568359375, + "loss": 1.1126, + "margin_dpo/margin_mean": 86.37178039550781, + "margin_dpo/margin_std": 117.17826843261719, + "step": 433 + }, + { + "KL/chosen_KL_mean": -147.67396545410156, + "KL/mean": -197.6832275390625, + "KL/rejected_KL_mean": -247.69247436523438, + "KL/std": 115.82525634765625, + "epoch": 0.656084656084656, + "fcm_dpo/beta": 0.004173349589109421, + "fcm_dpo/delta": -0.0761074498295784, + "fcm_dpo/margin": 100.01850891113281, + "fcm_dpo/q_t": 0.4028673768043518, + "grad_norm": 10.230304718017578, + "learning_rate": 1.6077844460203204e-07, + "logits/chosen": 0.945635199546814, + "logits/rejected": 0.8929431438446045, + "logps/chosen": -195.64727783203125, + "logps/ref_chosen": -47.97331619262695, + "logps/ref_rejected": -72.51132202148438, + "logps/rejected": -320.20379638671875, + "loss": 1.0938, + "margin_dpo/margin_mean": 100.01850891113281, + "margin_dpo/margin_std": 138.5220489501953, + "step": 434 + }, + { + "KL/chosen_KL_mean": -179.42828369140625, + "KL/mean": -215.53460693359375, + "KL/rejected_KL_mean": -251.64093017578125, + "KL/std": 111.70719909667969, + "epoch": 0.6575963718820862, + "fcm_dpo/beta": 0.004173349589109421, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 72.21266174316406, + "fcm_dpo/q_t": 0.42970752716064453, + "grad_norm": 11.32288646697998, + "learning_rate": 1.5954455004830878e-07, + "logits/chosen": 0.9255178570747375, + "logits/rejected": 0.8949824571609497, + "logps/chosen": -236.488525390625, + "logps/ref_chosen": -57.06024932861328, + "logps/ref_rejected": -71.69146728515625, + "logps/rejected": -323.3323974609375, + "loss": 1.1787, + "margin_dpo/margin_mean": 72.21266174316406, + "margin_dpo/margin_std": 129.89993286132812, + "step": 435 + }, + { + "KL/chosen_KL_mean": -183.29244995117188, + "KL/mean": -212.61312866210938, + "KL/rejected_KL_mean": -241.93377685546875, + "KL/std": 108.2564697265625, + "epoch": 0.6591080876795162, + "fcm_dpo/beta": 0.004173349589109421, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 58.641334533691406, + "fcm_dpo/q_t": 0.44333615899086, + "grad_norm": 12.016843795776367, + "learning_rate": 1.5831318572796847e-07, + "logits/chosen": 0.8693698644638062, + "logits/rejected": 0.8239619135856628, + "logps/chosen": -239.45050048828125, + "logps/ref_chosen": -56.158050537109375, + "logps/ref_rejected": -67.63787841796875, + "logps/rejected": -309.5716552734375, + "loss": 1.2306, + "margin_dpo/margin_mean": 58.641334533691406, + "margin_dpo/margin_std": 132.94003295898438, + "step": 436 + }, + { + "KL/chosen_KL_mean": -185.86080932617188, + "KL/mean": -225.59451293945312, + "KL/rejected_KL_mean": -265.3282165527344, + "KL/std": 121.49739074707031, + "epoch": 0.6606198034769464, + "fcm_dpo/beta": 0.004142164718359709, + "fcm_dpo/delta": -0.03750201314687729, + "fcm_dpo/margin": 79.46741485595703, + "fcm_dpo/q_t": 0.42540091276168823, + "grad_norm": 13.124794960021973, + "learning_rate": 1.5708438608491815e-07, + "logits/chosen": 0.8585692048072815, + "logits/rejected": 0.7549466490745544, + "logps/chosen": -242.84658813476562, + "logps/ref_chosen": -56.98578643798828, + "logps/ref_rejected": -85.61524963378906, + "logps/rejected": -350.9434814453125, + "loss": 1.1855, + "margin_dpo/margin_mean": 79.46741485595703, + "margin_dpo/margin_std": 155.61752319335938, + "step": 437 + }, + { + "KL/chosen_KL_mean": -160.57635498046875, + "KL/mean": -209.36996459960938, + "KL/rejected_KL_mean": -258.16357421875, + "KL/std": 124.45043182373047, + "epoch": 0.6621315192743764, + "fcm_dpo/beta": 0.004129474051296711, + "fcm_dpo/delta": -0.030732562765479088, + "fcm_dpo/margin": 97.58722686767578, + "fcm_dpo/q_t": 0.41016703844070435, + "grad_norm": 10.8823881149292, + "learning_rate": 1.558581854913253e-07, + "logits/chosen": 0.9030640125274658, + "logits/rejected": 0.8500392436981201, + "logps/chosen": -201.8541259765625, + "logps/ref_chosen": -41.27777862548828, + "logps/ref_rejected": -65.33840942382812, + "logps/rejected": -323.5019836425781, + "loss": 1.0969, + "margin_dpo/margin_mean": 97.58723449707031, + "margin_dpo/margin_std": 140.27755737304688, + "step": 438 + }, + { + "KL/chosen_KL_mean": -190.50978088378906, + "KL/mean": -234.99398803710938, + "KL/rejected_KL_mean": -279.47821044921875, + "KL/std": 127.0751953125, + "epoch": 0.6636432350718064, + "fcm_dpo/beta": 0.004095879383385181, + "fcm_dpo/delta": -0.05103587731719017, + "fcm_dpo/margin": 88.96844482421875, + "fcm_dpo/q_t": 0.4150954484939575, + "grad_norm": 11.545726776123047, + "learning_rate": 1.5463461824665658e-07, + "logits/chosen": 0.7603689432144165, + "logits/rejected": 0.7392828464508057, + "logps/chosen": -271.92742919921875, + "logps/ref_chosen": -81.41764831542969, + "logps/ref_rejected": -94.72309875488281, + "logps/rejected": -374.2012939453125, + "loss": 1.1227, + "margin_dpo/margin_mean": 88.96844482421875, + "margin_dpo/margin_std": 126.95797729492188, + "step": 439 + }, + { + "KL/chosen_KL_mean": -157.71478271484375, + "KL/mean": -202.0891876220703, + "KL/rejected_KL_mean": -246.4635772705078, + "KL/std": 111.21897888183594, + "epoch": 0.6651549508692366, + "fcm_dpo/beta": 0.004074975848197937, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 88.74879455566406, + "fcm_dpo/q_t": 0.41615790128707886, + "grad_norm": 13.960528373718262, + "learning_rate": 1.534137185767178e-07, + "logits/chosen": 0.8458383083343506, + "logits/rejected": 0.7661876678466797, + "logps/chosen": -200.2529754638672, + "logps/ref_chosen": -42.538185119628906, + "logps/ref_rejected": -69.78813934326172, + "logps/rejected": -316.251708984375, + "loss": 1.1194, + "margin_dpo/margin_mean": 88.74879455566406, + "margin_dpo/margin_std": 124.50497436523438, + "step": 440 + }, + { + "KL/chosen_KL_mean": -165.97714233398438, + "KL/mean": -213.51602172851562, + "KL/rejected_KL_mean": -261.05487060546875, + "KL/std": 123.1169204711914, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.004050433170050383, + "fcm_dpo/delta": -0.030205076560378075, + "fcm_dpo/margin": 95.07774353027344, + "fcm_dpo/q_t": 0.4091061055660248, + "grad_norm": 12.10502815246582, + "learning_rate": 1.521955206326976e-07, + "logits/chosen": 0.8075554370880127, + "logits/rejected": 0.7362128496170044, + "logps/chosen": -223.57037353515625, + "logps/ref_chosen": -57.593223571777344, + "logps/ref_rejected": -84.82878875732422, + "logps/rejected": -345.8836669921875, + "loss": 1.0876, + "margin_dpo/margin_mean": 95.07774353027344, + "margin_dpo/margin_std": 111.09236907958984, + "step": 441 + }, + { + "KL/chosen_KL_mean": -198.12374877929688, + "KL/mean": -246.94883728027344, + "KL/rejected_KL_mean": -295.77392578125, + "KL/std": 124.66996765136719, + "epoch": 0.6681783824640968, + "fcm_dpo/beta": 0.004028483759611845, + "fcm_dpo/delta": -0.0271687563508749, + "fcm_dpo/margin": 97.65019226074219, + "fcm_dpo/q_t": 0.40914517641067505, + "grad_norm": 12.60295295715332, + "learning_rate": 1.5098005849021078e-07, + "logits/chosen": 0.8383224606513977, + "logits/rejected": 0.8079402446746826, + "logps/chosen": -265.5849609375, + "logps/ref_chosen": -67.46121978759766, + "logps/ref_rejected": -89.0693588256836, + "logps/rejected": -384.8432922363281, + "loss": 1.0944, + "margin_dpo/margin_mean": 97.65019226074219, + "margin_dpo/margin_std": 127.72940063476562, + "step": 442 + }, + { + "KL/chosen_KL_mean": -163.28781127929688, + "KL/mean": -221.1153106689453, + "KL/rejected_KL_mean": -278.94281005859375, + "KL/std": 130.93421936035156, + "epoch": 0.6696900982615268, + "fcm_dpo/beta": 0.003985672257840633, + "fcm_dpo/delta": -0.0640106350183487, + "fcm_dpo/margin": 115.65498352050781, + "fcm_dpo/q_t": 0.3956124186515808, + "grad_norm": 11.192153930664062, + "learning_rate": 1.4976736614834662e-07, + "logits/chosen": 0.8276779055595398, + "logits/rejected": 0.7804521322250366, + "logps/chosen": -218.08392333984375, + "logps/ref_chosen": -54.79610061645508, + "logps/ref_rejected": -77.80781555175781, + "logps/rejected": -356.7506103515625, + "loss": 1.053, + "margin_dpo/margin_mean": 115.65498352050781, + "margin_dpo/margin_std": 147.14096069335938, + "step": 443 + }, + { + "KL/chosen_KL_mean": -195.40420532226562, + "KL/mean": -227.3412322998047, + "KL/rejected_KL_mean": -259.27825927734375, + "KL/std": 135.2494354248047, + "epoch": 0.671201814058957, + "fcm_dpo/beta": 0.00397723913192749, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 63.8740348815918, + "fcm_dpo/q_t": 0.4439532160758972, + "grad_norm": 10.997804641723633, + "learning_rate": 1.4855747752871654e-07, + "logits/chosen": 0.8664942979812622, + "logits/rejected": 0.7986875772476196, + "logps/chosen": -254.15325927734375, + "logps/ref_chosen": -58.749061584472656, + "logps/ref_rejected": -86.87396240234375, + "logps/rejected": -346.1522216796875, + "loss": 1.2352, + "margin_dpo/margin_mean": 63.8740348815918, + "margin_dpo/margin_std": 151.58612060546875, + "step": 444 + }, + { + "KL/chosen_KL_mean": -186.91659545898438, + "KL/mean": -236.37582397460938, + "KL/rejected_KL_mean": -285.83502197265625, + "KL/std": 114.96759796142578, + "epoch": 0.672713529856387, + "fcm_dpo/beta": 0.00397723913192749, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 98.91845703125, + "fcm_dpo/q_t": 0.40706494450569153, + "grad_norm": 9.45373821258545, + "learning_rate": 1.473504264745062e-07, + "logits/chosen": 0.8414635062217712, + "logits/rejected": 0.8434320092201233, + "logps/chosen": -247.8340301513672, + "logps/ref_chosen": -60.91743850708008, + "logps/ref_rejected": -71.5637435913086, + "logps/rejected": -357.3988037109375, + "loss": 1.0856, + "margin_dpo/margin_mean": 98.91845703125, + "margin_dpo/margin_std": 119.76266479492188, + "step": 445 + }, + { + "KL/chosen_KL_mean": -168.513916015625, + "KL/mean": -217.90615844726562, + "KL/rejected_KL_mean": -267.29840087890625, + "KL/std": 118.74134826660156, + "epoch": 0.674225245653817, + "fcm_dpo/beta": 0.00394941633567214, + "fcm_dpo/delta": -0.07044916599988937, + "fcm_dpo/margin": 98.78448486328125, + "fcm_dpo/q_t": 0.4086703062057495, + "grad_norm": 9.038079261779785, + "learning_rate": 1.461462467495284e-07, + "logits/chosen": 0.8681352138519287, + "logits/rejected": 0.807909369468689, + "logps/chosen": -217.3131561279297, + "logps/ref_chosen": -48.79924774169922, + "logps/ref_rejected": -71.8719482421875, + "logps/rejected": -339.17034912109375, + "loss": 1.0904, + "margin_dpo/margin_mean": 98.78448486328125, + "margin_dpo/margin_std": 117.52779388427734, + "step": 446 + }, + { + "KL/chosen_KL_mean": -164.670166015625, + "KL/mean": -223.03323364257812, + "KL/rejected_KL_mean": -281.39630126953125, + "KL/std": 118.19184875488281, + "epoch": 0.6757369614512472, + "fcm_dpo/beta": 0.0038791203405708075, + "fcm_dpo/delta": -0.054448507726192474, + "fcm_dpo/margin": 116.72611999511719, + "fcm_dpo/q_t": 0.39377325773239136, + "grad_norm": 9.447699546813965, + "learning_rate": 1.4494497203727843e-07, + "logits/chosen": 0.8478422164916992, + "logits/rejected": 0.7641305923461914, + "logps/chosen": -218.35289001464844, + "logps/ref_chosen": -53.682716369628906, + "logps/ref_rejected": -88.17315673828125, + "logps/rejected": -369.5694580078125, + "loss": 1.0486, + "margin_dpo/margin_mean": 116.72611999511719, + "margin_dpo/margin_std": 133.0546875, + "step": 447 + }, + { + "KL/chosen_KL_mean": -182.4579620361328, + "KL/mean": -227.93060302734375, + "KL/rejected_KL_mean": -273.4031982421875, + "KL/std": 124.02155303955078, + "epoch": 0.6772486772486772, + "fcm_dpo/beta": 0.0038791203405708075, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 90.94525146484375, + "fcm_dpo/q_t": 0.4181848466396332, + "grad_norm": 8.76012134552002, + "learning_rate": 1.4374663593999256e-07, + "logits/chosen": 0.8726906776428223, + "logits/rejected": 0.837250828742981, + "logps/chosen": -236.209228515625, + "logps/ref_chosen": -53.75125503540039, + "logps/ref_rejected": -77.17623901367188, + "logps/rejected": -350.5794677734375, + "loss": 1.1278, + "margin_dpo/margin_mean": 90.94525146484375, + "margin_dpo/margin_std": 134.71463012695312, + "step": 448 + }, + { + "KL/chosen_KL_mean": -219.57989501953125, + "KL/mean": -242.92726135253906, + "KL/rejected_KL_mean": -266.2746276855469, + "KL/std": 132.32901000976562, + "epoch": 0.6787603930461074, + "fcm_dpo/beta": 0.0038791203405708075, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 46.694740295410156, + "fcm_dpo/q_t": 0.45708128809928894, + "grad_norm": 16.812198638916016, + "learning_rate": 1.4255127197770707e-07, + "logits/chosen": 0.7664126753807068, + "logits/rejected": 0.7682603597640991, + "logps/chosen": -295.40728759765625, + "logps/ref_chosen": -75.82737731933594, + "logps/ref_rejected": -82.20687866210938, + "logps/rejected": -348.48150634765625, + "loss": 1.2788, + "margin_dpo/margin_mean": 46.69473648071289, + "margin_dpo/margin_std": 132.43252563476562, + "step": 449 + }, + { + "KL/chosen_KL_mean": -174.35791015625, + "KL/mean": -214.56973266601562, + "KL/rejected_KL_mean": -254.78155517578125, + "KL/std": 129.234130859375, + "epoch": 0.6802721088435374, + "fcm_dpo/beta": 0.0038791203405708075, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.42366027832031, + "fcm_dpo/q_t": 0.4304322600364685, + "grad_norm": 11.050307273864746, + "learning_rate": 1.4135891358732205e-07, + "logits/chosen": 0.9499484300613403, + "logits/rejected": 0.8608253002166748, + "logps/chosen": -221.4736328125, + "logps/ref_chosen": -47.11572265625, + "logps/ref_rejected": -78.7546615600586, + "logps/rejected": -333.5362243652344, + "loss": 1.1788, + "margin_dpo/margin_mean": 80.42366790771484, + "margin_dpo/margin_std": 149.21299743652344, + "step": 450 + }, + { + "KL/chosen_KL_mean": -178.43597412109375, + "KL/mean": -214.04747009277344, + "KL/rejected_KL_mean": -249.65896606445312, + "KL/std": 120.39581298828125, + "epoch": 0.6817838246409675, + "fcm_dpo/beta": 0.0038791203405708075, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 71.22297668457031, + "fcm_dpo/q_t": 0.4351150393486023, + "grad_norm": 10.856236457824707, + "learning_rate": 1.4016959412166437e-07, + "logits/chosen": 0.8059324026107788, + "logits/rejected": 0.7745248079299927, + "logps/chosen": -241.7864227294922, + "logps/ref_chosen": -63.350440979003906, + "logps/ref_rejected": -76.28530883789062, + "logps/rejected": -325.94427490234375, + "loss": 1.1836, + "margin_dpo/margin_mean": 71.22297668457031, + "margin_dpo/margin_std": 123.11032104492188, + "step": 451 + }, + { + "KL/chosen_KL_mean": -180.02713012695312, + "KL/mean": -219.95492553710938, + "KL/rejected_KL_mean": -259.88275146484375, + "KL/std": 116.23613739013672, + "epoch": 0.6832955404383976, + "fcm_dpo/beta": 0.0038791203405708075, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 79.85560607910156, + "fcm_dpo/q_t": 0.42794910073280334, + "grad_norm": 13.129495620727539, + "learning_rate": 1.3898334684855645e-07, + "logits/chosen": 0.8197082281112671, + "logits/rejected": 0.7570655345916748, + "logps/chosen": -235.6129608154297, + "logps/ref_chosen": -55.58583450317383, + "logps/ref_rejected": -77.68738555908203, + "logps/rejected": -337.57012939453125, + "loss": 1.1709, + "margin_dpo/margin_mean": 79.85560607910156, + "margin_dpo/margin_std": 140.17767333984375, + "step": 452 + }, + { + "KL/chosen_KL_mean": -182.94837951660156, + "KL/mean": -218.63784790039062, + "KL/rejected_KL_mean": -254.3273162841797, + "KL/std": 119.77302551269531, + "epoch": 0.6848072562358276, + "fcm_dpo/beta": 0.0038791203405708075, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 71.37896728515625, + "fcm_dpo/q_t": 0.43533623218536377, + "grad_norm": 10.636575698852539, + "learning_rate": 1.3780020494988445e-07, + "logits/chosen": 0.8267333507537842, + "logits/rejected": 0.8144370913505554, + "logps/chosen": -244.72657775878906, + "logps/ref_chosen": -61.778202056884766, + "logps/ref_rejected": -71.51403045654297, + "logps/rejected": -325.8413391113281, + "loss": 1.1923, + "margin_dpo/margin_mean": 71.37896728515625, + "margin_dpo/margin_std": 132.53733825683594, + "step": 453 + }, + { + "KL/chosen_KL_mean": -163.52182006835938, + "KL/mean": -207.54925537109375, + "KL/rejected_KL_mean": -251.57669067382812, + "KL/std": 117.58868408203125, + "epoch": 0.6863189720332578, + "fcm_dpo/beta": 0.0038791203405708075, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 88.05487060546875, + "fcm_dpo/q_t": 0.4201856553554535, + "grad_norm": 9.357555389404297, + "learning_rate": 1.366202015206706e-07, + "logits/chosen": 0.8628107309341431, + "logits/rejected": 0.837904691696167, + "logps/chosen": -215.11697387695312, + "logps/ref_chosen": -51.59515380859375, + "logps/ref_rejected": -63.96732711791992, + "logps/rejected": -315.54400634765625, + "loss": 1.1354, + "margin_dpo/margin_mean": 88.05487060546875, + "margin_dpo/margin_std": 133.06265258789062, + "step": 454 + }, + { + "KL/chosen_KL_mean": -196.3070068359375, + "KL/mean": -236.11773681640625, + "KL/rejected_KL_mean": -275.928466796875, + "KL/std": 121.60128784179688, + "epoch": 0.6878306878306878, + "fcm_dpo/beta": 0.0038791203405708075, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 79.6214599609375, + "fcm_dpo/q_t": 0.42936086654663086, + "grad_norm": 11.233712196350098, + "learning_rate": 1.354433695681474e-07, + "logits/chosen": 0.7930549383163452, + "logits/rejected": 0.7773592472076416, + "logps/chosen": -266.958740234375, + "logps/ref_chosen": -70.65170288085938, + "logps/ref_rejected": -77.44276428222656, + "logps/rejected": -353.3712463378906, + "loss": 1.1624, + "margin_dpo/margin_mean": 79.6214599609375, + "margin_dpo/margin_std": 131.82275390625, + "step": 455 + }, + { + "KL/chosen_KL_mean": -184.79290771484375, + "KL/mean": -225.66094970703125, + "KL/rejected_KL_mean": -266.52899169921875, + "KL/std": 121.29298400878906, + "epoch": 0.6893424036281179, + "fcm_dpo/beta": 0.0038791203405708075, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 81.73609161376953, + "fcm_dpo/q_t": 0.42831528186798096, + "grad_norm": 13.263677597045898, + "learning_rate": 1.3426974201083439e-07, + "logits/chosen": 0.7968763709068298, + "logits/rejected": 0.7505416870117188, + "logps/chosen": -241.19119262695312, + "logps/ref_chosen": -56.398284912109375, + "logps/ref_rejected": -82.61642456054688, + "logps/rejected": -349.14544677734375, + "loss": 1.1627, + "margin_dpo/margin_mean": 81.73609924316406, + "margin_dpo/margin_std": 141.47987365722656, + "step": 456 + }, + { + "KL/chosen_KL_mean": -181.234130859375, + "KL/mean": -221.42344665527344, + "KL/rejected_KL_mean": -261.61279296875, + "KL/std": 125.76825714111328, + "epoch": 0.690854119425548, + "fcm_dpo/beta": 0.0038791203405708075, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.37864685058594, + "fcm_dpo/q_t": 0.4260377287864685, + "grad_norm": 12.457381248474121, + "learning_rate": 1.3309935167761717e-07, + "logits/chosen": 0.9459880590438843, + "logits/rejected": 0.8794763684272766, + "logps/chosen": -225.9547119140625, + "logps/ref_chosen": -44.72057342529297, + "logps/ref_rejected": -68.1158676147461, + "logps/rejected": -329.7286376953125, + "loss": 1.1543, + "margin_dpo/margin_mean": 80.37864685058594, + "margin_dpo/margin_std": 124.81919860839844, + "step": 457 + }, + { + "KL/chosen_KL_mean": -174.767822265625, + "KL/mean": -222.24642944335938, + "KL/rejected_KL_mean": -269.72503662109375, + "KL/std": 126.41743469238281, + "epoch": 0.6923658352229781, + "fcm_dpo/beta": 0.0038791203405708075, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.95720672607422, + "fcm_dpo/q_t": 0.41461867094039917, + "grad_norm": 10.82316780090332, + "learning_rate": 1.3193223130682936e-07, + "logits/chosen": 0.8550417423248291, + "logits/rejected": 0.7658358812332153, + "logps/chosen": -224.77352905273438, + "logps/ref_chosen": -50.00569152832031, + "logps/ref_rejected": -87.50015258789062, + "logps/rejected": -357.2251892089844, + "loss": 1.1235, + "margin_dpo/margin_mean": 94.95721435546875, + "margin_dpo/margin_std": 142.54466247558594, + "step": 458 + }, + { + "KL/chosen_KL_mean": -177.70147705078125, + "KL/mean": -230.20147705078125, + "KL/rejected_KL_mean": -282.70147705078125, + "KL/std": 154.35580444335938, + "epoch": 0.6938775510204082, + "fcm_dpo/beta": 0.0038504954427480698, + "fcm_dpo/delta": -0.07434207946062088, + "fcm_dpo/margin": 104.99995422363281, + "fcm_dpo/q_t": 0.40416857600212097, + "grad_norm": 9.334559440612793, + "learning_rate": 1.3076841354533658e-07, + "logits/chosen": 0.8462595343589783, + "logits/rejected": 0.8283253908157349, + "logps/chosen": -243.07943725585938, + "logps/ref_chosen": -65.37794494628906, + "logps/ref_rejected": -88.19244384765625, + "logps/rejected": -370.8939208984375, + "loss": 1.0988, + "margin_dpo/margin_mean": 104.99995422363281, + "margin_dpo/margin_std": 143.6303253173828, + "step": 459 + }, + { + "KL/chosen_KL_mean": -185.249755859375, + "KL/mean": -238.11590576171875, + "KL/rejected_KL_mean": -290.9820556640625, + "KL/std": 130.9014129638672, + "epoch": 0.6953892668178382, + "fcm_dpo/beta": 0.003799508325755596, + "fcm_dpo/delta": -0.02934153750538826, + "fcm_dpo/margin": 105.73228454589844, + "fcm_dpo/q_t": 0.4084155559539795, + "grad_norm": 9.660005569458008, + "learning_rate": 1.2960793094762345e-07, + "logits/chosen": 0.8733299970626831, + "logits/rejected": 0.7844957113265991, + "logps/chosen": -249.81143188476562, + "logps/ref_chosen": -64.5616683959961, + "logps/ref_rejected": -88.67890167236328, + "logps/rejected": -379.66094970703125, + "loss": 1.0947, + "margin_dpo/margin_mean": 105.73228454589844, + "margin_dpo/margin_std": 144.70050048828125, + "step": 460 + }, + { + "KL/chosen_KL_mean": -154.37887573242188, + "KL/mean": -207.57666015625, + "KL/rejected_KL_mean": -260.77447509765625, + "KL/std": 122.10684204101562, + "epoch": 0.6969009826152683, + "fcm_dpo/beta": 0.0037420152220875025, + "fcm_dpo/delta": -0.07623682171106339, + "fcm_dpo/margin": 106.39558410644531, + "fcm_dpo/q_t": 0.4072979688644409, + "grad_norm": 9.591710090637207, + "learning_rate": 1.2845081597488286e-07, + "logits/chosen": 0.9570119380950928, + "logits/rejected": 0.8960863947868347, + "logps/chosen": -203.85678100585938, + "logps/ref_chosen": -49.4779167175293, + "logps/ref_rejected": -72.65262603759766, + "logps/rejected": -333.4271240234375, + "loss": 1.0899, + "margin_dpo/margin_mean": 106.39558410644531, + "margin_dpo/margin_std": 132.27688598632812, + "step": 461 + }, + { + "KL/chosen_KL_mean": -168.79248046875, + "KL/mean": -220.48451232910156, + "KL/rejected_KL_mean": -272.1765441894531, + "KL/std": 109.50112915039062, + "epoch": 0.6984126984126984, + "fcm_dpo/beta": 0.0037420152220875025, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 103.38409423828125, + "fcm_dpo/q_t": 0.4085114002227783, + "grad_norm": 10.171971321105957, + "learning_rate": 1.27297100994108e-07, + "logits/chosen": 0.8515751361846924, + "logits/rejected": 0.8222944736480713, + "logps/chosen": -229.28758239746094, + "logps/ref_chosen": -60.4951171875, + "logps/ref_rejected": -74.82136535644531, + "logps/rejected": -346.9979248046875, + "loss": 1.0875, + "margin_dpo/margin_mean": 103.38409423828125, + "margin_dpo/margin_std": 123.28450012207031, + "step": 462 + }, + { + "KL/chosen_KL_mean": -209.23985290527344, + "KL/mean": -237.67771911621094, + "KL/rejected_KL_mean": -266.1156005859375, + "KL/std": 116.87516784667969, + "epoch": 0.6999244142101285, + "fcm_dpo/beta": 0.0037420152220875025, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 56.87575912475586, + "fcm_dpo/q_t": 0.4490014910697937, + "grad_norm": 16.511831283569336, + "learning_rate": 1.2614681827718695e-07, + "logits/chosen": 0.8656213283538818, + "logits/rejected": 0.8616636991500854, + "logps/chosen": -276.9249572753906, + "logps/ref_chosen": -67.68511962890625, + "logps/ref_rejected": -71.32196044921875, + "logps/rejected": -337.43756103515625, + "loss": 1.2477, + "margin_dpo/margin_mean": 56.87575912475586, + "margin_dpo/margin_std": 135.13449096679688, + "step": 463 + }, + { + "KL/chosen_KL_mean": -185.364013671875, + "KL/mean": -234.14212036132812, + "KL/rejected_KL_mean": -282.92022705078125, + "KL/std": 136.61669921875, + "epoch": 0.7014361300075586, + "fcm_dpo/beta": 0.003737367456778884, + "fcm_dpo/delta": -0.01243579015135765, + "fcm_dpo/margin": 97.55622863769531, + "fcm_dpo/q_t": 0.41499170660972595, + "grad_norm": 9.446571350097656, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": 0.8816648721694946, + "logits/rejected": 0.8628978133201599, + "logps/chosen": -244.52964782714844, + "logps/ref_chosen": -59.16564178466797, + "logps/ref_rejected": -69.56146240234375, + "logps/rejected": -352.481689453125, + "loss": 1.1323, + "margin_dpo/margin_mean": 97.55621337890625, + "margin_dpo/margin_std": 151.31185913085938, + "step": 464 + }, + { + "KL/chosen_KL_mean": -191.65701293945312, + "KL/mean": -236.20742797851562, + "KL/rejected_KL_mean": -280.7578125, + "KL/std": 124.93704223632812, + "epoch": 0.7029478458049887, + "fcm_dpo/beta": 0.0037327196914702654, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.100830078125, + "fcm_dpo/q_t": 0.42286938428878784, + "grad_norm": 11.02184009552002, + "learning_rate": 1.238566782415197e-07, + "logits/chosen": 0.9379351139068604, + "logits/rejected": 0.8928974270820618, + "logps/chosen": -250.17068481445312, + "logps/ref_chosen": -58.513671875, + "logps/ref_rejected": -84.31745910644531, + "logps/rejected": -365.0752868652344, + "loss": 1.1481, + "margin_dpo/margin_mean": 89.10082244873047, + "margin_dpo/margin_std": 139.1053924560547, + "step": 465 + }, + { + "KL/chosen_KL_mean": -214.32485961914062, + "KL/mean": -233.25064086914062, + "KL/rejected_KL_mean": -252.1764373779297, + "KL/std": 137.39306640625, + "epoch": 0.7044595616024187, + "fcm_dpo/beta": 0.0037327196914702654, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 37.851585388183594, + "fcm_dpo/q_t": 0.4645908772945404, + "grad_norm": 15.143999099731445, + "learning_rate": 1.2271688498291334e-07, + "logits/chosen": 0.8913939595222473, + "logits/rejected": 0.8950200080871582, + "logps/chosen": -287.5906677246094, + "logps/ref_chosen": -73.26580810546875, + "logps/ref_rejected": -74.83621215820312, + "logps/rejected": -327.01263427734375, + "loss": 1.3203, + "margin_dpo/margin_mean": 37.85158920288086, + "margin_dpo/margin_std": 142.350341796875, + "step": 466 + }, + { + "KL/chosen_KL_mean": -182.82945251464844, + "KL/mean": -228.56005859375, + "KL/rejected_KL_mean": -274.2906799316406, + "KL/std": 138.61526489257812, + "epoch": 0.7059712773998488, + "fcm_dpo/beta": 0.0037327196914702654, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.46121978759766, + "fcm_dpo/q_t": 0.42408573627471924, + "grad_norm": 10.50765323638916, + "learning_rate": 1.2158065210664848e-07, + "logits/chosen": 0.8969777822494507, + "logits/rejected": 0.7978317737579346, + "logps/chosen": -230.408935546875, + "logps/ref_chosen": -47.57947540283203, + "logps/ref_rejected": -78.68522644042969, + "logps/rejected": -352.97589111328125, + "loss": 1.1481, + "margin_dpo/margin_mean": 91.46122741699219, + "margin_dpo/margin_std": 154.33111572265625, + "step": 467 + }, + { + "KL/chosen_KL_mean": -180.0341033935547, + "KL/mean": -227.7010040283203, + "KL/rejected_KL_mean": -275.367919921875, + "KL/std": 131.7947998046875, + "epoch": 0.7074829931972789, + "fcm_dpo/beta": 0.0037327196914702654, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 95.33381652832031, + "fcm_dpo/q_t": 0.4157509505748749, + "grad_norm": 12.930656433105469, + "learning_rate": 1.204480113956011e-07, + "logits/chosen": 0.8496984243392944, + "logits/rejected": 0.8403213024139404, + "logps/chosen": -243.96188354492188, + "logps/ref_chosen": -63.92778778076172, + "logps/ref_rejected": -76.51626586914062, + "logps/rejected": -351.8841857910156, + "loss": 1.1424, + "margin_dpo/margin_mean": 95.33382415771484, + "margin_dpo/margin_std": 156.22494506835938, + "step": 468 + }, + { + "KL/chosen_KL_mean": -180.99874877929688, + "KL/mean": -224.89773559570312, + "KL/rejected_KL_mean": -268.7967224121094, + "KL/std": 126.71222686767578, + "epoch": 0.708994708994709, + "fcm_dpo/beta": 0.0037327196914702654, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 87.7979736328125, + "fcm_dpo/q_t": 0.42392563819885254, + "grad_norm": 12.877175331115723, + "learning_rate": 1.1931899453216697e-07, + "logits/chosen": 0.9460161924362183, + "logits/rejected": 0.9314364194869995, + "logps/chosen": -240.05694580078125, + "logps/ref_chosen": -59.05818176269531, + "logps/ref_rejected": -75.67672729492188, + "logps/rejected": -344.47344970703125, + "loss": 1.1347, + "margin_dpo/margin_mean": 87.7979736328125, + "margin_dpo/margin_std": 122.22289276123047, + "step": 469 + }, + { + "KL/chosen_KL_mean": -166.74258422851562, + "KL/mean": -213.37454223632812, + "KL/rejected_KL_mean": -260.0065002441406, + "KL/std": 120.32038879394531, + "epoch": 0.7105064247921391, + "fcm_dpo/beta": 0.0037327196914702654, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 93.263916015625, + "fcm_dpo/q_t": 0.41877222061157227, + "grad_norm": 11.062674522399902, + "learning_rate": 1.1819363309737438e-07, + "logits/chosen": 0.9049103260040283, + "logits/rejected": 0.8607597351074219, + "logps/chosen": -214.61001586914062, + "logps/ref_chosen": -47.86743927001953, + "logps/ref_rejected": -65.96859741210938, + "logps/rejected": -325.97509765625, + "loss": 1.1311, + "margin_dpo/margin_mean": 93.263916015625, + "margin_dpo/margin_std": 137.35110473632812, + "step": 470 + }, + { + "KL/chosen_KL_mean": -166.78622436523438, + "KL/mean": -217.06524658203125, + "KL/rejected_KL_mean": -267.34429931640625, + "KL/std": 121.04692077636719, + "epoch": 0.7120181405895691, + "fcm_dpo/beta": 0.0037327196914702654, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 100.55805969238281, + "fcm_dpo/q_t": 0.41160130500793457, + "grad_norm": 10.453146934509277, + "learning_rate": 1.1707195857000215e-07, + "logits/chosen": 0.8833752870559692, + "logits/rejected": 0.8440606594085693, + "logps/chosen": -224.56407165527344, + "logps/ref_chosen": -57.777854919433594, + "logps/ref_rejected": -73.81172180175781, + "logps/rejected": -341.156005859375, + "loss": 1.1057, + "margin_dpo/margin_mean": 100.55805969238281, + "margin_dpo/margin_std": 132.71646118164062, + "step": 471 + }, + { + "KL/chosen_KL_mean": -167.60638427734375, + "KL/mean": -207.35028076171875, + "KL/rejected_KL_mean": -247.0941925048828, + "KL/std": 126.00967407226562, + "epoch": 0.7135298563869993, + "fcm_dpo/beta": 0.0037327196914702654, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 79.48780059814453, + "fcm_dpo/q_t": 0.43231454491615295, + "grad_norm": 11.560592651367188, + "learning_rate": 1.1595400232569768e-07, + "logits/chosen": 0.8736377954483032, + "logits/rejected": 0.8380210399627686, + "logps/chosen": -223.51504516601562, + "logps/ref_chosen": -55.908668518066406, + "logps/ref_rejected": -74.70294189453125, + "logps/rejected": -321.797119140625, + "loss": 1.1947, + "margin_dpo/margin_mean": 79.48780059814453, + "margin_dpo/margin_std": 160.8380126953125, + "step": 472 + }, + { + "KL/chosen_KL_mean": -170.9720458984375, + "KL/mean": -227.07763671875, + "KL/rejected_KL_mean": -283.1832275390625, + "KL/std": 155.66497802734375, + "epoch": 0.7150415721844293, + "fcm_dpo/beta": 0.0037327196914702654, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 112.21117401123047, + "fcm_dpo/q_t": 0.4082377254962921, + "grad_norm": 12.538541793823242, + "learning_rate": 1.1483979563610069e-07, + "logits/chosen": 0.970649003982544, + "logits/rejected": 0.8827682733535767, + "logps/chosen": -225.1329345703125, + "logps/ref_chosen": -54.16088104248047, + "logps/ref_rejected": -92.76789855957031, + "logps/rejected": -375.95111083984375, + "loss": 1.1122, + "margin_dpo/margin_mean": 112.21117401123047, + "margin_dpo/margin_std": 179.88638305664062, + "step": 473 + }, + { + "KL/chosen_KL_mean": -173.3741455078125, + "KL/mean": -217.32489013671875, + "KL/rejected_KL_mean": -261.275634765625, + "KL/std": 130.40184020996094, + "epoch": 0.7165532879818595, + "fcm_dpo/beta": 0.0037327196914702654, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 87.90148162841797, + "fcm_dpo/q_t": 0.4254756569862366, + "grad_norm": 16.006996154785156, + "learning_rate": 1.1372936966796709e-07, + "logits/chosen": 0.9420585632324219, + "logits/rejected": 0.8917109966278076, + "logps/chosen": -220.05984497070312, + "logps/ref_chosen": -46.685707092285156, + "logps/ref_rejected": -71.44731903076172, + "logps/rejected": -332.7229309082031, + "loss": 1.1583, + "margin_dpo/margin_mean": 87.90147399902344, + "margin_dpo/margin_std": 150.548828125, + "step": 474 + }, + { + "KL/chosen_KL_mean": -173.3091583251953, + "KL/mean": -232.6639404296875, + "KL/rejected_KL_mean": -292.01873779296875, + "KL/std": 132.6728057861328, + "epoch": 0.7180650037792895, + "fcm_dpo/beta": 0.003699835389852524, + "fcm_dpo/delta": -0.044244010001420975, + "fcm_dpo/margin": 118.70957946777344, + "fcm_dpo/q_t": 0.4001288414001465, + "grad_norm": 7.808788299560547, + "learning_rate": 1.126227554822985e-07, + "logits/chosen": 0.9036856889724731, + "logits/rejected": 0.8648196458816528, + "logps/chosen": -231.7964630126953, + "logps/ref_chosen": -58.4873046875, + "logps/ref_rejected": -87.00187683105469, + "logps/rejected": -379.0206298828125, + "loss": 1.0693, + "margin_dpo/margin_mean": 118.70957946777344, + "margin_dpo/margin_std": 152.59393310546875, + "step": 475 + }, + { + "KL/chosen_KL_mean": -208.72824096679688, + "KL/mean": -243.93551635742188, + "KL/rejected_KL_mean": -279.1427917480469, + "KL/std": 144.14910888671875, + "epoch": 0.7195767195767195, + "fcm_dpo/beta": 0.003699835389852524, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 70.41454315185547, + "fcm_dpo/q_t": 0.4408169984817505, + "grad_norm": 12.431097030639648, + "learning_rate": 1.1151998403347243e-07, + "logits/chosen": 0.8004306554794312, + "logits/rejected": 0.8054685592651367, + "logps/chosen": -284.10986328125, + "logps/ref_chosen": -75.38162231445312, + "logps/ref_rejected": -76.99822235107422, + "logps/rejected": -356.1410217285156, + "loss": 1.2173, + "margin_dpo/margin_mean": 70.41454315185547, + "margin_dpo/margin_std": 151.20233154296875, + "step": 476 + }, + { + "KL/chosen_KL_mean": -202.79141235351562, + "KL/mean": -247.09869384765625, + "KL/rejected_KL_mean": -291.40594482421875, + "KL/std": 144.90768432617188, + "epoch": 0.7210884353741497, + "fcm_dpo/beta": 0.003699835389852524, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 88.61453247070312, + "fcm_dpo/q_t": 0.4267289936542511, + "grad_norm": 11.736209869384766, + "learning_rate": 1.1042108616837692e-07, + "logits/chosen": 0.8813755512237549, + "logits/rejected": 0.8460351228713989, + "logps/chosen": -263.86480712890625, + "logps/ref_chosen": -61.073387145996094, + "logps/ref_rejected": -81.34375, + "logps/rejected": -372.74969482421875, + "loss": 1.1971, + "margin_dpo/margin_mean": 88.61453247070312, + "margin_dpo/margin_std": 187.2801055908203, + "step": 477 + }, + { + "KL/chosen_KL_mean": -178.40023803710938, + "KL/mean": -209.76089477539062, + "KL/rejected_KL_mean": -241.12158203125, + "KL/std": 112.0628890991211, + "epoch": 0.7226001511715797, + "fcm_dpo/beta": 0.003699835389852524, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 62.721343994140625, + "fcm_dpo/q_t": 0.44511181116104126, + "grad_norm": 11.34143352508545, + "learning_rate": 1.0932609262554746e-07, + "logits/chosen": 0.8605385422706604, + "logits/rejected": 0.8723839521408081, + "logps/chosen": -235.56756591796875, + "logps/ref_chosen": -57.16731643676758, + "logps/ref_rejected": -53.30917739868164, + "logps/rejected": -294.4307556152344, + "loss": 1.2306, + "margin_dpo/margin_mean": 62.72134017944336, + "margin_dpo/margin_std": 137.10507202148438, + "step": 478 + }, + { + "KL/chosen_KL_mean": -186.5111541748047, + "KL/mean": -215.21243286132812, + "KL/rejected_KL_mean": -243.91368103027344, + "KL/std": 118.79924774169922, + "epoch": 0.7241118669690099, + "fcm_dpo/beta": 0.003699835389852524, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.40254211425781, + "fcm_dpo/q_t": 0.45077085494995117, + "grad_norm": 11.206938743591309, + "learning_rate": 1.0823503403430734e-07, + "logits/chosen": 0.871839165687561, + "logits/rejected": 0.8502979874610901, + "logps/chosen": -245.42446899414062, + "logps/ref_chosen": -58.91331481933594, + "logps/ref_rejected": -63.7403450012207, + "logps/rejected": -307.654052734375, + "loss": 1.2523, + "margin_dpo/margin_mean": 57.40254211425781, + "margin_dpo/margin_std": 141.32350158691406, + "step": 479 + }, + { + "KL/chosen_KL_mean": -193.8066864013672, + "KL/mean": -237.73788452148438, + "KL/rejected_KL_mean": -281.6690673828125, + "KL/std": 121.91385650634766, + "epoch": 0.7256235827664399, + "fcm_dpo/beta": 0.003685213625431061, + "fcm_dpo/delta": -0.03967710956931114, + "fcm_dpo/margin": 87.86239624023438, + "fcm_dpo/q_t": 0.4248046875, + "grad_norm": 13.300537109375, + "learning_rate": 1.0714794091391072e-07, + "logits/chosen": 0.8580498099327087, + "logits/rejected": 0.8567924499511719, + "logps/chosen": -256.6072998046875, + "logps/ref_chosen": -62.80061340332031, + "logps/ref_rejected": -67.58859252929688, + "logps/rejected": -349.2576599121094, + "loss": 1.1564, + "margin_dpo/margin_mean": 87.86239624023438, + "margin_dpo/margin_std": 139.7371826171875, + "step": 480 + }, + { + "KL/chosen_KL_mean": -193.44094848632812, + "KL/mean": -233.70297241210938, + "KL/rejected_KL_mean": -273.96502685546875, + "KL/std": 130.8161163330078, + "epoch": 0.72713529856387, + "fcm_dpo/beta": 0.0036705918610095978, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.52407836914062, + "fcm_dpo/q_t": 0.43242886662483215, + "grad_norm": 10.15783977508545, + "learning_rate": 1.0606484367268906e-07, + "logits/chosen": 0.8588245511054993, + "logits/rejected": 0.8666401505470276, + "logps/chosen": -258.7274169921875, + "logps/ref_chosen": -65.28649139404297, + "logps/ref_rejected": -70.78668212890625, + "logps/rejected": -344.751708984375, + "loss": 1.1766, + "margin_dpo/margin_mean": 80.52407836914062, + "margin_dpo/margin_std": 143.02203369140625, + "step": 481 + }, + { + "KL/chosen_KL_mean": -211.6179962158203, + "KL/mean": -260.3857116699219, + "KL/rejected_KL_mean": -309.1534423828125, + "KL/std": 149.3299560546875, + "epoch": 0.7286470143613001, + "fcm_dpo/beta": 0.0036705918610095978, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 97.53544616699219, + "fcm_dpo/q_t": 0.4218747317790985, + "grad_norm": 11.276646614074707, + "learning_rate": 1.0498577260720048e-07, + "logits/chosen": 0.8682383298873901, + "logits/rejected": 0.7406269907951355, + "logps/chosen": -272.524169921875, + "logps/ref_chosen": -60.906185150146484, + "logps/ref_rejected": -103.44656372070312, + "logps/rejected": -412.6000061035156, + "loss": 1.1612, + "margin_dpo/margin_mean": 97.53544616699219, + "margin_dpo/margin_std": 183.6652069091797, + "step": 482 + }, + { + "KL/chosen_KL_mean": -177.52684020996094, + "KL/mean": -231.13912963867188, + "KL/rejected_KL_mean": -284.75140380859375, + "KL/std": 125.70189666748047, + "epoch": 0.7301587301587301, + "fcm_dpo/beta": 0.003663320792838931, + "fcm_dpo/delta": -0.01984819769859314, + "fcm_dpo/margin": 107.22454833984375, + "fcm_dpo/q_t": 0.4089610278606415, + "grad_norm": 11.108445167541504, + "learning_rate": 1.0391075790138232e-07, + "logits/chosen": 0.9367642402648926, + "logits/rejected": 0.8554052114486694, + "logps/chosen": -230.71885681152344, + "logps/ref_chosen": -53.192012786865234, + "logps/ref_rejected": -81.83927154541016, + "logps/rejected": -366.5906677246094, + "loss": 1.0983, + "margin_dpo/margin_mean": 107.22454833984375, + "margin_dpo/margin_std": 145.13848876953125, + "step": 483 + }, + { + "KL/chosen_KL_mean": -186.06248474121094, + "KL/mean": -219.39501953125, + "KL/rejected_KL_mean": -252.72755432128906, + "KL/std": 125.95346069335938, + "epoch": 0.7316704459561603, + "fcm_dpo/beta": 0.0036560497246682644, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.66506958007812, + "fcm_dpo/q_t": 0.4416685998439789, + "grad_norm": 13.961874961853027, + "learning_rate": 1.0283982962570681e-07, + "logits/chosen": 0.938569188117981, + "logits/rejected": 0.9175790548324585, + "logps/chosen": -243.83193969726562, + "logps/ref_chosen": -57.76945877075195, + "logps/ref_rejected": -71.6829833984375, + "logps/rejected": -324.4105224609375, + "loss": 1.1964, + "margin_dpo/margin_mean": 66.66506958007812, + "margin_dpo/margin_std": 109.29454803466797, + "step": 484 + }, + { + "KL/chosen_KL_mean": -191.56036376953125, + "KL/mean": -228.56719970703125, + "KL/rejected_KL_mean": -265.57403564453125, + "KL/std": 122.90736389160156, + "epoch": 0.7331821617535903, + "fcm_dpo/beta": 0.0036560497246682644, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 74.01366424560547, + "fcm_dpo/q_t": 0.4370569586753845, + "grad_norm": 11.334329605102539, + "learning_rate": 1.0177301773633992e-07, + "logits/chosen": 0.9071985483169556, + "logits/rejected": 0.896731972694397, + "logps/chosen": -248.19619750976562, + "logps/ref_chosen": -56.63584899902344, + "logps/ref_rejected": -70.85614013671875, + "logps/rejected": -336.43017578125, + "loss": 1.1859, + "margin_dpo/margin_mean": 74.013671875, + "margin_dpo/margin_std": 123.91354370117188, + "step": 485 + }, + { + "KL/chosen_KL_mean": -215.73367309570312, + "KL/mean": -254.3745574951172, + "KL/rejected_KL_mean": -293.01544189453125, + "KL/std": 147.834716796875, + "epoch": 0.7346938775510204, + "fcm_dpo/beta": 0.0036560497246682644, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 77.28172302246094, + "fcm_dpo/q_t": 0.4370615780353546, + "grad_norm": 9.572896957397461, + "learning_rate": 1.007103520743035e-07, + "logits/chosen": 0.9285538196563721, + "logits/rejected": 0.8437390923500061, + "logps/chosen": -272.0806884765625, + "logps/ref_chosen": -56.347023010253906, + "logps/ref_rejected": -85.97221374511719, + "logps/rejected": -378.9876403808594, + "loss": 1.2158, + "margin_dpo/margin_mean": 77.28172302246094, + "margin_dpo/margin_std": 172.080078125, + "step": 486 + }, + { + "KL/chosen_KL_mean": -195.94334411621094, + "KL/mean": -240.12217712402344, + "KL/rejected_KL_mean": -284.301025390625, + "KL/std": 138.95465087890625, + "epoch": 0.7362055933484505, + "fcm_dpo/beta": 0.0036560497246682644, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 88.35769653320312, + "fcm_dpo/q_t": 0.42905205488204956, + "grad_norm": 10.672550201416016, + "learning_rate": 9.965186236464046e-08, + "logits/chosen": 0.9684932231903076, + "logits/rejected": 0.9252954721450806, + "logps/chosen": -256.560546875, + "logps/ref_chosen": -60.617218017578125, + "logps/ref_rejected": -82.50975036621094, + "logps/rejected": -366.810791015625, + "loss": 1.1715, + "margin_dpo/margin_mean": 88.35769653320312, + "margin_dpo/margin_std": 167.52935791015625, + "step": 487 + }, + { + "KL/chosen_KL_mean": -182.96995544433594, + "KL/mean": -233.22586059570312, + "KL/rejected_KL_mean": -283.48175048828125, + "KL/std": 129.29685974121094, + "epoch": 0.7377173091458806, + "fcm_dpo/beta": 0.003645846154540777, + "fcm_dpo/delta": -0.027987273409962654, + "fcm_dpo/margin": 100.51178741455078, + "fcm_dpo/q_t": 0.4163171947002411, + "grad_norm": 13.601997375488281, + "learning_rate": 9.859757821558337e-08, + "logits/chosen": 0.8888124227523804, + "logits/rejected": 0.8562193512916565, + "logps/chosen": -246.07901000976562, + "logps/ref_chosen": -63.10905075073242, + "logps/ref_rejected": -82.49348449707031, + "logps/rejected": -365.9752197265625, + "loss": 1.1183, + "margin_dpo/margin_mean": 100.51178741455078, + "margin_dpo/margin_std": 144.21810913085938, + "step": 488 + }, + { + "KL/chosen_KL_mean": -216.40370178222656, + "KL/mean": -245.2534637451172, + "KL/rejected_KL_mean": -274.10321044921875, + "KL/std": 141.31771850585938, + "epoch": 0.7392290249433107, + "fcm_dpo/beta": 0.0036356423515826464, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.69950485229492, + "fcm_dpo/q_t": 0.4508482813835144, + "grad_norm": 9.973169326782227, + "learning_rate": 9.754752911772615e-08, + "logits/chosen": 0.8845810890197754, + "logits/rejected": 0.8505979776382446, + "logps/chosen": -281.3926696777344, + "logps/ref_chosen": -64.98896026611328, + "logps/ref_rejected": -84.39607238769531, + "logps/rejected": -358.4992980957031, + "loss": 1.2604, + "margin_dpo/margin_mean": 57.69950866699219, + "margin_dpo/margin_std": 151.286865234375, + "step": 489 + }, + { + "KL/chosen_KL_mean": -193.18675231933594, + "KL/mean": -229.94459533691406, + "KL/rejected_KL_mean": -266.7024230957031, + "KL/std": 135.72817993164062, + "epoch": 0.7407407407407407, + "fcm_dpo/beta": 0.0036356423515826464, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 73.51567077636719, + "fcm_dpo/q_t": 0.43982094526290894, + "grad_norm": 9.761716842651367, + "learning_rate": 9.650174444319956e-08, + "logits/chosen": 0.9422023296356201, + "logits/rejected": 0.9359936714172363, + "logps/chosen": -255.09548950195312, + "logps/ref_chosen": -61.90874481201172, + "logps/ref_rejected": -70.58566284179688, + "logps/rejected": -337.2880859375, + "loss": 1.2254, + "margin_dpo/margin_mean": 73.51567077636719, + "margin_dpo/margin_std": 169.10879516601562, + "step": 490 + }, + { + "KL/chosen_KL_mean": -186.3331298828125, + "KL/mean": -222.13389587402344, + "KL/rejected_KL_mean": -257.9346618652344, + "KL/std": 120.4219741821289, + "epoch": 0.7422524565381708, + "fcm_dpo/beta": 0.0036356423515826464, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 71.60151672363281, + "fcm_dpo/q_t": 0.43975383043289185, + "grad_norm": 8.621192932128906, + "learning_rate": 9.546025344484868e-08, + "logits/chosen": 0.8622674942016602, + "logits/rejected": 0.811538577079773, + "logps/chosen": -241.808837890625, + "logps/ref_chosen": -55.47570037841797, + "logps/ref_rejected": -78.70318603515625, + "logps/rejected": -336.6378173828125, + "loss": 1.1981, + "margin_dpo/margin_mean": 71.60152435302734, + "margin_dpo/margin_std": 129.99026489257812, + "step": 491 + }, + { + "KL/chosen_KL_mean": -220.4925537109375, + "KL/mean": -255.36636352539062, + "KL/rejected_KL_mean": -290.2402038574219, + "KL/std": 143.24710083007812, + "epoch": 0.7437641723356009, + "fcm_dpo/beta": 0.0036356423515826464, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 69.74764251708984, + "fcm_dpo/q_t": 0.44249725341796875, + "grad_norm": 12.263677597045898, + "learning_rate": 9.442308525541589e-08, + "logits/chosen": 0.8829400539398193, + "logits/rejected": 0.8344748020172119, + "logps/chosen": -287.7789306640625, + "logps/ref_chosen": -67.28638458251953, + "logps/ref_rejected": -82.78628540039062, + "logps/rejected": -373.0264892578125, + "loss": 1.2368, + "margin_dpo/margin_mean": 69.74765014648438, + "margin_dpo/margin_std": 165.14651489257812, + "step": 492 + }, + { + "KL/chosen_KL_mean": -184.6879425048828, + "KL/mean": -233.34588623046875, + "KL/rejected_KL_mean": -282.0038146972656, + "KL/std": 140.0909423828125, + "epoch": 0.745275888133031, + "fcm_dpo/beta": 0.0036191383842378855, + "fcm_dpo/delta": -0.045602187514305115, + "fcm_dpo/margin": 97.31587219238281, + "fcm_dpo/q_t": 0.42029207944869995, + "grad_norm": 10.198504447937012, + "learning_rate": 9.339026888672468e-08, + "logits/chosen": 0.8781353831291199, + "logits/rejected": 0.8281064033508301, + "logps/chosen": -240.61544799804688, + "logps/ref_chosen": -55.92750549316406, + "logps/ref_rejected": -79.12149810791016, + "logps/rejected": -361.12530517578125, + "loss": 1.1388, + "margin_dpo/margin_mean": 97.31587219238281, + "margin_dpo/margin_std": 152.83680725097656, + "step": 493 + }, + { + "KL/chosen_KL_mean": -191.65391540527344, + "KL/mean": -236.73977661132812, + "KL/rejected_KL_mean": -281.82568359375, + "KL/std": 144.91470336914062, + "epoch": 0.7467876039304611, + "fcm_dpo/beta": 0.0036026344168931246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 90.17176055908203, + "fcm_dpo/q_t": 0.42782270908355713, + "grad_norm": 12.270206451416016, + "learning_rate": 9.236183322886945e-08, + "logits/chosen": 0.8384512066841125, + "logits/rejected": 0.806420087814331, + "logps/chosen": -259.6080322265625, + "logps/ref_chosen": -67.95410919189453, + "logps/ref_rejected": -90.50865173339844, + "logps/rejected": -372.3343505859375, + "loss": 1.1845, + "margin_dpo/margin_mean": 90.17176818847656, + "margin_dpo/margin_std": 181.9389190673828, + "step": 494 + }, + { + "KL/chosen_KL_mean": -183.88514709472656, + "KL/mean": -219.1983642578125, + "KL/rejected_KL_mean": -254.51158142089844, + "KL/std": 138.21241760253906, + "epoch": 0.7482993197278912, + "fcm_dpo/beta": 0.0036026344168931246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 70.62643432617188, + "fcm_dpo/q_t": 0.4458218216896057, + "grad_norm": 14.293071746826172, + "learning_rate": 9.133780704940594e-08, + "logits/chosen": 0.9193370938301086, + "logits/rejected": 0.8782978653907776, + "logps/chosen": -236.5106201171875, + "logps/ref_chosen": -52.62546157836914, + "logps/ref_rejected": -72.06781005859375, + "logps/rejected": -326.57940673828125, + "loss": 1.237, + "margin_dpo/margin_mean": 70.62643432617188, + "margin_dpo/margin_std": 175.55023193359375, + "step": 495 + }, + { + "KL/chosen_KL_mean": -198.95504760742188, + "KL/mean": -249.46466064453125, + "KL/rejected_KL_mean": -299.9742431640625, + "KL/std": 150.4727325439453, + "epoch": 0.7498110355253212, + "fcm_dpo/beta": 0.0036026344168931246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 101.01922607421875, + "fcm_dpo/q_t": 0.4242492914199829, + "grad_norm": 9.73660659790039, + "learning_rate": 9.031821899254797e-08, + "logits/chosen": 0.9211750626564026, + "logits/rejected": 0.8287931680679321, + "logps/chosen": -256.5523681640625, + "logps/ref_chosen": -57.597320556640625, + "logps/ref_rejected": -94.36127471923828, + "logps/rejected": -394.3355407714844, + "loss": 1.1633, + "margin_dpo/margin_mean": 101.01922607421875, + "margin_dpo/margin_std": 196.87779235839844, + "step": 496 + }, + { + "KL/chosen_KL_mean": -206.773193359375, + "KL/mean": -251.5380859375, + "KL/rejected_KL_mean": -296.302978515625, + "KL/std": 133.81814575195312, + "epoch": 0.7513227513227513, + "fcm_dpo/beta": 0.0036026344168931246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.52980041503906, + "fcm_dpo/q_t": 0.42446810007095337, + "grad_norm": 10.825132369995117, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": 0.9078977108001709, + "logits/rejected": 0.8813755512237549, + "logps/chosen": -279.5631408691406, + "logps/ref_chosen": -72.78994750976562, + "logps/ref_rejected": -89.48483276367188, + "logps/rejected": -385.787841796875, + "loss": 1.1571, + "margin_dpo/margin_mean": 89.52980041503906, + "margin_dpo/margin_std": 148.03123474121094, + "step": 497 + }, + { + "KL/chosen_KL_mean": -182.50619506835938, + "KL/mean": -229.74530029296875, + "KL/rejected_KL_mean": -276.98443603515625, + "KL/std": 126.8770751953125, + "epoch": 0.7528344671201814, + "fcm_dpo/beta": 0.0036026344168931246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.47825622558594, + "fcm_dpo/q_t": 0.4190598726272583, + "grad_norm": 11.309755325317383, + "learning_rate": 8.829247120198563e-08, + "logits/chosen": 0.8609334230422974, + "logits/rejected": 0.8546430468559265, + "logps/chosen": -250.8719024658203, + "logps/ref_chosen": -68.36572265625, + "logps/ref_rejected": -71.28846740722656, + "logps/rejected": -348.27288818359375, + "loss": 1.1377, + "margin_dpo/margin_mean": 94.47825622558594, + "margin_dpo/margin_std": 142.64303588867188, + "step": 498 + }, + { + "KL/chosen_KL_mean": -186.2841796875, + "KL/mean": -240.4781494140625, + "KL/rejected_KL_mean": -294.672119140625, + "KL/std": 144.49862670898438, + "epoch": 0.7543461829176115, + "fcm_dpo/beta": 0.0036026344168931246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 108.387939453125, + "fcm_dpo/q_t": 0.4136818051338196, + "grad_norm": 14.066987991333008, + "learning_rate": 8.728636813280163e-08, + "logits/chosen": 0.8882684111595154, + "logits/rejected": 0.8408522605895996, + "logps/chosen": -248.19300842285156, + "logps/ref_chosen": -61.90882873535156, + "logps/ref_rejected": -91.9411392211914, + "logps/rejected": -386.61328125, + "loss": 1.1433, + "margin_dpo/margin_mean": 108.387939453125, + "margin_dpo/margin_std": 191.2751007080078, + "step": 499 + }, + { + "KL/chosen_KL_mean": -196.25576782226562, + "KL/mean": -235.48272705078125, + "KL/rejected_KL_mean": -274.7096862792969, + "KL/std": 126.16201782226562, + "epoch": 0.7558578987150416, + "fcm_dpo/beta": 0.0036026344168931246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 78.45390319824219, + "fcm_dpo/q_t": 0.43167075514793396, + "grad_norm": 11.740747451782227, + "learning_rate": 8.628481651367875e-08, + "logits/chosen": 0.84987473487854, + "logits/rejected": 0.866410493850708, + "logps/chosen": -266.4815979003906, + "logps/ref_chosen": -70.225830078125, + "logps/ref_rejected": -71.72203063964844, + "logps/rejected": -346.43170166015625, + "loss": 1.202, + "margin_dpo/margin_mean": 78.45390319824219, + "margin_dpo/margin_std": 159.15243530273438, + "step": 500 + }, + { + "KL/chosen_KL_mean": -196.87637329101562, + "KL/mean": -232.41775512695312, + "KL/rejected_KL_mean": -267.95916748046875, + "KL/std": 131.5418701171875, + "epoch": 0.7573696145124716, + "fcm_dpo/beta": 0.0036026344168931246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 71.08279418945312, + "fcm_dpo/q_t": 0.43857502937316895, + "grad_norm": 9.775738716125488, + "learning_rate": 8.528784436016878e-08, + "logits/chosen": 0.9283879995346069, + "logits/rejected": 0.9351827502250671, + "logps/chosen": -261.4751892089844, + "logps/ref_chosen": -64.59880828857422, + "logps/ref_rejected": -70.59329223632812, + "logps/rejected": -338.5524597167969, + "loss": 1.1816, + "margin_dpo/margin_mean": 71.08279418945312, + "margin_dpo/margin_std": 105.03961944580078, + "step": 501 + }, + { + "KL/chosen_KL_mean": -197.69915771484375, + "KL/mean": -237.3424072265625, + "KL/rejected_KL_mean": -276.98565673828125, + "KL/std": 147.6724853515625, + "epoch": 0.7588813303099018, + "fcm_dpo/beta": 0.0036026344168931246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 79.2864761352539, + "fcm_dpo/q_t": 0.4334084391593933, + "grad_norm": 12.1803617477417, + "learning_rate": 8.4295479559726e-08, + "logits/chosen": 0.9151396751403809, + "logits/rejected": 0.8784686326980591, + "logps/chosen": -263.165771484375, + "logps/ref_chosen": -65.46662902832031, + "logps/ref_rejected": -90.22233581542969, + "logps/rejected": -367.2079772949219, + "loss": 1.1803, + "margin_dpo/margin_mean": 79.28648376464844, + "margin_dpo/margin_std": 137.8919219970703, + "step": 502 + }, + { + "KL/chosen_KL_mean": -170.47628784179688, + "KL/mean": -209.97726440429688, + "KL/rejected_KL_mean": -249.47824096679688, + "KL/std": 115.76939392089844, + "epoch": 0.7603930461073318, + "fcm_dpo/beta": 0.0036026344168931246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 79.001953125, + "fcm_dpo/q_t": 0.43191272020339966, + "grad_norm": 9.445923805236816, + "learning_rate": 8.330774987092712e-08, + "logits/chosen": 0.85642409324646, + "logits/rejected": 0.8654800653457642, + "logps/chosen": -222.31106567382812, + "logps/ref_chosen": -51.83476257324219, + "logps/ref_rejected": -57.62522506713867, + "logps/rejected": -307.10345458984375, + "loss": 1.1774, + "margin_dpo/margin_mean": 79.00196075439453, + "margin_dpo/margin_std": 132.04278564453125, + "step": 503 + }, + { + "KL/chosen_KL_mean": -182.04151916503906, + "KL/mean": -233.4086151123047, + "KL/rejected_KL_mean": -284.77569580078125, + "KL/std": 120.95684814453125, + "epoch": 0.7619047619047619, + "fcm_dpo/beta": 0.0035869048442691565, + "fcm_dpo/delta": -0.04385312646627426, + "fcm_dpo/margin": 102.73417663574219, + "fcm_dpo/q_t": 0.41299694776535034, + "grad_norm": 8.542745590209961, + "learning_rate": 8.232468292269479e-08, + "logits/chosen": 0.8525612354278564, + "logits/rejected": 0.8485938906669617, + "logps/chosen": -250.69271850585938, + "logps/ref_chosen": -68.65119934082031, + "logps/ref_rejected": -77.91394805908203, + "logps/rejected": -362.68963623046875, + "loss": 1.1027, + "margin_dpo/margin_mean": 102.73417663574219, + "margin_dpo/margin_std": 125.10858154296875, + "step": 504 + }, + { + "KL/chosen_KL_mean": -193.14846801757812, + "KL/mean": -230.13601684570312, + "KL/rejected_KL_mean": -267.12359619140625, + "KL/std": 149.0281219482422, + "epoch": 0.763416477702192, + "fcm_dpo/beta": 0.0035711752716451883, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 73.97512817382812, + "fcm_dpo/q_t": 0.4398888349533081, + "grad_norm": 11.520407676696777, + "learning_rate": 8.134630621352483e-08, + "logits/chosen": 0.8992654085159302, + "logits/rejected": 0.8699280023574829, + "logps/chosen": -253.14730834960938, + "logps/ref_chosen": -59.99884796142578, + "logps/ref_rejected": -76.88048553466797, + "logps/rejected": -344.00408935546875, + "loss": 1.2187, + "margin_dpo/margin_mean": 73.97512817382812, + "margin_dpo/margin_std": 160.25271606445312, + "step": 505 + }, + { + "KL/chosen_KL_mean": -194.79971313476562, + "KL/mean": -233.82147216796875, + "KL/rejected_KL_mean": -272.8432312011719, + "KL/std": 128.04318237304688, + "epoch": 0.764928193499622, + "fcm_dpo/beta": 0.0035711752716451883, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 78.04351806640625, + "fcm_dpo/q_t": 0.43447548151016235, + "grad_norm": 10.453583717346191, + "learning_rate": 8.037264711071698e-08, + "logits/chosen": 0.8826917409896851, + "logits/rejected": 0.8774870038032532, + "logps/chosen": -264.8710021972656, + "logps/ref_chosen": -70.07130432128906, + "logps/ref_rejected": -82.03775024414062, + "logps/rejected": -354.8809814453125, + "loss": 1.206, + "margin_dpo/margin_mean": 78.04351806640625, + "margin_dpo/margin_std": 160.6054229736328, + "step": 506 + }, + { + "KL/chosen_KL_mean": -204.0850372314453, + "KL/mean": -251.54730224609375, + "KL/rejected_KL_mean": -299.00958251953125, + "KL/std": 149.53985595703125, + "epoch": 0.7664399092970522, + "fcm_dpo/beta": 0.0035711752716451883, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.924560546875, + "fcm_dpo/q_t": 0.42609667778015137, + "grad_norm": 11.234076499938965, + "learning_rate": 7.940373284960933e-08, + "logits/chosen": 0.8698856830596924, + "logits/rejected": 0.8384686708450317, + "logps/chosen": -276.0920715332031, + "logps/ref_chosen": -72.00703430175781, + "logps/ref_rejected": -93.94987487792969, + "logps/rejected": -392.95947265625, + "loss": 1.1703, + "margin_dpo/margin_mean": 94.92456817626953, + "margin_dpo/margin_std": 175.51181030273438, + "step": 507 + }, + { + "KL/chosen_KL_mean": -188.01617431640625, + "KL/mean": -245.24417114257812, + "KL/rejected_KL_mean": -302.47216796875, + "KL/std": 159.1145782470703, + "epoch": 0.7679516250944822, + "fcm_dpo/beta": 0.0035516000352799892, + "fcm_dpo/delta": -0.055117167532444, + "fcm_dpo/margin": 114.45600128173828, + "fcm_dpo/q_t": 0.4093967080116272, + "grad_norm": 12.474221229553223, + "learning_rate": 7.843959053281663e-08, + "logits/chosen": 0.8432912230491638, + "logits/rejected": 0.7599025964736938, + "logps/chosen": -248.23609924316406, + "logps/ref_chosen": -60.21992492675781, + "logps/ref_rejected": -95.9200668334961, + "logps/rejected": -398.3922424316406, + "loss": 1.1163, + "margin_dpo/margin_mean": 114.45600891113281, + "margin_dpo/margin_std": 178.3321990966797, + "step": 508 + }, + { + "KL/chosen_KL_mean": -199.39761352539062, + "KL/mean": -237.2610626220703, + "KL/rejected_KL_mean": -275.1245422363281, + "KL/std": 134.06805419921875, + "epoch": 0.7694633408919124, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 75.72692108154297, + "fcm_dpo/q_t": 0.43684712052345276, + "grad_norm": 12.999882698059082, + "learning_rate": 7.748024712947204e-08, + "logits/chosen": 0.8402736783027649, + "logits/rejected": 0.8276892900466919, + "logps/chosen": -265.66778564453125, + "logps/ref_chosen": -66.27017211914062, + "logps/ref_rejected": -71.73065185546875, + "logps/rejected": -346.85516357421875, + "loss": 1.2142, + "margin_dpo/margin_mean": 75.72692108154297, + "margin_dpo/margin_std": 160.88034057617188, + "step": 509 + }, + { + "KL/chosen_KL_mean": -196.36257934570312, + "KL/mean": -249.74359130859375, + "KL/rejected_KL_mean": -303.1246032714844, + "KL/std": 156.79202270507812, + "epoch": 0.7709750566893424, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 106.76203155517578, + "fcm_dpo/q_t": 0.4201943874359131, + "grad_norm": 10.616975784301758, + "learning_rate": 7.652572947447272e-08, + "logits/chosen": 0.9781365990638733, + "logits/rejected": 0.9099892377853394, + "logps/chosen": -249.90745544433594, + "logps/ref_chosen": -53.54487609863281, + "logps/ref_rejected": -91.36648559570312, + "logps/rejected": -394.4910888671875, + "loss": 1.1591, + "margin_dpo/margin_mean": 106.76203155517578, + "margin_dpo/margin_std": 205.65582275390625, + "step": 510 + }, + { + "KL/chosen_KL_mean": -184.873046875, + "KL/mean": -241.27972412109375, + "KL/rejected_KL_mean": -297.6864013671875, + "KL/std": 131.96929931640625, + "epoch": 0.7724867724867724, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 112.81336212158203, + "fcm_dpo/q_t": 0.4065815210342407, + "grad_norm": 12.444725036621094, + "learning_rate": 7.557606426772961e-08, + "logits/chosen": 0.9038701057434082, + "logits/rejected": 0.8630021810531616, + "logps/chosen": -240.71743774414062, + "logps/ref_chosen": -55.844383239746094, + "logps/ref_rejected": -86.49819946289062, + "logps/rejected": -384.1846008300781, + "loss": 1.0837, + "margin_dpo/margin_mean": 112.81336212158203, + "margin_dpo/margin_std": 135.35617065429688, + "step": 511 + }, + { + "KL/chosen_KL_mean": -194.4578857421875, + "KL/mean": -227.50833129882812, + "KL/rejected_KL_mean": -260.5587463378906, + "KL/std": 124.75822448730469, + "epoch": 0.7739984882842026, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.1008529663086, + "fcm_dpo/q_t": 0.4442126154899597, + "grad_norm": 18.04234504699707, + "learning_rate": 7.463127807341966e-08, + "logits/chosen": 0.8226577043533325, + "logits/rejected": 0.8273448944091797, + "logps/chosen": -256.1109313964844, + "logps/ref_chosen": -61.653038024902344, + "logps/ref_rejected": -72.83148193359375, + "logps/rejected": -333.3902587890625, + "loss": 1.2403, + "margin_dpo/margin_mean": 66.1008529663086, + "margin_dpo/margin_std": 154.18467712402344, + "step": 512 + }, + { + "KL/chosen_KL_mean": -172.45489501953125, + "KL/mean": -215.680419921875, + "KL/rejected_KL_mean": -258.90594482421875, + "KL/std": 121.78617858886719, + "epoch": 0.7755102040816326, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 86.45104217529297, + "fcm_dpo/q_t": 0.42885059118270874, + "grad_norm": 8.722911834716797, + "learning_rate": 7.369139731924401e-08, + "logits/chosen": 0.9772987365722656, + "logits/rejected": 0.9377813339233398, + "logps/chosen": -223.30746459960938, + "logps/ref_chosen": -50.85256576538086, + "logps/ref_rejected": -69.21754455566406, + "logps/rejected": -328.1235046386719, + "loss": 1.1564, + "margin_dpo/margin_mean": 86.4510498046875, + "margin_dpo/margin_std": 131.8882598876953, + "step": 513 + }, + { + "KL/chosen_KL_mean": -196.28350830078125, + "KL/mean": -246.86834716796875, + "KL/rejected_KL_mean": -297.45318603515625, + "KL/std": 137.37765502929688, + "epoch": 0.7770219198790628, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 101.169677734375, + "fcm_dpo/q_t": 0.41767847537994385, + "grad_norm": 14.140531539916992, + "learning_rate": 7.275644829568747e-08, + "logits/chosen": 0.8947337865829468, + "logits/rejected": 0.8782483339309692, + "logps/chosen": -265.66845703125, + "logps/ref_chosen": -69.38493347167969, + "logps/ref_rejected": -83.32447814941406, + "logps/rejected": -380.7776794433594, + "loss": 1.133, + "margin_dpo/margin_mean": 101.169677734375, + "margin_dpo/margin_std": 156.6019287109375, + "step": 514 + }, + { + "KL/chosen_KL_mean": -201.03335571289062, + "KL/mean": -242.63894653320312, + "KL/rejected_KL_mean": -284.2445373535156, + "KL/std": 127.74497985839844, + "epoch": 0.7785336356764928, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 83.21117401123047, + "fcm_dpo/q_t": 0.4302670359611511, + "grad_norm": 9.764320373535156, + "learning_rate": 7.182645715528435e-08, + "logits/chosen": 0.9377768039703369, + "logits/rejected": 0.8733525276184082, + "logps/chosen": -254.7203826904297, + "logps/ref_chosen": -53.687034606933594, + "logps/ref_rejected": -83.59614562988281, + "logps/rejected": -367.8406677246094, + "loss": 1.1769, + "margin_dpo/margin_mean": 83.211181640625, + "margin_dpo/margin_std": 144.85556030273438, + "step": 515 + }, + { + "KL/chosen_KL_mean": -175.96588134765625, + "KL/mean": -214.3978729248047, + "KL/rejected_KL_mean": -252.82989501953125, + "KL/std": 123.38099670410156, + "epoch": 0.780045351473923, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.86399841308594, + "fcm_dpo/q_t": 0.43540239334106445, + "grad_norm": 11.299661636352539, + "learning_rate": 7.090144991188568e-08, + "logits/chosen": 0.8877590894699097, + "logits/rejected": 0.8631953001022339, + "logps/chosen": -232.8675994873047, + "logps/ref_chosen": -56.9017219543457, + "logps/ref_rejected": -67.83477783203125, + "logps/rejected": -320.6646728515625, + "loss": 1.1956, + "margin_dpo/margin_mean": 76.86400604248047, + "margin_dpo/margin_std": 143.44354248046875, + "step": 516 + }, + { + "KL/chosen_KL_mean": -208.1798095703125, + "KL/mean": -237.89071655273438, + "KL/rejected_KL_mean": -267.60162353515625, + "KL/std": 135.55838012695312, + "epoch": 0.781557067271353, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 59.42182159423828, + "fcm_dpo/q_t": 0.4514128565788269, + "grad_norm": 11.521942138671875, + "learning_rate": 6.998145243993284e-08, + "logits/chosen": 0.9006946086883545, + "logits/rejected": 0.9093080759048462, + "logps/chosen": -269.9549560546875, + "logps/ref_chosen": -61.775142669677734, + "logps/ref_rejected": -62.88270950317383, + "logps/rejected": -330.484375, + "loss": 1.2447, + "margin_dpo/margin_mean": 59.42182922363281, + "margin_dpo/margin_std": 138.92967224121094, + "step": 517 + }, + { + "KL/chosen_KL_mean": -180.6609344482422, + "KL/mean": -224.84420776367188, + "KL/rejected_KL_mean": -269.0274353027344, + "KL/std": 133.86831665039062, + "epoch": 0.783068783068783, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 88.36648559570312, + "fcm_dpo/q_t": 0.4286819100379944, + "grad_norm": 10.708097457885742, + "learning_rate": 6.906649047373245e-08, + "logits/chosen": 0.8798240423202515, + "logits/rejected": 0.8539550304412842, + "logps/chosen": -242.68617248535156, + "logps/ref_chosen": -62.02523422241211, + "logps/ref_rejected": -79.06085205078125, + "logps/rejected": -348.0882873535156, + "loss": 1.1652, + "margin_dpo/margin_mean": 88.36649322509766, + "margin_dpo/margin_std": 151.16380310058594, + "step": 518 + }, + { + "KL/chosen_KL_mean": -209.5022430419922, + "KL/mean": -235.99777221679688, + "KL/rejected_KL_mean": -262.4932861328125, + "KL/std": 132.36302185058594, + "epoch": 0.7845804988662132, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 52.991058349609375, + "fcm_dpo/q_t": 0.45548588037490845, + "grad_norm": 13.631095886230469, + "learning_rate": 6.815658960673781e-08, + "logits/chosen": 0.8932880163192749, + "logits/rejected": 0.8629090785980225, + "logps/chosen": -271.1086120605469, + "logps/ref_chosen": -61.60636901855469, + "logps/ref_rejected": -74.50727844238281, + "logps/rejected": -337.0005798339844, + "loss": 1.2941, + "margin_dpo/margin_mean": 52.991058349609375, + "margin_dpo/margin_std": 169.62295532226562, + "step": 519 + }, + { + "KL/chosen_KL_mean": -195.7787322998047, + "KL/mean": -235.96630859375, + "KL/rejected_KL_mean": -276.1539001464844, + "KL/std": 137.5313262939453, + "epoch": 0.7860922146636432, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.37517547607422, + "fcm_dpo/q_t": 0.4349038004875183, + "grad_norm": 13.164137840270996, + "learning_rate": 6.725177529083209e-08, + "logits/chosen": 0.974045991897583, + "logits/rejected": 0.9345914125442505, + "logps/chosen": -258.65216064453125, + "logps/ref_chosen": -62.87343215942383, + "logps/ref_rejected": -76.505615234375, + "logps/rejected": -352.65948486328125, + "loss": 1.18, + "margin_dpo/margin_mean": 80.37517547607422, + "margin_dpo/margin_std": 138.4918212890625, + "step": 520 + }, + { + "KL/chosen_KL_mean": -191.95623779296875, + "KL/mean": -248.52978515625, + "KL/rejected_KL_mean": -305.1033630371094, + "KL/std": 137.93133544921875, + "epoch": 0.7876039304610734, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 113.1470947265625, + "fcm_dpo/q_t": 0.40972086787223816, + "grad_norm": 8.968234062194824, + "learning_rate": 6.63520728356167e-08, + "logits/chosen": 0.8188831806182861, + "logits/rejected": 0.7698487043380737, + "logps/chosen": -256.16290283203125, + "logps/ref_chosen": -64.20668029785156, + "logps/ref_rejected": -92.28083038330078, + "logps/rejected": -397.3841857910156, + "loss": 1.0978, + "margin_dpo/margin_mean": 113.1470947265625, + "margin_dpo/margin_std": 159.48582458496094, + "step": 521 + }, + { + "KL/chosen_KL_mean": -198.2237548828125, + "KL/mean": -229.06884765625, + "KL/rejected_KL_mean": -259.9139099121094, + "KL/std": 131.58770751953125, + "epoch": 0.7891156462585034, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.690155029296875, + "fcm_dpo/q_t": 0.44906526803970337, + "grad_norm": 11.111807823181152, + "learning_rate": 6.545750740770336e-08, + "logits/chosen": 0.9058156609535217, + "logits/rejected": 0.9006133079528809, + "logps/chosen": -256.5934753417969, + "logps/ref_chosen": -58.369720458984375, + "logps/ref_rejected": -68.79248046875, + "logps/rejected": -328.7063903808594, + "loss": 1.2654, + "margin_dpo/margin_mean": 61.690147399902344, + "margin_dpo/margin_std": 168.9842529296875, + "step": 522 + }, + { + "KL/chosen_KL_mean": -205.13241577148438, + "KL/mean": -247.29925537109375, + "KL/rejected_KL_mean": -289.466064453125, + "KL/std": 140.7245635986328, + "epoch": 0.7906273620559335, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 84.33366394042969, + "fcm_dpo/q_t": 0.42924371361732483, + "grad_norm": 14.811322212219238, + "learning_rate": 6.456810403001012e-08, + "logits/chosen": 0.9063102006912231, + "logits/rejected": 0.8146883249282837, + "logps/chosen": -270.84564208984375, + "logps/ref_chosen": -65.71324157714844, + "logps/ref_rejected": -91.98896789550781, + "logps/rejected": -381.455078125, + "loss": 1.1802, + "margin_dpo/margin_mean": 84.33366394042969, + "margin_dpo/margin_std": 153.331787109375, + "step": 523 + }, + { + "KL/chosen_KL_mean": -182.45616149902344, + "KL/mean": -222.56631469726562, + "KL/rejected_KL_mean": -262.6764831542969, + "KL/std": 133.9646453857422, + "epoch": 0.7921390778533636, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.22032928466797, + "fcm_dpo/q_t": 0.43342095613479614, + "grad_norm": 11.4309720993042, + "learning_rate": 6.368388758106134e-08, + "logits/chosen": 0.8162028789520264, + "logits/rejected": 0.8029775619506836, + "logps/chosen": -258.8074035644531, + "logps/ref_chosen": -76.35124969482422, + "logps/ref_rejected": -89.96072387695312, + "logps/rejected": -352.63720703125, + "loss": 1.1759, + "margin_dpo/margin_mean": 80.2203369140625, + "margin_dpo/margin_std": 133.57424926757812, + "step": 524 + }, + { + "KL/chosen_KL_mean": -204.7865447998047, + "KL/mean": -237.8707275390625, + "KL/rejected_KL_mean": -270.9549255371094, + "KL/std": 131.46104431152344, + "epoch": 0.7936507936507936, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.16841125488281, + "fcm_dpo/q_t": 0.4471135139465332, + "grad_norm": 14.039843559265137, + "learning_rate": 6.280488279429185e-08, + "logits/chosen": 0.7615352869033813, + "logits/rejected": 0.7642134428024292, + "logps/chosen": -280.2823486328125, + "logps/ref_chosen": -75.49578857421875, + "logps/ref_rejected": -84.04852294921875, + "logps/rejected": -355.00347900390625, + "loss": 1.2397, + "margin_dpo/margin_mean": 66.16841125488281, + "margin_dpo/margin_std": 158.4366455078125, + "step": 525 + }, + { + "KL/chosen_KL_mean": -206.23468017578125, + "KL/mean": -239.44537353515625, + "KL/rejected_KL_mean": -272.65606689453125, + "KL/std": 137.84564208984375, + "epoch": 0.7951625094482238, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.42141723632812, + "fcm_dpo/q_t": 0.44594311714172363, + "grad_norm": 11.381861686706543, + "learning_rate": 6.193111425735515e-08, + "logits/chosen": 0.9240107536315918, + "logits/rejected": 0.8726488351821899, + "logps/chosen": -267.527099609375, + "logps/ref_chosen": -61.29241943359375, + "logps/ref_rejected": -82.47763061523438, + "logps/rejected": -355.13372802734375, + "loss": 1.2292, + "margin_dpo/margin_mean": 66.42142486572266, + "margin_dpo/margin_std": 142.61502075195312, + "step": 526 + }, + { + "KL/chosen_KL_mean": -214.8679656982422, + "KL/mean": -244.40792846679688, + "KL/rejected_KL_mean": -273.9479064941406, + "KL/std": 135.75579833984375, + "epoch": 0.7966742252456538, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 59.0799560546875, + "fcm_dpo/q_t": 0.452206552028656, + "grad_norm": 11.494239807128906, + "learning_rate": 6.106260641143546e-08, + "logits/chosen": 0.9482017755508423, + "logits/rejected": 0.8839746713638306, + "logps/chosen": -276.340576171875, + "logps/ref_chosen": -61.472625732421875, + "logps/ref_rejected": -90.52831268310547, + "logps/rejected": -364.4762268066406, + "loss": 1.2573, + "margin_dpo/margin_mean": 59.079952239990234, + "margin_dpo/margin_std": 150.58551025390625, + "step": 527 + }, + { + "KL/chosen_KL_mean": -191.18624877929688, + "KL/mean": -223.8017120361328, + "KL/rejected_KL_mean": -256.41717529296875, + "KL/std": 124.5980224609375, + "epoch": 0.7981859410430839, + "fcm_dpo/beta": 0.00353202479891479, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 65.23095703125, + "fcm_dpo/q_t": 0.4472540318965912, + "grad_norm": 12.8545560836792, + "learning_rate": 6.019938355056422e-08, + "logits/chosen": 0.8528811931610107, + "logits/rejected": 0.8042967915534973, + "logps/chosen": -249.97825622558594, + "logps/ref_chosen": -58.792015075683594, + "logps/ref_rejected": -71.82516479492188, + "logps/rejected": -328.24237060546875, + "loss": 1.253, + "margin_dpo/margin_mean": 65.23095703125, + "margin_dpo/margin_std": 163.40753173828125, + "step": 528 + }, + { + "KL/chosen_KL_mean": -187.9537353515625, + "KL/mean": -245.75003051757812, + "KL/rejected_KL_mean": -303.54632568359375, + "KL/std": 121.35850524902344, + "epoch": 0.799697656840514, + "fcm_dpo/beta": 0.003525674808770418, + "fcm_dpo/delta": -0.01801120676100254, + "fcm_dpo/margin": 115.59262084960938, + "fcm_dpo/q_t": 0.40416377782821655, + "grad_norm": 11.939973831176758, + "learning_rate": 5.934146982094049e-08, + "logits/chosen": 0.8495355248451233, + "logits/rejected": 0.8155577778816223, + "logps/chosen": -243.02468872070312, + "logps/ref_chosen": -55.070960998535156, + "logps/ref_rejected": -75.44007873535156, + "logps/rejected": -378.9864196777344, + "loss": 1.0713, + "margin_dpo/margin_mean": 115.59260559082031, + "margin_dpo/margin_std": 131.84812927246094, + "step": 529 + }, + { + "KL/chosen_KL_mean": -184.20016479492188, + "KL/mean": -225.19178771972656, + "KL/rejected_KL_mean": -266.18341064453125, + "KL/std": 140.19061279296875, + "epoch": 0.8012093726379441, + "fcm_dpo/beta": 0.0035193245857954025, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 81.98326110839844, + "fcm_dpo/q_t": 0.4346047639846802, + "grad_norm": 12.730396270751953, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": 0.9051386117935181, + "logits/rejected": 0.874481201171875, + "logps/chosen": -240.9439697265625, + "logps/ref_chosen": -56.743812561035156, + "logps/ref_rejected": -76.6692123413086, + "logps/rejected": -342.8526306152344, + "loss": 1.1849, + "margin_dpo/margin_mean": 81.98326110839844, + "margin_dpo/margin_std": 149.03494262695312, + "step": 530 + }, + { + "KL/chosen_KL_mean": -184.72265625, + "KL/mean": -225.67088317871094, + "KL/rejected_KL_mean": -266.6191101074219, + "KL/std": 133.89361572265625, + "epoch": 0.8027210884353742, + "fcm_dpo/beta": 0.0035193245857954025, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 81.89643859863281, + "fcm_dpo/q_t": 0.4329376816749573, + "grad_norm": 10.601408958435059, + "learning_rate": 5.7641665597021435e-08, + "logits/chosen": 0.8985570669174194, + "logits/rejected": 0.8340739011764526, + "logps/chosen": -235.839111328125, + "logps/ref_chosen": -51.116455078125, + "logps/ref_rejected": -79.52884674072266, + "logps/rejected": -346.14794921875, + "loss": 1.1821, + "margin_dpo/margin_mean": 81.89643859863281, + "margin_dpo/margin_std": 146.56626892089844, + "step": 531 + }, + { + "KL/chosen_KL_mean": -214.75320434570312, + "KL/mean": -257.167236328125, + "KL/rejected_KL_mean": -299.5812683105469, + "KL/std": 135.9754638671875, + "epoch": 0.8042328042328042, + "fcm_dpo/beta": 0.0035193245857954025, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 84.8280258178711, + "fcm_dpo/q_t": 0.43083542585372925, + "grad_norm": 11.06413459777832, + "learning_rate": 5.679982264990424e-08, + "logits/chosen": 0.8801513910293579, + "logits/rejected": 0.8441125154495239, + "logps/chosen": -273.03314208984375, + "logps/ref_chosen": -58.279945373535156, + "logps/ref_rejected": -78.05426788330078, + "logps/rejected": -377.6355285644531, + "loss": 1.1777, + "margin_dpo/margin_mean": 84.8280258178711, + "margin_dpo/margin_std": 151.73892211914062, + "step": 532 + }, + { + "KL/chosen_KL_mean": -155.84539794921875, + "KL/mean": -205.1451416015625, + "KL/rejected_KL_mean": -254.4448699951172, + "KL/std": 144.37937927246094, + "epoch": 0.8057445200302343, + "fcm_dpo/beta": 0.0035193245857954025, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 98.59945678710938, + "fcm_dpo/q_t": 0.4208802580833435, + "grad_norm": 12.034300804138184, + "learning_rate": 5.596338392706076e-08, + "logits/chosen": 0.9465242624282837, + "logits/rejected": 0.9069874882698059, + "logps/chosen": -212.263427734375, + "logps/ref_chosen": -56.41801071166992, + "logps/ref_rejected": -73.89324951171875, + "logps/rejected": -328.338134765625, + "loss": 1.1378, + "margin_dpo/margin_mean": 98.5994644165039, + "margin_dpo/margin_std": 153.1092987060547, + "step": 533 + }, + { + "KL/chosen_KL_mean": -194.58856201171875, + "KL/mean": -231.89251708984375, + "KL/rejected_KL_mean": -269.1965026855469, + "KL/std": 126.38986206054688, + "epoch": 0.8072562358276644, + "fcm_dpo/beta": 0.0035193245857954025, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 74.60794830322266, + "fcm_dpo/q_t": 0.4392569661140442, + "grad_norm": 11.195599555969238, + "learning_rate": 5.513237282548033e-08, + "logits/chosen": 0.8730405569076538, + "logits/rejected": 0.8510438799858093, + "logps/chosen": -255.33724975585938, + "logps/ref_chosen": -60.748687744140625, + "logps/ref_rejected": -73.8623046875, + "logps/rejected": -343.0588073730469, + "loss": 1.2105, + "margin_dpo/margin_mean": 74.60794830322266, + "margin_dpo/margin_std": 153.75209045410156, + "step": 534 + }, + { + "KL/chosen_KL_mean": -204.45562744140625, + "KL/mean": -240.7137451171875, + "KL/rejected_KL_mean": -276.97186279296875, + "KL/std": 134.9508056640625, + "epoch": 0.8087679516250945, + "fcm_dpo/beta": 0.0035193245857954025, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 72.51627349853516, + "fcm_dpo/q_t": 0.4411739706993103, + "grad_norm": 11.336689949035645, + "learning_rate": 5.430681259032957e-08, + "logits/chosen": 0.8373583555221558, + "logits/rejected": 0.7964029908180237, + "logps/chosen": -266.093017578125, + "logps/ref_chosen": -61.637413024902344, + "logps/ref_rejected": -80.93138885498047, + "logps/rejected": -357.90325927734375, + "loss": 1.2167, + "margin_dpo/margin_mean": 72.51627349853516, + "margin_dpo/margin_std": 150.4508819580078, + "step": 535 + }, + { + "KL/chosen_KL_mean": -176.42593383789062, + "KL/mean": -233.67471313476562, + "KL/rejected_KL_mean": -290.9234924316406, + "KL/std": 134.7869873046875, + "epoch": 0.8102796674225246, + "fcm_dpo/beta": 0.0035193245857954025, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 114.49759674072266, + "fcm_dpo/q_t": 0.4069276452064514, + "grad_norm": 8.79976749420166, + "learning_rate": 5.3486726314303175e-08, + "logits/chosen": 0.9333071708679199, + "logits/rejected": 0.877731442451477, + "logps/chosen": -228.31491088867188, + "logps/ref_chosen": -51.88897705078125, + "logps/ref_rejected": -73.34864044189453, + "logps/rejected": -364.27215576171875, + "loss": 1.0802, + "margin_dpo/margin_mean": 114.49759674072266, + "margin_dpo/margin_std": 139.86199951171875, + "step": 536 + }, + { + "KL/chosen_KL_mean": -200.56451416015625, + "KL/mean": -251.96243286132812, + "KL/rejected_KL_mean": -303.3603515625, + "KL/std": 149.33303833007812, + "epoch": 0.8117913832199547, + "fcm_dpo/beta": 0.0035193245857954025, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 102.79583740234375, + "fcm_dpo/q_t": 0.41782182455062866, + "grad_norm": 10.906473159790039, + "learning_rate": 5.267213693697695e-08, + "logits/chosen": 0.9628849029541016, + "logits/rejected": 0.8948889970779419, + "logps/chosen": -254.81312561035156, + "logps/ref_chosen": -54.248619079589844, + "logps/ref_rejected": -94.94343566894531, + "logps/rejected": -398.30377197265625, + "loss": 1.1366, + "margin_dpo/margin_mean": 102.79583740234375, + "margin_dpo/margin_std": 165.4451904296875, + "step": 537 + }, + { + "KL/chosen_KL_mean": -206.56629943847656, + "KL/mean": -254.18389892578125, + "KL/rejected_KL_mean": -301.801513671875, + "KL/std": 152.03558349609375, + "epoch": 0.8133030990173847, + "fcm_dpo/beta": 0.0035193245857954025, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 95.23521423339844, + "fcm_dpo/q_t": 0.42235174775123596, + "grad_norm": 10.19690227508545, + "learning_rate": 5.1863067244167144e-08, + "logits/chosen": 0.9172987937927246, + "logits/rejected": 0.9068432450294495, + "logps/chosen": -276.65985107421875, + "logps/ref_chosen": -70.09353637695312, + "logps/ref_rejected": -79.49833679199219, + "logps/rejected": -381.2998352050781, + "loss": 1.1466, + "margin_dpo/margin_mean": 95.23521423339844, + "margin_dpo/margin_std": 152.82855224609375, + "step": 538 + }, + { + "KL/chosen_KL_mean": -206.97450256347656, + "KL/mean": -244.26101684570312, + "KL/rejected_KL_mean": -281.5475158691406, + "KL/std": 135.51553344726562, + "epoch": 0.8148148148148148, + "fcm_dpo/beta": 0.0035193245857954025, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 74.57301330566406, + "fcm_dpo/q_t": 0.4409199357032776, + "grad_norm": 12.181897163391113, + "learning_rate": 5.105953986729195e-08, + "logits/chosen": 0.869612455368042, + "logits/rejected": 0.8076390027999878, + "logps/chosen": -268.90618896484375, + "logps/ref_chosen": -61.93169403076172, + "logps/ref_rejected": -84.08946228027344, + "logps/rejected": -365.636962890625, + "loss": 1.2022, + "margin_dpo/margin_mean": 74.57301330566406, + "margin_dpo/margin_std": 146.1897735595703, + "step": 539 + }, + { + "KL/chosen_KL_mean": -191.78707885742188, + "KL/mean": -251.977783203125, + "KL/rejected_KL_mean": -312.16851806640625, + "KL/std": 161.7171630859375, + "epoch": 0.8163265306122449, + "fcm_dpo/beta": 0.003508302615955472, + "fcm_dpo/delta": -0.03141680359840393, + "fcm_dpo/margin": 120.38145446777344, + "fcm_dpo/q_t": 0.40351608395576477, + "grad_norm": 9.613325119018555, + "learning_rate": 5.026157728273966e-08, + "logits/chosen": 0.9319074749946594, + "logits/rejected": 0.8714286088943481, + "logps/chosen": -254.49131774902344, + "logps/ref_chosen": -62.704254150390625, + "logps/ref_rejected": -95.63597106933594, + "logps/rejected": -407.80450439453125, + "loss": 1.0779, + "margin_dpo/margin_mean": 120.3814697265625, + "margin_dpo/margin_std": 157.05987548828125, + "step": 540 + }, + { + "KL/chosen_KL_mean": -187.46072387695312, + "KL/mean": -230.84857177734375, + "KL/rejected_KL_mean": -274.2364196777344, + "KL/std": 133.34332275390625, + "epoch": 0.817838246409675, + "fcm_dpo/beta": 0.0034972806461155415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 86.77567291259766, + "fcm_dpo/q_t": 0.4274454712867737, + "grad_norm": 10.517534255981445, + "learning_rate": 4.9469201811239035e-08, + "logits/chosen": 0.9364334344863892, + "logits/rejected": 0.9676176309585571, + "logps/chosen": -249.94155883789062, + "logps/ref_chosen": -62.48084259033203, + "logps/ref_rejected": -57.55541229248047, + "logps/rejected": -331.79180908203125, + "loss": 1.1528, + "margin_dpo/margin_mean": 86.77566528320312, + "margin_dpo/margin_std": 125.50430297851562, + "step": 541 + }, + { + "KL/chosen_KL_mean": -163.31414794921875, + "KL/mean": -212.81866455078125, + "KL/rejected_KL_mean": -262.32318115234375, + "KL/std": 123.844482421875, + "epoch": 0.8193499622071051, + "fcm_dpo/beta": 0.0034972806461155415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 99.00904846191406, + "fcm_dpo/q_t": 0.41917315125465393, + "grad_norm": 10.483098983764648, + "learning_rate": 4.868243561723534e-08, + "logits/chosen": 0.9898308515548706, + "logits/rejected": 0.9567674398422241, + "logps/chosen": -212.76902770996094, + "logps/ref_chosen": -49.454891204833984, + "logps/ref_rejected": -65.33275604248047, + "logps/rejected": -327.65594482421875, + "loss": 1.1378, + "margin_dpo/margin_mean": 99.00904846191406, + "margin_dpo/margin_std": 152.8940887451172, + "step": 542 + }, + { + "KL/chosen_KL_mean": -176.56614685058594, + "KL/mean": -229.4498291015625, + "KL/rejected_KL_mean": -282.33349609375, + "KL/std": 136.6936798095703, + "epoch": 0.8208616780045351, + "fcm_dpo/beta": 0.0034972806461155415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 105.7673568725586, + "fcm_dpo/q_t": 0.41475868225097656, + "grad_norm": 9.79175090789795, + "learning_rate": 4.790130070827028e-08, + "logits/chosen": 0.9310851097106934, + "logits/rejected": 0.8702279329299927, + "logps/chosen": -227.66700744628906, + "logps/ref_chosen": -51.100860595703125, + "logps/ref_rejected": -76.06130981445312, + "logps/rejected": -358.39483642578125, + "loss": 1.1182, + "margin_dpo/margin_mean": 105.76736450195312, + "margin_dpo/margin_std": 151.32635498046875, + "step": 543 + }, + { + "KL/chosen_KL_mean": -192.3682861328125, + "KL/mean": -251.83297729492188, + "KL/rejected_KL_mean": -311.29766845703125, + "KL/std": 152.34619140625, + "epoch": 0.8223733938019653, + "fcm_dpo/beta": 0.0034972806461155415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 118.92938232421875, + "fcm_dpo/q_t": 0.40710359811782837, + "grad_norm": 12.77357292175293, + "learning_rate": 4.7125818934366454e-08, + "logits/chosen": 0.8962240219116211, + "logits/rejected": 0.8459259271621704, + "logps/chosen": -252.64552307128906, + "logps/ref_chosen": -60.2772331237793, + "logps/ref_rejected": -88.40553283691406, + "logps/rejected": -399.70318603515625, + "loss": 1.1037, + "margin_dpo/margin_mean": 118.92938232421875, + "margin_dpo/margin_std": 180.5123291015625, + "step": 544 + }, + { + "KL/chosen_KL_mean": -209.5484161376953, + "KL/mean": -240.4080810546875, + "KL/rejected_KL_mean": -271.2677307128906, + "KL/std": 137.64572143554688, + "epoch": 0.8238851095993953, + "fcm_dpo/beta": 0.0034972806461155415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.71931457519531, + "fcm_dpo/q_t": 0.4491823613643646, + "grad_norm": 12.780492782592773, + "learning_rate": 4.635601198741607e-08, + "logits/chosen": 0.9082489013671875, + "logits/rejected": 0.8583580851554871, + "logps/chosen": -271.1636657714844, + "logps/ref_chosen": -61.61524963378906, + "logps/ref_rejected": -78.71266174316406, + "logps/rejected": -349.9803771972656, + "loss": 1.2488, + "margin_dpo/margin_mean": 61.71931457519531, + "margin_dpo/margin_std": 150.1622314453125, + "step": 545 + }, + { + "KL/chosen_KL_mean": -198.5826416015625, + "KL/mean": -232.60609436035156, + "KL/rejected_KL_mean": -266.6295471191406, + "KL/std": 130.08131408691406, + "epoch": 0.8253968253968254, + "fcm_dpo/beta": 0.0034972806461155415, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 68.0468978881836, + "fcm_dpo/q_t": 0.443244993686676, + "grad_norm": 16.30405044555664, + "learning_rate": 4.559190140057428e-08, + "logits/chosen": 0.9910711646080017, + "logits/rejected": 0.9918454885482788, + "logps/chosen": -257.8959045410156, + "logps/ref_chosen": -59.313262939453125, + "logps/ref_rejected": -64.73631286621094, + "logps/rejected": -331.3658447265625, + "loss": 1.2364, + "margin_dpo/margin_mean": 68.04690551757812, + "margin_dpo/margin_std": 158.61746215820312, + "step": 546 + }, + { + "KL/chosen_KL_mean": -171.865966796875, + "KL/mean": -226.3336181640625, + "KL/rejected_KL_mean": -280.80126953125, + "KL/std": 134.1888885498047, + "epoch": 0.8269085411942555, + "fcm_dpo/beta": 0.0034795503597706556, + "fcm_dpo/delta": -0.0254131481051445, + "fcm_dpo/margin": 108.93531036376953, + "fcm_dpo/q_t": 0.41161462664604187, + "grad_norm": 10.086834907531738, + "learning_rate": 4.483350854765672e-08, + "logits/chosen": 0.8900845050811768, + "logits/rejected": 0.8488314747810364, + "logps/chosen": -226.84271240234375, + "logps/ref_chosen": -54.97674560546875, + "logps/ref_rejected": -75.35922241210938, + "logps/rejected": -356.16046142578125, + "loss": 1.1111, + "margin_dpo/margin_mean": 108.93531036376953, + "margin_dpo/margin_std": 154.0179901123047, + "step": 547 + }, + { + "KL/chosen_KL_mean": -204.906982421875, + "KL/mean": -238.65890502929688, + "KL/rejected_KL_mean": -272.4108581542969, + "KL/std": 137.80738830566406, + "epoch": 0.8284202569916855, + "fcm_dpo/beta": 0.0034795503597706556, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 67.50390625, + "fcm_dpo/q_t": 0.4453282952308655, + "grad_norm": 11.766864776611328, + "learning_rate": 4.4080854642541826e-08, + "logits/chosen": 0.8757404088973999, + "logits/rejected": 0.8392364978790283, + "logps/chosen": -268.1176452636719, + "logps/ref_chosen": -63.21067428588867, + "logps/ref_rejected": -81.23347473144531, + "logps/rejected": -353.64434814453125, + "loss": 1.2296, + "margin_dpo/margin_mean": 67.50390625, + "margin_dpo/margin_std": 149.1422882080078, + "step": 548 + }, + { + "KL/chosen_KL_mean": -206.6922149658203, + "KL/mean": -245.13987731933594, + "KL/rejected_KL_mean": -283.5875244140625, + "KL/std": 154.62350463867188, + "epoch": 0.8299319727891157, + "fcm_dpo/beta": 0.0034795503597706556, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.89530181884766, + "fcm_dpo/q_t": 0.4395865201950073, + "grad_norm": 14.56954288482666, + "learning_rate": 4.333396073857723e-08, + "logits/chosen": 0.9742203950881958, + "logits/rejected": 0.9250259399414062, + "logps/chosen": -270.9657287597656, + "logps/ref_chosen": -64.27351379394531, + "logps/ref_rejected": -92.31663513183594, + "logps/rejected": -375.9041748046875, + "loss": 1.232, + "margin_dpo/margin_mean": 76.89530181884766, + "margin_dpo/margin_std": 181.66064453125, + "step": 549 + }, + { + "KL/chosen_KL_mean": -200.8487548828125, + "KL/mean": -230.66412353515625, + "KL/rejected_KL_mean": -260.4795227050781, + "KL/std": 129.66531372070312, + "epoch": 0.8314436885865457, + "fcm_dpo/beta": 0.0034795503597706556, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 59.630733489990234, + "fcm_dpo/q_t": 0.4519397020339966, + "grad_norm": 11.026690483093262, + "learning_rate": 4.259284772799099e-08, + "logits/chosen": 0.9252485036849976, + "logits/rejected": 0.91241455078125, + "logps/chosen": -257.0791931152344, + "logps/ref_chosen": -56.230438232421875, + "logps/ref_rejected": -62.59788513183594, + "logps/rejected": -323.077392578125, + "loss": 1.25, + "margin_dpo/margin_mean": 59.6307373046875, + "margin_dpo/margin_std": 142.07293701171875, + "step": 550 + }, + { + "KL/chosen_KL_mean": -212.65078735351562, + "KL/mean": -249.81268310546875, + "KL/rejected_KL_mean": -286.974609375, + "KL/std": 144.7467498779297, + "epoch": 0.8329554043839759, + "fcm_dpo/beta": 0.0034795503597706556, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 74.32379150390625, + "fcm_dpo/q_t": 0.44245636463165283, + "grad_norm": 9.790303230285645, + "learning_rate": 4.1857536341307176e-08, + "logits/chosen": 0.9503294229507446, + "logits/rejected": 0.929543137550354, + "logps/chosen": -280.39801025390625, + "logps/ref_chosen": -67.74720764160156, + "logps/ref_rejected": -87.04285430908203, + "logps/rejected": -374.0174560546875, + "loss": 1.2062, + "margin_dpo/margin_mean": 74.32378387451172, + "margin_dpo/margin_std": 147.83872985839844, + "step": 551 + }, + { + "KL/chosen_KL_mean": -196.72901916503906, + "KL/mean": -239.97210693359375, + "KL/rejected_KL_mean": -283.2151794433594, + "KL/std": 154.41824340820312, + "epoch": 0.8344671201814059, + "fcm_dpo/beta": 0.0034795503597706556, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 86.48617553710938, + "fcm_dpo/q_t": 0.429646760225296, + "grad_norm": 13.021488189697266, + "learning_rate": 4.112804714676593e-08, + "logits/chosen": 0.9085184335708618, + "logits/rejected": 0.8709832429885864, + "logps/chosen": -259.6552734375, + "logps/ref_chosen": -62.92625427246094, + "logps/ref_rejected": -82.98365783691406, + "logps/rejected": -366.1988525390625, + "loss": 1.1662, + "margin_dpo/margin_mean": 86.48617553710938, + "margin_dpo/margin_std": 141.24722290039062, + "step": 552 + }, + { + "KL/chosen_KL_mean": -210.44729614257812, + "KL/mean": -255.28866577148438, + "KL/rejected_KL_mean": -300.1300048828125, + "KL/std": 151.55690002441406, + "epoch": 0.8359788359788359, + "fcm_dpo/beta": 0.0034795503597706556, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.68273162841797, + "fcm_dpo/q_t": 0.43007200956344604, + "grad_norm": 12.203096389770508, + "learning_rate": 4.0404400549748144e-08, + "logits/chosen": 0.9325939416885376, + "logits/rejected": 0.8622708320617676, + "logps/chosen": -266.48577880859375, + "logps/ref_chosen": -56.038490295410156, + "logps/ref_rejected": -84.48454284667969, + "logps/rejected": -384.61456298828125, + "loss": 1.1827, + "margin_dpo/margin_mean": 89.68273162841797, + "margin_dpo/margin_std": 174.0795440673828, + "step": 553 + }, + { + "KL/chosen_KL_mean": -189.37258911132812, + "KL/mean": -238.8749237060547, + "KL/rejected_KL_mean": -288.37725830078125, + "KL/std": 133.0142364501953, + "epoch": 0.8374905517762661, + "fcm_dpo/beta": 0.0034795503597706556, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 99.00468444824219, + "fcm_dpo/q_t": 0.4176701605319977, + "grad_norm": 10.483379364013672, + "learning_rate": 3.968661679220467e-08, + "logits/chosen": 0.8873802423477173, + "logits/rejected": 0.8916881084442139, + "logps/chosen": -253.90318298339844, + "logps/ref_chosen": -64.53059387207031, + "logps/ref_rejected": -71.2155990600586, + "logps/rejected": -359.5928955078125, + "loss": 1.1295, + "margin_dpo/margin_mean": 99.00468444824219, + "margin_dpo/margin_std": 140.27540588378906, + "step": 554 + }, + { + "KL/chosen_KL_mean": -207.7998809814453, + "KL/mean": -241.81884765625, + "KL/rejected_KL_mean": -275.83782958984375, + "KL/std": 131.1754913330078, + "epoch": 0.8390022675736961, + "fcm_dpo/beta": 0.0034795503597706556, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 68.0379409790039, + "fcm_dpo/q_t": 0.44281113147735596, + "grad_norm": 11.698552131652832, + "learning_rate": 3.89747159520904e-08, + "logits/chosen": 0.932090163230896, + "logits/rejected": 0.9236975908279419, + "logps/chosen": -274.4517822265625, + "logps/ref_chosen": -66.65191650390625, + "logps/ref_rejected": -68.6667251586914, + "logps/rejected": -344.5045471191406, + "loss": 1.236, + "margin_dpo/margin_mean": 68.0379409790039, + "margin_dpo/margin_std": 154.10720825195312, + "step": 555 + }, + { + "KL/chosen_KL_mean": -199.71800231933594, + "KL/mean": -238.94131469726562, + "KL/rejected_KL_mean": -278.16461181640625, + "KL/std": 134.803466796875, + "epoch": 0.8405139833711263, + "fcm_dpo/beta": 0.0034795503597706556, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 78.44662475585938, + "fcm_dpo/q_t": 0.4365660846233368, + "grad_norm": 11.139961242675781, + "learning_rate": 3.826871794280192e-08, + "logits/chosen": 0.9880878925323486, + "logits/rejected": 0.9601046442985535, + "logps/chosen": -252.5503692626953, + "logps/ref_chosen": -52.832366943359375, + "logps/ref_rejected": -64.49044036865234, + "logps/rejected": -342.65509033203125, + "loss": 1.1934, + "margin_dpo/margin_mean": 78.44662475585938, + "margin_dpo/margin_std": 143.8062286376953, + "step": 556 + }, + { + "KL/chosen_KL_mean": -199.31085205078125, + "KL/mean": -255.07276916503906, + "KL/rejected_KL_mean": -310.834716796875, + "KL/std": 142.0129852294922, + "epoch": 0.8420256991685563, + "fcm_dpo/beta": 0.0034691179171204567, + "fcm_dpo/delta": -0.01501347217708826, + "fcm_dpo/margin": 111.52387237548828, + "fcm_dpo/q_t": 0.41104036569595337, + "grad_norm": 10.035857200622559, + "learning_rate": 3.756864251262143e-08, + "logits/chosen": 0.9729760885238647, + "logits/rejected": 0.9261834621429443, + "logps/chosen": -254.34683227539062, + "logps/ref_chosen": -55.03598403930664, + "logps/ref_rejected": -75.80644989013672, + "logps/rejected": -386.64117431640625, + "loss": 1.0961, + "margin_dpo/margin_mean": 111.52386474609375, + "margin_dpo/margin_std": 141.70437622070312, + "step": 557 + }, + { + "KL/chosen_KL_mean": -191.17112731933594, + "KL/mean": -249.64356994628906, + "KL/rejected_KL_mean": -308.11602783203125, + "KL/std": 148.79632568359375, + "epoch": 0.8435374149659864, + "fcm_dpo/beta": 0.0034465533681213856, + "fcm_dpo/delta": -0.032628294080495834, + "fcm_dpo/margin": 116.94486999511719, + "fcm_dpo/q_t": 0.40802621841430664, + "grad_norm": 10.100082397460938, + "learning_rate": 3.687450924416341e-08, + "logits/chosen": 0.9450229406356812, + "logits/rejected": 0.9094198346138, + "logps/chosen": -254.39749145507812, + "logps/ref_chosen": -63.226348876953125, + "logps/ref_rejected": -91.46881866455078, + "logps/rejected": -399.5848388671875, + "loss": 1.0967, + "margin_dpo/margin_mean": 116.94486999511719, + "margin_dpo/margin_std": 162.83993530273438, + "step": 558 + }, + { + "KL/chosen_KL_mean": -192.72799682617188, + "KL/mean": -246.63272094726562, + "KL/rejected_KL_mean": -300.5374450683594, + "KL/std": 150.26181030273438, + "epoch": 0.8450491307634165, + "fcm_dpo/beta": 0.0034465533681213856, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 107.80944061279297, + "fcm_dpo/q_t": 0.4185447692871094, + "grad_norm": 11.648037910461426, + "learning_rate": 3.6186337553827743e-08, + "logits/chosen": 0.9157607555389404, + "logits/rejected": 0.8777155876159668, + "logps/chosen": -254.24964904785156, + "logps/ref_chosen": -61.521644592285156, + "logps/ref_rejected": -82.83859252929688, + "logps/rejected": -383.37603759765625, + "loss": 1.1398, + "margin_dpo/margin_mean": 107.8094482421875, + "margin_dpo/margin_std": 182.46499633789062, + "step": 559 + }, + { + "KL/chosen_KL_mean": -209.66896057128906, + "KL/mean": -258.95849609375, + "KL/rejected_KL_mean": -308.2480163574219, + "KL/std": 139.55429077148438, + "epoch": 0.8465608465608465, + "fcm_dpo/beta": 0.0034465533681213856, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 98.57904815673828, + "fcm_dpo/q_t": 0.4205133318901062, + "grad_norm": 12.194161415100098, + "learning_rate": 3.550414669125573e-08, + "logits/chosen": 0.9571900963783264, + "logits/rejected": 0.9336278438568115, + "logps/chosen": -270.3101806640625, + "logps/ref_chosen": -60.64122009277344, + "logps/ref_rejected": -78.75474548339844, + "logps/rejected": -387.00274658203125, + "loss": 1.1241, + "margin_dpo/margin_mean": 98.57905578613281, + "margin_dpo/margin_std": 129.6819305419922, + "step": 560 + }, + { + "KL/chosen_KL_mean": -197.01702880859375, + "KL/mean": -237.6276092529297, + "KL/rejected_KL_mean": -278.2381896972656, + "KL/std": 132.2579345703125, + "epoch": 0.8480725623582767, + "fcm_dpo/beta": 0.0034465533681213856, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 81.22117614746094, + "fcm_dpo/q_t": 0.43530896306037903, + "grad_norm": 10.928399085998535, + "learning_rate": 3.482795573879241e-08, + "logits/chosen": 0.8836110830307007, + "logits/rejected": 0.8617398738861084, + "logps/chosen": -259.515625, + "logps/ref_chosen": -62.49859619140625, + "logps/ref_rejected": -78.72064208984375, + "logps/rejected": -356.9588317871094, + "loss": 1.1879, + "margin_dpo/margin_mean": 81.2211685180664, + "margin_dpo/margin_std": 147.069580078125, + "step": 561 + }, + { + "KL/chosen_KL_mean": -213.68255615234375, + "KL/mean": -271.47039794921875, + "KL/rejected_KL_mean": -329.25823974609375, + "KL/std": 176.3778076171875, + "epoch": 0.8495842781557067, + "fcm_dpo/beta": 0.003401133930310607, + "fcm_dpo/delta": -0.06632918119430542, + "fcm_dpo/margin": 115.57572174072266, + "fcm_dpo/q_t": 0.41180339455604553, + "grad_norm": 13.891858100891113, + "learning_rate": 3.415778361095226e-08, + "logits/chosen": 0.9108099937438965, + "logits/rejected": 0.8991225957870483, + "logps/chosen": -288.46429443359375, + "logps/ref_chosen": -74.78173828125, + "logps/ref_rejected": -92.63499450683594, + "logps/rejected": -421.89324951171875, + "loss": 1.1111, + "margin_dpo/margin_mean": 115.57572937011719, + "margin_dpo/margin_std": 170.50723266601562, + "step": 562 + }, + { + "KL/chosen_KL_mean": -172.10968017578125, + "KL/mean": -219.07452392578125, + "KL/rejected_KL_mean": -266.03936767578125, + "KL/std": 117.71343994140625, + "epoch": 0.8510959939531368, + "fcm_dpo/beta": 0.003401133930310607, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 93.92967987060547, + "fcm_dpo/q_t": 0.424142062664032, + "grad_norm": 17.967735290527344, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": 1.005516529083252, + "logits/rejected": 0.9655479192733765, + "logps/chosen": -222.30819702148438, + "logps/ref_chosen": -50.19850158691406, + "logps/ref_rejected": -66.76687622070312, + "logps/rejected": -332.8062438964844, + "loss": 1.154, + "margin_dpo/margin_mean": 93.9296875, + "margin_dpo/margin_std": 148.81942749023438, + "step": 563 + }, + { + "KL/chosen_KL_mean": -172.90924072265625, + "KL/mean": -230.859619140625, + "KL/rejected_KL_mean": -288.80999755859375, + "KL/std": 130.04574584960938, + "epoch": 0.8526077097505669, + "fcm_dpo/beta": 0.003401133930310607, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 115.9007339477539, + "fcm_dpo/q_t": 0.40835243463516235, + "grad_norm": 11.180398941040039, + "learning_rate": 3.283557064487785e-08, + "logits/chosen": 0.90259850025177, + "logits/rejected": 0.8867555856704712, + "logps/chosen": -228.65008544921875, + "logps/ref_chosen": -55.7408447265625, + "logps/ref_rejected": -74.82323455810547, + "logps/rejected": -363.63323974609375, + "loss": 1.1012, + "margin_dpo/margin_mean": 115.90072631835938, + "margin_dpo/margin_std": 161.30313110351562, + "step": 564 + }, + { + "KL/chosen_KL_mean": -211.1297607421875, + "KL/mean": -259.3542785644531, + "KL/rejected_KL_mean": -307.57879638671875, + "KL/std": 142.05772399902344, + "epoch": 0.854119425547997, + "fcm_dpo/beta": 0.003401133930310607, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 96.44905090332031, + "fcm_dpo/q_t": 0.4226211905479431, + "grad_norm": 12.189798355102539, + "learning_rate": 3.218356679178252e-08, + "logits/chosen": 0.9537685513496399, + "logits/rejected": 0.9214394092559814, + "logps/chosen": -269.4671630859375, + "logps/ref_chosen": -58.33738327026367, + "logps/ref_rejected": -78.31776428222656, + "logps/rejected": -385.89654541015625, + "loss": 1.1404, + "margin_dpo/margin_mean": 96.44903564453125, + "margin_dpo/margin_std": 142.17813110351562, + "step": 565 + }, + { + "KL/chosen_KL_mean": -202.7068634033203, + "KL/mean": -244.68670654296875, + "KL/rejected_KL_mean": -286.6665344238281, + "KL/std": 151.67178344726562, + "epoch": 0.8556311413454271, + "fcm_dpo/beta": 0.003401133930310607, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 83.95967864990234, + "fcm_dpo/q_t": 0.4360388517379761, + "grad_norm": 9.914851188659668, + "learning_rate": 3.1537655732553764e-08, + "logits/chosen": 0.944286048412323, + "logits/rejected": 0.9478252530097961, + "logps/chosen": -273.93060302734375, + "logps/ref_chosen": -71.22373962402344, + "logps/ref_rejected": -71.11601257324219, + "logps/rejected": -357.78253173828125, + "loss": 1.2189, + "margin_dpo/margin_mean": 83.95967102050781, + "margin_dpo/margin_std": 190.03399658203125, + "step": 566 + }, + { + "KL/chosen_KL_mean": -187.10617065429688, + "KL/mean": -242.869873046875, + "KL/rejected_KL_mean": -298.63360595703125, + "KL/std": 141.89431762695312, + "epoch": 0.8571428571428571, + "fcm_dpo/beta": 0.003368604462593794, + "fcm_dpo/delta": -0.04805165156722069, + "fcm_dpo/margin": 111.52742767333984, + "fcm_dpo/q_t": 0.41300350427627563, + "grad_norm": 9.936294555664062, + "learning_rate": 3.089785553471233e-08, + "logits/chosen": 0.9251055121421814, + "logits/rejected": 0.8682990670204163, + "logps/chosen": -239.77545166015625, + "logps/ref_chosen": -52.669273376464844, + "logps/ref_rejected": -74.34785461425781, + "logps/rejected": -372.9814453125, + "loss": 1.1155, + "margin_dpo/margin_mean": 111.52742767333984, + "margin_dpo/margin_std": 159.84197998046875, + "step": 567 + }, + { + "KL/chosen_KL_mean": -175.10830688476562, + "KL/mean": -240.4864501953125, + "KL/rejected_KL_mean": -305.8646240234375, + "KL/std": 149.96920776367188, + "epoch": 0.8586545729402872, + "fcm_dpo/beta": 0.0033373315818607807, + "fcm_dpo/delta": -0.09370847791433334, + "fcm_dpo/margin": 130.75628662109375, + "fcm_dpo/q_t": 0.40074622631073, + "grad_norm": 13.38181209564209, + "learning_rate": 3.026418409484513e-08, + "logits/chosen": 0.9603248834609985, + "logits/rejected": 0.9052588939666748, + "logps/chosen": -227.28631591796875, + "logps/ref_chosen": -52.178001403808594, + "logps/ref_rejected": -85.8277587890625, + "logps/rejected": -391.6923828125, + "loss": 1.0673, + "margin_dpo/margin_mean": 130.75628662109375, + "margin_dpo/margin_std": 157.99542236328125, + "step": 568 + }, + { + "KL/chosen_KL_mean": -209.8172149658203, + "KL/mean": -239.02120971679688, + "KL/rejected_KL_mean": -268.2251892089844, + "KL/std": 151.32420349121094, + "epoch": 0.8601662887377173, + "fcm_dpo/beta": 0.003306058933958411, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 58.4079704284668, + "fcm_dpo/q_t": 0.4554460644721985, + "grad_norm": 12.692896842956543, + "learning_rate": 2.963665913810451e-08, + "logits/chosen": 0.8867205381393433, + "logits/rejected": 0.8692376017570496, + "logps/chosen": -272.46649169921875, + "logps/ref_chosen": -62.649261474609375, + "logps/ref_rejected": -75.4298324584961, + "logps/rejected": -343.655029296875, + "loss": 1.2721, + "margin_dpo/margin_mean": 58.4079704284668, + "margin_dpo/margin_std": 159.5178985595703, + "step": 569 + }, + { + "KL/chosen_KL_mean": -184.9730987548828, + "KL/mean": -242.1673126220703, + "KL/rejected_KL_mean": -299.36151123046875, + "KL/std": 139.96983337402344, + "epoch": 0.8616780045351474, + "fcm_dpo/beta": 0.003306058933958411, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 114.38844299316406, + "fcm_dpo/q_t": 0.41404989361763, + "grad_norm": 12.19923210144043, + "learning_rate": 2.9015298217712453e-08, + "logits/chosen": 0.928882896900177, + "logits/rejected": 0.8641595244407654, + "logps/chosen": -235.014892578125, + "logps/ref_chosen": -50.04179382324219, + "logps/ref_rejected": -78.27146911621094, + "logps/rejected": -377.63299560546875, + "loss": 1.1134, + "margin_dpo/margin_mean": 114.38844299316406, + "margin_dpo/margin_std": 166.88583374023438, + "step": 570 + }, + { + "KL/chosen_KL_mean": -192.01284790039062, + "KL/mean": -229.24810791015625, + "KL/rejected_KL_mean": -266.4833679199219, + "KL/std": 138.1226043701172, + "epoch": 0.8631897203325775, + "fcm_dpo/beta": 0.003306058933958411, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 74.47052001953125, + "fcm_dpo/q_t": 0.44437307119369507, + "grad_norm": 8.68694019317627, + "learning_rate": 2.840011871446962e-08, + "logits/chosen": 0.9437531232833862, + "logits/rejected": 0.9379255771636963, + "logps/chosen": -245.669677734375, + "logps/ref_chosen": -53.65681457519531, + "logps/ref_rejected": -66.13298034667969, + "logps/rejected": -332.6163330078125, + "loss": 1.2239, + "margin_dpo/margin_mean": 74.47051239013672, + "margin_dpo/margin_std": 161.92054748535156, + "step": 571 + }, + { + "KL/chosen_KL_mean": -208.37600708007812, + "KL/mean": -239.53163146972656, + "KL/rejected_KL_mean": -270.687255859375, + "KL/std": 139.2075653076172, + "epoch": 0.8647014361300076, + "fcm_dpo/beta": 0.003306058933958411, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 62.31123352050781, + "fcm_dpo/q_t": 0.45096078515052795, + "grad_norm": 12.564879417419434, + "learning_rate": 2.7791137836269158e-08, + "logits/chosen": 0.9028031826019287, + "logits/rejected": 0.9370099306106567, + "logps/chosen": -283.1939392089844, + "logps/ref_chosen": -74.81792449951172, + "logps/ref_rejected": -65.88681030273438, + "logps/rejected": -336.5740661621094, + "loss": 1.2508, + "margin_dpo/margin_mean": 62.31123352050781, + "margin_dpo/margin_std": 148.28436279296875, + "step": 572 + }, + { + "KL/chosen_KL_mean": -219.3136749267578, + "KL/mean": -264.943603515625, + "KL/rejected_KL_mean": -310.57354736328125, + "KL/std": 160.91592407226562, + "epoch": 0.8662131519274376, + "fcm_dpo/beta": 0.003306058933958411, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.25989532470703, + "fcm_dpo/q_t": 0.4338764250278473, + "grad_norm": 12.651811599731445, + "learning_rate": 2.718837261761528e-08, + "logits/chosen": 0.9531590938568115, + "logits/rejected": 0.9273461103439331, + "logps/chosen": -288.039306640625, + "logps/ref_chosen": -68.72564697265625, + "logps/ref_rejected": -88.16201782226562, + "logps/rejected": -398.735595703125, + "loss": 1.2021, + "margin_dpo/margin_mean": 91.25990295410156, + "margin_dpo/margin_std": 193.46453857421875, + "step": 573 + }, + { + "KL/chosen_KL_mean": -191.57583618164062, + "KL/mean": -251.47848510742188, + "KL/rejected_KL_mean": -311.3811340332031, + "KL/std": 150.62881469726562, + "epoch": 0.8677248677248677, + "fcm_dpo/beta": 0.003287060186266899, + "fcm_dpo/delta": -0.05779891833662987, + "fcm_dpo/margin": 119.80528259277344, + "fcm_dpo/q_t": 0.4098057150840759, + "grad_norm": 9.95071792602539, + "learning_rate": 2.659183991914696e-08, + "logits/chosen": 0.9709665775299072, + "logits/rejected": 0.9339627623558044, + "logps/chosen": -247.88925170898438, + "logps/ref_chosen": -56.31340026855469, + "logps/ref_rejected": -83.91553497314453, + "logps/rejected": -395.29669189453125, + "loss": 1.0883, + "margin_dpo/margin_mean": 119.80528259277344, + "margin_dpo/margin_std": 150.09042358398438, + "step": 574 + }, + { + "KL/chosen_KL_mean": -199.07955932617188, + "KL/mean": -242.06597900390625, + "KL/rejected_KL_mean": -285.05242919921875, + "KL/std": 151.01272583007812, + "epoch": 0.8692365835222978, + "fcm_dpo/beta": 0.0032680616714060307, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 85.97285461425781, + "fcm_dpo/q_t": 0.43860968947410583, + "grad_norm": 11.692818641662598, + "learning_rate": 2.600155642716606e-08, + "logits/chosen": 0.9394083023071289, + "logits/rejected": 0.8856104016304016, + "logps/chosen": -263.6636962890625, + "logps/ref_chosen": -64.5841293334961, + "logps/ref_rejected": -93.47034454345703, + "logps/rejected": -378.52276611328125, + "loss": 1.2128, + "margin_dpo/margin_mean": 85.97285461425781, + "margin_dpo/margin_std": 182.2880096435547, + "step": 575 + }, + { + "KL/chosen_KL_mean": -173.85931396484375, + "KL/mean": -238.97288513183594, + "KL/rejected_KL_mean": -304.08648681640625, + "KL/std": 149.41693115234375, + "epoch": 0.8707482993197279, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": -0.06305931508541107, + "fcm_dpo/margin": 130.22715759277344, + "fcm_dpo/q_t": 0.40280795097351074, + "grad_norm": 11.600000381469727, + "learning_rate": 2.5417538653170754e-08, + "logits/chosen": 0.9049896001815796, + "logits/rejected": 0.8447592854499817, + "logps/chosen": -227.1398468017578, + "logps/ref_chosen": -53.28052520751953, + "logps/ref_rejected": -84.2000503540039, + "logps/rejected": -388.2864990234375, + "loss": 1.0805, + "margin_dpo/margin_mean": 130.22715759277344, + "margin_dpo/margin_std": 166.29541015625, + "step": 576 + }, + { + "KL/chosen_KL_mean": -195.810791015625, + "KL/mean": -234.27926635742188, + "KL/rejected_KL_mean": -272.74774169921875, + "KL/std": 146.26927185058594, + "epoch": 0.872260015117158, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.93694305419922, + "fcm_dpo/q_t": 0.4415355920791626, + "grad_norm": 11.450098991394043, + "learning_rate": 2.4839802933393607e-08, + "logits/chosen": 0.9213416576385498, + "logits/rejected": 0.926072895526886, + "logps/chosen": -258.1354675292969, + "logps/ref_chosen": -62.32468795776367, + "logps/ref_rejected": -67.300537109375, + "logps/rejected": -340.04827880859375, + "loss": 1.2163, + "margin_dpo/margin_mean": 76.93695068359375, + "margin_dpo/margin_std": 156.5452117919922, + "step": 577 + }, + { + "KL/chosen_KL_mean": -191.14976501464844, + "KL/mean": -227.45904541015625, + "KL/rejected_KL_mean": -263.7683410644531, + "KL/std": 155.82366943359375, + "epoch": 0.873771730914588, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 72.61858367919922, + "fcm_dpo/q_t": 0.44642916321754456, + "grad_norm": 11.682079315185547, + "learning_rate": 2.4268365428344733e-08, + "logits/chosen": 0.9547611474990845, + "logits/rejected": 0.9533353447914124, + "logps/chosen": -247.80532836914062, + "logps/ref_chosen": -56.65557861328125, + "logps/ref_rejected": -68.21835327148438, + "logps/rejected": -331.9866943359375, + "loss": 1.2429, + "margin_dpo/margin_mean": 72.61858367919922, + "margin_dpo/margin_std": 178.0867919921875, + "step": 578 + }, + { + "KL/chosen_KL_mean": -193.22874450683594, + "KL/mean": -247.7565460205078, + "KL/rejected_KL_mean": -302.28436279296875, + "KL/std": 136.99658203125, + "epoch": 0.8752834467120182, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 109.05561828613281, + "fcm_dpo/q_t": 0.41695117950439453, + "grad_norm": 11.134893417358398, + "learning_rate": 2.3703242122359357e-08, + "logits/chosen": 0.8828812837600708, + "logits/rejected": 0.8775182962417603, + "logps/chosen": -250.03842163085938, + "logps/ref_chosen": -56.809661865234375, + "logps/ref_rejected": -68.09613037109375, + "logps/rejected": -370.3804931640625, + "loss": 1.1188, + "margin_dpo/margin_mean": 109.05561828613281, + "margin_dpo/margin_std": 147.0736083984375, + "step": 579 + }, + { + "KL/chosen_KL_mean": -197.42172241210938, + "KL/mean": -250.31234741210938, + "KL/rejected_KL_mean": -303.2030029296875, + "KL/std": 168.3189697265625, + "epoch": 0.8767951625094482, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 105.78125762939453, + "fcm_dpo/q_t": 0.42473307251930237, + "grad_norm": 12.491499900817871, + "learning_rate": 2.3144448823151392e-08, + "logits/chosen": 0.9132966995239258, + "logits/rejected": 0.8900126218795776, + "logps/chosen": -255.12184143066406, + "logps/ref_chosen": -57.70011520385742, + "logps/ref_rejected": -77.90664672851562, + "logps/rejected": -381.109619140625, + "loss": 1.1594, + "margin_dpo/margin_mean": 105.78126525878906, + "margin_dpo/margin_std": 187.40509033203125, + "step": 580 + }, + { + "KL/chosen_KL_mean": -212.98367309570312, + "KL/mean": -258.75372314453125, + "KL/rejected_KL_mean": -304.5237731933594, + "KL/std": 145.23110961914062, + "epoch": 0.8783068783068783, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.54010009765625, + "fcm_dpo/q_t": 0.4324272572994232, + "grad_norm": 9.804346084594727, + "learning_rate": 2.259200116137039e-08, + "logits/chosen": 0.9398747086524963, + "logits/rejected": 0.901383638381958, + "logps/chosen": -272.3160400390625, + "logps/ref_chosen": -59.332359313964844, + "logps/ref_rejected": -83.64482116699219, + "logps/rejected": -388.1685791015625, + "loss": 1.1886, + "margin_dpo/margin_mean": 91.54010009765625, + "margin_dpo/margin_std": 175.1453399658203, + "step": 581 + }, + { + "KL/chosen_KL_mean": -196.52120971679688, + "KL/mean": -236.0716094970703, + "KL/rejected_KL_mean": -275.62200927734375, + "KL/std": 129.60977172851562, + "epoch": 0.8798185941043084, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 79.10078430175781, + "fcm_dpo/q_t": 0.44115710258483887, + "grad_norm": 8.99724292755127, + "learning_rate": 2.204591459016525e-08, + "logits/chosen": 0.9279496073722839, + "logits/rejected": 0.9599611759185791, + "logps/chosen": -260.68408203125, + "logps/ref_chosen": -64.16285705566406, + "logps/ref_rejected": -58.632896423339844, + "logps/rejected": -334.2548828125, + "loss": 1.2082, + "margin_dpo/margin_mean": 79.10079193115234, + "margin_dpo/margin_std": 153.8829345703125, + "step": 582 + }, + { + "KL/chosen_KL_mean": -191.3851776123047, + "KL/mean": -253.97930908203125, + "KL/rejected_KL_mean": -316.57342529296875, + "KL/std": 171.81069946289062, + "epoch": 0.8813303099017384, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 125.1882553100586, + "fcm_dpo/q_t": 0.4104909300804138, + "grad_norm": 11.005936622619629, + "learning_rate": 2.1506204384751064e-08, + "logits/chosen": 1.0140917301177979, + "logits/rejected": 0.9544622898101807, + "logps/chosen": -243.257568359375, + "logps/ref_chosen": -51.87239456176758, + "logps/ref_rejected": -83.86331176757812, + "logps/rejected": -400.4367370605469, + "loss": 1.1127, + "margin_dpo/margin_mean": 125.18826293945312, + "margin_dpo/margin_std": 195.55584716796875, + "step": 583 + }, + { + "KL/chosen_KL_mean": -175.76718139648438, + "KL/mean": -224.52301025390625, + "KL/rejected_KL_mean": -273.27886962890625, + "KL/std": 139.64749145507812, + "epoch": 0.8828420256991686, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 97.51168823242188, + "fcm_dpo/q_t": 0.4288090467453003, + "grad_norm": 13.306455612182617, + "learning_rate": 2.09728856419826e-08, + "logits/chosen": 1.0285711288452148, + "logits/rejected": 0.957194983959198, + "logps/chosen": -222.33856201171875, + "logps/ref_chosen": -46.571388244628906, + "logps/ref_rejected": -80.67969512939453, + "logps/rejected": -353.95855712890625, + "loss": 1.1691, + "margin_dpo/margin_mean": 97.51168823242188, + "margin_dpo/margin_std": 173.30181884765625, + "step": 584 + }, + { + "KL/chosen_KL_mean": -211.24838256835938, + "KL/mean": -243.45950317382812, + "KL/rejected_KL_mean": -275.6706237792969, + "KL/std": 152.93052673339844, + "epoch": 0.8843537414965986, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 64.42221069335938, + "fcm_dpo/q_t": 0.45181816816329956, + "grad_norm": 11.961204528808594, + "learning_rate": 2.044597327993153e-08, + "logits/chosen": 0.9316689968109131, + "logits/rejected": 0.8908916711807251, + "logps/chosen": -269.3729248046875, + "logps/ref_chosen": -58.124534606933594, + "logps/ref_rejected": -79.00538635253906, + "logps/rejected": -354.676025390625, + "loss": 1.2475, + "margin_dpo/margin_mean": 64.42221069335938, + "margin_dpo/margin_std": 151.98226928710938, + "step": 585 + }, + { + "KL/chosen_KL_mean": -189.38601684570312, + "KL/mean": -230.9322052001953, + "KL/rejected_KL_mean": -272.4783935546875, + "KL/std": 120.71141052246094, + "epoch": 0.8858654572940288, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 83.09239196777344, + "fcm_dpo/q_t": 0.4359871745109558, + "grad_norm": 10.963727951049805, + "learning_rate": 1.9925482037469187e-08, + "logits/chosen": 0.9770946502685547, + "logits/rejected": 0.9540187120437622, + "logps/chosen": -243.48764038085938, + "logps/ref_chosen": -54.10163879394531, + "logps/ref_rejected": -63.72113037109375, + "logps/rejected": -336.19952392578125, + "loss": 1.1685, + "margin_dpo/margin_mean": 83.09239959716797, + "margin_dpo/margin_std": 113.74529266357422, + "step": 586 + }, + { + "KL/chosen_KL_mean": -208.8435821533203, + "KL/mean": -248.68414306640625, + "KL/rejected_KL_mean": -288.5246887207031, + "KL/std": 154.78927612304688, + "epoch": 0.8873771730914588, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 79.68111419677734, + "fcm_dpo/q_t": 0.43557557463645935, + "grad_norm": 14.696372985839844, + "learning_rate": 1.9411426473854687e-08, + "logits/chosen": 0.9748609066009521, + "logits/rejected": 0.9778647422790527, + "logps/chosen": -272.2607727050781, + "logps/ref_chosen": -63.41719436645508, + "logps/ref_rejected": -63.47003936767578, + "logps/rejected": -351.9947509765625, + "loss": 1.2461, + "margin_dpo/margin_mean": 79.68111419677734, + "margin_dpo/margin_std": 209.5115966796875, + "step": 587 + }, + { + "KL/chosen_KL_mean": -211.2041015625, + "KL/mean": -259.3454284667969, + "KL/rejected_KL_mean": -307.48675537109375, + "KL/std": 155.98251342773438, + "epoch": 0.8888888888888888, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 96.28263854980469, + "fcm_dpo/q_t": 0.42922869324684143, + "grad_norm": 11.775358200073242, + "learning_rate": 1.890382096832699e-08, + "logits/chosen": 0.9626029133796692, + "logits/rejected": 0.9362854361534119, + "logps/chosen": -273.4051513671875, + "logps/ref_chosen": -62.20103454589844, + "logps/ref_rejected": -82.10249328613281, + "logps/rejected": -389.5892333984375, + "loss": 1.1927, + "margin_dpo/margin_mean": 96.28263854980469, + "margin_dpo/margin_std": 196.510986328125, + "step": 588 + }, + { + "KL/chosen_KL_mean": -196.3404541015625, + "KL/mean": -242.31954956054688, + "KL/rejected_KL_mean": -288.29864501953125, + "KL/std": 142.04229736328125, + "epoch": 0.890400604686319, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.95817565917969, + "fcm_dpo/q_t": 0.4290810525417328, + "grad_norm": 8.143330574035645, + "learning_rate": 1.840267971970344e-08, + "logits/chosen": 0.9108644723892212, + "logits/rejected": 0.8894894123077393, + "logps/chosen": -253.0540771484375, + "logps/ref_chosen": -56.71361541748047, + "logps/ref_rejected": -76.7366943359375, + "logps/rejected": -365.03533935546875, + "loss": 1.1599, + "margin_dpo/margin_mean": 91.95817565917969, + "margin_dpo/margin_std": 138.236083984375, + "step": 589 + }, + { + "KL/chosen_KL_mean": -212.30551147460938, + "KL/mean": -253.93572998046875, + "KL/rejected_KL_mean": -295.56597900390625, + "KL/std": 127.766845703125, + "epoch": 0.891912320483749, + "fcm_dpo/beta": 0.003227104200050235, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 83.26048278808594, + "fcm_dpo/q_t": 0.43615347146987915, + "grad_norm": 13.782276153564453, + "learning_rate": 1.7908016745981856e-08, + "logits/chosen": 0.9059382677078247, + "logits/rejected": 0.8786763548851013, + "logps/chosen": -278.8193359375, + "logps/ref_chosen": -66.5138168334961, + "logps/ref_rejected": -85.70820617675781, + "logps/rejected": -381.274169921875, + "loss": 1.1835, + "margin_dpo/margin_mean": 83.26048278808594, + "margin_dpo/margin_std": 136.46484375, + "step": 590 + }, + { + "KL/chosen_KL_mean": -178.59005737304688, + "KL/mean": -238.56552124023438, + "KL/rejected_KL_mean": -298.5409851074219, + "KL/std": 159.26284790039062, + "epoch": 0.8934240362811792, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": -0.05250098556280136, + "fcm_dpo/margin": 119.950927734375, + "fcm_dpo/q_t": 0.41484156250953674, + "grad_norm": 10.59327220916748, + "learning_rate": 1.7419845883949098e-08, + "logits/chosen": 0.9709192514419556, + "logits/rejected": 0.9397531747817993, + "logps/chosen": -239.2872314453125, + "logps/ref_chosen": -60.697181701660156, + "logps/ref_rejected": -86.12278747558594, + "logps/rejected": -384.66375732421875, + "loss": 1.1338, + "margin_dpo/margin_mean": 119.950927734375, + "margin_dpo/margin_std": 200.14447021484375, + "step": 591 + }, + { + "KL/chosen_KL_mean": -193.6923065185547, + "KL/mean": -239.26446533203125, + "KL/rejected_KL_mean": -284.836669921875, + "KL/std": 129.58926391601562, + "epoch": 0.8949357520786092, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.14432525634766, + "fcm_dpo/q_t": 0.43314582109451294, + "grad_norm": 10.38012981414795, + "learning_rate": 1.6938180788793556e-08, + "logits/chosen": 0.9803141355514526, + "logits/rejected": 0.8997335433959961, + "logps/chosen": -244.92962646484375, + "logps/ref_chosen": -51.237327575683594, + "logps/ref_rejected": -81.60242462158203, + "logps/rejected": -366.4390869140625, + "loss": 1.1699, + "margin_dpo/margin_mean": 91.14432525634766, + "margin_dpo/margin_std": 148.00392150878906, + "step": 592 + }, + { + "KL/chosen_KL_mean": -159.02511596679688, + "KL/mean": -209.68447875976562, + "KL/rejected_KL_mean": -260.3438720703125, + "KL/std": 127.21440124511719, + "epoch": 0.8964474678760394, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 101.31876373291016, + "fcm_dpo/q_t": 0.42233753204345703, + "grad_norm": 11.570535659790039, + "learning_rate": 1.6463034933723336e-08, + "logits/chosen": 0.9546194076538086, + "logits/rejected": 0.8912557363510132, + "logps/chosen": -201.10511779785156, + "logps/ref_chosen": -42.08000183105469, + "logps/ref_rejected": -68.47499084472656, + "logps/rejected": -328.81884765625, + "loss": 1.1397, + "margin_dpo/margin_mean": 101.31876373291016, + "margin_dpo/margin_std": 143.3717041015625, + "step": 593 + }, + { + "KL/chosen_KL_mean": -208.95913696289062, + "KL/mean": -247.4608612060547, + "KL/rejected_KL_mean": -285.96258544921875, + "KL/std": 137.06649780273438, + "epoch": 0.8979591836734694, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 77.00344848632812, + "fcm_dpo/q_t": 0.4417816996574402, + "grad_norm": 11.07624340057373, + "learning_rate": 1.5994421609589385e-08, + "logits/chosen": 0.90055912733078, + "logits/rejected": 0.896392822265625, + "logps/chosen": -272.6177978515625, + "logps/ref_chosen": -63.658668518066406, + "logps/ref_rejected": -70.35597229003906, + "logps/rejected": -356.31854248046875, + "loss": 1.2068, + "margin_dpo/margin_mean": 77.00344848632812, + "margin_dpo/margin_std": 142.98709106445312, + "step": 594 + }, + { + "KL/chosen_KL_mean": -189.4154510498047, + "KL/mean": -244.88433837890625, + "KL/rejected_KL_mean": -300.35321044921875, + "KL/std": 143.3093719482422, + "epoch": 0.8994708994708994, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 110.93778991699219, + "fcm_dpo/q_t": 0.41879788041114807, + "grad_norm": 9.869068145751953, + "learning_rate": 1.553235392451377e-08, + "logits/chosen": 0.9638773202896118, + "logits/rejected": 0.9036699533462524, + "logps/chosen": -245.63421630859375, + "logps/ref_chosen": -56.21875762939453, + "logps/ref_rejected": -83.95773315429688, + "logps/rejected": -384.31097412109375, + "loss": 1.135, + "margin_dpo/margin_mean": 110.93778991699219, + "margin_dpo/margin_std": 173.9689178466797, + "step": 595 + }, + { + "KL/chosen_KL_mean": -220.83944702148438, + "KL/mean": -240.2639923095703, + "KL/rejected_KL_mean": -259.68853759765625, + "KL/std": 130.09234619140625, + "epoch": 0.9009826152683296, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 38.849117279052734, + "fcm_dpo/q_t": 0.47059816122055054, + "grad_norm": 12.449370384216309, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": 0.9170597195625305, + "logits/rejected": 0.9390828609466553, + "logps/chosen": -289.3203125, + "logps/ref_chosen": -68.48088073730469, + "logps/ref_rejected": -61.732967376708984, + "logps/rejected": -321.4215087890625, + "loss": 1.3083, + "margin_dpo/margin_mean": 38.849117279052734, + "margin_dpo/margin_std": 128.81515502929688, + "step": 596 + }, + { + "KL/chosen_KL_mean": -168.60281372070312, + "KL/mean": -208.48593139648438, + "KL/rejected_KL_mean": -248.3690185546875, + "KL/std": 111.42578125, + "epoch": 0.9024943310657596, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 79.7662353515625, + "fcm_dpo/q_t": 0.43913787603378296, + "grad_norm": 10.048505783081055, + "learning_rate": 1.4627906988186111e-08, + "logits/chosen": 0.9357984066009521, + "logits/rejected": 0.9276165962219238, + "logps/chosen": -217.4603271484375, + "logps/ref_chosen": -48.85750961303711, + "logps/ref_rejected": -55.068084716796875, + "logps/rejected": -303.4371337890625, + "loss": 1.1861, + "margin_dpo/margin_mean": 79.76622772216797, + "margin_dpo/margin_std": 123.33467102050781, + "step": 597 + }, + { + "KL/chosen_KL_mean": -221.0791015625, + "KL/mean": -251.432373046875, + "KL/rejected_KL_mean": -281.78564453125, + "KL/std": 139.46469116210938, + "epoch": 0.9040060468631897, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 60.70652389526367, + "fcm_dpo/q_t": 0.4551101326942444, + "grad_norm": 9.732013702392578, + "learning_rate": 1.4185553036259095e-08, + "logits/chosen": 0.9217236042022705, + "logits/rejected": 0.8709498047828674, + "logps/chosen": -279.96624755859375, + "logps/ref_chosen": -58.88715362548828, + "logps/ref_rejected": -81.43145751953125, + "logps/rejected": -363.21710205078125, + "loss": 1.2562, + "margin_dpo/margin_mean": 60.70652770996094, + "margin_dpo/margin_std": 148.96145629882812, + "step": 598 + }, + { + "KL/chosen_KL_mean": -220.0737762451172, + "KL/mean": -253.25234985351562, + "KL/rejected_KL_mean": -286.430908203125, + "KL/std": 134.68807983398438, + "epoch": 0.9055177626606198, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.35713958740234, + "fcm_dpo/q_t": 0.45046699047088623, + "grad_norm": 11.252570152282715, + "learning_rate": 1.3749795321332885e-08, + "logits/chosen": 0.982478141784668, + "logits/rejected": 0.9580433964729309, + "logps/chosen": -277.68096923828125, + "logps/ref_chosen": -57.60719299316406, + "logps/ref_rejected": -71.80469512939453, + "logps/rejected": -358.235595703125, + "loss": 1.247, + "margin_dpo/margin_mean": 66.35713958740234, + "margin_dpo/margin_std": 157.38815307617188, + "step": 599 + }, + { + "KL/chosen_KL_mean": -198.6517333984375, + "KL/mean": -238.66195678710938, + "KL/rejected_KL_mean": -278.67218017578125, + "KL/std": 147.49740600585938, + "epoch": 0.9070294784580499, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.02043914794922, + "fcm_dpo/q_t": 0.44202420115470886, + "grad_norm": 12.139007568359375, + "learning_rate": 1.3320646032487393e-08, + "logits/chosen": 0.93305504322052, + "logits/rejected": 0.8942174911499023, + "logps/chosen": -257.09405517578125, + "logps/ref_chosen": -58.44231414794922, + "logps/ref_rejected": -83.64639282226562, + "logps/rejected": -362.3185729980469, + "loss": 1.2126, + "margin_dpo/margin_mean": 80.02043151855469, + "margin_dpo/margin_std": 163.208251953125, + "step": 600 + }, + { + "KL/chosen_KL_mean": -181.49945068359375, + "KL/mean": -234.6309051513672, + "KL/rejected_KL_mean": -287.76239013671875, + "KL/std": 156.81146240234375, + "epoch": 0.90854119425548, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 106.2629165649414, + "fcm_dpo/q_t": 0.4221433997154236, + "grad_norm": 8.865299224853516, + "learning_rate": 1.2898117173950868e-08, + "logits/chosen": 0.9232147336006165, + "logits/rejected": 0.8683621883392334, + "logps/chosen": -237.09378051757812, + "logps/ref_chosen": -55.59432601928711, + "logps/ref_rejected": -83.68630981445312, + "logps/rejected": -371.44866943359375, + "loss": 1.1508, + "margin_dpo/margin_mean": 106.2629165649414, + "margin_dpo/margin_std": 176.8553009033203, + "step": 601 + }, + { + "KL/chosen_KL_mean": -172.61276245117188, + "KL/mean": -217.75296020507812, + "KL/rejected_KL_mean": -262.89312744140625, + "KL/std": 132.79425048828125, + "epoch": 0.91005291005291, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 90.28034973144531, + "fcm_dpo/q_t": 0.43157410621643066, + "grad_norm": 10.930935859680176, + "learning_rate": 1.2482220564763667e-08, + "logits/chosen": 0.9340589046478271, + "logits/rejected": 0.9145586490631104, + "logps/chosen": -228.9619598388672, + "logps/ref_chosen": -56.349185943603516, + "logps/ref_rejected": -71.9959716796875, + "logps/rejected": -334.88909912109375, + "loss": 1.1684, + "margin_dpo/margin_mean": 90.28034973144531, + "margin_dpo/margin_std": 141.6522216796875, + "step": 602 + }, + { + "KL/chosen_KL_mean": -186.05422973632812, + "KL/mean": -235.9767303466797, + "KL/rejected_KL_mean": -285.89923095703125, + "KL/std": 139.99981689453125, + "epoch": 0.9115646258503401, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 99.84503173828125, + "fcm_dpo/q_t": 0.427456796169281, + "grad_norm": 9.996122360229492, + "learning_rate": 1.2072967838448051e-08, + "logits/chosen": 0.9382352828979492, + "logits/rejected": 0.9089177846908569, + "logps/chosen": -239.2226104736328, + "logps/ref_chosen": -53.16838836669922, + "logps/ref_rejected": -73.8604736328125, + "logps/rejected": -359.7597351074219, + "loss": 1.1586, + "margin_dpo/margin_mean": 99.84503173828125, + "margin_dpo/margin_std": 166.58010864257812, + "step": 603 + }, + { + "KL/chosen_KL_mean": -187.37911987304688, + "KL/mean": -224.50912475585938, + "KL/rejected_KL_mean": -261.63909912109375, + "KL/std": 124.96034240722656, + "epoch": 0.9130763416477702, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 74.25997161865234, + "fcm_dpo/q_t": 0.4446590542793274, + "grad_norm": 10.817628860473633, + "learning_rate": 1.1670370442682459e-08, + "logits/chosen": 0.8739547729492188, + "logits/rejected": 0.8863567113876343, + "logps/chosen": -260.028564453125, + "logps/ref_chosen": -72.64942169189453, + "logps/ref_rejected": -69.8792724609375, + "logps/rejected": -331.51837158203125, + "loss": 1.2305, + "margin_dpo/margin_mean": 74.25996398925781, + "margin_dpo/margin_std": 166.822021484375, + "step": 604 + }, + { + "KL/chosen_KL_mean": -206.32119750976562, + "KL/mean": -251.2423553466797, + "KL/rejected_KL_mean": -296.16351318359375, + "KL/std": 132.05099487304688, + "epoch": 0.9145880574452003, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.84228515625, + "fcm_dpo/q_t": 0.43243077397346497, + "grad_norm": 10.091094970703125, + "learning_rate": 1.1274439638981532e-08, + "logits/chosen": 0.9632163643836975, + "logits/rejected": 0.9292812347412109, + "logps/chosen": -267.9340515136719, + "logps/ref_chosen": -61.61284637451172, + "logps/ref_rejected": -79.34398651123047, + "logps/rejected": -375.50750732421875, + "loss": 1.1792, + "margin_dpo/margin_mean": 89.84228515625, + "margin_dpo/margin_std": 154.51718139648438, + "step": 605 + }, + { + "KL/chosen_KL_mean": -179.38706970214844, + "KL/mean": -236.68408203125, + "KL/rejected_KL_mean": -293.9811096191406, + "KL/std": 143.79409790039062, + "epoch": 0.9160997732426304, + "fcm_dpo/beta": 0.003193396143615246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 114.59405517578125, + "fcm_dpo/q_t": 0.41510650515556335, + "grad_norm": 10.840108871459961, + "learning_rate": 1.0885186502381016e-08, + "logits/chosen": 0.9171103239059448, + "logits/rejected": 0.8769623041152954, + "logps/chosen": -233.85130310058594, + "logps/ref_chosen": -54.46424102783203, + "logps/ref_rejected": -79.62708282470703, + "logps/rejected": -373.6081848144531, + "loss": 1.1131, + "margin_dpo/margin_mean": 114.59405517578125, + "margin_dpo/margin_std": 156.9613494873047, + "step": 606 + }, + { + "KL/chosen_KL_mean": -200.49571228027344, + "KL/mean": -248.2861328125, + "KL/rejected_KL_mean": -296.0765380859375, + "KL/std": 139.8798370361328, + "epoch": 0.9176114890400605, + "fcm_dpo/beta": 0.003162774723023176, + "fcm_dpo/delta": -0.04817618429660797, + "fcm_dpo/margin": 95.58084106445312, + "fcm_dpo/q_t": 0.4292658865451813, + "grad_norm": 10.28044605255127, + "learning_rate": 1.0502621921127774e-08, + "logits/chosen": 0.9089508056640625, + "logits/rejected": 0.9013663530349731, + "logps/chosen": -263.3565673828125, + "logps/ref_chosen": -62.86086654663086, + "logps/ref_rejected": -72.5501937866211, + "logps/rejected": -368.6267395019531, + "loss": 1.1738, + "margin_dpo/margin_mean": 95.58084106445312, + "margin_dpo/margin_std": 158.3627471923828, + "step": 607 + }, + { + "KL/chosen_KL_mean": -206.0166473388672, + "KL/mean": -258.2984313964844, + "KL/rejected_KL_mean": -310.5802001953125, + "KL/std": 155.61599731445312, + "epoch": 0.9191232048374905, + "fcm_dpo/beta": 0.003162774723023176, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.56355285644531, + "fcm_dpo/q_t": 0.4240010380744934, + "grad_norm": 10.039460182189941, + "learning_rate": 1.0126756596375685e-08, + "logits/chosen": 0.9120040535926819, + "logits/rejected": 0.8582882881164551, + "logps/chosen": -269.1973876953125, + "logps/ref_chosen": -63.18071746826172, + "logps/ref_rejected": -99.15888214111328, + "logps/rejected": -409.73907470703125, + "loss": 1.1448, + "margin_dpo/margin_mean": 104.56355285644531, + "margin_dpo/margin_std": 163.28131103515625, + "step": 608 + }, + { + "KL/chosen_KL_mean": -180.6420440673828, + "KL/mean": -230.55471801757812, + "KL/rejected_KL_mean": -280.4674072265625, + "KL/std": 127.04386901855469, + "epoch": 0.9206349206349206, + "fcm_dpo/beta": 0.003162774723023176, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 99.82534790039062, + "fcm_dpo/q_t": 0.42423099279403687, + "grad_norm": 10.703655242919922, + "learning_rate": 9.757601041885694e-09, + "logits/chosen": 0.9837725162506104, + "logits/rejected": 0.9593820571899414, + "logps/chosen": -229.2652587890625, + "logps/ref_chosen": -48.62322235107422, + "logps/ref_rejected": -68.28271484375, + "logps/rejected": -348.7501220703125, + "loss": 1.1328, + "margin_dpo/margin_mean": 99.82534790039062, + "margin_dpo/margin_std": 120.62831115722656, + "step": 609 + }, + { + "KL/chosen_KL_mean": -210.98101806640625, + "KL/mean": -253.97109985351562, + "KL/rejected_KL_mean": -296.961181640625, + "KL/std": 153.73622131347656, + "epoch": 0.9221466364323507, + "fcm_dpo/beta": 0.003162774723023176, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 85.98018646240234, + "fcm_dpo/q_t": 0.434524804353714, + "grad_norm": 13.81159782409668, + "learning_rate": 9.395165583732379e-09, + "logits/chosen": 0.9371830224990845, + "logits/rejected": 0.937787652015686, + "logps/chosen": -283.6461181640625, + "logps/ref_chosen": -72.66513061523438, + "logps/ref_rejected": -87.15310668945312, + "logps/rejected": -384.1142883300781, + "loss": 1.2179, + "margin_dpo/margin_mean": 85.98019409179688, + "margin_dpo/margin_std": 185.64382934570312, + "step": 610 + }, + { + "KL/chosen_KL_mean": -180.6585693359375, + "KL/mean": -221.9508056640625, + "KL/rejected_KL_mean": -263.2430419921875, + "KL/std": 132.38990783691406, + "epoch": 0.9236583522297808, + "fcm_dpo/beta": 0.003162774723023176, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 82.58445739746094, + "fcm_dpo/q_t": 0.4378008246421814, + "grad_norm": 10.246078491210938, + "learning_rate": 9.03946036001449e-09, + "logits/chosen": 0.9471895098686218, + "logits/rejected": 0.917996883392334, + "logps/chosen": -228.9671630859375, + "logps/ref_chosen": -48.30857849121094, + "logps/ref_rejected": -70.6141128540039, + "logps/rejected": -333.857177734375, + "loss": 1.1824, + "margin_dpo/margin_mean": 82.58445739746094, + "margin_dpo/margin_std": 128.2288055419922, + "step": 611 + }, + { + "KL/chosen_KL_mean": -200.29458618164062, + "KL/mean": -261.31329345703125, + "KL/rejected_KL_mean": -322.33203125, + "KL/std": 136.66183471679688, + "epoch": 0.9251700680272109, + "fcm_dpo/beta": 0.003162774723023176, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 122.03746032714844, + "fcm_dpo/q_t": 0.4092690050601959, + "grad_norm": 10.354504585266113, + "learning_rate": 8.690495320571839e-09, + "logits/chosen": 0.8809033632278442, + "logits/rejected": 0.8318875432014465, + "logps/chosen": -261.526123046875, + "logps/ref_chosen": -61.23155975341797, + "logps/ref_rejected": -94.37979888916016, + "logps/rejected": -416.71185302734375, + "loss": 1.0896, + "margin_dpo/margin_mean": 122.03746795654297, + "margin_dpo/margin_std": 148.84161376953125, + "step": 612 + }, + { + "KL/chosen_KL_mean": -179.49685668945312, + "KL/mean": -227.58367919921875, + "KL/rejected_KL_mean": -275.6705322265625, + "KL/std": 156.89382934570312, + "epoch": 0.926681783824641, + "fcm_dpo/beta": 0.003162774723023176, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 96.17366027832031, + "fcm_dpo/q_t": 0.42703235149383545, + "grad_norm": 9.231825828552246, + "learning_rate": 8.348280226706722e-09, + "logits/chosen": 0.9118015766143799, + "logits/rejected": 0.9143195152282715, + "logps/chosen": -233.47996520996094, + "logps/ref_chosen": -53.98310852050781, + "logps/ref_rejected": -58.32208251953125, + "logps/rejected": -333.99261474609375, + "loss": 1.1612, + "margin_dpo/margin_mean": 96.17366027832031, + "margin_dpo/margin_std": 152.2427978515625, + "step": 613 + }, + { + "KL/chosen_KL_mean": -198.9600067138672, + "KL/mean": -245.7151641845703, + "KL/rejected_KL_mean": -292.47027587890625, + "KL/std": 128.48739624023438, + "epoch": 0.9281934996220711, + "fcm_dpo/beta": 0.003162774723023176, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 93.5102767944336, + "fcm_dpo/q_t": 0.4299646317958832, + "grad_norm": 10.526408195495605, + "learning_rate": 8.012824650910937e-09, + "logits/chosen": 0.9376695156097412, + "logits/rejected": 0.9362810850143433, + "logps/chosen": -259.20306396484375, + "logps/ref_chosen": -60.24303436279297, + "logps/ref_rejected": -72.26258850097656, + "logps/rejected": -364.7328796386719, + "loss": 1.169, + "margin_dpo/margin_mean": 93.5102767944336, + "margin_dpo/margin_std": 148.775634765625, + "step": 614 + }, + { + "KL/chosen_KL_mean": -211.63214111328125, + "KL/mean": -268.7047119140625, + "KL/rejected_KL_mean": -325.77728271484375, + "KL/std": 174.5651092529297, + "epoch": 0.9297052154195011, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": -0.05142123997211456, + "fcm_dpo/margin": 114.14509582519531, + "fcm_dpo/q_t": 0.41916823387145996, + "grad_norm": 9.828910827636719, + "learning_rate": 7.684137976598088e-09, + "logits/chosen": 0.9190930128097534, + "logits/rejected": 0.8757984042167664, + "logps/chosen": -283.726806640625, + "logps/ref_chosen": -72.09467315673828, + "logps/ref_rejected": -104.02980041503906, + "logps/rejected": -429.80706787109375, + "loss": 1.1426, + "margin_dpo/margin_mean": 114.14510345458984, + "margin_dpo/margin_std": 186.59616088867188, + "step": 615 + }, + { + "KL/chosen_KL_mean": -197.91896057128906, + "KL/mean": -242.6910400390625, + "KL/rejected_KL_mean": -287.4631042480469, + "KL/std": 172.01504516601562, + "epoch": 0.9312169312169312, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.54414367675781, + "fcm_dpo/q_t": 0.4340965747833252, + "grad_norm": 9.829987525939941, + "learning_rate": 7.36222939784098e-09, + "logits/chosen": 0.9627779722213745, + "logits/rejected": 0.9220662713050842, + "logps/chosen": -256.4496765136719, + "logps/ref_chosen": -58.530723571777344, + "logps/ref_rejected": -75.48025512695312, + "logps/rejected": -362.943359375, + "loss": 1.1764, + "margin_dpo/margin_mean": 89.54414367675781, + "margin_dpo/margin_std": 147.76666259765625, + "step": 616 + }, + { + "KL/chosen_KL_mean": -207.77853393554688, + "KL/mean": -246.21920776367188, + "KL/rejected_KL_mean": -284.65985107421875, + "KL/std": 143.22564697265625, + "epoch": 0.9327286470143613, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.88133239746094, + "fcm_dpo/q_t": 0.4445750117301941, + "grad_norm": 13.23380184173584, + "learning_rate": 7.047107919114586e-09, + "logits/chosen": 0.9405636787414551, + "logits/rejected": 0.907119631767273, + "logps/chosen": -265.38720703125, + "logps/ref_chosen": -57.608673095703125, + "logps/ref_rejected": -81.22109985351562, + "logps/rejected": -365.8809814453125, + "loss": 1.2199, + "margin_dpo/margin_mean": 76.88133239746094, + "margin_dpo/margin_std": 153.86878967285156, + "step": 617 + }, + { + "KL/chosen_KL_mean": -194.5389862060547, + "KL/mean": -245.77511596679688, + "KL/rejected_KL_mean": -297.0112609863281, + "KL/std": 160.71217346191406, + "epoch": 0.9342403628117913, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 102.47227478027344, + "fcm_dpo/q_t": 0.4275450110435486, + "grad_norm": 13.20183277130127, + "learning_rate": 6.738782355044048e-09, + "logits/chosen": 0.9345172643661499, + "logits/rejected": 0.8785445690155029, + "logps/chosen": -251.23492431640625, + "logps/ref_chosen": -56.69594192504883, + "logps/ref_rejected": -85.92362976074219, + "logps/rejected": -382.93487548828125, + "loss": 1.1553, + "margin_dpo/margin_mean": 102.47227478027344, + "margin_dpo/margin_std": 170.03724670410156, + "step": 618 + }, + { + "KL/chosen_KL_mean": -184.23806762695312, + "KL/mean": -236.51092529296875, + "KL/rejected_KL_mean": -288.78375244140625, + "KL/std": 148.20510864257812, + "epoch": 0.9357520786092215, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.54570007324219, + "fcm_dpo/q_t": 0.42483896017074585, + "grad_norm": 9.666030883789062, + "learning_rate": 6.437261330158206e-09, + "logits/chosen": 0.9987913966178894, + "logits/rejected": 0.9459144473075867, + "logps/chosen": -238.29649353027344, + "logps/ref_chosen": -54.05841827392578, + "logps/ref_rejected": -83.55493927001953, + "logps/rejected": -372.3387145996094, + "loss": 1.15, + "margin_dpo/margin_mean": 104.54570007324219, + "margin_dpo/margin_std": 168.18020629882812, + "step": 619 + }, + { + "KL/chosen_KL_mean": -193.63037109375, + "KL/mean": -230.4679718017578, + "KL/rejected_KL_mean": -267.3055725097656, + "KL/std": 135.5782470703125, + "epoch": 0.9372637944066515, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 73.67523193359375, + "fcm_dpo/q_t": 0.44435736536979675, + "grad_norm": 10.929662704467773, + "learning_rate": 6.142553278648238e-09, + "logits/chosen": 0.9473289251327515, + "logits/rejected": 0.9542825818061829, + "logps/chosen": -257.0000915527344, + "logps/ref_chosen": -63.36971664428711, + "logps/ref_rejected": -65.68269348144531, + "logps/rejected": -332.98828125, + "loss": 1.2239, + "margin_dpo/margin_mean": 73.67523193359375, + "margin_dpo/margin_std": 142.8062744140625, + "step": 620 + }, + { + "KL/chosen_KL_mean": -185.4219970703125, + "KL/mean": -238.8184356689453, + "KL/rejected_KL_mean": -292.2148742675781, + "KL/std": 160.7276611328125, + "epoch": 0.9387755102040817, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 106.79286193847656, + "fcm_dpo/q_t": 0.4272248148918152, + "grad_norm": 9.562493324279785, + "learning_rate": 5.854666444131934e-09, + "logits/chosen": 0.9666614532470703, + "logits/rejected": 0.8935528993606567, + "logps/chosen": -237.74322509765625, + "logps/ref_chosen": -52.321224212646484, + "logps/ref_rejected": -88.09001159667969, + "logps/rejected": -380.30487060546875, + "loss": 1.1668, + "margin_dpo/margin_mean": 106.79286193847656, + "margin_dpo/margin_std": 194.57687377929688, + "step": 621 + }, + { + "KL/chosen_KL_mean": -205.28720092773438, + "KL/mean": -252.37612915039062, + "KL/rejected_KL_mean": -299.46502685546875, + "KL/std": 157.73703002929688, + "epoch": 0.9402872260015117, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.17784118652344, + "fcm_dpo/q_t": 0.4326885938644409, + "grad_norm": 10.958165168762207, + "learning_rate": 5.573608879422875e-09, + "logits/chosen": 0.9199116230010986, + "logits/rejected": 0.8872038722038269, + "logps/chosen": -265.15264892578125, + "logps/ref_chosen": -59.86545944213867, + "logps/ref_rejected": -81.86668395996094, + "logps/rejected": -381.33172607421875, + "loss": 1.1775, + "margin_dpo/margin_mean": 94.17784118652344, + "margin_dpo/margin_std": 165.80418395996094, + "step": 622 + }, + { + "KL/chosen_KL_mean": -222.81295776367188, + "KL/mean": -274.02532958984375, + "KL/rejected_KL_mean": -325.23773193359375, + "KL/std": 174.5893096923828, + "epoch": 0.9417989417989417, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 102.42472076416016, + "fcm_dpo/q_t": 0.4269852638244629, + "grad_norm": 11.868815422058105, + "learning_rate": 5.299388446305342e-09, + "logits/chosen": 0.9369495511054993, + "logits/rejected": 0.9090813994407654, + "logps/chosen": -290.1814270019531, + "logps/ref_chosen": -67.36846160888672, + "logps/ref_rejected": -82.02733612060547, + "logps/rejected": -407.2650451660156, + "loss": 1.1739, + "margin_dpo/margin_mean": 102.42472076416016, + "margin_dpo/margin_std": 189.164794921875, + "step": 623 + }, + { + "KL/chosen_KL_mean": -185.4460906982422, + "KL/mean": -235.8998260498047, + "KL/rejected_KL_mean": -286.35357666015625, + "KL/std": 150.03134155273438, + "epoch": 0.9433106575963719, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 100.90745544433594, + "fcm_dpo/q_t": 0.42712944746017456, + "grad_norm": 11.591996192932129, + "learning_rate": 5.03201281531429e-09, + "logits/chosen": 0.9607518911361694, + "logits/rejected": 0.8958628177642822, + "logps/chosen": -236.47264099121094, + "logps/ref_chosen": -51.02655029296875, + "logps/ref_rejected": -76.49203491210938, + "logps/rejected": -362.8455810546875, + "loss": 1.1536, + "margin_dpo/margin_mean": 100.90745544433594, + "margin_dpo/margin_std": 158.26055908203125, + "step": 624 + }, + { + "KL/chosen_KL_mean": -186.66845703125, + "KL/mean": -226.89675903320312, + "KL/rejected_KL_mean": -267.12506103515625, + "KL/std": 141.98199462890625, + "epoch": 0.9448223733938019, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.45657348632812, + "fcm_dpo/q_t": 0.4423113167285919, + "grad_norm": 10.402351379394531, + "learning_rate": 4.7714894655209174e-09, + "logits/chosen": 0.9809439778327942, + "logits/rejected": 0.9170085191726685, + "logps/chosen": -240.87608337402344, + "logps/ref_chosen": -54.20761489868164, + "logps/ref_rejected": -84.93669128417969, + "logps/rejected": -352.061767578125, + "loss": 1.2162, + "margin_dpo/margin_mean": 80.45657348632812, + "margin_dpo/margin_std": 167.8040771484375, + "step": 625 + }, + { + "KL/chosen_KL_mean": -173.0820770263672, + "KL/mean": -229.86431884765625, + "KL/rejected_KL_mean": -286.6465759277344, + "KL/std": 159.24993896484375, + "epoch": 0.9463340891912321, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 113.56451416015625, + "fcm_dpo/q_t": 0.42139124870300293, + "grad_norm": 10.874939918518066, + "learning_rate": 4.517825684323323e-09, + "logits/chosen": 1.0064220428466797, + "logits/rejected": 0.9120746850967407, + "logps/chosen": -218.14410400390625, + "logps/ref_chosen": -45.06201934814453, + "logps/ref_rejected": -89.66368103027344, + "logps/rejected": -376.31024169921875, + "loss": 1.1387, + "margin_dpo/margin_mean": 113.56451416015625, + "margin_dpo/margin_std": 190.21609497070312, + "step": 626 + }, + { + "KL/chosen_KL_mean": -203.1885986328125, + "KL/mean": -264.7366943359375, + "KL/rejected_KL_mean": -326.2847900390625, + "KL/std": 168.15475463867188, + "epoch": 0.9478458049886621, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 123.09614562988281, + "fcm_dpo/q_t": 0.41006141901016235, + "grad_norm": 11.144246101379395, + "learning_rate": 4.271028567242818e-09, + "logits/chosen": 0.8766025304794312, + "logits/rejected": 0.7959402799606323, + "logps/chosen": -261.97967529296875, + "logps/ref_chosen": -58.791053771972656, + "logps/ref_rejected": -94.90802001953125, + "logps/rejected": -421.19281005859375, + "loss": 1.0933, + "margin_dpo/margin_mean": 123.09615325927734, + "margin_dpo/margin_std": 155.38308715820312, + "step": 627 + }, + { + "KL/chosen_KL_mean": -191.96481323242188, + "KL/mean": -244.04881286621094, + "KL/rejected_KL_mean": -296.1328125, + "KL/std": 150.7305908203125, + "epoch": 0.9493575207860923, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.16798400878906, + "fcm_dpo/q_t": 0.4224216043949127, + "grad_norm": 10.786967277526855, + "learning_rate": 4.0311050177251895e-09, + "logits/chosen": 0.9671989679336548, + "logits/rejected": 0.9418696761131287, + "logps/chosen": -244.7683868408203, + "logps/ref_chosen": -52.80357360839844, + "logps/ref_rejected": -76.49468994140625, + "logps/rejected": -372.62750244140625, + "loss": 1.1657, + "margin_dpo/margin_mean": 104.16798400878906, + "margin_dpo/margin_std": 174.9075164794922, + "step": 628 + }, + { + "KL/chosen_KL_mean": -203.937744140625, + "KL/mean": -241.90890502929688, + "KL/rejected_KL_mean": -279.88006591796875, + "KL/std": 139.17115783691406, + "epoch": 0.9508692365835223, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 75.94229125976562, + "fcm_dpo/q_t": 0.4455203413963318, + "grad_norm": 10.165343284606934, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": 0.9274606704711914, + "logits/rejected": 0.9357987642288208, + "logps/chosen": -274.6552429199219, + "logps/ref_chosen": -70.71749877929688, + "logps/ref_rejected": -78.96273803710938, + "logps/rejected": -358.8428039550781, + "loss": 1.2155, + "margin_dpo/margin_mean": 75.94229125976562, + "margin_dpo/margin_std": 150.8992919921875, + "step": 629 + }, + { + "KL/chosen_KL_mean": -183.7464599609375, + "KL/mean": -239.07704162597656, + "KL/rejected_KL_mean": -294.4075927734375, + "KL/std": 145.11636352539062, + "epoch": 0.9523809523809523, + "fcm_dpo/beta": 0.003130414756014943, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 110.66114807128906, + "fcm_dpo/q_t": 0.41840821504592896, + "grad_norm": 10.500893592834473, + "learning_rate": 3.5719052736323806e-09, + "logits/chosen": 0.9014256000518799, + "logits/rejected": 0.8830850124359131, + "logps/chosen": -239.9478759765625, + "logps/ref_chosen": -56.201412200927734, + "logps/ref_rejected": -74.69807434082031, + "logps/rejected": -369.1056823730469, + "loss": 1.1297, + "margin_dpo/margin_mean": 110.66114807128906, + "margin_dpo/margin_std": 160.73321533203125, + "step": 630 + }, + { + "KL/chosen_KL_mean": -180.64476013183594, + "KL/mean": -246.01486206054688, + "KL/rejected_KL_mean": -311.38494873046875, + "KL/std": 162.5005340576172, + "epoch": 0.9538926681783825, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": -0.05842360481619835, + "fcm_dpo/margin": 130.7401885986328, + "fcm_dpo/q_t": 0.40720421075820923, + "grad_norm": 9.99383544921875, + "learning_rate": 3.352641923861144e-09, + "logits/chosen": 1.0124080181121826, + "logits/rejected": 0.9354528784751892, + "logps/chosen": -239.46536254882812, + "logps/ref_chosen": -58.82059860229492, + "logps/ref_rejected": -96.51437377929688, + "logps/rejected": -407.8993225097656, + "loss": 1.0968, + "margin_dpo/margin_mean": 130.7401885986328, + "margin_dpo/margin_std": 179.59544372558594, + "step": 631 + }, + { + "KL/chosen_KL_mean": -180.45394897460938, + "KL/mean": -236.18548583984375, + "KL/rejected_KL_mean": -291.9169921875, + "KL/std": 135.77951049804688, + "epoch": 0.9554043839758125, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 111.46307373046875, + "fcm_dpo/q_t": 0.41834402084350586, + "grad_norm": 10.386039733886719, + "learning_rate": 3.140277830901428e-09, + "logits/chosen": 0.9802180528640747, + "logits/rejected": 0.9753730297088623, + "logps/chosen": -239.239990234375, + "logps/ref_chosen": -58.786048889160156, + "logps/ref_rejected": -67.21923828125, + "logps/rejected": -359.1362609863281, + "loss": 1.1238, + "margin_dpo/margin_mean": 111.46307373046875, + "margin_dpo/margin_std": 151.68695068359375, + "step": 632 + }, + { + "KL/chosen_KL_mean": -175.48123168945312, + "KL/mean": -222.42837524414062, + "KL/rejected_KL_mean": -269.3755187988281, + "KL/std": 135.81710815429688, + "epoch": 0.9569160997732427, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 93.894287109375, + "fcm_dpo/q_t": 0.43249619007110596, + "grad_norm": 12.328335762023926, + "learning_rate": 2.9348189350335007e-09, + "logits/chosen": 0.9596331119537354, + "logits/rejected": 0.9312918782234192, + "logps/chosen": -227.61141967773438, + "logps/ref_chosen": -52.13019561767578, + "logps/ref_rejected": -67.23016357421875, + "logps/rejected": -336.6056823730469, + "loss": 1.1702, + "margin_dpo/margin_mean": 93.894287109375, + "margin_dpo/margin_std": 150.7270050048828, + "step": 633 + }, + { + "KL/chosen_KL_mean": -211.42013549804688, + "KL/mean": -227.02139282226562, + "KL/rejected_KL_mean": -242.6226806640625, + "KL/std": 144.92550659179688, + "epoch": 0.9584278155706727, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 31.20253562927246, + "fcm_dpo/q_t": 0.47805124521255493, + "grad_norm": 11.825292587280273, + "learning_rate": 2.736270983384276e-09, + "logits/chosen": 0.9753604531288147, + "logits/rejected": 0.9930594563484192, + "logps/chosen": -272.3999328613281, + "logps/ref_chosen": -60.97979736328125, + "logps/ref_rejected": -58.50825119018555, + "logps/rejected": -301.13092041015625, + "loss": 1.3544, + "margin_dpo/margin_mean": 31.20253562927246, + "margin_dpo/margin_std": 162.56167602539062, + "step": 634 + }, + { + "KL/chosen_KL_mean": -230.13027954101562, + "KL/mean": -270.3157958984375, + "KL/rejected_KL_mean": -310.5013122558594, + "KL/std": 133.14523315429688, + "epoch": 0.9599395313681028, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.37104797363281, + "fcm_dpo/q_t": 0.44142985343933105, + "grad_norm": 10.514294624328613, + "learning_rate": 2.5446395297668287e-09, + "logits/chosen": 0.8804504871368408, + "logits/rejected": 0.8512506484985352, + "logps/chosen": -296.103271484375, + "logps/ref_chosen": -65.9730224609375, + "logps/ref_rejected": -85.61317443847656, + "logps/rejected": -396.114501953125, + "loss": 1.2082, + "margin_dpo/margin_mean": 80.37104797363281, + "margin_dpo/margin_std": 152.8987274169922, + "step": 635 + }, + { + "KL/chosen_KL_mean": -175.61082458496094, + "KL/mean": -228.16116333007812, + "KL/rejected_KL_mean": -280.71148681640625, + "KL/std": 136.18251037597656, + "epoch": 0.9614512471655329, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 105.10067749023438, + "fcm_dpo/q_t": 0.4229128658771515, + "grad_norm": 8.548711776733398, + "learning_rate": 2.359929934524829e-09, + "logits/chosen": 0.9548631310462952, + "logits/rejected": 0.8846620321273804, + "logps/chosen": -224.75099182128906, + "logps/ref_chosen": -49.140167236328125, + "logps/ref_rejected": -81.26971435546875, + "logps/rejected": -361.981201171875, + "loss": 1.134, + "margin_dpo/margin_mean": 105.10067749023438, + "margin_dpo/margin_std": 142.31265258789062, + "step": 636 + }, + { + "KL/chosen_KL_mean": -241.5238494873047, + "KL/mean": -282.18572998046875, + "KL/rejected_KL_mean": -322.84759521484375, + "KL/std": 149.25390625, + "epoch": 0.9629629629629629, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 81.32373809814453, + "fcm_dpo/q_t": 0.4420722424983978, + "grad_norm": 9.542378425598145, + "learning_rate": 2.1821473643827137e-09, + "logits/chosen": 0.9008264541625977, + "logits/rejected": 0.8717888593673706, + "logps/chosen": -315.2204284667969, + "logps/ref_chosen": -73.69658660888672, + "logps/ref_rejected": -83.01487731933594, + "logps/rejected": -405.86248779296875, + "loss": 1.2203, + "margin_dpo/margin_mean": 81.32373809814453, + "margin_dpo/margin_std": 176.375732421875, + "step": 637 + }, + { + "KL/chosen_KL_mean": -208.33538818359375, + "KL/mean": -256.4822082519531, + "KL/rejected_KL_mean": -304.6290283203125, + "KL/std": 140.94253540039062, + "epoch": 0.9644746787603931, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 96.29367065429688, + "fcm_dpo/q_t": 0.4311360716819763, + "grad_norm": 11.053569793701172, + "learning_rate": 2.0112967923011646e-09, + "logits/chosen": 0.9434064626693726, + "logits/rejected": 0.9192993640899658, + "logps/chosen": -271.1169738769531, + "logps/ref_chosen": -62.78158187866211, + "logps/ref_rejected": -85.40478515625, + "logps/rejected": -390.0338134765625, + "loss": 1.172, + "margin_dpo/margin_mean": 96.29366302490234, + "margin_dpo/margin_std": 164.99270629882812, + "step": 638 + }, + { + "KL/chosen_KL_mean": -187.1907958984375, + "KL/mean": -242.9512176513672, + "KL/rejected_KL_mean": -298.71160888671875, + "KL/std": 155.54104614257812, + "epoch": 0.9659863945578231, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 111.52082824707031, + "fcm_dpo/q_t": 0.42117875814437866, + "grad_norm": 10.140286445617676, + "learning_rate": 1.847382997337943e-09, + "logits/chosen": 0.9174301624298096, + "logits/rejected": 0.8663163185119629, + "logps/chosen": -240.95738220214844, + "logps/ref_chosen": -53.76658630371094, + "logps/ref_rejected": -72.30009460449219, + "logps/rejected": -371.01171875, + "loss": 1.139, + "margin_dpo/margin_mean": 111.52082824707031, + "margin_dpo/margin_std": 174.923828125, + "step": 639 + }, + { + "KL/chosen_KL_mean": -193.22474670410156, + "KL/mean": -241.06468200683594, + "KL/rejected_KL_mean": -288.9046325683594, + "KL/std": 144.21368408203125, + "epoch": 0.9674981103552532, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 95.67987823486328, + "fcm_dpo/q_t": 0.43061840534210205, + "grad_norm": 10.762125015258789, + "learning_rate": 1.690410564514244e-09, + "logits/chosen": 1.0091588497161865, + "logits/rejected": 0.966665506362915, + "logps/chosen": -244.64251708984375, + "logps/ref_chosen": -51.41777801513672, + "logps/ref_rejected": -77.27879333496094, + "logps/rejected": -366.18341064453125, + "loss": 1.1673, + "margin_dpo/margin_mean": 95.67987823486328, + "margin_dpo/margin_std": 154.81198120117188, + "step": 640 + }, + { + "KL/chosen_KL_mean": -208.03912353515625, + "KL/mean": -248.72650146484375, + "KL/rejected_KL_mean": -289.413818359375, + "KL/std": 132.9041290283203, + "epoch": 0.9690098261526833, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 81.37470245361328, + "fcm_dpo/q_t": 0.4405854344367981, + "grad_norm": 11.396415710449219, + "learning_rate": 1.5403838846864692e-09, + "logits/chosen": 0.9047503471374512, + "logits/rejected": 0.9040436148643494, + "logps/chosen": -279.09381103515625, + "logps/ref_chosen": -71.0546646118164, + "logps/ref_rejected": -82.2440185546875, + "logps/rejected": -371.6578369140625, + "loss": 1.1894, + "margin_dpo/margin_mean": 81.37470245361328, + "margin_dpo/margin_std": 129.62969970703125, + "step": 641 + }, + { + "KL/chosen_KL_mean": -216.7130584716797, + "KL/mean": -246.51898193359375, + "KL/rejected_KL_mean": -276.32489013671875, + "KL/std": 134.2535400390625, + "epoch": 0.9705215419501134, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 59.61183166503906, + "fcm_dpo/q_t": 0.4580966532230377, + "grad_norm": 12.750149726867676, + "learning_rate": 1.3973071544233218e-09, + "logits/chosen": 0.8720563650131226, + "logits/rejected": 0.8946826457977295, + "logps/chosen": -285.642333984375, + "logps/ref_chosen": -68.92927551269531, + "logps/ref_rejected": -70.85682678222656, + "logps/rejected": -347.18170166015625, + "loss": 1.2688, + "margin_dpo/margin_mean": 59.61183166503906, + "margin_dpo/margin_std": 158.3714141845703, + "step": 642 + }, + { + "KL/chosen_KL_mean": -195.25668334960938, + "KL/mean": -249.47900390625, + "KL/rejected_KL_mean": -303.7013244628906, + "KL/std": 150.85220336914062, + "epoch": 0.9720332577475435, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 108.44462585449219, + "fcm_dpo/q_t": 0.42308878898620605, + "grad_norm": 13.6078462600708, + "learning_rate": 1.261184375888541e-09, + "logits/chosen": 0.9121971130371094, + "logits/rejected": 0.8631089329719543, + "logps/chosen": -260.56573486328125, + "logps/ref_chosen": -65.30903625488281, + "logps/ref_rejected": -83.61613464355469, + "logps/rejected": -387.31744384765625, + "loss": 1.1468, + "margin_dpo/margin_mean": 108.44462585449219, + "margin_dpo/margin_std": 171.8424072265625, + "step": 643 + }, + { + "KL/chosen_KL_mean": -183.8955078125, + "KL/mean": -214.19424438476562, + "KL/rejected_KL_mean": -244.49301147460938, + "KL/std": 119.51637268066406, + "epoch": 0.9735449735449735, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 60.597511291503906, + "fcm_dpo/q_t": 0.45479634404182434, + "grad_norm": 8.376975059509277, + "learning_rate": 1.1320193567288527e-09, + "logits/chosen": 1.0118638277053833, + "logits/rejected": 0.9890410900115967, + "logps/chosen": -234.89810180664062, + "logps/ref_chosen": -51.002601623535156, + "logps/ref_rejected": -64.46372985839844, + "logps/rejected": -308.95672607421875, + "loss": 1.2544, + "margin_dpo/margin_mean": 60.597511291503906, + "margin_dpo/margin_std": 138.36883544921875, + "step": 644 + }, + { + "KL/chosen_KL_mean": -204.33132934570312, + "KL/mean": -242.13632202148438, + "KL/rejected_KL_mean": -279.94134521484375, + "KL/std": 136.10458374023438, + "epoch": 0.9750566893424036, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 75.61000061035156, + "fcm_dpo/q_t": 0.4438573122024536, + "grad_norm": 15.0194091796875, + "learning_rate": 1.0098157099674987e-09, + "logits/chosen": 0.9073389768600464, + "logits/rejected": 0.8987625241279602, + "logps/chosen": -265.29473876953125, + "logps/ref_chosen": -60.963409423828125, + "logps/ref_rejected": -69.73353576660156, + "logps/rejected": -349.67486572265625, + "loss": 1.2134, + "margin_dpo/margin_mean": 75.61000061035156, + "margin_dpo/margin_std": 142.1605682373047, + "step": 645 + }, + { + "KL/chosen_KL_mean": -222.45260620117188, + "KL/mean": -262.6767578125, + "KL/rejected_KL_mean": -302.90093994140625, + "KL/std": 172.28286743164062, + "epoch": 0.9765684051398337, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.44831848144531, + "fcm_dpo/q_t": 0.4418299198150635, + "grad_norm": 9.49789810180664, + "learning_rate": 8.945768539031783e-10, + "logits/chosen": 0.9376930594444275, + "logits/rejected": 0.8963108062744141, + "logps/chosen": -284.74267578125, + "logps/ref_chosen": -62.290069580078125, + "logps/ref_rejected": -85.54812622070312, + "logps/rejected": -388.44903564453125, + "loss": 1.2106, + "margin_dpo/margin_mean": 80.44831848144531, + "margin_dpo/margin_std": 158.8577423095703, + "step": 646 + }, + { + "KL/chosen_KL_mean": -219.18106079101562, + "KL/mean": -282.8184509277344, + "KL/rejected_KL_mean": -346.455810546875, + "KL/std": 167.93240356445312, + "epoch": 0.9780801209372638, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 127.27474975585938, + "fcm_dpo/q_t": 0.40855348110198975, + "grad_norm": 10.709985733032227, + "learning_rate": 7.863060120144316e-10, + "logits/chosen": 0.9660693407058716, + "logits/rejected": 0.8981690406799316, + "logps/chosen": -286.69696044921875, + "logps/ref_chosen": -67.515869140625, + "logps/ref_rejected": -101.50871276855469, + "logps/rejected": -447.96453857421875, + "loss": 1.0954, + "margin_dpo/margin_mean": 127.27474975585938, + "margin_dpo/margin_std": 169.9162139892578, + "step": 647 + }, + { + "KL/chosen_KL_mean": -219.24618530273438, + "KL/mean": -263.4234924316406, + "KL/rejected_KL_mean": -307.60076904296875, + "KL/std": 140.67422485351562, + "epoch": 0.9795918367346939, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 88.35462188720703, + "fcm_dpo/q_t": 0.437153160572052, + "grad_norm": 9.533730506896973, + "learning_rate": 6.850062128694045e-10, + "logits/chosen": 0.9046711921691895, + "logits/rejected": 0.8672827482223511, + "logps/chosen": -283.84210205078125, + "logps/ref_chosen": -64.59593963623047, + "logps/ref_rejected": -83.384033203125, + "logps/rejected": -390.98480224609375, + "loss": 1.2142, + "margin_dpo/margin_mean": 88.35462188720703, + "margin_dpo/margin_std": 183.80010986328125, + "step": 648 + }, + { + "KL/chosen_KL_mean": -195.35403442382812, + "KL/mean": -238.9105682373047, + "KL/rejected_KL_mean": -282.46710205078125, + "KL/std": 131.21609497070312, + "epoch": 0.981103552532124, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 87.11306762695312, + "fcm_dpo/q_t": 0.43540871143341064, + "grad_norm": 13.80373764038086, + "learning_rate": 5.906802900412788e-10, + "logits/chosen": 0.9699212312698364, + "logits/rejected": 0.9264802932739258, + "logps/chosen": -244.66368103027344, + "logps/ref_chosen": -49.30964660644531, + "logps/ref_rejected": -73.73710632324219, + "logps/rejected": -356.2042236328125, + "loss": 1.1903, + "margin_dpo/margin_mean": 87.11306762695312, + "margin_dpo/margin_std": 155.24761962890625, + "step": 649 + }, + { + "KL/chosen_KL_mean": -189.0690460205078, + "KL/mean": -237.6030731201172, + "KL/rejected_KL_mean": -286.1370849609375, + "KL/std": 138.7324676513672, + "epoch": 0.982615268329554, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 97.06804656982422, + "fcm_dpo/q_t": 0.4303385615348816, + "grad_norm": 9.39942741394043, + "learning_rate": 5.033308820289184e-10, + "logits/chosen": 0.9848513603210449, + "logits/rejected": 0.9468841552734375, + "logps/chosen": -244.13230895996094, + "logps/ref_chosen": -55.06325912475586, + "logps/ref_rejected": -77.39610290527344, + "logps/rejected": -363.533203125, + "loss": 1.1745, + "margin_dpo/margin_mean": 97.06805419921875, + "margin_dpo/margin_std": 170.67352294921875, + "step": 650 + }, + { + "KL/chosen_KL_mean": -187.72344970703125, + "KL/mean": -232.03823852539062, + "KL/rejected_KL_mean": -276.35302734375, + "KL/std": 137.58258056640625, + "epoch": 0.9841269841269841, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 88.62960815429688, + "fcm_dpo/q_t": 0.43634384870529175, + "grad_norm": 11.805054664611816, + "learning_rate": 4.2296043218295606e-10, + "logits/chosen": 0.9546950459480286, + "logits/rejected": 0.9093322157859802, + "logps/chosen": -241.78860473632812, + "logps/ref_chosen": -54.065162658691406, + "logps/ref_rejected": -77.79080200195312, + "logps/rejected": -354.14385986328125, + "loss": 1.1785, + "margin_dpo/margin_mean": 88.62960815429688, + "margin_dpo/margin_std": 143.01470947265625, + "step": 651 + }, + { + "KL/chosen_KL_mean": -223.95025634765625, + "KL/mean": -261.40289306640625, + "KL/rejected_KL_mean": -298.8555603027344, + "KL/std": 159.61614990234375, + "epoch": 0.9856386999244142, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 74.9052963256836, + "fcm_dpo/q_t": 0.4478684067726135, + "grad_norm": 11.329047203063965, + "learning_rate": 3.4957118863768176e-10, + "logits/chosen": 0.9421413540840149, + "logits/rejected": 0.9153538942337036, + "logps/chosen": -287.590576171875, + "logps/ref_chosen": -63.64030456542969, + "logps/ref_rejected": -78.86882019042969, + "logps/rejected": -377.724365234375, + "loss": 1.2524, + "margin_dpo/margin_mean": 74.9052963256836, + "margin_dpo/margin_std": 195.61993408203125, + "step": 652 + }, + { + "KL/chosen_KL_mean": -196.9767608642578, + "KL/mean": -241.781494140625, + "KL/rejected_KL_mean": -286.58624267578125, + "KL/std": 133.9491424560547, + "epoch": 0.9871504157218443, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.60951232910156, + "fcm_dpo/q_t": 0.4346145987510681, + "grad_norm": 13.65695858001709, + "learning_rate": 2.831652042480093e-10, + "logits/chosen": 0.9071061611175537, + "logits/rejected": 0.8793363571166992, + "logps/chosen": -258.6451416015625, + "logps/ref_chosen": -61.668373107910156, + "logps/ref_rejected": -73.83012390136719, + "logps/rejected": -360.4163818359375, + "loss": 1.1819, + "margin_dpo/margin_mean": 89.60951232910156, + "margin_dpo/margin_std": 151.9982147216797, + "step": 653 + }, + { + "KL/chosen_KL_mean": -205.64804077148438, + "KL/mean": -253.06338500976562, + "KL/rejected_KL_mean": -300.47869873046875, + "KL/std": 164.1302490234375, + "epoch": 0.9886621315192744, + "fcm_dpo/beta": 0.003094049636274576, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.83065795898438, + "fcm_dpo/q_t": 0.4318302869796753, + "grad_norm": 12.566394805908203, + "learning_rate": 2.2374433653205016e-10, + "logits/chosen": 0.914149284362793, + "logits/rejected": 0.8528527021408081, + "logps/chosen": -263.21630859375, + "logps/ref_chosen": -57.568267822265625, + "logps/ref_rejected": -87.74789428710938, + "logps/rejected": -388.2265930175781, + "loss": 1.1715, + "margin_dpo/margin_mean": 94.83065795898438, + "margin_dpo/margin_std": 154.06057739257812, + "step": 654 + }, + { + "KL/chosen_KL_mean": -161.66128540039062, + "KL/mean": -227.72610473632812, + "KL/rejected_KL_mean": -293.7909240722656, + "KL/std": 143.35321044921875, + "epoch": 0.9901738473167044, + "fcm_dpo/beta": 0.0030435477383434772, + "fcm_dpo/delta": -0.08228477090597153, + "fcm_dpo/margin": 132.12965393066406, + "fcm_dpo/q_t": 0.4063999652862549, + "grad_norm": 10.75922966003418, + "learning_rate": 1.7131024761923852e-10, + "logits/chosen": 0.9187965393066406, + "logits/rejected": 0.8673887848854065, + "logps/chosen": -213.8084259033203, + "logps/ref_chosen": -52.14714813232422, + "logps/ref_rejected": -80.85014343261719, + "logps/rejected": -374.64105224609375, + "loss": 1.0789, + "margin_dpo/margin_mean": 132.12965393066406, + "margin_dpo/margin_std": 152.25323486328125, + "step": 655 + }, + { + "KL/chosen_KL_mean": -219.64300537109375, + "KL/mean": -265.52117919921875, + "KL/rejected_KL_mean": -311.3993835449219, + "KL/std": 142.48965454101562, + "epoch": 0.9916855631141346, + "fcm_dpo/beta": 0.0030435477383434772, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.75637817382812, + "fcm_dpo/q_t": 0.43464866280555725, + "grad_norm": 9.470726013183594, + "learning_rate": 1.2586440420372934e-10, + "logits/chosen": 0.8957068920135498, + "logits/rejected": 0.8757596611976624, + "logps/chosen": -292.89971923828125, + "logps/ref_chosen": -73.25672912597656, + "logps/ref_rejected": -85.35127258300781, + "logps/rejected": -396.75067138671875, + "loss": 1.1805, + "margin_dpo/margin_mean": 91.75637817382812, + "margin_dpo/margin_std": 156.62033081054688, + "step": 656 + }, + { + "KL/chosen_KL_mean": -187.28375244140625, + "KL/mean": -245.9228057861328, + "KL/rejected_KL_mean": -304.5618591308594, + "KL/std": 159.7275848388672, + "epoch": 0.9931972789115646, + "fcm_dpo/beta": 0.0030435477383434772, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 117.27809143066406, + "fcm_dpo/q_t": 0.41826194524765015, + "grad_norm": 9.124080657958984, + "learning_rate": 8.740807750345913e-11, + "logits/chosen": 1.0158004760742188, + "logits/rejected": 0.9686501026153564, + "logps/chosen": -237.0071563720703, + "logps/ref_chosen": -49.72339630126953, + "logps/ref_rejected": -75.1568603515625, + "logps/rejected": -379.7187194824219, + "loss": 1.13, + "margin_dpo/margin_mean": 117.27810668945312, + "margin_dpo/margin_std": 178.3359375, + "step": 657 + }, + { + "KL/chosen_KL_mean": -205.32443237304688, + "KL/mean": -251.24359130859375, + "KL/rejected_KL_mean": -297.16278076171875, + "KL/std": 170.65353393554688, + "epoch": 0.9947089947089947, + "fcm_dpo/beta": 0.0030435477383434772, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.83834838867188, + "fcm_dpo/q_t": 0.4347524642944336, + "grad_norm": 8.767982482910156, + "learning_rate": 5.594234322453539e-11, + "logits/chosen": 0.953624963760376, + "logits/rejected": 0.9262589812278748, + "logps/chosen": -268.37078857421875, + "logps/ref_chosen": -63.04634094238281, + "logps/ref_rejected": -83.44963073730469, + "logps/rejected": -380.6123962402344, + "loss": 1.21, + "margin_dpo/margin_mean": 91.83834075927734, + "margin_dpo/margin_std": 194.5883331298828, + "step": 658 + }, + { + "KL/chosen_KL_mean": -207.29444885253906, + "KL/mean": -241.6499481201172, + "KL/rejected_KL_mean": -276.00543212890625, + "KL/std": 147.86619567871094, + "epoch": 0.9962207105064248, + "fcm_dpo/beta": 0.0030435477383434772, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 68.71096801757812, + "fcm_dpo/q_t": 0.4509459435939789, + "grad_norm": 11.185192108154297, + "learning_rate": 3.146808153123293e-11, + "logits/chosen": 0.9820291996002197, + "logits/rejected": 0.9410825967788696, + "logps/chosen": -262.3746337890625, + "logps/ref_chosen": -55.0802001953125, + "logps/ref_rejected": -71.91049194335938, + "logps/rejected": -347.9159240722656, + "loss": 1.2532, + "margin_dpo/margin_mean": 68.71096801757812, + "margin_dpo/margin_std": 169.84005737304688, + "step": 659 + }, + { + "KL/chosen_KL_mean": -190.37002563476562, + "KL/mean": -251.46902465820312, + "KL/rejected_KL_mean": -312.5680236816406, + "KL/std": 157.89157104492188, + "epoch": 0.9977324263038548, + "fcm_dpo/beta": 0.0030435477383434772, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 122.19801330566406, + "fcm_dpo/q_t": 0.4155905544757843, + "grad_norm": 11.91143798828125, + "learning_rate": 1.3985977021235829e-11, + "logits/chosen": 1.0178961753845215, + "logits/rejected": 0.983450174331665, + "logps/chosen": -244.89593505859375, + "logps/ref_chosen": -54.525917053222656, + "logps/ref_rejected": -81.23604583740234, + "logps/rejected": -393.8040771484375, + "loss": 1.1127, + "margin_dpo/margin_mean": 122.19801330566406, + "margin_dpo/margin_std": 177.11691284179688, + "step": 660 + }, + { + "KL/chosen_KL_mean": -225.02529907226562, + "KL/mean": -263.05596923828125, + "KL/rejected_KL_mean": -301.086669921875, + "KL/std": 154.0184326171875, + "epoch": 0.999244142101285, + "fcm_dpo/beta": 0.0030435477383434772, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.06137084960938, + "fcm_dpo/q_t": 0.4469204545021057, + "grad_norm": 8.742901802062988, + "learning_rate": 3.4965187065971735e-12, + "logits/chosen": 0.9296639561653137, + "logits/rejected": 0.8836817741394043, + "logps/chosen": -285.39794921875, + "logps/ref_chosen": -60.37263870239258, + "logps/ref_rejected": -77.42874145507812, + "logps/rejected": -378.51544189453125, + "loss": 1.2393, + "margin_dpo/margin_mean": 76.06137084960938, + "margin_dpo/margin_std": 180.522705078125, + "step": 661 + }, + { + "epoch": 0.999244142101285, + "step": 661, + "total_flos": 0.0, + "train_loss": 1.1812428684169693, + "train_runtime": 1648.6054, + "train_samples_per_second": 25.68, + "train_steps_per_second": 0.401 + } + ], + "logging_steps": 1, + "max_steps": 661, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}