commit 4d19c05e0b2e1ac2b3e4868e5bbe4f2590651c0b Author: ModelHub XC Date: Tue May 26 11:58:18 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.48 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..6c7e08c --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-helpful-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.48 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.48 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-helpful-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-helpful-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..731b007 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.0968060302664355, + "train_runtime": 1742.1859, + "train_samples": 43598, + "train_samples_per_second": 25.025, + "train_steps_per_second": 0.391 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..a3c8a0b --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63498525df2d3a7c5f20d2f23389f1b08b41af668a47ec9c510ba01601f32bbd +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..7274c11 --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d47eef568c7c71823ba1e2e878ad2ae1210f6ca0ce25b382197a33680d12d174 +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..a25ea63 --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbc9ce33379bdff824e3125a00b8a5680586f0f8e250775b32661b4489d7a0c5 +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..21b2426 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de90854879c3aa48dff8a0b37837942cabea310e31811988cd6cf2173344818b +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..e12d254 --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75c3bbd4c922c140f4753a9544c6804eef2fa979cfa672bdd2d9f4572c373068 +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..e5e4b67 --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5532e1fa57f3f94a401c55b82f563a409fa0a111781fd3ac017711aa265cad4c +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..be7c2b3 --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e0bd1db1613ae1ba715f004d0aa0e1e93c2173bba651843bf064036d8e1c8ae +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..731b007 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.0968060302664355, + "train_runtime": 1742.1859, + "train_samples": 43598, + "train_samples_per_second": 25.025, + "train_steps_per_second": 0.391 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..14c26b7 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 200, + "global_step": 681, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.00527191162109375, + "KL/mean": 0.016706019639968872, + "KL/rejected_KL_mean": 0.028141021728515625, + "KL/std": 0.272699236869812, + "epoch": 0.0014684287812041115, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02287006378173828, + "fcm_dpo/q_t": 0.5005706548690796, + "grad_norm": 83.52754974365234, + "learning_rate": 0.0, + "logits/chosen": -0.4974287748336792, + "logits/rejected": -0.43299180269241333, + "logps/chosen": -50.1435661315918, + "logps/ref_chosen": -50.14883804321289, + "logps/ref_rejected": -74.1280517578125, + "logps/rejected": -74.09991455078125, + "loss": 1.389, + "margin_dpo/margin_mean": -0.02287048101425171, + "margin_dpo/margin_std": 0.41920793056488037, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.03498649597167969, + "KL/mean": -0.00212840735912323, + "KL/rejected_KL_mean": 0.030735015869140625, + "KL/std": 0.24797174334526062, + "epoch": 0.002936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06572261452674866, + "fcm_dpo/q_t": 0.5016425848007202, + "grad_norm": 72.21381378173828, + "learning_rate": 7.246376811594203e-09, + "logits/chosen": -0.49536412954330444, + "logits/rejected": -0.4594460427761078, + "logps/chosen": -52.65568923950195, + "logps/ref_chosen": -52.620704650878906, + "logps/ref_rejected": -75.30413818359375, + "logps/rejected": -75.27340698242188, + "loss": 1.3932, + "margin_dpo/margin_mean": -0.06572240591049194, + "margin_dpo/margin_std": 0.35048407316207886, + "step": 2 + }, + { + "KL/chosen_KL_mean": 0.012119293212890625, + "KL/mean": 0.023418381810188293, + "KL/rejected_KL_mean": 0.034717559814453125, + "KL/std": 0.2618991434574127, + "epoch": 0.004405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02260264754295349, + "fcm_dpo/q_t": 0.5005650520324707, + "grad_norm": 70.9487533569336, + "learning_rate": 1.4492753623188406e-08, + "logits/chosen": -0.48166990280151367, + "logits/rejected": -0.442186176776886, + "logps/chosen": -60.96947479248047, + "logps/ref_chosen": -60.981597900390625, + "logps/ref_rejected": -68.67259216308594, + "logps/rejected": -68.63787841796875, + "loss": 1.3889, + "margin_dpo/margin_mean": -0.022602319717407227, + "margin_dpo/margin_std": 0.3861812651157379, + "step": 3 + }, + { + "KL/chosen_KL_mean": -0.04089927673339844, + "KL/mean": 0.003831237554550171, + "KL/rejected_KL_mean": 0.04856109619140625, + "KL/std": 0.2447841912508011, + "epoch": 0.005873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.08945882320404053, + "fcm_dpo/q_t": 0.5022358894348145, + "grad_norm": 72.29135131835938, + "learning_rate": 2.1739130434782606e-08, + "logits/chosen": -0.46891048550605774, + "logits/rejected": -0.44123539328575134, + "logps/chosen": -56.80860900878906, + "logps/ref_chosen": -56.7677116394043, + "logps/ref_rejected": -86.64710998535156, + "logps/rejected": -86.59854888916016, + "loss": 1.3956, + "margin_dpo/margin_mean": -0.08945959806442261, + "margin_dpo/margin_std": 0.3571521043777466, + "step": 4 + }, + { + "KL/chosen_KL_mean": 0.04132270812988281, + "KL/mean": 0.02954573929309845, + "KL/rejected_KL_mean": 0.017772674560546875, + "KL/std": 0.2609584927558899, + "epoch": 0.007342143906020558, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0235537588596344, + "fcm_dpo/q_t": 0.49941110610961914, + "grad_norm": 89.83948516845703, + "learning_rate": 2.898550724637681e-08, + "logits/chosen": -0.5144683122634888, + "logits/rejected": -0.4707663655281067, + "logps/chosen": -53.81805419921875, + "logps/ref_chosen": -53.859375, + "logps/ref_rejected": -84.14918518066406, + "logps/rejected": -84.13140869140625, + "loss": 1.3843, + "margin_dpo/margin_mean": 0.023554116487503052, + "margin_dpo/margin_std": 0.35333937406539917, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.00206756591796875, + "KL/mean": -0.015875771641731262, + "KL/rejected_KL_mean": -0.02968597412109375, + "KL/std": 0.24448521435260773, + "epoch": 0.00881057268722467, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.027621835470199585, + "fcm_dpo/q_t": 0.49930986762046814, + "grad_norm": 92.0187759399414, + "learning_rate": 3.6231884057971014e-08, + "logits/chosen": -0.5049769282341003, + "logits/rejected": -0.4625431299209595, + "logps/chosen": -63.009552001953125, + "logps/ref_chosen": -63.007484436035156, + "logps/ref_rejected": -92.64534759521484, + "logps/rejected": -92.67503356933594, + "loss": 1.3839, + "margin_dpo/margin_mean": 0.02762138843536377, + "margin_dpo/margin_std": 0.3585347533226013, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.05124092102050781, + "KL/mean": 0.04420331120491028, + "KL/rejected_KL_mean": 0.03716278076171875, + "KL/std": 0.3070700764656067, + "epoch": 0.010279001468428781, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.014078021049499512, + "fcm_dpo/q_t": 0.499647855758667, + "grad_norm": 82.254150390625, + "learning_rate": 4.347826086956521e-08, + "logits/chosen": -0.49819037318229675, + "logits/rejected": -0.4643648862838745, + "logps/chosen": -57.72357940673828, + "logps/ref_chosen": -57.774818420410156, + "logps/ref_rejected": -103.92059326171875, + "logps/rejected": -103.88343048095703, + "loss": 1.3854, + "margin_dpo/margin_mean": 0.014078229665756226, + "margin_dpo/margin_std": 0.46783095598220825, + "step": 7 + }, + { + "KL/chosen_KL_mean": -0.0028095245361328125, + "KL/mean": 0.0157100111246109, + "KL/rejected_KL_mean": 0.034236907958984375, + "KL/std": 0.2974522113800049, + "epoch": 0.011747430249632892, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.03704729676246643, + "fcm_dpo/q_t": 0.5009254813194275, + "grad_norm": 79.46379852294922, + "learning_rate": 5.0724637681159424e-08, + "logits/chosen": -0.5174297094345093, + "logits/rejected": -0.492509663105011, + "logps/chosen": -58.71884536743164, + "logps/ref_chosen": -58.716033935546875, + "logps/ref_rejected": -79.3114242553711, + "logps/rejected": -79.27719116210938, + "loss": 1.3904, + "margin_dpo/margin_mean": -0.037046968936920166, + "margin_dpo/margin_std": 0.3789815306663513, + "step": 8 + }, + { + "KL/chosen_KL_mean": 0.02559661865234375, + "KL/mean": 0.013599887490272522, + "KL/rejected_KL_mean": 0.0016021728515625, + "KL/std": 0.29548415541648865, + "epoch": 0.013215859030837005, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0239962637424469, + "fcm_dpo/q_t": 0.4993996322154999, + "grad_norm": 85.05180358886719, + "learning_rate": 5.797101449275362e-08, + "logits/chosen": -0.4923360347747803, + "logits/rejected": -0.4455389380455017, + "logps/chosen": -69.84124755859375, + "logps/ref_chosen": -69.8668441772461, + "logps/ref_rejected": -99.6026611328125, + "logps/rejected": -99.60105895996094, + "loss": 1.3843, + "margin_dpo/margin_mean": 0.023995906114578247, + "margin_dpo/margin_std": 0.4077387750148773, + "step": 9 + }, + { + "KL/chosen_KL_mean": -0.0124359130859375, + "KL/mean": 0.0032117292284965515, + "KL/rejected_KL_mean": 0.01885986328125, + "KL/std": 0.2599485218524933, + "epoch": 0.014684287812041116, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.031296178698539734, + "fcm_dpo/q_t": 0.5007817149162292, + "grad_norm": 71.18731689453125, + "learning_rate": 6.521739130434782e-08, + "logits/chosen": -0.48350363969802856, + "logits/rejected": -0.43980133533477783, + "logps/chosen": -48.370121002197266, + "logps/ref_chosen": -48.35768508911133, + "logps/ref_rejected": -80.37206268310547, + "logps/rejected": -80.35320281982422, + "loss": 1.3898, + "margin_dpo/margin_mean": -0.03129580616950989, + "margin_dpo/margin_std": 0.3704303503036499, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.01000213623046875, + "KL/mean": 0.0064017027616500854, + "KL/rejected_KL_mean": 0.002803802490234375, + "KL/std": 0.24205231666564941, + "epoch": 0.016152716593245228, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.007197290658950806, + "fcm_dpo/q_t": 0.4998197555541992, + "grad_norm": 68.69322204589844, + "learning_rate": 7.246376811594203e-08, + "logits/chosen": -0.45256781578063965, + "logits/rejected": -0.4272083044052124, + "logps/chosen": -53.00685119628906, + "logps/ref_chosen": -53.01685333251953, + "logps/ref_rejected": -87.78038024902344, + "logps/rejected": -87.77757263183594, + "loss": 1.3859, + "margin_dpo/margin_mean": 0.007197737693786621, + "margin_dpo/margin_std": 0.33006855845451355, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.028219223022460938, + "KL/mean": -0.04441457986831665, + "KL/rejected_KL_mean": -0.060604095458984375, + "KL/std": 0.2953580617904663, + "epoch": 0.01762114537444934, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03238654136657715, + "fcm_dpo/q_t": 0.4991909861564636, + "grad_norm": 89.39579772949219, + "learning_rate": 7.971014492753623e-08, + "logits/chosen": -0.5462939739227295, + "logits/rejected": -0.5103648900985718, + "logps/chosen": -61.833656311035156, + "logps/ref_chosen": -61.80543518066406, + "logps/ref_rejected": -104.8582763671875, + "logps/rejected": -104.91887664794922, + "loss": 1.3835, + "margin_dpo/margin_mean": 0.03238588571548462, + "margin_dpo/margin_std": 0.43585580587387085, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.0037994384765625, + "KL/mean": 0.034382060170173645, + "KL/rejected_KL_mean": 0.06496810913085938, + "KL/std": 0.27900394797325134, + "epoch": 0.01908957415565345, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06116214394569397, + "fcm_dpo/q_t": 0.5015291571617126, + "grad_norm": 79.63651275634766, + "learning_rate": 8.695652173913042e-08, + "logits/chosen": -0.470973938703537, + "logits/rejected": -0.44221845269203186, + "logps/chosen": -64.25655364990234, + "logps/ref_chosen": -64.2603530883789, + "logps/ref_rejected": -87.20307922363281, + "logps/rejected": -87.13811492919922, + "loss": 1.3927, + "margin_dpo/margin_mean": -0.06116211414337158, + "margin_dpo/margin_std": 0.3499847948551178, + "step": 13 + }, + { + "KL/chosen_KL_mean": -0.02565765380859375, + "KL/mean": -0.022577762603759766, + "KL/rejected_KL_mean": -0.019500732421875, + "KL/std": 0.24927328526973724, + "epoch": 0.020558002936857563, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.00615239143371582, + "fcm_dpo/q_t": 0.5001521706581116, + "grad_norm": 85.70947265625, + "learning_rate": 9.420289855072464e-08, + "logits/chosen": -0.48932701349258423, + "logits/rejected": -0.4504287838935852, + "logps/chosen": -58.135868072509766, + "logps/ref_chosen": -58.11021041870117, + "logps/ref_rejected": -104.04708099365234, + "logps/rejected": -104.06658172607422, + "loss": 1.3872, + "margin_dpo/margin_mean": -0.006151974201202393, + "margin_dpo/margin_std": 0.3624458312988281, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.018096923828125, + "KL/mean": -0.0443209707736969, + "KL/rejected_KL_mean": -0.07054519653320312, + "KL/std": 0.2593069076538086, + "epoch": 0.022026431718061675, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05244654417037964, + "fcm_dpo/q_t": 0.49868789315223694, + "grad_norm": 64.14149475097656, + "learning_rate": 1.0144927536231885e-07, + "logits/chosen": -0.48956209421157837, + "logits/rejected": -0.47070273756980896, + "logps/chosen": -56.985008239746094, + "logps/ref_chosen": -56.96691131591797, + "logps/ref_rejected": -80.80863952636719, + "logps/rejected": -80.87918853759766, + "loss": 1.3815, + "margin_dpo/margin_mean": 0.05244651436805725, + "margin_dpo/margin_std": 0.39674949645996094, + "step": 15 + }, + { + "KL/chosen_KL_mean": -0.03409004211425781, + "KL/mean": -0.04335886240005493, + "KL/rejected_KL_mean": -0.052623748779296875, + "KL/std": 0.273632287979126, + "epoch": 0.023494860499265784, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.01853311061859131, + "fcm_dpo/q_t": 0.49953699111938477, + "grad_norm": 84.44381713867188, + "learning_rate": 1.0869565217391303e-07, + "logits/chosen": -0.5264363288879395, + "logits/rejected": -0.4855811595916748, + "logps/chosen": -61.773983001708984, + "logps/ref_chosen": -61.739891052246094, + "logps/ref_rejected": -84.36947631835938, + "logps/rejected": -84.4220962524414, + "loss": 1.3849, + "margin_dpo/margin_mean": 0.018532603979110718, + "margin_dpo/margin_std": 0.41814303398132324, + "step": 16 + }, + { + "KL/chosen_KL_mean": 0.052093505859375, + "KL/mean": 0.00013424456119537354, + "KL/rejected_KL_mean": -0.05182647705078125, + "KL/std": 0.24351900815963745, + "epoch": 0.024963289280469897, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1039208471775055, + "fcm_dpo/q_t": 0.49740248918533325, + "grad_norm": 78.52274322509766, + "learning_rate": 1.1594202898550725e-07, + "logits/chosen": -0.48946413397789, + "logits/rejected": -0.450802206993103, + "logps/chosen": -67.65824127197266, + "logps/ref_chosen": -67.71033477783203, + "logps/ref_rejected": -85.37865447998047, + "logps/rejected": -85.43048095703125, + "loss": 1.3762, + "margin_dpo/margin_mean": 0.10392042994499207, + "margin_dpo/margin_std": 0.3155348598957062, + "step": 17 + }, + { + "KL/chosen_KL_mean": -0.03583717346191406, + "KL/mean": -0.04290063679218292, + "KL/rejected_KL_mean": -0.049968719482421875, + "KL/std": 0.2571260333061218, + "epoch": 0.02643171806167401, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.01413002610206604, + "fcm_dpo/q_t": 0.49964603781700134, + "grad_norm": 81.87183380126953, + "learning_rate": 1.2318840579710146e-07, + "logits/chosen": -0.4920946955680847, + "logits/rejected": -0.4363882541656494, + "logps/chosen": -47.77532958984375, + "logps/ref_chosen": -47.7394905090332, + "logps/ref_rejected": -75.4722900390625, + "logps/rejected": -75.52225494384766, + "loss": 1.3853, + "margin_dpo/margin_mean": 0.014129400253295898, + "margin_dpo/margin_std": 0.3615719676017761, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.031063079833984375, + "KL/mean": -0.03421039879322052, + "KL/rejected_KL_mean": -0.09947967529296875, + "KL/std": 0.24964235723018646, + "epoch": 0.027900146842878122, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1305374801158905, + "fcm_dpo/q_t": 0.4967383146286011, + "grad_norm": 72.95270538330078, + "learning_rate": 1.3043478260869563e-07, + "logits/chosen": -0.489202618598938, + "logits/rejected": -0.4391380548477173, + "logps/chosen": -70.17430114746094, + "logps/ref_chosen": -70.20536041259766, + "logps/ref_rejected": -89.7575912475586, + "logps/rejected": -89.85707092285156, + "loss": 1.3736, + "margin_dpo/margin_mean": 0.13053762912750244, + "margin_dpo/margin_std": 0.32683902978897095, + "step": 19 + }, + { + "KL/chosen_KL_mean": 0.040554046630859375, + "KL/mean": -0.01780860126018524, + "KL/rejected_KL_mean": -0.07616806030273438, + "KL/std": 0.2401285469532013, + "epoch": 0.02936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.11672148108482361, + "fcm_dpo/q_t": 0.49708235263824463, + "grad_norm": 74.03209686279297, + "learning_rate": 1.3768115942028986e-07, + "logits/chosen": -0.5423327684402466, + "logits/rejected": -0.4851418733596802, + "logps/chosen": -50.76268768310547, + "logps/ref_chosen": -50.80324172973633, + "logps/ref_rejected": -78.82334899902344, + "logps/rejected": -78.89952087402344, + "loss": 1.3749, + "margin_dpo/margin_mean": 0.11672136187553406, + "margin_dpo/margin_std": 0.3035232722759247, + "step": 20 + }, + { + "KL/chosen_KL_mean": -0.004669189453125, + "KL/mean": -0.06208917498588562, + "KL/rejected_KL_mean": -0.11950302124023438, + "KL/std": 0.27453505992889404, + "epoch": 0.030837004405286344, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.11483883857727051, + "fcm_dpo/q_t": 0.49713045358657837, + "grad_norm": 77.23396301269531, + "learning_rate": 1.4492753623188405e-07, + "logits/chosen": -0.49135324358940125, + "logits/rejected": -0.46715474128723145, + "logps/chosen": -50.06768798828125, + "logps/ref_chosen": -50.063018798828125, + "logps/ref_rejected": -77.86878967285156, + "logps/rejected": -77.98829650878906, + "loss": 1.3751, + "margin_dpo/margin_mean": 0.11483970284461975, + "margin_dpo/margin_std": 0.3439750671386719, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.0177459716796875, + "KL/mean": -0.06365428864955902, + "KL/rejected_KL_mean": -0.14505386352539062, + "KL/std": 0.2868351936340332, + "epoch": 0.032305433186490456, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1628032624721527, + "fcm_dpo/q_t": 0.4959312379360199, + "grad_norm": 84.80654907226562, + "learning_rate": 1.5217391304347825e-07, + "logits/chosen": -0.47701001167297363, + "logits/rejected": -0.43303921818733215, + "logps/chosen": -59.03989028930664, + "logps/ref_chosen": -59.05763626098633, + "logps/ref_rejected": -97.50466918945312, + "logps/rejected": -97.64972686767578, + "loss": 1.3704, + "margin_dpo/margin_mean": 0.1628030240535736, + "margin_dpo/margin_std": 0.35856860876083374, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.05772972106933594, + "KL/mean": -0.029708027839660645, + "KL/rejected_KL_mean": -0.1171417236328125, + "KL/std": 0.32750219106674194, + "epoch": 0.033773861967694566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1748729944229126, + "fcm_dpo/q_t": 0.49562904238700867, + "grad_norm": 80.86152648925781, + "learning_rate": 1.5942028985507245e-07, + "logits/chosen": -0.47440969944000244, + "logits/rejected": -0.45099538564682007, + "logps/chosen": -60.01996612548828, + "logps/ref_chosen": -60.07769775390625, + "logps/ref_rejected": -81.13955688476562, + "logps/rejected": -81.25669860839844, + "loss": 1.3694, + "margin_dpo/margin_mean": 0.17487305402755737, + "margin_dpo/margin_std": 0.4415150582790375, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.04306221008300781, + "KL/mean": -0.06060684472322464, + "KL/rejected_KL_mean": -0.16427993774414062, + "KL/std": 0.27947989106178284, + "epoch": 0.03524229074889868, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.20734088122844696, + "fcm_dpo/q_t": 0.4948195815086365, + "grad_norm": 82.94523620605469, + "learning_rate": 1.6666666666666665e-07, + "logits/chosen": -0.5204753875732422, + "logits/rejected": -0.5036641955375671, + "logps/chosen": -44.24797058105469, + "logps/ref_chosen": -44.29103469848633, + "logps/ref_rejected": -99.12521362304688, + "logps/rejected": -99.28948974609375, + "loss": 1.3661, + "margin_dpo/margin_mean": 0.20734065771102905, + "margin_dpo/margin_std": 0.39581578969955444, + "step": 24 + }, + { + "KL/chosen_KL_mean": 0.007762908935546875, + "KL/mean": -0.0868898332118988, + "KL/rejected_KL_mean": -0.18154144287109375, + "KL/std": 0.332895964384079, + "epoch": 0.03671071953010279, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.18930190801620483, + "fcm_dpo/q_t": 0.4952710270881653, + "grad_norm": 73.9625244140625, + "learning_rate": 1.7391304347826085e-07, + "logits/chosen": -0.48971864581108093, + "logits/rejected": -0.4594026803970337, + "logps/chosen": -52.52928924560547, + "logps/ref_chosen": -52.537052154541016, + "logps/ref_rejected": -89.34219360351562, + "logps/rejected": -89.52372741699219, + "loss": 1.368, + "margin_dpo/margin_mean": 0.18930193781852722, + "margin_dpo/margin_std": 0.4436342120170593, + "step": 25 + }, + { + "KL/chosen_KL_mean": 0.08230400085449219, + "KL/mean": -0.10930752754211426, + "KL/rejected_KL_mean": -0.3009147644042969, + "KL/std": 0.40695035457611084, + "epoch": 0.0381791483113069, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.383225679397583, + "fcm_dpo/q_t": 0.49042731523513794, + "grad_norm": 87.4775390625, + "learning_rate": 1.8115942028985507e-07, + "logits/chosen": -0.5517945289611816, + "logits/rejected": -0.52059006690979, + "logps/chosen": -53.84050369262695, + "logps/ref_chosen": -53.92280578613281, + "logps/ref_rejected": -103.35971069335938, + "logps/rejected": -103.66062927246094, + "loss": 1.3489, + "margin_dpo/margin_mean": 0.3832261562347412, + "margin_dpo/margin_std": 0.4770854711532593, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.12397003173828125, + "KL/mean": -0.1301194131374359, + "KL/rejected_KL_mean": -0.38420867919921875, + "KL/std": 0.47300222516059875, + "epoch": 0.039647577092511016, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5081788301467896, + "fcm_dpo/q_t": 0.48731520771980286, + "grad_norm": 92.92485046386719, + "learning_rate": 1.8840579710144927e-07, + "logits/chosen": -0.5351539850234985, + "logits/rejected": -0.49897927045822144, + "logps/chosen": -42.774559020996094, + "logps/ref_chosen": -42.898529052734375, + "logps/ref_rejected": -98.72419738769531, + "logps/rejected": -99.10841369628906, + "loss": 1.337, + "margin_dpo/margin_mean": 0.5081784725189209, + "margin_dpo/margin_std": 0.578755795955658, + "step": 27 + }, + { + "KL/chosen_KL_mean": 0.037380218505859375, + "KL/mean": -0.13808989524841309, + "KL/rejected_KL_mean": -0.3135643005371094, + "KL/std": 0.395422101020813, + "epoch": 0.041116005873715125, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.35093891620635986, + "fcm_dpo/q_t": 0.49123209714889526, + "grad_norm": 76.57662963867188, + "learning_rate": 1.9565217391304347e-07, + "logits/chosen": -0.5218425989151001, + "logits/rejected": -0.46810781955718994, + "logps/chosen": -60.51911926269531, + "logps/ref_chosen": -60.55650329589844, + "logps/ref_rejected": -91.40111541748047, + "logps/rejected": -91.71467590332031, + "loss": 1.3521, + "margin_dpo/margin_mean": 0.3509393632411957, + "margin_dpo/margin_std": 0.4972136616706848, + "step": 28 + }, + { + "KL/chosen_KL_mean": 0.07178878784179688, + "KL/mean": -0.212729811668396, + "KL/rejected_KL_mean": -0.4972419738769531, + "KL/std": 0.48936769366264343, + "epoch": 0.042584434654919234, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5690329670906067, + "fcm_dpo/q_t": 0.48579180240631104, + "grad_norm": 90.49282836914062, + "learning_rate": 2.028985507246377e-07, + "logits/chosen": -0.5380536317825317, + "logits/rejected": -0.4905041456222534, + "logps/chosen": -57.73599624633789, + "logps/ref_chosen": -57.80778503417969, + "logps/ref_rejected": -97.39434814453125, + "logps/rejected": -97.89159393310547, + "loss": 1.331, + "margin_dpo/margin_mean": 0.5690339803695679, + "margin_dpo/margin_std": 0.5614117980003357, + "step": 29 + }, + { + "KL/chosen_KL_mean": 0.12009620666503906, + "KL/mean": -0.20123910903930664, + "KL/rejected_KL_mean": -0.5225753784179688, + "KL/std": 0.5947461724281311, + "epoch": 0.04405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6426688432693481, + "fcm_dpo/q_t": 0.48396098613739014, + "grad_norm": 87.52836608886719, + "learning_rate": 2.1014492753623187e-07, + "logits/chosen": -0.47559118270874023, + "logits/rejected": -0.4440673589706421, + "logps/chosen": -52.457275390625, + "logps/ref_chosen": -52.577369689941406, + "logps/ref_rejected": -98.48920440673828, + "logps/rejected": -99.01177978515625, + "loss": 1.3242, + "margin_dpo/margin_mean": 0.642669141292572, + "margin_dpo/margin_std": 0.6606372594833374, + "step": 30 + }, + { + "KL/chosen_KL_mean": 0.09145545959472656, + "KL/mean": -0.1492789387702942, + "KL/rejected_KL_mean": -0.3900146484375, + "KL/std": 0.5775303840637207, + "epoch": 0.04552129221732746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4814666509628296, + "fcm_dpo/q_t": 0.4879956841468811, + "grad_norm": 67.94648742675781, + "learning_rate": 2.1739130434782607e-07, + "logits/chosen": -0.4888034462928772, + "logits/rejected": -0.4421996772289276, + "logps/chosen": -63.7154655456543, + "logps/ref_chosen": -63.806922912597656, + "logps/ref_rejected": -72.89400482177734, + "logps/rejected": -73.28401947021484, + "loss": 1.3403, + "margin_dpo/margin_mean": 0.4814665913581848, + "margin_dpo/margin_std": 0.7748575210571289, + "step": 31 + }, + { + "KL/chosen_KL_mean": 0.15587234497070312, + "KL/mean": -0.18827323615550995, + "KL/rejected_KL_mean": -0.5324172973632812, + "KL/std": 0.6637758612632751, + "epoch": 0.04698972099853157, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6882903575897217, + "fcm_dpo/q_t": 0.48283424973487854, + "grad_norm": 83.79432678222656, + "learning_rate": 2.2463768115942027e-07, + "logits/chosen": -0.5092083215713501, + "logits/rejected": -0.4677237272262573, + "logps/chosen": -62.58365249633789, + "logps/ref_chosen": -62.739524841308594, + "logps/ref_rejected": -89.3175048828125, + "logps/rejected": -89.84991455078125, + "loss": 1.3202, + "margin_dpo/margin_mean": 0.6882905960083008, + "margin_dpo/margin_std": 0.776940643787384, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.12227249145507812, + "KL/mean": -0.19456741213798523, + "KL/rejected_KL_mean": -0.5114059448242188, + "KL/std": 0.5843777656555176, + "epoch": 0.048458149779735685, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6336773633956909, + "fcm_dpo/q_t": 0.48418158292770386, + "grad_norm": 71.57157135009766, + "learning_rate": 2.318840579710145e-07, + "logits/chosen": -0.4876636266708374, + "logits/rejected": -0.46141698956489563, + "logps/chosen": -53.138702392578125, + "logps/ref_chosen": -53.26097106933594, + "logps/ref_rejected": -87.8851318359375, + "logps/rejected": -88.39654541015625, + "loss": 1.325, + "margin_dpo/margin_mean": 0.6336771249771118, + "margin_dpo/margin_std": 0.6494045257568359, + "step": 33 + }, + { + "KL/chosen_KL_mean": 0.09919548034667969, + "KL/mean": -0.3045317530632019, + "KL/rejected_KL_mean": -0.7082595825195312, + "KL/std": 0.7005865573883057, + "epoch": 0.049926578560939794, + "fcm_dpo/beta": 0.10320889204740524, + "fcm_dpo/delta": 0.1579238921403885, + "fcm_dpo/margin": 0.8074537515640259, + "fcm_dpo/q_t": 0.4795566201210022, + "grad_norm": 81.28727722167969, + "learning_rate": 2.391304347826087e-07, + "logits/chosen": -0.525603175163269, + "logits/rejected": -0.5081762671470642, + "logps/chosen": -50.71813201904297, + "logps/ref_chosen": -50.81732940673828, + "logps/ref_rejected": -101.92184448242188, + "logps/rejected": -102.6301040649414, + "loss": 1.3065, + "margin_dpo/margin_mean": 0.8074527382850647, + "margin_dpo/margin_std": 0.8205959796905518, + "step": 34 + }, + { + "KL/chosen_KL_mean": 0.16864395141601562, + "KL/mean": -0.4522685408592224, + "KL/rejected_KL_mean": -1.0731773376464844, + "KL/std": 0.9934526681900024, + "epoch": 0.0513950073421439, + "fcm_dpo/beta": 0.10746243596076965, + "fcm_dpo/delta": 0.2701945900917053, + "fcm_dpo/margin": 1.2418255805969238, + "fcm_dpo/q_t": 0.4677559733390808, + "grad_norm": 87.83883666992188, + "learning_rate": 2.463768115942029e-07, + "logits/chosen": -0.5124801397323608, + "logits/rejected": -0.47577357292175293, + "logps/chosen": -50.8558464050293, + "logps/ref_chosen": -51.02449035644531, + "logps/ref_rejected": -106.82443237304688, + "logps/rejected": -107.8976058959961, + "loss": 1.2608, + "margin_dpo/margin_mean": 1.2418255805969238, + "margin_dpo/margin_std": 1.0855079889297485, + "step": 35 + }, + { + "KL/chosen_KL_mean": 0.013975143432617188, + "KL/mean": -0.5303745865821838, + "KL/rejected_KL_mean": -1.0747222900390625, + "KL/std": 1.0487756729125977, + "epoch": 0.05286343612334802, + "fcm_dpo/beta": 0.11381839960813522, + "fcm_dpo/delta": 0.27891629934310913, + "fcm_dpo/margin": 1.0886973142623901, + "fcm_dpo/q_t": 0.46992403268814087, + "grad_norm": 81.70454406738281, + "learning_rate": 2.536231884057971e-07, + "logits/chosen": -0.5460728406906128, + "logits/rejected": -0.5091354846954346, + "logps/chosen": -51.977516174316406, + "logps/ref_chosen": -51.991493225097656, + "logps/ref_rejected": -86.0406265258789, + "logps/rejected": -87.11534881591797, + "loss": 1.2707, + "margin_dpo/margin_mean": 1.0886969566345215, + "margin_dpo/margin_std": 1.171996831893921, + "step": 36 + }, + { + "KL/chosen_KL_mean": 0.023801803588867188, + "KL/mean": -0.5040315985679626, + "KL/rejected_KL_mean": -1.0318717956542969, + "KL/std": 1.0373797416687012, + "epoch": 0.05433186490455213, + "fcm_dpo/beta": 0.1198909655213356, + "fcm_dpo/delta": 0.2773377299308777, + "fcm_dpo/margin": 1.0556702613830566, + "fcm_dpo/q_t": 0.46964770555496216, + "grad_norm": 72.01265716552734, + "learning_rate": 2.6086956521739126e-07, + "logits/chosen": -0.5176362991333008, + "logits/rejected": -0.4745715260505676, + "logps/chosen": -62.78330612182617, + "logps/ref_chosen": -62.807106018066406, + "logps/ref_rejected": -77.89507293701172, + "logps/rejected": -78.92694091796875, + "loss": 1.2708, + "margin_dpo/margin_mean": 1.0556700229644775, + "margin_dpo/margin_std": 1.3116538524627686, + "step": 37 + }, + { + "KL/chosen_KL_mean": 0.17965316772460938, + "KL/mean": -0.5454572439193726, + "KL/rejected_KL_mean": -1.2705650329589844, + "KL/std": 1.3868043422698975, + "epoch": 0.055800293685756244, + "fcm_dpo/beta": 0.12598319351673126, + "fcm_dpo/delta": 0.22110190987586975, + "fcm_dpo/margin": 1.4502217769622803, + "fcm_dpo/q_t": 0.45613712072372437, + "grad_norm": 84.13441467285156, + "learning_rate": 2.681159420289855e-07, + "logits/chosen": -0.5286298394203186, + "logits/rejected": -0.49596402049064636, + "logps/chosen": -48.21086883544922, + "logps/ref_chosen": -48.39051818847656, + "logps/ref_rejected": -97.91244506835938, + "logps/rejected": -99.1830062866211, + "loss": 1.2227, + "margin_dpo/margin_mean": 1.4502220153808594, + "margin_dpo/margin_std": 1.6653861999511719, + "step": 38 + }, + { + "KL/chosen_KL_mean": 0.07242012023925781, + "KL/mean": -0.7552523016929626, + "KL/rejected_KL_mean": -1.5829238891601562, + "KL/std": 1.3359938859939575, + "epoch": 0.05726872246696035, + "fcm_dpo/beta": 0.13071495294570923, + "fcm_dpo/delta": 0.18790489435195923, + "fcm_dpo/margin": 1.6553417444229126, + "fcm_dpo/q_t": 0.44766077399253845, + "grad_norm": 111.4832534790039, + "learning_rate": 2.753623188405797e-07, + "logits/chosen": -0.5785598754882812, + "logits/rejected": -0.5401351451873779, + "logps/chosen": -50.67805480957031, + "logps/ref_chosen": -50.75047302246094, + "logps/ref_rejected": -78.56951141357422, + "logps/rejected": -80.15243530273438, + "loss": 1.1908, + "margin_dpo/margin_mean": 1.6553418636322021, + "margin_dpo/margin_std": 1.4348034858703613, + "step": 39 + }, + { + "KL/chosen_KL_mean": 0.14400482177734375, + "KL/mean": -0.60335373878479, + "KL/rejected_KL_mean": -1.3507156372070312, + "KL/std": 1.3741190433502197, + "epoch": 0.05873715124816446, + "fcm_dpo/beta": 0.13639971613883972, + "fcm_dpo/delta": 0.19976407289505005, + "fcm_dpo/margin": 1.494720220565796, + "fcm_dpo/q_t": 0.45102059841156006, + "grad_norm": 78.08142852783203, + "learning_rate": 2.8260869565217386e-07, + "logits/chosen": -0.5232949256896973, + "logits/rejected": -0.4935886859893799, + "logps/chosen": -57.841064453125, + "logps/ref_chosen": -57.985069274902344, + "logps/ref_rejected": -74.3000717163086, + "logps/rejected": -75.65078735351562, + "loss": 1.2059, + "margin_dpo/margin_mean": 1.4947205781936646, + "margin_dpo/margin_std": 1.6571329832077026, + "step": 40 + }, + { + "KL/chosen_KL_mean": 0.04059600830078125, + "KL/mean": -0.9012515544891357, + "KL/rejected_KL_mean": -1.8430938720703125, + "KL/std": 1.8287923336029053, + "epoch": 0.06020558002936858, + "fcm_dpo/beta": 0.1405719816684723, + "fcm_dpo/delta": 0.1386878788471222, + "fcm_dpo/margin": 1.8836982250213623, + "fcm_dpo/q_t": 0.43659090995788574, + "grad_norm": 89.06941223144531, + "learning_rate": 2.898550724637681e-07, + "logits/chosen": -0.5429688692092896, + "logits/rejected": -0.5066103339195251, + "logps/chosen": -62.65522003173828, + "logps/ref_chosen": -62.69581604003906, + "logps/ref_rejected": -97.02352905273438, + "logps/rejected": -98.86662292480469, + "loss": 1.1586, + "margin_dpo/margin_mean": 1.8836987018585205, + "margin_dpo/margin_std": 1.968984842300415, + "step": 41 + }, + { + "KL/chosen_KL_mean": 0.21233558654785156, + "KL/mean": -1.0596989393234253, + "KL/rejected_KL_mean": -2.3317337036132812, + "KL/std": 2.1389851570129395, + "epoch": 0.06167400881057269, + "fcm_dpo/beta": 0.14226916432380676, + "fcm_dpo/delta": 0.03947313129901886, + "fcm_dpo/margin": 2.5440683364868164, + "fcm_dpo/q_t": 0.4139750599861145, + "grad_norm": 101.21937561035156, + "learning_rate": 2.971014492753623e-07, + "logits/chosen": -0.5372546315193176, + "logits/rejected": -0.49066978693008423, + "logps/chosen": -58.75408935546875, + "logps/ref_chosen": -58.966426849365234, + "logps/ref_rejected": -109.90837097167969, + "logps/rejected": -112.2401123046875, + "loss": 1.084, + "margin_dpo/margin_mean": 2.5440685749053955, + "margin_dpo/margin_std": 2.3744425773620605, + "step": 42 + }, + { + "KL/chosen_KL_mean": 0.5405921936035156, + "KL/mean": -0.6949355602264404, + "KL/rejected_KL_mean": -1.9304618835449219, + "KL/std": 1.8301477432250977, + "epoch": 0.0631424375917768, + "fcm_dpo/beta": 0.14327695965766907, + "fcm_dpo/delta": 0.047662679105997086, + "fcm_dpo/margin": 2.4710512161254883, + "fcm_dpo/q_t": 0.4148353338241577, + "grad_norm": 93.92181396484375, + "learning_rate": 3.043478260869565e-07, + "logits/chosen": -0.5310695171356201, + "logits/rejected": -0.5056588649749756, + "logps/chosen": -53.61540222167969, + "logps/ref_chosen": -54.15599822998047, + "logps/ref_rejected": -96.48019409179688, + "logps/rejected": -98.41065216064453, + "loss": 1.0826, + "margin_dpo/margin_mean": 2.47105073928833, + "margin_dpo/margin_std": 1.9682235717773438, + "step": 43 + }, + { + "KL/chosen_KL_mean": 0.2531108856201172, + "KL/mean": -1.1532858610153198, + "KL/rejected_KL_mean": -2.559680938720703, + "KL/std": 2.2799549102783203, + "epoch": 0.06461086637298091, + "fcm_dpo/beta": 0.14362391829490662, + "fcm_dpo/delta": -0.004183035343885422, + "fcm_dpo/margin": 2.8127894401550293, + "fcm_dpo/q_t": 0.4032197594642639, + "grad_norm": 101.86164855957031, + "learning_rate": 3.115942028985507e-07, + "logits/chosen": -0.4643116593360901, + "logits/rejected": -0.4446447789669037, + "logps/chosen": -49.82538604736328, + "logps/ref_chosen": -50.07849884033203, + "logps/ref_rejected": -108.78376007080078, + "logps/rejected": -111.34344482421875, + "loss": 1.0471, + "margin_dpo/margin_mean": 2.812788963317871, + "margin_dpo/margin_std": 2.241828203201294, + "step": 44 + }, + { + "KL/chosen_KL_mean": 0.11113929748535156, + "KL/mean": -0.9870209693908691, + "KL/rejected_KL_mean": -2.0851821899414062, + "KL/std": 2.0045013427734375, + "epoch": 0.06607929515418502, + "fcm_dpo/beta": 0.14574244618415833, + "fcm_dpo/delta": 0.08238682150840759, + "fcm_dpo/margin": 2.1963233947753906, + "fcm_dpo/q_t": 0.4246715307235718, + "grad_norm": 82.72820281982422, + "learning_rate": 3.188405797101449e-07, + "logits/chosen": -0.4895831346511841, + "logits/rejected": -0.47717034816741943, + "logps/chosen": -48.30378723144531, + "logps/ref_chosen": -48.4149284362793, + "logps/ref_rejected": -77.93643188476562, + "logps/rejected": -80.0216064453125, + "loss": 1.1231, + "margin_dpo/margin_mean": 2.1963229179382324, + "margin_dpo/margin_std": 2.4882490634918213, + "step": 45 + }, + { + "KL/chosen_KL_mean": 0.18288421630859375, + "KL/mean": -1.276892900466919, + "KL/rejected_KL_mean": -2.736675262451172, + "KL/std": 2.590786933898926, + "epoch": 0.06754772393538913, + "fcm_dpo/beta": 0.14583346247673035, + "fcm_dpo/delta": -0.026936061680316925, + "fcm_dpo/margin": 2.9195570945739746, + "fcm_dpo/q_t": 0.4010394215583801, + "grad_norm": 87.75068664550781, + "learning_rate": 3.260869565217391e-07, + "logits/chosen": -0.5153135657310486, + "logits/rejected": -0.463581919670105, + "logps/chosen": -55.81654357910156, + "logps/ref_chosen": -55.999427795410156, + "logps/ref_rejected": -95.652587890625, + "logps/rejected": -98.38926696777344, + "loss": 1.0532, + "margin_dpo/margin_mean": 2.9195570945739746, + "margin_dpo/margin_std": 3.142143726348877, + "step": 46 + }, + { + "KL/chosen_KL_mean": 0.387847900390625, + "KL/mean": -1.0973193645477295, + "KL/rejected_KL_mean": -2.582489013671875, + "KL/std": 2.5982861518859863, + "epoch": 0.06901615271659324, + "fcm_dpo/beta": 0.14528387784957886, + "fcm_dpo/delta": -0.03321390599012375, + "fcm_dpo/margin": 2.970334529876709, + "fcm_dpo/q_t": 0.39802086353302, + "grad_norm": 84.18961334228516, + "learning_rate": 3.333333333333333e-07, + "logits/chosen": -0.5865252614021301, + "logits/rejected": -0.5347657799720764, + "logps/chosen": -57.538230895996094, + "logps/ref_chosen": -57.92607879638672, + "logps/ref_rejected": -94.67920684814453, + "logps/rejected": -97.2616958618164, + "loss": 1.0386, + "margin_dpo/margin_mean": 2.9703352451324463, + "margin_dpo/margin_std": 2.69755482673645, + "step": 47 + }, + { + "KL/chosen_KL_mean": 0.10560417175292969, + "KL/mean": -1.441537857055664, + "KL/rejected_KL_mean": -2.988677978515625, + "KL/std": 2.643078565597534, + "epoch": 0.07048458149779736, + "fcm_dpo/beta": 0.14288130402565002, + "fcm_dpo/delta": -0.04455402493476868, + "fcm_dpo/margin": 3.0942819118499756, + "fcm_dpo/q_t": 0.39582157135009766, + "grad_norm": 93.56952667236328, + "learning_rate": 3.4057971014492755e-07, + "logits/chosen": -0.5885308980941772, + "logits/rejected": -0.5303641557693481, + "logps/chosen": -57.08246994018555, + "logps/ref_chosen": -57.188072204589844, + "logps/ref_rejected": -88.0166015625, + "logps/rejected": -91.00527954101562, + "loss": 1.0395, + "margin_dpo/margin_mean": 3.0942819118499756, + "margin_dpo/margin_std": 2.9218857288360596, + "step": 48 + }, + { + "KL/chosen_KL_mean": 0.33449745178222656, + "KL/mean": -1.5316673517227173, + "KL/rejected_KL_mean": -3.3978309631347656, + "KL/std": 3.353222370147705, + "epoch": 0.07195301027900147, + "fcm_dpo/beta": 0.1409154236316681, + "fcm_dpo/delta": -0.13273698091506958, + "fcm_dpo/margin": 3.732323169708252, + "fcm_dpo/q_t": 0.38071030378341675, + "grad_norm": 92.1107177734375, + "learning_rate": 3.478260869565217e-07, + "logits/chosen": -0.5483835935592651, + "logits/rejected": -0.4899318218231201, + "logps/chosen": -61.350772857666016, + "logps/ref_chosen": -61.685272216796875, + "logps/ref_rejected": -83.76747131347656, + "logps/rejected": -87.1653060913086, + "loss": 0.9959, + "margin_dpo/margin_mean": 3.7323226928710938, + "margin_dpo/margin_std": 3.8772082328796387, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.16253089904785156, + "KL/mean": -2.2148306369781494, + "KL/rejected_KL_mean": -4.267131805419922, + "KL/std": 3.460838794708252, + "epoch": 0.07342143906020558, + "fcm_dpo/beta": 0.13572102785110474, + "fcm_dpo/delta": -0.16670596599578857, + "fcm_dpo/margin": 4.104592800140381, + "fcm_dpo/q_t": 0.371456503868103, + "grad_norm": 75.59302520751953, + "learning_rate": 3.5507246376811595e-07, + "logits/chosen": -0.5448007583618164, + "logits/rejected": -0.5087927579879761, + "logps/chosen": -58.88666915893555, + "logps/ref_chosen": -58.72413635253906, + "logps/ref_rejected": -96.35814666748047, + "logps/rejected": -100.62527465820312, + "loss": 0.9677, + "margin_dpo/margin_mean": 4.104592800140381, + "margin_dpo/margin_std": 3.9096031188964844, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.28087806701660156, + "KL/mean": -2.270071506500244, + "KL/rejected_KL_mean": -4.259269714355469, + "KL/std": 4.135315895080566, + "epoch": 0.07488986784140969, + "fcm_dpo/beta": 0.1313726007938385, + "fcm_dpo/delta": -0.1310398280620575, + "fcm_dpo/margin": 3.978391408920288, + "fcm_dpo/q_t": 0.3860105872154236, + "grad_norm": 61.80961990356445, + "learning_rate": 3.6231884057971015e-07, + "logits/chosen": -0.5152701735496521, + "logits/rejected": -0.4818602204322815, + "logps/chosen": -61.65454864501953, + "logps/ref_chosen": -61.3736686706543, + "logps/ref_rejected": -76.00199890136719, + "logps/rejected": -80.26127624511719, + "loss": 1.0265, + "margin_dpo/margin_mean": 3.978390693664551, + "margin_dpo/margin_std": 5.0646772384643555, + "step": 51 + }, + { + "KL/chosen_KL_mean": 0.36811256408691406, + "KL/mean": -2.65940523147583, + "KL/rejected_KL_mean": -5.686923980712891, + "KL/std": 4.684706687927246, + "epoch": 0.0763582966226138, + "fcm_dpo/beta": 0.12406842410564423, + "fcm_dpo/delta": -0.3800222873687744, + "fcm_dpo/margin": 6.0550360679626465, + "fcm_dpo/q_t": 0.33053886890411377, + "grad_norm": 65.30122375488281, + "learning_rate": 3.695652173913043e-07, + "logits/chosen": -0.5419540405273438, + "logits/rejected": -0.48573756217956543, + "logps/chosen": -51.96924591064453, + "logps/ref_chosen": -52.33735656738281, + "logps/ref_rejected": -79.97391510009766, + "logps/rejected": -85.66084289550781, + "loss": 0.8506, + "margin_dpo/margin_mean": 6.055036544799805, + "margin_dpo/margin_std": 4.9989800453186035, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.18677711486816406, + "KL/mean": -3.213822841644287, + "KL/rejected_KL_mean": -6.240867614746094, + "KL/std": 5.253150939941406, + "epoch": 0.07782672540381791, + "fcm_dpo/beta": 0.11692243814468384, + "fcm_dpo/delta": -0.3300025165081024, + "fcm_dpo/margin": 6.054093360900879, + "fcm_dpo/q_t": 0.34951359033584595, + "grad_norm": 64.69686126708984, + "learning_rate": 3.7681159420289855e-07, + "logits/chosen": -0.6237785816192627, + "logits/rejected": -0.6024104356765747, + "logps/chosen": -53.501426696777344, + "logps/ref_chosen": -53.31465148925781, + "logps/ref_rejected": -91.78359985351562, + "logps/rejected": -98.02445983886719, + "loss": 0.9121, + "margin_dpo/margin_mean": 6.054093360900879, + "margin_dpo/margin_std": 6.0293989181518555, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.3544273376464844, + "KL/mean": -3.032489538192749, + "KL/rejected_KL_mean": -5.710548400878906, + "KL/std": 5.030124664306641, + "epoch": 0.07929515418502203, + "fcm_dpo/beta": 0.1107134222984314, + "fcm_dpo/delta": -0.2048681676387787, + "fcm_dpo/margin": 5.356122970581055, + "fcm_dpo/q_t": 0.36526209115982056, + "grad_norm": 58.66542053222656, + "learning_rate": 3.8405797101449274e-07, + "logits/chosen": -0.5997574329376221, + "logits/rejected": -0.5465754270553589, + "logps/chosen": -51.0430908203125, + "logps/ref_chosen": -50.68865966796875, + "logps/ref_rejected": -91.71539306640625, + "logps/rejected": -97.42594146728516, + "loss": 0.9456, + "margin_dpo/margin_mean": 5.3561224937438965, + "margin_dpo/margin_std": 5.040410995483398, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.8778724670410156, + "KL/mean": -4.0574259757995605, + "KL/rejected_KL_mean": -7.236972808837891, + "KL/std": 6.0431365966796875, + "epoch": 0.08076358296622614, + "fcm_dpo/beta": 0.1051725447177887, + "fcm_dpo/delta": -0.2873228192329407, + "fcm_dpo/margin": 6.359100341796875, + "fcm_dpo/q_t": 0.3560294508934021, + "grad_norm": 55.624202728271484, + "learning_rate": 3.9130434782608694e-07, + "logits/chosen": -0.6527860760688782, + "logits/rejected": -0.5910372734069824, + "logps/chosen": -63.493106842041016, + "logps/ref_chosen": -62.615234375, + "logps/ref_rejected": -88.99349975585938, + "logps/rejected": -96.23047637939453, + "loss": 0.9494, + "margin_dpo/margin_mean": 6.359099388122559, + "margin_dpo/margin_std": 7.310431480407715, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.6478157043457031, + "KL/mean": -3.6710574626922607, + "KL/rejected_KL_mean": -6.694305419921875, + "KL/std": 5.903786659240723, + "epoch": 0.08223201174743025, + "fcm_dpo/beta": 0.10002656280994415, + "fcm_dpo/delta": -0.21787334978580475, + "fcm_dpo/margin": 6.0464887619018555, + "fcm_dpo/q_t": 0.3679868280887604, + "grad_norm": 50.561851501464844, + "learning_rate": 3.9855072463768114e-07, + "logits/chosen": -0.6278807520866394, + "logits/rejected": -0.586266040802002, + "logps/chosen": -58.580543518066406, + "logps/ref_chosen": -57.9327278137207, + "logps/ref_rejected": -94.1744384765625, + "logps/rejected": -100.86874389648438, + "loss": 0.9811, + "margin_dpo/margin_mean": 6.0464887619018555, + "margin_dpo/margin_std": 7.142303466796875, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.682220458984375, + "KL/mean": -4.073773384094238, + "KL/rejected_KL_mean": -7.46533203125, + "KL/std": 5.824618339538574, + "epoch": 0.08370044052863436, + "fcm_dpo/beta": 0.0954844132065773, + "fcm_dpo/delta": -0.2640697658061981, + "fcm_dpo/margin": 6.783110618591309, + "fcm_dpo/q_t": 0.3528316020965576, + "grad_norm": 53.0193977355957, + "learning_rate": 4.057971014492754e-07, + "logits/chosen": -0.5914736986160278, + "logits/rejected": -0.563991129398346, + "logps/chosen": -71.17750549316406, + "logps/ref_chosen": -70.49528503417969, + "logps/ref_rejected": -95.56546020507812, + "logps/rejected": -103.03079223632812, + "loss": 0.9247, + "margin_dpo/margin_mean": 6.78311014175415, + "margin_dpo/margin_std": 6.405632972717285, + "step": 57 + }, + { + "KL/chosen_KL_mean": -0.9351482391357422, + "KL/mean": -4.74384069442749, + "KL/rejected_KL_mean": -8.552532196044922, + "KL/std": 6.671572208404541, + "epoch": 0.08516886930983847, + "fcm_dpo/beta": 0.0899924710392952, + "fcm_dpo/delta": -0.3056134581565857, + "fcm_dpo/margin": 7.6173810958862305, + "fcm_dpo/q_t": 0.3502853810787201, + "grad_norm": 54.50084686279297, + "learning_rate": 4.1304347826086954e-07, + "logits/chosen": -0.6031397581100464, + "logits/rejected": -0.5278700590133667, + "logps/chosen": -63.06808853149414, + "logps/ref_chosen": -62.13294219970703, + "logps/ref_rejected": -84.61729431152344, + "logps/rejected": -93.16983032226562, + "loss": 0.9236, + "margin_dpo/margin_mean": 7.617381572723389, + "margin_dpo/margin_std": 7.944563865661621, + "step": 58 + }, + { + "KL/chosen_KL_mean": -1.396148681640625, + "KL/mean": -5.524988174438477, + "KL/rejected_KL_mean": -9.653823852539062, + "KL/std": 7.2611308097839355, + "epoch": 0.08663729809104258, + "fcm_dpo/beta": 0.08342745900154114, + "fcm_dpo/delta": -0.3148040175437927, + "fcm_dpo/margin": 8.257675170898438, + "fcm_dpo/q_t": 0.3485872745513916, + "grad_norm": 51.20856475830078, + "learning_rate": 4.2028985507246374e-07, + "logits/chosen": -0.6402565836906433, + "logits/rejected": -0.6007430553436279, + "logps/chosen": -53.32867431640625, + "logps/ref_chosen": -51.932525634765625, + "logps/ref_rejected": -88.88520050048828, + "logps/rejected": -98.53902435302734, + "loss": 0.9173, + "margin_dpo/margin_mean": 8.257675170898438, + "margin_dpo/margin_std": 8.405501365661621, + "step": 59 + }, + { + "KL/chosen_KL_mean": -2.5095481872558594, + "KL/mean": -5.817296028137207, + "KL/rejected_KL_mean": -9.12503433227539, + "KL/std": 6.605817794799805, + "epoch": 0.0881057268722467, + "fcm_dpo/beta": 0.08078856766223907, + "fcm_dpo/delta": -0.14282643795013428, + "fcm_dpo/margin": 6.615485191345215, + "fcm_dpo/q_t": 0.37698274850845337, + "grad_norm": 54.96247100830078, + "learning_rate": 4.2753623188405794e-07, + "logits/chosen": -0.6214213371276855, + "logits/rejected": -0.5622453093528748, + "logps/chosen": -63.45173645019531, + "logps/ref_chosen": -60.94218826293945, + "logps/ref_rejected": -85.39340209960938, + "logps/rejected": -94.5184326171875, + "loss": 1.0065, + "margin_dpo/margin_mean": 6.615485191345215, + "margin_dpo/margin_std": 7.516720771789551, + "step": 60 + }, + { + "KL/chosen_KL_mean": -1.354644775390625, + "KL/mean": -5.456735610961914, + "KL/rejected_KL_mean": -9.558830261230469, + "KL/std": 8.855253219604492, + "epoch": 0.08957415565345081, + "fcm_dpo/beta": 0.07721322774887085, + "fcm_dpo/delta": -0.25074315071105957, + "fcm_dpo/margin": 8.20418930053711, + "fcm_dpo/q_t": 0.36832770705223083, + "grad_norm": 45.67434310913086, + "learning_rate": 4.3478260869565214e-07, + "logits/chosen": -0.597309947013855, + "logits/rejected": -0.5627981424331665, + "logps/chosen": -61.98816680908203, + "logps/ref_chosen": -60.633522033691406, + "logps/ref_rejected": -89.85249328613281, + "logps/rejected": -99.41133117675781, + "loss": 0.9848, + "margin_dpo/margin_mean": 8.20418930053711, + "margin_dpo/margin_std": 10.933375358581543, + "step": 61 + }, + { + "KL/chosen_KL_mean": -1.5534076690673828, + "KL/mean": -4.591529369354248, + "KL/rejected_KL_mean": -7.629646301269531, + "KL/std": 6.465494155883789, + "epoch": 0.09104258443465492, + "fcm_dpo/beta": 0.07611910998821259, + "fcm_dpo/delta": -0.06562487781047821, + "fcm_dpo/margin": 6.076238632202148, + "fcm_dpo/q_t": 0.39654144644737244, + "grad_norm": 44.995994567871094, + "learning_rate": 4.420289855072464e-07, + "logits/chosen": -0.6019065380096436, + "logits/rejected": -0.5675424337387085, + "logps/chosen": -57.70417785644531, + "logps/ref_chosen": -56.15077209472656, + "logps/ref_rejected": -75.56619262695312, + "logps/rejected": -83.19584655761719, + "loss": 1.0601, + "margin_dpo/margin_mean": 6.076239109039307, + "margin_dpo/margin_std": 7.934353351593018, + "step": 62 + }, + { + "KL/chosen_KL_mean": -2.580839157104492, + "KL/mean": -6.7052154541015625, + "KL/rejected_KL_mean": -10.82958984375, + "KL/std": 8.269981384277344, + "epoch": 0.09251101321585903, + "fcm_dpo/beta": 0.07275725901126862, + "fcm_dpo/delta": -0.21487507224082947, + "fcm_dpo/margin": 8.248749732971191, + "fcm_dpo/q_t": 0.3637925684452057, + "grad_norm": 45.84245681762695, + "learning_rate": 4.4927536231884053e-07, + "logits/chosen": -0.6152663826942444, + "logits/rejected": -0.5709649920463562, + "logps/chosen": -75.72823333740234, + "logps/ref_chosen": -73.14739227294922, + "logps/ref_rejected": -97.61006164550781, + "logps/rejected": -108.43965148925781, + "loss": 0.9641, + "margin_dpo/margin_mean": 8.248749732971191, + "margin_dpo/margin_std": 8.703377723693848, + "step": 63 + }, + { + "KL/chosen_KL_mean": -0.9510021209716797, + "KL/mean": -5.819749355316162, + "KL/rejected_KL_mean": -10.688499450683594, + "KL/std": 8.974802017211914, + "epoch": 0.09397944199706314, + "fcm_dpo/beta": 0.06858538091182709, + "fcm_dpo/delta": -0.2923140823841095, + "fcm_dpo/margin": 9.737493515014648, + "fcm_dpo/q_t": 0.3547326326370239, + "grad_norm": 42.15247344970703, + "learning_rate": 4.5652173913043473e-07, + "logits/chosen": -0.5690383911132812, + "logits/rejected": -0.5363543033599854, + "logps/chosen": -54.94960021972656, + "logps/ref_chosen": -53.998600006103516, + "logps/ref_rejected": -93.53019714355469, + "logps/rejected": -104.21870422363281, + "loss": 0.9457, + "margin_dpo/margin_mean": 9.737493515014648, + "margin_dpo/margin_std": 11.002016067504883, + "step": 64 + }, + { + "KL/chosen_KL_mean": -3.15191650390625, + "KL/mean": -8.072813034057617, + "KL/rejected_KL_mean": -12.99371337890625, + "KL/std": 10.051557540893555, + "epoch": 0.09544787077826726, + "fcm_dpo/beta": 0.06531103700399399, + "fcm_dpo/delta": -0.2614961266517639, + "fcm_dpo/margin": 9.841796875, + "fcm_dpo/q_t": 0.3564883768558502, + "grad_norm": 42.087825775146484, + "learning_rate": 4.63768115942029e-07, + "logits/chosen": -0.6592050790786743, + "logits/rejected": -0.646178662776947, + "logps/chosen": -67.9879150390625, + "logps/ref_chosen": -64.83599853515625, + "logps/ref_rejected": -109.94645690917969, + "logps/rejected": -122.94017028808594, + "loss": 0.9588, + "margin_dpo/margin_mean": 9.841795921325684, + "margin_dpo/margin_std": 11.036256790161133, + "step": 65 + }, + { + "KL/chosen_KL_mean": -2.923826217651367, + "KL/mean": -7.444736480712891, + "KL/rejected_KL_mean": -11.965641021728516, + "KL/std": 9.111772537231445, + "epoch": 0.09691629955947137, + "fcm_dpo/beta": 0.06262894719839096, + "fcm_dpo/delta": -0.1786663979291916, + "fcm_dpo/margin": 9.041810989379883, + "fcm_dpo/q_t": 0.3762148320674896, + "grad_norm": 37.77988815307617, + "learning_rate": 4.7101449275362313e-07, + "logits/chosen": -0.6611512899398804, + "logits/rejected": -0.6294161081314087, + "logps/chosen": -54.36735153198242, + "logps/ref_chosen": -51.44352722167969, + "logps/ref_rejected": -75.63629913330078, + "logps/rejected": -87.60194396972656, + "loss": 0.9981, + "margin_dpo/margin_mean": 9.041810989379883, + "margin_dpo/margin_std": 11.0140380859375, + "step": 66 + }, + { + "KL/chosen_KL_mean": -2.5404319763183594, + "KL/mean": -7.237859725952148, + "KL/rejected_KL_mean": -11.935287475585938, + "KL/std": 9.200343132019043, + "epoch": 0.09838472834067548, + "fcm_dpo/beta": 0.061006784439086914, + "fcm_dpo/delta": -0.1833253800868988, + "fcm_dpo/margin": 9.394853591918945, + "fcm_dpo/q_t": 0.3741205930709839, + "grad_norm": 37.25181579589844, + "learning_rate": 4.782608695652174e-07, + "logits/chosen": -0.5752357244491577, + "logits/rejected": -0.5304913520812988, + "logps/chosen": -61.8812370300293, + "logps/ref_chosen": -59.34080505371094, + "logps/ref_rejected": -72.78728485107422, + "logps/rejected": -84.72257232666016, + "loss": 0.991, + "margin_dpo/margin_mean": 9.394853591918945, + "margin_dpo/margin_std": 11.157440185546875, + "step": 67 + }, + { + "KL/chosen_KL_mean": -3.007139205932617, + "KL/mean": -7.437971115112305, + "KL/rejected_KL_mean": -11.86880111694336, + "KL/std": 8.222919464111328, + "epoch": 0.09985315712187959, + "fcm_dpo/beta": 0.05923429876565933, + "fcm_dpo/delta": -0.13163843750953674, + "fcm_dpo/margin": 8.861660957336426, + "fcm_dpo/q_t": 0.3803149461746216, + "grad_norm": 37.139434814453125, + "learning_rate": 4.855072463768116e-07, + "logits/chosen": -0.6381473541259766, + "logits/rejected": -0.5780563354492188, + "logps/chosen": -68.21297454833984, + "logps/ref_chosen": -65.2058334350586, + "logps/ref_rejected": -77.20724487304688, + "logps/rejected": -89.0760498046875, + "loss": 0.9965, + "margin_dpo/margin_mean": 8.86166000366211, + "margin_dpo/margin_std": 9.258892059326172, + "step": 68 + }, + { + "KL/chosen_KL_mean": -3.585367202758789, + "KL/mean": -9.017387390136719, + "KL/rejected_KL_mean": -14.449409484863281, + "KL/std": 9.787298202514648, + "epoch": 0.1013215859030837, + "fcm_dpo/beta": 0.056480493396520615, + "fcm_dpo/delta": -0.22901439666748047, + "fcm_dpo/margin": 10.864038467407227, + "fcm_dpo/q_t": 0.3623312711715698, + "grad_norm": 39.13850402832031, + "learning_rate": 4.927536231884058e-07, + "logits/chosen": -0.6019381284713745, + "logits/rejected": -0.5787031650543213, + "logps/chosen": -63.40460968017578, + "logps/ref_chosen": -59.81924057006836, + "logps/ref_rejected": -103.38886260986328, + "logps/rejected": -117.83827209472656, + "loss": 0.9433, + "margin_dpo/margin_mean": 10.864038467407227, + "margin_dpo/margin_std": 10.782807350158691, + "step": 69 + }, + { + "KL/chosen_KL_mean": -5.221946716308594, + "KL/mean": -11.105976104736328, + "KL/rejected_KL_mean": -16.990005493164062, + "KL/std": 12.004703521728516, + "epoch": 0.1027900146842878, + "fcm_dpo/beta": 0.054179318249225616, + "fcm_dpo/delta": -0.2531682252883911, + "fcm_dpo/margin": 11.768058776855469, + "fcm_dpo/q_t": 0.3599928617477417, + "grad_norm": 40.161224365234375, + "learning_rate": 5e-07, + "logits/chosen": -0.5996398329734802, + "logits/rejected": -0.5637354850769043, + "logps/chosen": -67.152587890625, + "logps/ref_chosen": -61.930641174316406, + "logps/ref_rejected": -91.06078338623047, + "logps/rejected": -108.05078887939453, + "loss": 0.951, + "margin_dpo/margin_mean": 11.768059730529785, + "margin_dpo/margin_std": 12.794807434082031, + "step": 70 + }, + { + "KL/chosen_KL_mean": -5.026617050170898, + "KL/mean": -11.98879623413086, + "KL/rejected_KL_mean": -18.95098114013672, + "KL/std": 12.658889770507812, + "epoch": 0.10425844346549193, + "fcm_dpo/beta": 0.050611525774002075, + "fcm_dpo/delta": -0.3290429711341858, + "fcm_dpo/margin": 13.924360275268555, + "fcm_dpo/q_t": 0.34701576828956604, + "grad_norm": 36.80656814575195, + "learning_rate": 4.999967061337492e-07, + "logits/chosen": -0.6452410221099854, + "logits/rejected": -0.6052732467651367, + "logps/chosen": -66.7769546508789, + "logps/ref_chosen": -61.750335693359375, + "logps/ref_rejected": -97.33662414550781, + "logps/rejected": -116.28759765625, + "loss": 0.9036, + "margin_dpo/margin_mean": 13.924360275268555, + "margin_dpo/margin_std": 13.787056922912598, + "step": 71 + }, + { + "KL/chosen_KL_mean": -5.833978652954102, + "KL/mean": -13.214212417602539, + "KL/rejected_KL_mean": -20.59444808959961, + "KL/std": 13.704126358032227, + "epoch": 0.10572687224669604, + "fcm_dpo/beta": 0.047448836266994476, + "fcm_dpo/delta": -0.3239469826221466, + "fcm_dpo/margin": 14.76047420501709, + "fcm_dpo/q_t": 0.34694284200668335, + "grad_norm": 37.849456787109375, + "learning_rate": 4.999868246217933e-07, + "logits/chosen": -0.6686690449714661, + "logits/rejected": -0.6333932280540466, + "logps/chosen": -71.88739013671875, + "logps/ref_chosen": -66.05341339111328, + "logps/ref_rejected": -95.2869873046875, + "logps/rejected": -115.88143920898438, + "loss": 0.9216, + "margin_dpo/margin_mean": 14.76047420501709, + "margin_dpo/margin_std": 15.948001861572266, + "step": 72 + }, + { + "KL/chosen_KL_mean": -7.895063400268555, + "KL/mean": -15.281990051269531, + "KL/rejected_KL_mean": -22.668926239013672, + "KL/std": 17.924169540405273, + "epoch": 0.10719530102790015, + "fcm_dpo/beta": 0.04465475678443909, + "fcm_dpo/delta": -0.27992942929267883, + "fcm_dpo/margin": 14.7738618850708, + "fcm_dpo/q_t": 0.36867302656173706, + "grad_norm": 36.75870132446289, + "learning_rate": 4.999703557245192e-07, + "logits/chosen": -0.6891875267028809, + "logits/rejected": -0.6485644578933716, + "logps/chosen": -74.15133666992188, + "logps/ref_chosen": -66.25627136230469, + "logps/ref_rejected": -90.45613098144531, + "logps/rejected": -113.12506103515625, + "loss": 1.0298, + "margin_dpo/margin_mean": 14.773862838745117, + "margin_dpo/margin_std": 22.38170623779297, + "step": 73 + }, + { + "KL/chosen_KL_mean": -8.47895622253418, + "KL/mean": -16.591014862060547, + "KL/rejected_KL_mean": -24.70307159423828, + "KL/std": 19.87729263305664, + "epoch": 0.10866372980910426, + "fcm_dpo/beta": 0.04214803874492645, + "fcm_dpo/delta": -0.30570995807647705, + "fcm_dpo/margin": 16.224117279052734, + "fcm_dpo/q_t": 0.36257338523864746, + "grad_norm": 38.15811538696289, + "learning_rate": 4.999472998758977e-07, + "logits/chosen": -0.6253733038902283, + "logits/rejected": -0.6167181730270386, + "logps/chosen": -61.903839111328125, + "logps/ref_chosen": -53.42488098144531, + "logps/ref_rejected": -95.94693756103516, + "logps/rejected": -120.65000915527344, + "loss": 0.9935, + "margin_dpo/margin_mean": 16.224117279052734, + "margin_dpo/margin_std": 24.648841857910156, + "step": 74 + }, + { + "KL/chosen_KL_mean": -8.299543380737305, + "KL/mean": -19.795719146728516, + "KL/rejected_KL_mean": -31.291885375976562, + "KL/std": 20.560977935791016, + "epoch": 0.11013215859030837, + "fcm_dpo/beta": 0.03871320188045502, + "fcm_dpo/delta": -0.5353958010673523, + "fcm_dpo/margin": 22.992351531982422, + "fcm_dpo/q_t": 0.317760705947876, + "grad_norm": 32.1464729309082, + "learning_rate": 4.999176576834721e-07, + "logits/chosen": -0.6416128873825073, + "logits/rejected": -0.6341755390167236, + "logps/chosen": -60.16120529174805, + "logps/ref_chosen": -51.861663818359375, + "logps/ref_rejected": -111.25398254394531, + "logps/rejected": -142.54586791992188, + "loss": 0.8396, + "margin_dpo/margin_mean": 22.992347717285156, + "margin_dpo/margin_std": 23.244129180908203, + "step": 75 + }, + { + "KL/chosen_KL_mean": -10.1375732421875, + "KL/mean": -17.43459701538086, + "KL/rejected_KL_mean": -24.731616973876953, + "KL/std": 16.729257583618164, + "epoch": 0.11160058737151249, + "fcm_dpo/beta": 0.036620307713747025, + "fcm_dpo/delta": -0.14195583760738373, + "fcm_dpo/margin": 14.594053268432617, + "fcm_dpo/q_t": 0.3785336911678314, + "grad_norm": 32.55298614501953, + "learning_rate": 4.998814299283415e-07, + "logits/chosen": -0.6936876773834229, + "logits/rejected": -0.6513668894767761, + "logps/chosen": -63.40361022949219, + "logps/ref_chosen": -53.26603698730469, + "logps/ref_rejected": -78.21662902832031, + "logps/rejected": -102.9482421875, + "loss": 1.0197, + "margin_dpo/margin_mean": 14.594053268432617, + "margin_dpo/margin_std": 18.254350662231445, + "step": 76 + }, + { + "KL/chosen_KL_mean": -9.1678466796875, + "KL/mean": -21.043556213378906, + "KL/rejected_KL_mean": -32.91926574707031, + "KL/std": 22.885698318481445, + "epoch": 0.1130690161527166, + "fcm_dpo/beta": 0.033799584954977036, + "fcm_dpo/delta": -0.44206297397613525, + "fcm_dpo/margin": 23.751419067382812, + "fcm_dpo/q_t": 0.32698968052864075, + "grad_norm": 33.31752395629883, + "learning_rate": 4.998386175651409e-07, + "logits/chosen": -0.6848742961883545, + "logits/rejected": -0.6468466520309448, + "logps/chosen": -67.2645263671875, + "logps/ref_chosen": -58.0966796875, + "logps/ref_rejected": -93.77361297607422, + "logps/rejected": -126.69287872314453, + "loss": 0.8878, + "margin_dpo/margin_mean": 23.751419067382812, + "margin_dpo/margin_std": 25.314167022705078, + "step": 77 + }, + { + "KL/chosen_KL_mean": -9.164955139160156, + "KL/mean": -18.119544982910156, + "KL/rejected_KL_mean": -27.074134826660156, + "KL/std": 18.446874618530273, + "epoch": 0.1145374449339207, + "fcm_dpo/beta": 0.03241837024688721, + "fcm_dpo/delta": -0.19134587049484253, + "fcm_dpo/margin": 17.909175872802734, + "fcm_dpo/q_t": 0.37250542640686035, + "grad_norm": 30.10005760192871, + "learning_rate": 4.997892217220159e-07, + "logits/chosen": -0.6356874704360962, + "logits/rejected": -0.6102886199951172, + "logps/chosen": -64.77873992919922, + "logps/ref_chosen": -55.61378479003906, + "logps/ref_rejected": -84.93436431884766, + "logps/rejected": -112.00849914550781, + "loss": 0.9917, + "margin_dpo/margin_mean": 17.909175872802734, + "margin_dpo/margin_std": 21.184471130371094, + "step": 78 + }, + { + "KL/chosen_KL_mean": -9.568305969238281, + "KL/mean": -19.395065307617188, + "KL/rejected_KL_mean": -29.221824645996094, + "KL/std": 20.979488372802734, + "epoch": 0.11600587371512482, + "fcm_dpo/beta": 0.030958333984017372, + "fcm_dpo/delta": -0.22205796837806702, + "fcm_dpo/margin": 19.65351676940918, + "fcm_dpo/q_t": 0.3720153868198395, + "grad_norm": 26.722917556762695, + "learning_rate": 4.997332437005931e-07, + "logits/chosen": -0.6731311082839966, + "logits/rejected": -0.6446127891540527, + "logps/chosen": -65.01879119873047, + "logps/ref_chosen": -55.45048522949219, + "logps/ref_rejected": -87.64756774902344, + "logps/rejected": -116.869384765625, + "loss": 1.0015, + "margin_dpo/margin_mean": 19.653514862060547, + "margin_dpo/margin_std": 25.67258071899414, + "step": 79 + }, + { + "KL/chosen_KL_mean": -12.641990661621094, + "KL/mean": -21.817380905151367, + "KL/rejected_KL_mean": -30.99276351928711, + "KL/std": 22.139293670654297, + "epoch": 0.11747430249632893, + "fcm_dpo/beta": 0.02968379482626915, + "fcm_dpo/delta": -0.155572310090065, + "fcm_dpo/margin": 18.35076904296875, + "fcm_dpo/q_t": 0.3862488269805908, + "grad_norm": 29.31393814086914, + "learning_rate": 4.996706849759452e-07, + "logits/chosen": -0.708533763885498, + "logits/rejected": -0.6655611991882324, + "logps/chosen": -71.16128540039062, + "logps/ref_chosen": -58.519290924072266, + "logps/ref_rejected": -87.54750061035156, + "logps/rejected": -118.54026794433594, + "loss": 1.0441, + "margin_dpo/margin_mean": 18.35076904296875, + "margin_dpo/margin_std": 25.68282699584961, + "step": 80 + }, + { + "KL/chosen_KL_mean": -11.637100219726562, + "KL/mean": -24.316699981689453, + "KL/rejected_KL_mean": -36.996299743652344, + "KL/std": 26.420780181884766, + "epoch": 0.11894273127753303, + "fcm_dpo/beta": 0.02805560827255249, + "fcm_dpo/delta": -0.33825811743736267, + "fcm_dpo/margin": 25.35919761657715, + "fcm_dpo/q_t": 0.351850688457489, + "grad_norm": 29.6393985748291, + "learning_rate": 4.996015471965529e-07, + "logits/chosen": -0.6757654547691345, + "logits/rejected": -0.6454110145568848, + "logps/chosen": -78.08596801757812, + "logps/ref_chosen": -66.44886779785156, + "logps/ref_rejected": -129.66270446777344, + "logps/rejected": -166.65899658203125, + "loss": 0.9463, + "margin_dpo/margin_mean": 25.35919761657715, + "margin_dpo/margin_std": 31.365787506103516, + "step": 81 + }, + { + "KL/chosen_KL_mean": -14.165742874145508, + "KL/mean": -24.178695678710938, + "KL/rejected_KL_mean": -34.19164276123047, + "KL/std": 24.08298110961914, + "epoch": 0.12041116005873716, + "fcm_dpo/beta": 0.027383368462324142, + "fcm_dpo/delta": -0.15754976868629456, + "fcm_dpo/margin": 20.025901794433594, + "fcm_dpo/q_t": 0.38632169365882874, + "grad_norm": 31.63576316833496, + "learning_rate": 4.995258321842611e-07, + "logits/chosen": -0.6542444229125977, + "logits/rejected": -0.6435472965240479, + "logps/chosen": -66.39813232421875, + "logps/ref_chosen": -52.232383728027344, + "logps/ref_rejected": -90.74325561523438, + "logps/rejected": -124.93489074707031, + "loss": 1.0894, + "margin_dpo/margin_mean": 20.025901794433594, + "margin_dpo/margin_std": 33.26299285888672, + "step": 82 + }, + { + "KL/chosen_KL_mean": -14.488626480102539, + "KL/mean": -26.423978805541992, + "KL/rejected_KL_mean": -38.35934066772461, + "KL/std": 25.066070556640625, + "epoch": 0.12187958883994127, + "fcm_dpo/beta": 0.025875981897115707, + "fcm_dpo/delta": -0.23306897282600403, + "fcm_dpo/margin": 23.870704650878906, + "fcm_dpo/q_t": 0.36692357063293457, + "grad_norm": 29.748979568481445, + "learning_rate": 4.994435419342304e-07, + "logits/chosen": -0.6859611868858337, + "logits/rejected": -0.6524330377578735, + "logps/chosen": -70.31600952148438, + "logps/ref_chosen": -55.82738494873047, + "logps/ref_rejected": -103.71589660644531, + "logps/rejected": -142.0752410888672, + "loss": 0.9878, + "margin_dpo/margin_mean": 23.870704650878906, + "margin_dpo/margin_std": 29.676780700683594, + "step": 83 + }, + { + "KL/chosen_KL_mean": -13.53270149230957, + "KL/mean": -23.642658233642578, + "KL/rejected_KL_mean": -33.75260925292969, + "KL/std": 21.36370849609375, + "epoch": 0.12334801762114538, + "fcm_dpo/beta": 0.025026695802807808, + "fcm_dpo/delta": -0.11385335773229599, + "fcm_dpo/margin": 20.219905853271484, + "fcm_dpo/q_t": 0.38283252716064453, + "grad_norm": 26.598854064941406, + "learning_rate": 4.993546786148857e-07, + "logits/chosen": -0.6387369632720947, + "logits/rejected": -0.5981898307800293, + "logps/chosen": -80.70886993408203, + "logps/ref_chosen": -67.1761703491211, + "logps/ref_rejected": -87.29859924316406, + "logps/rejected": -121.05120849609375, + "loss": 1.0144, + "margin_dpo/margin_mean": 20.21990394592285, + "margin_dpo/margin_std": 21.734418869018555, + "step": 84 + }, + { + "KL/chosen_KL_mean": -13.872922897338867, + "KL/mean": -24.366085052490234, + "KL/rejected_KL_mean": -34.85926055908203, + "KL/std": 21.958358764648438, + "epoch": 0.12481644640234948, + "fcm_dpo/beta": 0.02479904145002365, + "fcm_dpo/delta": -0.12728425860404968, + "fcm_dpo/margin": 20.98633575439453, + "fcm_dpo/q_t": 0.38485896587371826, + "grad_norm": 26.617475509643555, + "learning_rate": 4.992592445678582e-07, + "logits/chosen": -0.6399697065353394, + "logits/rejected": -0.6071797013282776, + "logps/chosen": -72.279541015625, + "logps/ref_chosen": -58.4066162109375, + "logps/ref_rejected": -78.63880157470703, + "logps/rejected": -113.49806213378906, + "loss": 1.0344, + "margin_dpo/margin_mean": 20.98633575439453, + "margin_dpo/margin_std": 26.402019500732422, + "step": 85 + }, + { + "KL/chosen_KL_mean": -18.034120559692383, + "KL/mean": -29.181171417236328, + "KL/rejected_KL_mean": -40.32822036743164, + "KL/std": 29.792917251586914, + "epoch": 0.1262848751835536, + "fcm_dpo/beta": 0.024033017456531525, + "fcm_dpo/delta": -0.14324147999286652, + "fcm_dpo/margin": 22.294099807739258, + "fcm_dpo/q_t": 0.39599549770355225, + "grad_norm": 30.38920021057129, + "learning_rate": 4.991572423079235e-07, + "logits/chosen": -0.6997466683387756, + "logits/rejected": -0.6884916424751282, + "logps/chosen": -74.17158508300781, + "logps/ref_chosen": -56.13746643066406, + "logps/ref_rejected": -88.12165069580078, + "logps/rejected": -128.4498748779297, + "loss": 1.1146, + "margin_dpo/margin_mean": 22.29410171508789, + "margin_dpo/margin_std": 41.11421203613281, + "step": 86 + }, + { + "KL/chosen_KL_mean": -17.486053466796875, + "KL/mean": -30.587915420532227, + "KL/rejected_KL_mean": -43.68976974487305, + "KL/std": 29.706398010253906, + "epoch": 0.1277533039647577, + "fcm_dpo/beta": 0.022885756567120552, + "fcm_dpo/delta": -0.21425558626651764, + "fcm_dpo/margin": 26.203716278076172, + "fcm_dpo/q_t": 0.3703950047492981, + "grad_norm": 25.73578453063965, + "learning_rate": 4.990486745229364e-07, + "logits/chosen": -0.7155952453613281, + "logits/rejected": -0.6913902163505554, + "logps/chosen": -73.12214660644531, + "logps/ref_chosen": -55.63609313964844, + "logps/ref_rejected": -95.46757507324219, + "logps/rejected": -139.1573486328125, + "loss": 1.0227, + "margin_dpo/margin_mean": 26.203716278076172, + "margin_dpo/margin_std": 36.01057052612305, + "step": 87 + }, + { + "KL/chosen_KL_mean": -22.227901458740234, + "KL/mean": -32.0677375793457, + "KL/rejected_KL_mean": -41.90757369995117, + "KL/std": 30.92605972290039, + "epoch": 0.12922173274596183, + "fcm_dpo/beta": 0.022482173517346382, + "fcm_dpo/delta": -0.0452876091003418, + "fcm_dpo/margin": 19.679677963256836, + "fcm_dpo/q_t": 0.40572047233581543, + "grad_norm": 26.90105438232422, + "learning_rate": 4.989335440737586e-07, + "logits/chosen": -0.670428991317749, + "logits/rejected": -0.6643358469009399, + "logps/chosen": -95.8990478515625, + "logps/ref_chosen": -73.67115020751953, + "logps/ref_rejected": -106.70849609375, + "logps/rejected": -148.61607360839844, + "loss": 1.1257, + "margin_dpo/margin_mean": 19.679677963256836, + "margin_dpo/margin_std": 32.507869720458984, + "step": 88 + }, + { + "KL/chosen_KL_mean": -13.619379043579102, + "KL/mean": -24.656768798828125, + "KL/rejected_KL_mean": -35.69416046142578, + "KL/std": 26.05047607421875, + "epoch": 0.13069016152716592, + "fcm_dpo/beta": 0.022331606596708298, + "fcm_dpo/delta": -0.09776569902896881, + "fcm_dpo/margin": 22.074787139892578, + "fcm_dpo/q_t": 0.3905707895755768, + "grad_norm": 24.52141571044922, + "learning_rate": 4.988118539941847e-07, + "logits/chosen": -0.7306783199310303, + "logits/rejected": -0.6993080973625183, + "logps/chosen": -74.24429321289062, + "logps/ref_chosen": -60.624916076660156, + "logps/ref_rejected": -82.08354949951172, + "logps/rejected": -117.7777099609375, + "loss": 1.0475, + "margin_dpo/margin_mean": 22.074787139892578, + "margin_dpo/margin_std": 29.647079467773438, + "step": 89 + }, + { + "KL/chosen_KL_mean": -17.485197067260742, + "KL/mean": -32.5256233215332, + "KL/rejected_KL_mean": -47.5660514831543, + "KL/std": 35.56610870361328, + "epoch": 0.13215859030837004, + "fcm_dpo/beta": 0.021488668397068977, + "fcm_dpo/delta": -0.26283973455429077, + "fcm_dpo/margin": 30.08085823059082, + "fcm_dpo/q_t": 0.37226438522338867, + "grad_norm": 26.66068458557129, + "learning_rate": 4.986836074908615e-07, + "logits/chosen": -0.6556218862533569, + "logits/rejected": -0.6689947843551636, + "logps/chosen": -70.7705078125, + "logps/ref_chosen": -53.285308837890625, + "logps/ref_rejected": -111.54470825195312, + "logps/rejected": -159.1107635498047, + "loss": 1.0278, + "margin_dpo/margin_mean": 30.080860137939453, + "margin_dpo/margin_std": 44.20441818237305, + "step": 90 + }, + { + "KL/chosen_KL_mean": -18.215688705444336, + "KL/mean": -30.59687042236328, + "KL/rejected_KL_mean": -42.978057861328125, + "KL/std": 28.502140045166016, + "epoch": 0.13362701908957417, + "fcm_dpo/beta": 0.02069549262523651, + "fcm_dpo/delta": -0.11838565766811371, + "fcm_dpo/margin": 24.762371063232422, + "fcm_dpo/q_t": 0.39080309867858887, + "grad_norm": 25.125621795654297, + "learning_rate": 4.985488079432037e-07, + "logits/chosen": -0.7024148106575012, + "logits/rejected": -0.6713524460792542, + "logps/chosen": -80.01864624023438, + "logps/ref_chosen": -61.802955627441406, + "logps/ref_rejected": -87.87395477294922, + "logps/rejected": -130.85202026367188, + "loss": 1.0666, + "margin_dpo/margin_mean": 24.762371063232422, + "margin_dpo/margin_std": 37.02970886230469, + "step": 91 + }, + { + "KL/chosen_KL_mean": -16.36530113220215, + "KL/mean": -28.872386932373047, + "KL/rejected_KL_mean": -41.37947082519531, + "KL/std": 29.21990203857422, + "epoch": 0.13509544787077826, + "fcm_dpo/beta": 0.02026374265551567, + "fcm_dpo/delta": -0.1125737801194191, + "fcm_dpo/margin": 25.01416778564453, + "fcm_dpo/q_t": 0.39040741324424744, + "grad_norm": 22.92947769165039, + "learning_rate": 4.984074589033043e-07, + "logits/chosen": -0.6997867822647095, + "logits/rejected": -0.676429271697998, + "logps/chosen": -68.00607299804688, + "logps/ref_chosen": -51.640769958496094, + "logps/ref_rejected": -77.88117980957031, + "logps/rejected": -119.26065063476562, + "loss": 1.0572, + "margin_dpo/margin_mean": 25.01416778564453, + "margin_dpo/margin_std": 35.404052734375, + "step": 92 + }, + { + "KL/chosen_KL_mean": -18.400798797607422, + "KL/mean": -30.97686004638672, + "KL/rejected_KL_mean": -43.552921295166016, + "KL/std": 27.59107780456543, + "epoch": 0.13656387665198239, + "fcm_dpo/beta": 0.019758395850658417, + "fcm_dpo/delta": -0.10195006430149078, + "fcm_dpo/margin": 25.15212631225586, + "fcm_dpo/q_t": 0.38974255323410034, + "grad_norm": 23.7645320892334, + "learning_rate": 4.982595640958425e-07, + "logits/chosen": -0.7242586612701416, + "logits/rejected": -0.6749112606048584, + "logps/chosen": -70.93003845214844, + "logps/ref_chosen": -52.529239654541016, + "logps/ref_rejected": -77.16075134277344, + "logps/rejected": -120.71366882324219, + "loss": 1.0348, + "margin_dpo/margin_mean": 25.15212631225586, + "margin_dpo/margin_std": 31.98870086669922, + "step": 93 + }, + { + "KL/chosen_KL_mean": -19.79575538635254, + "KL/mean": -34.32554626464844, + "KL/rejected_KL_mean": -48.8553352355957, + "KL/std": 31.645320892333984, + "epoch": 0.13803230543318648, + "fcm_dpo/beta": 0.019052118062973022, + "fcm_dpo/delta": -0.1642438918352127, + "fcm_dpo/margin": 29.0595760345459, + "fcm_dpo/q_t": 0.37634679675102234, + "grad_norm": 23.290563583374023, + "learning_rate": 4.98105127417984e-07, + "logits/chosen": -0.6614178419113159, + "logits/rejected": -0.6468695402145386, + "logps/chosen": -81.01837158203125, + "logps/ref_chosen": -61.22261047363281, + "logps/ref_rejected": -99.59902954101562, + "logps/rejected": -148.45437622070312, + "loss": 0.997, + "margin_dpo/margin_mean": 29.05957794189453, + "margin_dpo/margin_std": 33.09284591674805, + "step": 94 + }, + { + "KL/chosen_KL_mean": -18.84189796447754, + "KL/mean": -30.522144317626953, + "KL/rejected_KL_mean": -42.202392578125, + "KL/std": 30.20883560180664, + "epoch": 0.1395007342143906, + "fcm_dpo/beta": 0.018812362104654312, + "fcm_dpo/delta": -0.04170980304479599, + "fcm_dpo/margin": 23.360492706298828, + "fcm_dpo/q_t": 0.3993641436100006, + "grad_norm": 21.823162078857422, + "learning_rate": 4.979441529392784e-07, + "logits/chosen": -0.6893196105957031, + "logits/rejected": -0.6614104509353638, + "logps/chosen": -71.36553955078125, + "logps/ref_chosen": -52.523643493652344, + "logps/ref_rejected": -75.8803482055664, + "logps/rejected": -118.0827407836914, + "loss": 1.0766, + "margin_dpo/margin_mean": 23.360492706298828, + "margin_dpo/margin_std": 31.601409912109375, + "step": 95 + }, + { + "KL/chosen_KL_mean": -18.52091407775879, + "KL/mean": -34.392066955566406, + "KL/rejected_KL_mean": -50.263206481933594, + "KL/std": 34.390037536621094, + "epoch": 0.14096916299559473, + "fcm_dpo/beta": 0.01818418689072132, + "fcm_dpo/delta": -0.19050875306129456, + "fcm_dpo/margin": 31.74229621887207, + "fcm_dpo/q_t": 0.3741267919540405, + "grad_norm": 22.391244888305664, + "learning_rate": 4.977766449015534e-07, + "logits/chosen": -0.702680766582489, + "logits/rejected": -0.6744130849838257, + "logps/chosen": -80.67788696289062, + "logps/ref_chosen": -62.15697479248047, + "logps/ref_rejected": -96.59601593017578, + "logps/rejected": -146.85922241210938, + "loss": 0.9906, + "margin_dpo/margin_mean": 31.742298126220703, + "margin_dpo/margin_std": 37.9951171875, + "step": 96 + }, + { + "KL/chosen_KL_mean": -19.829116821289062, + "KL/mean": -32.06022262573242, + "KL/rejected_KL_mean": -44.29132843017578, + "KL/std": 28.707332611083984, + "epoch": 0.14243759177679882, + "fcm_dpo/beta": 0.01816154085099697, + "fcm_dpo/delta": -0.04735935106873512, + "fcm_dpo/margin": 24.46221351623535, + "fcm_dpo/q_t": 0.39719265699386597, + "grad_norm": 23.29534339904785, + "learning_rate": 4.976026077188012e-07, + "logits/chosen": -0.6568596363067627, + "logits/rejected": -0.6160587668418884, + "logps/chosen": -74.47547912597656, + "logps/ref_chosen": -54.646366119384766, + "logps/ref_rejected": -76.96475219726562, + "logps/rejected": -121.2560806274414, + "loss": 1.0603, + "margin_dpo/margin_mean": 24.46221351623535, + "margin_dpo/margin_std": 28.667625427246094, + "step": 97 + }, + { + "KL/chosen_KL_mean": -24.416702270507812, + "KL/mean": -38.547080993652344, + "KL/rejected_KL_mean": -52.677459716796875, + "KL/std": 32.80169677734375, + "epoch": 0.14390602055800295, + "fcm_dpo/beta": 0.01765655353665352, + "fcm_dpo/delta": -0.10444936156272888, + "fcm_dpo/margin": 28.260757446289062, + "fcm_dpo/q_t": 0.38620907068252563, + "grad_norm": 24.261281967163086, + "learning_rate": 4.974220459770639e-07, + "logits/chosen": -0.6890856027603149, + "logits/rejected": -0.6740926504135132, + "logps/chosen": -89.67533111572266, + "logps/ref_chosen": -65.25862884521484, + "logps/ref_rejected": -96.5274887084961, + "logps/rejected": -149.2049560546875, + "loss": 1.0602, + "margin_dpo/margin_mean": 28.260757446289062, + "margin_dpo/margin_std": 39.32505798339844, + "step": 98 + }, + { + "KL/chosen_KL_mean": -19.188392639160156, + "KL/mean": -36.50630187988281, + "KL/rejected_KL_mean": -53.82420349121094, + "KL/std": 35.373687744140625, + "epoch": 0.14537444933920704, + "fcm_dpo/beta": 0.017020825296640396, + "fcm_dpo/delta": -0.20224523544311523, + "fcm_dpo/margin": 34.63581848144531, + "fcm_dpo/q_t": 0.3741142749786377, + "grad_norm": 21.056440353393555, + "learning_rate": 4.972349644343108e-07, + "logits/chosen": -0.6723522543907166, + "logits/rejected": -0.6737344861030579, + "logps/chosen": -64.82687377929688, + "logps/ref_chosen": -45.638484954833984, + "logps/ref_rejected": -86.43793487548828, + "logps/rejected": -140.26214599609375, + "loss": 0.994, + "margin_dpo/margin_mean": 34.63581848144531, + "margin_dpo/margin_std": 43.95652389526367, + "step": 99 + }, + { + "KL/chosen_KL_mean": -21.610239028930664, + "KL/mean": -31.642812728881836, + "KL/rejected_KL_mean": -41.675384521484375, + "KL/std": 28.504859924316406, + "epoch": 0.14684287812041116, + "fcm_dpo/beta": 0.017114732414484024, + "fcm_dpo/delta": 0.0580127090215683, + "fcm_dpo/margin": 20.065147399902344, + "fcm_dpo/q_t": 0.4225125014781952, + "grad_norm": 23.509201049804688, + "learning_rate": 4.970413680203148e-07, + "logits/chosen": -0.6509604454040527, + "logits/rejected": -0.6082332730293274, + "logps/chosen": -79.2042236328125, + "logps/ref_chosen": -57.59397888183594, + "logps/ref_rejected": -74.06021118164062, + "logps/rejected": -115.735595703125, + "loss": 1.1693, + "margin_dpo/margin_mean": 20.065147399902344, + "margin_dpo/margin_std": 36.753334045410156, + "step": 100 + }, + { + "KL/chosen_KL_mean": -26.565208435058594, + "KL/mean": -38.76583480834961, + "KL/rejected_KL_mean": -50.96645736694336, + "KL/std": 35.17367935180664, + "epoch": 0.14831130690161526, + "fcm_dpo/beta": 0.01696072332561016, + "fcm_dpo/delta": -0.015185698866844177, + "fcm_dpo/margin": 24.401235580444336, + "fcm_dpo/q_t": 0.41169631481170654, + "grad_norm": 23.37774658203125, + "learning_rate": 4.968412618365215e-07, + "logits/chosen": -0.6405997276306152, + "logits/rejected": -0.6082361936569214, + "logps/chosen": -88.21406555175781, + "logps/ref_chosen": -61.64885330200195, + "logps/ref_rejected": -83.18968200683594, + "logps/rejected": -134.15614318847656, + "loss": 1.1319, + "margin_dpo/margin_mean": 24.401235580444336, + "margin_dpo/margin_std": 42.36262512207031, + "step": 101 + }, + { + "KL/chosen_KL_mean": -28.67186164855957, + "KL/mean": -37.669090270996094, + "KL/rejected_KL_mean": -46.66631317138672, + "KL/std": 32.99787139892578, + "epoch": 0.14977973568281938, + "fcm_dpo/beta": 0.01739579439163208, + "fcm_dpo/delta": 0.08802524209022522, + "fcm_dpo/margin": 17.994457244873047, + "fcm_dpo/q_t": 0.43094637989997864, + "grad_norm": 27.00950813293457, + "learning_rate": 4.966346511559149e-07, + "logits/chosen": -0.6696982383728027, + "logits/rejected": -0.6241832971572876, + "logps/chosen": -92.75074005126953, + "logps/ref_chosen": -64.0788803100586, + "logps/ref_rejected": -68.18707275390625, + "logps/rejected": -114.8533935546875, + "loss": 1.2095, + "margin_dpo/margin_mean": 17.994457244873047, + "margin_dpo/margin_std": 39.09446716308594, + "step": 102 + }, + { + "KL/chosen_KL_mean": -22.722579956054688, + "KL/mean": -40.756980895996094, + "KL/rejected_KL_mean": -58.7913932800293, + "KL/std": 36.61017608642578, + "epoch": 0.1512481644640235, + "fcm_dpo/beta": 0.016756968572735786, + "fcm_dpo/delta": -0.2181204855442047, + "fcm_dpo/margin": 36.068809509277344, + "fcm_dpo/q_t": 0.3694770336151123, + "grad_norm": 22.91532325744629, + "learning_rate": 4.964215414228785e-07, + "logits/chosen": -0.6852983236312866, + "logits/rejected": -0.6532962322235107, + "logps/chosen": -84.02185821533203, + "logps/ref_chosen": -61.299278259277344, + "logps/ref_rejected": -93.57270812988281, + "logps/rejected": -152.36410522460938, + "loss": 0.9828, + "margin_dpo/margin_mean": 36.068809509277344, + "margin_dpo/margin_std": 44.13578796386719, + "step": 103 + }, + { + "KL/chosen_KL_mean": -24.786264419555664, + "KL/mean": -41.86579895019531, + "KL/rejected_KL_mean": -58.945335388183594, + "KL/std": 40.4488639831543, + "epoch": 0.1527165932452276, + "fcm_dpo/beta": 0.01623380184173584, + "fcm_dpo/delta": -0.1633920669555664, + "fcm_dpo/margin": 34.15907669067383, + "fcm_dpo/q_t": 0.3847202956676483, + "grad_norm": 22.349903106689453, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": -0.6932718753814697, + "logits/rejected": -0.664044976234436, + "logps/chosen": -79.15904235839844, + "logps/ref_chosen": -54.372772216796875, + "logps/ref_rejected": -89.5647201538086, + "logps/rejected": -148.5100555419922, + "loss": 1.0435, + "margin_dpo/margin_mean": 34.159080505371094, + "margin_dpo/margin_std": 49.494911193847656, + "step": 104 + }, + { + "KL/chosen_KL_mean": -23.31475257873535, + "KL/mean": -47.551963806152344, + "KL/rejected_KL_mean": -71.7891845703125, + "KL/std": 39.350242614746094, + "epoch": 0.15418502202643172, + "fcm_dpo/beta": 0.015241267159581184, + "fcm_dpo/delta": -0.36551716923713684, + "fcm_dpo/margin": 48.474430084228516, + "fcm_dpo/q_t": 0.3338563144207001, + "grad_norm": 22.70827293395996, + "learning_rate": 4.959758474331832e-07, + "logits/chosen": -0.6466401219367981, + "logits/rejected": -0.6229440569877625, + "logps/chosen": -77.95369720458984, + "logps/ref_chosen": -54.638946533203125, + "logps/ref_rejected": -97.97351837158203, + "logps/rejected": -169.7626953125, + "loss": 0.8715, + "margin_dpo/margin_mean": 48.474430084228516, + "margin_dpo/margin_std": 43.423583984375, + "step": 105 + }, + { + "KL/chosen_KL_mean": -26.223730087280273, + "KL/mean": -41.38634490966797, + "KL/rejected_KL_mean": -56.54896545410156, + "KL/std": 34.416595458984375, + "epoch": 0.15565345080763582, + "fcm_dpo/beta": 0.014795205555856228, + "fcm_dpo/delta": -0.05113736912608147, + "fcm_dpo/margin": 30.32522964477539, + "fcm_dpo/q_t": 0.3969998359680176, + "grad_norm": 21.981822967529297, + "learning_rate": 4.957432749209755e-07, + "logits/chosen": -0.6268575191497803, + "logits/rejected": -0.597021222114563, + "logps/chosen": -81.05662536621094, + "logps/ref_chosen": -54.83289337158203, + "logps/ref_rejected": -85.22461700439453, + "logps/rejected": -141.77359008789062, + "loss": 1.0591, + "margin_dpo/margin_mean": 30.32522964477539, + "margin_dpo/margin_std": 37.44782638549805, + "step": 106 + }, + { + "KL/chosen_KL_mean": -31.174667358398438, + "KL/mean": -48.2926025390625, + "KL/rejected_KL_mean": -65.41053771972656, + "KL/std": 41.864192962646484, + "epoch": 0.15712187958883994, + "fcm_dpo/beta": 0.014499913901090622, + "fcm_dpo/delta": -0.10227993130683899, + "fcm_dpo/margin": 34.235870361328125, + "fcm_dpo/q_t": 0.38768211007118225, + "grad_norm": 21.1258487701416, + "learning_rate": 4.955042268449307e-07, + "logits/chosen": -0.6795566082000732, + "logits/rejected": -0.6355477571487427, + "logps/chosen": -100.88247680664062, + "logps/ref_chosen": -69.70780944824219, + "logps/ref_rejected": -94.73950958251953, + "logps/rejected": -160.15005493164062, + "loss": 1.0454, + "margin_dpo/margin_mean": 34.235870361328125, + "margin_dpo/margin_std": 44.14703369140625, + "step": 107 + }, + { + "KL/chosen_KL_mean": -27.941110610961914, + "KL/mean": -48.69300079345703, + "KL/rejected_KL_mean": -69.44490051269531, + "KL/std": 46.46894836425781, + "epoch": 0.15859030837004406, + "fcm_dpo/beta": 0.014074774459004402, + "fcm_dpo/delta": -0.1955530345439911, + "fcm_dpo/margin": 41.5037956237793, + "fcm_dpo/q_t": 0.37944066524505615, + "grad_norm": 21.240692138671875, + "learning_rate": 4.952587095041881e-07, + "logits/chosen": -0.6599795818328857, + "logits/rejected": -0.6397461891174316, + "logps/chosen": -83.95099639892578, + "logps/ref_chosen": -56.0098876953125, + "logps/ref_rejected": -95.79601287841797, + "logps/rejected": -165.24090576171875, + "loss": 1.0361, + "margin_dpo/margin_mean": 41.50379180908203, + "margin_dpo/margin_std": 60.09625244140625, + "step": 108 + }, + { + "KL/chosen_KL_mean": -26.06736946105957, + "KL/mean": -48.12309646606445, + "KL/rejected_KL_mean": -70.1788101196289, + "KL/std": 44.416900634765625, + "epoch": 0.16005873715124816, + "fcm_dpo/beta": 0.013461511582136154, + "fcm_dpo/delta": -0.20704127848148346, + "fcm_dpo/margin": 44.111446380615234, + "fcm_dpo/q_t": 0.3681620955467224, + "grad_norm": 22.13717269897461, + "learning_rate": 4.95006729368358e-07, + "logits/chosen": -0.59981369972229, + "logits/rejected": -0.578811526298523, + "logps/chosen": -88.9528579711914, + "logps/ref_chosen": -62.88549041748047, + "logps/ref_rejected": -98.68573760986328, + "logps/rejected": -168.8645477294922, + "loss": 0.9887, + "margin_dpo/margin_mean": 44.1114501953125, + "margin_dpo/margin_std": 52.54209518432617, + "step": 109 + }, + { + "KL/chosen_KL_mean": -26.961872100830078, + "KL/mean": -46.808937072753906, + "KL/rejected_KL_mean": -66.65599822998047, + "KL/std": 44.2524528503418, + "epoch": 0.16152716593245228, + "fcm_dpo/beta": 0.01295989565551281, + "fcm_dpo/delta": -0.12524738907814026, + "fcm_dpo/margin": 39.694129943847656, + "fcm_dpo/q_t": 0.3869009017944336, + "grad_norm": 19.041017532348633, + "learning_rate": 4.947482930773511e-07, + "logits/chosen": -0.6018489003181458, + "logits/rejected": -0.566160261631012, + "logps/chosen": -85.71556091308594, + "logps/ref_chosen": -58.753684997558594, + "logps/ref_rejected": -79.75001525878906, + "logps/rejected": -146.406005859375, + "loss": 1.0523, + "margin_dpo/margin_mean": 39.694129943847656, + "margin_dpo/margin_std": 53.206546783447266, + "step": 110 + }, + { + "KL/chosen_KL_mean": -30.57050132751465, + "KL/mean": -53.029212951660156, + "KL/rejected_KL_mean": -75.48793029785156, + "KL/std": 50.006378173828125, + "epoch": 0.16299559471365638, + "fcm_dpo/beta": 0.012622907757759094, + "fcm_dpo/delta": -0.17867523431777954, + "fcm_dpo/margin": 44.91743469238281, + "fcm_dpo/q_t": 0.37669897079467773, + "grad_norm": 21.740154266357422, + "learning_rate": 4.944834074412042e-07, + "logits/chosen": -0.6179641485214233, + "logits/rejected": -0.594010591506958, + "logps/chosen": -99.19461059570312, + "logps/ref_chosen": -68.62410736083984, + "logps/ref_rejected": -98.42886352539062, + "logps/rejected": -173.9167938232422, + "loss": 1.0337, + "margin_dpo/margin_mean": 44.91743469238281, + "margin_dpo/margin_std": 61.63525390625, + "step": 111 + }, + { + "KL/chosen_KL_mean": -28.242313385009766, + "KL/mean": -42.492279052734375, + "KL/rejected_KL_mean": -56.74223709106445, + "KL/std": 34.65959930419922, + "epoch": 0.1644640234948605, + "fcm_dpo/beta": 0.012602494098246098, + "fcm_dpo/delta": 0.04215133190155029, + "fcm_dpo/margin": 28.499923706054688, + "fcm_dpo/q_t": 0.41772669553756714, + "grad_norm": 19.57350730895996, + "learning_rate": 4.942120794399002e-07, + "logits/chosen": -0.6415656805038452, + "logits/rejected": -0.6061877608299255, + "logps/chosen": -78.49195861816406, + "logps/ref_chosen": -50.24964141845703, + "logps/ref_rejected": -64.77442932128906, + "logps/rejected": -121.51666259765625, + "loss": 1.1268, + "margin_dpo/margin_mean": 28.499921798706055, + "margin_dpo/margin_std": 41.986968994140625, + "step": 112 + }, + { + "KL/chosen_KL_mean": -35.24995803833008, + "KL/mean": -50.625701904296875, + "KL/rejected_KL_mean": -66.00144958496094, + "KL/std": 36.226314544677734, + "epoch": 0.16593245227606462, + "fcm_dpo/beta": 0.012705144472420216, + "fcm_dpo/delta": 0.009673002175986767, + "fcm_dpo/margin": 30.751483917236328, + "fcm_dpo/q_t": 0.4100167751312256, + "grad_norm": 19.772661209106445, + "learning_rate": 4.939343162231841e-07, + "logits/chosen": -0.6053025722503662, + "logits/rejected": -0.5635230541229248, + "logps/chosen": -101.96290588378906, + "logps/ref_chosen": -66.71295166015625, + "logps/ref_rejected": -77.96870422363281, + "logps/rejected": -143.97015380859375, + "loss": 1.0939, + "margin_dpo/margin_mean": 30.751482009887695, + "margin_dpo/margin_std": 40.09980773925781, + "step": 113 + }, + { + "KL/chosen_KL_mean": -32.449310302734375, + "KL/mean": -56.777923583984375, + "KL/rejected_KL_mean": -81.10653686523438, + "KL/std": 54.06468963623047, + "epoch": 0.16740088105726872, + "fcm_dpo/beta": 0.01223907433450222, + "fcm_dpo/delta": -0.21012035012245178, + "fcm_dpo/margin": 48.65721893310547, + "fcm_dpo/q_t": 0.37521958351135254, + "grad_norm": 21.462278366088867, + "learning_rate": 4.936501251103751e-07, + "logits/chosen": -0.6051023006439209, + "logits/rejected": -0.5736086964607239, + "logps/chosen": -90.23439025878906, + "logps/ref_chosen": -57.78507995605469, + "logps/ref_rejected": -87.10966491699219, + "logps/rejected": -168.2161865234375, + "loss": 1.0021, + "margin_dpo/margin_mean": 48.65721893310547, + "margin_dpo/margin_std": 65.560791015625, + "step": 114 + }, + { + "KL/chosen_KL_mean": -42.36359786987305, + "KL/mean": -58.933746337890625, + "KL/rejected_KL_mean": -75.50389099121094, + "KL/std": 51.648048400878906, + "epoch": 0.16886930983847284, + "fcm_dpo/beta": 0.012213023379445076, + "fcm_dpo/delta": -0.005111444741487503, + "fcm_dpo/margin": 33.140296936035156, + "fcm_dpo/q_t": 0.4153473973274231, + "grad_norm": 27.328691482543945, + "learning_rate": 4.933595135901732e-07, + "logits/chosen": -0.6272869110107422, + "logits/rejected": -0.6080690622329712, + "logps/chosen": -107.94624328613281, + "logps/ref_chosen": -65.5826416015625, + "logps/ref_rejected": -98.56552124023438, + "logps/rejected": -174.0694122314453, + "loss": 1.1709, + "margin_dpo/margin_mean": 33.140296936035156, + "margin_dpo/margin_std": 68.08973693847656, + "step": 115 + }, + { + "KL/chosen_KL_mean": -31.53169822692871, + "KL/mean": -49.997901916503906, + "KL/rejected_KL_mean": -68.464111328125, + "KL/std": 43.47674560546875, + "epoch": 0.17033773861967694, + "fcm_dpo/beta": 0.012136522680521011, + "fcm_dpo/delta": -0.0506804920732975, + "fcm_dpo/margin": 36.932403564453125, + "fcm_dpo/q_t": 0.3989343047142029, + "grad_norm": 21.574514389038086, + "learning_rate": 4.930624893204624e-07, + "logits/chosen": -0.6011572480201721, + "logits/rejected": -0.5915130376815796, + "logps/chosen": -82.9320068359375, + "logps/ref_chosen": -51.40031433105469, + "logps/ref_rejected": -80.5218505859375, + "logps/rejected": -148.9859619140625, + "loss": 1.0625, + "margin_dpo/margin_mean": 36.932403564453125, + "margin_dpo/margin_std": 47.14096450805664, + "step": 116 + }, + { + "KL/chosen_KL_mean": -40.54469299316406, + "KL/mean": -55.65740966796875, + "KL/rejected_KL_mean": -70.77013397216797, + "KL/std": 46.44588851928711, + "epoch": 0.17180616740088106, + "fcm_dpo/beta": 0.012116730213165283, + "fcm_dpo/delta": 0.035039689391851425, + "fcm_dpo/margin": 30.225439071655273, + "fcm_dpo/q_t": 0.4179823696613312, + "grad_norm": 27.370105743408203, + "learning_rate": 4.927590601281083e-07, + "logits/chosen": -0.5658215284347534, + "logits/rejected": -0.5278164744377136, + "logps/chosen": -109.84310150146484, + "logps/ref_chosen": -69.29840850830078, + "logps/ref_rejected": -66.583984375, + "logps/rejected": -137.3541259765625, + "loss": 1.146, + "margin_dpo/margin_mean": 30.225440979003906, + "margin_dpo/margin_std": 52.66984176635742, + "step": 117 + }, + { + "KL/chosen_KL_mean": -32.08608627319336, + "KL/mean": -50.51774597167969, + "KL/rejected_KL_mean": -68.94939422607422, + "KL/std": 41.92464065551758, + "epoch": 0.17327459618208516, + "fcm_dpo/beta": 0.012059303931891918, + "fcm_dpo/delta": -0.04659657180309296, + "fcm_dpo/margin": 36.863319396972656, + "fcm_dpo/q_t": 0.4000033140182495, + "grad_norm": 20.587656021118164, + "learning_rate": 4.924492340087524e-07, + "logits/chosen": -0.6026766300201416, + "logits/rejected": -0.5824264287948608, + "logps/chosen": -87.72706604003906, + "logps/ref_chosen": -55.6409797668457, + "logps/ref_rejected": -75.66905975341797, + "logps/rejected": -144.6184539794922, + "loss": 1.0657, + "margin_dpo/margin_mean": 36.86331558227539, + "margin_dpo/margin_std": 47.96271514892578, + "step": 118 + }, + { + "KL/chosen_KL_mean": -43.64602279663086, + "KL/mean": -61.894630432128906, + "KL/rejected_KL_mean": -80.14323425292969, + "KL/std": 47.0084228515625, + "epoch": 0.17474302496328928, + "fcm_dpo/beta": 0.01187710091471672, + "fcm_dpo/delta": -0.03619098290801048, + "fcm_dpo/margin": 36.49721145629883, + "fcm_dpo/q_t": 0.40584415197372437, + "grad_norm": 23.66024398803711, + "learning_rate": 4.92133019126601e-07, + "logits/chosen": -0.6127524375915527, + "logits/rejected": -0.6005865931510925, + "logps/chosen": -117.15621948242188, + "logps/ref_chosen": -73.51019287109375, + "logps/ref_rejected": -102.977294921875, + "logps/rejected": -183.1205291748047, + "loss": 1.1046, + "margin_dpo/margin_mean": 36.49720764160156, + "margin_dpo/margin_std": 57.2733154296875, + "step": 119 + }, + { + "KL/chosen_KL_mean": -45.010643005371094, + "KL/mean": -70.27703857421875, + "KL/rejected_KL_mean": -95.54344177246094, + "KL/std": 58.90607452392578, + "epoch": 0.1762114537444934, + "fcm_dpo/beta": 0.011582771316170692, + "fcm_dpo/delta": -0.19700661301612854, + "fcm_dpo/margin": 50.532806396484375, + "fcm_dpo/q_t": 0.3717060089111328, + "grad_norm": 21.907379150390625, + "learning_rate": 4.918104238142103e-07, + "logits/chosen": -0.6179412603378296, + "logits/rejected": -0.5869693160057068, + "logps/chosen": -121.79147338867188, + "logps/ref_chosen": -76.78083801269531, + "logps/ref_rejected": -108.02374267578125, + "logps/rejected": -203.5671844482422, + "loss": 0.9953, + "margin_dpo/margin_mean": 50.532806396484375, + "margin_dpo/margin_std": 61.917076110839844, + "step": 120 + }, + { + "KL/chosen_KL_mean": -43.67778015136719, + "KL/mean": -71.53456115722656, + "KL/rejected_KL_mean": -99.39134216308594, + "KL/std": 57.339202880859375, + "epoch": 0.1776798825256975, + "fcm_dpo/beta": 0.011032504960894585, + "fcm_dpo/delta": -0.23044228553771973, + "fcm_dpo/margin": 55.71354675292969, + "fcm_dpo/q_t": 0.3678060472011566, + "grad_norm": 23.938167572021484, + "learning_rate": 4.91481456572267e-07, + "logits/chosen": -0.5758655071258545, + "logits/rejected": -0.572211742401123, + "logps/chosen": -105.46768188476562, + "logps/ref_chosen": -61.789894104003906, + "logps/ref_rejected": -109.99456787109375, + "logps/rejected": -209.3859100341797, + "loss": 0.9973, + "margin_dpo/margin_mean": 55.71355056762695, + "margin_dpo/margin_std": 71.77114868164062, + "step": 121 + }, + { + "KL/chosen_KL_mean": -39.53837203979492, + "KL/mean": -74.14775848388672, + "KL/rejected_KL_mean": -108.75715637207031, + "KL/std": 65.19731140136719, + "epoch": 0.17914831130690162, + "fcm_dpo/beta": 0.010416809469461441, + "fcm_dpo/delta": -0.34584206342697144, + "fcm_dpo/margin": 69.2187728881836, + "fcm_dpo/q_t": 0.34151148796081543, + "grad_norm": 23.229158401489258, + "learning_rate": 4.911461260693638e-07, + "logits/chosen": -0.5348167419433594, + "logits/rejected": -0.5510457158088684, + "logps/chosen": -86.44058227539062, + "logps/ref_chosen": -46.9022102355957, + "logps/ref_rejected": -106.71418762207031, + "logps/rejected": -215.47134399414062, + "loss": 0.8958, + "margin_dpo/margin_mean": 69.2187728881836, + "margin_dpo/margin_std": 68.12580871582031, + "step": 122 + }, + { + "KL/chosen_KL_mean": -43.5894775390625, + "KL/mean": -66.2512435913086, + "KL/rejected_KL_mean": -88.91300964355469, + "KL/std": 55.7255859375, + "epoch": 0.18061674008810572, + "fcm_dpo/beta": 0.010076490230858326, + "fcm_dpo/delta": -0.060425400733947754, + "fcm_dpo/margin": 45.323524475097656, + "fcm_dpo/q_t": 0.4018062949180603, + "grad_norm": 20.704429626464844, + "learning_rate": 4.908044411417711e-07, + "logits/chosen": -0.5436292290687561, + "logits/rejected": -0.5264102220535278, + "logps/chosen": -104.92811584472656, + "logps/ref_chosen": -61.33863830566406, + "logps/ref_rejected": -87.775390625, + "logps/rejected": -176.6884002685547, + "loss": 1.1086, + "margin_dpo/margin_mean": 45.32352828979492, + "margin_dpo/margin_std": 74.68168640136719, + "step": 123 + }, + { + "KL/chosen_KL_mean": -50.04446029663086, + "KL/mean": -83.56463623046875, + "KL/rejected_KL_mean": -117.08482360839844, + "KL/std": 76.33003234863281, + "epoch": 0.18208516886930984, + "fcm_dpo/beta": 0.009682193398475647, + "fcm_dpo/delta": -0.26734763383865356, + "fcm_dpo/margin": 67.04035949707031, + "fcm_dpo/q_t": 0.3692913055419922, + "grad_norm": 23.1252498626709, + "learning_rate": 4.904564107932048e-07, + "logits/chosen": -0.5410699248313904, + "logits/rejected": -0.5437477827072144, + "logps/chosen": -121.4927978515625, + "logps/ref_chosen": -71.44833374023438, + "logps/ref_rejected": -117.58056640625, + "logps/rejected": -234.66539001464844, + "loss": 1.0151, + "margin_dpo/margin_mean": 67.04035949707031, + "margin_dpo/margin_std": 95.87313842773438, + "step": 124 + }, + { + "KL/chosen_KL_mean": -41.3752326965332, + "KL/mean": -71.00807189941406, + "KL/rejected_KL_mean": -100.64091491699219, + "KL/std": 64.5465316772461, + "epoch": 0.18355359765051396, + "fcm_dpo/beta": 0.009364111348986626, + "fcm_dpo/delta": -0.16384468972682953, + "fcm_dpo/margin": 59.265682220458984, + "fcm_dpo/q_t": 0.38098064064979553, + "grad_norm": 19.144956588745117, + "learning_rate": 4.90102044194588e-07, + "logits/chosen": -0.5046694278717041, + "logits/rejected": -0.507087230682373, + "logps/chosen": -91.51217651367188, + "logps/ref_chosen": -50.136940002441406, + "logps/ref_rejected": -83.98861694335938, + "logps/rejected": -184.62953186035156, + "loss": 1.0301, + "margin_dpo/margin_mean": 59.26567840576172, + "margin_dpo/margin_std": 80.23193359375, + "step": 125 + }, + { + "KL/chosen_KL_mean": -44.953895568847656, + "KL/mean": -72.60568237304688, + "KL/rejected_KL_mean": -100.2574691772461, + "KL/std": 58.28239440917969, + "epoch": 0.18502202643171806, + "fcm_dpo/beta": 0.009100959636271, + "fcm_dpo/delta": -0.10908694565296173, + "fcm_dpo/margin": 55.30357360839844, + "fcm_dpo/q_t": 0.38956084847450256, + "grad_norm": 20.653635025024414, + "learning_rate": 4.897413506838102e-07, + "logits/chosen": -0.5091671943664551, + "logits/rejected": -0.503969132900238, + "logps/chosen": -100.62096405029297, + "logps/ref_chosen": -55.66706848144531, + "logps/ref_rejected": -98.1297607421875, + "logps/rejected": -198.38723754882812, + "loss": 1.0457, + "margin_dpo/margin_mean": 55.30357360839844, + "margin_dpo/margin_std": 74.20457458496094, + "step": 126 + }, + { + "KL/chosen_KL_mean": -43.32256317138672, + "KL/mean": -63.5421142578125, + "KL/rejected_KL_mean": -83.76165771484375, + "KL/std": 50.99864959716797, + "epoch": 0.18649045521292218, + "fcm_dpo/beta": 0.00911656953394413, + "fcm_dpo/delta": 0.032512567937374115, + "fcm_dpo/margin": 40.43910217285156, + "fcm_dpo/q_t": 0.41489607095718384, + "grad_norm": 20.905221939086914, + "learning_rate": 4.89374339765481e-07, + "logits/chosen": -0.5238068103790283, + "logits/rejected": -0.5045550465583801, + "logps/chosen": -99.87724304199219, + "logps/ref_chosen": -56.55467987060547, + "logps/ref_rejected": -76.7957763671875, + "logps/rejected": -160.55743408203125, + "loss": 1.1317, + "margin_dpo/margin_mean": 40.43910217285156, + "margin_dpo/margin_std": 64.11962127685547, + "step": 127 + }, + { + "KL/chosen_KL_mean": -46.23556137084961, + "KL/mean": -67.52534484863281, + "KL/rejected_KL_mean": -88.81513977050781, + "KL/std": 59.858497619628906, + "epoch": 0.18795888399412627, + "fcm_dpo/beta": 0.009177174419164658, + "fcm_dpo/delta": 0.009238209575414658, + "fcm_dpo/margin": 42.5795783996582, + "fcm_dpo/q_t": 0.4133646786212921, + "grad_norm": 29.049198150634766, + "learning_rate": 4.890010211106795e-07, + "logits/chosen": -0.5192145109176636, + "logits/rejected": -0.4998999834060669, + "logps/chosen": -104.35652160644531, + "logps/ref_chosen": -58.12095642089844, + "logps/ref_rejected": -76.43896484375, + "logps/rejected": -165.25408935546875, + "loss": 1.1434, + "margin_dpo/margin_mean": 42.5795783996582, + "margin_dpo/margin_std": 75.051513671875, + "step": 128 + }, + { + "KL/chosen_KL_mean": -56.05470657348633, + "KL/mean": -78.45681762695312, + "KL/rejected_KL_mean": -100.85891723632812, + "KL/std": 65.97745513916016, + "epoch": 0.1894273127753304, + "fcm_dpo/beta": 0.009138467721641064, + "fcm_dpo/delta": -0.009851408191025257, + "fcm_dpo/margin": 44.80420684814453, + "fcm_dpo/q_t": 0.41433221101760864, + "grad_norm": 20.67636489868164, + "learning_rate": 4.88621404556699e-07, + "logits/chosen": -0.5321705341339111, + "logits/rejected": -0.5218693614006042, + "logps/chosen": -122.97108459472656, + "logps/ref_chosen": -66.91637420654297, + "logps/ref_rejected": -96.6422119140625, + "logps/rejected": -197.50112915039062, + "loss": 1.15, + "margin_dpo/margin_mean": 44.80420684814453, + "margin_dpo/margin_std": 85.82166290283203, + "step": 129 + }, + { + "KL/chosen_KL_mean": -41.49655532836914, + "KL/mean": -75.23775482177734, + "KL/rejected_KL_mean": -108.97895050048828, + "KL/std": 67.84368896484375, + "epoch": 0.19089574155653452, + "fcm_dpo/beta": 0.008932676166296005, + "fcm_dpo/delta": -0.21578986942768097, + "fcm_dpo/margin": 67.48239135742188, + "fcm_dpo/q_t": 0.37061649560928345, + "grad_norm": 21.26862907409668, + "learning_rate": 4.882355001067891e-07, + "logits/chosen": -0.49152904748916626, + "logits/rejected": -0.4860014319419861, + "logps/chosen": -86.16340637207031, + "logps/ref_chosen": -44.66685104370117, + "logps/ref_rejected": -82.78165435791016, + "logps/rejected": -191.76060485839844, + "loss": 0.9985, + "margin_dpo/margin_mean": 67.48239135742188, + "margin_dpo/margin_std": 82.64620971679688, + "step": 130 + }, + { + "KL/chosen_KL_mean": -36.96665954589844, + "KL/mean": -70.82868194580078, + "KL/rejected_KL_mean": -104.69070434570312, + "KL/std": 67.29373168945312, + "epoch": 0.19236417033773862, + "fcm_dpo/beta": 0.008490066044032574, + "fcm_dpo/delta": -0.1859792321920395, + "fcm_dpo/margin": 67.72404479980469, + "fcm_dpo/q_t": 0.36927857995033264, + "grad_norm": 29.203166961669922, + "learning_rate": 4.878433179298909e-07, + "logits/chosen": -0.5128257274627686, + "logits/rejected": -0.5198142528533936, + "logps/chosen": -81.89125061035156, + "logps/ref_chosen": -44.924591064453125, + "logps/ref_rejected": -88.44401550292969, + "logps/rejected": -193.13470458984375, + "loss": 0.9808, + "margin_dpo/margin_mean": 67.72404479980469, + "margin_dpo/margin_std": 73.8796615600586, + "step": 131 + }, + { + "KL/chosen_KL_mean": -49.28675079345703, + "KL/mean": -76.96174621582031, + "KL/rejected_KL_mean": -104.63676452636719, + "KL/std": 67.005615234375, + "epoch": 0.19383259911894274, + "fcm_dpo/beta": 0.008325127884745598, + "fcm_dpo/delta": -0.06394054740667343, + "fcm_dpo/margin": 55.35000228881836, + "fcm_dpo/q_t": 0.40161028504371643, + "grad_norm": 19.584449768066406, + "learning_rate": 4.874448683603694e-07, + "logits/chosen": -0.5270289778709412, + "logits/rejected": -0.5260351896286011, + "logps/chosen": -108.28783416748047, + "logps/ref_chosen": -59.00108337402344, + "logps/ref_rejected": -87.89215087890625, + "logps/rejected": -192.52891540527344, + "loss": 1.0894, + "margin_dpo/margin_mean": 55.350006103515625, + "margin_dpo/margin_std": 87.00371551513672, + "step": 132 + }, + { + "KL/chosen_KL_mean": -57.79051208496094, + "KL/mean": -82.15497589111328, + "KL/rejected_KL_mean": -106.51943969726562, + "KL/std": 60.27544403076172, + "epoch": 0.19530102790014683, + "fcm_dpo/beta": 0.008292624726891518, + "fcm_dpo/delta": -0.004351507872343063, + "fcm_dpo/margin": 48.728919982910156, + "fcm_dpo/q_t": 0.4111335277557373, + "grad_norm": 27.549100875854492, + "learning_rate": 4.870401618977415e-07, + "logits/chosen": -0.511991024017334, + "logits/rejected": -0.4981684684753418, + "logps/chosen": -124.39501190185547, + "logps/ref_chosen": -66.60449981689453, + "logps/ref_rejected": -96.33355712890625, + "logps/rejected": -202.85299682617188, + "loss": 1.1121, + "margin_dpo/margin_mean": 48.728919982910156, + "margin_dpo/margin_std": 76.42304229736328, + "step": 133 + }, + { + "KL/chosen_KL_mean": -46.12035369873047, + "KL/mean": -72.27875518798828, + "KL/rejected_KL_mean": -98.43716430664062, + "KL/std": 58.89271926879883, + "epoch": 0.19676945668135096, + "fcm_dpo/beta": 0.008291056379675865, + "fcm_dpo/delta": -0.03539953753352165, + "fcm_dpo/margin": 52.316802978515625, + "fcm_dpo/q_t": 0.40254372358322144, + "grad_norm": 18.665939331054688, + "learning_rate": 4.866292092063986e-07, + "logits/chosen": -0.4636070132255554, + "logits/rejected": -0.44918179512023926, + "logps/chosen": -98.18960571289062, + "logps/ref_chosen": -52.06925582885742, + "logps/ref_rejected": -87.6545181274414, + "logps/rejected": -186.0916748046875, + "loss": 1.0695, + "margin_dpo/margin_mean": 52.316802978515625, + "margin_dpo/margin_std": 68.16081237792969, + "step": 134 + }, + { + "KL/chosen_KL_mean": -50.93290710449219, + "KL/mean": -88.6656265258789, + "KL/rejected_KL_mean": -126.39834594726562, + "KL/std": 78.58366394042969, + "epoch": 0.19823788546255505, + "fcm_dpo/beta": 0.007986115291714668, + "fcm_dpo/delta": -0.21575535833835602, + "fcm_dpo/margin": 75.46543884277344, + "fcm_dpo/q_t": 0.3711055815219879, + "grad_norm": 22.26523780822754, + "learning_rate": 4.862120211153265e-07, + "logits/chosen": -0.4720418155193329, + "logits/rejected": -0.5066670775413513, + "logps/chosen": -101.28677368164062, + "logps/ref_chosen": -50.353858947753906, + "logps/ref_rejected": -115.97975158691406, + "logps/rejected": -242.3780975341797, + "loss": 0.9955, + "margin_dpo/margin_mean": 75.4654312133789, + "margin_dpo/margin_std": 94.92060089111328, + "step": 135 + }, + { + "KL/chosen_KL_mean": -60.29087829589844, + "KL/mean": -85.43303680419922, + "KL/rejected_KL_mean": -110.5751953125, + "KL/std": 70.91878509521484, + "epoch": 0.19970631424375918, + "fcm_dpo/beta": 0.00785889569669962, + "fcm_dpo/delta": 0.004331991076469421, + "fcm_dpo/margin": 50.2843132019043, + "fcm_dpo/q_t": 0.41889041662216187, + "grad_norm": 20.16181182861328, + "learning_rate": 4.857886086178193e-07, + "logits/chosen": -0.49389830231666565, + "logits/rejected": -0.4865124225616455, + "logps/chosen": -125.36338806152344, + "logps/ref_chosen": -65.072509765625, + "logps/ref_rejected": -96.32122802734375, + "logps/rejected": -206.89642333984375, + "loss": 1.1418, + "margin_dpo/margin_mean": 50.2843132019043, + "margin_dpo/margin_std": 90.57024383544922, + "step": 136 + }, + { + "KL/chosen_KL_mean": -57.76277542114258, + "KL/mean": -97.14057922363281, + "KL/rejected_KL_mean": -136.5183868408203, + "KL/std": 94.00605010986328, + "epoch": 0.2011747430249633, + "fcm_dpo/beta": 0.0076486412435770035, + "fcm_dpo/delta": -0.21565069258213043, + "fcm_dpo/margin": 78.755615234375, + "fcm_dpo/q_t": 0.37695467472076416, + "grad_norm": 18.521162033081055, + "learning_rate": 4.853589828711902e-07, + "logits/chosen": -0.4430182874202728, + "logits/rejected": -0.4700878858566284, + "logps/chosen": -106.52188873291016, + "logps/ref_chosen": -48.759117126464844, + "logps/ref_rejected": -113.86376953125, + "logps/rejected": -250.3821563720703, + "loss": 1.0248, + "margin_dpo/margin_mean": 78.755615234375, + "margin_dpo/margin_std": 112.62461853027344, + "step": 137 + }, + { + "KL/chosen_KL_mean": -60.64649963378906, + "KL/mean": -90.4873275756836, + "KL/rejected_KL_mean": -120.32815551757812, + "KL/std": 71.90509033203125, + "epoch": 0.2026431718061674, + "fcm_dpo/beta": 0.0075285714119672775, + "fcm_dpo/delta": -0.05161011964082718, + "fcm_dpo/margin": 59.68164825439453, + "fcm_dpo/q_t": 0.39715737104415894, + "grad_norm": 20.577062606811523, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": -0.42421677708625793, + "logits/rejected": -0.41229647397994995, + "logps/chosen": -121.16615295410156, + "logps/ref_chosen": -60.519649505615234, + "logps/ref_rejected": -93.19694519042969, + "logps/rejected": -213.5251007080078, + "loss": 1.0575, + "margin_dpo/margin_mean": 59.68164825439453, + "margin_dpo/margin_std": 73.65308380126953, + "step": 138 + }, + { + "KL/chosen_KL_mean": -50.748390197753906, + "KL/mean": -86.53669738769531, + "KL/rejected_KL_mean": -122.32501220703125, + "KL/std": 68.32363891601562, + "epoch": 0.20411160058737152, + "fcm_dpo/beta": 0.007364056073129177, + "fcm_dpo/delta": -0.13397160172462463, + "fcm_dpo/margin": 71.57662200927734, + "fcm_dpo/q_t": 0.38309505581855774, + "grad_norm": 18.44870948791504, + "learning_rate": 4.844811370781446e-07, + "logits/chosen": -0.4444272220134735, + "logits/rejected": -0.43558651208877563, + "logps/chosen": -97.6397705078125, + "logps/ref_chosen": -46.89138412475586, + "logps/ref_rejected": -79.72798156738281, + "logps/rejected": -202.05299377441406, + "loss": 1.0191, + "margin_dpo/margin_mean": 71.57662963867188, + "margin_dpo/margin_std": 88.49024963378906, + "step": 139 + }, + { + "KL/chosen_KL_mean": -61.39569091796875, + "KL/mean": -93.86390686035156, + "KL/rejected_KL_mean": -126.33212280273438, + "KL/std": 76.08689880371094, + "epoch": 0.2055800293685756, + "fcm_dpo/beta": 0.007215442135930061, + "fcm_dpo/delta": -0.07196357846260071, + "fcm_dpo/margin": 64.9364242553711, + "fcm_dpo/q_t": 0.395868718624115, + "grad_norm": 21.35501480102539, + "learning_rate": 4.840329401637809e-07, + "logits/chosen": -0.4337591230869293, + "logits/rejected": -0.42049670219421387, + "logps/chosen": -120.37040710449219, + "logps/ref_chosen": -58.97471618652344, + "logps/ref_rejected": -83.28410339355469, + "logps/rejected": -209.61622619628906, + "loss": 1.0684, + "margin_dpo/margin_mean": 64.93641662597656, + "margin_dpo/margin_std": 91.21418762207031, + "step": 140 + }, + { + "KL/chosen_KL_mean": -69.03340148925781, + "KL/mean": -99.73199462890625, + "KL/rejected_KL_mean": -130.4305877685547, + "KL/std": 83.39578247070312, + "epoch": 0.20704845814977973, + "fcm_dpo/beta": 0.007154383696615696, + "fcm_dpo/delta": -0.041043903678655624, + "fcm_dpo/margin": 61.39719009399414, + "fcm_dpo/q_t": 0.40136831998825073, + "grad_norm": 25.930021286010742, + "learning_rate": 4.83578576263792e-07, + "logits/chosen": -0.4134112596511841, + "logits/rejected": -0.40002214908599854, + "logps/chosen": -144.10906982421875, + "logps/ref_chosen": -75.07566833496094, + "logps/ref_rejected": -98.1922607421875, + "logps/rejected": -228.6228485107422, + "loss": 1.1017, + "margin_dpo/margin_mean": 61.397193908691406, + "margin_dpo/margin_std": 96.28628540039062, + "step": 141 + }, + { + "KL/chosen_KL_mean": -70.42542266845703, + "KL/mean": -105.56607055664062, + "KL/rejected_KL_mean": -140.70672607421875, + "KL/std": 91.09803009033203, + "epoch": 0.20851688693098386, + "fcm_dpo/beta": 0.0070582120679318905, + "fcm_dpo/delta": -0.10101625323295593, + "fcm_dpo/margin": 70.28129577636719, + "fcm_dpo/q_t": 0.39278340339660645, + "grad_norm": 26.267784118652344, + "learning_rate": 4.83118057351089e-07, + "logits/chosen": -0.42163795232772827, + "logits/rejected": -0.4213171601295471, + "logps/chosen": -128.45335388183594, + "logps/ref_chosen": -58.027931213378906, + "logps/ref_rejected": -94.58222961425781, + "logps/rejected": -235.2889404296875, + "loss": 1.0825, + "margin_dpo/margin_mean": 70.28129577636719, + "margin_dpo/margin_std": 107.12841796875, + "step": 142 + }, + { + "KL/chosen_KL_mean": -74.68773651123047, + "KL/mean": -97.2497787475586, + "KL/rejected_KL_mean": -119.81182098388672, + "KL/std": 80.62870788574219, + "epoch": 0.20998531571218795, + "fcm_dpo/beta": 0.0070522744208574295, + "fcm_dpo/delta": 0.08447334915399551, + "fcm_dpo/margin": 45.12409210205078, + "fcm_dpo/q_t": 0.43191662430763245, + "grad_norm": 23.529150009155273, + "learning_rate": 4.826513955607734e-07, + "logits/chosen": -0.3745364546775818, + "logits/rejected": -0.3672657012939453, + "logps/chosen": -132.2841796875, + "logps/ref_chosen": -57.59645080566406, + "logps/ref_rejected": -78.99957275390625, + "logps/rejected": -198.8114013671875, + "loss": 1.1956, + "margin_dpo/margin_mean": 45.12409210205078, + "margin_dpo/margin_std": 94.2938003540039, + "step": 143 + }, + { + "KL/chosen_KL_mean": -66.37725067138672, + "KL/mean": -93.94914245605469, + "KL/rejected_KL_mean": -121.52104187011719, + "KL/std": 68.17041015625, + "epoch": 0.21145374449339208, + "fcm_dpo/beta": 0.007112512364983559, + "fcm_dpo/delta": 0.008096275851130486, + "fcm_dpo/margin": 55.14379119873047, + "fcm_dpo/q_t": 0.4112512767314911, + "grad_norm": 20.934486389160156, + "learning_rate": 4.821786031898176e-07, + "logits/chosen": -0.4165929853916168, + "logits/rejected": -0.40653660893440247, + "logps/chosen": -126.28361511230469, + "logps/ref_chosen": -59.90636444091797, + "logps/ref_rejected": -82.00025939941406, + "logps/rejected": -203.52130126953125, + "loss": 1.1075, + "margin_dpo/margin_mean": 55.14379119873047, + "margin_dpo/margin_std": 79.93777465820312, + "step": 144 + }, + { + "KL/chosen_KL_mean": -64.00596618652344, + "KL/mean": -93.86626434326172, + "KL/rejected_KL_mean": -123.72657775878906, + "KL/std": 68.75868225097656, + "epoch": 0.21292217327459617, + "fcm_dpo/beta": 0.0070916153490543365, + "fcm_dpo/delta": -0.02458004653453827, + "fcm_dpo/margin": 59.720611572265625, + "fcm_dpo/q_t": 0.4040692448616028, + "grad_norm": 24.859176635742188, + "learning_rate": 4.816996926967401e-07, + "logits/chosen": -0.4160414934158325, + "logits/rejected": -0.4003496766090393, + "logps/chosen": -120.60662841796875, + "logps/ref_chosen": -56.60066604614258, + "logps/ref_rejected": -77.86631774902344, + "logps/rejected": -201.5928955078125, + "loss": 1.087, + "margin_dpo/margin_mean": 59.720611572265625, + "margin_dpo/margin_std": 83.28811645507812, + "step": 145 + }, + { + "KL/chosen_KL_mean": -86.9118881225586, + "KL/mean": -109.9712142944336, + "KL/rejected_KL_mean": -133.03054809570312, + "KL/std": 74.28648376464844, + "epoch": 0.2143906020558003, + "fcm_dpo/beta": 0.0071436744183301926, + "fcm_dpo/delta": 0.07298312336206436, + "fcm_dpo/margin": 46.11865997314453, + "fcm_dpo/q_t": 0.4252440631389618, + "grad_norm": 26.424964904785156, + "learning_rate": 4.812146767012779e-07, + "logits/chosen": -0.39881205558776855, + "logits/rejected": -0.3725563585758209, + "logps/chosen": -152.91233825683594, + "logps/ref_chosen": -66.00045013427734, + "logps/ref_rejected": -81.70278930664062, + "logps/rejected": -214.73333740234375, + "loss": 1.1818, + "margin_dpo/margin_mean": 46.11865997314453, + "margin_dpo/margin_std": 89.22908020019531, + "step": 146 + }, + { + "KL/chosen_KL_mean": -63.9739990234375, + "KL/mean": -95.299072265625, + "KL/rejected_KL_mean": -126.6241455078125, + "KL/std": 75.37117004394531, + "epoch": 0.21585903083700442, + "fcm_dpo/beta": 0.007126647979021072, + "fcm_dpo/delta": -0.0488114133477211, + "fcm_dpo/margin": 62.65013122558594, + "fcm_dpo/q_t": 0.4014296233654022, + "grad_norm": 20.018695831298828, + "learning_rate": 4.807235679840536e-07, + "logits/chosen": -0.45268571376800537, + "logits/rejected": -0.4357437789440155, + "logps/chosen": -117.37948608398438, + "logps/ref_chosen": -53.405487060546875, + "logps/ref_rejected": -71.39060974121094, + "logps/rejected": -198.01475524902344, + "loss": 1.0888, + "margin_dpo/margin_mean": 62.65013122558594, + "margin_dpo/margin_std": 93.385009765625, + "step": 147 + }, + { + "KL/chosen_KL_mean": -62.910282135009766, + "KL/mean": -89.76136016845703, + "KL/rejected_KL_mean": -116.61244201660156, + "KL/std": 75.14965057373047, + "epoch": 0.2173274596182085, + "fcm_dpo/beta": 0.00720440037548542, + "fcm_dpo/delta": 0.01156807690858841, + "fcm_dpo/margin": 53.70217514038086, + "fcm_dpo/q_t": 0.41519132256507874, + "grad_norm": 19.996816635131836, + "learning_rate": 4.802263794862384e-07, + "logits/chosen": -0.46612483263015747, + "logits/rejected": -0.45894235372543335, + "logps/chosen": -127.84736633300781, + "logps/ref_chosen": -64.93708038330078, + "logps/ref_rejected": -103.09384155273438, + "logps/rejected": -219.706298828125, + "loss": 1.1206, + "margin_dpo/margin_mean": 53.70217514038086, + "margin_dpo/margin_std": 79.02860260009766, + "step": 148 + }, + { + "KL/chosen_KL_mean": -60.414554595947266, + "KL/mean": -93.79571533203125, + "KL/rejected_KL_mean": -127.1768569946289, + "KL/std": 67.92532348632812, + "epoch": 0.21879588839941264, + "fcm_dpo/beta": 0.007002708502113819, + "fcm_dpo/delta": -0.07285458594560623, + "fcm_dpo/margin": 66.7623062133789, + "fcm_dpo/q_t": 0.39336636662483215, + "grad_norm": 18.49657440185547, + "learning_rate": 4.797231243092118e-07, + "logits/chosen": -0.4581308960914612, + "logits/rejected": -0.4425523281097412, + "logps/chosen": -118.88832092285156, + "logps/ref_chosen": -58.47376251220703, + "logps/ref_rejected": -99.31474304199219, + "logps/rejected": -226.49160766601562, + "loss": 1.0507, + "margin_dpo/margin_mean": 66.7623062133789, + "margin_dpo/margin_std": 80.3822021484375, + "step": 149 + }, + { + "KL/chosen_KL_mean": -52.977989196777344, + "KL/mean": -85.69601440429688, + "KL/rejected_KL_mean": -118.41404724121094, + "KL/std": 79.58135223388672, + "epoch": 0.22026431718061673, + "fcm_dpo/beta": 0.006925541907548904, + "fcm_dpo/delta": -0.057165369391441345, + "fcm_dpo/margin": 65.43606567382812, + "fcm_dpo/q_t": 0.40269482135772705, + "grad_norm": 18.195709228515625, + "learning_rate": 4.792138157142157e-07, + "logits/chosen": -0.4748048186302185, + "logits/rejected": -0.4793627858161926, + "logps/chosen": -98.68379974365234, + "logps/ref_chosen": -45.705810546875, + "logps/ref_rejected": -83.34759521484375, + "logps/rejected": -201.7616424560547, + "loss": 1.0769, + "margin_dpo/margin_mean": 65.4360580444336, + "margin_dpo/margin_std": 95.0787582397461, + "step": 150 + }, + { + "KL/chosen_KL_mean": -66.27272033691406, + "KL/mean": -99.1185531616211, + "KL/rejected_KL_mean": -131.96438598632812, + "KL/std": 75.18177032470703, + "epoch": 0.22173274596182085, + "fcm_dpo/beta": 0.006895341910421848, + "fcm_dpo/delta": -0.05550111457705498, + "fcm_dpo/margin": 65.69166564941406, + "fcm_dpo/q_t": 0.3965303897857666, + "grad_norm": 20.076454162597656, + "learning_rate": 4.786984671220053e-07, + "logits/chosen": -0.547601580619812, + "logits/rejected": -0.5237374901771545, + "logps/chosen": -136.84356689453125, + "logps/ref_chosen": -70.57083129882812, + "logps/ref_rejected": -100.46382141113281, + "logps/rejected": -232.42820739746094, + "loss": 1.0574, + "margin_dpo/margin_mean": 65.69166564941406, + "margin_dpo/margin_std": 82.30619812011719, + "step": 151 + }, + { + "KL/chosen_KL_mean": -57.727760314941406, + "KL/mean": -97.53009796142578, + "KL/rejected_KL_mean": -137.33242797851562, + "KL/std": 77.0035400390625, + "epoch": 0.22320117474302498, + "fcm_dpo/beta": 0.006748279090970755, + "fcm_dpo/delta": -0.14475369453430176, + "fcm_dpo/margin": 79.60468292236328, + "fcm_dpo/q_t": 0.3807300627231598, + "grad_norm": 20.364458084106445, + "learning_rate": 4.78177092112495e-07, + "logits/chosen": -0.5026451945304871, + "logits/rejected": -0.5023806095123291, + "logps/chosen": -117.89215087890625, + "logps/ref_chosen": -60.16438674926758, + "logps/ref_rejected": -106.14045715332031, + "logps/rejected": -243.472900390625, + "loss": 1.0132, + "margin_dpo/margin_mean": 79.60467529296875, + "margin_dpo/margin_std": 94.24575805664062, + "step": 152 + }, + { + "KL/chosen_KL_mean": -58.48396301269531, + "KL/mean": -92.23340606689453, + "KL/rejected_KL_mean": -125.98284912109375, + "KL/std": 83.61231231689453, + "epoch": 0.22466960352422907, + "fcm_dpo/beta": 0.006644603796303272, + "fcm_dpo/delta": -0.05075981840491295, + "fcm_dpo/margin": 67.49888610839844, + "fcm_dpo/q_t": 0.40270525217056274, + "grad_norm": 15.50756549835205, + "learning_rate": 4.776497044244016e-07, + "logits/chosen": -0.4666762948036194, + "logits/rejected": -0.4615291357040405, + "logps/chosen": -114.79924011230469, + "logps/ref_chosen": -56.315277099609375, + "logps/ref_rejected": -85.65583801269531, + "logps/rejected": -211.638671875, + "loss": 1.0883, + "margin_dpo/margin_mean": 67.49888610839844, + "margin_dpo/margin_std": 102.39913940429688, + "step": 153 + }, + { + "KL/chosen_KL_mean": -70.078369140625, + "KL/mean": -102.3911361694336, + "KL/rejected_KL_mean": -134.70388793945312, + "KL/std": 84.00700378417969, + "epoch": 0.2261380323054332, + "fcm_dpo/beta": 0.006613458506762981, + "fcm_dpo/delta": -0.02885277569293976, + "fcm_dpo/margin": 64.62553405761719, + "fcm_dpo/q_t": 0.40623587369918823, + "grad_norm": 19.01382827758789, + "learning_rate": 4.771163179548808e-07, + "logits/chosen": -0.4539533853530884, + "logits/rejected": -0.4565533995628357, + "logps/chosen": -132.82093811035156, + "logps/ref_chosen": -62.74256896972656, + "logps/ref_rejected": -104.24420166015625, + "logps/rejected": -238.94808959960938, + "loss": 1.1195, + "margin_dpo/margin_mean": 64.62552642822266, + "margin_dpo/margin_std": 106.87834167480469, + "step": 154 + }, + { + "KL/chosen_KL_mean": -63.6666374206543, + "KL/mean": -96.20714569091797, + "KL/rejected_KL_mean": -128.74766540527344, + "KL/std": 77.72993469238281, + "epoch": 0.2276064610866373, + "fcm_dpo/beta": 0.006566190160810947, + "fcm_dpo/delta": -0.028627503663301468, + "fcm_dpo/margin": 65.0810317993164, + "fcm_dpo/q_t": 0.40416377782821655, + "grad_norm": 19.003984451293945, + "learning_rate": 4.7657694675916247e-07, + "logits/chosen": -0.466948926448822, + "logits/rejected": -0.44710594415664673, + "logps/chosen": -124.31982421875, + "logps/ref_chosen": -60.65318298339844, + "logps/ref_rejected": -77.49220275878906, + "logps/rejected": -206.2398681640625, + "loss": 1.0947, + "margin_dpo/margin_mean": 65.0810317993164, + "margin_dpo/margin_std": 96.49385070800781, + "step": 155 + }, + { + "KL/chosen_KL_mean": -89.43757629394531, + "KL/mean": -108.11923217773438, + "KL/rejected_KL_mean": -126.8009033203125, + "KL/std": 82.73648071289062, + "epoch": 0.2290748898678414, + "fcm_dpo/beta": 0.006675416603684425, + "fcm_dpo/delta": 0.15469492971897125, + "fcm_dpo/margin": 37.36332702636719, + "fcm_dpo/q_t": 0.4441591501235962, + "grad_norm": 28.448909759521484, + "learning_rate": 4.7603160505017893e-07, + "logits/chosen": -0.42967504262924194, + "logits/rejected": -0.42389848828315735, + "logps/chosen": -158.929443359375, + "logps/ref_chosen": -69.49188232421875, + "logps/ref_rejected": -77.16929626464844, + "logps/rejected": -203.97018432617188, + "loss": 1.2757, + "margin_dpo/margin_mean": 37.36332702636719, + "margin_dpo/margin_std": 108.8001708984375, + "step": 156 + }, + { + "KL/chosen_KL_mean": -80.1298828125, + "KL/mean": -121.85565185546875, + "KL/rejected_KL_mean": -163.5814208984375, + "KL/std": 90.1138916015625, + "epoch": 0.2305433186490455, + "fcm_dpo/beta": 0.00655252393335104, + "fcm_dpo/delta": -0.1568107306957245, + "fcm_dpo/margin": 83.45154571533203, + "fcm_dpo/q_t": 0.3757891058921814, + "grad_norm": 23.798870086669922, + "learning_rate": 4.7548030719819154e-07, + "logits/chosen": -0.38265031576156616, + "logits/rejected": -0.3891392946243286, + "logps/chosen": -141.49832153320312, + "logps/ref_chosen": -61.368438720703125, + "logps/ref_rejected": -107.64636993408203, + "logps/rejected": -271.227783203125, + "loss": 1.0205, + "margin_dpo/margin_mean": 83.45154571533203, + "margin_dpo/margin_std": 102.42288208007812, + "step": 157 + }, + { + "KL/chosen_KL_mean": -80.79515075683594, + "KL/mean": -125.07054138183594, + "KL/rejected_KL_mean": -169.345947265625, + "KL/std": 111.43318176269531, + "epoch": 0.23201174743024963, + "fcm_dpo/beta": 0.006371090654283762, + "fcm_dpo/delta": -0.17393946647644043, + "fcm_dpo/margin": 88.55078125, + "fcm_dpo/q_t": 0.3846416473388672, + "grad_norm": 20.157474517822266, + "learning_rate": 4.7492306773041136e-07, + "logits/chosen": -0.37623435258865356, + "logits/rejected": -0.39367592334747314, + "logps/chosen": -138.4080810546875, + "logps/ref_chosen": -57.612918853759766, + "logps/ref_rejected": -113.6946792602539, + "logps/rejected": -283.0406188964844, + "loss": 1.0498, + "margin_dpo/margin_mean": 88.55077362060547, + "margin_dpo/margin_std": 134.24615478515625, + "step": 158 + }, + { + "KL/chosen_KL_mean": -91.09815979003906, + "KL/mean": -121.51444244384766, + "KL/rejected_KL_mean": -151.93072509765625, + "KL/std": 98.36724853515625, + "epoch": 0.23348017621145375, + "fcm_dpo/beta": 0.0063539366237819195, + "fcm_dpo/delta": 0.013539198786020279, + "fcm_dpo/margin": 60.832557678222656, + "fcm_dpo/q_t": 0.4142197370529175, + "grad_norm": 21.495512008666992, + "learning_rate": 4.743599013306165e-07, + "logits/chosen": -0.40772944688796997, + "logits/rejected": -0.3778040409088135, + "logps/chosen": -172.65850830078125, + "logps/ref_chosen": -81.56034851074219, + "logps/ref_rejected": -88.89871215820312, + "logps/rejected": -240.82943725585938, + "loss": 1.1449, + "margin_dpo/margin_mean": 60.832557678222656, + "margin_dpo/margin_std": 106.78931427001953, + "step": 159 + }, + { + "KL/chosen_KL_mean": -92.82463073730469, + "KL/mean": -133.309326171875, + "KL/rejected_KL_mean": -173.79403686523438, + "KL/std": 104.54073333740234, + "epoch": 0.23494860499265785, + "fcm_dpo/beta": 0.0061985463835299015, + "fcm_dpo/delta": -0.10827823728322983, + "fcm_dpo/margin": 80.96941375732422, + "fcm_dpo/q_t": 0.39520615339279175, + "grad_norm": 23.35076332092285, + "learning_rate": 4.737908228387656e-07, + "logits/chosen": -0.36829066276550293, + "logits/rejected": -0.35965633392333984, + "logps/chosen": -158.55551147460938, + "logps/ref_chosen": -65.73088073730469, + "logps/ref_rejected": -97.21781921386719, + "logps/rejected": -271.0118408203125, + "loss": 1.0896, + "margin_dpo/margin_mean": 80.96941375732422, + "margin_dpo/margin_std": 131.15676879882812, + "step": 160 + }, + { + "KL/chosen_KL_mean": -79.95537567138672, + "KL/mean": -115.55949401855469, + "KL/rejected_KL_mean": -151.1636199951172, + "KL/std": 82.77009582519531, + "epoch": 0.23641703377386197, + "fcm_dpo/beta": 0.006156083196401596, + "fcm_dpo/delta": -0.040130577981472015, + "fcm_dpo/margin": 71.20826721191406, + "fcm_dpo/q_t": 0.40316489338874817, + "grad_norm": 22.19649887084961, + "learning_rate": 4.7321584725060594e-07, + "logits/chosen": -0.37738293409347534, + "logits/rejected": -0.37736430764198303, + "logps/chosen": -132.391845703125, + "logps/ref_chosen": -52.43647003173828, + "logps/ref_rejected": -83.43095397949219, + "logps/rejected": -234.59457397460938, + "loss": 1.0892, + "margin_dpo/margin_mean": 71.20826721191406, + "margin_dpo/margin_std": 104.3830795288086, + "step": 161 + }, + { + "KL/chosen_KL_mean": -76.72035217285156, + "KL/mean": -112.05348205566406, + "KL/rejected_KL_mean": -147.3865966796875, + "KL/std": 91.60896301269531, + "epoch": 0.23788546255506607, + "fcm_dpo/beta": 0.0060644932091236115, + "fcm_dpo/delta": -0.031555503606796265, + "fcm_dpo/margin": 70.66625213623047, + "fcm_dpo/q_t": 0.40564799308776855, + "grad_norm": 22.75583267211914, + "learning_rate": 4.7263498971727905e-07, + "logits/chosen": -0.42984259128570557, + "logits/rejected": -0.414145290851593, + "logps/chosen": -139.3309326171875, + "logps/ref_chosen": -62.6105842590332, + "logps/ref_rejected": -89.39057922363281, + "logps/rejected": -236.77719116210938, + "loss": 1.1054, + "margin_dpo/margin_mean": 70.666259765625, + "margin_dpo/margin_std": 108.65666961669922, + "step": 162 + }, + { + "KL/chosen_KL_mean": -86.96597290039062, + "KL/mean": -121.1256103515625, + "KL/rejected_KL_mean": -155.28524780273438, + "KL/std": 92.2059326171875, + "epoch": 0.2393538913362702, + "fcm_dpo/beta": 0.006092265248298645, + "fcm_dpo/delta": -0.016920365393161774, + "fcm_dpo/margin": 68.31927490234375, + "fcm_dpo/q_t": 0.40940380096435547, + "grad_norm": 22.025920867919922, + "learning_rate": 4.720482655449212e-07, + "logits/chosen": -0.3410716652870178, + "logits/rejected": -0.3217606544494629, + "logps/chosen": -141.98760986328125, + "logps/ref_chosen": -55.021629333496094, + "logps/ref_rejected": -75.418212890625, + "logps/rejected": -230.70347595214844, + "loss": 1.1158, + "margin_dpo/margin_mean": 68.31927490234375, + "margin_dpo/margin_std": 111.27782440185547, + "step": 163 + }, + { + "KL/chosen_KL_mean": -78.96578979492188, + "KL/mean": -120.85997009277344, + "KL/rejected_KL_mean": -162.754150390625, + "KL/std": 89.56776428222656, + "epoch": 0.24082232011747431, + "fcm_dpo/beta": 0.005949662998318672, + "fcm_dpo/delta": -0.1053713783621788, + "fcm_dpo/margin": 83.78836822509766, + "fcm_dpo/q_t": 0.3873593211174011, + "grad_norm": 21.903921127319336, + "learning_rate": 4.714556901942599e-07, + "logits/chosen": -0.3370760381221771, + "logits/rejected": -0.32179197669029236, + "logps/chosen": -134.60646057128906, + "logps/ref_chosen": -55.64066696166992, + "logps/ref_rejected": -79.66463470458984, + "logps/rejected": -242.41879272460938, + "loss": 1.0341, + "margin_dpo/margin_mean": 83.78837585449219, + "margin_dpo/margin_std": 102.59002685546875, + "step": 164 + }, + { + "KL/chosen_KL_mean": -84.470703125, + "KL/mean": -111.83903503417969, + "KL/rejected_KL_mean": -139.20736694335938, + "KL/std": 75.84541320800781, + "epoch": 0.2422907488986784, + "fcm_dpo/beta": 0.006008328404277563, + "fcm_dpo/delta": 0.0735950842499733, + "fcm_dpo/margin": 54.73666000366211, + "fcm_dpo/q_t": 0.4263015389442444, + "grad_norm": 22.944101333618164, + "learning_rate": 4.708572792802069e-07, + "logits/chosen": -0.3864282965660095, + "logits/rejected": -0.36028358340263367, + "logps/chosen": -145.78138732910156, + "logps/ref_chosen": -61.310691833496094, + "logps/ref_rejected": -73.67060852050781, + "logps/rejected": -212.8779754638672, + "loss": 1.1705, + "margin_dpo/margin_mean": 54.736656188964844, + "margin_dpo/margin_std": 100.56073760986328, + "step": 165 + }, + { + "KL/chosen_KL_mean": -74.94708251953125, + "KL/mean": -125.40420532226562, + "KL/rejected_KL_mean": -175.861328125, + "KL/std": 110.8086929321289, + "epoch": 0.24375917767988253, + "fcm_dpo/beta": 0.005840961821377277, + "fcm_dpo/delta": -0.20264847576618195, + "fcm_dpo/margin": 100.91423034667969, + "fcm_dpo/q_t": 0.37943559885025024, + "grad_norm": 17.69853401184082, + "learning_rate": 4.702530485714461e-07, + "logits/chosen": -0.33663398027420044, + "logits/rejected": -0.3471418023109436, + "logps/chosen": -125.9306869506836, + "logps/ref_chosen": -50.98360061645508, + "logps/ref_rejected": -98.09512329101562, + "logps/rejected": -273.9564514160156, + "loss": 1.0146, + "margin_dpo/margin_mean": 100.91423034667969, + "margin_dpo/margin_std": 139.43954467773438, + "step": 166 + }, + { + "KL/chosen_KL_mean": -75.11039733886719, + "KL/mean": -128.34188842773438, + "KL/rejected_KL_mean": -181.57339477539062, + "KL/std": 100.28507232666016, + "epoch": 0.24522760646108663, + "fcm_dpo/beta": 0.005624993704259396, + "fcm_dpo/delta": -0.21160906553268433, + "fcm_dpo/margin": 106.46297454833984, + "fcm_dpo/q_t": 0.3662090003490448, + "grad_norm": 23.289794921875, + "learning_rate": 4.6964301399001877e-07, + "logits/chosen": -0.33210843801498413, + "logits/rejected": -0.33472272753715515, + "logps/chosen": -125.53449249267578, + "logps/ref_chosen": -50.424095153808594, + "logps/ref_rejected": -96.03042602539062, + "logps/rejected": -277.60382080078125, + "loss": 0.968, + "margin_dpo/margin_mean": 106.46297454833984, + "margin_dpo/margin_std": 115.56710815429688, + "step": 167 + }, + { + "KL/chosen_KL_mean": -81.955078125, + "KL/mean": -120.9576187133789, + "KL/rejected_KL_mean": -159.9601593017578, + "KL/std": 94.59892272949219, + "epoch": 0.24669603524229075, + "fcm_dpo/beta": 0.005527859553694725, + "fcm_dpo/delta": -0.03275269269943237, + "fcm_dpo/margin": 78.00506591796875, + "fcm_dpo/q_t": 0.40346774458885193, + "grad_norm": 20.936565399169922, + "learning_rate": 4.690271916109034e-07, + "logits/chosen": -0.3593074083328247, + "logits/rejected": -0.35054898262023926, + "logps/chosen": -131.41790771484375, + "logps/ref_chosen": -49.462825775146484, + "logps/ref_rejected": -75.30855560302734, + "logps/rejected": -235.26870727539062, + "loss": 1.078, + "margin_dpo/margin_mean": 78.00506591796875, + "margin_dpo/margin_std": 105.96997833251953, + "step": 168 + }, + { + "KL/chosen_KL_mean": -83.62703704833984, + "KL/mean": -117.71241760253906, + "KL/rejected_KL_mean": -151.7978057861328, + "KL/std": 92.74655151367188, + "epoch": 0.24816446402349487, + "fcm_dpo/beta": 0.005502847954630852, + "fcm_dpo/delta": 0.02455396205186844, + "fcm_dpo/margin": 68.17076110839844, + "fcm_dpo/q_t": 0.41967761516571045, + "grad_norm": 20.715503692626953, + "learning_rate": 4.6840559766159235e-07, + "logits/chosen": -0.36550360918045044, + "logits/rejected": -0.34896135330200195, + "logps/chosen": -143.43048095703125, + "logps/ref_chosen": -59.803443908691406, + "logps/ref_rejected": -83.34574890136719, + "logps/rejected": -235.1435546875, + "loss": 1.1596, + "margin_dpo/margin_mean": 68.17076110839844, + "margin_dpo/margin_std": 127.58938598632812, + "step": 169 + }, + { + "KL/chosen_KL_mean": -73.78776550292969, + "KL/mean": -114.02574157714844, + "KL/rejected_KL_mean": -154.26370239257812, + "KL/std": 88.0094985961914, + "epoch": 0.24963289280469897, + "fcm_dpo/beta": 0.005487416870892048, + "fcm_dpo/delta": -0.04451485723257065, + "fcm_dpo/margin": 80.47593688964844, + "fcm_dpo/q_t": 0.39898285269737244, + "grad_norm": 18.808998107910156, + "learning_rate": 4.6777824852166437e-07, + "logits/chosen": -0.3425253629684448, + "logits/rejected": -0.3321423828601837, + "logps/chosen": -123.25953674316406, + "logps/ref_chosen": -49.471771240234375, + "logps/ref_rejected": -75.91734313964844, + "logps/rejected": -230.18104553222656, + "loss": 1.0729, + "margin_dpo/margin_mean": 80.47593688964844, + "margin_dpo/margin_std": 105.5477066040039, + "step": 170 + }, + { + "KL/chosen_KL_mean": -107.6456298828125, + "KL/mean": -140.05889892578125, + "KL/rejected_KL_mean": -172.47218322753906, + "KL/std": 101.28620910644531, + "epoch": 0.2511013215859031, + "fcm_dpo/beta": 0.005539648234844208, + "fcm_dpo/delta": 0.04242313653230667, + "fcm_dpo/margin": 64.82654571533203, + "fcm_dpo/q_t": 0.4240008294582367, + "grad_norm": 29.54085922241211, + "learning_rate": 4.6714516072235273e-07, + "logits/chosen": -0.34802329540252686, + "logits/rejected": -0.32956576347351074, + "logps/chosen": -192.1449432373047, + "logps/ref_chosen": -84.49931335449219, + "logps/ref_rejected": -109.38209533691406, + "logps/rejected": -281.8542785644531, + "loss": 1.1712, + "margin_dpo/margin_mean": 64.82654571533203, + "margin_dpo/margin_std": 129.8333282470703, + "step": 171 + }, + { + "KL/chosen_KL_mean": -94.89369201660156, + "KL/mean": -128.6036376953125, + "KL/rejected_KL_mean": -162.31356811523438, + "KL/std": 97.58683776855469, + "epoch": 0.2525697503671072, + "fcm_dpo/beta": 0.005568898282945156, + "fcm_dpo/delta": 0.025504067540168762, + "fcm_dpo/margin": 67.41987609863281, + "fcm_dpo/q_t": 0.41584065556526184, + "grad_norm": 24.327449798583984, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": -0.3986096680164337, + "logits/rejected": -0.3816487789154053, + "logps/chosen": -163.547607421875, + "logps/ref_chosen": -68.65391540527344, + "logps/ref_rejected": -85.43667602539062, + "logps/rejected": -247.750244140625, + "loss": 1.1322, + "margin_dpo/margin_mean": 67.41987609863281, + "margin_dpo/margin_std": 110.49188232421875, + "step": 172 + }, + { + "KL/chosen_KL_mean": -87.27731323242188, + "KL/mean": -121.42498779296875, + "KL/rejected_KL_mean": -155.57266235351562, + "KL/std": 92.45211791992188, + "epoch": 0.2540381791483113, + "fcm_dpo/beta": 0.005617057904601097, + "fcm_dpo/delta": 0.016708562150597572, + "fcm_dpo/margin": 68.29533386230469, + "fcm_dpo/q_t": 0.41303884983062744, + "grad_norm": 20.406827926635742, + "learning_rate": 4.6586183602616687e-07, + "logits/chosen": -0.41884762048721313, + "logits/rejected": -0.39233148097991943, + "logps/chosen": -150.32818603515625, + "logps/ref_chosen": -63.050880432128906, + "logps/ref_rejected": -78.68392181396484, + "logps/rejected": -234.256591796875, + "loss": 1.1075, + "margin_dpo/margin_mean": 68.29534149169922, + "margin_dpo/margin_std": 96.916748046875, + "step": 173 + }, + { + "KL/chosen_KL_mean": -83.62928771972656, + "KL/mean": -124.2885971069336, + "KL/rejected_KL_mean": -164.94790649414062, + "KL/std": 97.81199645996094, + "epoch": 0.2555066079295154, + "fcm_dpo/beta": 0.005588578060269356, + "fcm_dpo/delta": -0.05747688561677933, + "fcm_dpo/margin": 81.31861114501953, + "fcm_dpo/q_t": 0.40022996068000793, + "grad_norm": 21.319252014160156, + "learning_rate": 4.652116329460919e-07, + "logits/chosen": -0.3324238061904907, + "logits/rejected": -0.34984150528907776, + "logps/chosen": -136.99224853515625, + "logps/ref_chosen": -53.36296844482422, + "logps/ref_rejected": -101.91120910644531, + "logps/rejected": -266.859130859375, + "loss": 1.0818, + "margin_dpo/margin_mean": 81.31861114501953, + "margin_dpo/margin_std": 116.7562026977539, + "step": 174 + }, + { + "KL/chosen_KL_mean": -77.55169677734375, + "KL/mean": -133.1820068359375, + "KL/rejected_KL_mean": -188.81231689453125, + "KL/std": 105.05592346191406, + "epoch": 0.25697503671071953, + "fcm_dpo/beta": 0.0053607383742928505, + "fcm_dpo/delta": -0.20923927426338196, + "fcm_dpo/margin": 111.2606201171875, + "fcm_dpo/q_t": 0.3642469048500061, + "grad_norm": 26.155202865600586, + "learning_rate": 4.645557588393406e-07, + "logits/chosen": -0.28884798288345337, + "logits/rejected": -0.2741377353668213, + "logps/chosen": -122.96946716308594, + "logps/ref_chosen": -45.417762756347656, + "logps/ref_rejected": -89.50579833984375, + "logps/rejected": -278.318115234375, + "loss": 0.9525, + "margin_dpo/margin_mean": 111.2606201171875, + "margin_dpo/margin_std": 109.48503875732422, + "step": 175 + }, + { + "KL/chosen_KL_mean": -84.14366149902344, + "KL/mean": -130.95176696777344, + "KL/rejected_KL_mean": -177.7598876953125, + "KL/std": 104.69883728027344, + "epoch": 0.25844346549192365, + "fcm_dpo/beta": 0.005244849249720573, + "fcm_dpo/delta": -0.09559769183397293, + "fcm_dpo/margin": 93.61622619628906, + "fcm_dpo/q_t": 0.3924563527107239, + "grad_norm": 19.009029388427734, + "learning_rate": 4.638942309888058e-07, + "logits/chosen": -0.3173462450504303, + "logits/rejected": -0.3352587819099426, + "logps/chosen": -134.59649658203125, + "logps/ref_chosen": -50.452842712402344, + "logps/ref_rejected": -95.5589599609375, + "logps/rejected": -273.31884765625, + "loss": 1.0433, + "margin_dpo/margin_mean": 93.61622619628906, + "margin_dpo/margin_std": 121.37356567382812, + "step": 176 + }, + { + "KL/chosen_KL_mean": -98.31048583984375, + "KL/mean": -144.74362182617188, + "KL/rejected_KL_mean": -191.17674255371094, + "KL/std": 113.88668823242188, + "epoch": 0.2599118942731278, + "fcm_dpo/beta": 0.005161338951438665, + "fcm_dpo/delta": -0.08324076235294342, + "fcm_dpo/margin": 92.86624145507812, + "fcm_dpo/q_t": 0.3939231038093567, + "grad_norm": 26.65350914001465, + "learning_rate": 4.6322706682636137e-07, + "logits/chosen": -0.316158264875412, + "logits/rejected": -0.30725324153900146, + "logps/chosen": -159.52694702148438, + "logps/ref_chosen": -61.216468811035156, + "logps/ref_rejected": -95.89378356933594, + "logps/rejected": -287.0705261230469, + "loss": 1.0487, + "margin_dpo/margin_mean": 92.86624145507812, + "margin_dpo/margin_std": 120.88250732421875, + "step": 177 + }, + { + "KL/chosen_KL_mean": -109.49505615234375, + "KL/mean": -168.2081298828125, + "KL/rejected_KL_mean": -226.9211883544922, + "KL/std": 134.9578094482422, + "epoch": 0.26138032305433184, + "fcm_dpo/beta": 0.004957615397870541, + "fcm_dpo/delta": -0.19456958770751953, + "fcm_dpo/margin": 117.42613983154297, + "fcm_dpo/q_t": 0.3746492266654968, + "grad_norm": 25.899173736572266, + "learning_rate": 4.6255428393240354e-07, + "logits/chosen": -0.247529536485672, + "logits/rejected": -0.23764248192310333, + "logps/chosen": -167.7598419189453, + "logps/ref_chosen": -58.26478958129883, + "logps/ref_rejected": -105.3653335571289, + "logps/rejected": -332.2865295410156, + "loss": 1.0001, + "margin_dpo/margin_mean": 117.42613983154297, + "margin_dpo/margin_std": 148.1576385498047, + "step": 178 + }, + { + "KL/chosen_KL_mean": -113.17926025390625, + "KL/mean": -154.49534606933594, + "KL/rejected_KL_mean": -195.81143188476562, + "KL/std": 114.84059143066406, + "epoch": 0.26284875183553597, + "fcm_dpo/beta": 0.004900245927274227, + "fcm_dpo/delta": -0.005644813179969788, + "fcm_dpo/margin": 82.63217163085938, + "fcm_dpo/q_t": 0.4111664593219757, + "grad_norm": 32.68442153930664, + "learning_rate": 4.6187590003538724e-07, + "logits/chosen": -0.27366673946380615, + "logits/rejected": -0.28250646591186523, + "logps/chosen": -174.23757934570312, + "logps/ref_chosen": -61.05832290649414, + "logps/ref_rejected": -90.52782440185547, + "logps/rejected": -286.3392639160156, + "loss": 1.1353, + "margin_dpo/margin_mean": 82.63216400146484, + "margin_dpo/margin_std": 142.4680633544922, + "step": 179 + }, + { + "KL/chosen_KL_mean": -98.1256103515625, + "KL/mean": -151.30715942382812, + "KL/rejected_KL_mean": -204.48873901367188, + "KL/std": 103.5050048828125, + "epoch": 0.2643171806167401, + "fcm_dpo/beta": 0.004829989746212959, + "fcm_dpo/delta": -0.12006718665361404, + "fcm_dpo/margin": 106.36312866210938, + "fcm_dpo/q_t": 0.38355523347854614, + "grad_norm": 20.321218490600586, + "learning_rate": 4.611919330113591e-07, + "logits/chosen": -0.30043309926986694, + "logits/rejected": -0.2955361604690552, + "logps/chosen": -152.46832275390625, + "logps/ref_chosen": -54.34272003173828, + "logps/ref_rejected": -98.21183776855469, + "logps/rejected": -302.7005615234375, + "loss": 1.0233, + "margin_dpo/margin_mean": 106.36312866210938, + "margin_dpo/margin_std": 128.10833740234375, + "step": 180 + }, + { + "KL/chosen_KL_mean": -84.74370574951172, + "KL/mean": -117.97935485839844, + "KL/rejected_KL_mean": -151.21499633789062, + "KL/std": 94.81138610839844, + "epoch": 0.2657856093979442, + "fcm_dpo/beta": 0.0048685576766729355, + "fcm_dpo/delta": 0.07881682366132736, + "fcm_dpo/margin": 66.4712905883789, + "fcm_dpo/q_t": 0.42652463912963867, + "grad_norm": 18.8144474029541, + "learning_rate": 4.605024008834863e-07, + "logits/chosen": -0.2951802611351013, + "logits/rejected": -0.2693043351173401, + "logps/chosen": -139.74417114257812, + "logps/ref_chosen": -55.000457763671875, + "logps/ref_rejected": -61.656166076660156, + "logps/rejected": -212.87115478515625, + "loss": 1.1676, + "margin_dpo/margin_mean": 66.4712905883789, + "margin_dpo/margin_std": 119.32975006103516, + "step": 181 + }, + { + "KL/chosen_KL_mean": -79.9010009765625, + "KL/mean": -137.60540771484375, + "KL/rejected_KL_mean": -195.309814453125, + "KL/std": 114.05545043945312, + "epoch": 0.26725403817914833, + "fcm_dpo/beta": 0.004749820567667484, + "fcm_dpo/delta": -0.157757967710495, + "fcm_dpo/margin": 115.4088363647461, + "fcm_dpo/q_t": 0.37575432658195496, + "grad_norm": 18.484952926635742, + "learning_rate": 4.598073218215817e-07, + "logits/chosen": -0.2895713448524475, + "logits/rejected": -0.29972368478775024, + "logps/chosen": -121.00885009765625, + "logps/ref_chosen": -41.107852935791016, + "logps/ref_rejected": -89.5215835571289, + "logps/rejected": -284.8314208984375, + "loss": 1.0091, + "margin_dpo/margin_mean": 115.4088363647461, + "margin_dpo/margin_std": 137.26278686523438, + "step": 182 + }, + { + "KL/chosen_KL_mean": -117.18058776855469, + "KL/mean": -148.19338989257812, + "KL/rejected_KL_mean": -179.2061767578125, + "KL/std": 94.16122436523438, + "epoch": 0.2687224669603524, + "fcm_dpo/beta": 0.004760707728564739, + "fcm_dpo/delta": 0.10700409859418869, + "fcm_dpo/margin": 62.02560043334961, + "fcm_dpo/q_t": 0.4322977662086487, + "grad_norm": 21.09385108947754, + "learning_rate": 4.5910671414162484e-07, + "logits/chosen": -0.2902984619140625, + "logits/rejected": -0.2783670127391815, + "logps/chosen": -174.7051544189453, + "logps/ref_chosen": -57.52456283569336, + "logps/ref_rejected": -75.97572326660156, + "logps/rejected": -255.18190002441406, + "loss": 1.1813, + "margin_dpo/margin_mean": 62.02560043334961, + "margin_dpo/margin_std": 103.5242691040039, + "step": 183 + }, + { + "KL/chosen_KL_mean": -97.4403076171875, + "KL/mean": -130.1512908935547, + "KL/rejected_KL_mean": -162.86228942871094, + "KL/std": 88.96430969238281, + "epoch": 0.2701908957415565, + "fcm_dpo/beta": 0.004917686805129051, + "fcm_dpo/delta": 0.07984302192926407, + "fcm_dpo/margin": 65.42196655273438, + "fcm_dpo/q_t": 0.42779117822647095, + "grad_norm": 19.217758178710938, + "learning_rate": 4.5840059630527985e-07, + "logits/chosen": -0.34908443689346313, + "logits/rejected": -0.3400712013244629, + "logps/chosen": -155.98526000976562, + "logps/ref_chosen": -58.544952392578125, + "logps/ref_rejected": -76.63406372070312, + "logps/rejected": -239.496337890625, + "loss": 1.1633, + "margin_dpo/margin_mean": 65.42196655273438, + "margin_dpo/margin_std": 111.91808319091797, + "step": 184 + }, + { + "KL/chosen_KL_mean": -104.18730926513672, + "KL/mean": -129.8303680419922, + "KL/rejected_KL_mean": -155.47341918945312, + "KL/std": 100.83888244628906, + "epoch": 0.27165932452276065, + "fcm_dpo/beta": 0.00501694530248642, + "fcm_dpo/delta": 0.14654436707496643, + "fcm_dpo/margin": 51.286109924316406, + "fcm_dpo/q_t": 0.44505059719085693, + "grad_norm": 21.227956771850586, + "learning_rate": 4.5768898691940836e-07, + "logits/chosen": -0.29932302236557007, + "logits/rejected": -0.27377188205718994, + "logps/chosen": -166.21316528320312, + "logps/ref_chosen": -62.025848388671875, + "logps/ref_rejected": -73.7625961303711, + "logps/rejected": -229.23602294921875, + "loss": 1.2302, + "margin_dpo/margin_mean": 51.286109924316406, + "margin_dpo/margin_std": 121.88388061523438, + "step": 185 + }, + { + "KL/chosen_KL_mean": -95.56036376953125, + "KL/mean": -144.005126953125, + "KL/rejected_KL_mean": -192.44989013671875, + "KL/std": 101.79664611816406, + "epoch": 0.27312775330396477, + "fcm_dpo/beta": 0.004984917119145393, + "fcm_dpo/delta": -0.08735480904579163, + "fcm_dpo/margin": 96.88951873779297, + "fcm_dpo/q_t": 0.3907889723777771, + "grad_norm": 28.150287628173828, + "learning_rate": 4.5697190473557947e-07, + "logits/chosen": -0.3715486228466034, + "logits/rejected": -0.3490146994590759, + "logps/chosen": -164.91383361816406, + "logps/ref_chosen": -69.35346984863281, + "logps/ref_rejected": -88.07244873046875, + "logps/rejected": -280.5223388671875, + "loss": 1.0364, + "margin_dpo/margin_mean": 96.88951873779297, + "margin_dpo/margin_std": 116.3585205078125, + "step": 186 + }, + { + "KL/chosen_KL_mean": -89.00492858886719, + "KL/mean": -129.27084350585938, + "KL/rejected_KL_mean": -169.53675842285156, + "KL/std": 96.89251708984375, + "epoch": 0.2745961820851689, + "fcm_dpo/beta": 0.004988536238670349, + "fcm_dpo/delta": -0.0021523088216781616, + "fcm_dpo/margin": 80.53184509277344, + "fcm_dpo/q_t": 0.40814200043678284, + "grad_norm": 23.734664916992188, + "learning_rate": 4.5624936864957555e-07, + "logits/chosen": -0.3111017942428589, + "logits/rejected": -0.3041296601295471, + "logps/chosen": -141.76138305664062, + "logps/ref_chosen": -52.7564582824707, + "logps/ref_rejected": -81.96910095214844, + "logps/rejected": -251.505859375, + "loss": 1.0907, + "margin_dpo/margin_mean": 80.5318374633789, + "margin_dpo/margin_std": 105.92333984375, + "step": 187 + }, + { + "KL/chosen_KL_mean": -81.93511962890625, + "KL/mean": -130.56826782226562, + "KL/rejected_KL_mean": -179.20138549804688, + "KL/std": 106.78981018066406, + "epoch": 0.27606461086637296, + "fcm_dpo/beta": 0.0049115014262497425, + "fcm_dpo/delta": -0.0815688967704773, + "fcm_dpo/margin": 97.26626586914062, + "fcm_dpo/q_t": 0.3923574984073639, + "grad_norm": 31.012367248535156, + "learning_rate": 4.5552139770089454e-07, + "logits/chosen": -0.3215259909629822, + "logits/rejected": -0.3264486491680145, + "logps/chosen": -131.35060119628906, + "logps/ref_chosen": -49.415489196777344, + "logps/ref_rejected": -89.54043579101562, + "logps/rejected": -268.7418212890625, + "loss": 1.039, + "margin_dpo/margin_mean": 97.26626586914062, + "margin_dpo/margin_std": 116.94786071777344, + "step": 188 + }, + { + "KL/chosen_KL_mean": -94.69155883789062, + "KL/mean": -135.35601806640625, + "KL/rejected_KL_mean": -176.02047729492188, + "KL/std": 106.91317749023438, + "epoch": 0.2775330396475771, + "fcm_dpo/beta": 0.004893161356449127, + "fcm_dpo/delta": 0.002108026295900345, + "fcm_dpo/margin": 81.32891082763672, + "fcm_dpo/q_t": 0.41269636154174805, + "grad_norm": 29.160770416259766, + "learning_rate": 4.5478801107224794e-07, + "logits/chosen": -0.34372690320014954, + "logits/rejected": -0.3232707381248474, + "logps/chosen": -147.09051513671875, + "logps/ref_chosen": -52.39896011352539, + "logps/ref_rejected": -72.16735076904297, + "logps/rejected": -248.1878204345703, + "loss": 1.1171, + "margin_dpo/margin_mean": 81.32891845703125, + "margin_dpo/margin_std": 131.2293701171875, + "step": 189 + }, + { + "KL/chosen_KL_mean": -101.5174560546875, + "KL/mean": -149.2252197265625, + "KL/rejected_KL_mean": -196.93295288085938, + "KL/std": 114.80913543701172, + "epoch": 0.2790014684287812, + "fcm_dpo/beta": 0.004881042987108231, + "fcm_dpo/delta": -0.06988224387168884, + "fcm_dpo/margin": 95.41551971435547, + "fcm_dpo/q_t": 0.3969564437866211, + "grad_norm": 19.110261917114258, + "learning_rate": 4.5404922808905543e-07, + "logits/chosen": -0.3549377918243408, + "logits/rejected": -0.3403692841529846, + "logps/chosen": -166.20053100585938, + "logps/ref_chosen": -64.68305969238281, + "logps/ref_rejected": -102.55052185058594, + "logps/rejected": -299.4834899902344, + "loss": 1.075, + "margin_dpo/margin_mean": 95.41551971435547, + "margin_dpo/margin_std": 132.25625610351562, + "step": 190 + }, + { + "KL/chosen_KL_mean": -94.4166259765625, + "KL/mean": -162.89239501953125, + "KL/rejected_KL_mean": -231.3681640625, + "KL/std": 133.07308959960938, + "epoch": 0.28046989720998533, + "fcm_dpo/beta": 0.004625319503247738, + "fcm_dpo/delta": -0.250690221786499, + "fcm_dpo/margin": 136.9515380859375, + "fcm_dpo/q_t": 0.3601779639720917, + "grad_norm": 20.274974822998047, + "learning_rate": 4.5330506821893565e-07, + "logits/chosen": -0.34608668088912964, + "logits/rejected": -0.3248242735862732, + "logps/chosen": -163.07550048828125, + "logps/ref_chosen": -68.65887451171875, + "logps/ref_rejected": -110.1396713256836, + "logps/rejected": -341.5078125, + "loss": 0.9469, + "margin_dpo/margin_mean": 136.9515380859375, + "margin_dpo/margin_std": 146.31927490234375, + "step": 191 + }, + { + "KL/chosen_KL_mean": -122.44120788574219, + "KL/mean": -167.78265380859375, + "KL/rejected_KL_mean": -213.12411499023438, + "KL/std": 115.74710845947266, + "epoch": 0.28193832599118945, + "fcm_dpo/beta": 0.004585623275488615, + "fcm_dpo/delta": -0.016613848507404327, + "fcm_dpo/margin": 90.68290710449219, + "fcm_dpo/q_t": 0.4083861708641052, + "grad_norm": 23.432592391967773, + "learning_rate": 4.5255555107119336e-07, + "logits/chosen": -0.32005661725997925, + "logits/rejected": -0.3190295696258545, + "logps/chosen": -192.16812133789062, + "logps/ref_chosen": -69.72691345214844, + "logps/ref_rejected": -103.32135009765625, + "logps/rejected": -316.4454650878906, + "loss": 1.1089, + "margin_dpo/margin_mean": 90.68290710449219, + "margin_dpo/margin_std": 142.26962280273438, + "step": 192 + }, + { + "KL/chosen_KL_mean": -123.967041015625, + "KL/mean": -150.6930694580078, + "KL/rejected_KL_mean": -177.4190673828125, + "KL/std": 106.99697875976562, + "epoch": 0.2834067547723935, + "fcm_dpo/beta": 0.004696063697338104, + "fcm_dpo/delta": 0.15172256529331207, + "fcm_dpo/margin": 53.45201873779297, + "fcm_dpo/q_t": 0.4412054121494293, + "grad_norm": 27.434032440185547, + "learning_rate": 4.5180069639630236e-07, + "logits/chosen": -0.3420638144016266, + "logits/rejected": -0.3330894708633423, + "logps/chosen": -184.1575469970703, + "logps/ref_chosen": -60.19049835205078, + "logps/ref_rejected": -76.40755462646484, + "logps/rejected": -253.82662963867188, + "loss": 1.249, + "margin_dpo/margin_mean": 53.4520263671875, + "margin_dpo/margin_std": 136.81741333007812, + "step": 193 + }, + { + "KL/chosen_KL_mean": -77.48358154296875, + "KL/mean": -120.73272705078125, + "KL/rejected_KL_mean": -163.98187255859375, + "KL/std": 89.46562957763672, + "epoch": 0.28487518355359764, + "fcm_dpo/beta": 0.004693003837019205, + "fcm_dpo/delta": -0.006373826414346695, + "fcm_dpo/margin": 86.498291015625, + "fcm_dpo/q_t": 0.40509384870529175, + "grad_norm": 18.315134048461914, + "learning_rate": 4.510405240853854e-07, + "logits/chosen": -0.25825223326683044, + "logits/rejected": -0.24261374771595, + "logps/chosen": -115.32395935058594, + "logps/ref_chosen": -37.84037399291992, + "logps/ref_rejected": -60.684783935546875, + "logps/rejected": -224.66665649414062, + "loss": 1.0718, + "margin_dpo/margin_mean": 86.498291015625, + "margin_dpo/margin_std": 98.0810775756836, + "step": 194 + }, + { + "KL/chosen_KL_mean": -122.55229949951172, + "KL/mean": -169.00955200195312, + "KL/rejected_KL_mean": -215.466796875, + "KL/std": 110.25206756591797, + "epoch": 0.28634361233480177, + "fcm_dpo/beta": 0.004678967874497175, + "fcm_dpo/delta": -0.03632781654596329, + "fcm_dpo/margin": 92.91448974609375, + "fcm_dpo/q_t": 0.4003722071647644, + "grad_norm": 21.463722229003906, + "learning_rate": 4.5027505416968985e-07, + "logits/chosen": -0.28670287132263184, + "logits/rejected": -0.3056129813194275, + "logps/chosen": -177.44387817382812, + "logps/ref_chosen": -54.891571044921875, + "logps/ref_rejected": -96.77095794677734, + "logps/rejected": -312.23773193359375, + "loss": 1.0636, + "margin_dpo/margin_mean": 92.91448974609375, + "margin_dpo/margin_std": 114.20263671875, + "step": 195 + }, + { + "KL/chosen_KL_mean": -96.20404815673828, + "KL/mean": -149.17095947265625, + "KL/rejected_KL_mean": -202.1378631591797, + "KL/std": 113.6821060180664, + "epoch": 0.2878120411160059, + "fcm_dpo/beta": 0.004583077039569616, + "fcm_dpo/delta": -0.09096772968769073, + "fcm_dpo/margin": 105.9338150024414, + "fcm_dpo/q_t": 0.39174580574035645, + "grad_norm": 18.06390380859375, + "learning_rate": 4.495043068200599e-07, + "logits/chosen": -0.32519471645355225, + "logits/rejected": -0.31019407510757446, + "logps/chosen": -149.4492950439453, + "logps/ref_chosen": -53.245243072509766, + "logps/ref_rejected": -76.05294799804688, + "logps/rejected": -278.1907958984375, + "loss": 1.0516, + "margin_dpo/margin_mean": 105.9338150024414, + "margin_dpo/margin_std": 136.15292358398438, + "step": 196 + }, + { + "KL/chosen_KL_mean": -99.3115234375, + "KL/mean": -140.2286376953125, + "KL/rejected_KL_mean": -181.14573669433594, + "KL/std": 98.75188446044922, + "epoch": 0.28928046989721, + "fcm_dpo/beta": 0.004617801867425442, + "fcm_dpo/delta": 0.022662952542304993, + "fcm_dpo/margin": 81.83421325683594, + "fcm_dpo/q_t": 0.41381216049194336, + "grad_norm": 18.633390426635742, + "learning_rate": 4.4872830234640493e-07, + "logits/chosen": -0.31022053956985474, + "logits/rejected": -0.30425357818603516, + "logps/chosen": -159.7318572998047, + "logps/ref_chosen": -60.42033386230469, + "logps/ref_rejected": -77.20890808105469, + "logps/rejected": -258.3546447753906, + "loss": 1.1065, + "margin_dpo/margin_mean": 81.83421325683594, + "margin_dpo/margin_std": 111.92158508300781, + "step": 197 + }, + { + "KL/chosen_KL_mean": -112.12559509277344, + "KL/mean": -164.85427856445312, + "KL/rejected_KL_mean": -217.58297729492188, + "KL/std": 124.64435577392578, + "epoch": 0.2907488986784141, + "fcm_dpo/beta": 0.004552255384624004, + "fcm_dpo/delta": -0.08407382667064667, + "fcm_dpo/margin": 105.45738220214844, + "fcm_dpo/q_t": 0.3944292962551117, + "grad_norm": 22.357629776000977, + "learning_rate": 4.479470611971645e-07, + "logits/chosen": -0.32955610752105713, + "logits/rejected": -0.32924240827560425, + "logps/chosen": -167.16177368164062, + "logps/ref_chosen": -55.03618621826172, + "logps/ref_rejected": -97.24325561523438, + "logps/rejected": -314.82623291015625, + "loss": 1.0521, + "margin_dpo/margin_mean": 105.45738983154297, + "margin_dpo/margin_std": 141.28903198242188, + "step": 198 + }, + { + "KL/chosen_KL_mean": -107.11283874511719, + "KL/mean": -159.80538940429688, + "KL/rejected_KL_mean": -212.49794006347656, + "KL/std": 113.97561645507812, + "epoch": 0.2922173274596182, + "fcm_dpo/beta": 0.004454015754163265, + "fcm_dpo/delta": -0.07400257885456085, + "fcm_dpo/margin": 105.38510131835938, + "fcm_dpo/q_t": 0.3943500220775604, + "grad_norm": 21.528413772583008, + "learning_rate": 4.471606039587695e-07, + "logits/chosen": -0.3109109401702881, + "logits/rejected": -0.2927241027355194, + "logps/chosen": -163.94166564941406, + "logps/ref_chosen": -56.828826904296875, + "logps/ref_rejected": -84.64820861816406, + "logps/rejected": -297.1461486816406, + "loss": 1.0586, + "margin_dpo/margin_mean": 105.38510131835938, + "margin_dpo/margin_std": 137.27755737304688, + "step": 199 + }, + { + "KL/chosen_KL_mean": -106.08656311035156, + "KL/mean": -158.6329345703125, + "KL/rejected_KL_mean": -211.1792755126953, + "KL/std": 122.60960388183594, + "epoch": 0.2936857562408223, + "fcm_dpo/beta": 0.004403954371809959, + "fcm_dpo/delta": -0.06639470160007477, + "fcm_dpo/margin": 105.09272003173828, + "fcm_dpo/q_t": 0.3984246253967285, + "grad_norm": 21.341712951660156, + "learning_rate": 4.4636895135509966e-07, + "logits/chosen": -0.27302879095077515, + "logits/rejected": -0.2546115517616272, + "logps/chosen": -159.15362548828125, + "logps/ref_chosen": -53.06706237792969, + "logps/ref_rejected": -80.60843658447266, + "logps/rejected": -291.7877197265625, + "loss": 1.0826, + "margin_dpo/margin_mean": 105.09272003173828, + "margin_dpo/margin_std": 156.76052856445312, + "step": 200 + }, + { + "KL/chosen_KL_mean": -112.1373062133789, + "KL/mean": -163.8036651611328, + "KL/rejected_KL_mean": -215.47003173828125, + "KL/std": 128.93569946289062, + "epoch": 0.29515418502202645, + "fcm_dpo/beta": 0.0043801432475447655, + "fcm_dpo/delta": -0.05509566143155098, + "fcm_dpo/margin": 103.33273315429688, + "fcm_dpo/q_t": 0.39835184812545776, + "grad_norm": 21.340782165527344, + "learning_rate": 4.455721242469372e-07, + "logits/chosen": -0.37157952785491943, + "logits/rejected": -0.36827552318573, + "logps/chosen": -187.53952026367188, + "logps/ref_chosen": -75.4022216796875, + "logps/ref_rejected": -114.80821990966797, + "logps/rejected": -330.27825927734375, + "loss": 1.0749, + "margin_dpo/margin_mean": 103.33273315429688, + "margin_dpo/margin_std": 143.89541625976562, + "step": 201 + }, + { + "KL/chosen_KL_mean": -114.76101684570312, + "KL/mean": -151.6962127685547, + "KL/rejected_KL_mean": -188.63140869140625, + "KL/std": 110.6797866821289, + "epoch": 0.2966226138032305, + "fcm_dpo/beta": 0.004410090856254101, + "fcm_dpo/delta": 0.07666227966547012, + "fcm_dpo/margin": 73.87037658691406, + "fcm_dpo/q_t": 0.42786508798599243, + "grad_norm": 22.023473739624023, + "learning_rate": 4.4477014363141755e-07, + "logits/chosen": -0.28071317076683044, + "logits/rejected": -0.2946799397468567, + "logps/chosen": -164.86233520507812, + "logps/ref_chosen": -50.101318359375, + "logps/ref_rejected": -86.98503112792969, + "logps/rejected": -275.616455078125, + "loss": 1.1785, + "margin_dpo/margin_mean": 73.87037658691406, + "margin_dpo/margin_std": 142.80360412597656, + "step": 202 + }, + { + "KL/chosen_KL_mean": -114.09391784667969, + "KL/mean": -159.502197265625, + "KL/rejected_KL_mean": -204.91046142578125, + "KL/std": 113.20623779296875, + "epoch": 0.29809104258443464, + "fcm_dpo/beta": 0.004417533054947853, + "fcm_dpo/delta": -0.0012725908309221268, + "fcm_dpo/margin": 90.81655883789062, + "fcm_dpo/q_t": 0.4084164500236511, + "grad_norm": 23.425310134887695, + "learning_rate": 4.439630306414758e-07, + "logits/chosen": -0.3172285556793213, + "logits/rejected": -0.3071226179599762, + "logps/chosen": -174.70361328125, + "logps/ref_chosen": -60.60969543457031, + "logps/ref_rejected": -85.89596557617188, + "logps/rejected": -290.80645751953125, + "loss": 1.0936, + "margin_dpo/margin_mean": 90.81655883789062, + "margin_dpo/margin_std": 123.70480346679688, + "step": 203 + }, + { + "KL/chosen_KL_mean": -127.60630798339844, + "KL/mean": -169.1259002685547, + "KL/rejected_KL_mean": -210.64549255371094, + "KL/std": 124.28507995605469, + "epoch": 0.29955947136563876, + "fcm_dpo/beta": 0.0044425297528505325, + "fcm_dpo/delta": 0.032297275960445404, + "fcm_dpo/margin": 83.0391845703125, + "fcm_dpo/q_t": 0.4190390110015869, + "grad_norm": 23.64316749572754, + "learning_rate": 4.431508065452897e-07, + "logits/chosen": -0.4175642132759094, + "logits/rejected": -0.38105836510658264, + "logps/chosen": -207.77127075195312, + "logps/ref_chosen": -80.16496276855469, + "logps/ref_rejected": -87.69590759277344, + "logps/rejected": -298.3414001464844, + "loss": 1.1456, + "margin_dpo/margin_mean": 83.0391845703125, + "margin_dpo/margin_std": 144.03652954101562, + "step": 204 + }, + { + "KL/chosen_KL_mean": -121.0059814453125, + "KL/mean": -176.66131591796875, + "KL/rejected_KL_mean": -232.31663513183594, + "KL/std": 126.79728698730469, + "epoch": 0.3010279001468429, + "fcm_dpo/beta": 0.004353957716375589, + "fcm_dpo/delta": -0.09127674996852875, + "fcm_dpo/margin": 111.31065368652344, + "fcm_dpo/q_t": 0.3891245722770691, + "grad_norm": 20.96511459350586, + "learning_rate": 4.4233349274571974e-07, + "logits/chosen": -0.3102046549320221, + "logits/rejected": -0.27934738993644714, + "logps/chosen": -180.39071655273438, + "logps/ref_chosen": -59.384735107421875, + "logps/ref_rejected": -85.12505340576172, + "logps/rejected": -317.4416809082031, + "loss": 1.0485, + "margin_dpo/margin_mean": 111.31065368652344, + "margin_dpo/margin_std": 137.294677734375, + "step": 205 + }, + { + "KL/chosen_KL_mean": -110.99642181396484, + "KL/mean": -170.51837158203125, + "KL/rejected_KL_mean": -230.0403594970703, + "KL/std": 118.66337585449219, + "epoch": 0.302496328928047, + "fcm_dpo/beta": 0.004280811175704002, + "fcm_dpo/delta": -0.1161835789680481, + "fcm_dpo/margin": 119.0439224243164, + "fcm_dpo/q_t": 0.38180607557296753, + "grad_norm": 25.16000747680664, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": -0.2330382764339447, + "logits/rejected": -0.23527702689170837, + "logps/chosen": -157.96092224121094, + "logps/ref_chosen": -46.964500427246094, + "logps/ref_rejected": -98.9534912109375, + "logps/rejected": -328.99383544921875, + "loss": 1.0108, + "margin_dpo/margin_mean": 119.0439224243164, + "margin_dpo/margin_std": 128.36184692382812, + "step": 206 + }, + { + "KL/chosen_KL_mean": -102.92005920410156, + "KL/mean": -171.51202392578125, + "KL/rejected_KL_mean": -240.10397338867188, + "KL/std": 136.4534912109375, + "epoch": 0.3039647577092511, + "fcm_dpo/beta": 0.004162299912422895, + "fcm_dpo/delta": -0.18118005990982056, + "fcm_dpo/margin": 137.18392944335938, + "fcm_dpo/q_t": 0.3742104470729828, + "grad_norm": 23.807788848876953, + "learning_rate": 4.4068368231789365e-07, + "logits/chosen": -0.33577364683151245, + "logits/rejected": -0.3098195493221283, + "logps/chosen": -158.976318359375, + "logps/ref_chosen": -56.05625915527344, + "logps/ref_rejected": -84.44779968261719, + "logps/rejected": -324.5517578125, + "loss": 0.9884, + "margin_dpo/margin_mean": 137.18392944335938, + "margin_dpo/margin_std": 158.6349334716797, + "step": 207 + }, + { + "KL/chosen_KL_mean": -158.25042724609375, + "KL/mean": -212.32925415039062, + "KL/rejected_KL_mean": -266.4080810546875, + "KL/std": 131.32470703125, + "epoch": 0.3054331864904552, + "fcm_dpo/beta": 0.00408403854817152, + "fcm_dpo/delta": -0.043813541531562805, + "fcm_dpo/margin": 108.15763092041016, + "fcm_dpo/q_t": 0.4003239870071411, + "grad_norm": 26.89968490600586, + "learning_rate": 4.398512291636768e-07, + "logits/chosen": -0.3362833261489868, + "logits/rejected": -0.3170960247516632, + "logps/chosen": -225.31805419921875, + "logps/ref_chosen": -67.06761169433594, + "logps/ref_rejected": -94.28689575195312, + "logps/rejected": -360.6949462890625, + "loss": 1.0899, + "margin_dpo/margin_mean": 108.15763092041016, + "margin_dpo/margin_std": 159.3949737548828, + "step": 208 + }, + { + "KL/chosen_KL_mean": -132.40557861328125, + "KL/mean": -179.732421875, + "KL/rejected_KL_mean": -227.05926513671875, + "KL/std": 117.1972427368164, + "epoch": 0.3069016152716593, + "fcm_dpo/beta": 0.004092153161764145, + "fcm_dpo/delta": 0.013167150318622589, + "fcm_dpo/margin": 94.65370178222656, + "fcm_dpo/q_t": 0.41229110956192017, + "grad_norm": 28.013294219970703, + "learning_rate": 4.3901377325300857e-07, + "logits/chosen": -0.24393108487129211, + "logits/rejected": -0.23185241222381592, + "logps/chosen": -188.58726501464844, + "logps/ref_chosen": -56.18169403076172, + "logps/ref_rejected": -80.94152069091797, + "logps/rejected": -308.00079345703125, + "loss": 1.1249, + "margin_dpo/margin_mean": 94.6537094116211, + "margin_dpo/margin_std": 148.58241271972656, + "step": 209 + }, + { + "KL/chosen_KL_mean": -121.67991638183594, + "KL/mean": -176.57435607910156, + "KL/rejected_KL_mean": -231.46881103515625, + "KL/std": 121.48208618164062, + "epoch": 0.30837004405286345, + "fcm_dpo/beta": 0.004080562852323055, + "fcm_dpo/delta": -0.0504092276096344, + "fcm_dpo/margin": 109.78889465332031, + "fcm_dpo/q_t": 0.39947184920310974, + "grad_norm": 26.426563262939453, + "learning_rate": 4.381713366536311e-07, + "logits/chosen": -0.27577272057533264, + "logits/rejected": -0.26780596375465393, + "logps/chosen": -168.05172729492188, + "logps/ref_chosen": -46.371822357177734, + "logps/ref_rejected": -76.68162536621094, + "logps/rejected": -308.1504211425781, + "loss": 1.0716, + "margin_dpo/margin_mean": 109.78888702392578, + "margin_dpo/margin_std": 147.1390838623047, + "step": 210 + }, + { + "KL/chosen_KL_mean": -172.16531372070312, + "KL/mean": -219.18504333496094, + "KL/rejected_KL_mean": -266.20477294921875, + "KL/std": 139.02566528320312, + "epoch": 0.30983847283406757, + "fcm_dpo/beta": 0.004065372049808502, + "fcm_dpo/delta": 0.018387429416179657, + "fcm_dpo/margin": 94.03945922851562, + "fcm_dpo/q_t": 0.4178032875061035, + "grad_norm": 36.138675689697266, + "learning_rate": 4.373239415645323e-07, + "logits/chosen": -0.31168001890182495, + "logits/rejected": -0.27302902936935425, + "logps/chosen": -251.09767150878906, + "logps/ref_chosen": -78.93235778808594, + "logps/ref_rejected": -86.82098388671875, + "logps/rejected": -353.0257568359375, + "loss": 1.1431, + "margin_dpo/margin_mean": 94.03945922851562, + "margin_dpo/margin_std": 163.8000030517578, + "step": 211 + }, + { + "KL/chosen_KL_mean": -140.9364776611328, + "KL/mean": -207.66456604003906, + "KL/rejected_KL_mean": -274.39263916015625, + "KL/std": 148.56857299804688, + "epoch": 0.31130690161527164, + "fcm_dpo/beta": 0.003955514170229435, + "fcm_dpo/delta": -0.13721655309200287, + "fcm_dpo/margin": 133.4561767578125, + "fcm_dpo/q_t": 0.382011741399765, + "grad_norm": 24.687803268432617, + "learning_rate": 4.3647161031536086e-07, + "logits/chosen": -0.31145745515823364, + "logits/rejected": -0.30414634943008423, + "logps/chosen": -199.13348388671875, + "logps/ref_chosen": -58.19701385498047, + "logps/ref_rejected": -103.05785369873047, + "logps/rejected": -377.45050048828125, + "loss": 1.0306, + "margin_dpo/margin_mean": 133.4561767578125, + "margin_dpo/margin_std": 164.7611846923828, + "step": 212 + }, + { + "KL/chosen_KL_mean": -132.23367309570312, + "KL/mean": -195.90211486816406, + "KL/rejected_KL_mean": -259.570556640625, + "KL/std": 128.409912109375, + "epoch": 0.31277533039647576, + "fcm_dpo/beta": 0.0038843729998916388, + "fcm_dpo/delta": -0.10020861029624939, + "fcm_dpo/margin": 127.33690643310547, + "fcm_dpo/q_t": 0.3873175084590912, + "grad_norm": 31.009477615356445, + "learning_rate": 4.3561436536583774e-07, + "logits/chosen": -0.30835479497909546, + "logits/rejected": -0.2814680337905884, + "logps/chosen": -199.74636840820312, + "logps/ref_chosen": -67.51271057128906, + "logps/ref_rejected": -93.91471862792969, + "logps/rejected": -353.48529052734375, + "loss": 1.0333, + "margin_dpo/margin_mean": 127.33689880371094, + "margin_dpo/margin_std": 152.68551635742188, + "step": 213 + }, + { + "KL/chosen_KL_mean": -113.12945556640625, + "KL/mean": -170.78250122070312, + "KL/rejected_KL_mean": -228.435546875, + "KL/std": 127.31915283203125, + "epoch": 0.3142437591776799, + "fcm_dpo/beta": 0.003852786961942911, + "fcm_dpo/delta": -0.046301376074552536, + "fcm_dpo/margin": 115.30607604980469, + "fcm_dpo/q_t": 0.39957842230796814, + "grad_norm": 23.4969425201416, + "learning_rate": 4.3475222930516473e-07, + "logits/chosen": -0.21872687339782715, + "logits/rejected": -0.22276321053504944, + "logps/chosen": -154.73434448242188, + "logps/ref_chosen": -41.604888916015625, + "logps/ref_rejected": -77.51741027832031, + "logps/rejected": -305.95294189453125, + "loss": 1.0659, + "margin_dpo/margin_mean": 115.30607604980469, + "margin_dpo/margin_std": 148.88754272460938, + "step": 214 + }, + { + "KL/chosen_KL_mean": -133.77352905273438, + "KL/mean": -193.89248657226562, + "KL/rejected_KL_mean": -254.01144409179688, + "KL/std": 130.903076171875, + "epoch": 0.315712187958884, + "fcm_dpo/beta": 0.0037993849255144596, + "fcm_dpo/delta": -0.059836942702531815, + "fcm_dpo/margin": 120.23790740966797, + "fcm_dpo/q_t": 0.39470916986465454, + "grad_norm": 25.146631240844727, + "learning_rate": 4.3388522485142885e-07, + "logits/chosen": -0.2989373803138733, + "logits/rejected": -0.29156550765037537, + "logps/chosen": -187.05279541015625, + "logps/ref_chosen": -53.279266357421875, + "logps/ref_rejected": -89.96464538574219, + "logps/rejected": -343.97607421875, + "loss": 1.0433, + "margin_dpo/margin_mean": 120.23790740966797, + "margin_dpo/margin_std": 136.16165161132812, + "step": 215 + }, + { + "KL/chosen_KL_mean": -137.1045379638672, + "KL/mean": -197.48190307617188, + "KL/rejected_KL_mean": -257.85931396484375, + "KL/std": 137.61212158203125, + "epoch": 0.31718061674008813, + "fcm_dpo/beta": 0.0037784408777952194, + "fcm_dpo/delta": -0.059020306915044785, + "fcm_dpo/margin": 120.75474548339844, + "fcm_dpo/q_t": 0.39792197942733765, + "grad_norm": 22.765789031982422, + "learning_rate": 4.330133748510036e-07, + "logits/chosen": -0.2852722406387329, + "logits/rejected": -0.26953035593032837, + "logps/chosen": -185.99234008789062, + "logps/ref_chosen": -48.887794494628906, + "logps/ref_rejected": -77.19892883300781, + "logps/rejected": -335.0582275390625, + "loss": 1.0761, + "margin_dpo/margin_mean": 120.75474548339844, + "margin_dpo/margin_std": 169.55313110351562, + "step": 216 + }, + { + "KL/chosen_KL_mean": -138.76242065429688, + "KL/mean": -209.293701171875, + "KL/rejected_KL_mean": -279.824951171875, + "KL/std": 140.5864715576172, + "epoch": 0.3186490455212922, + "fcm_dpo/beta": 0.003676855470985174, + "fcm_dpo/delta": -0.12545619904994965, + "fcm_dpo/margin": 141.06256103515625, + "fcm_dpo/q_t": 0.3828177750110626, + "grad_norm": 20.623205184936523, + "learning_rate": 4.3213670227794757e-07, + "logits/chosen": -0.2778016924858093, + "logits/rejected": -0.27289605140686035, + "logps/chosen": -188.60772705078125, + "logps/ref_chosen": -49.845306396484375, + "logps/ref_rejected": -100.07832336425781, + "logps/rejected": -379.9032897949219, + "loss": 1.0089, + "margin_dpo/margin_mean": 141.06256103515625, + "margin_dpo/margin_std": 158.33660888671875, + "step": 217 + }, + { + "KL/chosen_KL_mean": -147.40609741210938, + "KL/mean": -201.98513793945312, + "KL/rejected_KL_mean": -256.5641784667969, + "KL/std": 138.4207763671875, + "epoch": 0.3201174743024963, + "fcm_dpo/beta": 0.003653950057923794, + "fcm_dpo/delta": 0.0011254064738750458, + "fcm_dpo/margin": 109.15806579589844, + "fcm_dpo/q_t": 0.410250723361969, + "grad_norm": 21.11243438720703, + "learning_rate": 4.3125523023339815e-07, + "logits/chosen": -0.2692360579967499, + "logits/rejected": -0.2633015513420105, + "logps/chosen": -205.9827880859375, + "logps/ref_chosen": -58.576683044433594, + "logps/ref_rejected": -87.84639739990234, + "logps/rejected": -344.41058349609375, + "loss": 1.108, + "margin_dpo/margin_mean": 109.15806579589844, + "margin_dpo/margin_std": 161.23675537109375, + "step": 218 + }, + { + "KL/chosen_KL_mean": -157.17605590820312, + "KL/mean": -204.95626831054688, + "KL/rejected_KL_mean": -252.73646545410156, + "KL/std": 144.8588104248047, + "epoch": 0.32158590308370044, + "fcm_dpo/beta": 0.003704340197145939, + "fcm_dpo/delta": 0.04699729382991791, + "fcm_dpo/margin": 95.56039428710938, + "fcm_dpo/q_t": 0.420939564704895, + "grad_norm": 30.59574317932129, + "learning_rate": 4.303689819449636e-07, + "logits/chosen": -0.3211911916732788, + "logits/rejected": -0.3160223066806793, + "logps/chosen": -218.25991821289062, + "logps/ref_chosen": -61.083858489990234, + "logps/ref_rejected": -85.83042907714844, + "logps/rejected": -338.56689453125, + "loss": 1.1644, + "margin_dpo/margin_mean": 95.56039428710938, + "margin_dpo/margin_std": 177.19955444335938, + "step": 219 + }, + { + "KL/chosen_KL_mean": -177.92498779296875, + "KL/mean": -217.5970001220703, + "KL/rejected_KL_mean": -257.26898193359375, + "KL/std": 125.33465576171875, + "epoch": 0.32305433186490456, + "fcm_dpo/beta": 0.003746958449482918, + "fcm_dpo/delta": 0.10597775876522064, + "fcm_dpo/margin": 79.34402465820312, + "fcm_dpo/q_t": 0.4305725693702698, + "grad_norm": 28.42041015625, + "learning_rate": 4.2947798076611047e-07, + "logits/chosen": -0.30295485258102417, + "logits/rejected": -0.28097742795944214, + "logps/chosen": -247.95626831054688, + "logps/ref_chosen": -70.03128051757812, + "logps/ref_rejected": -87.68551635742188, + "logps/rejected": -344.95452880859375, + "loss": 1.1672, + "margin_dpo/margin_mean": 79.34402465820312, + "margin_dpo/margin_std": 129.00704956054688, + "step": 220 + }, + { + "KL/chosen_KL_mean": -154.29933166503906, + "KL/mean": -240.99725341796875, + "KL/rejected_KL_mean": -327.6951904296875, + "KL/std": 159.48818969726562, + "epoch": 0.3245227606461087, + "fcm_dpo/beta": 0.0036492447834461927, + "fcm_dpo/delta": -0.24793776869773865, + "fcm_dpo/margin": 173.39584350585938, + "fcm_dpo/q_t": 0.35594797134399414, + "grad_norm": 23.717262268066406, + "learning_rate": 4.285822501755485e-07, + "logits/chosen": -0.270913690328598, + "logits/rejected": -0.2767089307308197, + "logps/chosen": -206.45404052734375, + "logps/ref_chosen": -52.15470886230469, + "logps/ref_rejected": -106.46768188476562, + "logps/rejected": -434.162841796875, + "loss": 0.9334, + "margin_dpo/margin_mean": 173.39584350585938, + "margin_dpo/margin_std": 163.9197998046875, + "step": 221 + }, + { + "KL/chosen_KL_mean": -159.87034606933594, + "KL/mean": -225.4850616455078, + "KL/rejected_KL_mean": -291.09979248046875, + "KL/std": 147.8228759765625, + "epoch": 0.32599118942731276, + "fcm_dpo/beta": 0.0035684143658727407, + "fcm_dpo/delta": -0.07168504595756531, + "fcm_dpo/margin": 131.2294464111328, + "fcm_dpo/q_t": 0.3936772048473358, + "grad_norm": 20.67112922668457, + "learning_rate": 4.276818137766118e-07, + "logits/chosen": -0.3012784719467163, + "logits/rejected": -0.30357640981674194, + "logps/chosen": -220.84144592285156, + "logps/ref_chosen": -60.971099853515625, + "logps/ref_rejected": -100.00115203857422, + "logps/rejected": -391.1009521484375, + "loss": 1.0517, + "margin_dpo/margin_mean": 131.22943115234375, + "margin_dpo/margin_std": 164.2476348876953, + "step": 222 + }, + { + "KL/chosen_KL_mean": -169.24302673339844, + "KL/mean": -224.17355346679688, + "KL/rejected_KL_mean": -279.1040954589844, + "KL/std": 143.19537353515625, + "epoch": 0.3274596182085169, + "fcm_dpo/beta": 0.003542605321854353, + "fcm_dpo/delta": 0.011229310184717178, + "fcm_dpo/margin": 109.86106872558594, + "fcm_dpo/q_t": 0.412454217672348, + "grad_norm": 24.183177947998047, + "learning_rate": 4.2677669529663686e-07, + "logits/chosen": -0.22766795754432678, + "logits/rejected": -0.22294780611991882, + "logps/chosen": -221.88360595703125, + "logps/ref_chosen": -52.64057540893555, + "logps/ref_rejected": -82.82502746582031, + "logps/rejected": -361.92913818359375, + "loss": 1.1302, + "margin_dpo/margin_mean": 109.86106872558594, + "margin_dpo/margin_std": 182.3267364501953, + "step": 223 + }, + { + "KL/chosen_KL_mean": -147.57431030273438, + "KL/mean": -209.97825622558594, + "KL/rejected_KL_mean": -272.3822021484375, + "KL/std": 158.82989501953125, + "epoch": 0.328928046989721, + "fcm_dpo/beta": 0.003500790800899267, + "fcm_dpo/delta": -0.04011113941669464, + "fcm_dpo/margin": 124.80789184570312, + "fcm_dpo/q_t": 0.40373706817626953, + "grad_norm": 25.53409767150879, + "learning_rate": 4.2586691858633747e-07, + "logits/chosen": -0.25661876797676086, + "logits/rejected": -0.23753324151039124, + "logps/chosen": -196.1697235107422, + "logps/ref_chosen": -48.59541320800781, + "logps/ref_rejected": -77.11648559570312, + "logps/rejected": -349.4986572265625, + "loss": 1.0905, + "margin_dpo/margin_mean": 124.8078842163086, + "margin_dpo/margin_std": 181.3145294189453, + "step": 224 + }, + { + "KL/chosen_KL_mean": -164.8527069091797, + "KL/mean": -239.45616149902344, + "KL/rejected_KL_mean": -314.05963134765625, + "KL/std": 152.40457153320312, + "epoch": 0.3303964757709251, + "fcm_dpo/beta": 0.003453275188803673, + "fcm_dpo/delta": -0.12168119102716446, + "fcm_dpo/margin": 149.2069091796875, + "fcm_dpo/q_t": 0.38449984788894653, + "grad_norm": 24.14275550842285, + "learning_rate": 4.249525076191759e-07, + "logits/chosen": -0.27758195996284485, + "logits/rejected": -0.266676127910614, + "logps/chosen": -222.85317993164062, + "logps/ref_chosen": -58.000465393066406, + "logps/ref_rejected": -99.90291595458984, + "logps/rejected": -413.9625244140625, + "loss": 1.0292, + "margin_dpo/margin_mean": 149.2069091796875, + "margin_dpo/margin_std": 187.06063842773438, + "step": 225 + }, + { + "KL/chosen_KL_mean": -138.45205688476562, + "KL/mean": -197.54412841796875, + "KL/rejected_KL_mean": -256.63623046875, + "KL/std": 149.68759155273438, + "epoch": 0.33186490455212925, + "fcm_dpo/beta": 0.0034129605628550053, + "fcm_dpo/delta": -0.004264920949935913, + "fcm_dpo/margin": 118.18417358398438, + "fcm_dpo/q_t": 0.41034334897994995, + "grad_norm": 29.583045959472656, + "learning_rate": 4.2403348649073167e-07, + "logits/chosen": -0.38879233598709106, + "logits/rejected": -0.3535831570625305, + "logps/chosen": -197.35086059570312, + "logps/ref_chosen": -58.898799896240234, + "logps/ref_rejected": -78.68775939941406, + "logps/rejected": -335.323974609375, + "loss": 1.1057, + "margin_dpo/margin_mean": 118.18417358398438, + "margin_dpo/margin_std": 173.61318969726562, + "step": 226 + }, + { + "KL/chosen_KL_mean": -158.3021697998047, + "KL/mean": -232.66543579101562, + "KL/rejected_KL_mean": -307.0286865234375, + "KL/std": 167.4959716796875, + "epoch": 0.3333333333333333, + "fcm_dpo/beta": 0.003365355310961604, + "fcm_dpo/delta": -0.10656450688838959, + "fcm_dpo/margin": 148.72653198242188, + "fcm_dpo/q_t": 0.3860970139503479, + "grad_norm": 20.811311721801758, + "learning_rate": 4.2310987941806615e-07, + "logits/chosen": -0.3414561152458191, + "logits/rejected": -0.3285638093948364, + "logps/chosen": -217.3743438720703, + "logps/ref_chosen": -59.072181701660156, + "logps/ref_rejected": -99.41236877441406, + "logps/rejected": -406.4410705566406, + "loss": 1.0252, + "margin_dpo/margin_mean": 148.72653198242188, + "margin_dpo/margin_std": 174.76016235351562, + "step": 227 + }, + { + "KL/chosen_KL_mean": -167.0030975341797, + "KL/mean": -219.13742065429688, + "KL/rejected_KL_mean": -271.271728515625, + "KL/std": 138.55203247070312, + "epoch": 0.33480176211453744, + "fcm_dpo/beta": 0.0033898483961820602, + "fcm_dpo/delta": 0.04812644422054291, + "fcm_dpo/margin": 104.26863098144531, + "fcm_dpo/q_t": 0.4190364480018616, + "grad_norm": 23.566688537597656, + "learning_rate": 4.2218171073908463e-07, + "logits/chosen": -0.31667008996009827, + "logits/rejected": -0.29874318838119507, + "logps/chosen": -232.89437866210938, + "logps/ref_chosen": -65.89128875732422, + "logps/ref_rejected": -91.04875183105469, + "logps/rejected": -362.32049560546875, + "loss": 1.141, + "margin_dpo/margin_mean": 104.26863098144531, + "margin_dpo/margin_std": 169.02606201171875, + "step": 228 + }, + { + "KL/chosen_KL_mean": -172.6579132080078, + "KL/mean": -228.57550048828125, + "KL/rejected_KL_mean": -284.4930419921875, + "KL/std": 159.468017578125, + "epoch": 0.33627019089574156, + "fcm_dpo/beta": 0.003405486699193716, + "fcm_dpo/delta": 0.01987878978252411, + "fcm_dpo/margin": 111.83513641357422, + "fcm_dpo/q_t": 0.41233646869659424, + "grad_norm": 26.473695755004883, + "learning_rate": 4.212490049118951e-07, + "logits/chosen": -0.41149044036865234, + "logits/rejected": -0.38208431005477905, + "logps/chosen": -243.36428833007812, + "logps/ref_chosen": -70.70637512207031, + "logps/ref_rejected": -84.52741241455078, + "logps/rejected": -369.0204772949219, + "loss": 1.1162, + "margin_dpo/margin_mean": 111.83513641357422, + "margin_dpo/margin_std": 166.87164306640625, + "step": 229 + }, + { + "KL/chosen_KL_mean": -133.82962036132812, + "KL/mean": -219.05960083007812, + "KL/rejected_KL_mean": -304.2895812988281, + "KL/std": 151.04354858398438, + "epoch": 0.3377386196769457, + "fcm_dpo/beta": 0.0033145418856292963, + "fcm_dpo/delta": -0.1751500815153122, + "fcm_dpo/margin": 170.4599609375, + "fcm_dpo/q_t": 0.36932283639907837, + "grad_norm": 25.753311157226562, + "learning_rate": 4.203117865141635e-07, + "logits/chosen": -0.2865590751171112, + "logits/rejected": -0.29057538509368896, + "logps/chosen": -173.1116180419922, + "logps/ref_chosen": -39.282005310058594, + "logps/ref_rejected": -85.62191009521484, + "logps/rejected": -389.9114990234375, + "loss": 0.9682, + "margin_dpo/margin_mean": 170.4599609375, + "margin_dpo/margin_std": 166.1871337890625, + "step": 230 + }, + { + "KL/chosen_KL_mean": -153.423828125, + "KL/mean": -212.75845336914062, + "KL/rejected_KL_mean": -272.0930480957031, + "KL/std": 134.63912963867188, + "epoch": 0.3392070484581498, + "fcm_dpo/beta": 0.0032932045869529247, + "fcm_dpo/delta": 0.00957314670085907, + "fcm_dpo/margin": 118.6692123413086, + "fcm_dpo/q_t": 0.41134652495384216, + "grad_norm": 20.769832611083984, + "learning_rate": 4.1937008024246625e-07, + "logits/chosen": -0.35538873076438904, + "logits/rejected": -0.3243277072906494, + "logps/chosen": -216.70028686523438, + "logps/ref_chosen": -63.27644348144531, + "logps/ref_rejected": -74.1239013671875, + "logps/rejected": -346.2169494628906, + "loss": 1.0945, + "margin_dpo/margin_mean": 118.66921997070312, + "margin_dpo/margin_std": 156.93490600585938, + "step": 231 + }, + { + "KL/chosen_KL_mean": -189.22994995117188, + "KL/mean": -235.84271240234375, + "KL/rejected_KL_mean": -282.4554443359375, + "KL/std": 156.42123413085938, + "epoch": 0.3406754772393539, + "fcm_dpo/beta": 0.003335074055939913, + "fcm_dpo/delta": 0.09204444289207458, + "fcm_dpo/margin": 93.22552490234375, + "fcm_dpo/q_t": 0.43068164587020874, + "grad_norm": 26.37661361694336, + "learning_rate": 4.1842391091163933e-07, + "logits/chosen": -0.3877544403076172, + "logits/rejected": -0.36684519052505493, + "logps/chosen": -259.97869873046875, + "logps/ref_chosen": -70.74876403808594, + "logps/ref_rejected": -83.97706604003906, + "logps/rejected": -366.4325256347656, + "loss": 1.1658, + "margin_dpo/margin_mean": 93.22552490234375, + "margin_dpo/margin_std": 160.78085327148438, + "step": 232 + }, + { + "KL/chosen_KL_mean": -168.43820190429688, + "KL/mean": -242.55459594726562, + "KL/rejected_KL_mean": -316.6710205078125, + "KL/std": 165.54281616210938, + "epoch": 0.342143906020558, + "fcm_dpo/beta": 0.003311819862574339, + "fcm_dpo/delta": -0.09555429965257645, + "fcm_dpo/margin": 148.23281860351562, + "fcm_dpo/q_t": 0.3932535648345947, + "grad_norm": 24.77134132385254, + "learning_rate": 4.174733034541245e-07, + "logits/chosen": -0.3793497085571289, + "logits/rejected": -0.38402658700942993, + "logps/chosen": -223.32113647460938, + "logps/ref_chosen": -54.8829345703125, + "logps/ref_rejected": -107.4800796508789, + "logps/rejected": -424.15106201171875, + "loss": 1.0634, + "margin_dpo/margin_mean": 148.23281860351562, + "margin_dpo/margin_std": 211.1150665283203, + "step": 233 + }, + { + "KL/chosen_KL_mean": -164.03988647460938, + "KL/mean": -241.57525634765625, + "KL/rejected_KL_mean": -319.110595703125, + "KL/std": 147.59521484375, + "epoch": 0.3436123348017621, + "fcm_dpo/beta": 0.003218206809833646, + "fcm_dpo/delta": -0.10609423369169235, + "fcm_dpo/margin": 155.07070922851562, + "fcm_dpo/q_t": 0.3857956826686859, + "grad_norm": 33.62549591064453, + "learning_rate": 4.165182829193126e-07, + "logits/chosen": -0.36793646216392517, + "logits/rejected": -0.3934020400047302, + "logps/chosen": -208.13441467285156, + "logps/ref_chosen": -44.094520568847656, + "logps/ref_rejected": -100.00663757324219, + "logps/rejected": -419.11724853515625, + "loss": 1.0175, + "margin_dpo/margin_mean": 155.07069396972656, + "margin_dpo/margin_std": 166.5502471923828, + "step": 234 + }, + { + "KL/chosen_KL_mean": -194.43746948242188, + "KL/mean": -243.66107177734375, + "KL/rejected_KL_mean": -292.8846435546875, + "KL/std": 138.89715576171875, + "epoch": 0.34508076358296624, + "fcm_dpo/beta": 0.0032716416753828526, + "fcm_dpo/delta": 0.08014119416475296, + "fcm_dpo/margin": 98.44720458984375, + "fcm_dpo/q_t": 0.4257309138774872, + "grad_norm": 23.623655319213867, + "learning_rate": 4.1555887447288255e-07, + "logits/chosen": -0.3998695909976959, + "logits/rejected": -0.3791996240615845, + "logps/chosen": -256.6753845214844, + "logps/ref_chosen": -62.237911224365234, + "logps/ref_rejected": -90.39506530761719, + "logps/rejected": -383.27972412109375, + "loss": 1.1598, + "margin_dpo/margin_mean": 98.44720458984375, + "margin_dpo/margin_std": 164.85972595214844, + "step": 235 + }, + { + "KL/chosen_KL_mean": -138.26724243164062, + "KL/mean": -217.79342651367188, + "KL/rejected_KL_mean": -297.31964111328125, + "KL/std": 148.53077697753906, + "epoch": 0.3465491923641703, + "fcm_dpo/beta": 0.0032193802762776613, + "fcm_dpo/delta": -0.1180877834558487, + "fcm_dpo/margin": 159.05239868164062, + "fcm_dpo/q_t": 0.3797944188117981, + "grad_norm": 44.529022216796875, + "learning_rate": 4.1459510339613946e-07, + "logits/chosen": -0.39753884077072144, + "logits/rejected": -0.3979169726371765, + "logps/chosen": -187.6085968017578, + "logps/ref_chosen": -49.34136199951172, + "logps/ref_rejected": -103.51162719726562, + "logps/rejected": -400.8312683105469, + "loss": 0.9862, + "margin_dpo/margin_mean": 159.05239868164062, + "margin_dpo/margin_std": 140.80845642089844, + "step": 236 + }, + { + "KL/chosen_KL_mean": -186.890380859375, + "KL/mean": -246.773193359375, + "KL/rejected_KL_mean": -306.656005859375, + "KL/std": 148.77780151367188, + "epoch": 0.34801762114537443, + "fcm_dpo/beta": 0.0032119196839630604, + "fcm_dpo/delta": 0.015867752954363823, + "fcm_dpo/margin": 119.76559448242188, + "fcm_dpo/q_t": 0.41194605827331543, + "grad_norm": 26.484649658203125, + "learning_rate": 4.136269950853473e-07, + "logits/chosen": -0.37552815675735474, + "logits/rejected": -0.3661985993385315, + "logps/chosen": -241.05850219726562, + "logps/ref_chosen": -54.168121337890625, + "logps/ref_rejected": -94.78036499023438, + "logps/rejected": -401.43634033203125, + "loss": 1.107, + "margin_dpo/margin_mean": 119.76559448242188, + "margin_dpo/margin_std": 169.92410278320312, + "step": 237 + }, + { + "KL/chosen_KL_mean": -174.848388671875, + "KL/mean": -235.38360595703125, + "KL/rejected_KL_mean": -295.91888427734375, + "KL/std": 152.73675537109375, + "epoch": 0.34948604992657856, + "fcm_dpo/beta": 0.0032016821205615997, + "fcm_dpo/delta": 0.012528903782367706, + "fcm_dpo/margin": 121.0704574584961, + "fcm_dpo/q_t": 0.41281700134277344, + "grad_norm": 28.03104019165039, + "learning_rate": 4.126545750510605e-07, + "logits/chosen": -0.35810092091560364, + "logits/rejected": -0.37316906452178955, + "logps/chosen": -228.82150268554688, + "logps/ref_chosen": -53.973121643066406, + "logps/ref_rejected": -89.41795349121094, + "logps/rejected": -385.3367919921875, + "loss": 1.102, + "margin_dpo/margin_mean": 121.0704574584961, + "margin_dpo/margin_std": 166.73324584960938, + "step": 238 + }, + { + "KL/chosen_KL_mean": -192.3235321044922, + "KL/mean": -261.5285339355469, + "KL/rejected_KL_mean": -330.7334899902344, + "KL/std": 148.68975830078125, + "epoch": 0.3509544787077827, + "fcm_dpo/beta": 0.003175197634845972, + "fcm_dpo/delta": -0.04259705916047096, + "fcm_dpo/margin": 138.40997314453125, + "fcm_dpo/q_t": 0.3980611562728882, + "grad_norm": 35.199913024902344, + "learning_rate": 4.116778689174514e-07, + "logits/chosen": -0.3678056597709656, + "logits/rejected": -0.3555825352668762, + "logps/chosen": -250.42135620117188, + "logps/ref_chosen": -58.09782409667969, + "logps/ref_rejected": -93.59294128417969, + "logps/rejected": -424.3264465332031, + "loss": 1.0604, + "margin_dpo/margin_mean": 138.40997314453125, + "margin_dpo/margin_std": 161.06385803222656, + "step": 239 + }, + { + "KL/chosen_KL_mean": -206.96327209472656, + "KL/mean": -264.1934814453125, + "KL/rejected_KL_mean": -321.42364501953125, + "KL/std": 160.36300659179688, + "epoch": 0.3524229074889868, + "fcm_dpo/beta": 0.0031964327208697796, + "fcm_dpo/delta": 0.035312630236148834, + "fcm_dpo/margin": 114.46038818359375, + "fcm_dpo/q_t": 0.41712719202041626, + "grad_norm": 34.83024597167969, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": -0.40968841314315796, + "logits/rejected": -0.3899762034416199, + "logps/chosen": -267.5777587890625, + "logps/ref_chosen": -60.6144905090332, + "logps/ref_rejected": -74.1185302734375, + "logps/rejected": -395.54217529296875, + "loss": 1.145, + "margin_dpo/margin_mean": 114.46038818359375, + "margin_dpo/margin_std": 193.00048828125, + "step": 240 + }, + { + "KL/chosen_KL_mean": -178.86370849609375, + "KL/mean": -267.1542663574219, + "KL/rejected_KL_mean": -355.44482421875, + "KL/std": 176.8538818359375, + "epoch": 0.35389133627019087, + "fcm_dpo/beta": 0.0031412378884851933, + "fcm_dpo/delta": -0.16355636715888977, + "fcm_dpo/margin": 176.58108520507812, + "fcm_dpo/q_t": 0.3761065602302551, + "grad_norm": 29.53580665588379, + "learning_rate": 4.097117014129903e-07, + "logits/chosen": -0.3840961456298828, + "logits/rejected": -0.35689496994018555, + "logps/chosen": -244.9547882080078, + "logps/ref_chosen": -66.091064453125, + "logps/ref_rejected": -88.06088256835938, + "logps/rejected": -443.50567626953125, + "loss": 0.9973, + "margin_dpo/margin_mean": 176.5811004638672, + "margin_dpo/margin_std": 199.89828491210938, + "step": 241 + }, + { + "KL/chosen_KL_mean": -196.65576171875, + "KL/mean": -260.2139892578125, + "KL/rejected_KL_mean": -323.7721862792969, + "KL/std": 153.70497131347656, + "epoch": 0.355359765051395, + "fcm_dpo/beta": 0.0031248326413333416, + "fcm_dpo/delta": 0.0024800747632980347, + "fcm_dpo/margin": 127.11642456054688, + "fcm_dpo/q_t": 0.4107505977153778, + "grad_norm": 27.897132873535156, + "learning_rate": 4.087222918524807e-07, + "logits/chosen": -0.3300558924674988, + "logits/rejected": -0.3024280071258545, + "logps/chosen": -264.5196838378906, + "logps/ref_chosen": -67.86392974853516, + "logps/ref_rejected": -83.36033630371094, + "logps/rejected": -407.13250732421875, + "loss": 1.1016, + "margin_dpo/margin_mean": 127.11642456054688, + "margin_dpo/margin_std": 180.156005859375, + "step": 242 + }, + { + "KL/chosen_KL_mean": -192.8182830810547, + "KL/mean": -271.3682861328125, + "KL/rejected_KL_mean": -349.918212890625, + "KL/std": 168.34677124023438, + "epoch": 0.3568281938325991, + "fcm_dpo/beta": 0.003067805664613843, + "fcm_dpo/delta": -0.08624139428138733, + "fcm_dpo/margin": 157.09994506835938, + "fcm_dpo/q_t": 0.39058151841163635, + "grad_norm": 24.1042537689209, + "learning_rate": 4.07728699811968e-07, + "logits/chosen": -0.3527655601501465, + "logits/rejected": -0.3175613582134247, + "logps/chosen": -255.90252685546875, + "logps/ref_chosen": -63.0842399597168, + "logps/ref_rejected": -76.33563232421875, + "logps/rejected": -426.25384521484375, + "loss": 1.0386, + "margin_dpo/margin_mean": 157.09994506835938, + "margin_dpo/margin_std": 191.3821563720703, + "step": 243 + }, + { + "KL/chosen_KL_mean": -164.286376953125, + "KL/mean": -245.10650634765625, + "KL/rejected_KL_mean": -325.9266357421875, + "KL/std": 160.13262939453125, + "epoch": 0.35829662261380324, + "fcm_dpo/beta": 0.003011933993548155, + "fcm_dpo/delta": -0.09149331599473953, + "fcm_dpo/margin": 161.64024353027344, + "fcm_dpo/q_t": 0.38753992319107056, + "grad_norm": 32.73558044433594, + "learning_rate": 4.067309514735267e-07, + "logits/chosen": -0.4291718900203705, + "logits/rejected": -0.42000579833984375, + "logps/chosen": -225.4270782470703, + "logps/ref_chosen": -61.140689849853516, + "logps/ref_rejected": -94.89193725585938, + "logps/rejected": -420.81854248046875, + "loss": 1.0157, + "margin_dpo/margin_mean": 161.6402587890625, + "margin_dpo/margin_std": 165.7441864013672, + "step": 244 + }, + { + "KL/chosen_KL_mean": -187.01638793945312, + "KL/mean": -250.25192260742188, + "KL/rejected_KL_mean": -313.4874267578125, + "KL/std": 156.05523681640625, + "epoch": 0.35976505139500736, + "fcm_dpo/beta": 0.0029890513978898525, + "fcm_dpo/delta": 0.02175918221473694, + "fcm_dpo/margin": 126.47102355957031, + "fcm_dpo/q_t": 0.41278743743896484, + "grad_norm": 26.198579788208008, + "learning_rate": 4.057290731287531e-07, + "logits/chosen": -0.4349704682826996, + "logits/rejected": -0.40197300910949707, + "logps/chosen": -254.2786865234375, + "logps/ref_chosen": -67.26228332519531, + "logps/ref_rejected": -87.64010620117188, + "logps/rejected": -401.12750244140625, + "loss": 1.1113, + "margin_dpo/margin_mean": 126.47102355957031, + "margin_dpo/margin_std": 172.0400390625, + "step": 245 + }, + { + "KL/chosen_KL_mean": -181.31773376464844, + "KL/mean": -244.63754272460938, + "KL/rejected_KL_mean": -307.95733642578125, + "KL/std": 169.26904296875, + "epoch": 0.36123348017621143, + "fcm_dpo/beta": 0.003016488626599312, + "fcm_dpo/delta": 0.01862441748380661, + "fcm_dpo/margin": 126.63958740234375, + "fcm_dpo/q_t": 0.4137783348560333, + "grad_norm": 25.956335067749023, + "learning_rate": 4.047230911780736e-07, + "logits/chosen": -0.45792657136917114, + "logits/rejected": -0.4178639054298401, + "logps/chosen": -248.01470947265625, + "logps/ref_chosen": -66.69696807861328, + "logps/ref_rejected": -84.34634399414062, + "logps/rejected": -392.3036804199219, + "loss": 1.1079, + "margin_dpo/margin_mean": 126.63959503173828, + "margin_dpo/margin_std": 180.6471405029297, + "step": 246 + }, + { + "KL/chosen_KL_mean": -212.8636474609375, + "KL/mean": -301.8062438964844, + "KL/rejected_KL_mean": -390.7489013671875, + "KL/std": 178.96267700195312, + "epoch": 0.36270190895741555, + "fcm_dpo/beta": 0.002963733160868287, + "fcm_dpo/delta": -0.1344645619392395, + "fcm_dpo/margin": 177.8852081298828, + "fcm_dpo/q_t": 0.37880799174308777, + "grad_norm": 27.184240341186523, + "learning_rate": 4.0371303213004814e-07, + "logits/chosen": -0.41092449426651, + "logits/rejected": -0.4103108048439026, + "logps/chosen": -269.468994140625, + "logps/ref_chosen": -56.6053466796875, + "logps/ref_rejected": -106.29326629638672, + "logps/rejected": -497.0421447753906, + "loss": 1.0015, + "margin_dpo/margin_mean": 177.8852081298828, + "margin_dpo/margin_std": 190.83111572265625, + "step": 247 + }, + { + "KL/chosen_KL_mean": -184.45071411132812, + "KL/mean": -263.3221740722656, + "KL/rejected_KL_mean": -342.19366455078125, + "KL/std": 146.73001098632812, + "epoch": 0.3641703377386197, + "fcm_dpo/beta": 0.002908772323280573, + "fcm_dpo/delta": -0.06217579543590546, + "fcm_dpo/margin": 157.74295043945312, + "fcm_dpo/q_t": 0.3912145495414734, + "grad_norm": 22.271547317504883, + "learning_rate": 4.0269892260067197e-07, + "logits/chosen": -0.37712520360946655, + "logits/rejected": -0.3957890272140503, + "logps/chosen": -228.49392700195312, + "logps/ref_chosen": -44.043216705322266, + "logps/ref_rejected": -91.85687255859375, + "logps/rejected": -434.050537109375, + "loss": 1.0207, + "margin_dpo/margin_mean": 157.74295043945312, + "margin_dpo/margin_std": 142.253173828125, + "step": 248 + }, + { + "KL/chosen_KL_mean": -246.08953857421875, + "KL/mean": -288.27056884765625, + "KL/rejected_KL_mean": -330.45159912109375, + "KL/std": 161.197265625, + "epoch": 0.3656387665198238, + "fcm_dpo/beta": 0.0029767826199531555, + "fcm_dpo/delta": 0.15274295210838318, + "fcm_dpo/margin": 84.36204528808594, + "fcm_dpo/q_t": 0.4428468942642212, + "grad_norm": 45.0234260559082, + "learning_rate": 4.0168078931267426e-07, + "logits/chosen": -0.4169383645057678, + "logits/rejected": -0.39167478680610657, + "logps/chosen": -308.5318908691406, + "logps/ref_chosen": -62.442352294921875, + "logps/ref_rejected": -80.46806335449219, + "logps/rejected": -410.9196472167969, + "loss": 1.2249, + "margin_dpo/margin_mean": 84.36204528808594, + "margin_dpo/margin_std": 186.65567016601562, + "step": 249 + }, + { + "KL/chosen_KL_mean": -216.86962890625, + "KL/mean": -296.193115234375, + "KL/rejected_KL_mean": -375.5166015625, + "KL/std": 161.82952880859375, + "epoch": 0.3671071953010279, + "fcm_dpo/beta": 0.002979879267513752, + "fcm_dpo/delta": -0.07642242312431335, + "fcm_dpo/margin": 158.64694213867188, + "fcm_dpo/q_t": 0.38990771770477295, + "grad_norm": 39.645294189453125, + "learning_rate": 4.006586590948141e-07, + "logits/chosen": -0.3875870704650879, + "logits/rejected": -0.32922881841659546, + "logps/chosen": -282.50634765625, + "logps/ref_chosen": -65.63668823242188, + "logps/ref_rejected": -73.87184143066406, + "logps/rejected": -449.388427734375, + "loss": 1.0248, + "margin_dpo/margin_mean": 158.64694213867188, + "margin_dpo/margin_std": 162.55426025390625, + "step": 250 + }, + { + "KL/chosen_KL_mean": -228.77435302734375, + "KL/mean": -284.546875, + "KL/rejected_KL_mean": -340.31939697265625, + "KL/std": 171.0687255859375, + "epoch": 0.368575624082232, + "fcm_dpo/beta": 0.0029769130051136017, + "fcm_dpo/delta": 0.0702948123216629, + "fcm_dpo/margin": 111.5450210571289, + "fcm_dpo/q_t": 0.42472678422927856, + "grad_norm": 27.457468032836914, + "learning_rate": 3.9963255888117325e-07, + "logits/chosen": -0.3817696273326874, + "logits/rejected": -0.3497239947319031, + "logps/chosen": -285.95709228515625, + "logps/ref_chosen": -57.182716369628906, + "logps/ref_rejected": -77.66343688964844, + "logps/rejected": -417.98284912109375, + "loss": 1.1561, + "margin_dpo/margin_mean": 111.5450210571289, + "margin_dpo/margin_std": 183.6023406982422, + "step": 251 + }, + { + "KL/chosen_KL_mean": -225.74676513671875, + "KL/mean": -299.53564453125, + "KL/rejected_KL_mean": -373.324462890625, + "KL/std": 150.81161499023438, + "epoch": 0.3700440528634361, + "fcm_dpo/beta": 0.0029721325263381004, + "fcm_dpo/delta": -0.040585629642009735, + "fcm_dpo/margin": 147.57772827148438, + "fcm_dpo/q_t": 0.3952495753765106, + "grad_norm": 26.062089920043945, + "learning_rate": 3.9860251571044666e-07, + "logits/chosen": -0.4397411346435547, + "logits/rejected": -0.39890581369400024, + "logps/chosen": -297.43243408203125, + "logps/ref_chosen": -71.68563842773438, + "logps/ref_rejected": -84.75799560546875, + "logps/rejected": -458.08245849609375, + "loss": 1.0434, + "margin_dpo/margin_mean": 147.57772827148438, + "margin_dpo/margin_std": 150.42495727539062, + "step": 252 + }, + { + "KL/chosen_KL_mean": -182.36927795410156, + "KL/mean": -253.0843505859375, + "KL/rejected_KL_mean": -323.7994079589844, + "KL/std": 155.97808837890625, + "epoch": 0.37151248164464024, + "fcm_dpo/beta": 0.0029799549374729395, + "fcm_dpo/delta": -0.02296675369143486, + "fcm_dpo/margin": 141.4301300048828, + "fcm_dpo/q_t": 0.40165138244628906, + "grad_norm": 22.8338680267334, + "learning_rate": 3.9756855672522986e-07, + "logits/chosen": -0.4326644539833069, + "logits/rejected": -0.42504405975341797, + "logps/chosen": -251.50320434570312, + "logps/ref_chosen": -69.1339340209961, + "logps/ref_rejected": -98.70252990722656, + "logps/rejected": -422.501953125, + "loss": 1.0752, + "margin_dpo/margin_mean": 141.4301300048828, + "margin_dpo/margin_std": 173.86398315429688, + "step": 253 + }, + { + "KL/chosen_KL_mean": -176.87899780273438, + "KL/mean": -238.10438537597656, + "KL/rejected_KL_mean": -299.32977294921875, + "KL/std": 163.08511352539062, + "epoch": 0.37298091042584436, + "fcm_dpo/beta": 0.0029733264818787575, + "fcm_dpo/delta": 0.03728723153471947, + "fcm_dpo/margin": 122.45077514648438, + "fcm_dpo/q_t": 0.41952478885650635, + "grad_norm": 22.23654556274414, + "learning_rate": 3.965307091713037e-07, + "logits/chosen": -0.4106729328632355, + "logits/rejected": -0.39602309465408325, + "logps/chosen": -231.03399658203125, + "logps/ref_chosen": -54.154998779296875, + "logps/ref_rejected": -90.30764770507812, + "logps/rejected": -389.6374206542969, + "loss": 1.1379, + "margin_dpo/margin_mean": 122.45077514648438, + "margin_dpo/margin_std": 204.29995727539062, + "step": 254 + }, + { + "KL/chosen_KL_mean": -181.30291748046875, + "KL/mean": -246.3142547607422, + "KL/rejected_KL_mean": -311.32562255859375, + "KL/std": 142.95404052734375, + "epoch": 0.3744493392070485, + "fcm_dpo/beta": 0.0029753465205430984, + "fcm_dpo/delta": 0.01340005174279213, + "fcm_dpo/margin": 130.02267456054688, + "fcm_dpo/q_t": 0.4105185270309448, + "grad_norm": 18.779788970947266, + "learning_rate": 3.954890003969163e-07, + "logits/chosen": -0.3276970386505127, + "logits/rejected": -0.3166770339012146, + "logps/chosen": -238.44459533691406, + "logps/ref_chosen": -57.14167022705078, + "logps/ref_rejected": -90.2085952758789, + "logps/rejected": -401.5341796875, + "loss": 1.1068, + "margin_dpo/margin_mean": 130.02267456054688, + "margin_dpo/margin_std": 181.08340454101562, + "step": 255 + }, + { + "KL/chosen_KL_mean": -158.61541748046875, + "KL/mean": -228.71539306640625, + "KL/rejected_KL_mean": -298.81536865234375, + "KL/std": 156.46206665039062, + "epoch": 0.37591776798825255, + "fcm_dpo/beta": 0.0029811577405780554, + "fcm_dpo/delta": -0.01873961091041565, + "fcm_dpo/margin": 140.199951171875, + "fcm_dpo/q_t": 0.4038965106010437, + "grad_norm": 23.397459030151367, + "learning_rate": 3.944434578520628e-07, + "logits/chosen": -0.30310070514678955, + "logits/rejected": -0.3115818500518799, + "logps/chosen": -213.77891540527344, + "logps/ref_chosen": -55.163490295410156, + "logps/ref_rejected": -92.56291961669922, + "logps/rejected": -391.3782958984375, + "loss": 1.0792, + "margin_dpo/margin_mean": 140.199951171875, + "margin_dpo/margin_std": 181.5375213623047, + "step": 256 + }, + { + "KL/chosen_KL_mean": -152.9769287109375, + "KL/mean": -225.80526733398438, + "KL/rejected_KL_mean": -298.6336364746094, + "KL/std": 159.47927856445312, + "epoch": 0.37738619676945667, + "fcm_dpo/beta": 0.0029455325566232204, + "fcm_dpo/delta": -0.03134532645344734, + "fcm_dpo/margin": 145.6566925048828, + "fcm_dpo/q_t": 0.4020821452140808, + "grad_norm": 19.92885971069336, + "learning_rate": 3.933941090877615e-07, + "logits/chosen": -0.3385705053806305, + "logits/rejected": -0.3252190947532654, + "logps/chosen": -202.40061950683594, + "logps/ref_chosen": -49.42369842529297, + "logps/ref_rejected": -79.53791809082031, + "logps/rejected": -378.1715393066406, + "loss": 1.0739, + "margin_dpo/margin_mean": 145.6566925048828, + "margin_dpo/margin_std": 181.281982421875, + "step": 257 + }, + { + "KL/chosen_KL_mean": -198.8714599609375, + "KL/mean": -269.2030944824219, + "KL/rejected_KL_mean": -339.53472900390625, + "KL/std": 167.5731201171875, + "epoch": 0.3788546255506608, + "fcm_dpo/beta": 0.0029567554593086243, + "fcm_dpo/delta": -0.016622664406895638, + "fcm_dpo/margin": 140.6632843017578, + "fcm_dpo/q_t": 0.4036473035812378, + "grad_norm": 20.93349838256836, + "learning_rate": 3.923409817553284e-07, + "logits/chosen": -0.3120572865009308, + "logits/rejected": -0.31056955456733704, + "logps/chosen": -258.2555847167969, + "logps/ref_chosen": -59.384124755859375, + "logps/ref_rejected": -95.99010467529297, + "logps/rejected": -435.52484130859375, + "loss": 1.0943, + "margin_dpo/margin_mean": 140.66329956054688, + "margin_dpo/margin_std": 199.73696899414062, + "step": 258 + }, + { + "KL/chosen_KL_mean": -186.13726806640625, + "KL/mean": -246.82554626464844, + "KL/rejected_KL_mean": -307.51385498046875, + "KL/std": 158.20399475097656, + "epoch": 0.3803230543318649, + "fcm_dpo/beta": 0.0029596006497740746, + "fcm_dpo/delta": 0.04224412143230438, + "fcm_dpo/margin": 121.37655639648438, + "fcm_dpo/q_t": 0.417808473110199, + "grad_norm": 24.50770378112793, + "learning_rate": 3.9128410360564793e-07, + "logits/chosen": -0.3589467704296112, + "logits/rejected": -0.35707467794418335, + "logps/chosen": -238.96560668945312, + "logps/ref_chosen": -52.828346252441406, + "logps/ref_rejected": -89.191650390625, + "logps/rejected": -396.70550537109375, + "loss": 1.1253, + "margin_dpo/margin_mean": 121.37657165527344, + "margin_dpo/margin_std": 178.23170471191406, + "step": 259 + }, + { + "KL/chosen_KL_mean": -195.42129516601562, + "KL/mean": -275.30938720703125, + "KL/rejected_KL_mean": -355.197509765625, + "KL/std": 164.61087036132812, + "epoch": 0.38179148311306904, + "fcm_dpo/beta": 0.0029522283002734184, + "fcm_dpo/delta": -0.07524219155311584, + "fcm_dpo/margin": 159.77621459960938, + "fcm_dpo/q_t": 0.3926173746585846, + "grad_norm": 28.655542373657227, + "learning_rate": 3.9022350248844246e-07, + "logits/chosen": -0.3197569251060486, + "logits/rejected": -0.33596664667129517, + "logps/chosen": -242.83897399902344, + "logps/ref_chosen": -47.41767501831055, + "logps/ref_rejected": -95.08978271484375, + "logps/rejected": -450.28729248046875, + "loss": 1.0311, + "margin_dpo/margin_mean": 159.77621459960938, + "margin_dpo/margin_std": 177.47177124023438, + "step": 260 + }, + { + "KL/chosen_KL_mean": -201.64346313476562, + "KL/mean": -277.6656494140625, + "KL/rejected_KL_mean": -353.6878662109375, + "KL/std": 180.62509155273438, + "epoch": 0.3832599118942731, + "fcm_dpo/beta": 0.00290453783236444, + "fcm_dpo/delta": -0.04374488443136215, + "fcm_dpo/margin": 152.04437255859375, + "fcm_dpo/q_t": 0.4000317454338074, + "grad_norm": 22.516796112060547, + "learning_rate": 3.891592063515376e-07, + "logits/chosen": -0.29051870107650757, + "logits/rejected": -0.2918134927749634, + "logps/chosen": -254.67483520507812, + "logps/ref_chosen": -53.03137969970703, + "logps/ref_rejected": -88.51494598388672, + "logps/rejected": -442.20281982421875, + "loss": 1.0725, + "margin_dpo/margin_mean": 152.04437255859375, + "margin_dpo/margin_std": 203.62240600585938, + "step": 261 + }, + { + "KL/chosen_KL_mean": -240.88412475585938, + "KL/mean": -304.42376708984375, + "KL/rejected_KL_mean": -367.96343994140625, + "KL/std": 165.10328674316406, + "epoch": 0.38472834067547723, + "fcm_dpo/beta": 0.0029267538338899612, + "fcm_dpo/delta": 0.028837885707616806, + "fcm_dpo/margin": 127.07933044433594, + "fcm_dpo/q_t": 0.4135780334472656, + "grad_norm": 24.18539047241211, + "learning_rate": 3.880912432401264e-07, + "logits/chosen": -0.3253632187843323, + "logits/rejected": -0.3014422059059143, + "logps/chosen": -300.5042724609375, + "logps/ref_chosen": -59.620140075683594, + "logps/ref_rejected": -86.41853332519531, + "logps/rejected": -454.3819885253906, + "loss": 1.1037, + "margin_dpo/margin_mean": 127.07933044433594, + "margin_dpo/margin_std": 165.640869140625, + "step": 262 + }, + { + "KL/chosen_KL_mean": -227.05230712890625, + "KL/mean": -316.44732666015625, + "KL/rejected_KL_mean": -405.8423156738281, + "KL/std": 193.70480346679688, + "epoch": 0.38619676945668135, + "fcm_dpo/beta": 0.0028580743819475174, + "fcm_dpo/delta": -0.1178855374455452, + "fcm_dpo/margin": 178.79000854492188, + "fcm_dpo/q_t": 0.3831760883331299, + "grad_norm": 25.265499114990234, + "learning_rate": 3.870196412960302e-07, + "logits/chosen": -0.3177950382232666, + "logits/rejected": -0.29410669207572937, + "logps/chosen": -286.4732666015625, + "logps/ref_chosen": -59.42094421386719, + "logps/ref_rejected": -96.85720825195312, + "logps/rejected": -502.69952392578125, + "loss": 1.018, + "margin_dpo/margin_mean": 178.7899932861328, + "margin_dpo/margin_std": 203.7642822265625, + "step": 263 + }, + { + "KL/chosen_KL_mean": -241.2745361328125, + "KL/mean": -318.86224365234375, + "KL/rejected_KL_mean": -396.44989013671875, + "KL/std": 177.32969665527344, + "epoch": 0.3876651982378855, + "fcm_dpo/beta": 0.002816273830831051, + "fcm_dpo/delta": -0.04003193974494934, + "fcm_dpo/margin": 155.17535400390625, + "fcm_dpo/q_t": 0.40031126141548157, + "grad_norm": 24.550264358520508, + "learning_rate": 3.8594442875695665e-07, + "logits/chosen": -0.35513001680374146, + "logits/rejected": -0.3457460403442383, + "logps/chosen": -303.99664306640625, + "logps/ref_chosen": -62.722084045410156, + "logps/ref_rejected": -93.85620880126953, + "logps/rejected": -490.30609130859375, + "loss": 1.0724, + "margin_dpo/margin_mean": 155.1753692626953, + "margin_dpo/margin_std": 197.9132080078125, + "step": 264 + }, + { + "KL/chosen_KL_mean": -272.23443603515625, + "KL/mean": -345.8155517578125, + "KL/rejected_KL_mean": -419.3966064453125, + "KL/std": 205.12991333007812, + "epoch": 0.3891336270190896, + "fcm_dpo/beta": 0.0028229644522070885, + "fcm_dpo/delta": -0.01610906980931759, + "fcm_dpo/margin": 147.16217041015625, + "fcm_dpo/q_t": 0.4075104594230652, + "grad_norm": 32.928955078125, + "learning_rate": 3.848656339557562e-07, + "logits/chosen": -0.2887270450592041, + "logits/rejected": -0.27040165662765503, + "logps/chosen": -334.2059020996094, + "logps/ref_chosen": -61.971466064453125, + "logps/ref_rejected": -88.02059936523438, + "logps/rejected": -507.417236328125, + "loss": 1.1221, + "margin_dpo/margin_mean": 147.16217041015625, + "margin_dpo/margin_std": 246.03475952148438, + "step": 265 + }, + { + "KL/chosen_KL_mean": -277.2935485839844, + "KL/mean": -338.205322265625, + "KL/rejected_KL_mean": -399.11712646484375, + "KL/std": 174.17181396484375, + "epoch": 0.39060205580029367, + "fcm_dpo/beta": 0.0028457948938012123, + "fcm_dpo/delta": 0.05520811304450035, + "fcm_dpo/margin": 121.82360076904297, + "fcm_dpo/q_t": 0.4216112792491913, + "grad_norm": 60.411834716796875, + "learning_rate": 3.8378328531967507e-07, + "logits/chosen": -0.3695454001426697, + "logits/rejected": -0.32945775985717773, + "logps/chosen": -344.3932189941406, + "logps/ref_chosen": -67.09967041015625, + "logps/ref_rejected": -67.97122192382812, + "logps/rejected": -467.08837890625, + "loss": 1.1455, + "margin_dpo/margin_mean": 121.82359313964844, + "margin_dpo/margin_std": 201.16543579101562, + "step": 266 + }, + { + "KL/chosen_KL_mean": -251.60833740234375, + "KL/mean": -329.51861572265625, + "KL/rejected_KL_mean": -407.428955078125, + "KL/std": 190.1357421875, + "epoch": 0.3920704845814978, + "fcm_dpo/beta": 0.002824952360242605, + "fcm_dpo/delta": -0.042434047907590866, + "fcm_dpo/margin": 155.82058715820312, + "fcm_dpo/q_t": 0.39909815788269043, + "grad_norm": 35.148372650146484, + "learning_rate": 3.8269741136960646e-07, + "logits/chosen": -0.3738306760787964, + "logits/rejected": -0.34012115001678467, + "logps/chosen": -320.5791015625, + "logps/ref_chosen": -68.97075653076172, + "logps/ref_rejected": -90.16844940185547, + "logps/rejected": -497.5973815917969, + "loss": 1.0761, + "margin_dpo/margin_mean": 155.82058715820312, + "margin_dpo/margin_std": 211.06903076171875, + "step": 267 + }, + { + "KL/chosen_KL_mean": -259.2694396972656, + "KL/mean": -333.34063720703125, + "KL/rejected_KL_mean": -407.41180419921875, + "KL/std": 180.32666015625, + "epoch": 0.3935389133627019, + "fcm_dpo/beta": 0.002812296850606799, + "fcm_dpo/delta": -0.0176074281334877, + "fcm_dpo/margin": 148.14236450195312, + "fcm_dpo/q_t": 0.40650928020477295, + "grad_norm": 36.6395378112793, + "learning_rate": 3.8160804071933894e-07, + "logits/chosen": -0.34564322233200073, + "logits/rejected": -0.35396748781204224, + "logps/chosen": -315.16973876953125, + "logps/ref_chosen": -55.90031051635742, + "logps/ref_rejected": -101.64763641357422, + "logps/rejected": -509.0594482421875, + "loss": 1.0918, + "margin_dpo/margin_mean": 148.14236450195312, + "margin_dpo/margin_std": 211.45758056640625, + "step": 268 + }, + { + "KL/chosen_KL_mean": -271.8604431152344, + "KL/mean": -358.281982421875, + "KL/rejected_KL_mean": -444.70355224609375, + "KL/std": 175.954345703125, + "epoch": 0.39500734214390604, + "fcm_dpo/beta": 0.002785563003271818, + "fcm_dpo/delta": -0.08552718162536621, + "fcm_dpo/margin": 172.84307861328125, + "fcm_dpo/q_t": 0.38992932438850403, + "grad_norm": 30.89337158203125, + "learning_rate": 3.8051520207480204e-07, + "logits/chosen": -0.4149805009365082, + "logits/rejected": -0.39908021688461304, + "logps/chosen": -341.8999938964844, + "logps/ref_chosen": -70.03955841064453, + "logps/ref_rejected": -107.34937286376953, + "logps/rejected": -552.0529174804688, + "loss": 1.0491, + "margin_dpo/margin_mean": 172.84307861328125, + "margin_dpo/margin_std": 220.37840270996094, + "step": 269 + }, + { + "KL/chosen_KL_mean": -228.65283203125, + "KL/mean": -294.26617431640625, + "KL/rejected_KL_mean": -359.8794860839844, + "KL/std": 157.0207977294922, + "epoch": 0.3964757709251101, + "fcm_dpo/beta": 0.002793004736304283, + "fcm_dpo/delta": 0.03450280427932739, + "fcm_dpo/margin": 131.22665405273438, + "fcm_dpo/q_t": 0.41584137082099915, + "grad_norm": 34.00802230834961, + "learning_rate": 3.794189242333106e-07, + "logits/chosen": -0.4382932186126709, + "logits/rejected": -0.4344383478164673, + "logps/chosen": -298.1863098144531, + "logps/ref_chosen": -69.53347778320312, + "logps/ref_rejected": -109.92864990234375, + "logps/rejected": -469.8081359863281, + "loss": 1.1246, + "margin_dpo/margin_mean": 131.22665405273438, + "margin_dpo/margin_std": 197.15127563476562, + "step": 270 + }, + { + "KL/chosen_KL_mean": -203.8043975830078, + "KL/mean": -284.2559509277344, + "KL/rejected_KL_mean": -364.70745849609375, + "KL/std": 154.94090270996094, + "epoch": 0.39794419970631423, + "fcm_dpo/beta": 0.002762623131275177, + "fcm_dpo/delta": -0.0468977689743042, + "fcm_dpo/margin": 160.903076171875, + "fcm_dpo/q_t": 0.3969581723213196, + "grad_norm": 27.79316520690918, + "learning_rate": 3.7831923608280514e-07, + "logits/chosen": -0.3980172276496887, + "logits/rejected": -0.38150689005851746, + "logps/chosen": -260.5689697265625, + "logps/ref_chosen": -56.76456832885742, + "logps/ref_rejected": -92.51383972167969, + "logps/rejected": -457.2213134765625, + "loss": 1.0442, + "margin_dpo/margin_mean": 160.903076171875, + "margin_dpo/margin_std": 173.694580078125, + "step": 271 + }, + { + "KL/chosen_KL_mean": -182.1749725341797, + "KL/mean": -278.8963623046875, + "KL/rejected_KL_mean": -375.6177978515625, + "KL/std": 174.50149536132812, + "epoch": 0.39941262848751835, + "fcm_dpo/beta": 0.0027207238599658012, + "fcm_dpo/delta": -0.13311508297920227, + "fcm_dpo/margin": 193.4428253173828, + "fcm_dpo/q_t": 0.3770027756690979, + "grad_norm": 39.809661865234375, + "learning_rate": 3.772161666010912e-07, + "logits/chosen": -0.37144580483436584, + "logits/rejected": -0.3810126781463623, + "logps/chosen": -231.672119140625, + "logps/ref_chosen": -49.497154235839844, + "logps/ref_rejected": -105.54279327392578, + "logps/rejected": -481.16058349609375, + "loss": 0.9857, + "margin_dpo/margin_mean": 193.4428253173828, + "margin_dpo/margin_std": 180.5559844970703, + "step": 272 + }, + { + "KL/chosen_KL_mean": -212.50933837890625, + "KL/mean": -307.0111083984375, + "KL/rejected_KL_mean": -401.51287841796875, + "KL/std": 174.13787841796875, + "epoch": 0.4008810572687225, + "fcm_dpo/beta": 0.0026398475747555494, + "fcm_dpo/delta": -0.10482463985681534, + "fcm_dpo/margin": 189.00352478027344, + "fcm_dpo/q_t": 0.3841009736061096, + "grad_norm": 25.719608306884766, + "learning_rate": 3.761097448550755e-07, + "logits/chosen": -0.3823262155056, + "logits/rejected": -0.36422067880630493, + "logps/chosen": -275.4847412109375, + "logps/ref_chosen": -62.97539520263672, + "logps/ref_rejected": -92.49858093261719, + "logps/rejected": -494.0114440917969, + "loss": 1.0094, + "margin_dpo/margin_mean": 189.00352478027344, + "margin_dpo/margin_std": 193.2362823486328, + "step": 273 + }, + { + "KL/chosen_KL_mean": -262.0535888671875, + "KL/mean": -333.47491455078125, + "KL/rejected_KL_mean": -404.896240234375, + "KL/std": 162.6813201904297, + "epoch": 0.4023494860499266, + "fcm_dpo/beta": 0.0026513845659792423, + "fcm_dpo/delta": 0.02189909853041172, + "fcm_dpo/margin": 142.8426513671875, + "fcm_dpo/q_t": 0.4122008681297302, + "grad_norm": 22.938400268554688, + "learning_rate": 3.75e-07, + "logits/chosen": -0.3189348578453064, + "logits/rejected": -0.3026675581932068, + "logps/chosen": -317.7213134765625, + "logps/ref_chosen": -55.66770935058594, + "logps/ref_rejected": -77.33308410644531, + "logps/rejected": -482.2293395996094, + "loss": 1.0992, + "margin_dpo/margin_mean": 142.8426513671875, + "margin_dpo/margin_std": 183.34677124023438, + "step": 274 + }, + { + "KL/chosen_KL_mean": -207.0484619140625, + "KL/mean": -285.15045166015625, + "KL/rejected_KL_mean": -363.25244140625, + "KL/std": 170.67037963867188, + "epoch": 0.40381791483113066, + "fcm_dpo/beta": 0.0026485861744731665, + "fcm_dpo/delta": -0.014418797567486763, + "fcm_dpo/margin": 156.20401000976562, + "fcm_dpo/q_t": 0.40412938594818115, + "grad_norm": 26.63812255859375, + "learning_rate": 3.738869612786737e-07, + "logits/chosen": -0.35331815481185913, + "logits/rejected": -0.35876357555389404, + "logps/chosen": -255.64315795898438, + "logps/ref_chosen": -48.594703674316406, + "logps/ref_rejected": -93.30369567871094, + "logps/rejected": -456.55615234375, + "loss": 1.0773, + "margin_dpo/margin_mean": 156.2040252685547, + "margin_dpo/margin_std": 193.69981384277344, + "step": 275 + }, + { + "KL/chosen_KL_mean": -230.7656707763672, + "KL/mean": -310.7945861816406, + "KL/rejected_KL_mean": -390.82354736328125, + "KL/std": 179.71499633789062, + "epoch": 0.4052863436123348, + "fcm_dpo/beta": 0.0026255841366946697, + "fcm_dpo/delta": -0.021273206919431686, + "fcm_dpo/margin": 160.05784606933594, + "fcm_dpo/q_t": 0.40462201833724976, + "grad_norm": 26.980419158935547, + "learning_rate": 3.7277065802070204e-07, + "logits/chosen": -0.3262450098991394, + "logits/rejected": -0.3033270239830017, + "logps/chosen": -287.34307861328125, + "logps/ref_chosen": -56.57740783691406, + "logps/ref_rejected": -70.36566925048828, + "logps/rejected": -461.189208984375, + "loss": 1.084, + "margin_dpo/margin_mean": 160.05784606933594, + "margin_dpo/margin_std": 216.21392822265625, + "step": 276 + }, + { + "KL/chosen_KL_mean": -258.88909912109375, + "KL/mean": -339.9987487792969, + "KL/rejected_KL_mean": -421.1083679199219, + "KL/std": 182.54701232910156, + "epoch": 0.4067547723935389, + "fcm_dpo/beta": 0.0026227221824228764, + "fcm_dpo/delta": -0.026605986058712006, + "fcm_dpo/margin": 162.21926879882812, + "fcm_dpo/q_t": 0.4024323523044586, + "grad_norm": 28.496627807617188, + "learning_rate": 3.71651119641714e-07, + "logits/chosen": -0.3310853838920593, + "logits/rejected": -0.31346580386161804, + "logps/chosen": -315.16064453125, + "logps/ref_chosen": -56.27156066894531, + "logps/ref_rejected": -92.88127136230469, + "logps/rejected": -513.9896240234375, + "loss": 1.0807, + "margin_dpo/margin_mean": 162.2192840576172, + "margin_dpo/margin_std": 215.76904296875, + "step": 277 + }, + { + "KL/chosen_KL_mean": -231.9629364013672, + "KL/mean": -326.51513671875, + "KL/rejected_KL_mean": -421.0672607421875, + "KL/std": 190.9918975830078, + "epoch": 0.40822320117474303, + "fcm_dpo/beta": 0.0025728708133101463, + "fcm_dpo/delta": -0.09131142497062683, + "fcm_dpo/margin": 189.1043243408203, + "fcm_dpo/q_t": 0.38871318101882935, + "grad_norm": 23.81670570373535, + "learning_rate": 3.705283756425872e-07, + "logits/chosen": -0.31821274757385254, + "logits/rejected": -0.32499927282333374, + "logps/chosen": -284.9048767089844, + "logps/ref_chosen": -52.94194030761719, + "logps/ref_rejected": -91.25357818603516, + "logps/rejected": -512.3208618164062, + "loss": 1.0258, + "margin_dpo/margin_mean": 189.1043243408203, + "margin_dpo/margin_std": 212.00811767578125, + "step": 278 + }, + { + "KL/chosen_KL_mean": -274.34991455078125, + "KL/mean": -368.72027587890625, + "KL/rejected_KL_mean": -463.09063720703125, + "KL/std": 205.50079345703125, + "epoch": 0.40969162995594716, + "fcm_dpo/beta": 0.002521257847547531, + "fcm_dpo/delta": -0.0808742344379425, + "fcm_dpo/margin": 188.74072265625, + "fcm_dpo/q_t": 0.39357665181159973, + "grad_norm": 27.415855407714844, + "learning_rate": 3.6940245560867e-07, + "logits/chosen": -0.27504926919937134, + "logits/rejected": -0.27438250184059143, + "logps/chosen": -322.9912109375, + "logps/ref_chosen": -48.641319274902344, + "logps/ref_rejected": -87.8514404296875, + "logps/rejected": -550.9420776367188, + "loss": 1.0599, + "margin_dpo/margin_mean": 188.74072265625, + "margin_dpo/margin_std": 252.65435791015625, + "step": 279 + }, + { + "KL/chosen_KL_mean": -273.220458984375, + "KL/mean": -370.7410583496094, + "KL/rejected_KL_mean": -468.26165771484375, + "KL/std": 192.02523803710938, + "epoch": 0.4111600587371512, + "fcm_dpo/beta": 0.002490004990249872, + "fcm_dpo/delta": -0.09012575447559357, + "fcm_dpo/margin": 195.04116821289062, + "fcm_dpo/q_t": 0.38661879301071167, + "grad_norm": 24.108890533447266, + "learning_rate": 3.6827338920900253e-07, + "logits/chosen": -0.2909424304962158, + "logits/rejected": -0.2941422462463379, + "logps/chosen": -332.0176086425781, + "logps/ref_chosen": -58.797122955322266, + "logps/ref_rejected": -98.61885070800781, + "logps/rejected": -566.8804931640625, + "loss": 1.0206, + "margin_dpo/margin_mean": 195.04116821289062, + "margin_dpo/margin_std": 207.14418029785156, + "step": 280 + }, + { + "KL/chosen_KL_mean": -244.1348876953125, + "KL/mean": -332.23809814453125, + "KL/rejected_KL_mean": -420.34130859375, + "KL/std": 186.86834716796875, + "epoch": 0.41262848751835535, + "fcm_dpo/beta": 0.00247568404302001, + "fcm_dpo/delta": -0.03801019489765167, + "fcm_dpo/margin": 176.2063751220703, + "fcm_dpo/q_t": 0.3967971205711365, + "grad_norm": 23.407094955444336, + "learning_rate": 3.6714120619553435e-07, + "logits/chosen": -0.3744785487651825, + "logits/rejected": -0.35385334491729736, + "logps/chosen": -299.6234130859375, + "logps/ref_chosen": -55.488521575927734, + "logps/ref_rejected": -80.88258361816406, + "logps/rejected": -501.223876953125, + "loss": 1.058, + "margin_dpo/margin_mean": 176.2063751220703, + "margin_dpo/margin_std": 202.88414001464844, + "step": 281 + }, + { + "KL/chosen_KL_mean": -265.88043212890625, + "KL/mean": -335.734619140625, + "KL/rejected_KL_mean": -405.58880615234375, + "KL/std": 192.28700256347656, + "epoch": 0.41409691629955947, + "fcm_dpo/beta": 0.00249258684925735, + "fcm_dpo/delta": 0.05299558490514755, + "fcm_dpo/margin": 139.70834350585938, + "fcm_dpo/q_t": 0.4222215414047241, + "grad_norm": 23.37168312072754, + "learning_rate": 3.660059364023408e-07, + "logits/chosen": -0.403659462928772, + "logits/rejected": -0.3835230767726898, + "logps/chosen": -338.9505615234375, + "logps/ref_chosen": -73.07014465332031, + "logps/ref_rejected": -95.35098266601562, + "logps/rejected": -500.93975830078125, + "loss": 1.1354, + "margin_dpo/margin_mean": 139.70834350585938, + "margin_dpo/margin_std": 219.26756286621094, + "step": 282 + }, + { + "KL/chosen_KL_mean": -278.4122314453125, + "KL/mean": -378.302490234375, + "KL/rejected_KL_mean": -478.1927795410156, + "KL/std": 221.9765625, + "epoch": 0.4155653450807636, + "fcm_dpo/beta": 0.002461041323840618, + "fcm_dpo/delta": -0.09631384164094925, + "fcm_dpo/margin": 199.78057861328125, + "fcm_dpo/q_t": 0.3858485817909241, + "grad_norm": 34.04173278808594, + "learning_rate": 3.6486760974483685e-07, + "logits/chosen": -0.3402835726737976, + "logits/rejected": -0.3434496521949768, + "logps/chosen": -340.3106689453125, + "logps/ref_chosen": -61.89844512939453, + "logps/ref_rejected": -96.98655700683594, + "logps/rejected": -575.1793212890625, + "loss": 1.0199, + "margin_dpo/margin_mean": 199.78057861328125, + "margin_dpo/margin_std": 213.38031005859375, + "step": 283 + }, + { + "KL/chosen_KL_mean": -261.8716125488281, + "KL/mean": -354.55419921875, + "KL/rejected_KL_mean": -447.2367858886719, + "KL/std": 204.1056671142578, + "epoch": 0.4170337738619677, + "fcm_dpo/beta": 0.0024192240089178085, + "fcm_dpo/delta": -0.05081501603126526, + "fcm_dpo/margin": 185.36517333984375, + "fcm_dpo/q_t": 0.3958737552165985, + "grad_norm": 36.13024139404297, + "learning_rate": 3.6372625621898863e-07, + "logits/chosen": -0.41608667373657227, + "logits/rejected": -0.40421050786972046, + "logps/chosen": -320.30712890625, + "logps/ref_chosen": -58.4355354309082, + "logps/ref_rejected": -93.46926879882812, + "logps/rejected": -540.7060546875, + "loss": 1.0411, + "margin_dpo/margin_mean": 185.36517333984375, + "margin_dpo/margin_std": 199.91265869140625, + "step": 284 + }, + { + "KL/chosen_KL_mean": -288.56036376953125, + "KL/mean": -369.67523193359375, + "KL/rejected_KL_mean": -450.7901306152344, + "KL/std": 178.81466674804688, + "epoch": 0.4185022026431718, + "fcm_dpo/beta": 0.0024310359731316566, + "fcm_dpo/delta": 0.005226004868745804, + "fcm_dpo/margin": 162.2297821044922, + "fcm_dpo/q_t": 0.4067595601081848, + "grad_norm": 25.638004302978516, + "learning_rate": 3.625819059005228e-07, + "logits/chosen": -0.37708234786987305, + "logits/rejected": -0.3645085096359253, + "logps/chosen": -354.79254150390625, + "logps/ref_chosen": -66.23219299316406, + "logps/ref_rejected": -99.1268310546875, + "logps/rejected": -549.9169921875, + "loss": 1.0887, + "margin_dpo/margin_mean": 162.2297821044922, + "margin_dpo/margin_std": 199.06863403320312, + "step": 285 + }, + { + "KL/chosen_KL_mean": -281.560791015625, + "KL/mean": -375.9154357910156, + "KL/rejected_KL_mean": -470.27008056640625, + "KL/std": 201.82308959960938, + "epoch": 0.4199706314243759, + "fcm_dpo/beta": 0.0024013579823076725, + "fcm_dpo/delta": -0.05564514547586441, + "fcm_dpo/margin": 188.70924377441406, + "fcm_dpo/q_t": 0.39494001865386963, + "grad_norm": 21.66739845275879, + "learning_rate": 3.614345889441346e-07, + "logits/chosen": -0.360429048538208, + "logits/rejected": -0.3457929491996765, + "logps/chosen": -354.5118103027344, + "logps/ref_chosen": -72.95100402832031, + "logps/ref_rejected": -88.58845520019531, + "logps/rejected": -558.8585205078125, + "loss": 1.0474, + "margin_dpo/margin_mean": 188.70924377441406, + "margin_dpo/margin_std": 217.59698486328125, + "step": 286 + }, + { + "KL/chosen_KL_mean": -260.7840576171875, + "KL/mean": -335.9070129394531, + "KL/rejected_KL_mean": -411.02996826171875, + "KL/std": 174.77919006347656, + "epoch": 0.42143906020558003, + "fcm_dpo/beta": 0.0024088136851787567, + "fcm_dpo/delta": 0.03944006562232971, + "fcm_dpo/margin": 150.24591064453125, + "fcm_dpo/q_t": 0.41572195291519165, + "grad_norm": 26.050071716308594, + "learning_rate": 3.6028433558269275e-07, + "logits/chosen": -0.3475659489631653, + "logits/rejected": -0.32449665665626526, + "logps/chosen": -322.3251953125, + "logps/ref_chosen": -61.54115295410156, + "logps/ref_rejected": -77.69607543945312, + "logps/rejected": -488.7260437011719, + "loss": 1.1087, + "margin_dpo/margin_mean": 150.24591064453125, + "margin_dpo/margin_std": 195.06973266601562, + "step": 287 + }, + { + "KL/chosen_KL_mean": -248.489990234375, + "KL/mean": -345.380126953125, + "KL/rejected_KL_mean": -442.270263671875, + "KL/std": 185.03494262695312, + "epoch": 0.42290748898678415, + "fcm_dpo/beta": 0.0023706420324742794, + "fcm_dpo/delta": -0.06380210071802139, + "fcm_dpo/margin": 193.78025817871094, + "fcm_dpo/q_t": 0.39201951026916504, + "grad_norm": 19.817523956298828, + "learning_rate": 3.5913117612644327e-07, + "logits/chosen": -0.3824455738067627, + "logits/rejected": -0.36902564764022827, + "logps/chosen": -305.1512145996094, + "logps/ref_chosen": -56.661224365234375, + "logps/ref_rejected": -87.33570098876953, + "logps/rejected": -529.60595703125, + "loss": 1.029, + "margin_dpo/margin_mean": 193.7802734375, + "margin_dpo/margin_std": 186.56927490234375, + "step": 288 + }, + { + "KL/chosen_KL_mean": -244.011474609375, + "KL/mean": -348.7763671875, + "KL/rejected_KL_mean": -453.54119873046875, + "KL/std": 203.1474609375, + "epoch": 0.4243759177679883, + "fcm_dpo/beta": 0.0023524933494627476, + "fcm_dpo/delta": -0.09762175381183624, + "fcm_dpo/margin": 209.52972412109375, + "fcm_dpo/q_t": 0.3861865997314453, + "grad_norm": 24.97647476196289, + "learning_rate": 3.5797514096221024e-07, + "logits/chosen": -0.30401328206062317, + "logits/rejected": -0.3052119314670563, + "logps/chosen": -289.24188232421875, + "logps/ref_chosen": -45.23039245605469, + "logps/ref_rejected": -87.64266967773438, + "logps/rejected": -541.183837890625, + "loss": 1.0157, + "margin_dpo/margin_mean": 209.52969360351562, + "margin_dpo/margin_std": 219.72787475585938, + "step": 289 + }, + { + "KL/chosen_KL_mean": -264.4161071777344, + "KL/mean": -370.1914367675781, + "KL/rejected_KL_mean": -475.966796875, + "KL/std": 215.30389404296875, + "epoch": 0.42584434654919234, + "fcm_dpo/beta": 0.002299440558999777, + "fcm_dpo/delta": -0.0910457968711853, + "fcm_dpo/margin": 211.55064392089844, + "fcm_dpo/q_t": 0.3896278738975525, + "grad_norm": 22.546796798706055, + "learning_rate": 3.568162605525952e-07, + "logits/chosen": -0.3152487874031067, + "logits/rejected": -0.3381202518939972, + "logps/chosen": -319.8876037597656, + "logps/ref_chosen": -55.47149658203125, + "logps/ref_rejected": -116.70857238769531, + "logps/rejected": -592.6753540039062, + "loss": 1.0352, + "margin_dpo/margin_mean": 211.5506591796875, + "margin_dpo/margin_std": 254.94503784179688, + "step": 290 + }, + { + "KL/chosen_KL_mean": -228.40325927734375, + "KL/mean": -325.32037353515625, + "KL/rejected_KL_mean": -422.2374572753906, + "KL/std": 190.18911743164062, + "epoch": 0.42731277533039647, + "fcm_dpo/beta": 0.002286990638822317, + "fcm_dpo/delta": -0.04551897943019867, + "fcm_dpo/margin": 193.83419799804688, + "fcm_dpo/q_t": 0.39684972167015076, + "grad_norm": 24.015735626220703, + "learning_rate": 3.5565456543517485e-07, + "logits/chosen": -0.3624478578567505, + "logits/rejected": -0.35021036863327026, + "logps/chosen": -291.66363525390625, + "logps/ref_chosen": -63.26036834716797, + "logps/ref_rejected": -89.29708862304688, + "logps/rejected": -511.5345458984375, + "loss": 1.0569, + "margin_dpo/margin_mean": 193.83419799804688, + "margin_dpo/margin_std": 227.18531799316406, + "step": 291 + }, + { + "KL/chosen_KL_mean": -259.12725830078125, + "KL/mean": -364.222412109375, + "KL/rejected_KL_mean": -469.3175048828125, + "KL/std": 228.16139221191406, + "epoch": 0.4287812041116006, + "fcm_dpo/beta": 0.0022533410228788853, + "fcm_dpo/delta": -0.077273428440094, + "fcm_dpo/margin": 210.19024658203125, + "fcm_dpo/q_t": 0.39149677753448486, + "grad_norm": 24.348356246948242, + "learning_rate": 3.5449008622169583e-07, + "logits/chosen": -0.3501250445842743, + "logits/rejected": -0.3385492265224457, + "logps/chosen": -313.0458068847656, + "logps/ref_chosen": -53.91852951049805, + "logps/ref_rejected": -89.96138000488281, + "logps/rejected": -559.2788696289062, + "loss": 1.0506, + "margin_dpo/margin_mean": 210.19024658203125, + "margin_dpo/margin_std": 265.28570556640625, + "step": 292 + }, + { + "KL/chosen_KL_mean": -262.6152648925781, + "KL/mean": -341.5986328125, + "KL/rejected_KL_mean": -420.58203125, + "KL/std": 223.62734985351562, + "epoch": 0.4302496328928047, + "fcm_dpo/beta": 0.002254858613014221, + "fcm_dpo/delta": 0.045318976044654846, + "fcm_dpo/margin": 157.96676635742188, + "fcm_dpo/q_t": 0.42001551389694214, + "grad_norm": 35.9681510925293, + "learning_rate": 3.5332285359726846e-07, + "logits/chosen": -0.3428913354873657, + "logits/rejected": -0.3345603942871094, + "logps/chosen": -322.99127197265625, + "logps/ref_chosen": -60.376033782958984, + "logps/ref_rejected": -77.85244750976562, + "logps/rejected": -498.4344482421875, + "loss": 1.1362, + "margin_dpo/margin_mean": 157.96676635742188, + "margin_dpo/margin_std": 253.5558319091797, + "step": 293 + }, + { + "KL/chosen_KL_mean": -239.33828735351562, + "KL/mean": -329.43267822265625, + "KL/rejected_KL_mean": -419.52703857421875, + "KL/std": 200.11917114257812, + "epoch": 0.43171806167400884, + "fcm_dpo/beta": 0.0022502196952700615, + "fcm_dpo/delta": -0.005795244127511978, + "fcm_dpo/margin": 180.18875122070312, + "fcm_dpo/q_t": 0.40808266401290894, + "grad_norm": 22.653549194335938, + "learning_rate": 3.5215289831955786e-07, + "logits/chosen": -0.33727580308914185, + "logits/rejected": -0.34320569038391113, + "logps/chosen": -287.42584228515625, + "logps/ref_chosen": -48.0875358581543, + "logps/ref_rejected": -81.89698791503906, + "logps/rejected": -501.4240417480469, + "loss": 1.0933, + "margin_dpo/margin_mean": 180.18875122070312, + "margin_dpo/margin_std": 248.12245178222656, + "step": 294 + }, + { + "KL/chosen_KL_mean": -292.0207824707031, + "KL/mean": -394.706298828125, + "KL/rejected_KL_mean": -497.391845703125, + "KL/std": 232.46151733398438, + "epoch": 0.4331864904552129, + "fcm_dpo/beta": 0.002237509936094284, + "fcm_dpo/delta": -0.062341004610061646, + "fcm_dpo/margin": 205.37106323242188, + "fcm_dpo/q_t": 0.39664924144744873, + "grad_norm": 32.122962951660156, + "learning_rate": 3.509802512179737e-07, + "logits/chosen": -0.329035222530365, + "logits/rejected": -0.33049476146698, + "logps/chosen": -341.9454345703125, + "logps/ref_chosen": -49.92467498779297, + "logps/ref_rejected": -87.45632934570312, + "logps/rejected": -584.84814453125, + "loss": 1.0669, + "margin_dpo/margin_mean": 205.37107849121094, + "margin_dpo/margin_std": 275.5567321777344, + "step": 295 + }, + { + "KL/chosen_KL_mean": -373.27532958984375, + "KL/mean": -442.79229736328125, + "KL/rejected_KL_mean": -512.3092651367188, + "KL/std": 223.4410400390625, + "epoch": 0.434654919236417, + "fcm_dpo/beta": 0.002242644317448139, + "fcm_dpo/delta": 0.09080183506011963, + "fcm_dpo/margin": 139.03390502929688, + "fcm_dpo/q_t": 0.42925071716308594, + "grad_norm": 37.06538009643555, + "learning_rate": 3.498049431928577e-07, + "logits/chosen": -0.3686225414276123, + "logits/rejected": -0.3517499566078186, + "logps/chosen": -438.7666015625, + "logps/ref_chosen": -65.49124145507812, + "logps/ref_rejected": -93.08908081054688, + "logps/rejected": -605.3983154296875, + "loss": 1.1995, + "margin_dpo/margin_mean": 139.03392028808594, + "margin_dpo/margin_std": 286.92205810546875, + "step": 296 + }, + { + "KL/chosen_KL_mean": -318.09564208984375, + "KL/mean": -401.1370849609375, + "KL/rejected_KL_mean": -484.1784973144531, + "KL/std": 211.25762939453125, + "epoch": 0.43612334801762115, + "fcm_dpo/beta": 0.002271018922328949, + "fcm_dpo/delta": 0.023693598806858063, + "fcm_dpo/margin": 166.08282470703125, + "fcm_dpo/q_t": 0.41268670558929443, + "grad_norm": 33.412715911865234, + "learning_rate": 3.486270052146694e-07, + "logits/chosen": -0.40956276655197144, + "logits/rejected": -0.41560155153274536, + "logps/chosen": -374.5726318359375, + "logps/ref_chosen": -56.476951599121094, + "logps/ref_rejected": -95.1385498046875, + "logps/rejected": -579.3170166015625, + "loss": 1.1004, + "margin_dpo/margin_mean": 166.08282470703125, + "margin_dpo/margin_std": 214.5130615234375, + "step": 297 + }, + { + "KL/chosen_KL_mean": -343.67779541015625, + "KL/mean": -453.3380432128906, + "KL/rejected_KL_mean": -562.998291015625, + "KL/std": 277.4004211425781, + "epoch": 0.43759177679882527, + "fcm_dpo/beta": 0.002254146384075284, + "fcm_dpo/delta": -0.09923798590898514, + "fcm_dpo/margin": 219.32052612304688, + "fcm_dpo/q_t": 0.3934746980667114, + "grad_norm": 27.909835815429688, + "learning_rate": 3.474464683231698e-07, + "logits/chosen": -0.4167177379131317, + "logits/rejected": -0.4354039132595062, + "logps/chosen": -411.0029296875, + "logps/ref_chosen": -67.32516479492188, + "logps/ref_rejected": -116.66217041015625, + "logps/rejected": -679.6604614257812, + "loss": 1.0671, + "margin_dpo/margin_mean": 219.32052612304688, + "margin_dpo/margin_std": 325.619873046875, + "step": 298 + }, + { + "KL/chosen_KL_mean": -277.94677734375, + "KL/mean": -368.5576171875, + "KL/rejected_KL_mean": -459.16839599609375, + "KL/std": 209.79385375976562, + "epoch": 0.4390602055800294, + "fcm_dpo/beta": 0.002244377974420786, + "fcm_dpo/delta": -0.0077451057732105255, + "fcm_dpo/margin": 181.2216339111328, + "fcm_dpo/q_t": 0.40871721506118774, + "grad_norm": 34.998390197753906, + "learning_rate": 3.462633636266041e-07, + "logits/chosen": -0.3771274983882904, + "logits/rejected": -0.38257941603660583, + "logps/chosen": -326.90887451171875, + "logps/ref_chosen": -48.96209716796875, + "logps/ref_rejected": -84.32823944091797, + "logps/rejected": -543.4966430664062, + "loss": 1.1056, + "margin_dpo/margin_mean": 181.22164916992188, + "margin_dpo/margin_std": 266.51416015625, + "step": 299 + }, + { + "KL/chosen_KL_mean": -348.21954345703125, + "KL/mean": -457.40155029296875, + "KL/rejected_KL_mean": -566.5836181640625, + "KL/std": 245.39015197753906, + "epoch": 0.44052863436123346, + "fcm_dpo/beta": 0.0022067520767450333, + "fcm_dpo/delta": -0.08593274652957916, + "fcm_dpo/margin": 218.36404418945312, + "fcm_dpo/q_t": 0.39113539457321167, + "grad_norm": 38.547908782958984, + "learning_rate": 3.4507772230088147e-07, + "logits/chosen": -0.41613638401031494, + "logits/rejected": -0.4217377305030823, + "logps/chosen": -407.29327392578125, + "logps/ref_chosen": -59.07371139526367, + "logps/ref_rejected": -95.9664535522461, + "logps/rejected": -662.550048828125, + "loss": 1.0635, + "margin_dpo/margin_mean": 218.36404418945312, + "margin_dpo/margin_std": 299.53173828125, + "step": 300 + }, + { + "KL/chosen_KL_mean": -289.61981201171875, + "KL/mean": -399.01641845703125, + "KL/rejected_KL_mean": -508.4130554199219, + "KL/std": 224.61441040039062, + "epoch": 0.4419970631424376, + "fcm_dpo/beta": 0.002160984091460705, + "fcm_dpo/delta": -0.0767727866768837, + "fcm_dpo/margin": 218.79324340820312, + "fcm_dpo/q_t": 0.3935708999633789, + "grad_norm": 22.94151496887207, + "learning_rate": 3.4388957558875316e-07, + "logits/chosen": -0.4170186519622803, + "logits/rejected": -0.41876429319381714, + "logps/chosen": -346.8691711425781, + "logps/ref_chosen": -57.249366760253906, + "logps/ref_rejected": -92.35354614257812, + "logps/rejected": -600.7666015625, + "loss": 1.0439, + "margin_dpo/margin_mean": 218.79324340820312, + "margin_dpo/margin_std": 268.8450012207031, + "step": 301 + }, + { + "KL/chosen_KL_mean": -249.56222534179688, + "KL/mean": -341.4454345703125, + "KL/rejected_KL_mean": -433.32861328125, + "KL/std": 198.3855438232422, + "epoch": 0.4434654919236417, + "fcm_dpo/beta": 0.0021501141600310802, + "fcm_dpo/delta": 0.004691842943429947, + "fcm_dpo/margin": 183.76638793945312, + "fcm_dpo/q_t": 0.4091954827308655, + "grad_norm": 24.27311134338379, + "learning_rate": 3.426989547989902e-07, + "logits/chosen": -0.38404256105422974, + "logits/rejected": -0.3903312683105469, + "logps/chosen": -300.7602233886719, + "logps/ref_chosen": -51.197994232177734, + "logps/ref_rejected": -97.22636413574219, + "logps/rejected": -530.5549926757812, + "loss": 1.0904, + "margin_dpo/margin_mean": 183.76638793945312, + "margin_dpo/margin_std": 234.3460693359375, + "step": 302 + }, + { + "KL/chosen_KL_mean": -257.18695068359375, + "KL/mean": -340.093994140625, + "KL/rejected_KL_mean": -423.0010070800781, + "KL/std": 207.89459228515625, + "epoch": 0.44493392070484583, + "fcm_dpo/beta": 0.002169003477320075, + "fcm_dpo/delta": 0.04184654355049133, + "fcm_dpo/margin": 165.81407165527344, + "fcm_dpo/q_t": 0.41749584674835205, + "grad_norm": 22.197315216064453, + "learning_rate": 3.4150589130555773e-07, + "logits/chosen": -0.4156029224395752, + "logits/rejected": -0.4042498767375946, + "logps/chosen": -323.90087890625, + "logps/ref_chosen": -66.71394348144531, + "logps/ref_rejected": -86.94542694091797, + "logps/rejected": -509.9464416503906, + "loss": 1.127, + "margin_dpo/margin_mean": 165.8140869140625, + "margin_dpo/margin_std": 246.3565216064453, + "step": 303 + }, + { + "KL/chosen_KL_mean": -225.50621032714844, + "KL/mean": -318.69146728515625, + "KL/rejected_KL_mean": -411.8767395019531, + "KL/std": 186.12899780273438, + "epoch": 0.44640234948604995, + "fcm_dpo/beta": 0.002184029668569565, + "fcm_dpo/delta": -0.007586956024169922, + "fcm_dpo/margin": 186.37051391601562, + "fcm_dpo/q_t": 0.4025821089744568, + "grad_norm": 34.385562896728516, + "learning_rate": 3.403104165467883e-07, + "logits/chosen": -0.4139406085014343, + "logits/rejected": -0.4038363993167877, + "logps/chosen": -297.4569091796875, + "logps/ref_chosen": -71.95069885253906, + "logps/ref_rejected": -90.47203063964844, + "logps/rejected": -502.3487548828125, + "loss": 1.0524, + "margin_dpo/margin_mean": 186.37051391601562, + "margin_dpo/margin_std": 163.2909393310547, + "step": 304 + }, + { + "KL/chosen_KL_mean": -241.08352661132812, + "KL/mean": -322.9089050292969, + "KL/rejected_KL_mean": -404.7342834472656, + "KL/std": 209.6416015625, + "epoch": 0.447870778267254, + "fcm_dpo/beta": 0.002178459893912077, + "fcm_dpo/delta": 0.044787101447582245, + "fcm_dpo/margin": 163.65077209472656, + "fcm_dpo/q_t": 0.4187085032463074, + "grad_norm": 21.086244583129883, + "learning_rate": 3.391125620245535e-07, + "logits/chosen": -0.45557549595832825, + "logits/rejected": -0.4429972171783447, + "logps/chosen": -307.8787841796875, + "logps/ref_chosen": -66.79523468017578, + "logps/ref_rejected": -92.75459289550781, + "logps/rejected": -497.4888916015625, + "loss": 1.121, + "margin_dpo/margin_mean": 163.65077209472656, + "margin_dpo/margin_std": 227.79843139648438, + "step": 305 + }, + { + "KL/chosen_KL_mean": -230.5140380859375, + "KL/mean": -311.62640380859375, + "KL/rejected_KL_mean": -392.73876953125, + "KL/std": 184.592529296875, + "epoch": 0.44933920704845814, + "fcm_dpo/beta": 0.0022113663144409657, + "fcm_dpo/delta": 0.04271689057350159, + "fcm_dpo/margin": 162.22476196289062, + "fcm_dpo/q_t": 0.4160733222961426, + "grad_norm": 26.13888168334961, + "learning_rate": 3.3791235930343417e-07, + "logits/chosen": -0.39813679456710815, + "logits/rejected": -0.3748471140861511, + "logps/chosen": -300.19793701171875, + "logps/ref_chosen": -69.68389892578125, + "logps/ref_rejected": -85.15919494628906, + "logps/rejected": -477.89794921875, + "loss": 1.1031, + "margin_dpo/margin_mean": 162.22474670410156, + "margin_dpo/margin_std": 195.34988403320312, + "step": 306 + }, + { + "KL/chosen_KL_mean": -216.44622802734375, + "KL/mean": -298.306396484375, + "KL/rejected_KL_mean": -380.1665344238281, + "KL/std": 168.08889770507812, + "epoch": 0.45080763582966227, + "fcm_dpo/beta": 0.0022216022480279207, + "fcm_dpo/delta": 0.037642501294612885, + "fcm_dpo/margin": 163.72030639648438, + "fcm_dpo/q_t": 0.41505366563796997, + "grad_norm": 25.627588272094727, + "learning_rate": 3.367098400098881e-07, + "logits/chosen": -0.4154399633407593, + "logits/rejected": -0.3966638445854187, + "logps/chosen": -286.61163330078125, + "logps/ref_chosen": -70.16542053222656, + "logps/ref_rejected": -86.97230529785156, + "logps/rejected": -467.13885498046875, + "loss": 1.1052, + "margin_dpo/margin_mean": 163.72030639648438, + "margin_dpo/margin_std": 206.83914184570312, + "step": 307 + }, + { + "KL/chosen_KL_mean": -226.32872009277344, + "KL/mean": -319.50994873046875, + "KL/rejected_KL_mean": -412.69122314453125, + "KL/std": 194.25296020507812, + "epoch": 0.4522760646108664, + "fcm_dpo/beta": 0.0022258548997342587, + "fcm_dpo/delta": -0.015454130247235298, + "fcm_dpo/margin": 186.36245727539062, + "fcm_dpo/q_t": 0.4018438756465912, + "grad_norm": 30.474700927734375, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": -0.38194745779037476, + "logits/rejected": -0.3681604266166687, + "logps/chosen": -281.57373046875, + "logps/ref_chosen": -55.2449951171875, + "logps/ref_rejected": -79.37226104736328, + "logps/rejected": -492.0634765625, + "loss": 1.0564, + "margin_dpo/margin_mean": 186.36245727539062, + "margin_dpo/margin_std": 188.6147918701172, + "step": 308 + }, + { + "KL/chosen_KL_mean": -239.09133911132812, + "KL/mean": -332.51025390625, + "KL/rejected_KL_mean": -425.92913818359375, + "KL/std": 211.6947479248047, + "epoch": 0.45374449339207046, + "fcm_dpo/beta": 0.002228262834250927, + "fcm_dpo/delta": -0.017414994537830353, + "fcm_dpo/margin": 186.83782958984375, + "fcm_dpo/q_t": 0.4025392532348633, + "grad_norm": 27.617185592651367, + "learning_rate": 3.3429797851573183e-07, + "logits/chosen": -0.3523610830307007, + "logits/rejected": -0.34558966755867004, + "logps/chosen": -288.0504150390625, + "logps/ref_chosen": -48.959083557128906, + "logps/ref_rejected": -82.34072875976562, + "logps/rejected": -508.2698669433594, + "loss": 1.0739, + "margin_dpo/margin_mean": 186.83782958984375, + "margin_dpo/margin_std": 224.22433471679688, + "step": 309 + }, + { + "KL/chosen_KL_mean": -277.80377197265625, + "KL/mean": -365.20257568359375, + "KL/rejected_KL_mean": -452.6014099121094, + "KL/std": 186.077880859375, + "epoch": 0.4552129221732746, + "fcm_dpo/beta": 0.0022279408294707537, + "fcm_dpo/delta": 0.010630182921886444, + "fcm_dpo/margin": 174.79759216308594, + "fcm_dpo/q_t": 0.40788841247558594, + "grad_norm": 24.81958770751953, + "learning_rate": 3.3308869986991487e-07, + "logits/chosen": -0.3655666410923004, + "logits/rejected": -0.3523295223712921, + "logps/chosen": -340.54559326171875, + "logps/ref_chosen": -62.74177932739258, + "logps/ref_rejected": -79.9302978515625, + "logps/rejected": -532.53173828125, + "loss": 1.0754, + "margin_dpo/margin_mean": 174.79759216308594, + "margin_dpo/margin_std": 181.52981567382812, + "step": 310 + }, + { + "KL/chosen_KL_mean": -288.7535095214844, + "KL/mean": -389.1260681152344, + "KL/rejected_KL_mean": -489.4986572265625, + "KL/std": 242.73895263671875, + "epoch": 0.4566813509544787, + "fcm_dpo/beta": 0.002205129014328122, + "fcm_dpo/delta": -0.04465709626674652, + "fcm_dpo/margin": 200.74514770507812, + "fcm_dpo/q_t": 0.39941084384918213, + "grad_norm": 23.147109985351562, + "learning_rate": 3.3187723175958346e-07, + "logits/chosen": -0.34936434030532837, + "logits/rejected": -0.3253268897533417, + "logps/chosen": -341.781494140625, + "logps/ref_chosen": -53.02798080444336, + "logps/ref_rejected": -77.43820190429688, + "logps/rejected": -566.9368896484375, + "loss": 1.064, + "margin_dpo/margin_mean": 200.74514770507812, + "margin_dpo/margin_std": 254.4142303466797, + "step": 311 + }, + { + "KL/chosen_KL_mean": -274.93408203125, + "KL/mean": -365.4719543457031, + "KL/rejected_KL_mean": -456.00982666015625, + "KL/std": 207.9627685546875, + "epoch": 0.4581497797356828, + "fcm_dpo/beta": 0.0022081173956394196, + "fcm_dpo/delta": -6.601214408874512e-05, + "fcm_dpo/margin": 181.0757293701172, + "fcm_dpo/q_t": 0.4093562364578247, + "grad_norm": 25.84002685546875, + "learning_rate": 3.306636061080487e-07, + "logits/chosen": -0.3078731298446655, + "logits/rejected": -0.29903510212898254, + "logps/chosen": -324.3262939453125, + "logps/ref_chosen": -49.39221954345703, + "logps/ref_rejected": -75.79280853271484, + "logps/rejected": -531.8026123046875, + "loss": 1.0932, + "margin_dpo/margin_mean": 181.07574462890625, + "margin_dpo/margin_std": 246.26904296875, + "step": 312 + }, + { + "KL/chosen_KL_mean": -258.62164306640625, + "KL/mean": -353.00494384765625, + "KL/rejected_KL_mean": -447.38824462890625, + "KL/std": 222.29296875, + "epoch": 0.45961820851688695, + "fcm_dpo/beta": 0.002209829166531563, + "fcm_dpo/delta": -0.018744412809610367, + "fcm_dpo/margin": 188.7665557861328, + "fcm_dpo/q_t": 0.4038567841053009, + "grad_norm": 23.65576171875, + "learning_rate": 3.2944785489547537e-07, + "logits/chosen": -0.3879469037055969, + "logits/rejected": -0.3879814147949219, + "logps/chosen": -308.7744140625, + "logps/ref_chosen": -50.152740478515625, + "logps/ref_rejected": -86.40620422363281, + "logps/rejected": -533.79443359375, + "loss": 1.0832, + "margin_dpo/margin_mean": 188.7665557861328, + "margin_dpo/margin_std": 242.31149291992188, + "step": 313 + }, + { + "KL/chosen_KL_mean": -279.6490783691406, + "KL/mean": -368.64263916015625, + "KL/rejected_KL_mean": -457.63616943359375, + "KL/std": 213.9251708984375, + "epoch": 0.461086637298091, + "fcm_dpo/beta": 0.002186702797189355, + "fcm_dpo/delta": 0.010914817452430725, + "fcm_dpo/margin": 177.98709106445312, + "fcm_dpo/q_t": 0.41287338733673096, + "grad_norm": 21.331357955932617, + "learning_rate": 3.2823001015803857e-07, + "logits/chosen": -0.4344632625579834, + "logits/rejected": -0.43607085943222046, + "logps/chosen": -336.88665771484375, + "logps/ref_chosen": -57.237579345703125, + "logps/ref_rejected": -97.5965347290039, + "logps/rejected": -555.232666015625, + "loss": 1.1167, + "margin_dpo/margin_mean": 177.98707580566406, + "margin_dpo/margin_std": 270.8641662597656, + "step": 314 + }, + { + "KL/chosen_KL_mean": -259.404052734375, + "KL/mean": -342.66021728515625, + "KL/rejected_KL_mean": -425.91644287109375, + "KL/std": 190.9168701171875, + "epoch": 0.46255506607929514, + "fcm_dpo/beta": 0.0022047478705644608, + "fcm_dpo/delta": 0.03413639962673187, + "fcm_dpo/margin": 166.5124053955078, + "fcm_dpo/q_t": 0.41494593024253845, + "grad_norm": 22.818117141723633, + "learning_rate": 3.270101039870797e-07, + "logits/chosen": -0.337954044342041, + "logits/rejected": -0.34216612577438354, + "logps/chosen": -308.4736328125, + "logps/ref_chosen": -49.06958770751953, + "logps/ref_rejected": -85.68087768554688, + "logps/rejected": -511.59735107421875, + "loss": 1.1044, + "margin_dpo/margin_mean": 166.51239013671875, + "margin_dpo/margin_std": 212.81405639648438, + "step": 315 + }, + { + "KL/chosen_KL_mean": -262.956298828125, + "KL/mean": -379.14678955078125, + "KL/rejected_KL_mean": -495.3372497558594, + "KL/std": 229.5782470703125, + "epoch": 0.46402349486049926, + "fcm_dpo/beta": 0.002165478654205799, + "fcm_dpo/delta": -0.10975506901741028, + "fcm_dpo/margin": 232.38096618652344, + "fcm_dpo/q_t": 0.38564929366111755, + "grad_norm": 27.47342300415039, + "learning_rate": 3.2578816852826086e-07, + "logits/chosen": -0.3632145822048187, + "logits/rejected": -0.3669757843017578, + "logps/chosen": -317.217041015625, + "logps/ref_chosen": -54.26074981689453, + "logps/ref_rejected": -101.2814712524414, + "logps/rejected": -596.6187133789062, + "loss": 1.0149, + "margin_dpo/margin_mean": 232.3809814453125, + "margin_dpo/margin_std": 254.15489196777344, + "step": 316 + }, + { + "KL/chosen_KL_mean": -273.8968811035156, + "KL/mean": -402.15899658203125, + "KL/rejected_KL_mean": -530.421142578125, + "KL/std": 214.53253173828125, + "epoch": 0.4654919236417034, + "fcm_dpo/beta": 0.002123830374330282, + "fcm_dpo/delta": -0.152874156832695, + "fcm_dpo/margin": 256.52423095703125, + "fcm_dpo/q_t": 0.3734695017337799, + "grad_norm": 23.029985427856445, + "learning_rate": 3.2456423598071783e-07, + "logits/chosen": -0.37268176674842834, + "logits/rejected": -0.36425209045410156, + "logps/chosen": -329.9910888671875, + "logps/ref_chosen": -56.094207763671875, + "logps/ref_rejected": -100.69905090332031, + "logps/rejected": -631.1201782226562, + "loss": 0.9752, + "margin_dpo/margin_mean": 256.52423095703125, + "margin_dpo/margin_std": 236.38705444335938, + "step": 317 + }, + { + "KL/chosen_KL_mean": -306.3728942871094, + "KL/mean": -398.26910400390625, + "KL/rejected_KL_mean": -490.16534423828125, + "KL/std": 222.15777587890625, + "epoch": 0.4669603524229075, + "fcm_dpo/beta": 0.0021082893945276737, + "fcm_dpo/delta": 0.012857386842370033, + "fcm_dpo/margin": 183.79248046875, + "fcm_dpo/q_t": 0.410823255777359, + "grad_norm": 26.980371475219727, + "learning_rate": 3.233383385962115e-07, + "logits/chosen": -0.43110066652297974, + "logits/rejected": -0.4063284993171692, + "logps/chosen": -371.0185852050781, + "logps/ref_chosen": -64.64569854736328, + "logps/ref_rejected": -82.76425170898438, + "logps/rejected": -572.9295654296875, + "loss": 1.0961, + "margin_dpo/margin_mean": 183.79248046875, + "margin_dpo/margin_std": 241.28457641601562, + "step": 318 + }, + { + "KL/chosen_KL_mean": -274.31536865234375, + "KL/mean": -389.5890197753906, + "KL/rejected_KL_mean": -504.8626708984375, + "KL/std": 245.08279418945312, + "epoch": 0.4684287812041116, + "fcm_dpo/beta": 0.002078516175970435, + "fcm_dpo/delta": -0.08318348973989487, + "fcm_dpo/margin": 230.54730224609375, + "fcm_dpo/q_t": 0.3892419934272766, + "grad_norm": 24.00583267211914, + "learning_rate": 3.2211050867827805e-07, + "logits/chosen": -0.4252866208553314, + "logits/rejected": -0.4395906329154968, + "logps/chosen": -323.69915771484375, + "logps/ref_chosen": -49.383758544921875, + "logps/ref_rejected": -113.90650939941406, + "logps/rejected": -618.7691650390625, + "loss": 1.0327, + "margin_dpo/margin_mean": 230.54730224609375, + "margin_dpo/margin_std": 264.1103515625, + "step": 319 + }, + { + "KL/chosen_KL_mean": -281.8694152832031, + "KL/mean": -408.36529541015625, + "KL/rejected_KL_mean": -534.8611450195312, + "KL/std": 252.33741760253906, + "epoch": 0.4698972099853157, + "fcm_dpo/beta": 0.002026338130235672, + "fcm_dpo/delta": -0.11938206106424332, + "fcm_dpo/margin": 252.99169921875, + "fcm_dpo/q_t": 0.382274866104126, + "grad_norm": 31.98560333251953, + "learning_rate": 3.208807785813777e-07, + "logits/chosen": -0.4378474950790405, + "logits/rejected": -0.4452739953994751, + "logps/chosen": -341.37432861328125, + "logps/ref_chosen": -59.50489044189453, + "logps/ref_rejected": -97.66717529296875, + "logps/rejected": -632.5283203125, + "loss": 1.0061, + "margin_dpo/margin_mean": 252.99171447753906, + "margin_dpo/margin_std": 269.489013671875, + "step": 320 + }, + { + "KL/chosen_KL_mean": -346.11138916015625, + "KL/mean": -450.9620666503906, + "KL/rejected_KL_mean": -555.812744140625, + "KL/std": 253.70281982421875, + "epoch": 0.4713656387665198, + "fcm_dpo/beta": 0.002002749592065811, + "fcm_dpo/delta": -0.021664388477802277, + "fcm_dpo/margin": 209.70138549804688, + "fcm_dpo/q_t": 0.40391805768013, + "grad_norm": 22.900249481201172, + "learning_rate": 3.1964918071004217e-07, + "logits/chosen": -0.3867838978767395, + "logits/rejected": -0.3807498812675476, + "logps/chosen": -407.6600646972656, + "logps/ref_chosen": -61.548683166503906, + "logps/ref_rejected": -91.64103698730469, + "logps/rejected": -647.4537963867188, + "loss": 1.0809, + "margin_dpo/margin_mean": 209.7013702392578, + "margin_dpo/margin_std": 269.6270751953125, + "step": 321 + }, + { + "KL/chosen_KL_mean": -296.47418212890625, + "KL/mean": -411.1878662109375, + "KL/rejected_KL_mean": -525.9015502929688, + "KL/std": 223.96456909179688, + "epoch": 0.47283406754772395, + "fcm_dpo/beta": 0.001987969968467951, + "fcm_dpo/delta": -0.05918703228235245, + "fcm_dpo/margin": 229.42739868164062, + "fcm_dpo/q_t": 0.3931780457496643, + "grad_norm": 23.179550170898438, + "learning_rate": 3.184157475180207e-07, + "logits/chosen": -0.41528087854385376, + "logits/rejected": -0.41565677523612976, + "logps/chosen": -353.76422119140625, + "logps/ref_chosen": -57.29003143310547, + "logps/ref_rejected": -95.74992370605469, + "logps/rejected": -621.6514892578125, + "loss": 1.0349, + "margin_dpo/margin_mean": 229.42739868164062, + "margin_dpo/margin_std": 237.61573791503906, + "step": 322 + }, + { + "KL/chosen_KL_mean": -306.5753173828125, + "KL/mean": -404.47186279296875, + "KL/rejected_KL_mean": -502.368408203125, + "KL/std": 220.96112060546875, + "epoch": 0.47430249632892807, + "fcm_dpo/beta": 0.00198989175260067, + "fcm_dpo/delta": 0.010807945393025875, + "fcm_dpo/margin": 195.7930908203125, + "fcm_dpo/q_t": 0.4091640114784241, + "grad_norm": 34.06732940673828, + "learning_rate": 3.171805115074251e-07, + "logits/chosen": -0.43661847710609436, + "logits/rejected": -0.4351438283920288, + "logps/chosen": -357.80926513671875, + "logps/ref_chosen": -51.23395919799805, + "logps/ref_rejected": -75.06192016601562, + "logps/rejected": -577.4302978515625, + "loss": 1.0912, + "margin_dpo/margin_mean": 195.7930908203125, + "margin_dpo/margin_std": 242.7722930908203, + "step": 323 + }, + { + "KL/chosen_KL_mean": -353.3026123046875, + "KL/mean": -450.3382568359375, + "KL/rejected_KL_mean": -547.3738403320312, + "KL/std": 240.54566955566406, + "epoch": 0.47577092511013214, + "fcm_dpo/beta": 0.002010114025324583, + "fcm_dpo/delta": 0.009144224226474762, + "fcm_dpo/margin": 194.07122802734375, + "fcm_dpo/q_t": 0.4112786650657654, + "grad_norm": 37.594154357910156, + "learning_rate": 3.1594350522787295e-07, + "logits/chosen": -0.4559288024902344, + "logits/rejected": -0.4483965337276459, + "logps/chosen": -418.43780517578125, + "logps/ref_chosen": -65.13516998291016, + "logps/ref_rejected": -86.47750854492188, + "logps/rejected": -633.851318359375, + "loss": 1.1081, + "margin_dpo/margin_mean": 194.0712432861328, + "margin_dpo/margin_std": 265.3919677734375, + "step": 324 + }, + { + "KL/chosen_KL_mean": -268.78985595703125, + "KL/mean": -350.67626953125, + "KL/rejected_KL_mean": -432.562744140625, + "KL/std": 201.10546875, + "epoch": 0.47723935389133626, + "fcm_dpo/beta": 0.0020195613615214825, + "fcm_dpo/delta": 0.07154245674610138, + "fcm_dpo/margin": 163.77285766601562, + "fcm_dpo/q_t": 0.422224223613739, + "grad_norm": 25.190866470336914, + "learning_rate": 3.147047612756302e-07, + "logits/chosen": -0.495430052280426, + "logits/rejected": -0.4804334044456482, + "logps/chosen": -325.00543212890625, + "logps/ref_chosen": -56.215599060058594, + "logps/ref_rejected": -70.08592987060547, + "logps/rejected": -502.648681640625, + "loss": 1.1267, + "margin_dpo/margin_mean": 163.7728729248047, + "margin_dpo/margin_std": 205.93533325195312, + "step": 325 + }, + { + "KL/chosen_KL_mean": -294.3157958984375, + "KL/mean": -368.50244140625, + "KL/rejected_KL_mean": -442.6891174316406, + "KL/std": 189.423095703125, + "epoch": 0.4787077826725404, + "fcm_dpo/beta": 0.0020513960625976324, + "fcm_dpo/delta": 0.09874353557825089, + "fcm_dpo/margin": 148.37332153320312, + "fcm_dpo/q_t": 0.4277682900428772, + "grad_norm": 50.60403823852539, + "learning_rate": 3.134643122927519e-07, + "logits/chosen": -0.47400808334350586, + "logits/rejected": -0.45543599128723145, + "logps/chosen": -367.040771484375, + "logps/ref_chosen": -72.72496032714844, + "logps/ref_rejected": -79.8467788696289, + "logps/rejected": -522.535888671875, + "loss": 1.1407, + "margin_dpo/margin_mean": 148.37332153320312, + "margin_dpo/margin_std": 184.3029022216797, + "step": 326 + }, + { + "KL/chosen_KL_mean": -258.0245056152344, + "KL/mean": -365.32470703125, + "KL/rejected_KL_mean": -472.6249084472656, + "KL/std": 198.78585815429688, + "epoch": 0.4801762114537445, + "fcm_dpo/beta": 0.0020476686768233776, + "fcm_dpo/delta": -0.04147752374410629, + "fcm_dpo/margin": 214.6004180908203, + "fcm_dpo/q_t": 0.39583897590637207, + "grad_norm": 48.44367218017578, + "learning_rate": 3.1222219096622264e-07, + "logits/chosen": -0.4509025812149048, + "logits/rejected": -0.4456767737865448, + "logps/chosen": -327.158935546875, + "logps/ref_chosen": -69.13441467285156, + "logps/ref_rejected": -111.93377685546875, + "logps/rejected": -584.5587158203125, + "loss": 1.0387, + "margin_dpo/margin_mean": 214.6004180908203, + "margin_dpo/margin_std": 209.97634887695312, + "step": 327 + }, + { + "KL/chosen_KL_mean": -267.9549865722656, + "KL/mean": -367.7935791015625, + "KL/rejected_KL_mean": -467.6322021484375, + "KL/std": 218.40206909179688, + "epoch": 0.48164464023494863, + "fcm_dpo/beta": 0.002048112917691469, + "fcm_dpo/delta": -0.009362122975289822, + "fcm_dpo/margin": 199.6772003173828, + "fcm_dpo/q_t": 0.40534746646881104, + "grad_norm": 34.26046371459961, + "learning_rate": 3.1097843002709427e-07, + "logits/chosen": -0.4282514452934265, + "logits/rejected": -0.43282201886177063, + "logps/chosen": -327.6421813964844, + "logps/ref_chosen": -59.68719482421875, + "logps/ref_rejected": -90.85499572753906, + "logps/rejected": -558.4871826171875, + "loss": 1.0743, + "margin_dpo/margin_mean": 199.67721557617188, + "margin_dpo/margin_std": 239.71859741210938, + "step": 328 + }, + { + "KL/chosen_KL_mean": -314.18572998046875, + "KL/mean": -422.6015625, + "KL/rejected_KL_mean": -531.017333984375, + "KL/std": 242.86639404296875, + "epoch": 0.4831130690161527, + "fcm_dpo/beta": 0.002023177221417427, + "fcm_dpo/delta": -0.041052184998989105, + "fcm_dpo/margin": 216.8316192626953, + "fcm_dpo/q_t": 0.3982122540473938, + "grad_norm": 27.216005325317383, + "learning_rate": 3.0973306224962437e-07, + "logits/chosen": -0.4262320399284363, + "logits/rejected": -0.4210492968559265, + "logps/chosen": -379.4319152832031, + "logps/ref_chosen": -65.2461929321289, + "logps/ref_rejected": -100.69770812988281, + "logps/rejected": -631.715087890625, + "loss": 1.0655, + "margin_dpo/margin_mean": 216.83163452148438, + "margin_dpo/margin_std": 263.0550231933594, + "step": 329 + }, + { + "KL/chosen_KL_mean": -271.18463134765625, + "KL/mean": -386.3155822753906, + "KL/rejected_KL_mean": -501.446533203125, + "KL/std": 238.26080322265625, + "epoch": 0.4845814977973568, + "fcm_dpo/beta": 0.002008675830438733, + "fcm_dpo/delta": -0.06554098427295685, + "fcm_dpo/margin": 230.26187133789062, + "fcm_dpo/q_t": 0.39291393756866455, + "grad_norm": 26.819442749023438, + "learning_rate": 3.084861204504122e-07, + "logits/chosen": -0.3993980884552002, + "logits/rejected": -0.3997897505760193, + "logps/chosen": -318.1829833984375, + "logps/ref_chosen": -46.998348236083984, + "logps/ref_rejected": -86.87684631347656, + "logps/rejected": -588.3233642578125, + "loss": 1.0381, + "margin_dpo/margin_mean": 230.26187133789062, + "margin_dpo/margin_std": 257.1680908203125, + "step": 330 + }, + { + "KL/chosen_KL_mean": -301.6651306152344, + "KL/mean": -418.078125, + "KL/rejected_KL_mean": -534.4911499023438, + "KL/std": 204.872314453125, + "epoch": 0.48604992657856094, + "fcm_dpo/beta": 0.001982024870812893, + "fcm_dpo/delta": -0.06445472687482834, + "fcm_dpo/margin": 232.82598876953125, + "fcm_dpo/q_t": 0.390749990940094, + "grad_norm": 30.03919792175293, + "learning_rate": 3.072376374875335e-07, + "logits/chosen": -0.41282713413238525, + "logits/rejected": -0.40853530168533325, + "logps/chosen": -352.18939208984375, + "logps/ref_chosen": -50.52424621582031, + "logps/ref_rejected": -89.01544189453125, + "logps/rejected": -623.506591796875, + "loss": 1.0137, + "margin_dpo/margin_mean": 232.82598876953125, + "margin_dpo/margin_std": 201.24801635742188, + "step": 331 + }, + { + "KL/chosen_KL_mean": -300.2237243652344, + "KL/mean": -383.75494384765625, + "KL/rejected_KL_mean": -467.2861328125, + "KL/std": 211.57131958007812, + "epoch": 0.48751835535976507, + "fcm_dpo/beta": 0.0019986871629953384, + "fcm_dpo/delta": 0.06825611740350723, + "fcm_dpo/margin": 167.06240844726562, + "fcm_dpo/q_t": 0.4234786927700043, + "grad_norm": 21.762527465820312, + "learning_rate": 3.059876462596758e-07, + "logits/chosen": -0.4211878478527069, + "logits/rejected": -0.4023297131061554, + "logps/chosen": -349.40399169921875, + "logps/ref_chosen": -49.18028259277344, + "logps/ref_rejected": -76.48515319824219, + "logps/rejected": -543.7713012695312, + "loss": 1.1323, + "margin_dpo/margin_mean": 167.0624237060547, + "margin_dpo/margin_std": 237.78700256347656, + "step": 332 + }, + { + "KL/chosen_KL_mean": -331.29693603515625, + "KL/mean": -439.12548828125, + "KL/rejected_KL_mean": -546.9540405273438, + "KL/std": 251.02923583984375, + "epoch": 0.4889867841409692, + "fcm_dpo/beta": 0.0019800434820353985, + "fcm_dpo/delta": -0.029480591416358948, + "fcm_dpo/margin": 215.6570281982422, + "fcm_dpo/q_t": 0.4032571315765381, + "grad_norm": 23.11104965209961, + "learning_rate": 3.0473617970527015e-07, + "logits/chosen": -0.42099469900131226, + "logits/rejected": -0.4137647747993469, + "logps/chosen": -395.0527038574219, + "logps/ref_chosen": -63.75574493408203, + "logps/ref_rejected": -95.04411315917969, + "logps/rejected": -641.9981689453125, + "loss": 1.0919, + "margin_dpo/margin_mean": 215.6570281982422, + "margin_dpo/margin_std": 301.61212158203125, + "step": 333 + }, + { + "KL/chosen_KL_mean": -313.2780456542969, + "KL/mean": -411.32177734375, + "KL/rejected_KL_mean": -509.3655090332031, + "KL/std": 270.41351318359375, + "epoch": 0.49045521292217326, + "fcm_dpo/beta": 0.001991923665627837, + "fcm_dpo/delta": 0.009776053950190544, + "fcm_dpo/margin": 196.08746337890625, + "fcm_dpo/q_t": 0.4119870066642761, + "grad_norm": 22.336389541625977, + "learning_rate": 3.034832708016243e-07, + "logits/chosen": -0.4335794150829315, + "logits/rejected": -0.43230512738227844, + "logps/chosen": -380.2578125, + "logps/ref_chosen": -66.97975158691406, + "logps/ref_rejected": -95.31692504882812, + "logps/rejected": -604.6824340820312, + "loss": 1.121, + "margin_dpo/margin_mean": 196.08746337890625, + "margin_dpo/margin_std": 306.477294921875, + "step": 334 + }, + { + "KL/chosen_KL_mean": -335.70196533203125, + "KL/mean": -414.1585998535156, + "KL/rejected_KL_mean": -492.6152038574219, + "KL/std": 244.75711059570312, + "epoch": 0.4919236417033774, + "fcm_dpo/beta": 0.0020306003279983997, + "fcm_dpo/delta": 0.08306629955768585, + "fcm_dpo/margin": 156.9132080078125, + "fcm_dpo/q_t": 0.4281030297279358, + "grad_norm": 29.569700241088867, + "learning_rate": 3.022289525640531e-07, + "logits/chosen": -0.4642852246761322, + "logits/rejected": -0.4429335594177246, + "logps/chosen": -398.24444580078125, + "logps/ref_chosen": -62.54248046875, + "logps/ref_rejected": -87.61770629882812, + "logps/rejected": -580.23291015625, + "loss": 1.1652, + "margin_dpo/margin_mean": 156.9132080078125, + "margin_dpo/margin_std": 268.2879333496094, + "step": 335 + }, + { + "KL/chosen_KL_mean": -330.2154541015625, + "KL/mean": -455.31500244140625, + "KL/rejected_KL_mean": -580.41455078125, + "KL/std": 285.79803466796875, + "epoch": 0.4933920704845815, + "fcm_dpo/beta": 0.0020034818444401026, + "fcm_dpo/delta": -0.10648685693740845, + "fcm_dpo/margin": 250.1991424560547, + "fcm_dpo/q_t": 0.38840019702911377, + "grad_norm": 31.805938720703125, + "learning_rate": 3.009732580450086e-07, + "logits/chosen": -0.4141591787338257, + "logits/rejected": -0.41369566321372986, + "logps/chosen": -384.74658203125, + "logps/ref_chosen": -54.53115463256836, + "logps/ref_rejected": -104.40424346923828, + "logps/rejected": -684.8187866210938, + "loss": 1.0525, + "margin_dpo/margin_mean": 250.1991424560547, + "margin_dpo/margin_std": 338.71246337890625, + "step": 336 + }, + { + "KL/chosen_KL_mean": -295.72308349609375, + "KL/mean": -417.3601379394531, + "KL/rejected_KL_mean": -538.9971923828125, + "KL/std": 227.58993530273438, + "epoch": 0.4948604992657856, + "fcm_dpo/beta": 0.001958114095032215, + "fcm_dpo/delta": -0.08043714612722397, + "fcm_dpo/margin": 243.27413940429688, + "fcm_dpo/q_t": 0.3905276656150818, + "grad_norm": 29.914546966552734, + "learning_rate": 2.9971622033320914e-07, + "logits/chosen": -0.42889440059661865, + "logits/rejected": -0.41781020164489746, + "logps/chosen": -360.85174560546875, + "logps/ref_chosen": -65.12869262695312, + "logps/ref_rejected": -101.72701263427734, + "logps/rejected": -640.7242431640625, + "loss": 1.0313, + "margin_dpo/margin_mean": 243.27413940429688, + "margin_dpo/margin_std": 274.2711486816406, + "step": 337 + }, + { + "KL/chosen_KL_mean": -267.119384765625, + "KL/mean": -383.24462890625, + "KL/rejected_KL_mean": -499.36981201171875, + "KL/std": 221.5072021484375, + "epoch": 0.49632892804698975, + "fcm_dpo/beta": 0.0019371946109458804, + "fcm_dpo/delta": -0.05233225226402283, + "fcm_dpo/margin": 232.25042724609375, + "fcm_dpo/q_t": 0.3943594694137573, + "grad_norm": 32.328243255615234, + "learning_rate": 2.984578725527675e-07, + "logits/chosen": -0.43586158752441406, + "logits/rejected": -0.4274016320705414, + "logps/chosen": -325.5421142578125, + "logps/ref_chosen": -58.422706604003906, + "logps/ref_rejected": -89.06854248046875, + "logps/rejected": -588.4383544921875, + "loss": 1.0337, + "margin_dpo/margin_mean": 232.25042724609375, + "margin_dpo/margin_std": 232.92489624023438, + "step": 338 + }, + { + "KL/chosen_KL_mean": -294.05352783203125, + "KL/mean": -407.55291748046875, + "KL/rejected_KL_mean": -521.05224609375, + "KL/std": 231.62216186523438, + "epoch": 0.4977973568281938, + "fcm_dpo/beta": 0.0019379344303160906, + "fcm_dpo/delta": -0.04284397512674332, + "fcm_dpo/margin": 226.99868774414062, + "fcm_dpo/q_t": 0.3972986936569214, + "grad_norm": 29.154096603393555, + "learning_rate": 2.9719824786231796e-07, + "logits/chosen": -0.4619607925415039, + "logits/rejected": -0.44948840141296387, + "logps/chosen": -354.0488586425781, + "logps/ref_chosen": -59.99531555175781, + "logps/ref_rejected": -103.9109115600586, + "logps/rejected": -624.963134765625, + "loss": 1.0493, + "margin_dpo/margin_mean": 226.99868774414062, + "margin_dpo/margin_std": 233.55419921875, + "step": 339 + }, + { + "KL/chosen_KL_mean": -315.4582824707031, + "KL/mean": -416.0123291015625, + "KL/rejected_KL_mean": -516.5663452148438, + "KL/std": 227.33596801757812, + "epoch": 0.49926578560939794, + "fcm_dpo/beta": 0.0019145121332257986, + "fcm_dpo/delta": 0.015345301479101181, + "fcm_dpo/margin": 201.10806274414062, + "fcm_dpo/q_t": 0.41142672300338745, + "grad_norm": 28.22606658935547, + "learning_rate": 2.959373794541426e-07, + "logits/chosen": -0.3919587731361389, + "logits/rejected": -0.37107688188552856, + "logps/chosen": -368.28851318359375, + "logps/ref_chosen": -52.83022689819336, + "logps/ref_rejected": -73.10723114013672, + "logps/rejected": -589.673583984375, + "loss": 1.104, + "margin_dpo/margin_mean": 201.10806274414062, + "margin_dpo/margin_std": 275.59588623046875, + "step": 340 + }, + { + "KL/chosen_KL_mean": -324.9350280761719, + "KL/mean": -440.717041015625, + "KL/rejected_KL_mean": -556.4990234375, + "KL/std": 254.97799682617188, + "epoch": 0.5007342143906021, + "fcm_dpo/beta": 0.0019036681624129415, + "fcm_dpo/delta": -0.04316044598817825, + "fcm_dpo/margin": 231.56407165527344, + "fcm_dpo/q_t": 0.3988724946975708, + "grad_norm": 25.336105346679688, + "learning_rate": 2.946753005532965e-07, + "logits/chosen": -0.40650486946105957, + "logits/rejected": -0.405281662940979, + "logps/chosen": -372.8348388671875, + "logps/ref_chosen": -47.899803161621094, + "logps/ref_rejected": -101.80987548828125, + "logps/rejected": -658.3089599609375, + "loss": 1.0573, + "margin_dpo/margin_mean": 231.56407165527344, + "margin_dpo/margin_std": 270.78424072265625, + "step": 341 + }, + { + "KL/chosen_KL_mean": -327.50567626953125, + "KL/mean": -430.53558349609375, + "KL/rejected_KL_mean": -533.5655517578125, + "KL/std": 242.9896697998047, + "epoch": 0.5022026431718062, + "fcm_dpo/beta": 0.0019156227353960276, + "fcm_dpo/delta": 0.005151804536581039, + "fcm_dpo/margin": 206.05987548828125, + "fcm_dpo/q_t": 0.4093712568283081, + "grad_norm": 26.062049865722656, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": -0.4568237066268921, + "logits/rejected": -0.43443238735198975, + "logps/chosen": -399.5023193359375, + "logps/ref_chosen": -71.99664306640625, + "logps/ref_rejected": -92.58959197998047, + "logps/rejected": -626.1551513671875, + "loss": 1.101, + "margin_dpo/margin_mean": 206.05987548828125, + "margin_dpo/margin_std": 287.116943359375, + "step": 342 + }, + { + "KL/chosen_KL_mean": -322.3934326171875, + "KL/mean": -443.8787536621094, + "KL/rejected_KL_mean": -565.3640747070312, + "KL/std": 246.6431884765625, + "epoch": 0.5036710719530103, + "fcm_dpo/beta": 0.0018897566478699446, + "fcm_dpo/delta": -0.062056764960289, + "fcm_dpo/margin": 242.97064208984375, + "fcm_dpo/q_t": 0.3914814591407776, + "grad_norm": 25.208070755004883, + "learning_rate": 2.9214764433242476e-07, + "logits/chosen": -0.4649735689163208, + "logits/rejected": -0.46656423807144165, + "logps/chosen": -376.799072265625, + "logps/ref_chosen": -54.405616760253906, + "logps/ref_rejected": -111.04142761230469, + "logps/rejected": -676.405517578125, + "loss": 1.0218, + "margin_dpo/margin_mean": 242.97064208984375, + "margin_dpo/margin_std": 221.15011596679688, + "step": 343 + }, + { + "KL/chosen_KL_mean": -315.85748291015625, + "KL/mean": -426.89324951171875, + "KL/rejected_KL_mean": -537.9290161132812, + "KL/std": 271.2740173339844, + "epoch": 0.5051395007342144, + "fcm_dpo/beta": 0.0018969422671943903, + "fcm_dpo/delta": -0.023906365036964417, + "fcm_dpo/margin": 222.07159423828125, + "fcm_dpo/q_t": 0.4061945974826813, + "grad_norm": 32.413124084472656, + "learning_rate": 2.9088213361849126e-07, + "logits/chosen": -0.4314219653606415, + "logits/rejected": -0.4362150728702545, + "logps/chosen": -369.8221435546875, + "logps/ref_chosen": -53.96466827392578, + "logps/ref_rejected": -90.62336730957031, + "logps/rejected": -628.5523681640625, + "loss": 1.0819, + "margin_dpo/margin_mean": 222.0715789794922, + "margin_dpo/margin_std": 282.13812255859375, + "step": 344 + }, + { + "KL/chosen_KL_mean": -371.65093994140625, + "KL/mean": -492.77777099609375, + "KL/rejected_KL_mean": -613.904541015625, + "KL/std": 254.54132080078125, + "epoch": 0.5066079295154186, + "fcm_dpo/beta": 0.001865363446995616, + "fcm_dpo/delta": -0.05433148890733719, + "fcm_dpo/margin": 242.25363159179688, + "fcm_dpo/q_t": 0.395372211933136, + "grad_norm": 22.87287139892578, + "learning_rate": 2.896155456223163e-07, + "logits/chosen": -0.4728688597679138, + "logits/rejected": -0.47000813484191895, + "logps/chosen": -433.3366394042969, + "logps/ref_chosen": -61.685699462890625, + "logps/ref_rejected": -99.49041748046875, + "logps/rejected": -713.39501953125, + "loss": 1.0481, + "margin_dpo/margin_mean": 242.25364685058594, + "margin_dpo/margin_std": 279.2045593261719, + "step": 345 + }, + { + "KL/chosen_KL_mean": -372.73223876953125, + "KL/mean": -485.113525390625, + "KL/rejected_KL_mean": -597.494873046875, + "KL/std": 246.56961059570312, + "epoch": 0.5080763582966226, + "fcm_dpo/beta": 0.0018515766132622957, + "fcm_dpo/delta": -0.016864469274878502, + "fcm_dpo/margin": 224.76260375976562, + "fcm_dpo/q_t": 0.40249505639076233, + "grad_norm": 20.88530158996582, + "learning_rate": 2.883479137196714e-07, + "logits/chosen": -0.43277889490127563, + "logits/rejected": -0.4222702980041504, + "logps/chosen": -427.988525390625, + "logps/ref_chosen": -55.256263732910156, + "logps/ref_rejected": -77.41532135009766, + "logps/rejected": -674.91015625, + "loss": 1.0697, + "margin_dpo/margin_mean": 224.76263427734375, + "margin_dpo/margin_std": 265.6940002441406, + "step": 346 + }, + { + "KL/chosen_KL_mean": -361.72607421875, + "KL/mean": -476.1159362792969, + "KL/rejected_KL_mean": -590.5057373046875, + "KL/std": 262.36614990234375, + "epoch": 0.5095447870778267, + "fcm_dpo/beta": 0.0018435390666127205, + "fcm_dpo/delta": -0.022720448672771454, + "fcm_dpo/margin": 228.77969360351562, + "fcm_dpo/q_t": 0.40304189920425415, + "grad_norm": 22.485294342041016, + "learning_rate": 2.8707927131383614e-07, + "logits/chosen": -0.43125152587890625, + "logits/rejected": -0.42406296730041504, + "logps/chosen": -419.2923278808594, + "logps/ref_chosen": -57.56623840332031, + "logps/ref_rejected": -92.35509490966797, + "logps/rejected": -682.86083984375, + "loss": 1.0752, + "margin_dpo/margin_mean": 228.77969360351562, + "margin_dpo/margin_std": 290.30322265625, + "step": 347 + }, + { + "KL/chosen_KL_mean": -323.0507507324219, + "KL/mean": -417.467041015625, + "KL/rejected_KL_mean": -511.8833312988281, + "KL/std": 221.05445861816406, + "epoch": 0.5110132158590308, + "fcm_dpo/beta": 0.0018442126456648111, + "fcm_dpo/delta": 0.053140588104724884, + "fcm_dpo/margin": 188.83261108398438, + "fcm_dpo/q_t": 0.41966158151626587, + "grad_norm": 22.27375602722168, + "learning_rate": 2.858096518347179e-07, + "logits/chosen": -0.47590553760528564, + "logits/rejected": -0.47713130712509155, + "logps/chosen": -379.36846923828125, + "logps/ref_chosen": -56.31770324707031, + "logps/ref_rejected": -89.13836669921875, + "logps/rejected": -601.021728515625, + "loss": 1.1241, + "margin_dpo/margin_mean": 188.83261108398438, + "margin_dpo/margin_std": 255.472900390625, + "step": 348 + }, + { + "KL/chosen_KL_mean": -299.5309143066406, + "KL/mean": -407.94049072265625, + "KL/rejected_KL_mean": -516.3500366210938, + "KL/std": 246.6162109375, + "epoch": 0.5124816446402349, + "fcm_dpo/beta": 0.0018662457587197423, + "fcm_dpo/delta": -0.005084618926048279, + "fcm_dpo/margin": 216.81912231445312, + "fcm_dpo/q_t": 0.4084588885307312, + "grad_norm": 21.248226165771484, + "learning_rate": 2.845390887379706e-07, + "logits/chosen": -0.41005945205688477, + "logits/rejected": -0.4089745283126831, + "logps/chosen": -357.5564270019531, + "logps/ref_chosen": -58.025516510009766, + "logps/ref_rejected": -97.50515747070312, + "logps/rejected": -613.855224609375, + "loss": 1.0995, + "margin_dpo/margin_mean": 216.81912231445312, + "margin_dpo/margin_std": 308.3006896972656, + "step": 349 + }, + { + "KL/chosen_KL_mean": -315.78411865234375, + "KL/mean": -420.061767578125, + "KL/rejected_KL_mean": -524.3394775390625, + "KL/std": 230.697021484375, + "epoch": 0.5139500734214391, + "fcm_dpo/beta": 0.0018557433504611254, + "fcm_dpo/delta": 0.013263138011097908, + "fcm_dpo/margin": 208.5552978515625, + "fcm_dpo/q_t": 0.40980684757232666, + "grad_norm": 33.216182708740234, + "learning_rate": 2.8326761550411346e-07, + "logits/chosen": -0.47512534260749817, + "logits/rejected": -0.4778461754322052, + "logps/chosen": -380.1146240234375, + "logps/ref_chosen": -64.33049011230469, + "logps/ref_rejected": -89.87164306640625, + "logps/rejected": -614.2110595703125, + "loss": 1.1033, + "margin_dpo/margin_mean": 208.5552978515625, + "margin_dpo/margin_std": 284.32781982421875, + "step": 350 + }, + { + "KL/chosen_KL_mean": -295.28631591796875, + "KL/mean": -421.78533935546875, + "KL/rejected_KL_mean": -548.2843017578125, + "KL/std": 270.7213134765625, + "epoch": 0.5154185022026432, + "fcm_dpo/beta": 0.0018403417197987437, + "fcm_dpo/delta": -0.06909814476966858, + "fcm_dpo/margin": 252.99801635742188, + "fcm_dpo/q_t": 0.39455610513687134, + "grad_norm": 22.35758399963379, + "learning_rate": 2.819952656376487e-07, + "logits/chosen": -0.4860556423664093, + "logits/rejected": -0.48423847556114197, + "logps/chosen": -355.9584655761719, + "logps/ref_chosen": -60.6721305847168, + "logps/ref_rejected": -101.5654296875, + "logps/rejected": -649.8497314453125, + "loss": 1.0492, + "margin_dpo/margin_mean": 252.99801635742188, + "margin_dpo/margin_std": 310.3662414550781, + "step": 351 + }, + { + "KL/chosen_KL_mean": -352.4783020019531, + "KL/mean": -432.419921875, + "KL/rejected_KL_mean": -512.3615112304688, + "KL/std": 252.67904663085938, + "epoch": 0.5168869309838473, + "fcm_dpo/beta": 0.001861312659457326, + "fcm_dpo/delta": 0.10567327588796616, + "fcm_dpo/margin": 159.88320922851562, + "fcm_dpo/q_t": 0.4318529963493347, + "grad_norm": 29.07950210571289, + "learning_rate": 2.8072207266617854e-07, + "logits/chosen": -0.46844974160194397, + "logits/rejected": -0.4366309642791748, + "logps/chosen": -423.4217529296875, + "logps/ref_chosen": -70.9434585571289, + "logps/ref_rejected": -76.6419677734375, + "logps/rejected": -589.0034790039062, + "loss": 1.1751, + "margin_dpo/margin_mean": 159.88320922851562, + "margin_dpo/margin_std": 276.7126159667969, + "step": 352 + }, + { + "KL/chosen_KL_mean": -332.9418640136719, + "KL/mean": -436.4351806640625, + "KL/rejected_KL_mean": -539.928466796875, + "KL/std": 252.6245574951172, + "epoch": 0.5183553597650514, + "fcm_dpo/beta": 0.0018715719925239682, + "fcm_dpo/delta": 0.01264815405011177, + "fcm_dpo/margin": 206.98663330078125, + "fcm_dpo/q_t": 0.41147467494010925, + "grad_norm": 32.52266311645508, + "learning_rate": 2.794480701395219e-07, + "logits/chosen": -0.5107035636901855, + "logits/rejected": -0.5004223585128784, + "logps/chosen": -391.33721923828125, + "logps/ref_chosen": -58.39533996582031, + "logps/ref_rejected": -80.33553314208984, + "logps/rejected": -620.2640380859375, + "loss": 1.1022, + "margin_dpo/margin_mean": 206.98663330078125, + "margin_dpo/margin_std": 278.4654541015625, + "step": 353 + }, + { + "KL/chosen_KL_mean": -288.1324462890625, + "KL/mean": -404.28253173828125, + "KL/rejected_KL_mean": -520.4326171875, + "KL/std": 234.51644897460938, + "epoch": 0.5198237885462555, + "fcm_dpo/beta": 0.001870082225650549, + "fcm_dpo/delta": -0.03605414927005768, + "fcm_dpo/margin": 232.30015563964844, + "fcm_dpo/q_t": 0.3978366255760193, + "grad_norm": 22.32083511352539, + "learning_rate": 2.781732916288303e-07, + "logits/chosen": -0.47122490406036377, + "logits/rejected": -0.4608303904533386, + "logps/chosen": -347.9354248046875, + "logps/ref_chosen": -59.80299377441406, + "logps/ref_rejected": -88.75750732421875, + "logps/rejected": -609.1900634765625, + "loss": 1.0421, + "margin_dpo/margin_mean": 232.3001708984375, + "margin_dpo/margin_std": 231.21661376953125, + "step": 354 + }, + { + "KL/chosen_KL_mean": -286.2269287109375, + "KL/mean": -399.2731018066406, + "KL/rejected_KL_mean": -512.3192749023438, + "KL/std": 236.06967163085938, + "epoch": 0.5212922173274597, + "fcm_dpo/beta": 0.0018591878470033407, + "fcm_dpo/delta": -0.021364610642194748, + "fcm_dpo/margin": 226.09228515625, + "fcm_dpo/q_t": 0.4009643793106079, + "grad_norm": 33.417415618896484, + "learning_rate": 2.7689777072570284e-07, + "logits/chosen": -0.5266305208206177, + "logits/rejected": -0.5164577960968018, + "logps/chosen": -340.35546875, + "logps/ref_chosen": -54.12849807739258, + "logps/ref_rejected": -82.40606689453125, + "logps/rejected": -594.725341796875, + "loss": 1.056, + "margin_dpo/margin_mean": 226.09228515625, + "margin_dpo/margin_std": 236.9951171875, + "step": 355 + }, + { + "KL/chosen_KL_mean": -351.47430419921875, + "KL/mean": -416.1058654785156, + "KL/rejected_KL_mean": -480.7374267578125, + "KL/std": 247.41590881347656, + "epoch": 0.5227606461086637, + "fcm_dpo/beta": 0.0019163393881171942, + "fcm_dpo/delta": 0.1543731987476349, + "fcm_dpo/margin": 129.26312255859375, + "fcm_dpo/q_t": 0.44436901807785034, + "grad_norm": 38.0467643737793, + "learning_rate": 2.7562154104130176e-07, + "logits/chosen": -0.4860239028930664, + "logits/rejected": -0.46484631299972534, + "logps/chosen": -416.14813232421875, + "logps/ref_chosen": -64.6738052368164, + "logps/ref_rejected": -75.89926147460938, + "logps/rejected": -556.63671875, + "loss": 1.2369, + "margin_dpo/margin_mean": 129.26312255859375, + "margin_dpo/margin_std": 304.29168701171875, + "step": 356 + }, + { + "KL/chosen_KL_mean": -321.70184326171875, + "KL/mean": -426.21234130859375, + "KL/rejected_KL_mean": -530.7227783203125, + "KL/std": 239.44512939453125, + "epoch": 0.5242290748898678, + "fcm_dpo/beta": 0.0019112075679004192, + "fcm_dpo/delta": 0.00017703324556350708, + "fcm_dpo/margin": 209.02096557617188, + "fcm_dpo/q_t": 0.406343549489975, + "grad_norm": 24.745952606201172, + "learning_rate": 2.7434463620546594e-07, + "logits/chosen": -0.47398656606674194, + "logits/rejected": -0.4626181721687317, + "logps/chosen": -374.4276123046875, + "logps/ref_chosen": -52.725799560546875, + "logps/ref_rejected": -86.84115600585938, + "logps/rejected": -617.56396484375, + "loss": 1.0775, + "margin_dpo/margin_mean": 209.02096557617188, + "margin_dpo/margin_std": 240.1793212890625, + "step": 357 + }, + { + "KL/chosen_KL_mean": -295.3409729003906, + "KL/mean": -391.8931884765625, + "KL/rejected_KL_mean": -488.4454345703125, + "KL/std": 233.13546752929688, + "epoch": 0.5256975036710719, + "fcm_dpo/beta": 0.0019269022159278393, + "fcm_dpo/delta": 0.028992902487516403, + "fcm_dpo/margin": 193.10446166992188, + "fcm_dpo/q_t": 0.4144141972064972, + "grad_norm": 27.003738403320312, + "learning_rate": 2.730670898658255e-07, + "logits/chosen": -0.5157039165496826, + "logits/rejected": -0.5015970468521118, + "logps/chosen": -358.5464172363281, + "logps/ref_chosen": -63.20543670654297, + "logps/ref_rejected": -88.373291015625, + "logps/rejected": -576.8187255859375, + "loss": 1.1041, + "margin_dpo/margin_mean": 193.10446166992188, + "margin_dpo/margin_std": 253.48260498046875, + "step": 358 + }, + { + "KL/chosen_KL_mean": -329.5625305175781, + "KL/mean": -441.9998474121094, + "KL/rejected_KL_mean": -554.4371948242188, + "KL/std": 232.54946899414062, + "epoch": 0.527165932452276, + "fcm_dpo/beta": 0.0019179590744897723, + "fcm_dpo/delta": -0.03281618282198906, + "fcm_dpo/margin": 224.8746337890625, + "fcm_dpo/q_t": 0.40181970596313477, + "grad_norm": 28.059730529785156, + "learning_rate": 2.717889356869146e-07, + "logits/chosen": -0.4727786183357239, + "logits/rejected": -0.4688911736011505, + "logps/chosen": -385.9327392578125, + "logps/ref_chosen": -56.370216369628906, + "logps/ref_rejected": -82.17375183105469, + "logps/rejected": -636.6109619140625, + "loss": 1.0721, + "margin_dpo/margin_mean": 224.8746337890625, + "margin_dpo/margin_std": 285.3248596191406, + "step": 359 + }, + { + "KL/chosen_KL_mean": -333.458740234375, + "KL/mean": -414.25750732421875, + "KL/rejected_KL_mean": -495.05621337890625, + "KL/std": 196.81692504882812, + "epoch": 0.5286343612334802, + "fcm_dpo/beta": 0.0019466321682557464, + "fcm_dpo/delta": 0.0879821926355362, + "fcm_dpo/margin": 161.5974578857422, + "fcm_dpo/q_t": 0.4257877469062805, + "grad_norm": 42.480712890625, + "learning_rate": 2.7051020734928443e-07, + "logits/chosen": -0.4350992441177368, + "logits/rejected": -0.42406588792800903, + "logps/chosen": -384.91912841796875, + "logps/ref_chosen": -51.460384368896484, + "logps/ref_rejected": -69.83892059326172, + "logps/rejected": -564.8951416015625, + "loss": 1.1332, + "margin_dpo/margin_mean": 161.59744262695312, + "margin_dpo/margin_std": 199.5988311767578, + "step": 360 + }, + { + "KL/chosen_KL_mean": -343.19219970703125, + "KL/mean": -426.57568359375, + "KL/rejected_KL_mean": -509.95916748046875, + "KL/std": 232.37318420410156, + "epoch": 0.5301027900146843, + "fcm_dpo/beta": 0.0019770534709095955, + "fcm_dpo/delta": 0.07246075570583344, + "fcm_dpo/margin": 166.7669677734375, + "fcm_dpo/q_t": 0.42386797070503235, + "grad_norm": 34.09297180175781, + "learning_rate": 2.6923093854861593e-07, + "logits/chosen": -0.43081650137901306, + "logits/rejected": -0.4273498058319092, + "logps/chosen": -397.06170654296875, + "logps/ref_chosen": -53.86951446533203, + "logps/ref_rejected": -90.7692642211914, + "logps/rejected": -600.7283935546875, + "loss": 1.1452, + "margin_dpo/margin_mean": 166.7669677734375, + "margin_dpo/margin_std": 257.58953857421875, + "step": 361 + }, + { + "KL/chosen_KL_mean": -295.250244140625, + "KL/mean": -429.8717346191406, + "KL/rejected_KL_mean": -564.4932861328125, + "KL/std": 243.06881713867188, + "epoch": 0.5315712187958884, + "fcm_dpo/beta": 0.001937607303261757, + "fcm_dpo/delta": -0.12887535989284515, + "fcm_dpo/margin": 269.2430114746094, + "fcm_dpo/q_t": 0.3792804479598999, + "grad_norm": 29.63429069519043, + "learning_rate": 2.679511629948319e-07, + "logits/chosen": -0.43117934465408325, + "logits/rejected": -0.4422128200531006, + "logps/chosen": -353.8893127441406, + "logps/ref_chosen": -58.639060974121094, + "logps/ref_rejected": -105.58195495605469, + "logps/rejected": -670.0751953125, + "loss": 0.9869, + "margin_dpo/margin_mean": 269.2430114746094, + "margin_dpo/margin_std": 257.28125, + "step": 362 + }, + { + "KL/chosen_KL_mean": -264.6348571777344, + "KL/mean": -406.2776184082031, + "KL/rejected_KL_mean": -547.92041015625, + "KL/std": 246.21340942382812, + "epoch": 0.5330396475770925, + "fcm_dpo/beta": 0.0018945279298350215, + "fcm_dpo/delta": -0.1442004144191742, + "fcm_dpo/margin": 283.2854919433594, + "fcm_dpo/q_t": 0.375781774520874, + "grad_norm": 24.405099868774414, + "learning_rate": 2.6667091441120816e-07, + "logits/chosen": -0.41785454750061035, + "logits/rejected": -0.4117404818534851, + "logps/chosen": -309.1932373046875, + "logps/ref_chosen": -44.558380126953125, + "logps/ref_rejected": -74.69496154785156, + "logps/rejected": -622.6153564453125, + "loss": 0.9783, + "margin_dpo/margin_mean": 283.2854919433594, + "margin_dpo/margin_std": 266.176025390625, + "step": 363 + }, + { + "KL/chosen_KL_mean": -288.89862060546875, + "KL/mean": -392.5069580078125, + "KL/rejected_KL_mean": -496.11529541015625, + "KL/std": 236.96873474121094, + "epoch": 0.5345080763582967, + "fcm_dpo/beta": 0.001885156612843275, + "fcm_dpo/delta": 0.0093899667263031, + "fcm_dpo/margin": 207.2166748046875, + "fcm_dpo/q_t": 0.4092685282230377, + "grad_norm": 21.653566360473633, + "learning_rate": 2.6539022653348575e-07, + "logits/chosen": -0.44824981689453125, + "logits/rejected": -0.45862144231796265, + "logps/chosen": -337.793212890625, + "logps/ref_chosen": -48.894622802734375, + "logps/ref_rejected": -91.395751953125, + "logps/rejected": -587.5110473632812, + "loss": 1.097, + "margin_dpo/margin_mean": 207.21669006347656, + "margin_dpo/margin_std": 273.45452880859375, + "step": 364 + }, + { + "KL/chosen_KL_mean": -273.8106689453125, + "KL/mean": -381.73956298828125, + "KL/rejected_KL_mean": -489.6685485839844, + "KL/std": 248.11196899414062, + "epoch": 0.5359765051395007, + "fcm_dpo/beta": 0.0018767972942441702, + "fcm_dpo/delta": -0.0053392937406897545, + "fcm_dpo/margin": 215.85787963867188, + "fcm_dpo/q_t": 0.4069938659667969, + "grad_norm": 21.937294006347656, + "learning_rate": 2.641091331089811e-07, + "logits/chosen": -0.4212133288383484, + "logits/rejected": -0.43191930651664734, + "logps/chosen": -325.30340576171875, + "logps/ref_chosen": -51.49274444580078, + "logps/ref_rejected": -92.70166778564453, + "logps/rejected": -582.3702392578125, + "loss": 1.0753, + "margin_dpo/margin_mean": 215.85787963867188, + "margin_dpo/margin_std": 255.764892578125, + "step": 365 + }, + { + "KL/chosen_KL_mean": -245.94454956054688, + "KL/mean": -353.47900390625, + "KL/rejected_KL_mean": -461.013427734375, + "KL/std": 230.1700897216797, + "epoch": 0.5374449339207048, + "fcm_dpo/beta": 0.001868388382717967, + "fcm_dpo/delta": -0.002255551517009735, + "fcm_dpo/margin": 215.06887817382812, + "fcm_dpo/q_t": 0.4071570634841919, + "grad_norm": 21.38697052001953, + "learning_rate": 2.6282766789569736e-07, + "logits/chosen": -0.4181390404701233, + "logits/rejected": -0.4320718050003052, + "logps/chosen": -290.6651306152344, + "logps/ref_chosen": -44.7205696105957, + "logps/ref_rejected": -83.31040954589844, + "logps/rejected": -544.3238525390625, + "loss": 1.0873, + "margin_dpo/margin_mean": 215.06887817382812, + "margin_dpo/margin_std": 274.4052429199219, + "step": 366 + }, + { + "KL/chosen_KL_mean": -257.902099609375, + "KL/mean": -347.42755126953125, + "KL/rejected_KL_mean": -436.95306396484375, + "KL/std": 208.62741088867188, + "epoch": 0.5389133627019089, + "fcm_dpo/beta": 0.001896706409752369, + "fcm_dpo/delta": 0.062337495386600494, + "fcm_dpo/margin": 179.05096435546875, + "fcm_dpo/q_t": 0.4197365641593933, + "grad_norm": 22.023393630981445, + "learning_rate": 2.615458646614349e-07, + "logits/chosen": -0.4201430678367615, + "logits/rejected": -0.4021303057670593, + "logps/chosen": -316.3074951171875, + "logps/ref_chosen": -58.405418395996094, + "logps/ref_rejected": -76.75132751464844, + "logps/rejected": -513.7044067382812, + "loss": 1.1196, + "margin_dpo/margin_mean": 179.05096435546875, + "margin_dpo/margin_std": 226.32080078125, + "step": 367 + }, + { + "KL/chosen_KL_mean": -236.77053833007812, + "KL/mean": -378.0199890136719, + "KL/rejected_KL_mean": -519.2694091796875, + "KL/std": 237.63888549804688, + "epoch": 0.540381791483113, + "fcm_dpo/beta": 0.0018626508535817266, + "fcm_dpo/delta": -0.1331697702407837, + "fcm_dpo/margin": 282.4989013671875, + "fcm_dpo/q_t": 0.37490114569664, + "grad_norm": 36.5501823425293, + "learning_rate": 2.6026375718290083e-07, + "logits/chosen": -0.4193364977836609, + "logits/rejected": -0.42513972520828247, + "logps/chosen": -281.2230529785156, + "logps/ref_chosen": -44.452518463134766, + "logps/ref_rejected": -98.55526733398438, + "logps/rejected": -617.82470703125, + "loss": 0.9663, + "margin_dpo/margin_mean": 282.4989013671875, + "margin_dpo/margin_std": 217.8269500732422, + "step": 368 + }, + { + "KL/chosen_KL_mean": -326.0372314453125, + "KL/mean": -400.8064880371094, + "KL/rejected_KL_mean": -475.5757141113281, + "KL/std": 242.97593688964844, + "epoch": 0.5418502202643172, + "fcm_dpo/beta": 0.001889625797048211, + "fcm_dpo/delta": 0.12015080451965332, + "fcm_dpo/margin": 149.5384979248047, + "fcm_dpo/q_t": 0.4347541332244873, + "grad_norm": 25.68787384033203, + "learning_rate": 2.589813792448196e-07, + "logits/chosen": -0.44025710225105286, + "logits/rejected": -0.42140746116638184, + "logps/chosen": -397.41876220703125, + "logps/ref_chosen": -71.38150024414062, + "logps/ref_rejected": -91.29582214355469, + "logps/rejected": -566.87158203125, + "loss": 1.19, + "margin_dpo/margin_mean": 149.5384979248047, + "margin_dpo/margin_std": 279.17926025390625, + "step": 369 + }, + { + "KL/chosen_KL_mean": -332.7804870605469, + "KL/mean": -408.59906005859375, + "KL/rejected_KL_mean": -484.41766357421875, + "KL/std": 246.52499389648438, + "epoch": 0.5433186490455213, + "fcm_dpo/beta": 0.001930559752508998, + "fcm_dpo/delta": 0.11008969694375992, + "fcm_dpo/margin": 151.63714599609375, + "fcm_dpo/q_t": 0.4327510893344879, + "grad_norm": 27.80760955810547, + "learning_rate": 2.5769876463904263e-07, + "logits/chosen": -0.4499947726726532, + "logits/rejected": -0.44359490275382996, + "logps/chosen": -404.38800048828125, + "logps/ref_chosen": -71.60749816894531, + "logps/ref_rejected": -97.25978088378906, + "logps/rejected": -581.677490234375, + "loss": 1.1832, + "margin_dpo/margin_mean": 151.63714599609375, + "margin_dpo/margin_std": 275.9014892578125, + "step": 370 + }, + { + "KL/chosen_KL_mean": -326.5951843261719, + "KL/mean": -433.36822509765625, + "KL/rejected_KL_mean": -540.1412963867188, + "KL/std": 257.112548828125, + "epoch": 0.5447870778267254, + "fcm_dpo/beta": 0.0019381106831133366, + "fcm_dpo/delta": -0.014613218605518341, + "fcm_dpo/margin": 213.54608154296875, + "fcm_dpo/q_t": 0.4060685932636261, + "grad_norm": 21.661701202392578, + "learning_rate": 2.5641594716365744e-07, + "logits/chosen": -0.5168248414993286, + "logits/rejected": -0.5063666105270386, + "logps/chosen": -396.0096740722656, + "logps/ref_chosen": -69.41448974609375, + "logps/ref_rejected": -99.17217254638672, + "logps/rejected": -639.3134765625, + "loss": 1.0939, + "margin_dpo/margin_mean": 213.54608154296875, + "margin_dpo/margin_std": 299.3140869140625, + "step": 371 + }, + { + "KL/chosen_KL_mean": -314.97894287109375, + "KL/mean": -441.3173522949219, + "KL/rejected_KL_mean": -567.65576171875, + "KL/std": 292.46392822265625, + "epoch": 0.5462555066079295, + "fcm_dpo/beta": 0.0019034836441278458, + "fcm_dpo/delta": -0.08524809032678604, + "fcm_dpo/margin": 252.6768341064453, + "fcm_dpo/q_t": 0.3927803039550781, + "grad_norm": 20.759563446044922, + "learning_rate": 2.551329606220976e-07, + "logits/chosen": -0.46310439705848694, + "logits/rejected": -0.4459260106086731, + "logps/chosen": -376.79693603515625, + "logps/ref_chosen": -61.8179931640625, + "logps/ref_rejected": -78.53948974609375, + "logps/rejected": -646.1952514648438, + "loss": 1.0493, + "margin_dpo/margin_mean": 252.6768341064453, + "margin_dpo/margin_std": 329.3980407714844, + "step": 372 + }, + { + "KL/chosen_KL_mean": -346.1135559082031, + "KL/mean": -465.76202392578125, + "KL/rejected_KL_mean": -585.4105224609375, + "KL/std": 286.3425598144531, + "epoch": 0.5477239353891337, + "fcm_dpo/beta": 0.001893439912237227, + "fcm_dpo/delta": -0.05597112700343132, + "fcm_dpo/margin": 239.29693603515625, + "fcm_dpo/q_t": 0.39477214217185974, + "grad_norm": 37.95475769042969, + "learning_rate": 2.538498388222517e-07, + "logits/chosen": -0.4648551344871521, + "logits/rejected": -0.44541704654693604, + "logps/chosen": -410.3306884765625, + "logps/ref_chosen": -64.21713256835938, + "logps/ref_rejected": -85.95960998535156, + "logps/rejected": -671.3701171875, + "loss": 1.05, + "margin_dpo/margin_mean": 239.29693603515625, + "margin_dpo/margin_std": 269.4728698730469, + "step": 373 + }, + { + "KL/chosen_KL_mean": -320.47308349609375, + "KL/mean": -428.29461669921875, + "KL/rejected_KL_mean": -536.1160888671875, + "KL/std": 311.81317138671875, + "epoch": 0.5491923641703378, + "fcm_dpo/beta": 0.00186735182069242, + "fcm_dpo/delta": -0.003267388790845871, + "fcm_dpo/margin": 215.6429901123047, + "fcm_dpo/q_t": 0.411865770816803, + "grad_norm": 22.822025299072266, + "learning_rate": 2.525666155755725e-07, + "logits/chosen": -0.4975176453590393, + "logits/rejected": -0.4788419008255005, + "logps/chosen": -391.123291015625, + "logps/ref_chosen": -70.65018463134766, + "logps/ref_rejected": -93.64016723632812, + "logps/rejected": -629.7562255859375, + "loss": 1.1191, + "margin_dpo/margin_mean": 215.6429901123047, + "margin_dpo/margin_std": 348.92791748046875, + "step": 374 + }, + { + "KL/chosen_KL_mean": -333.2713623046875, + "KL/mean": -435.57318115234375, + "KL/rejected_KL_mean": -537.8748779296875, + "KL/std": 250.6235809326172, + "epoch": 0.5506607929515418, + "fcm_dpo/beta": 0.0018701537046581507, + "fcm_dpo/delta": 0.017490137368440628, + "fcm_dpo/margin": 204.60354614257812, + "fcm_dpo/q_t": 0.41193336248397827, + "grad_norm": 35.44966125488281, + "learning_rate": 2.512833246961859e-07, + "logits/chosen": -0.48623552918434143, + "logits/rejected": -0.4854326844215393, + "logps/chosen": -393.35162353515625, + "logps/ref_chosen": -60.080223083496094, + "logps/ref_rejected": -88.93830871582031, + "logps/rejected": -626.813232421875, + "loss": 1.1195, + "margin_dpo/margin_mean": 204.60354614257812, + "margin_dpo/margin_std": 305.8067626953125, + "step": 375 + }, + { + "KL/chosen_KL_mean": -329.8739929199219, + "KL/mean": -463.758544921875, + "KL/rejected_KL_mean": -597.6431884765625, + "KL/std": 283.9222106933594, + "epoch": 0.5521292217327459, + "fcm_dpo/beta": 0.001852883491665125, + "fcm_dpo/delta": -0.10125644505023956, + "fcm_dpo/margin": 267.7691650390625, + "fcm_dpo/q_t": 0.38710707426071167, + "grad_norm": 24.405960083007812, + "learning_rate": 2.5e-07, + "logits/chosen": -0.4748380184173584, + "logits/rejected": -0.4657718539237976, + "logps/chosen": -392.5343017578125, + "logps/ref_chosen": -62.660308837890625, + "logps/ref_rejected": -105.52660369873047, + "logps/rejected": -703.1697998046875, + "loss": 1.0372, + "margin_dpo/margin_mean": 267.7691650390625, + "margin_dpo/margin_std": 331.33099365234375, + "step": 376 + }, + { + "KL/chosen_KL_mean": -334.75823974609375, + "KL/mean": -461.15093994140625, + "KL/rejected_KL_mean": -587.543701171875, + "KL/std": 287.4979248046875, + "epoch": 0.55359765051395, + "fcm_dpo/beta": 0.0018305475823581219, + "fcm_dpo/delta": -0.06573105603456497, + "fcm_dpo/margin": 252.7854461669922, + "fcm_dpo/q_t": 0.3940247893333435, + "grad_norm": 23.72755241394043, + "learning_rate": 2.487166753038141e-07, + "logits/chosen": -0.3885685205459595, + "logits/rejected": -0.385770320892334, + "logps/chosen": -389.2369384765625, + "logps/ref_chosen": -54.478736877441406, + "logps/ref_rejected": -98.70335388183594, + "logps/rejected": -686.2470092773438, + "loss": 1.0482, + "margin_dpo/margin_mean": 252.7854461669922, + "margin_dpo/margin_std": 304.465087890625, + "step": 377 + }, + { + "KL/chosen_KL_mean": -306.72894287109375, + "KL/mean": -441.921875, + "KL/rejected_KL_mean": -577.1148681640625, + "KL/std": 269.91107177734375, + "epoch": 0.5550660792951542, + "fcm_dpo/beta": 0.0017946755979210138, + "fcm_dpo/delta": -0.08971542119979858, + "fcm_dpo/margin": 270.3858642578125, + "fcm_dpo/q_t": 0.3866649270057678, + "grad_norm": 21.382219314575195, + "learning_rate": 2.4743338442442754e-07, + "logits/chosen": -0.40874114632606506, + "logits/rejected": -0.41893163323402405, + "logps/chosen": -351.7494812011719, + "logps/ref_chosen": -45.02053451538086, + "logps/ref_rejected": -88.0469741821289, + "logps/rejected": -665.1618041992188, + "loss": 1.0224, + "margin_dpo/margin_mean": 270.3858642578125, + "margin_dpo/margin_std": 291.0148010253906, + "step": 378 + }, + { + "KL/chosen_KL_mean": -351.1424560546875, + "KL/mean": -483.84320068359375, + "KL/rejected_KL_mean": -616.5439453125, + "KL/std": 272.0304260253906, + "epoch": 0.5565345080763583, + "fcm_dpo/beta": 0.0017617056146264076, + "fcm_dpo/delta": -0.07156138122081757, + "fcm_dpo/margin": 265.40155029296875, + "fcm_dpo/q_t": 0.39362668991088867, + "grad_norm": 24.216798782348633, + "learning_rate": 2.461501611777483e-07, + "logits/chosen": -0.4043707847595215, + "logits/rejected": -0.4257460832595825, + "logps/chosen": -404.32452392578125, + "logps/ref_chosen": -53.182098388671875, + "logps/ref_rejected": -114.3001708984375, + "logps/rejected": -730.8441162109375, + "loss": 1.0483, + "margin_dpo/margin_mean": 265.4015197753906, + "margin_dpo/margin_std": 324.57073974609375, + "step": 379 + }, + { + "KL/chosen_KL_mean": -344.0395812988281, + "KL/mean": -488.787109375, + "KL/rejected_KL_mean": -633.53466796875, + "KL/std": 300.50885009765625, + "epoch": 0.5580029368575624, + "fcm_dpo/beta": 0.0017366987885907292, + "fcm_dpo/delta": -0.10811088979244232, + "fcm_dpo/margin": 289.4950866699219, + "fcm_dpo/q_t": 0.3833885192871094, + "grad_norm": 24.989524841308594, + "learning_rate": 2.4486703937790243e-07, + "logits/chosen": -0.41599875688552856, + "logits/rejected": -0.44077515602111816, + "logps/chosen": -395.3926086425781, + "logps/ref_chosen": -51.3530387878418, + "logps/ref_rejected": -104.19169616699219, + "logps/rejected": -737.726318359375, + "loss": 1.0199, + "margin_dpo/margin_mean": 289.4950866699219, + "margin_dpo/margin_std": 327.95684814453125, + "step": 380 + }, + { + "KL/chosen_KL_mean": -352.6854248046875, + "KL/mean": -451.4104309082031, + "KL/rejected_KL_mean": -550.135498046875, + "KL/std": 252.61831665039062, + "epoch": 0.5594713656387665, + "fcm_dpo/beta": 0.0017375698080286384, + "fcm_dpo/delta": 0.05898090451955795, + "fcm_dpo/margin": 197.4500732421875, + "fcm_dpo/q_t": 0.42169779539108276, + "grad_norm": 25.88216781616211, + "learning_rate": 2.435840528363426e-07, + "logits/chosen": -0.4213838577270508, + "logits/rejected": -0.4027274549007416, + "logps/chosen": -410.48846435546875, + "logps/ref_chosen": -57.80306625366211, + "logps/ref_rejected": -79.21940612792969, + "logps/rejected": -629.3548583984375, + "loss": 1.1561, + "margin_dpo/margin_mean": 197.4500732421875, + "margin_dpo/margin_std": 342.58575439453125, + "step": 381 + }, + { + "KL/chosen_KL_mean": -350.3328857421875, + "KL/mean": -471.0789489746094, + "KL/rejected_KL_mean": -591.8250122070312, + "KL/std": 237.4173583984375, + "epoch": 0.5609397944199707, + "fcm_dpo/beta": 0.0017401074292138219, + "fcm_dpo/delta": -0.021109461784362793, + "fcm_dpo/margin": 241.49209594726562, + "fcm_dpo/q_t": 0.40183016657829285, + "grad_norm": 32.546287536621094, + "learning_rate": 2.4230123536095745e-07, + "logits/chosen": -0.4856158494949341, + "logits/rejected": -0.4916708171367645, + "logps/chosen": -416.35321044921875, + "logps/ref_chosen": -66.02030181884766, + "logps/ref_rejected": -110.71016693115234, + "logps/rejected": -702.53515625, + "loss": 1.0602, + "margin_dpo/margin_mean": 241.49209594726562, + "margin_dpo/margin_std": 266.0589904785156, + "step": 382 + }, + { + "KL/chosen_KL_mean": -351.69720458984375, + "KL/mean": -473.2536926269531, + "KL/rejected_KL_mean": -594.8101806640625, + "KL/std": 264.37579345703125, + "epoch": 0.5624082232011748, + "fcm_dpo/beta": 0.0017301496118307114, + "fcm_dpo/delta": -0.021601904183626175, + "fcm_dpo/margin": 243.11300659179688, + "fcm_dpo/q_t": 0.40308916568756104, + "grad_norm": 23.824296951293945, + "learning_rate": 2.4101862075518037e-07, + "logits/chosen": -0.4409335255622864, + "logits/rejected": -0.45157590508461, + "logps/chosen": -402.08868408203125, + "logps/ref_chosen": -50.39148712158203, + "logps/ref_rejected": -93.71589660644531, + "logps/rejected": -688.5260009765625, + "loss": 1.091, + "margin_dpo/margin_mean": 243.11300659179688, + "margin_dpo/margin_std": 342.62615966796875, + "step": 383 + }, + { + "KL/chosen_KL_mean": -353.783447265625, + "KL/mean": -453.2200927734375, + "KL/rejected_KL_mean": -552.65673828125, + "KL/std": 246.05821228027344, + "epoch": 0.5638766519823789, + "fcm_dpo/beta": 0.0017507347511127591, + "fcm_dpo/delta": 0.05325193330645561, + "fcm_dpo/margin": 198.87332153320312, + "fcm_dpo/q_t": 0.4177956283092499, + "grad_norm": 26.645626068115234, + "learning_rate": 2.397362428170992e-07, + "logits/chosen": -0.49639368057250977, + "logits/rejected": -0.4920317530632019, + "logps/chosen": -405.82952880859375, + "logps/ref_chosen": -52.046104431152344, + "logps/ref_rejected": -85.76089477539062, + "logps/rejected": -638.4176635742188, + "loss": 1.109, + "margin_dpo/margin_mean": 198.8733367919922, + "margin_dpo/margin_std": 234.27737426757812, + "step": 384 + }, + { + "KL/chosen_KL_mean": -335.99951171875, + "KL/mean": -452.42498779296875, + "KL/rejected_KL_mean": -568.8504638671875, + "KL/std": 211.4859619140625, + "epoch": 0.5653450807635829, + "fcm_dpo/beta": 0.0017469192389398813, + "fcm_dpo/delta": -0.007062938995659351, + "fcm_dpo/margin": 232.8509521484375, + "fcm_dpo/q_t": 0.4036387503147125, + "grad_norm": 32.45009231567383, + "learning_rate": 2.3845413533856514e-07, + "logits/chosen": -0.4798526167869568, + "logits/rejected": -0.4585825204849243, + "logps/chosen": -401.5516662597656, + "logps/ref_chosen": -65.55215454101562, + "logps/ref_rejected": -77.82792663574219, + "logps/rejected": -646.6783447265625, + "loss": 1.0606, + "margin_dpo/margin_mean": 232.8509521484375, + "margin_dpo/margin_std": 235.7921142578125, + "step": 385 + }, + { + "KL/chosen_KL_mean": -356.20208740234375, + "KL/mean": -485.5333251953125, + "KL/rejected_KL_mean": -614.864501953125, + "KL/std": 275.59869384765625, + "epoch": 0.566813509544787, + "fcm_dpo/beta": 0.0017344809602946043, + "fcm_dpo/delta": -0.050910089164972305, + "fcm_dpo/margin": 258.6624450683594, + "fcm_dpo/q_t": 0.3971731662750244, + "grad_norm": 24.357194900512695, + "learning_rate": 2.3717233210430254e-07, + "logits/chosen": -0.47782373428344727, + "logits/rejected": -0.4760916233062744, + "logps/chosen": -414.4239196777344, + "logps/ref_chosen": -58.22185516357422, + "logps/ref_rejected": -92.32742309570312, + "logps/rejected": -707.19189453125, + "loss": 1.0564, + "margin_dpo/margin_mean": 258.6624450683594, + "margin_dpo/margin_std": 317.303466796875, + "step": 386 + }, + { + "KL/chosen_KL_mean": -365.7936096191406, + "KL/mean": -475.52777099609375, + "KL/rejected_KL_mean": -585.2620239257812, + "KL/std": 252.08740234375, + "epoch": 0.5682819383259912, + "fcm_dpo/beta": 0.001727396622300148, + "fcm_dpo/delta": 0.021443448960781097, + "fcm_dpo/margin": 219.4683837890625, + "fcm_dpo/q_t": 0.4107184112071991, + "grad_norm": 25.59358024597168, + "learning_rate": 2.3589086689101889e-07, + "logits/chosen": -0.4887543320655823, + "logits/rejected": -0.4731796383857727, + "logps/chosen": -432.21307373046875, + "logps/ref_chosen": -66.41944885253906, + "logps/ref_rejected": -92.16915893554688, + "logps/rejected": -677.43115234375, + "loss": 1.0906, + "margin_dpo/margin_mean": 219.4683837890625, + "margin_dpo/margin_std": 254.8131103515625, + "step": 387 + }, + { + "KL/chosen_KL_mean": -350.8273010253906, + "KL/mean": -490.6996154785156, + "KL/rejected_KL_mean": -630.5719604492188, + "KL/std": 284.7591552734375, + "epoch": 0.5697503671071953, + "fcm_dpo/beta": 0.0017110585467889905, + "fcm_dpo/delta": -0.08294045180082321, + "fcm_dpo/margin": 279.74468994140625, + "fcm_dpo/q_t": 0.39071470499038696, + "grad_norm": 25.492565155029297, + "learning_rate": 2.3460977346651428e-07, + "logits/chosen": -0.3969477415084839, + "logits/rejected": -0.40795570611953735, + "logps/chosen": -400.9567565917969, + "logps/ref_chosen": -50.129459381103516, + "logps/ref_rejected": -104.43305969238281, + "logps/rejected": -735.0050048828125, + "loss": 1.0314, + "margin_dpo/margin_mean": 279.7446594238281, + "margin_dpo/margin_std": 319.7544250488281, + "step": 388 + }, + { + "KL/chosen_KL_mean": -381.65643310546875, + "KL/mean": -502.360595703125, + "KL/rejected_KL_mean": -623.0647583007812, + "KL/std": 286.53057861328125, + "epoch": 0.5712187958883994, + "fcm_dpo/beta": 0.0017027594149112701, + "fcm_dpo/delta": -0.011586458422243595, + "fcm_dpo/margin": 241.40829467773438, + "fcm_dpo/q_t": 0.40512824058532715, + "grad_norm": 28.0815486907959, + "learning_rate": 2.3332908558879177e-07, + "logits/chosen": -0.4462489187717438, + "logits/rejected": -0.43941187858581543, + "logps/chosen": -439.56304931640625, + "logps/ref_chosen": -57.906593322753906, + "logps/ref_rejected": -77.91454315185547, + "logps/rejected": -700.9793090820312, + "loss": 1.0814, + "margin_dpo/margin_mean": 241.40829467773438, + "margin_dpo/margin_std": 308.45904541015625, + "step": 389 + }, + { + "KL/chosen_KL_mean": -375.6517333984375, + "KL/mean": -493.81524658203125, + "KL/rejected_KL_mean": -611.978759765625, + "KL/std": 276.1775207519531, + "epoch": 0.5726872246696035, + "fcm_dpo/beta": 0.001697941217571497, + "fcm_dpo/delta": -0.001557283103466034, + "fcm_dpo/margin": 236.3270263671875, + "fcm_dpo/q_t": 0.4103020429611206, + "grad_norm": 24.159801483154297, + "learning_rate": 2.320488370051681e-07, + "logits/chosen": -0.4154917895793915, + "logits/rejected": -0.40859878063201904, + "logps/chosen": -424.8776550292969, + "logps/ref_chosen": -49.22591781616211, + "logps/ref_rejected": -85.5281982421875, + "logps/rejected": -697.5069580078125, + "loss": 1.1063, + "margin_dpo/margin_mean": 236.32704162597656, + "margin_dpo/margin_std": 351.1764221191406, + "step": 390 + }, + { + "KL/chosen_KL_mean": -365.83709716796875, + "KL/mean": -436.49969482421875, + "KL/rejected_KL_mean": -507.162353515625, + "KL/std": 257.5408020019531, + "epoch": 0.5741556534508077, + "fcm_dpo/beta": 0.0017448479775339365, + "fcm_dpo/delta": 0.1572088599205017, + "fcm_dpo/margin": 141.32522583007812, + "fcm_dpo/q_t": 0.44321030378341675, + "grad_norm": 39.628028869628906, + "learning_rate": 2.3076906145138405e-07, + "logits/chosen": -0.45562559366226196, + "logits/rejected": -0.4495752155780792, + "logps/chosen": -430.166748046875, + "logps/ref_chosen": -64.32965087890625, + "logps/ref_rejected": -86.73820495605469, + "logps/rejected": -593.9005126953125, + "loss": 1.2098, + "margin_dpo/margin_mean": 141.32522583007812, + "margin_dpo/margin_std": 270.3441467285156, + "step": 391 + }, + { + "KL/chosen_KL_mean": -317.95721435546875, + "KL/mean": -458.79046630859375, + "KL/rejected_KL_mean": -599.6236572265625, + "KL/std": 278.5746154785156, + "epoch": 0.5756240822320118, + "fcm_dpo/beta": 0.0017387724947184324, + "fcm_dpo/delta": -0.09427893161773682, + "fcm_dpo/margin": 281.6664123535156, + "fcm_dpo/q_t": 0.38599228858947754, + "grad_norm": 25.61530876159668, + "learning_rate": 2.294897926507156e-07, + "logits/chosen": -0.4354037046432495, + "logits/rejected": -0.42951297760009766, + "logps/chosen": -371.4612121582031, + "logps/ref_chosen": -53.50397872924805, + "logps/ref_rejected": -102.34584045410156, + "logps/rejected": -701.969482421875, + "loss": 1.0072, + "margin_dpo/margin_mean": 281.6664123535156, + "margin_dpo/margin_std": 269.19384765625, + "step": 392 + }, + { + "KL/chosen_KL_mean": -308.1512451171875, + "KL/mean": -415.21820068359375, + "KL/rejected_KL_mean": -522.2850952148438, + "KL/std": 262.9591064453125, + "epoch": 0.5770925110132159, + "fcm_dpo/beta": 0.0017301738262176514, + "fcm_dpo/delta": 0.030642738565802574, + "fcm_dpo/margin": 214.13385009765625, + "fcm_dpo/q_t": 0.4171079993247986, + "grad_norm": 21.046979904174805, + "learning_rate": 2.2821106431308543e-07, + "logits/chosen": -0.41253232955932617, + "logits/rejected": -0.4099976420402527, + "logps/chosen": -354.62518310546875, + "logps/ref_chosen": -46.473915100097656, + "logps/ref_rejected": -71.96885681152344, + "logps/rejected": -594.2539672851562, + "loss": 1.123, + "margin_dpo/margin_mean": 214.13385009765625, + "margin_dpo/margin_std": 329.9991455078125, + "step": 393 + }, + { + "KL/chosen_KL_mean": -352.0980224609375, + "KL/mean": -468.20489501953125, + "KL/rejected_KL_mean": -584.311767578125, + "KL/std": 285.9789733886719, + "epoch": 0.57856093979442, + "fcm_dpo/beta": 0.0017325542867183685, + "fcm_dpo/delta": -0.0024658460170030594, + "fcm_dpo/margin": 232.21368408203125, + "fcm_dpo/q_t": 0.4075622856616974, + "grad_norm": 25.002628326416016, + "learning_rate": 2.2693291013417452e-07, + "logits/chosen": -0.4068412184715271, + "logits/rejected": -0.406298965215683, + "logps/chosen": -405.00958251953125, + "logps/ref_chosen": -52.91154861450195, + "logps/ref_rejected": -90.8226318359375, + "logps/rejected": -675.1343994140625, + "loss": 1.084, + "margin_dpo/margin_mean": 232.21368408203125, + "margin_dpo/margin_std": 293.77191162109375, + "step": 394 + }, + { + "KL/chosen_KL_mean": -350.91033935546875, + "KL/mean": -474.3104248046875, + "KL/rejected_KL_mean": -597.7105712890625, + "KL/std": 282.8669738769531, + "epoch": 0.580029368575624, + "fcm_dpo/beta": 0.001723103690892458, + "fcm_dpo/delta": -0.026621468365192413, + "fcm_dpo/margin": 246.8002166748047, + "fcm_dpo/q_t": 0.40307846665382385, + "grad_norm": 26.7552433013916, + "learning_rate": 2.2565536379453404e-07, + "logits/chosen": -0.46250438690185547, + "logits/rejected": -0.46052122116088867, + "logps/chosen": -413.4564208984375, + "logps/ref_chosen": -62.546112060546875, + "logps/ref_rejected": -83.78262329101562, + "logps/rejected": -681.4931640625, + "loss": 1.0793, + "margin_dpo/margin_mean": 246.80023193359375, + "margin_dpo/margin_std": 326.7712097167969, + "step": 395 + }, + { + "KL/chosen_KL_mean": -351.205078125, + "KL/mean": -466.0959777832031, + "KL/rejected_KL_mean": -580.9869384765625, + "KL/std": 271.4552307128906, + "epoch": 0.5814977973568282, + "fcm_dpo/beta": 0.0017233972903341055, + "fcm_dpo/delta": 0.004093697294592857, + "fcm_dpo/margin": 229.7818145751953, + "fcm_dpo/q_t": 0.40719085931777954, + "grad_norm": 22.856327056884766, + "learning_rate": 2.2437845895869825e-07, + "logits/chosen": -0.49015119671821594, + "logits/rejected": -0.47574925422668457, + "logps/chosen": -420.20098876953125, + "logps/ref_chosen": -68.99594116210938, + "logps/ref_rejected": -88.64665985107422, + "logps/rejected": -669.633544921875, + "loss": 1.0721, + "margin_dpo/margin_mean": 229.7818145751953, + "margin_dpo/margin_std": 249.69090270996094, + "step": 396 + }, + { + "KL/chosen_KL_mean": -333.80255126953125, + "KL/mean": -477.57720947265625, + "KL/rejected_KL_mean": -621.3518676757812, + "KL/std": 273.3784484863281, + "epoch": 0.5829662261380323, + "fcm_dpo/beta": 0.001697493251413107, + "fcm_dpo/delta": -0.0931856781244278, + "fcm_dpo/margin": 287.54931640625, + "fcm_dpo/q_t": 0.3860146105289459, + "grad_norm": 30.467397689819336, + "learning_rate": 2.2310222927429716e-07, + "logits/chosen": -0.40329790115356445, + "logits/rejected": -0.40474754571914673, + "logps/chosen": -395.0797119140625, + "logps/ref_chosen": -61.27716827392578, + "logps/ref_rejected": -103.11612701416016, + "logps/rejected": -724.468017578125, + "loss": 1.0114, + "margin_dpo/margin_mean": 287.54931640625, + "margin_dpo/margin_std": 285.5010681152344, + "step": 397 + }, + { + "KL/chosen_KL_mean": -361.37225341796875, + "KL/mean": -491.65283203125, + "KL/rejected_KL_mean": -621.9334716796875, + "KL/std": 285.1490478515625, + "epoch": 0.5844346549192364, + "fcm_dpo/beta": 0.0016844018828123808, + "fcm_dpo/delta": -0.040688835084438324, + "fcm_dpo/margin": 260.5611877441406, + "fcm_dpo/q_t": 0.40008544921875, + "grad_norm": 21.616304397583008, + "learning_rate": 2.2182670837116972e-07, + "logits/chosen": -0.48934173583984375, + "logits/rejected": -0.4850488007068634, + "logps/chosen": -429.5238037109375, + "logps/ref_chosen": -68.15155029296875, + "logps/ref_rejected": -108.52360534667969, + "logps/rejected": -730.45703125, + "loss": 1.0645, + "margin_dpo/margin_mean": 260.5611877441406, + "margin_dpo/margin_std": 329.8893127441406, + "step": 398 + }, + { + "KL/chosen_KL_mean": -320.106201171875, + "KL/mean": -435.54656982421875, + "KL/rejected_KL_mean": -550.9869384765625, + "KL/std": 252.38217163085938, + "epoch": 0.5859030837004405, + "fcm_dpo/beta": 0.0016788092907518148, + "fcm_dpo/delta": 0.012697199359536171, + "fcm_dpo/margin": 230.88070678710938, + "fcm_dpo/q_t": 0.4103388786315918, + "grad_norm": 26.99171257019043, + "learning_rate": 2.2055192986047804e-07, + "logits/chosen": -0.47362208366394043, + "logits/rejected": -0.43797028064727783, + "logps/chosen": -380.9960021972656, + "logps/ref_chosen": -60.889801025390625, + "logps/ref_rejected": -77.965576171875, + "logps/rejected": -628.9525146484375, + "loss": 1.1009, + "margin_dpo/margin_mean": 230.8806915283203, + "margin_dpo/margin_std": 310.9993591308594, + "step": 399 + }, + { + "KL/chosen_KL_mean": -306.8255615234375, + "KL/mean": -475.8355712890625, + "KL/rejected_KL_mean": -644.8455810546875, + "KL/std": 272.01824951171875, + "epoch": 0.5873715124816447, + "fcm_dpo/beta": 0.0016412277473136783, + "fcm_dpo/delta": -0.16435033082962036, + "fcm_dpo/margin": 338.02001953125, + "fcm_dpo/q_t": 0.36962372064590454, + "grad_norm": 22.292245864868164, + "learning_rate": 2.192779273338215e-07, + "logits/chosen": -0.45513978600502014, + "logits/rejected": -0.4522014260292053, + "logps/chosen": -370.46917724609375, + "logps/ref_chosen": -63.64359664916992, + "logps/ref_rejected": -105.252685546875, + "logps/rejected": -750.0982666015625, + "loss": 0.9618, + "margin_dpo/margin_mean": 338.02001953125, + "margin_dpo/margin_std": 297.86004638671875, + "step": 400 + }, + { + "KL/chosen_KL_mean": -357.10784912109375, + "KL/mean": -451.53125, + "KL/rejected_KL_mean": -545.95458984375, + "KL/std": 285.63330078125, + "epoch": 0.5888399412628488, + "fcm_dpo/beta": 0.0016490190755575895, + "fcm_dpo/delta": 0.09153245389461517, + "fcm_dpo/margin": 188.84674072265625, + "fcm_dpo/q_t": 0.4310322701931, + "grad_norm": 23.559465408325195, + "learning_rate": 2.1800473436235136e-07, + "logits/chosen": -0.45114535093307495, + "logits/rejected": -0.44320282340049744, + "logps/chosen": -414.2708740234375, + "logps/ref_chosen": -57.16303253173828, + "logps/ref_rejected": -83.79249572753906, + "logps/rejected": -629.7471313476562, + "loss": 1.1949, + "margin_dpo/margin_mean": 188.84677124023438, + "margin_dpo/margin_std": 386.6992492675781, + "step": 401 + }, + { + "KL/chosen_KL_mean": -268.1985778808594, + "KL/mean": -443.2376708984375, + "KL/rejected_KL_mean": -618.2767333984375, + "KL/std": 305.66064453125, + "epoch": 0.5903083700440529, + "fcm_dpo/beta": 0.0016179950907826424, + "fcm_dpo/delta": -0.1764608919620514, + "fcm_dpo/margin": 350.0781555175781, + "fcm_dpo/q_t": 0.36862170696258545, + "grad_norm": 20.347381591796875, + "learning_rate": 2.1673238449588665e-07, + "logits/chosen": -0.4648516774177551, + "logits/rejected": -0.4514453411102295, + "logps/chosen": -318.93896484375, + "logps/ref_chosen": -50.74037170410156, + "logps/ref_rejected": -81.0460433959961, + "logps/rejected": -699.32275390625, + "loss": 0.9548, + "margin_dpo/margin_mean": 350.078125, + "margin_dpo/margin_std": 312.98248291015625, + "step": 402 + }, + { + "KL/chosen_KL_mean": -298.02740478515625, + "KL/mean": -428.8231201171875, + "KL/rejected_KL_mean": -559.6187744140625, + "KL/std": 291.15869140625, + "epoch": 0.591776798825257, + "fcm_dpo/beta": 0.0015987367369234562, + "fcm_dpo/delta": -0.019017325714230537, + "fcm_dpo/margin": 261.5914306640625, + "fcm_dpo/q_t": 0.40307024121284485, + "grad_norm": 25.700403213500977, + "learning_rate": 2.154609112620295e-07, + "logits/chosen": -0.44661349058151245, + "logits/rejected": -0.4446351230144501, + "logps/chosen": -345.1747131347656, + "logps/ref_chosen": -47.14731216430664, + "logps/ref_rejected": -77.2666015625, + "logps/rejected": -636.8854370117188, + "loss": 1.0672, + "margin_dpo/margin_mean": 261.5914306640625, + "margin_dpo/margin_std": 303.60894775390625, + "step": 403 + }, + { + "KL/chosen_KL_mean": -324.199951171875, + "KL/mean": -455.5159606933594, + "KL/rejected_KL_mean": -586.8319091796875, + "KL/std": 273.1387634277344, + "epoch": 0.593245227606461, + "fcm_dpo/beta": 0.0015905757900327444, + "fcm_dpo/delta": -0.018594812601804733, + "fcm_dpo/margin": 262.6319885253906, + "fcm_dpo/q_t": 0.4038143754005432, + "grad_norm": 27.593461990356445, + "learning_rate": 2.1419034816528218e-07, + "logits/chosen": -0.46002256870269775, + "logits/rejected": -0.45228320360183716, + "logps/chosen": -372.0752258300781, + "logps/ref_chosen": -47.875274658203125, + "logps/ref_rejected": -77.15499877929688, + "logps/rejected": -663.9869384765625, + "loss": 1.0834, + "margin_dpo/margin_mean": 262.6319885253906, + "margin_dpo/margin_std": 347.2093505859375, + "step": 404 + }, + { + "KL/chosen_KL_mean": -376.49700927734375, + "KL/mean": -483.60076904296875, + "KL/rejected_KL_mean": -590.7044677734375, + "KL/std": 307.7935791015625, + "epoch": 0.5947136563876652, + "fcm_dpo/beta": 0.001616810099221766, + "fcm_dpo/delta": 0.054005078971385956, + "fcm_dpo/margin": 214.20745849609375, + "fcm_dpo/q_t": 0.4231037497520447, + "grad_norm": 28.185096740722656, + "learning_rate": 2.129207286861638e-07, + "logits/chosen": -0.4508972465991974, + "logits/rejected": -0.44256657361984253, + "logps/chosen": -441.659912109375, + "logps/ref_chosen": -65.16290283203125, + "logps/ref_rejected": -87.18678283691406, + "logps/rejected": -677.8912353515625, + "loss": 1.1654, + "margin_dpo/margin_mean": 214.20745849609375, + "margin_dpo/margin_std": 385.24530029296875, + "step": 405 + }, + { + "KL/chosen_KL_mean": -331.07080078125, + "KL/mean": -467.66162109375, + "KL/rejected_KL_mean": -604.2523803710938, + "KL/std": 297.1915588378906, + "epoch": 0.5961820851688693, + "fcm_dpo/beta": 0.0016056987224146724, + "fcm_dpo/delta": -0.04058470577001572, + "fcm_dpo/margin": 273.1816101074219, + "fcm_dpo/q_t": 0.39960601925849915, + "grad_norm": 24.00153160095215, + "learning_rate": 2.1165208628032861e-07, + "logits/chosen": -0.4905382990837097, + "logits/rejected": -0.49678927659988403, + "logps/chosen": -380.8116149902344, + "logps/ref_chosen": -49.740814208984375, + "logps/ref_rejected": -92.07862854003906, + "logps/rejected": -696.3310546875, + "loss": 1.0601, + "margin_dpo/margin_mean": 273.1816101074219, + "margin_dpo/margin_std": 328.4947814941406, + "step": 406 + }, + { + "KL/chosen_KL_mean": -363.4420166015625, + "KL/mean": -452.39483642578125, + "KL/rejected_KL_mean": -541.34765625, + "KL/std": 244.7425537109375, + "epoch": 0.5976505139500734, + "fcm_dpo/beta": 0.0016330704092979431, + "fcm_dpo/delta": 0.1112029105424881, + "fcm_dpo/margin": 177.90560913085938, + "fcm_dpo/q_t": 0.4322769343852997, + "grad_norm": 26.514976501464844, + "learning_rate": 2.1038445437768375e-07, + "logits/chosen": -0.470073401927948, + "logits/rejected": -0.4409082531929016, + "logps/chosen": -419.7727355957031, + "logps/ref_chosen": -56.33069610595703, + "logps/ref_rejected": -77.51209259033203, + "logps/rejected": -618.8597412109375, + "loss": 1.1879, + "margin_dpo/margin_mean": 177.9055938720703, + "margin_dpo/margin_std": 325.5947265625, + "step": 407 + }, + { + "KL/chosen_KL_mean": -365.21722412109375, + "KL/mean": -465.7977294921875, + "KL/rejected_KL_mean": -566.3782958984375, + "KL/std": 237.32412719726562, + "epoch": 0.5991189427312775, + "fcm_dpo/beta": 0.0016487594693899155, + "fcm_dpo/delta": 0.07069391012191772, + "fcm_dpo/margin": 201.16098022460938, + "fcm_dpo/q_t": 0.42220157384872437, + "grad_norm": 21.884017944335938, + "learning_rate": 2.0911786638150872e-07, + "logits/chosen": -0.4787837266921997, + "logits/rejected": -0.4536997079849243, + "logps/chosen": -435.0065612792969, + "logps/ref_chosen": -69.789306640625, + "logps/ref_rejected": -90.09693908691406, + "logps/rejected": -656.4752197265625, + "loss": 1.1279, + "margin_dpo/margin_mean": 201.16098022460938, + "margin_dpo/margin_std": 266.10650634765625, + "step": 408 + }, + { + "KL/chosen_KL_mean": -356.888671875, + "KL/mean": -453.8036804199219, + "KL/rejected_KL_mean": -550.71875, + "KL/std": 255.46417236328125, + "epoch": 0.6005873715124816, + "fcm_dpo/beta": 0.0016783413011580706, + "fcm_dpo/delta": 0.07688158750534058, + "fcm_dpo/margin": 193.83004760742188, + "fcm_dpo/q_t": 0.42450258135795593, + "grad_norm": 29.449851989746094, + "learning_rate": 2.0785235566757517e-07, + "logits/chosen": -0.45866113901138306, + "logits/rejected": -0.44195905327796936, + "logps/chosen": -424.20611572265625, + "logps/ref_chosen": -67.31744384765625, + "logps/ref_rejected": -84.904296875, + "logps/rejected": -635.623046875, + "loss": 1.1479, + "margin_dpo/margin_mean": 193.83004760742188, + "margin_dpo/margin_std": 298.97589111328125, + "step": 409 + }, + { + "KL/chosen_KL_mean": -321.3678283691406, + "KL/mean": -441.85791015625, + "KL/rejected_KL_mean": -562.3480224609375, + "KL/std": 258.2183837890625, + "epoch": 0.6020558002936858, + "fcm_dpo/beta": 0.001676855026744306, + "fcm_dpo/delta": -0.004324676468968391, + "fcm_dpo/margin": 240.9801483154297, + "fcm_dpo/q_t": 0.4049345850944519, + "grad_norm": 23.625701904296875, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": -0.46683716773986816, + "logits/rejected": -0.4663783311843872, + "logps/chosen": -372.83319091796875, + "logps/ref_chosen": -51.465354919433594, + "logps/ref_rejected": -83.198974609375, + "logps/rejected": -645.5469970703125, + "loss": 1.0765, + "margin_dpo/margin_mean": 240.98016357421875, + "margin_dpo/margin_std": 284.5562744140625, + "step": 410 + }, + { + "KL/chosen_KL_mean": -347.264404296875, + "KL/mean": -463.2464599609375, + "KL/rejected_KL_mean": -579.228515625, + "KL/std": 278.7031555175781, + "epoch": 0.6035242290748899, + "fcm_dpo/beta": 0.001669498858973384, + "fcm_dpo/delta": 0.012138403952121735, + "fcm_dpo/margin": 231.9641876220703, + "fcm_dpo/q_t": 0.41212886571884155, + "grad_norm": 23.38148307800293, + "learning_rate": 2.0532469944670343e-07, + "logits/chosen": -0.4481458365917206, + "logits/rejected": -0.45459499955177307, + "logps/chosen": -399.5716552734375, + "logps/ref_chosen": -52.30727005004883, + "logps/ref_rejected": -80.69495391845703, + "logps/rejected": -659.9235229492188, + "loss": 1.1047, + "margin_dpo/margin_mean": 231.9641876220703, + "margin_dpo/margin_std": 314.99127197265625, + "step": 411 + }, + { + "KL/chosen_KL_mean": -357.5395202636719, + "KL/mean": -476.832763671875, + "KL/rejected_KL_mean": -596.1259765625, + "KL/std": 277.5762939453125, + "epoch": 0.604992657856094, + "fcm_dpo/beta": 0.0016845406498759985, + "fcm_dpo/delta": -0.00200699083507061, + "fcm_dpo/margin": 238.58645629882812, + "fcm_dpo/q_t": 0.40782129764556885, + "grad_norm": 41.79743194580078, + "learning_rate": 2.0406262054585738e-07, + "logits/chosen": -0.49470025300979614, + "logits/rejected": -0.5206660628318787, + "logps/chosen": -410.68365478515625, + "logps/ref_chosen": -53.144126892089844, + "logps/ref_rejected": -100.0608139038086, + "logps/rejected": -696.186767578125, + "loss": 1.0941, + "margin_dpo/margin_mean": 238.58645629882812, + "margin_dpo/margin_std": 326.08343505859375, + "step": 412 + }, + { + "KL/chosen_KL_mean": -381.70831298828125, + "KL/mean": -499.70184326171875, + "KL/rejected_KL_mean": -617.6954345703125, + "KL/std": 265.35125732421875, + "epoch": 0.6064610866372981, + "fcm_dpo/beta": 0.001688747899606824, + "fcm_dpo/delta": 0.0012981817126274109, + "fcm_dpo/margin": 235.98709106445312, + "fcm_dpo/q_t": 0.406361848115921, + "grad_norm": 25.019241333007812, + "learning_rate": 2.0280175213768205e-07, + "logits/chosen": -0.47015732526779175, + "logits/rejected": -0.47621041536331177, + "logps/chosen": -443.290283203125, + "logps/ref_chosen": -61.58196258544922, + "logps/ref_rejected": -99.47340393066406, + "logps/rejected": -717.1688232421875, + "loss": 1.0904, + "margin_dpo/margin_mean": 235.9871063232422, + "margin_dpo/margin_std": 305.47210693359375, + "step": 413 + }, + { + "KL/chosen_KL_mean": -334.1068115234375, + "KL/mean": -468.125732421875, + "KL/rejected_KL_mean": -602.1447143554688, + "KL/std": 256.7291259765625, + "epoch": 0.6079295154185022, + "fcm_dpo/beta": 0.001681898720562458, + "fcm_dpo/delta": -0.053896043449640274, + "fcm_dpo/margin": 268.03790283203125, + "fcm_dpo/q_t": 0.3947378396987915, + "grad_norm": 29.387535095214844, + "learning_rate": 2.0154212744723247e-07, + "logits/chosen": -0.4133932590484619, + "logits/rejected": -0.40770968794822693, + "logps/chosen": -380.73828125, + "logps/ref_chosen": -46.63148498535156, + "logps/ref_rejected": -87.64653015136719, + "logps/rejected": -689.791259765625, + "loss": 1.0452, + "margin_dpo/margin_mean": 268.03790283203125, + "margin_dpo/margin_std": 288.237548828125, + "step": 414 + }, + { + "KL/chosen_KL_mean": -390.490966796875, + "KL/mean": -491.0526123046875, + "KL/rejected_KL_mean": -591.6142578125, + "KL/std": 267.60589599609375, + "epoch": 0.6093979441997063, + "fcm_dpo/beta": 0.0016747894696891308, + "fcm_dpo/delta": 0.06530318409204483, + "fcm_dpo/margin": 201.123291015625, + "fcm_dpo/q_t": 0.4213160574436188, + "grad_norm": 26.970125198364258, + "learning_rate": 2.002837796667909e-07, + "logits/chosen": -0.448618084192276, + "logits/rejected": -0.4439224600791931, + "logps/chosen": -469.1092529296875, + "logps/ref_chosen": -78.6182861328125, + "logps/ref_rejected": -100.47752380371094, + "logps/rejected": -692.091796875, + "loss": 1.1369, + "margin_dpo/margin_mean": 201.123291015625, + "margin_dpo/margin_std": 294.55072021484375, + "step": 415 + }, + { + "KL/chosen_KL_mean": -348.8500061035156, + "KL/mean": -504.3551330566406, + "KL/rejected_KL_mean": -659.8602294921875, + "KL/std": 290.89202880859375, + "epoch": 0.6108663729809104, + "fcm_dpo/beta": 0.001661974936723709, + "fcm_dpo/delta": -0.1230802834033966, + "fcm_dpo/margin": 311.01019287109375, + "fcm_dpo/q_t": 0.37868714332580566, + "grad_norm": 31.66852569580078, + "learning_rate": 1.990267419549914e-07, + "logits/chosen": -0.4952217638492584, + "logits/rejected": -0.5007729530334473, + "logps/chosen": -407.129150390625, + "logps/ref_chosen": -58.27912521362305, + "logps/ref_rejected": -90.56871795654297, + "logps/rejected": -750.428955078125, + "loss": 0.9841, + "margin_dpo/margin_mean": 311.01019287109375, + "margin_dpo/margin_std": 273.85443115234375, + "step": 416 + }, + { + "KL/chosen_KL_mean": -348.1707763671875, + "KL/mean": -476.07635498046875, + "KL/rejected_KL_mean": -603.98193359375, + "KL/std": 262.30242919921875, + "epoch": 0.6123348017621145, + "fcm_dpo/beta": 0.001643989235162735, + "fcm_dpo/delta": -0.02149224281311035, + "fcm_dpo/margin": 255.8111572265625, + "fcm_dpo/q_t": 0.40100353956222534, + "grad_norm": 24.26694107055664, + "learning_rate": 1.9777104743594686e-07, + "logits/chosen": -0.43751639127731323, + "logits/rejected": -0.4203481674194336, + "logps/chosen": -398.3695068359375, + "logps/ref_chosen": -50.1987190246582, + "logps/ref_rejected": -68.15184020996094, + "logps/rejected": -672.1337280273438, + "loss": 1.0534, + "margin_dpo/margin_mean": 255.8111572265625, + "margin_dpo/margin_std": 262.398193359375, + "step": 417 + }, + { + "KL/chosen_KL_mean": -375.5062255859375, + "KL/mean": -509.50982666015625, + "KL/rejected_KL_mean": -643.513427734375, + "KL/std": 293.5723571777344, + "epoch": 0.6138032305433186, + "fcm_dpo/beta": 0.0016407601069658995, + "fcm_dpo/delta": -0.042371716350317, + "fcm_dpo/margin": 268.0071716308594, + "fcm_dpo/q_t": 0.4010557532310486, + "grad_norm": 27.076887130737305, + "learning_rate": 1.965167291983757e-07, + "logits/chosen": -0.5030707120895386, + "logits/rejected": -0.48085683584213257, + "logps/chosen": -457.4847106933594, + "logps/ref_chosen": -81.97846984863281, + "logps/ref_rejected": -104.69148254394531, + "logps/rejected": -748.2048950195312, + "loss": 1.0719, + "margin_dpo/margin_mean": 268.0072021484375, + "margin_dpo/margin_std": 345.320068359375, + "step": 418 + }, + { + "KL/chosen_KL_mean": -347.1816101074219, + "KL/mean": -493.19476318359375, + "KL/rejected_KL_mean": -639.2078857421875, + "KL/std": 260.90594482421875, + "epoch": 0.6152716593245228, + "fcm_dpo/beta": 0.0016074271406978369, + "fcm_dpo/delta": -0.07283525168895721, + "fcm_dpo/margin": 292.02630615234375, + "fcm_dpo/q_t": 0.3896368443965912, + "grad_norm": 30.91160011291504, + "learning_rate": 1.9526382029472988e-07, + "logits/chosen": -0.49828916788101196, + "logits/rejected": -0.49863529205322266, + "logps/chosen": -400.1302490234375, + "logps/ref_chosen": -52.948646545410156, + "logps/ref_rejected": -91.58309936523438, + "logps/rejected": -730.791015625, + "loss": 1.0255, + "margin_dpo/margin_mean": 292.02630615234375, + "margin_dpo/margin_std": 299.172119140625, + "step": 419 + }, + { + "KL/chosen_KL_mean": -439.3621520996094, + "KL/mean": -528.0501098632812, + "KL/rejected_KL_mean": -616.7380981445312, + "KL/std": 280.856689453125, + "epoch": 0.6167400881057269, + "fcm_dpo/beta": 0.001630987972021103, + "fcm_dpo/delta": 0.11383231729269028, + "fcm_dpo/margin": 177.37596130371094, + "fcm_dpo/q_t": 0.43548548221588135, + "grad_norm": 60.948829650878906, + "learning_rate": 1.9401235374032425e-07, + "logits/chosen": -0.5088773369789124, + "logits/rejected": -0.47673243284225464, + "logps/chosen": -517.132080078125, + "logps/ref_chosen": -77.7699203491211, + "logps/ref_rejected": -69.31985473632812, + "logps/rejected": -686.0579833984375, + "loss": 1.2097, + "margin_dpo/margin_mean": 177.3759765625, + "margin_dpo/margin_std": 384.9363098144531, + "step": 420 + }, + { + "KL/chosen_KL_mean": -349.59552001953125, + "KL/mean": -450.66204833984375, + "KL/rejected_KL_mean": -551.728515625, + "KL/std": 265.665283203125, + "epoch": 0.618208516886931, + "fcm_dpo/beta": 0.0016611446626484394, + "fcm_dpo/delta": 0.06588370352983475, + "fcm_dpo/margin": 202.13302612304688, + "fcm_dpo/q_t": 0.4204084873199463, + "grad_norm": 26.5180606842041, + "learning_rate": 1.9276236251246653e-07, + "logits/chosen": -0.519219160079956, + "logits/rejected": -0.5066835880279541, + "logps/chosen": -403.36138916015625, + "logps/ref_chosen": -53.765865325927734, + "logps/ref_rejected": -89.28144836425781, + "logps/rejected": -641.010009765625, + "loss": 1.1353, + "margin_dpo/margin_mean": 202.13302612304688, + "margin_dpo/margin_std": 287.11669921875, + "step": 421 + }, + { + "KL/chosen_KL_mean": -403.3852233886719, + "KL/mean": -521.2222900390625, + "KL/rejected_KL_mean": -639.059326171875, + "KL/std": 274.61279296875, + "epoch": 0.6196769456681351, + "fcm_dpo/beta": 0.001660653855651617, + "fcm_dpo/delta": 0.008965391665697098, + "fcm_dpo/margin": 235.6741485595703, + "fcm_dpo/q_t": 0.4084932208061218, + "grad_norm": 31.94446563720703, + "learning_rate": 1.9151387954958792e-07, + "logits/chosen": -0.5387804508209229, + "logits/rejected": -0.5378388166427612, + "logps/chosen": -472.01898193359375, + "logps/ref_chosen": -68.6337661743164, + "logps/ref_rejected": -87.86351013183594, + "logps/rejected": -726.9228515625, + "loss": 1.1001, + "margin_dpo/margin_mean": 235.6741485595703, + "margin_dpo/margin_std": 322.131103515625, + "step": 422 + }, + { + "KL/chosen_KL_mean": -375.15228271484375, + "KL/mean": -501.4312744140625, + "KL/rejected_KL_mean": -627.710205078125, + "KL/std": 263.3236083984375, + "epoch": 0.6211453744493393, + "fcm_dpo/beta": 0.0016572801396250725, + "fcm_dpo/delta": -0.019391080364584923, + "fcm_dpo/margin": 252.5579833984375, + "fcm_dpo/q_t": 0.4022686183452606, + "grad_norm": 31.633256912231445, + "learning_rate": 1.902669377503756e-07, + "logits/chosen": -0.5140703916549683, + "logits/rejected": -0.519053041934967, + "logps/chosen": -430.142578125, + "logps/ref_chosen": -54.99030303955078, + "logps/ref_rejected": -86.30654907226562, + "logps/rejected": -714.0167846679688, + "loss": 1.0628, + "margin_dpo/margin_mean": 252.5579833984375, + "margin_dpo/margin_std": 284.78033447265625, + "step": 423 + }, + { + "KL/chosen_KL_mean": -334.7992858886719, + "KL/mean": -459.7820739746094, + "KL/rejected_KL_mean": -584.7648315429688, + "KL/std": 269.6551513671875, + "epoch": 0.6226138032305433, + "fcm_dpo/beta": 0.0016480376943945885, + "fcm_dpo/delta": -0.012738510966300964, + "fcm_dpo/margin": 249.96554565429688, + "fcm_dpo/q_t": 0.4058811068534851, + "grad_norm": 32.633731842041016, + "learning_rate": 1.890215699729057e-07, + "logits/chosen": -0.5273067951202393, + "logits/rejected": -0.5043067932128906, + "logps/chosen": -390.81121826171875, + "logps/ref_chosen": -56.01192092895508, + "logps/ref_rejected": -66.47896575927734, + "logps/rejected": -651.2437744140625, + "loss": 1.0844, + "margin_dpo/margin_mean": 249.96554565429688, + "margin_dpo/margin_std": 328.8323974609375, + "step": 424 + }, + { + "KL/chosen_KL_mean": -362.98529052734375, + "KL/mean": -461.9312744140625, + "KL/rejected_KL_mean": -560.877197265625, + "KL/std": 245.79722595214844, + "epoch": 0.6240822320117474, + "fcm_dpo/beta": 0.0016736630350351334, + "fcm_dpo/delta": 0.07090499997138977, + "fcm_dpo/margin": 197.89198303222656, + "fcm_dpo/q_t": 0.4216083884239197, + "grad_norm": 25.201358795166016, + "learning_rate": 1.8777780903377732e-07, + "logits/chosen": -0.50770503282547, + "logits/rejected": -0.5075915455818176, + "logps/chosen": -409.8542785644531, + "logps/ref_chosen": -46.86899948120117, + "logps/ref_rejected": -95.92545318603516, + "logps/rejected": -656.802734375, + "loss": 1.1421, + "margin_dpo/margin_mean": 197.89198303222656, + "margin_dpo/margin_std": 296.4322509765625, + "step": 425 + }, + { + "KL/chosen_KL_mean": -336.8538818359375, + "KL/mean": -458.18670654296875, + "KL/rejected_KL_mean": -579.51953125, + "KL/std": 251.29937744140625, + "epoch": 0.6255506607929515, + "fcm_dpo/beta": 0.0016819476149976254, + "fcm_dpo/delta": -0.008926056325435638, + "fcm_dpo/margin": 242.66563415527344, + "fcm_dpo/q_t": 0.40451914072036743, + "grad_norm": 25.522932052612305, + "learning_rate": 1.8653568770724803e-07, + "logits/chosen": -0.5204522609710693, + "logits/rejected": -0.4901271462440491, + "logps/chosen": -413.43743896484375, + "logps/ref_chosen": -76.58354187011719, + "logps/ref_rejected": -81.26658630371094, + "logps/rejected": -660.7861328125, + "loss": 1.0736, + "margin_dpo/margin_mean": 242.66561889648438, + "margin_dpo/margin_std": 277.174560546875, + "step": 426 + }, + { + "KL/chosen_KL_mean": -315.9246520996094, + "KL/mean": -404.6652526855469, + "KL/rejected_KL_mean": -493.4059143066406, + "KL/std": 217.07647705078125, + "epoch": 0.6270190895741556, + "fcm_dpo/beta": 0.0016970799770206213, + "fcm_dpo/delta": 0.10195031762123108, + "fcm_dpo/margin": 177.48121643066406, + "fcm_dpo/q_t": 0.42938411235809326, + "grad_norm": 23.957731246948242, + "learning_rate": 1.8529523872436977e-07, + "logits/chosen": -0.5153839588165283, + "logits/rejected": -0.49676990509033203, + "logps/chosen": -380.7785339355469, + "logps/ref_chosen": -64.8538818359375, + "logps/ref_rejected": -78.5660171508789, + "logps/rejected": -571.971923828125, + "loss": 1.1509, + "margin_dpo/margin_mean": 177.48121643066406, + "margin_dpo/margin_std": 249.63307189941406, + "step": 427 + }, + { + "KL/chosen_KL_mean": -394.0665283203125, + "KL/mean": -510.107421875, + "KL/rejected_KL_mean": -626.1483154296875, + "KL/std": 281.40484619140625, + "epoch": 0.6284875183553598, + "fcm_dpo/beta": 0.0017051963368430734, + "fcm_dpo/delta": 0.0043541621416807175, + "fcm_dpo/margin": 232.08180236816406, + "fcm_dpo/q_t": 0.40895557403564453, + "grad_norm": 30.00721549987793, + "learning_rate": 1.8405649477212697e-07, + "logits/chosen": -0.5060927271842957, + "logits/rejected": -0.5087655782699585, + "logps/chosen": -456.70318603515625, + "logps/ref_chosen": -62.63666534423828, + "logps/ref_rejected": -103.28181457519531, + "logps/rejected": -729.43017578125, + "loss": 1.1043, + "margin_dpo/margin_mean": 232.081787109375, + "margin_dpo/margin_std": 331.1031494140625, + "step": 428 + }, + { + "KL/chosen_KL_mean": -389.79180908203125, + "KL/mean": -480.70843505859375, + "KL/rejected_KL_mean": -571.6251220703125, + "KL/std": 241.70196533203125, + "epoch": 0.6299559471365639, + "fcm_dpo/beta": 0.0017423585522919893, + "fcm_dpo/delta": 0.08442431688308716, + "fcm_dpo/margin": 181.8333282470703, + "fcm_dpo/q_t": 0.42665740847587585, + "grad_norm": 35.86955642700195, + "learning_rate": 1.828194884925749e-07, + "logits/chosen": -0.5312508940696716, + "logits/rejected": -0.5112993717193604, + "logps/chosen": -471.02581787109375, + "logps/ref_chosen": -81.23401641845703, + "logps/ref_rejected": -91.79493713378906, + "logps/rejected": -663.4200439453125, + "loss": 1.163, + "margin_dpo/margin_mean": 181.83334350585938, + "margin_dpo/margin_std": 292.9608154296875, + "step": 429 + }, + { + "KL/chosen_KL_mean": -317.3845520019531, + "KL/mean": -419.97271728515625, + "KL/rejected_KL_mean": -522.560791015625, + "KL/std": 234.30323791503906, + "epoch": 0.631424375917768, + "fcm_dpo/beta": 0.001752232201397419, + "fcm_dpo/delta": 0.04184893146157265, + "fcm_dpo/margin": 205.17626953125, + "fcm_dpo/q_t": 0.4163983464241028, + "grad_norm": 30.209016799926758, + "learning_rate": 1.8158425248197928e-07, + "logits/chosen": -0.5189450979232788, + "logits/rejected": -0.5163074731826782, + "logps/chosen": -378.30487060546875, + "logps/ref_chosen": -60.920326232910156, + "logps/ref_rejected": -104.42280578613281, + "logps/rejected": -626.983642578125, + "loss": 1.1061, + "margin_dpo/margin_mean": 205.17623901367188, + "margin_dpo/margin_std": 255.99505615234375, + "step": 430 + }, + { + "KL/chosen_KL_mean": -305.75433349609375, + "KL/mean": -447.6547546386719, + "KL/rejected_KL_mean": -589.55517578125, + "KL/std": 262.2629089355469, + "epoch": 0.6328928046989721, + "fcm_dpo/beta": 0.0017262442270293832, + "fcm_dpo/delta": -0.094680055975914, + "fcm_dpo/margin": 283.80078125, + "fcm_dpo/q_t": 0.3853592872619629, + "grad_norm": 27.11848258972168, + "learning_rate": 1.8035081928995788e-07, + "logits/chosen": -0.4816255271434784, + "logits/rejected": -0.48415422439575195, + "logps/chosen": -363.10308837890625, + "logps/ref_chosen": -57.34874725341797, + "logps/ref_rejected": -92.84022521972656, + "logps/rejected": -682.3953857421875, + "loss": 1.0109, + "margin_dpo/margin_mean": 283.80078125, + "margin_dpo/margin_std": 278.22015380859375, + "step": 431 + }, + { + "KL/chosen_KL_mean": -311.2400817871094, + "KL/mean": -443.8277893066406, + "KL/rejected_KL_mean": -576.41552734375, + "KL/std": 255.052490234375, + "epoch": 0.6343612334801763, + "fcm_dpo/beta": 0.0017163490410894156, + "fcm_dpo/delta": -0.05826277658343315, + "fcm_dpo/margin": 265.17547607421875, + "fcm_dpo/q_t": 0.39427268505096436, + "grad_norm": 25.935993194580078, + "learning_rate": 1.791192214186223e-07, + "logits/chosen": -0.48976317048072815, + "logits/rejected": -0.4787473678588867, + "logps/chosen": -382.31488037109375, + "logps/ref_chosen": -71.07479095458984, + "logps/ref_rejected": -98.57952880859375, + "logps/rejected": -674.9950561523438, + "loss": 1.0351, + "margin_dpo/margin_mean": 265.1754455566406, + "margin_dpo/margin_std": 264.5519104003906, + "step": 432 + }, + { + "KL/chosen_KL_mean": -400.88958740234375, + "KL/mean": -498.4219970703125, + "KL/rejected_KL_mean": -595.9544067382812, + "KL/std": 252.05137634277344, + "epoch": 0.6358296622613803, + "fcm_dpo/beta": 0.0017208922654390335, + "fcm_dpo/delta": 0.06628023833036423, + "fcm_dpo/margin": 195.0648193359375, + "fcm_dpo/q_t": 0.42144525051116943, + "grad_norm": 28.61791229248047, + "learning_rate": 1.7788949132172193e-07, + "logits/chosen": -0.5252653360366821, + "logits/rejected": -0.514911413192749, + "logps/chosen": -459.16278076171875, + "logps/ref_chosen": -58.273193359375, + "logps/ref_rejected": -95.95089721679688, + "logps/rejected": -691.9052734375, + "loss": 1.1435, + "margin_dpo/margin_mean": 195.0648193359375, + "margin_dpo/margin_std": 301.6573486328125, + "step": 433 + }, + { + "KL/chosen_KL_mean": -365.26397705078125, + "KL/mean": -477.8414306640625, + "KL/rejected_KL_mean": -590.4189453125, + "KL/std": 274.9288024902344, + "epoch": 0.6372980910425844, + "fcm_dpo/beta": 0.0017205406911671162, + "fcm_dpo/delta": 0.012956922873854637, + "fcm_dpo/margin": 225.1549530029297, + "fcm_dpo/q_t": 0.4162324070930481, + "grad_norm": 27.521728515625, + "learning_rate": 1.7666166140378853e-07, + "logits/chosen": -0.5110592842102051, + "logits/rejected": -0.5141887664794922, + "logps/chosen": -427.2376708984375, + "logps/ref_chosen": -61.97370147705078, + "logps/ref_rejected": -78.49861145019531, + "logps/rejected": -668.91748046875, + "loss": 1.1143, + "margin_dpo/margin_mean": 225.1549530029297, + "margin_dpo/margin_std": 354.01708984375, + "step": 434 + }, + { + "KL/chosen_KL_mean": -334.04791259765625, + "KL/mean": -459.4658203125, + "KL/rejected_KL_mean": -584.8836669921875, + "KL/std": 268.1673583984375, + "epoch": 0.6387665198237885, + "fcm_dpo/beta": 0.0017172786174342036, + "fcm_dpo/delta": -0.03218632563948631, + "fcm_dpo/margin": 250.83578491210938, + "fcm_dpo/q_t": 0.40059614181518555, + "grad_norm": 23.964658737182617, + "learning_rate": 1.7543576401928218e-07, + "logits/chosen": -0.5041155219078064, + "logits/rejected": -0.4924761950969696, + "logps/chosen": -385.54998779296875, + "logps/ref_chosen": -51.502052307128906, + "logps/ref_rejected": -87.56689453125, + "logps/rejected": -672.4505615234375, + "loss": 1.0677, + "margin_dpo/margin_mean": 250.83578491210938, + "margin_dpo/margin_std": 302.60870361328125, + "step": 435 + }, + { + "KL/chosen_KL_mean": -356.885986328125, + "KL/mean": -461.7406005859375, + "KL/rejected_KL_mean": -566.5952758789062, + "KL/std": 239.44638061523438, + "epoch": 0.6402349486049926, + "fcm_dpo/beta": 0.0017226835479959846, + "fcm_dpo/delta": 0.040182169526815414, + "fcm_dpo/margin": 209.70928955078125, + "fcm_dpo/q_t": 0.4159030616283417, + "grad_norm": 49.06702423095703, + "learning_rate": 1.742118314717391e-07, + "logits/chosen": -0.5320940613746643, + "logits/rejected": -0.5051020979881287, + "logps/chosen": -428.2896728515625, + "logps/ref_chosen": -71.40371704101562, + "logps/ref_rejected": -82.72775268554688, + "logps/rejected": -649.322998046875, + "loss": 1.1129, + "margin_dpo/margin_mean": 209.70928955078125, + "margin_dpo/margin_std": 280.531982421875, + "step": 436 + }, + { + "KL/chosen_KL_mean": -356.98883056640625, + "KL/mean": -465.893310546875, + "KL/rejected_KL_mean": -574.7978515625, + "KL/std": 229.29736328125, + "epoch": 0.6417033773861968, + "fcm_dpo/beta": 0.0017339956248179078, + "fcm_dpo/delta": 0.023197388276457787, + "fcm_dpo/margin": 217.80899047851562, + "fcm_dpo/q_t": 0.4120755195617676, + "grad_norm": 27.109731674194336, + "learning_rate": 1.7298989601292036e-07, + "logits/chosen": -0.537100613117218, + "logits/rejected": -0.5201215147972107, + "logps/chosen": -421.73309326171875, + "logps/ref_chosen": -64.7442626953125, + "logps/ref_rejected": -82.04356384277344, + "logps/rejected": -656.8413696289062, + "loss": 1.1006, + "margin_dpo/margin_mean": 217.80899047851562, + "margin_dpo/margin_std": 282.6852111816406, + "step": 437 + }, + { + "KL/chosen_KL_mean": -356.2583923339844, + "KL/mean": -477.8333740234375, + "KL/rejected_KL_mean": -599.4083251953125, + "KL/std": 254.6542510986328, + "epoch": 0.6431718061674009, + "fcm_dpo/beta": 0.0017272335244342685, + "fcm_dpo/delta": -0.02110411413013935, + "fcm_dpo/margin": 243.14990234375, + "fcm_dpo/q_t": 0.401958167552948, + "grad_norm": 36.0264892578125, + "learning_rate": 1.7176998984196144e-07, + "logits/chosen": -0.47867268323898315, + "logits/rejected": -0.4559587240219116, + "logps/chosen": -415.2770690917969, + "logps/ref_chosen": -59.0186653137207, + "logps/ref_rejected": -83.07682800292969, + "logps/rejected": -682.485107421875, + "loss": 1.0646, + "margin_dpo/margin_mean": 243.14990234375, + "margin_dpo/margin_std": 278.13189697265625, + "step": 438 + }, + { + "KL/chosen_KL_mean": -355.92974853515625, + "KL/mean": -459.84600830078125, + "KL/rejected_KL_mean": -563.76220703125, + "KL/std": 248.40817260742188, + "epoch": 0.644640234948605, + "fcm_dpo/beta": 0.0017257456202059984, + "fcm_dpo/delta": 0.041571587324142456, + "fcm_dpo/margin": 207.83248901367188, + "fcm_dpo/q_t": 0.4176570177078247, + "grad_norm": 27.175893783569336, + "learning_rate": 1.7055214510452458e-07, + "logits/chosen": -0.49970486760139465, + "logits/rejected": -0.5002373456954956, + "logps/chosen": -409.7138366699219, + "logps/ref_chosen": -53.78407669067383, + "logps/ref_rejected": -83.98545837402344, + "logps/rejected": -647.7476806640625, + "loss": 1.1219, + "margin_dpo/margin_mean": 207.83248901367188, + "margin_dpo/margin_std": 278.52850341796875, + "step": 439 + }, + { + "KL/chosen_KL_mean": -379.3209533691406, + "KL/mean": -483.71697998046875, + "KL/rejected_KL_mean": -588.113037109375, + "KL/std": 282.49066162109375, + "epoch": 0.6461086637298091, + "fcm_dpo/beta": 0.0017588550690561533, + "fcm_dpo/delta": 0.0337531603872776, + "fcm_dpo/margin": 208.7920684814453, + "fcm_dpo/q_t": 0.4173220992088318, + "grad_norm": 26.570308685302734, + "learning_rate": 1.6933639389195134e-07, + "logits/chosen": -0.5298205614089966, + "logits/rejected": -0.5196566581726074, + "logps/chosen": -457.8876953125, + "logps/ref_chosen": -78.56671905517578, + "logps/ref_rejected": -96.49775695800781, + "logps/rejected": -684.6107788085938, + "loss": 1.1246, + "margin_dpo/margin_mean": 208.79208374023438, + "margin_dpo/margin_std": 317.7086181640625, + "step": 440 + }, + { + "KL/chosen_KL_mean": -393.6994323730469, + "KL/mean": -507.71136474609375, + "KL/rejected_KL_mean": -621.7232666015625, + "KL/std": 289.2242431640625, + "epoch": 0.6475770925110133, + "fcm_dpo/beta": 0.0017611184157431126, + "fcm_dpo/delta": -0.0017841961234807968, + "fcm_dpo/margin": 228.02386474609375, + "fcm_dpo/q_t": 0.40944066643714905, + "grad_norm": 30.262908935546875, + "learning_rate": 1.681227682404166e-07, + "logits/chosen": -0.4817603826522827, + "logits/rejected": -0.4669860601425171, + "logps/chosen": -454.52386474609375, + "logps/ref_chosen": -60.824440002441406, + "logps/ref_rejected": -96.47080993652344, + "logps/rejected": -718.194091796875, + "loss": 1.1168, + "margin_dpo/margin_mean": 228.02386474609375, + "margin_dpo/margin_std": 352.9437255859375, + "step": 441 + }, + { + "KL/chosen_KL_mean": -332.66845703125, + "KL/mean": -464.35418701171875, + "KL/rejected_KL_mean": -596.0399169921875, + "KL/std": 273.4464416503906, + "epoch": 0.6490455212922174, + "fcm_dpo/beta": 0.0017507702577859163, + "fcm_dpo/delta": -0.06448774039745331, + "fcm_dpo/margin": 263.3714599609375, + "fcm_dpo/q_t": 0.39469367265701294, + "grad_norm": 22.49814796447754, + "learning_rate": 1.669113001300851e-07, + "logits/chosen": -0.4911942481994629, + "logits/rejected": -0.4789756238460541, + "logps/chosen": -379.6796875, + "logps/ref_chosen": -47.01121520996094, + "logps/ref_rejected": -76.53926086425781, + "logps/rejected": -672.5791625976562, + "loss": 1.0467, + "margin_dpo/margin_mean": 263.3714599609375, + "margin_dpo/margin_std": 305.1097717285156, + "step": 442 + }, + { + "KL/chosen_KL_mean": -378.4925842285156, + "KL/mean": -465.4745788574219, + "KL/rejected_KL_mean": -552.45654296875, + "KL/std": 265.3682861328125, + "epoch": 0.6505139500734214, + "fcm_dpo/beta": 0.0017513078637421131, + "fcm_dpo/delta": 0.09834998100996017, + "fcm_dpo/margin": 173.96397399902344, + "fcm_dpo/q_t": 0.4313589036464691, + "grad_norm": 29.68878746032715, + "learning_rate": 1.6570202148426815e-07, + "logits/chosen": -0.4677377939224243, + "logits/rejected": -0.4488041400909424, + "logps/chosen": -449.765625, + "logps/ref_chosen": -71.27301788330078, + "logps/ref_rejected": -86.679931640625, + "logps/rejected": -639.136474609375, + "loss": 1.1875, + "margin_dpo/margin_mean": 173.96395874023438, + "margin_dpo/margin_std": 334.8567810058594, + "step": 443 + }, + { + "KL/chosen_KL_mean": -357.6883850097656, + "KL/mean": -494.31622314453125, + "KL/rejected_KL_mean": -630.944091796875, + "KL/std": 298.2752685546875, + "epoch": 0.6519823788546255, + "fcm_dpo/beta": 0.0017422267701476812, + "fcm_dpo/delta": -0.08027061074972153, + "fcm_dpo/margin": 273.25567626953125, + "fcm_dpo/q_t": 0.39155757427215576, + "grad_norm": 29.274503707885742, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": -0.45265936851501465, + "logits/rejected": -0.4536314010620117, + "logps/chosen": -414.902099609375, + "logps/ref_chosen": -57.213706970214844, + "logps/ref_rejected": -97.25489807128906, + "logps/rejected": -728.198974609375, + "loss": 1.0395, + "margin_dpo/margin_mean": 273.25567626953125, + "margin_dpo/margin_std": 326.13409423828125, + "step": 444 + }, + { + "KL/chosen_KL_mean": -323.1199035644531, + "KL/mean": -449.0135803222656, + "KL/rejected_KL_mean": -574.9072265625, + "KL/std": 250.39508056640625, + "epoch": 0.6534508076358296, + "fcm_dpo/beta": 0.001730577670969069, + "fcm_dpo/delta": -0.037361592054367065, + "fcm_dpo/margin": 251.78732299804688, + "fcm_dpo/q_t": 0.40040573477745056, + "grad_norm": 30.908275604248047, + "learning_rate": 1.6329015999011182e-07, + "logits/chosen": -0.46896958351135254, + "logits/rejected": -0.4585559368133545, + "logps/chosen": -390.4197082519531, + "logps/ref_chosen": -67.29979705810547, + "logps/ref_rejected": -92.68267059326172, + "logps/rejected": -667.5899658203125, + "loss": 1.0632, + "margin_dpo/margin_mean": 251.78732299804688, + "margin_dpo/margin_std": 303.667724609375, + "step": 445 + }, + { + "KL/chosen_KL_mean": -299.8826904296875, + "KL/mean": -439.4361877441406, + "KL/rejected_KL_mean": -578.9896850585938, + "KL/std": 266.3847961425781, + "epoch": 0.6549192364170338, + "fcm_dpo/beta": 0.0017157963011413813, + "fcm_dpo/delta": -0.08306018263101578, + "fcm_dpo/margin": 279.10699462890625, + "fcm_dpo/q_t": 0.38841933012008667, + "grad_norm": 24.125913619995117, + "learning_rate": 1.6208764069656578e-07, + "logits/chosen": -0.46759241819381714, + "logits/rejected": -0.4777601361274719, + "logps/chosen": -358.98114013671875, + "logps/ref_chosen": -59.098487854003906, + "logps/ref_rejected": -101.26419067382812, + "logps/rejected": -680.25390625, + "loss": 1.018, + "margin_dpo/margin_mean": 279.10699462890625, + "margin_dpo/margin_std": 273.501220703125, + "step": 446 + }, + { + "KL/chosen_KL_mean": -321.5812683105469, + "KL/mean": -465.3448486328125, + "KL/rejected_KL_mean": -609.1083984375, + "KL/std": 309.42169189453125, + "epoch": 0.6563876651982379, + "fcm_dpo/beta": 0.0016721412539482117, + "fcm_dpo/delta": -0.08531050384044647, + "fcm_dpo/margin": 287.5271301269531, + "fcm_dpo/q_t": 0.39167481660842896, + "grad_norm": 30.128538131713867, + "learning_rate": 1.608874379754465e-07, + "logits/chosen": -0.5374941229820251, + "logits/rejected": -0.542584240436554, + "logps/chosen": -377.6566162109375, + "logps/ref_chosen": -56.07533264160156, + "logps/ref_rejected": -98.69475555419922, + "logps/rejected": -707.8031616210938, + "loss": 1.0343, + "margin_dpo/margin_mean": 287.52716064453125, + "margin_dpo/margin_std": 343.70501708984375, + "step": 447 + }, + { + "KL/chosen_KL_mean": -364.1661071777344, + "KL/mean": -509.53765869140625, + "KL/rejected_KL_mean": -654.9091796875, + "KL/std": 271.6998291015625, + "epoch": 0.657856093979442, + "fcm_dpo/beta": 0.0016555753536522388, + "fcm_dpo/delta": -0.08545278757810593, + "fcm_dpo/margin": 290.74310302734375, + "fcm_dpo/q_t": 0.38933056592941284, + "grad_norm": 40.963924407958984, + "learning_rate": 1.5968958345321177e-07, + "logits/chosen": -0.47368836402893066, + "logits/rejected": -0.4778289794921875, + "logps/chosen": -424.1699523925781, + "logps/ref_chosen": -60.00384521484375, + "logps/ref_rejected": -102.26465606689453, + "logps/rejected": -757.173828125, + "loss": 1.0235, + "margin_dpo/margin_mean": 290.74310302734375, + "margin_dpo/margin_std": 306.6837463378906, + "step": 448 + }, + { + "KL/chosen_KL_mean": -390.13323974609375, + "KL/mean": -518.724853515625, + "KL/rejected_KL_mean": -647.3164672851562, + "KL/std": 310.909912109375, + "epoch": 0.6593245227606461, + "fcm_dpo/beta": 0.001634610234759748, + "fcm_dpo/delta": -0.021336160600185394, + "fcm_dpo/margin": 257.1832275390625, + "fcm_dpo/q_t": 0.40584796667099, + "grad_norm": 25.04412078857422, + "learning_rate": 1.584941086944423e-07, + "logits/chosen": -0.5128796100616455, + "logits/rejected": -0.5053173899650574, + "logps/chosen": -457.65985107421875, + "logps/ref_chosen": -67.52661895751953, + "logps/ref_rejected": -88.59690856933594, + "logps/rejected": -735.9133911132812, + "loss": 1.0971, + "margin_dpo/margin_mean": 257.1832275390625, + "margin_dpo/margin_std": 382.7791748046875, + "step": 449 + }, + { + "KL/chosen_KL_mean": -325.1204833984375, + "KL/mean": -481.97296142578125, + "KL/rejected_KL_mean": -638.825439453125, + "KL/std": 312.12579345703125, + "epoch": 0.6607929515418502, + "fcm_dpo/beta": 0.0016077951295301318, + "fcm_dpo/delta": -0.10993964970111847, + "fcm_dpo/margin": 313.70501708984375, + "fcm_dpo/q_t": 0.3819790780544281, + "grad_norm": 31.933448791503906, + "learning_rate": 1.573010452010098e-07, + "logits/chosen": -0.5362136363983154, + "logits/rejected": -0.5440558195114136, + "logps/chosen": -382.22857666015625, + "logps/ref_chosen": -57.10811996459961, + "logps/ref_rejected": -102.75494384765625, + "logps/rejected": -741.5804443359375, + "loss": 1.0019, + "margin_dpo/margin_mean": 313.7049865722656, + "margin_dpo/margin_std": 309.9493408203125, + "step": 450 + }, + { + "KL/chosen_KL_mean": -407.0479431152344, + "KL/mean": -521.1527099609375, + "KL/rejected_KL_mean": -635.2574462890625, + "KL/std": 311.9068603515625, + "epoch": 0.6622613803230544, + "fcm_dpo/beta": 0.0016091731376945972, + "fcm_dpo/delta": 0.03394605964422226, + "fcm_dpo/margin": 228.20953369140625, + "fcm_dpo/q_t": 0.41256624460220337, + "grad_norm": 28.686208724975586, + "learning_rate": 1.5611042441124687e-07, + "logits/chosen": -0.4843297302722931, + "logits/rejected": -0.4612458348274231, + "logps/chosen": -465.51678466796875, + "logps/ref_chosen": -58.46883010864258, + "logps/ref_rejected": -72.92941284179688, + "logps/rejected": -708.1868896484375, + "loss": 1.1327, + "margin_dpo/margin_mean": 228.2095184326172, + "margin_dpo/margin_std": 363.1060791015625, + "step": 451 + }, + { + "KL/chosen_KL_mean": -299.3313293457031, + "KL/mean": -445.92633056640625, + "KL/rejected_KL_mean": -592.5213012695312, + "KL/std": 276.49298095703125, + "epoch": 0.6637298091042585, + "fcm_dpo/beta": 0.0015910858055576682, + "fcm_dpo/delta": -0.0698796957731247, + "fcm_dpo/margin": 293.19000244140625, + "fcm_dpo/q_t": 0.39026927947998047, + "grad_norm": 20.935033798217773, + "learning_rate": 1.549222776991186e-07, + "logits/chosen": -0.4869954288005829, + "logits/rejected": -0.5046002864837646, + "logps/chosen": -349.7218933105469, + "logps/ref_chosen": -50.39055252075195, + "logps/ref_rejected": -97.77142333984375, + "logps/rejected": -690.292724609375, + "loss": 1.0204, + "margin_dpo/margin_mean": 293.1899719238281, + "margin_dpo/margin_std": 282.45556640625, + "step": 452 + }, + { + "KL/chosen_KL_mean": -366.5586242675781, + "KL/mean": -491.7527160644531, + "KL/rejected_KL_mean": -616.94677734375, + "KL/std": 276.7699279785156, + "epoch": 0.6651982378854625, + "fcm_dpo/beta": 0.0015832162462174892, + "fcm_dpo/delta": 0.0035196300595998764, + "fcm_dpo/margin": 250.38816833496094, + "fcm_dpo/q_t": 0.4097445607185364, + "grad_norm": 32.47494888305664, + "learning_rate": 1.5373663637339584e-07, + "logits/chosen": -0.5162127614021301, + "logits/rejected": -0.4957190752029419, + "logps/chosen": -424.27349853515625, + "logps/ref_chosen": -57.71485137939453, + "logps/ref_rejected": -82.20741271972656, + "logps/rejected": -699.1542358398438, + "loss": 1.0915, + "margin_dpo/margin_mean": 250.38815307617188, + "margin_dpo/margin_std": 329.19891357421875, + "step": 453 + }, + { + "KL/chosen_KL_mean": -425.7057189941406, + "KL/mean": -568.9153442382812, + "KL/rejected_KL_mean": -712.125, + "KL/std": 310.9266357421875, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.0015724084805697203, + "fcm_dpo/delta": -0.052992694079875946, + "fcm_dpo/margin": 286.4192810058594, + "fcm_dpo/q_t": 0.39700382947921753, + "grad_norm": 29.691364288330078, + "learning_rate": 1.5255353167683017e-07, + "logits/chosen": -0.5279435515403748, + "logits/rejected": -0.5177085995674133, + "logps/chosen": -486.6513671875, + "logps/ref_chosen": -60.945648193359375, + "logps/ref_rejected": -84.95079040527344, + "logps/rejected": -797.0758056640625, + "loss": 1.0566, + "margin_dpo/margin_mean": 286.4192810058594, + "margin_dpo/margin_std": 355.6832275390625, + "step": 454 + }, + { + "KL/chosen_KL_mean": -374.63739013671875, + "KL/mean": -549.975341796875, + "KL/rejected_KL_mean": -725.3132934570312, + "KL/std": 342.216064453125, + "epoch": 0.6681350954478708, + "fcm_dpo/beta": 0.0015454981476068497, + "fcm_dpo/delta": -0.14988133311271667, + "fcm_dpo/margin": 350.6759033203125, + "fcm_dpo/q_t": 0.3785054683685303, + "grad_norm": 39.72151565551758, + "learning_rate": 1.5137299478533064e-07, + "logits/chosen": -0.5236682295799255, + "logits/rejected": -0.5419257283210754, + "logps/chosen": -419.5240783691406, + "logps/ref_chosen": -44.88671112060547, + "logps/ref_rejected": -115.30147552490234, + "logps/rejected": -840.61474609375, + "loss": 1.0004, + "margin_dpo/margin_mean": 350.6759033203125, + "margin_dpo/margin_std": 388.170166015625, + "step": 455 + }, + { + "KL/chosen_KL_mean": -396.84381103515625, + "KL/mean": -554.3494262695312, + "KL/rejected_KL_mean": -711.85498046875, + "KL/std": 319.39385986328125, + "epoch": 0.6696035242290749, + "fcm_dpo/beta": 0.001508980873040855, + "fcm_dpo/delta": -0.07905428111553192, + "fcm_dpo/margin": 315.0111389160156, + "fcm_dpo/q_t": 0.38942915201187134, + "grad_norm": 20.266353607177734, + "learning_rate": 1.5019505680714232e-07, + "logits/chosen": -0.5147756338119507, + "logits/rejected": -0.5317715406417847, + "logps/chosen": -453.880615234375, + "logps/ref_chosen": -57.036781311035156, + "logps/ref_rejected": -105.21784210205078, + "logps/rejected": -817.0728149414062, + "loss": 1.0175, + "margin_dpo/margin_mean": 315.01116943359375, + "margin_dpo/margin_std": 312.59521484375, + "step": 456 + }, + { + "KL/chosen_KL_mean": -372.20489501953125, + "KL/mean": -528.918212890625, + "KL/rejected_KL_mean": -685.631591796875, + "KL/std": 302.023193359375, + "epoch": 0.671071953010279, + "fcm_dpo/beta": 0.001482290681451559, + "fcm_dpo/delta": -0.06809078902006149, + "fcm_dpo/margin": 313.4267272949219, + "fcm_dpo/q_t": 0.3901534080505371, + "grad_norm": 25.995811462402344, + "learning_rate": 1.4901974878202627e-07, + "logits/chosen": -0.5356336832046509, + "logits/rejected": -0.5320440530776978, + "logps/chosen": -426.44744873046875, + "logps/ref_chosen": -54.24253845214844, + "logps/ref_rejected": -85.10956573486328, + "logps/rejected": -770.7412109375, + "loss": 1.0214, + "margin_dpo/margin_mean": 313.4267272949219, + "margin_dpo/margin_std": 297.42291259765625, + "step": 457 + }, + { + "KL/chosen_KL_mean": -381.38238525390625, + "KL/mean": -524.291748046875, + "KL/rejected_KL_mean": -667.201171875, + "KL/std": 277.86822509765625, + "epoch": 0.6725403817914831, + "fcm_dpo/beta": 0.001474408432841301, + "fcm_dpo/delta": -0.022421175613999367, + "fcm_dpo/margin": 285.8187255859375, + "fcm_dpo/q_t": 0.4017055928707123, + "grad_norm": 24.2961483001709, + "learning_rate": 1.4784710168044212e-07, + "logits/chosen": -0.5444470047950745, + "logits/rejected": -0.5421079397201538, + "logps/chosen": -436.7912902832031, + "logps/ref_chosen": -55.40888214111328, + "logps/ref_rejected": -97.68325805664062, + "logps/rejected": -764.8843994140625, + "loss": 1.06, + "margin_dpo/margin_mean": 285.8187255859375, + "margin_dpo/margin_std": 318.3499450683594, + "step": 458 + }, + { + "KL/chosen_KL_mean": -400.17242431640625, + "KL/mean": -548.4536743164062, + "KL/rejected_KL_mean": -696.73486328125, + "KL/std": 310.79791259765625, + "epoch": 0.6740088105726872, + "fcm_dpo/beta": 0.0014622649177908897, + "fcm_dpo/delta": -0.035495489835739136, + "fcm_dpo/margin": 296.5623779296875, + "fcm_dpo/q_t": 0.399467408657074, + "grad_norm": 21.809309005737305, + "learning_rate": 1.466771464027316e-07, + "logits/chosen": -0.5215349793434143, + "logits/rejected": -0.5344473719596863, + "logps/chosen": -446.72991943359375, + "logps/ref_chosen": -46.55748748779297, + "logps/ref_rejected": -86.16854095458984, + "logps/rejected": -782.9033813476562, + "loss": 1.0613, + "margin_dpo/margin_mean": 296.5624084472656, + "margin_dpo/margin_std": 346.4207458496094, + "step": 459 + }, + { + "KL/chosen_KL_mean": -457.33087158203125, + "KL/mean": -610.467041015625, + "KL/rejected_KL_mean": -763.6031494140625, + "KL/std": 300.7550354003906, + "epoch": 0.6754772393538914, + "fcm_dpo/beta": 0.0014558184193447232, + "fcm_dpo/delta": -0.047990500926971436, + "fcm_dpo/margin": 306.27227783203125, + "fcm_dpo/q_t": 0.3961849808692932, + "grad_norm": 37.35396957397461, + "learning_rate": 1.4550991377830423e-07, + "logits/chosen": -0.5847633481025696, + "logits/rejected": -0.6056011915206909, + "logps/chosen": -508.96575927734375, + "logps/ref_chosen": -51.63489532470703, + "logps/ref_rejected": -104.11935424804688, + "logps/rejected": -867.7225341796875, + "loss": 1.0468, + "margin_dpo/margin_mean": 306.2723083496094, + "margin_dpo/margin_std": 339.9352722167969, + "step": 460 + }, + { + "KL/chosen_KL_mean": -445.0636901855469, + "KL/mean": -572.2376708984375, + "KL/rejected_KL_mean": -699.4116821289062, + "KL/std": 290.94036865234375, + "epoch": 0.6769456681350955, + "fcm_dpo/beta": 0.0014602546580135822, + "fcm_dpo/delta": 0.029491884633898735, + "fcm_dpo/margin": 254.34796142578125, + "fcm_dpo/q_t": 0.4137762188911438, + "grad_norm": 22.10498046875, + "learning_rate": 1.4434543456482518e-07, + "logits/chosen": -0.5995633006095886, + "logits/rejected": -0.6073707342147827, + "logps/chosen": -500.24566650390625, + "logps/ref_chosen": -55.18195724487305, + "logps/ref_rejected": -86.47689819335938, + "logps/rejected": -785.8885498046875, + "loss": 1.1051, + "margin_dpo/margin_mean": 254.34796142578125, + "margin_dpo/margin_std": 332.9735107421875, + "step": 461 + }, + { + "KL/chosen_KL_mean": -459.8350830078125, + "KL/mean": -553.7359619140625, + "KL/rejected_KL_mean": -647.6368408203125, + "KL/std": 295.6435852050781, + "epoch": 0.6784140969162996, + "fcm_dpo/beta": 0.0014856795314699411, + "fcm_dpo/delta": 0.12450301647186279, + "fcm_dpo/margin": 187.80177307128906, + "fcm_dpo/q_t": 0.4358844757080078, + "grad_norm": 43.47697067260742, + "learning_rate": 1.4318373944740484e-07, + "logits/chosen": -0.6230775117874146, + "logits/rejected": -0.616161584854126, + "logps/chosen": -529.7631225585938, + "logps/ref_chosen": -69.92803192138672, + "logps/ref_rejected": -78.84111022949219, + "logps/rejected": -726.4779052734375, + "loss": 1.1844, + "margin_dpo/margin_mean": 187.8017578125, + "margin_dpo/margin_std": 329.5667419433594, + "step": 462 + }, + { + "KL/chosen_KL_mean": -415.004150390625, + "KL/mean": -553.5238037109375, + "KL/rejected_KL_mean": -692.04345703125, + "KL/std": 322.8453674316406, + "epoch": 0.6798825256975036, + "fcm_dpo/beta": 0.001497290562838316, + "fcm_dpo/delta": -0.015637099742889404, + "fcm_dpo/margin": 277.03924560546875, + "fcm_dpo/q_t": 0.40619271993637085, + "grad_norm": 33.27143478393555, + "learning_rate": 1.4202485903778976e-07, + "logits/chosen": -0.6098886728286743, + "logits/rejected": -0.6173523664474487, + "logps/chosen": -470.27850341796875, + "logps/ref_chosen": -55.27437210083008, + "logps/ref_rejected": -89.02497863769531, + "logps/rejected": -781.068359375, + "loss": 1.0932, + "margin_dpo/margin_mean": 277.03924560546875, + "margin_dpo/margin_std": 387.570068359375, + "step": 463 + }, + { + "KL/chosen_KL_mean": -402.9927978515625, + "KL/mean": -604.0712890625, + "KL/rejected_KL_mean": -805.1497802734375, + "KL/std": 352.389892578125, + "epoch": 0.6813509544787077, + "fcm_dpo/beta": 0.0014413490425795317, + "fcm_dpo/delta": -0.1922934651374817, + "fcm_dpo/margin": 402.1569519042969, + "fcm_dpo/q_t": 0.3662247657775879, + "grad_norm": 29.33611488342285, + "learning_rate": 1.4086882387355658e-07, + "logits/chosen": -0.5622963309288025, + "logits/rejected": -0.6052860021591187, + "logps/chosen": -453.9051208496094, + "logps/ref_chosen": -50.91230010986328, + "logps/ref_rejected": -102.4893798828125, + "logps/rejected": -907.63916015625, + "loss": 0.9519, + "margin_dpo/margin_mean": 402.156982421875, + "margin_dpo/margin_std": 368.00384521484375, + "step": 464 + }, + { + "KL/chosen_KL_mean": -372.560791015625, + "KL/mean": -539.6180419921875, + "KL/rejected_KL_mean": -706.67529296875, + "KL/std": 334.50079345703125, + "epoch": 0.6828193832599119, + "fcm_dpo/beta": 0.0014234642731025815, + "fcm_dpo/delta": -0.07937896996736526, + "fcm_dpo/margin": 334.114501953125, + "fcm_dpo/q_t": 0.3893335163593292, + "grad_norm": 27.498571395874023, + "learning_rate": 1.3971566441730714e-07, + "logits/chosen": -0.5741344094276428, + "logits/rejected": -0.5896367430686951, + "logps/chosen": -432.6776428222656, + "logps/ref_chosen": -60.116851806640625, + "logps/ref_rejected": -113.94602966308594, + "logps/rejected": -820.621337890625, + "loss": 1.022, + "margin_dpo/margin_mean": 334.114501953125, + "margin_dpo/margin_std": 343.14959716796875, + "step": 465 + }, + { + "KL/chosen_KL_mean": -411.8160400390625, + "KL/mean": -545.8191528320312, + "KL/rejected_KL_mean": -679.822265625, + "KL/std": 299.4898681640625, + "epoch": 0.684287812041116, + "fcm_dpo/beta": 0.0014133648946881294, + "fcm_dpo/delta": 0.021980978548526764, + "fcm_dpo/margin": 268.0061950683594, + "fcm_dpo/q_t": 0.4110508859157562, + "grad_norm": 29.728809356689453, + "learning_rate": 1.3856541105586545e-07, + "logits/chosen": -0.5952455997467041, + "logits/rejected": -0.5951662063598633, + "logps/chosen": -464.73699951171875, + "logps/ref_chosen": -52.920921325683594, + "logps/ref_rejected": -90.3154296875, + "logps/rejected": -770.1376953125, + "loss": 1.0968, + "margin_dpo/margin_mean": 268.00616455078125, + "margin_dpo/margin_std": 334.8175354003906, + "step": 466 + }, + { + "KL/chosen_KL_mean": -508.6435546875, + "KL/mean": -648.5333251953125, + "KL/rejected_KL_mean": -788.423095703125, + "KL/std": 349.01898193359375, + "epoch": 0.6857562408223201, + "fcm_dpo/beta": 0.0014083052519708872, + "fcm_dpo/delta": 0.005007736384868622, + "fcm_dpo/margin": 279.7795715332031, + "fcm_dpo/q_t": 0.4113559424877167, + "grad_norm": 33.90370559692383, + "learning_rate": 1.3741809409947729e-07, + "logits/chosen": -0.6591016054153442, + "logits/rejected": -0.6488548517227173, + "logps/chosen": -587.359375, + "logps/ref_chosen": -78.7158203125, + "logps/ref_rejected": -102.86019897460938, + "logps/rejected": -891.2833251953125, + "loss": 1.1234, + "margin_dpo/margin_mean": 279.779541015625, + "margin_dpo/margin_std": 433.38726806640625, + "step": 467 + }, + { + "KL/chosen_KL_mean": -386.95941162109375, + "KL/mean": -549.1633911132812, + "KL/rejected_KL_mean": -711.3673706054688, + "KL/std": 344.0048522949219, + "epoch": 0.6872246696035242, + "fcm_dpo/beta": 0.0014025418786332011, + "fcm_dpo/delta": -0.05807999521493912, + "fcm_dpo/margin": 324.407958984375, + "fcm_dpo/q_t": 0.39575815200805664, + "grad_norm": 39.35798645019531, + "learning_rate": 1.362737437810114e-07, + "logits/chosen": -0.632330596446991, + "logits/rejected": -0.6243791580200195, + "logps/chosen": -456.894775390625, + "logps/ref_chosen": -69.93536376953125, + "logps/ref_rejected": -101.02880859375, + "logps/rejected": -812.3961791992188, + "loss": 1.0463, + "margin_dpo/margin_mean": 324.407958984375, + "margin_dpo/margin_std": 376.43096923828125, + "step": 468 + }, + { + "KL/chosen_KL_mean": -436.9048767089844, + "KL/mean": -599.1432495117188, + "KL/rejected_KL_mean": -761.3817138671875, + "KL/std": 320.456787109375, + "epoch": 0.6886930983847284, + "fcm_dpo/beta": 0.0013855324359610677, + "fcm_dpo/delta": -0.0526653453707695, + "fcm_dpo/margin": 324.476806640625, + "fcm_dpo/q_t": 0.3954392969608307, + "grad_norm": 27.19671058654785, + "learning_rate": 1.351323902551631e-07, + "logits/chosen": -0.6334969997406006, + "logits/rejected": -0.6272901296615601, + "logps/chosen": -505.0295715332031, + "logps/ref_chosen": -68.12469482421875, + "logps/ref_rejected": -104.78640747070312, + "logps/rejected": -866.1680908203125, + "loss": 1.0435, + "margin_dpo/margin_mean": 324.476806640625, + "margin_dpo/margin_std": 351.4849853515625, + "step": 469 + }, + { + "KL/chosen_KL_mean": -332.91339111328125, + "KL/mean": -487.83050537109375, + "KL/rejected_KL_mean": -642.7476806640625, + "KL/std": 311.11309814453125, + "epoch": 0.6901615271659325, + "fcm_dpo/beta": 0.0013843157794326544, + "fcm_dpo/delta": -0.030201081186532974, + "fcm_dpo/margin": 309.83428955078125, + "fcm_dpo/q_t": 0.39850544929504395, + "grad_norm": 25.313594818115234, + "learning_rate": 1.339940635976592e-07, + "logits/chosen": -0.6387758851051331, + "logits/rejected": -0.6436039209365845, + "logps/chosen": -376.7052917480469, + "logps/ref_chosen": -43.791927337646484, + "logps/ref_rejected": -82.70285034179688, + "logps/rejected": -725.4505004882812, + "loss": 1.0555, + "margin_dpo/margin_mean": 309.83428955078125, + "margin_dpo/margin_std": 340.318115234375, + "step": 470 + }, + { + "KL/chosen_KL_mean": -412.908447265625, + "KL/mean": -547.954833984375, + "KL/rejected_KL_mean": -683.0012817382812, + "KL/std": 324.60595703125, + "epoch": 0.6916299559471366, + "fcm_dpo/beta": 0.0013804736081510782, + "fcm_dpo/delta": 0.027975287288427353, + "fcm_dpo/margin": 270.0928039550781, + "fcm_dpo/q_t": 0.41345933079719543, + "grad_norm": 24.690170288085938, + "learning_rate": 1.3285879380446563e-07, + "logits/chosen": -0.6649577617645264, + "logits/rejected": -0.6651647090911865, + "logps/chosen": -476.24798583984375, + "logps/ref_chosen": -63.33952331542969, + "logps/ref_rejected": -83.61048126220703, + "logps/rejected": -766.6117553710938, + "loss": 1.0987, + "margin_dpo/margin_mean": 270.0928039550781, + "margin_dpo/margin_std": 332.7794189453125, + "step": 471 + }, + { + "KL/chosen_KL_mean": -438.15020751953125, + "KL/mean": -597.1299438476562, + "KL/rejected_KL_mean": -756.1097412109375, + "KL/std": 392.02294921875, + "epoch": 0.6930983847283406, + "fcm_dpo/beta": 0.001379038905724883, + "fcm_dpo/delta": -0.04026825353503227, + "fcm_dpo/margin": 317.9595031738281, + "fcm_dpo/q_t": 0.40268874168395996, + "grad_norm": 25.000944137573242, + "learning_rate": 1.317266107909975e-07, + "logits/chosen": -0.7143632173538208, + "logits/rejected": -0.6818025708198547, + "logps/chosen": -521.8162841796875, + "logps/ref_chosen": -83.66610717773438, + "logps/ref_rejected": -117.20919799804688, + "logps/rejected": -873.3189086914062, + "loss": 1.0767, + "margin_dpo/margin_mean": 317.95953369140625, + "margin_dpo/margin_std": 434.4294738769531, + "step": 472 + }, + { + "KL/chosen_KL_mean": -489.4718322753906, + "KL/mean": -592.4940185546875, + "KL/rejected_KL_mean": -695.5161743164062, + "KL/std": 372.8263244628906, + "epoch": 0.6945668135095447, + "fcm_dpo/beta": 0.0014005075208842754, + "fcm_dpo/delta": 0.11475996673107147, + "fcm_dpo/margin": 206.04432678222656, + "fcm_dpo/q_t": 0.43647685647010803, + "grad_norm": 41.77243423461914, + "learning_rate": 1.3059754439133002e-07, + "logits/chosen": -0.6815752983093262, + "logits/rejected": -0.6562888622283936, + "logps/chosen": -552.9688110351562, + "logps/ref_chosen": -63.49696731567383, + "logps/ref_rejected": -81.14657592773438, + "logps/rejected": -776.6627197265625, + "loss": 1.2097, + "margin_dpo/margin_mean": 206.04432678222656, + "margin_dpo/margin_std": 453.6682434082031, + "step": 473 + }, + { + "KL/chosen_KL_mean": -412.56927490234375, + "KL/mean": -540.0201416015625, + "KL/rejected_KL_mean": -667.470947265625, + "KL/std": 328.03204345703125, + "epoch": 0.6960352422907489, + "fcm_dpo/beta": 0.001424451475031674, + "fcm_dpo/delta": 0.03749527409672737, + "fcm_dpo/margin": 254.90162658691406, + "fcm_dpo/q_t": 0.41584312915802, + "grad_norm": 30.2032527923584, + "learning_rate": 1.2947162435741277e-07, + "logits/chosen": -0.6723858118057251, + "logits/rejected": -0.6669915914535522, + "logps/chosen": -465.18121337890625, + "logps/ref_chosen": -52.6119384765625, + "logps/ref_rejected": -90.08041381835938, + "logps/rejected": -757.5513305664062, + "loss": 1.1276, + "margin_dpo/margin_mean": 254.90164184570312, + "margin_dpo/margin_std": 379.5084228515625, + "step": 474 + }, + { + "KL/chosen_KL_mean": -334.7874755859375, + "KL/mean": -509.8450622558594, + "KL/rejected_KL_mean": -684.9027099609375, + "KL/std": 321.20361328125, + "epoch": 0.697503671071953, + "fcm_dpo/beta": 0.0013988735154271126, + "fcm_dpo/delta": -0.09445011615753174, + "fcm_dpo/margin": 350.11517333984375, + "fcm_dpo/q_t": 0.3866940140724182, + "grad_norm": 22.809551239013672, + "learning_rate": 1.2834888035828596e-07, + "logits/chosen": -0.7078989744186401, + "logits/rejected": -0.7233259677886963, + "logps/chosen": -377.2826843261719, + "logps/ref_chosen": -42.49519348144531, + "logps/ref_rejected": -90.06294250488281, + "logps/rejected": -774.9656372070312, + "loss": 1.0106, + "margin_dpo/margin_mean": 350.1152038574219, + "margin_dpo/margin_std": 354.17852783203125, + "step": 475 + }, + { + "KL/chosen_KL_mean": -391.2220458984375, + "KL/mean": -534.6827392578125, + "KL/rejected_KL_mean": -678.1434326171875, + "KL/std": 323.22772216796875, + "epoch": 0.6989720998531571, + "fcm_dpo/beta": 0.001390953897498548, + "fcm_dpo/delta": 0.0008967118337750435, + "fcm_dpo/margin": 286.92138671875, + "fcm_dpo/q_t": 0.4077247977256775, + "grad_norm": 29.844114303588867, + "learning_rate": 1.2722934197929802e-07, + "logits/chosen": -0.7228225469589233, + "logits/rejected": -0.7230457663536072, + "logps/chosen": -434.17138671875, + "logps/ref_chosen": -42.94938278198242, + "logps/ref_rejected": -73.71023559570312, + "logps/rejected": -751.8536376953125, + "loss": 1.0791, + "margin_dpo/margin_mean": 286.92138671875, + "margin_dpo/margin_std": 342.6464538574219, + "step": 476 + }, + { + "KL/chosen_KL_mean": -419.1278991699219, + "KL/mean": -544.8071899414062, + "KL/rejected_KL_mean": -670.4865112304688, + "KL/std": 317.6999816894531, + "epoch": 0.7004405286343612, + "fcm_dpo/beta": 0.0014057450462132692, + "fcm_dpo/delta": 0.0482083223760128, + "fcm_dpo/margin": 251.35861206054688, + "fcm_dpo/q_t": 0.4184876084327698, + "grad_norm": 27.383092880249023, + "learning_rate": 1.2611303872132631e-07, + "logits/chosen": -0.7424951791763306, + "logits/rejected": -0.70060133934021, + "logps/chosen": -489.9005126953125, + "logps/ref_chosen": -70.77261352539062, + "logps/ref_rejected": -76.13737487792969, + "logps/rejected": -746.6239013671875, + "loss": 1.1313, + "margin_dpo/margin_mean": 251.35861206054688, + "margin_dpo/margin_std": 380.20428466796875, + "step": 477 + }, + { + "KL/chosen_KL_mean": -347.18548583984375, + "KL/mean": -512.42431640625, + "KL/rejected_KL_mean": -677.6632080078125, + "KL/std": 301.6866455078125, + "epoch": 0.7019089574155654, + "fcm_dpo/beta": 0.001397792948409915, + "fcm_dpo/delta": -0.06495002657175064, + "fcm_dpo/margin": 330.4776611328125, + "fcm_dpo/q_t": 0.3927929997444153, + "grad_norm": 28.49869155883789, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": -0.6548566818237305, + "logits/rejected": -0.6655555367469788, + "logps/chosen": -388.6259765625, + "logps/ref_chosen": -41.440513610839844, + "logps/ref_rejected": -85.36196899414062, + "logps/rejected": -763.025146484375, + "loss": 1.0329, + "margin_dpo/margin_mean": 330.4776611328125, + "margin_dpo/margin_std": 345.8775634765625, + "step": 478 + }, + { + "KL/chosen_KL_mean": -487.4216613769531, + "KL/mean": -634.49609375, + "KL/rejected_KL_mean": -781.5704345703125, + "KL/std": 378.451416015625, + "epoch": 0.7033773861967695, + "fcm_dpo/beta": 0.0013954101596027613, + "fcm_dpo/delta": -0.011742450296878815, + "fcm_dpo/margin": 294.1488037109375, + "fcm_dpo/q_t": 0.4071451425552368, + "grad_norm": 25.941673278808594, + "learning_rate": 1.2389025514492456e-07, + "logits/chosen": -0.7130151987075806, + "logits/rejected": -0.7304632663726807, + "logps/chosen": -541.32958984375, + "logps/ref_chosen": -53.907920837402344, + "logps/ref_rejected": -95.1163330078125, + "logps/rejected": -876.686767578125, + "loss": 1.0994, + "margin_dpo/margin_mean": 294.1488037109375, + "margin_dpo/margin_std": 420.4874572753906, + "step": 479 + }, + { + "KL/chosen_KL_mean": -545.10693359375, + "KL/mean": -653.86328125, + "KL/rejected_KL_mean": -762.61962890625, + "KL/std": 339.1335754394531, + "epoch": 0.7048458149779736, + "fcm_dpo/beta": 0.0014152206713333726, + "fcm_dpo/delta": 0.09325779229402542, + "fcm_dpo/margin": 217.51266479492188, + "fcm_dpo/q_t": 0.42927083373069763, + "grad_norm": 57.129112243652344, + "learning_rate": 1.227838333989088e-07, + "logits/chosen": -0.7444524765014648, + "logits/rejected": -0.7359182834625244, + "logps/chosen": -603.7896118164062, + "logps/ref_chosen": -58.682701110839844, + "logps/ref_rejected": -82.93248748779297, + "logps/rejected": -845.5521240234375, + "loss": 1.1837, + "margin_dpo/margin_mean": 217.5126953125, + "margin_dpo/margin_std": 391.7486572265625, + "step": 480 + }, + { + "KL/chosen_KL_mean": -469.24285888671875, + "KL/mean": -651.1468505859375, + "KL/rejected_KL_mean": -833.05078125, + "KL/std": 393.1416320800781, + "epoch": 0.7063142437591777, + "fcm_dpo/beta": 0.0013865388464182615, + "fcm_dpo/delta": -0.110165536403656, + "fcm_dpo/margin": 363.80792236328125, + "fcm_dpo/q_t": 0.38523510098457336, + "grad_norm": 45.171932220458984, + "learning_rate": 1.2168076391719489e-07, + "logits/chosen": -0.7980456352233887, + "logits/rejected": -0.8220420479774475, + "logps/chosen": -524.2071533203125, + "logps/ref_chosen": -54.964271545410156, + "logps/ref_rejected": -92.42044067382812, + "logps/rejected": -925.47119140625, + "loss": 1.0202, + "margin_dpo/margin_mean": 363.80792236328125, + "margin_dpo/margin_std": 419.04107666015625, + "step": 481 + }, + { + "KL/chosen_KL_mean": -532.6513671875, + "KL/mean": -629.147705078125, + "KL/rejected_KL_mean": -725.64404296875, + "KL/std": 367.68963623046875, + "epoch": 0.7077826725403817, + "fcm_dpo/beta": 0.0014093199279159307, + "fcm_dpo/delta": 0.13144272565841675, + "fcm_dpo/margin": 192.99264526367188, + "fcm_dpo/q_t": 0.4362224340438843, + "grad_norm": 43.460269927978516, + "learning_rate": 1.2058107576668938e-07, + "logits/chosen": -0.7118823528289795, + "logits/rejected": -0.7040268182754517, + "logps/chosen": -600.204833984375, + "logps/ref_chosen": -67.553466796875, + "logps/ref_rejected": -87.58953857421875, + "logps/rejected": -813.2335815429688, + "loss": 1.2274, + "margin_dpo/margin_mean": 192.99264526367188, + "margin_dpo/margin_std": 455.07916259765625, + "step": 482 + }, + { + "KL/chosen_KL_mean": -494.7724304199219, + "KL/mean": -688.48193359375, + "KL/rejected_KL_mean": -882.1915283203125, + "KL/std": 397.38714599609375, + "epoch": 0.7092511013215859, + "fcm_dpo/beta": 0.0013840030878782272, + "fcm_dpo/delta": -0.14435875415802002, + "fcm_dpo/margin": 387.4190979003906, + "fcm_dpo/q_t": 0.3799830973148346, + "grad_norm": 40.94780349731445, + "learning_rate": 1.194847979251979e-07, + "logits/chosen": -0.7805616855621338, + "logits/rejected": -0.7874066233634949, + "logps/chosen": -558.102294921875, + "logps/ref_chosen": -63.32981872558594, + "logps/ref_rejected": -95.78697204589844, + "logps/rejected": -977.978515625, + "loss": 1.0135, + "margin_dpo/margin_mean": 387.4190673828125, + "margin_dpo/margin_std": 465.99285888671875, + "step": 483 + }, + { + "KL/chosen_KL_mean": -447.64361572265625, + "KL/mean": -612.8146362304688, + "KL/rejected_KL_mean": -777.985595703125, + "KL/std": 382.26495361328125, + "epoch": 0.71071953010279, + "fcm_dpo/beta": 0.0013740049907937646, + "fcm_dpo/delta": -0.056898586452007294, + "fcm_dpo/margin": 330.342041015625, + "fcm_dpo/q_t": 0.39715200662612915, + "grad_norm": 36.74729537963867, + "learning_rate": 1.1839195928066101e-07, + "logits/chosen": -0.7907060384750366, + "logits/rejected": -0.8072522282600403, + "logps/chosen": -506.78173828125, + "logps/ref_chosen": -59.13812255859375, + "logps/ref_rejected": -84.37144470214844, + "logps/rejected": -862.3570556640625, + "loss": 1.0552, + "margin_dpo/margin_mean": 330.342041015625, + "margin_dpo/margin_std": 402.3882141113281, + "step": 484 + }, + { + "KL/chosen_KL_mean": -473.51666259765625, + "KL/mean": -629.485107421875, + "KL/rejected_KL_mean": -785.4535522460938, + "KL/std": 376.95941162109375, + "epoch": 0.7121879588839941, + "fcm_dpo/beta": 0.0013583770487457514, + "fcm_dpo/delta": -0.024785015732049942, + "fcm_dpo/margin": 311.9368896484375, + "fcm_dpo/q_t": 0.4046768844127655, + "grad_norm": 34.864871978759766, + "learning_rate": 1.1730258863039347e-07, + "logits/chosen": -0.7671700716018677, + "logits/rejected": -0.785019040107727, + "logps/chosen": -532.3662109375, + "logps/ref_chosen": -58.849571228027344, + "logps/ref_rejected": -103.36408233642578, + "logps/rejected": -888.817626953125, + "loss": 1.0916, + "margin_dpo/margin_mean": 311.9368896484375, + "margin_dpo/margin_std": 449.0213317871094, + "step": 485 + }, + { + "KL/chosen_KL_mean": -506.8376770019531, + "KL/mean": -694.2506103515625, + "KL/rejected_KL_mean": -881.6634521484375, + "KL/std": 416.7193603515625, + "epoch": 0.7136563876651982, + "fcm_dpo/beta": 0.0013335715048015118, + "fcm_dpo/delta": -0.10512037575244904, + "fcm_dpo/margin": 374.8258056640625, + "fcm_dpo/q_t": 0.3889124393463135, + "grad_norm": 39.17012023925781, + "learning_rate": 1.1621671468032493e-07, + "logits/chosen": -0.8197872638702393, + "logits/rejected": -0.8387131094932556, + "logps/chosen": -562.0973510742188, + "logps/ref_chosen": -55.25966262817383, + "logps/ref_rejected": -92.13936614990234, + "logps/rejected": -973.8028564453125, + "loss": 1.0538, + "margin_dpo/margin_mean": 374.8258056640625, + "margin_dpo/margin_std": 518.553466796875, + "step": 486 + }, + { + "KL/chosen_KL_mean": -560.105224609375, + "KL/mean": -713.3970947265625, + "KL/rejected_KL_mean": -866.6890258789062, + "KL/std": 408.19464111328125, + "epoch": 0.7151248164464024, + "fcm_dpo/beta": 0.0013294760137796402, + "fcm_dpo/delta": -0.008133085444569588, + "fcm_dpo/margin": 306.58380126953125, + "fcm_dpo/q_t": 0.4039134085178375, + "grad_norm": 61.269317626953125, + "learning_rate": 1.1513436604424378e-07, + "logits/chosen": -0.8197336196899414, + "logits/rejected": -0.8259349465370178, + "logps/chosen": -613.1685791015625, + "logps/ref_chosen": -53.06330871582031, + "logps/ref_rejected": -92.41883087158203, + "logps/rejected": -959.1078491210938, + "loss": 1.0925, + "margin_dpo/margin_mean": 306.58380126953125, + "margin_dpo/margin_std": 416.7705078125, + "step": 487 + }, + { + "KL/chosen_KL_mean": -450.8982238769531, + "KL/mean": -607.0410766601562, + "KL/rejected_KL_mean": -763.1839599609375, + "KL/std": 365.67449951171875, + "epoch": 0.7165932452276065, + "fcm_dpo/beta": 0.0013271074276417494, + "fcm_dpo/delta": -0.015382722020149231, + "fcm_dpo/margin": 312.2857360839844, + "fcm_dpo/q_t": 0.4043211042881012, + "grad_norm": 38.323486328125, + "learning_rate": 1.1405557124304335e-07, + "logits/chosen": -0.7659053802490234, + "logits/rejected": -0.7713078260421753, + "logps/chosen": -503.1263732910156, + "logps/ref_chosen": -52.22815704345703, + "logps/ref_rejected": -84.00656127929688, + "logps/rejected": -847.1905517578125, + "loss": 1.0717, + "margin_dpo/margin_mean": 312.2857666015625, + "margin_dpo/margin_std": 371.44061279296875, + "step": 488 + }, + { + "KL/chosen_KL_mean": -421.9764709472656, + "KL/mean": -567.7413330078125, + "KL/rejected_KL_mean": -713.5062255859375, + "KL/std": 350.5052185058594, + "epoch": 0.7180616740088106, + "fcm_dpo/beta": 0.0013246666640043259, + "fcm_dpo/delta": 0.014329822733998299, + "fcm_dpo/margin": 291.52972412109375, + "fcm_dpo/q_t": 0.4118216931819916, + "grad_norm": 31.000072479248047, + "learning_rate": 1.1298035870396985e-07, + "logits/chosen": -0.8222206234931946, + "logits/rejected": -0.8239779472351074, + "logps/chosen": -477.9660949707031, + "logps/ref_chosen": -55.989627838134766, + "logps/ref_rejected": -79.39812469482422, + "logps/rejected": -792.904296875, + "loss": 1.0936, + "margin_dpo/margin_mean": 291.52972412109375, + "margin_dpo/margin_std": 376.18890380859375, + "step": 489 + }, + { + "KL/chosen_KL_mean": -505.61871337890625, + "KL/mean": -669.4989013671875, + "KL/rejected_KL_mean": -833.3790893554688, + "KL/std": 459.050048828125, + "epoch": 0.7195301027900147, + "fcm_dpo/beta": 0.0013201385736465454, + "fcm_dpo/delta": -0.03418659418821335, + "fcm_dpo/margin": 327.7603759765625, + "fcm_dpo/q_t": 0.4046947658061981, + "grad_norm": 35.84400177001953, + "learning_rate": 1.1190875675987355e-07, + "logits/chosen": -0.8307057619094849, + "logits/rejected": -0.8715535402297974, + "logps/chosen": -557.985107421875, + "logps/ref_chosen": -52.36639404296875, + "logps/ref_rejected": -110.4090576171875, + "logps/rejected": -943.7881469726562, + "loss": 1.1084, + "margin_dpo/margin_mean": 327.7603759765625, + "margin_dpo/margin_std": 522.5004272460938, + "step": 490 + }, + { + "KL/chosen_KL_mean": -479.70025634765625, + "KL/mean": -585.4370727539062, + "KL/rejected_KL_mean": -691.173828125, + "KL/std": 367.56024169921875, + "epoch": 0.7209985315712188, + "fcm_dpo/beta": 0.0013417748268693686, + "fcm_dpo/delta": 0.11926029622554779, + "fcm_dpo/margin": 211.4735870361328, + "fcm_dpo/q_t": 0.43593573570251465, + "grad_norm": 30.89192008972168, + "learning_rate": 1.1084079364846241e-07, + "logits/chosen": -0.7931652069091797, + "logits/rejected": -0.7898507714271545, + "logps/chosen": -539.8165283203125, + "logps/ref_chosen": -60.11626434326172, + "logps/ref_rejected": -73.27278900146484, + "logps/rejected": -764.4466552734375, + "loss": 1.1864, + "margin_dpo/margin_mean": 211.47357177734375, + "margin_dpo/margin_std": 388.8064270019531, + "step": 491 + }, + { + "KL/chosen_KL_mean": -498.99969482421875, + "KL/mean": -605.6187744140625, + "KL/rejected_KL_mean": -712.2379150390625, + "KL/std": 378.2779541015625, + "epoch": 0.7224669603524229, + "fcm_dpo/beta": 0.0013660395052284002, + "fcm_dpo/delta": 0.11212383210659027, + "fcm_dpo/margin": 213.23822021484375, + "fcm_dpo/q_t": 0.4349971413612366, + "grad_norm": 34.141265869140625, + "learning_rate": 1.097764975115576e-07, + "logits/chosen": -0.8712701797485352, + "logits/rejected": -0.8622636795043945, + "logps/chosen": -552.993896484375, + "logps/ref_chosen": -53.994178771972656, + "logps/ref_rejected": -72.65962219238281, + "logps/rejected": -784.8974609375, + "loss": 1.2029, + "margin_dpo/margin_mean": 213.23822021484375, + "margin_dpo/margin_std": 443.1600341796875, + "step": 492 + }, + { + "KL/chosen_KL_mean": -544.2478637695312, + "KL/mean": -646.387451171875, + "KL/rejected_KL_mean": -748.5269775390625, + "KL/std": 407.29376220703125, + "epoch": 0.723935389133627, + "fcm_dpo/beta": 0.0013930967543274164, + "fcm_dpo/delta": 0.11872616410255432, + "fcm_dpo/margin": 204.27920532226562, + "fcm_dpo/q_t": 0.43394380807876587, + "grad_norm": 40.129520416259766, + "learning_rate": 1.0871589639435203e-07, + "logits/chosen": -0.9206933379173279, + "logits/rejected": -0.8987519145011902, + "logps/chosen": -619.7451171875, + "logps/ref_chosen": -75.49723815917969, + "logps/ref_rejected": -87.32301330566406, + "logps/rejected": -835.8500366210938, + "loss": 1.1997, + "margin_dpo/margin_mean": 204.27920532226562, + "margin_dpo/margin_std": 401.0374755859375, + "step": 493 + }, + { + "KL/chosen_KL_mean": -415.2734375, + "KL/mean": -600.3134765625, + "KL/rejected_KL_mean": -785.353515625, + "KL/std": 375.8345642089844, + "epoch": 0.7254038179148311, + "fcm_dpo/beta": 0.001385992392897606, + "fcm_dpo/delta": -0.11901067197322845, + "fcm_dpo/margin": 370.080078125, + "fcm_dpo/q_t": 0.38057684898376465, + "grad_norm": 46.4956169128418, + "learning_rate": 1.0765901824467166e-07, + "logits/chosen": -0.7722222805023193, + "logits/rejected": -0.8065148591995239, + "logps/chosen": -456.6326904296875, + "logps/ref_chosen": -41.35926818847656, + "logps/ref_rejected": -86.09136962890625, + "logps/rejected": -871.4448852539062, + "loss": 0.9966, + "margin_dpo/margin_mean": 370.0801086425781, + "margin_dpo/margin_std": 362.3285827636719, + "step": 494 + }, + { + "KL/chosen_KL_mean": -454.084228515625, + "KL/mean": -611.7135009765625, + "KL/rejected_KL_mean": -769.3427124023438, + "KL/std": 360.3024597167969, + "epoch": 0.7268722466960352, + "fcm_dpo/beta": 0.0013737970730289817, + "fcm_dpo/delta": -0.03465069457888603, + "fcm_dpo/margin": 315.25848388671875, + "fcm_dpo/q_t": 0.40194958448410034, + "grad_norm": 42.17742919921875, + "learning_rate": 1.0660589091223854e-07, + "logits/chosen": -0.8575547933578491, + "logits/rejected": -0.8662198781967163, + "logps/chosen": -517.6192626953125, + "logps/ref_chosen": -63.53507995605469, + "logps/ref_rejected": -91.42443084716797, + "logps/rejected": -860.76708984375, + "loss": 1.0748, + "margin_dpo/margin_mean": 315.25848388671875, + "margin_dpo/margin_std": 420.1734619140625, + "step": 495 + }, + { + "KL/chosen_KL_mean": -543.9061279296875, + "KL/mean": -646.899658203125, + "KL/rejected_KL_mean": -749.89306640625, + "KL/std": 301.260498046875, + "epoch": 0.7283406754772394, + "fcm_dpo/beta": 0.0013906970852985978, + "fcm_dpo/delta": 0.11696639657020569, + "fcm_dpo/margin": 205.9869842529297, + "fcm_dpo/q_t": 0.43300455808639526, + "grad_norm": 38.68362808227539, + "learning_rate": 1.0555654214793722e-07, + "logits/chosen": -0.8373208045959473, + "logits/rejected": -0.8159177303314209, + "logps/chosen": -616.498046875, + "logps/ref_chosen": -72.5919189453125, + "logps/ref_rejected": -84.32933807373047, + "logps/rejected": -834.222412109375, + "loss": 1.1732, + "margin_dpo/margin_mean": 205.9869842529297, + "margin_dpo/margin_std": 331.5146484375, + "step": 496 + }, + { + "KL/chosen_KL_mean": -514.3320922851562, + "KL/mean": -602.7001953125, + "KL/rejected_KL_mean": -691.068359375, + "KL/std": 345.37384033203125, + "epoch": 0.7298091042584435, + "fcm_dpo/beta": 0.0014234920963644981, + "fcm_dpo/delta": 0.15240351855754852, + "fcm_dpo/margin": 176.73617553710938, + "fcm_dpo/q_t": 0.44170081615448, + "grad_norm": 47.40353012084961, + "learning_rate": 1.0451099960308374e-07, + "logits/chosen": -0.7741077542304993, + "logits/rejected": -0.7563482522964478, + "logps/chosen": -572.9260864257812, + "logps/ref_chosen": -58.59397506713867, + "logps/ref_rejected": -76.28836822509766, + "logps/rejected": -767.356689453125, + "loss": 1.2084, + "margin_dpo/margin_mean": 176.73619079589844, + "margin_dpo/margin_std": 327.4029541015625, + "step": 497 + }, + { + "KL/chosen_KL_mean": -505.73297119140625, + "KL/mean": -641.7274169921875, + "KL/rejected_KL_mean": -777.7219848632812, + "KL/std": 385.7830810546875, + "epoch": 0.7312775330396476, + "fcm_dpo/beta": 0.0014440558152273297, + "fcm_dpo/delta": 0.007523723877966404, + "fcm_dpo/margin": 271.9889831542969, + "fcm_dpo/q_t": 0.4101245403289795, + "grad_norm": 38.1798210144043, + "learning_rate": 1.0346929082869641e-07, + "logits/chosen": -0.8347723484039307, + "logits/rejected": -0.8251057267189026, + "logps/chosen": -576.9386596679688, + "logps/ref_chosen": -71.20565795898438, + "logps/ref_rejected": -83.95803833007812, + "logps/rejected": -861.6800537109375, + "loss": 1.1289, + "margin_dpo/margin_mean": 271.98895263671875, + "margin_dpo/margin_std": 450.01959228515625, + "step": 498 + }, + { + "KL/chosen_KL_mean": -444.7691345214844, + "KL/mean": -620.37109375, + "KL/rejected_KL_mean": -795.97314453125, + "KL/std": 380.4271240234375, + "epoch": 0.7327459618208517, + "fcm_dpo/beta": 0.00142248859629035, + "fcm_dpo/delta": -0.10488341003656387, + "fcm_dpo/margin": 351.2040710449219, + "fcm_dpo/q_t": 0.3861340880393982, + "grad_norm": 28.65833854675293, + "learning_rate": 1.0243144327477013e-07, + "logits/chosen": -0.7777875065803528, + "logits/rejected": -0.8081082105636597, + "logps/chosen": -496.0243225097656, + "logps/ref_chosen": -51.25519561767578, + "logps/ref_rejected": -101.07870483398438, + "logps/rejected": -897.0518798828125, + "loss": 1.0342, + "margin_dpo/margin_mean": 351.20404052734375, + "margin_dpo/margin_std": 435.44036865234375, + "step": 499 + }, + { + "KL/chosen_KL_mean": -509.4615478515625, + "KL/mean": -653.1812744140625, + "KL/rejected_KL_mean": -796.90087890625, + "KL/std": 342.46307373046875, + "epoch": 0.7342143906020558, + "fcm_dpo/beta": 0.0014125257730484009, + "fcm_dpo/delta": -0.006309333257377148, + "fcm_dpo/margin": 287.4393310546875, + "fcm_dpo/q_t": 0.4078175127506256, + "grad_norm": 32.10559844970703, + "learning_rate": 1.0139748428955333e-07, + "logits/chosen": -0.756510853767395, + "logits/rejected": -0.7809162735939026, + "logps/chosen": -566.489013671875, + "logps/ref_chosen": -57.027442932128906, + "logps/ref_rejected": -93.93421173095703, + "logps/rejected": -890.8350830078125, + "loss": 1.1072, + "margin_dpo/margin_mean": 287.4393310546875, + "margin_dpo/margin_std": 430.96832275390625, + "step": 500 + }, + { + "KL/chosen_KL_mean": -445.49664306640625, + "KL/mean": -589.9784545898438, + "KL/rejected_KL_mean": -734.460205078125, + "KL/std": 344.7445068359375, + "epoch": 0.73568281938326, + "fcm_dpo/beta": 0.00141524663195014, + "fcm_dpo/delta": -0.00943204015493393, + "fcm_dpo/margin": 288.9635314941406, + "fcm_dpo/q_t": 0.4075087308883667, + "grad_norm": 35.74781799316406, + "learning_rate": 1.0036744111882672e-07, + "logits/chosen": -0.7470359206199646, + "logits/rejected": -0.7275895476341248, + "logps/chosen": -499.856201171875, + "logps/ref_chosen": -54.359527587890625, + "logps/ref_rejected": -80.15670013427734, + "logps/rejected": -814.616943359375, + "loss": 1.1037, + "margin_dpo/margin_mean": 288.96356201171875, + "margin_dpo/margin_std": 428.626708984375, + "step": 501 + }, + { + "KL/chosen_KL_mean": -402.1201171875, + "KL/mean": -555.928955078125, + "KL/rejected_KL_mean": -709.73779296875, + "KL/std": 308.30999755859375, + "epoch": 0.737151248164464, + "fcm_dpo/beta": 0.0014068656601011753, + "fcm_dpo/delta": -0.034274935722351074, + "fcm_dpo/margin": 307.6176452636719, + "fcm_dpo/q_t": 0.3976163864135742, + "grad_norm": 46.3470573425293, + "learning_rate": 9.934134090518592e-08, + "logits/chosen": -0.7059692144393921, + "logits/rejected": -0.6937296390533447, + "logps/chosen": -469.7206115722656, + "logps/ref_chosen": -67.60050964355469, + "logps/ref_rejected": -82.94876098632812, + "logps/rejected": -792.6865234375, + "loss": 1.0425, + "margin_dpo/margin_mean": 307.6176452636719, + "margin_dpo/margin_std": 298.7598876953125, + "step": 502 + }, + { + "KL/chosen_KL_mean": -417.1700439453125, + "KL/mean": -571.521240234375, + "KL/rejected_KL_mean": -725.8723754882812, + "KL/std": 324.25213623046875, + "epoch": 0.7386196769456681, + "fcm_dpo/beta": 0.001392916077747941, + "fcm_dpo/delta": -0.03143535181879997, + "fcm_dpo/margin": 308.7023620605469, + "fcm_dpo/q_t": 0.4012370705604553, + "grad_norm": 24.095443725585938, + "learning_rate": 9.831921068732571e-08, + "logits/chosen": -0.6967535614967346, + "logits/rejected": -0.6839867830276489, + "logps/chosen": -472.2484130859375, + "logps/ref_chosen": -55.078407287597656, + "logps/ref_rejected": -82.50544738769531, + "logps/rejected": -808.3778076171875, + "loss": 1.0592, + "margin_dpo/margin_mean": 308.7023620605469, + "margin_dpo/margin_std": 360.79278564453125, + "step": 503 + }, + { + "KL/chosen_KL_mean": -498.46856689453125, + "KL/mean": -675.3897705078125, + "KL/rejected_KL_mean": -852.3109130859375, + "KL/std": 413.42071533203125, + "epoch": 0.7400881057268722, + "fcm_dpo/beta": 0.0013741697184741497, + "fcm_dpo/delta": -0.09064232558012009, + "fcm_dpo/margin": 353.84228515625, + "fcm_dpo/q_t": 0.3909027874469757, + "grad_norm": 31.768949508666992, + "learning_rate": 9.730107739932805e-08, + "logits/chosen": -0.7702962160110474, + "logits/rejected": -0.7931787967681885, + "logps/chosen": -558.434326171875, + "logps/ref_chosen": -59.96575164794922, + "logps/ref_rejected": -103.76212310791016, + "logps/rejected": -956.072998046875, + "loss": 1.0515, + "margin_dpo/margin_mean": 353.84228515625, + "margin_dpo/margin_std": 460.11822509765625, + "step": 504 + }, + { + "KL/chosen_KL_mean": -557.1580810546875, + "KL/mean": -650.727783203125, + "KL/rejected_KL_mean": -744.2975463867188, + "KL/std": 392.6376037597656, + "epoch": 0.7415565345080763, + "fcm_dpo/beta": 0.0013989988947287202, + "fcm_dpo/delta": 0.14166060090065002, + "fcm_dpo/margin": 187.13946533203125, + "fcm_dpo/q_t": 0.4404255449771881, + "grad_norm": 34.36621856689453, + "learning_rate": 9.628696786995188e-08, + "logits/chosen": -0.7928653955459595, + "logits/rejected": -0.7725697159767151, + "logps/chosen": -633.31298828125, + "logps/ref_chosen": -76.1549072265625, + "logps/ref_rejected": -88.58537292480469, + "logps/rejected": -832.8829345703125, + "loss": 1.2135, + "margin_dpo/margin_mean": 187.13946533203125, + "margin_dpo/margin_std": 389.143310546875, + "step": 505 + }, + { + "KL/chosen_KL_mean": -444.1624450683594, + "KL/mean": -609.744140625, + "KL/rejected_KL_mean": -775.3258666992188, + "KL/std": 375.8392333984375, + "epoch": 0.7430249632892805, + "fcm_dpo/beta": 0.0013877790188416839, + "fcm_dpo/delta": -0.06312032043933868, + "fcm_dpo/margin": 331.1634521484375, + "fcm_dpo/q_t": 0.3947676420211792, + "grad_norm": 24.712621688842773, + "learning_rate": 9.527690882192635e-08, + "logits/chosen": -0.7426855564117432, + "logits/rejected": -0.7506879568099976, + "logps/chosen": -493.1229248046875, + "logps/ref_chosen": -48.96050262451172, + "logps/ref_rejected": -78.41505432128906, + "logps/rejected": -853.7409057617188, + "loss": 1.0503, + "margin_dpo/margin_mean": 331.1634521484375, + "margin_dpo/margin_std": 399.01446533203125, + "step": 506 + }, + { + "KL/chosen_KL_mean": -518.7902221679688, + "KL/mean": -660.9124755859375, + "KL/rejected_KL_mean": -803.0347900390625, + "KL/std": 432.55389404296875, + "epoch": 0.7444933920704846, + "fcm_dpo/beta": 0.0013897776370868087, + "fcm_dpo/delta": 0.005165234208106995, + "fcm_dpo/margin": 284.24456787109375, + "fcm_dpo/q_t": 0.4138604402542114, + "grad_norm": 36.502052307128906, + "learning_rate": 9.427092687124691e-08, + "logits/chosen": -0.7783904075622559, + "logits/rejected": -0.7834802269935608, + "logps/chosen": -585.5916748046875, + "logps/ref_chosen": -66.80149841308594, + "logps/ref_rejected": -95.37289428710938, + "logps/rejected": -898.4076538085938, + "loss": 1.1169, + "margin_dpo/margin_mean": 284.2445373535156, + "margin_dpo/margin_std": 456.73846435546875, + "step": 507 + }, + { + "KL/chosen_KL_mean": -553.865966796875, + "KL/mean": -671.4600830078125, + "KL/rejected_KL_mean": -789.0540161132812, + "KL/std": 411.0580749511719, + "epoch": 0.7459618208516887, + "fcm_dpo/beta": 0.0014068866148591042, + "fcm_dpo/delta": 0.07140786945819855, + "fcm_dpo/margin": 235.18801879882812, + "fcm_dpo/q_t": 0.4281538724899292, + "grad_norm": 52.258079528808594, + "learning_rate": 9.326904852647344e-08, + "logits/chosen": -0.7359493970870972, + "logits/rejected": -0.7305607795715332, + "logps/chosen": -625.16943359375, + "logps/ref_chosen": -71.303466796875, + "logps/ref_rejected": -95.6275405883789, + "logps/rejected": -884.6815795898438, + "loss": 1.1927, + "margin_dpo/margin_mean": 235.18801879882812, + "margin_dpo/margin_std": 488.09149169921875, + "step": 508 + }, + { + "KL/chosen_KL_mean": -454.72467041015625, + "KL/mean": -576.01513671875, + "KL/rejected_KL_mean": -697.3055419921875, + "KL/std": 319.4141845703125, + "epoch": 0.7474302496328928, + "fcm_dpo/beta": 0.0014271587133407593, + "fcm_dpo/delta": 0.05532916262745857, + "fcm_dpo/margin": 242.58090209960938, + "fcm_dpo/q_t": 0.42185845971107483, + "grad_norm": 53.33353805541992, + "learning_rate": 9.227130018803195e-08, + "logits/chosen": -0.7149254083633423, + "logits/rejected": -0.7081091403961182, + "logps/chosen": -518.5435791015625, + "logps/ref_chosen": -63.81895065307617, + "logps/ref_rejected": -83.25643920898438, + "logps/rejected": -780.56201171875, + "loss": 1.1447, + "margin_dpo/margin_mean": 242.58090209960938, + "margin_dpo/margin_std": 395.3203125, + "step": 509 + }, + { + "KL/chosen_KL_mean": -523.7684326171875, + "KL/mean": -695.6725463867188, + "KL/rejected_KL_mean": -867.57666015625, + "KL/std": 365.83953857421875, + "epoch": 0.748898678414097, + "fcm_dpo/beta": 0.0014092667261138558, + "fcm_dpo/delta": -0.08875143527984619, + "fcm_dpo/margin": 343.80816650390625, + "fcm_dpo/q_t": 0.3868986666202545, + "grad_norm": 34.31364059448242, + "learning_rate": 9.127770814751932e-08, + "logits/chosen": -0.6882549524307251, + "logits/rejected": -0.7085413932800293, + "logps/chosen": -575.6468505859375, + "logps/ref_chosen": -51.878448486328125, + "logps/ref_rejected": -102.7651596069336, + "logps/rejected": -970.341796875, + "loss": 1.0208, + "margin_dpo/margin_mean": 343.80816650390625, + "margin_dpo/margin_std": 363.1875, + "step": 510 + }, + { + "KL/chosen_KL_mean": -496.804443359375, + "KL/mean": -639.4493408203125, + "KL/rejected_KL_mean": -782.0941162109375, + "KL/std": 395.576416015625, + "epoch": 0.750367107195301, + "fcm_dpo/beta": 0.0014029962476342916, + "fcm_dpo/delta": -0.0002991212531924248, + "fcm_dpo/margin": 285.2897644042969, + "fcm_dpo/q_t": 0.40967386960983276, + "grad_norm": 41.153446197509766, + "learning_rate": 9.028829858700973e-08, + "logits/chosen": -0.7291253805160522, + "logits/rejected": -0.7314298152923584, + "logps/chosen": -557.0425415039062, + "logps/ref_chosen": -60.23811721801758, + "logps/ref_rejected": -92.85676574707031, + "logps/rejected": -874.950927734375, + "loss": 1.1254, + "margin_dpo/margin_mean": 285.2897644042969, + "margin_dpo/margin_std": 475.3314208984375, + "step": 511 + }, + { + "KL/chosen_KL_mean": -421.3726806640625, + "KL/mean": -607.905517578125, + "KL/rejected_KL_mean": -794.4383544921875, + "KL/std": 373.86981201171875, + "epoch": 0.7518355359765051, + "fcm_dpo/beta": 0.001376358326524496, + "fcm_dpo/delta": -0.11963961273431778, + "fcm_dpo/margin": 373.065673828125, + "fcm_dpo/q_t": 0.3809126019477844, + "grad_norm": 37.31480026245117, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": -0.7321999073028564, + "logits/rejected": -0.746883749961853, + "logps/chosen": -476.2781677246094, + "logps/ref_chosen": -54.905494689941406, + "logps/ref_rejected": -81.87586975097656, + "logps/rejected": -876.314208984375, + "loss": 0.9998, + "margin_dpo/margin_mean": 373.065673828125, + "margin_dpo/margin_std": 378.77947998046875, + "step": 512 + }, + { + "KL/chosen_KL_mean": -525.3746337890625, + "KL/mean": -649.6375732421875, + "KL/rejected_KL_mean": -773.9005737304688, + "KL/std": 345.8955078125, + "epoch": 0.7533039647577092, + "fcm_dpo/beta": 0.001371821272186935, + "fcm_dpo/delta": 0.060696233063936234, + "fcm_dpo/margin": 248.52593994140625, + "fcm_dpo/q_t": 0.4223766028881073, + "grad_norm": 43.97354507446289, + "learning_rate": 8.832213108254863e-08, + "logits/chosen": -0.78472501039505, + "logits/rejected": -0.7688409090042114, + "logps/chosen": -590.2910766601562, + "logps/ref_chosen": -64.91644287109375, + "logps/ref_rejected": -76.06245422363281, + "logps/rejected": -849.9630126953125, + "loss": 1.1507, + "margin_dpo/margin_mean": 248.52593994140625, + "margin_dpo/margin_std": 403.25299072265625, + "step": 513 + }, + { + "KL/chosen_KL_mean": -545.6099243164062, + "KL/mean": -675.9661865234375, + "KL/rejected_KL_mean": -806.322509765625, + "KL/std": 379.65130615234375, + "epoch": 0.7547723935389133, + "fcm_dpo/beta": 0.0013956009643152356, + "fcm_dpo/delta": 0.0373426228761673, + "fcm_dpo/margin": 260.7126159667969, + "fcm_dpo/q_t": 0.4192820191383362, + "grad_norm": 38.2194709777832, + "learning_rate": 8.734542494893954e-08, + "logits/chosen": -0.7396078705787659, + "logits/rejected": -0.7294738292694092, + "logps/chosen": -619.8394775390625, + "logps/ref_chosen": -74.22957611083984, + "logps/ref_rejected": -78.945556640625, + "logps/rejected": -885.26806640625, + "loss": 1.1391, + "margin_dpo/margin_mean": 260.7126159667969, + "margin_dpo/margin_std": 434.5355224609375, + "step": 514 + }, + { + "KL/chosen_KL_mean": -449.01666259765625, + "KL/mean": -563.1497802734375, + "KL/rejected_KL_mean": -677.2828979492188, + "KL/std": 333.6114196777344, + "epoch": 0.7562408223201175, + "fcm_dpo/beta": 0.0014119221596047282, + "fcm_dpo/delta": 0.08030429482460022, + "fcm_dpo/margin": 228.26622009277344, + "fcm_dpo/q_t": 0.4252746105194092, + "grad_norm": 49.32374954223633, + "learning_rate": 8.637300491465272e-08, + "logits/chosen": -0.7190433740615845, + "logits/rejected": -0.7307946681976318, + "logps/chosen": -499.4182434082031, + "logps/ref_chosen": -50.40156555175781, + "logps/ref_rejected": -87.09774780273438, + "logps/rejected": -764.380615234375, + "loss": 1.159, + "margin_dpo/margin_mean": 228.26620483398438, + "margin_dpo/margin_std": 386.7580261230469, + "step": 515 + }, + { + "KL/chosen_KL_mean": -490.2201843261719, + "KL/mean": -656.5306396484375, + "KL/rejected_KL_mean": -822.841064453125, + "KL/std": 385.257568359375, + "epoch": 0.7577092511013216, + "fcm_dpo/beta": 0.001406691619195044, + "fcm_dpo/delta": -0.07118304073810577, + "fcm_dpo/margin": 332.62091064453125, + "fcm_dpo/q_t": 0.3913043141365051, + "grad_norm": 38.687232971191406, + "learning_rate": 8.540489660386064e-08, + "logits/chosen": -0.7688533067703247, + "logits/rejected": -0.7967864871025085, + "logps/chosen": -554.8697509765625, + "logps/ref_chosen": -64.64956665039062, + "logps/ref_rejected": -111.72237396240234, + "logps/rejected": -934.5634765625, + "loss": 1.0369, + "margin_dpo/margin_mean": 332.62091064453125, + "margin_dpo/margin_std": 372.27069091796875, + "step": 516 + }, + { + "KL/chosen_KL_mean": -515.1275024414062, + "KL/mean": -695.8507080078125, + "KL/rejected_KL_mean": -876.5738525390625, + "KL/std": 418.8572082519531, + "epoch": 0.7591776798825257, + "fcm_dpo/beta": 0.0013741871807724237, + "fcm_dpo/delta": -0.1020156666636467, + "fcm_dpo/margin": 361.4462890625, + "fcm_dpo/q_t": 0.3897242248058319, + "grad_norm": 26.913551330566406, + "learning_rate": 8.444112552711752e-08, + "logits/chosen": -0.7710027694702148, + "logits/rejected": -0.7722501754760742, + "logps/chosen": -576.0410766601562, + "logps/ref_chosen": -60.913551330566406, + "logps/ref_rejected": -89.08308410644531, + "logps/rejected": -965.656982421875, + "loss": 1.0368, + "margin_dpo/margin_mean": 361.4462890625, + "margin_dpo/margin_std": 460.18988037109375, + "step": 517 + }, + { + "KL/chosen_KL_mean": -483.17401123046875, + "KL/mean": -631.18896484375, + "KL/rejected_KL_mean": -779.2039794921875, + "KL/std": 357.8949279785156, + "epoch": 0.7606461086637298, + "fcm_dpo/beta": 0.0013638620730489492, + "fcm_dpo/delta": -0.00417877733707428, + "fcm_dpo/margin": 296.0299072265625, + "fcm_dpo/q_t": 0.4065293073654175, + "grad_norm": 49.986297607421875, + "learning_rate": 8.348171708068747e-08, + "logits/chosen": -0.7640267610549927, + "logits/rejected": -0.7799776792526245, + "logps/chosen": -540.6298828125, + "logps/ref_chosen": -57.45589065551758, + "logps/ref_rejected": -85.31269836425781, + "logps/rejected": -864.5166015625, + "loss": 1.0928, + "margin_dpo/margin_mean": 296.0299072265625, + "margin_dpo/margin_std": 397.06072998046875, + "step": 518 + }, + { + "KL/chosen_KL_mean": -480.3702697753906, + "KL/mean": -592.9776000976562, + "KL/rejected_KL_mean": -705.5848388671875, + "KL/std": 326.01904296875, + "epoch": 0.762114537444934, + "fcm_dpo/beta": 0.0013907022075727582, + "fcm_dpo/delta": 0.08918791264295578, + "fcm_dpo/margin": 225.21456909179688, + "fcm_dpo/q_t": 0.4279451370239258, + "grad_norm": 30.363061904907227, + "learning_rate": 8.25266965458755e-08, + "logits/chosen": -0.7486674189567566, + "logits/rejected": -0.732239305973053, + "logps/chosen": -554.43359375, + "logps/ref_chosen": -74.06331634521484, + "logps/ref_rejected": -104.44416809082031, + "logps/rejected": -810.029052734375, + "loss": 1.1746, + "margin_dpo/margin_mean": 225.21456909179688, + "margin_dpo/margin_std": 407.32513427734375, + "step": 519 + }, + { + "KL/chosen_KL_mean": -511.0517578125, + "KL/mean": -647.2855834960938, + "KL/rejected_KL_mean": -783.5194091796875, + "KL/std": 358.12750244140625, + "epoch": 0.7635829662261381, + "fcm_dpo/beta": 0.0013931184075772762, + "fcm_dpo/delta": 0.021092122420668602, + "fcm_dpo/margin": 272.4676208496094, + "fcm_dpo/q_t": 0.4135820269584656, + "grad_norm": 35.9310302734375, + "learning_rate": 8.15760890883607e-08, + "logits/chosen": -0.7427826523780823, + "logits/rejected": -0.7525472640991211, + "logps/chosen": -581.3515625, + "logps/ref_chosen": -70.2998275756836, + "logps/ref_rejected": -99.98133850097656, + "logps/rejected": -883.500732421875, + "loss": 1.1106, + "margin_dpo/margin_mean": 272.4676208496094, + "margin_dpo/margin_std": 379.3016357421875, + "step": 520 + }, + { + "KL/chosen_KL_mean": -464.24853515625, + "KL/mean": -612.560791015625, + "KL/rejected_KL_mean": -760.8731079101562, + "KL/std": 374.5102844238281, + "epoch": 0.7650513950073421, + "fcm_dpo/beta": 0.0014033226761966944, + "fcm_dpo/delta": -0.017528019845485687, + "fcm_dpo/margin": 296.62457275390625, + "fcm_dpo/q_t": 0.404031366109848, + "grad_norm": 31.091215133666992, + "learning_rate": 8.062991975753378e-08, + "logits/chosen": -0.774972677230835, + "logits/rejected": -0.7772347331047058, + "logps/chosen": -522.3914794921875, + "logps/ref_chosen": -58.14292526245117, + "logps/ref_rejected": -83.28060913085938, + "logps/rejected": -844.1536865234375, + "loss": 1.0795, + "margin_dpo/margin_mean": 296.62457275390625, + "margin_dpo/margin_std": 372.91729736328125, + "step": 521 + }, + { + "KL/chosen_KL_mean": -514.0162353515625, + "KL/mean": -652.778564453125, + "KL/rejected_KL_mean": -791.5409545898438, + "KL/std": 397.7562255859375, + "epoch": 0.7665198237885462, + "fcm_dpo/beta": 0.0013970948057249188, + "fcm_dpo/delta": 0.01276197750121355, + "fcm_dpo/margin": 277.5247497558594, + "fcm_dpo/q_t": 0.4113382399082184, + "grad_norm": 36.46541976928711, + "learning_rate": 7.968821348583643e-08, + "logits/chosen": -0.7818618416786194, + "logits/rejected": -0.7891424894332886, + "logps/chosen": -560.5638427734375, + "logps/ref_chosen": -46.54766845703125, + "logps/ref_rejected": -66.01388549804688, + "logps/rejected": -857.5548095703125, + "loss": 1.1174, + "margin_dpo/margin_mean": 277.5247497558594, + "margin_dpo/margin_std": 421.7791748046875, + "step": 522 + }, + { + "KL/chosen_KL_mean": -539.9659423828125, + "KL/mean": -685.8133544921875, + "KL/rejected_KL_mean": -831.66064453125, + "KL/std": 432.6346435546875, + "epoch": 0.7679882525697503, + "fcm_dpo/beta": 0.0013963168021291494, + "fcm_dpo/delta": -0.00760743347927928, + "fcm_dpo/margin": 291.6947326660156, + "fcm_dpo/q_t": 0.4086528420448303, + "grad_norm": 36.53788757324219, + "learning_rate": 7.875099508810484e-08, + "logits/chosen": -0.8017454147338867, + "logits/rejected": -0.8047584295272827, + "logps/chosen": -601.735595703125, + "logps/ref_chosen": -61.76960372924805, + "logps/ref_rejected": -83.76141357421875, + "logps/rejected": -915.4220581054688, + "loss": 1.1164, + "margin_dpo/margin_mean": 291.6947326660156, + "margin_dpo/margin_std": 465.686279296875, + "step": 523 + }, + { + "KL/chosen_KL_mean": -532.7937622070312, + "KL/mean": -692.2679443359375, + "KL/rejected_KL_mean": -851.7421264648438, + "KL/std": 435.285400390625, + "epoch": 0.7694566813509545, + "fcm_dpo/beta": 0.0013786728959530592, + "fcm_dpo/delta": -0.04249449074268341, + "fcm_dpo/margin": 318.9483337402344, + "fcm_dpo/q_t": 0.398825079202652, + "grad_norm": 43.52704620361328, + "learning_rate": 7.781828926091535e-08, + "logits/chosen": -0.8273106813430786, + "logits/rejected": -0.8241423964500427, + "logps/chosen": -610.8658447265625, + "logps/ref_chosen": -78.0720443725586, + "logps/ref_rejected": -81.30198669433594, + "logps/rejected": -933.0441284179688, + "loss": 1.0843, + "margin_dpo/margin_mean": 318.9483337402344, + "margin_dpo/margin_std": 438.8890380859375, + "step": 524 + }, + { + "KL/chosen_KL_mean": -513.194580078125, + "KL/mean": -706.9784545898438, + "KL/rejected_KL_mean": -900.7623291015625, + "KL/std": 436.58209228515625, + "epoch": 0.7709251101321586, + "fcm_dpo/beta": 0.0013523332308977842, + "fcm_dpo/delta": -0.13166292011737823, + "fcm_dpo/margin": 387.5677490234375, + "fcm_dpo/q_t": 0.3833525776863098, + "grad_norm": 43.69297790527344, + "learning_rate": 7.689012058193384e-08, + "logits/chosen": -0.7880626916885376, + "logits/rejected": -0.8217567205429077, + "logps/chosen": -564.0224609375, + "logps/ref_chosen": -50.827857971191406, + "logps/ref_rejected": -100.05294036865234, + "logps/rejected": -1000.8153076171875, + "loss": 1.0179, + "margin_dpo/margin_mean": 387.5677490234375, + "margin_dpo/margin_std": 468.5898742675781, + "step": 525 + }, + { + "KL/chosen_KL_mean": -560.8143310546875, + "KL/mean": -753.6343994140625, + "KL/rejected_KL_mean": -946.4544067382812, + "KL/std": 424.5549621582031, + "epoch": 0.7723935389133627, + "fcm_dpo/beta": 0.001329383347183466, + "fcm_dpo/delta": -0.11859972029924393, + "fcm_dpo/margin": 385.64013671875, + "fcm_dpo/q_t": 0.3826919198036194, + "grad_norm": 35.98128128051758, + "learning_rate": 7.596651350926836e-08, + "logits/chosen": -0.8110041618347168, + "logits/rejected": -0.8155307769775391, + "logps/chosen": -623.9815673828125, + "logps/ref_chosen": -63.167236328125, + "logps/ref_rejected": -86.30934143066406, + "logps/rejected": -1032.7637939453125, + "loss": 1.029, + "margin_dpo/margin_mean": 385.64013671875, + "margin_dpo/margin_std": 466.6395263671875, + "step": 526 + }, + { + "KL/chosen_KL_mean": -580.950439453125, + "KL/mean": -715.412109375, + "KL/rejected_KL_mean": -849.8738403320312, + "KL/std": 442.2706298828125, + "epoch": 0.7738619676945668, + "fcm_dpo/beta": 0.0013231317279860377, + "fcm_dpo/delta": 0.04583089426159859, + "fcm_dpo/margin": 268.92340087890625, + "fcm_dpo/q_t": 0.4169968366622925, + "grad_norm": 36.08964538574219, + "learning_rate": 7.504749238082414e-08, + "logits/chosen": -0.9514946937561035, + "logits/rejected": -0.9313217997550964, + "logps/chosen": -652.0791015625, + "logps/ref_chosen": -71.12867736816406, + "logps/ref_rejected": -78.3425521850586, + "logps/rejected": -928.2164306640625, + "loss": 1.1218, + "margin_dpo/margin_mean": 268.92340087890625, + "margin_dpo/margin_std": 378.7649230957031, + "step": 527 + }, + { + "KL/chosen_KL_mean": -611.4620361328125, + "KL/mean": -767.3988037109375, + "KL/rejected_KL_mean": -923.3355102539062, + "KL/std": 427.5599365234375, + "epoch": 0.775330396475771, + "fcm_dpo/beta": 0.0013234864454716444, + "fcm_dpo/delta": -0.013384838588535786, + "fcm_dpo/margin": 311.8734130859375, + "fcm_dpo/q_t": 0.40960851311683655, + "grad_norm": 42.53367614746094, + "learning_rate": 7.413308141366254e-08, + "logits/chosen": -0.9003564119338989, + "logits/rejected": -0.8869940042495728, + "logps/chosen": -679.551513671875, + "logps/ref_chosen": -68.0894546508789, + "logps/ref_rejected": -93.91006469726562, + "logps/rejected": -1017.24560546875, + "loss": 1.1211, + "margin_dpo/margin_mean": 311.8734130859375, + "margin_dpo/margin_std": 515.3593139648438, + "step": 528 + }, + { + "KL/chosen_KL_mean": -696.36767578125, + "KL/mean": -806.6844482421875, + "KL/rejected_KL_mean": -917.001220703125, + "KL/std": 397.0294494628906, + "epoch": 0.7767988252569751, + "fcm_dpo/beta": 0.001338808098807931, + "fcm_dpo/delta": 0.10776834189891815, + "fcm_dpo/margin": 220.63357543945312, + "fcm_dpo/q_t": 0.43243926763534546, + "grad_norm": 42.70530700683594, + "learning_rate": 7.322330470336313e-08, + "logits/chosen": -0.9018471240997314, + "logits/rejected": -0.9128869771957397, + "logps/chosen": -751.942626953125, + "logps/ref_chosen": -55.57495880126953, + "logps/ref_rejected": -89.20909118652344, + "logps/rejected": -1006.2103271484375, + "loss": 1.2233, + "margin_dpo/margin_mean": 220.63357543945312, + "margin_dpo/margin_std": 502.29888916015625, + "step": 529 + }, + { + "KL/chosen_KL_mean": -592.1580810546875, + "KL/mean": -767.647705078125, + "KL/rejected_KL_mean": -943.1373291015625, + "KL/std": 471.09625244140625, + "epoch": 0.7782672540381792, + "fcm_dpo/beta": 0.0013396573485806584, + "fcm_dpo/delta": -0.07361838221549988, + "fcm_dpo/margin": 350.97930908203125, + "fcm_dpo/q_t": 0.3977614939212799, + "grad_norm": 49.951263427734375, + "learning_rate": 7.231818622338822e-08, + "logits/chosen": -0.8480117321014404, + "logits/rejected": -0.846229076385498, + "logps/chosen": -639.7594604492188, + "logps/ref_chosen": -47.601417541503906, + "logps/ref_rejected": -87.2845230102539, + "logps/rejected": -1030.421875, + "loss": 1.1034, + "margin_dpo/margin_mean": 350.97930908203125, + "margin_dpo/margin_std": 581.3236694335938, + "step": 530 + }, + { + "KL/chosen_KL_mean": -643.9351806640625, + "KL/mean": -796.6851806640625, + "KL/rejected_KL_mean": -949.4352416992188, + "KL/std": 499.3472900390625, + "epoch": 0.7797356828193832, + "fcm_dpo/beta": 0.0013343794271349907, + "fcm_dpo/delta": -0.008019059896469116, + "fcm_dpo/margin": 305.5000305175781, + "fcm_dpo/q_t": 0.40879255533218384, + "grad_norm": 36.798309326171875, + "learning_rate": 7.141774982445147e-08, + "logits/chosen": -0.9161735773086548, + "logits/rejected": -0.9046072959899902, + "logps/chosen": -699.1812744140625, + "logps/ref_chosen": -55.246063232421875, + "logps/ref_rejected": -70.60598754882812, + "logps/rejected": -1020.041259765625, + "loss": 1.1112, + "margin_dpo/margin_mean": 305.5000305175781, + "margin_dpo/margin_std": 474.65972900390625, + "step": 531 + }, + { + "KL/chosen_KL_mean": -626.92822265625, + "KL/mean": -798.6822509765625, + "KL/rejected_KL_mean": -970.436279296875, + "KL/std": 478.4112548828125, + "epoch": 0.7812041116005873, + "fcm_dpo/beta": 0.0013115124311298132, + "fcm_dpo/delta": -0.054340578615665436, + "fcm_dpo/margin": 343.50811767578125, + "fcm_dpo/q_t": 0.399822860956192, + "grad_norm": 50.558799743652344, + "learning_rate": 7.052201923388953e-08, + "logits/chosen": -0.8500086069107056, + "logits/rejected": -0.8299944400787354, + "logps/chosen": -697.2142333984375, + "logps/ref_chosen": -70.28601837158203, + "logps/ref_rejected": -86.5913314819336, + "logps/rejected": -1057.027587890625, + "loss": 1.1119, + "margin_dpo/margin_mean": 343.50811767578125, + "margin_dpo/margin_std": 558.3848876953125, + "step": 532 + }, + { + "KL/chosen_KL_mean": -598.7301025390625, + "KL/mean": -717.7310791015625, + "KL/rejected_KL_mean": -836.7320556640625, + "KL/std": 411.3270263671875, + "epoch": 0.7826725403817915, + "fcm_dpo/beta": 0.0013432963751256466, + "fcm_dpo/delta": 0.08121401071548462, + "fcm_dpo/margin": 238.00205993652344, + "fcm_dpo/q_t": 0.4286707639694214, + "grad_norm": 53.88816833496094, + "learning_rate": 6.963101805503646e-08, + "logits/chosen": -0.8658875226974487, + "logits/rejected": -0.8455657958984375, + "logps/chosen": -663.585205078125, + "logps/ref_chosen": -64.8551025390625, + "logps/ref_rejected": -76.58805847167969, + "logps/rejected": -913.3201904296875, + "loss": 1.1979, + "margin_dpo/margin_mean": 238.0020751953125, + "margin_dpo/margin_std": 496.88311767578125, + "step": 533 + }, + { + "KL/chosen_KL_mean": -611.169189453125, + "KL/mean": -767.7733154296875, + "KL/rejected_KL_mean": -924.37744140625, + "KL/std": 438.1583557128906, + "epoch": 0.7841409691629956, + "fcm_dpo/beta": 0.0013270191848278046, + "fcm_dpo/delta": -0.01723591610789299, + "fcm_dpo/margin": 313.2082824707031, + "fcm_dpo/q_t": 0.4059499502182007, + "grad_norm": 34.78287124633789, + "learning_rate": 6.874476976660184e-08, + "logits/chosen": -0.843059778213501, + "logits/rejected": -0.8394489288330078, + "logps/chosen": -671.28857421875, + "logps/ref_chosen": -60.119388580322266, + "logps/ref_rejected": -78.54347229003906, + "logps/rejected": -1002.9208984375, + "loss": 1.0967, + "margin_dpo/margin_mean": 313.2082824707031, + "margin_dpo/margin_std": 444.6209716796875, + "step": 534 + }, + { + "KL/chosen_KL_mean": -533.5681762695312, + "KL/mean": -717.9885864257812, + "KL/rejected_KL_mean": -902.408935546875, + "KL/std": 436.5372314453125, + "epoch": 0.7856093979441997, + "fcm_dpo/beta": 0.001325107179582119, + "fcm_dpo/delta": -0.09366312623023987, + "fcm_dpo/margin": 368.8407897949219, + "fcm_dpo/q_t": 0.39030542969703674, + "grad_norm": 29.257259368896484, + "learning_rate": 6.786329772205246e-08, + "logits/chosen": -0.8032083511352539, + "logits/rejected": -0.8018609285354614, + "logps/chosen": -587.8984375, + "logps/ref_chosen": -54.330238342285156, + "logps/ref_rejected": -96.30763244628906, + "logps/rejected": -998.716552734375, + "loss": 1.0469, + "margin_dpo/margin_mean": 368.8408203125, + "margin_dpo/margin_std": 462.23028564453125, + "step": 535 + }, + { + "KL/chosen_KL_mean": -478.49005126953125, + "KL/mean": -697.0470581054688, + "KL/rejected_KL_mean": -915.6041259765625, + "KL/std": 505.50726318359375, + "epoch": 0.7870778267254038, + "fcm_dpo/beta": 0.0012766944710165262, + "fcm_dpo/delta": -0.1674920618534088, + "fcm_dpo/margin": 437.11407470703125, + "fcm_dpo/q_t": 0.38029739260673523, + "grad_norm": 31.428855895996094, + "learning_rate": 6.698662514899638e-08, + "logits/chosen": -0.7796632051467896, + "logits/rejected": -0.8067115545272827, + "logps/chosen": -525.570556640625, + "logps/ref_chosen": -47.08053207397461, + "logps/ref_rejected": -89.09783935546875, + "logps/rejected": -1004.701904296875, + "loss": 1.0216, + "margin_dpo/margin_mean": 437.11407470703125, + "margin_dpo/margin_std": 601.05322265625, + "step": 536 + }, + { + "KL/chosen_KL_mean": -491.5775146484375, + "KL/mean": -637.9295654296875, + "KL/rejected_KL_mean": -784.2815551757812, + "KL/std": 396.8555908203125, + "epoch": 0.788546255506608, + "fcm_dpo/beta": 0.0012780611868947744, + "fcm_dpo/delta": 0.026200681924819946, + "fcm_dpo/margin": 292.7039794921875, + "fcm_dpo/q_t": 0.4136512577533722, + "grad_norm": 37.28866195678711, + "learning_rate": 6.611477514857114e-08, + "logits/chosen": -0.8498191833496094, + "logits/rejected": -0.8356490135192871, + "logps/chosen": -549.3250122070312, + "logps/ref_chosen": -57.747467041015625, + "logps/ref_rejected": -70.43838500976562, + "logps/rejected": -854.719970703125, + "loss": 1.136, + "margin_dpo/margin_mean": 292.7039794921875, + "margin_dpo/margin_std": 474.5274353027344, + "step": 537 + }, + { + "KL/chosen_KL_mean": -613.0432739257812, + "KL/mean": -787.3213500976562, + "KL/rejected_KL_mean": -961.5994262695312, + "KL/std": 445.82562255859375, + "epoch": 0.7900146842878121, + "fcm_dpo/beta": 0.001262559788301587, + "fcm_dpo/delta": -0.04207906872034073, + "fcm_dpo/margin": 348.55621337890625, + "fcm_dpo/q_t": 0.4003813862800598, + "grad_norm": 28.12194061279297, + "learning_rate": 6.524777069483525e-08, + "logits/chosen": -0.8495669364929199, + "logits/rejected": -0.8379867076873779, + "logps/chosen": -679.459228515625, + "logps/ref_chosen": -66.41594696044922, + "logps/ref_rejected": -84.22808837890625, + "logps/rejected": -1045.8275146484375, + "loss": 1.0696, + "margin_dpo/margin_mean": 348.55621337890625, + "margin_dpo/margin_std": 457.8056335449219, + "step": 538 + }, + { + "KL/chosen_KL_mean": -530.21875, + "KL/mean": -680.2120361328125, + "KL/rejected_KL_mean": -830.205322265625, + "KL/std": 353.58636474609375, + "epoch": 0.7914831130690162, + "fcm_dpo/beta": 0.0012676361948251724, + "fcm_dpo/delta": 0.0204551313072443, + "fcm_dpo/margin": 299.98663330078125, + "fcm_dpo/q_t": 0.4122438430786133, + "grad_norm": 25.66387176513672, + "learning_rate": 6.438563463416221e-08, + "logits/chosen": -0.8782637119293213, + "logits/rejected": -0.8664537668228149, + "logps/chosen": -588.7115478515625, + "logps/ref_chosen": -58.492855072021484, + "logps/ref_rejected": -91.85395050048828, + "logps/rejected": -922.059326171875, + "loss": 1.1019, + "margin_dpo/margin_mean": 299.98663330078125, + "margin_dpo/margin_std": 396.3056335449219, + "step": 539 + }, + { + "KL/chosen_KL_mean": -508.0380859375, + "KL/mean": -721.2171630859375, + "KL/rejected_KL_mean": -934.3963623046875, + "KL/std": 451.8458251953125, + "epoch": 0.7929515418502202, + "fcm_dpo/beta": 0.00124639505520463, + "fcm_dpo/delta": -0.13855527341365814, + "fcm_dpo/margin": 426.35821533203125, + "fcm_dpo/q_t": 0.3821779489517212, + "grad_norm": 36.2562370300293, + "learning_rate": 6.352838968463919e-08, + "logits/chosen": -0.8005784749984741, + "logits/rejected": -0.8170750141143799, + "logps/chosen": -571.5206298828125, + "logps/ref_chosen": -63.482513427734375, + "logps/ref_rejected": -116.42999267578125, + "logps/rejected": -1050.8262939453125, + "loss": 1.0202, + "margin_dpo/margin_mean": 426.35821533203125, + "margin_dpo/margin_std": 517.6613159179688, + "step": 540 + }, + { + "KL/chosen_KL_mean": -607.1495361328125, + "KL/mean": -720.770263671875, + "KL/rejected_KL_mean": -834.390869140625, + "KL/std": 394.2872619628906, + "epoch": 0.7944199706314243, + "fcm_dpo/beta": 0.0012464348692446947, + "fcm_dpo/delta": 0.12000001966953278, + "fcm_dpo/margin": 227.2414093017578, + "fcm_dpo/q_t": 0.4358953833580017, + "grad_norm": 39.796974182128906, + "learning_rate": 6.267605843546767e-08, + "logits/chosen": -0.8924692273139954, + "logits/rejected": -0.8837276697158813, + "logps/chosen": -685.4298706054688, + "logps/ref_chosen": -78.28036499023438, + "logps/ref_rejected": -103.273681640625, + "logps/rejected": -937.66455078125, + "loss": 1.2091, + "margin_dpo/margin_mean": 227.24142456054688, + "margin_dpo/margin_std": 467.9242248535156, + "step": 541 + }, + { + "KL/chosen_KL_mean": -537.729248046875, + "KL/mean": -746.4808959960938, + "KL/rejected_KL_mean": -955.2325439453125, + "KL/std": 462.728759765625, + "epoch": 0.7958883994126285, + "fcm_dpo/beta": 0.001229890389367938, + "fcm_dpo/delta": -0.1220366433262825, + "fcm_dpo/margin": 417.50335693359375, + "fcm_dpo/q_t": 0.3854454755783081, + "grad_norm": 39.909820556640625, + "learning_rate": 6.182866334636888e-08, + "logits/chosen": -0.8912371397018433, + "logits/rejected": -0.9178431034088135, + "logps/chosen": -595.2142333984375, + "logps/ref_chosen": -57.48497009277344, + "logps/ref_rejected": -96.47506713867188, + "logps/rejected": -1051.7076416015625, + "loss": 1.0364, + "margin_dpo/margin_mean": 417.5033264160156, + "margin_dpo/margin_std": 530.0671997070312, + "step": 542 + }, + { + "KL/chosen_KL_mean": -606.9930419921875, + "KL/mean": -751.1796264648438, + "KL/rejected_KL_mean": -895.3662109375, + "KL/std": 553.8678588867188, + "epoch": 0.7973568281938326, + "fcm_dpo/beta": 0.0012381270062178373, + "fcm_dpo/delta": 0.04455637186765671, + "fcm_dpo/margin": 288.3731994628906, + "fcm_dpo/q_t": 0.43025535345077515, + "grad_norm": 39.136451721191406, + "learning_rate": 6.098622674699147e-08, + "logits/chosen": -0.8726138472557068, + "logits/rejected": -0.8934823274612427, + "logps/chosen": -667.6105346679688, + "logps/ref_chosen": -60.61750793457031, + "logps/ref_rejected": -105.59896850585938, + "logps/rejected": -1000.9652099609375, + "loss": 1.1918, + "margin_dpo/margin_mean": 288.3731994628906, + "margin_dpo/margin_std": 641.470703125, + "step": 543 + }, + { + "KL/chosen_KL_mean": -598.482177734375, + "KL/mean": -781.916748046875, + "KL/rejected_KL_mean": -965.3511962890625, + "KL/std": 443.2710266113281, + "epoch": 0.7988252569750367, + "fcm_dpo/beta": 0.0012327476870268583, + "fcm_dpo/delta": -0.054760560393333435, + "fcm_dpo/margin": 366.86907958984375, + "fcm_dpo/q_t": 0.397747278213501, + "grad_norm": 33.70466232299805, + "learning_rate": 6.01487708363232e-08, + "logits/chosen": -0.8226778507232666, + "logits/rejected": -0.8345876932144165, + "logps/chosen": -658.12451171875, + "logps/ref_chosen": -59.642303466796875, + "logps/ref_rejected": -100.95469665527344, + "logps/rejected": -1066.305908203125, + "loss": 1.0675, + "margin_dpo/margin_mean": 366.86907958984375, + "margin_dpo/margin_std": 492.79803466796875, + "step": 544 + }, + { + "KL/chosen_KL_mean": -577.6429443359375, + "KL/mean": -770.2427978515625, + "KL/rejected_KL_mean": -962.8426513671875, + "KL/std": 431.33111572265625, + "epoch": 0.8002936857562408, + "fcm_dpo/beta": 0.0012169883120805025, + "fcm_dpo/delta": -0.07215145230293274, + "fcm_dpo/margin": 385.19970703125, + "fcm_dpo/q_t": 0.3939034640789032, + "grad_norm": 37.47612380981445, + "learning_rate": 5.9316317682106294e-08, + "logits/chosen": -0.8058052062988281, + "logits/rejected": -0.8308186531066895, + "logps/chosen": -645.29150390625, + "logps/ref_chosen": -67.64859771728516, + "logps/ref_rejected": -95.90800476074219, + "logps/rejected": -1058.7506103515625, + "loss": 1.0537, + "margin_dpo/margin_mean": 385.19970703125, + "margin_dpo/margin_std": 491.25616455078125, + "step": 545 + }, + { + "KL/chosen_KL_mean": -527.8880004882812, + "KL/mean": -677.0577392578125, + "KL/rejected_KL_mean": -826.2274169921875, + "KL/std": 387.1895751953125, + "epoch": 0.801762114537445, + "fcm_dpo/beta": 0.00121834478341043, + "fcm_dpo/delta": 0.037912800908088684, + "fcm_dpo/margin": 298.3394470214844, + "fcm_dpo/q_t": 0.41466593742370605, + "grad_norm": 26.89870262145996, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": -0.877615213394165, + "logits/rejected": -0.8663809299468994, + "logps/chosen": -578.6322021484375, + "logps/ref_chosen": -50.744232177734375, + "logps/ref_rejected": -81.86622619628906, + "logps/rejected": -908.0936279296875, + "loss": 1.1145, + "margin_dpo/margin_mean": 298.33941650390625, + "margin_dpo/margin_std": 405.95281982421875, + "step": 546 + }, + { + "KL/chosen_KL_mean": -543.017333984375, + "KL/mean": -709.8662109375, + "KL/rejected_KL_mean": -876.715087890625, + "KL/std": 433.509521484375, + "epoch": 0.8032305433186491, + "fcm_dpo/beta": 0.0012220775242894888, + "fcm_dpo/delta": -0.00818348303437233, + "fcm_dpo/margin": 333.69775390625, + "fcm_dpo/q_t": 0.4070436358451843, + "grad_norm": 49.96892166137695, + "learning_rate": 5.7666507254280265e-08, + "logits/chosen": -0.8075991272926331, + "logits/rejected": -0.812584400177002, + "logps/chosen": -616.705078125, + "logps/ref_chosen": -73.6877212524414, + "logps/ref_rejected": -90.76136779785156, + "logps/rejected": -967.4764404296875, + "loss": 1.0976, + "margin_dpo/margin_mean": 333.69775390625, + "margin_dpo/margin_std": 473.7314758300781, + "step": 547 + }, + { + "KL/chosen_KL_mean": -549.0206298828125, + "KL/mean": -712.397705078125, + "KL/rejected_KL_mean": -875.7747802734375, + "KL/std": 449.38262939453125, + "epoch": 0.8046989720998532, + "fcm_dpo/beta": 0.0012183936778455973, + "fcm_dpo/delta": 0.0019378187134861946, + "fcm_dpo/margin": 326.75421142578125, + "fcm_dpo/q_t": 0.41230571269989014, + "grad_norm": 25.078372955322266, + "learning_rate": 5.684919345471029e-08, + "logits/chosen": -0.8877569437026978, + "logits/rejected": -0.8838699460029602, + "logps/chosen": -614.2669677734375, + "logps/ref_chosen": -65.24634552001953, + "logps/ref_rejected": -94.11807250976562, + "logps/rejected": -969.8928833007812, + "loss": 1.1042, + "margin_dpo/margin_mean": 326.75421142578125, + "margin_dpo/margin_std": 487.56951904296875, + "step": 548 + }, + { + "KL/chosen_KL_mean": -589.3922729492188, + "KL/mean": -711.4036254882812, + "KL/rejected_KL_mean": -833.4149780273438, + "KL/std": 379.79595947265625, + "epoch": 0.8061674008810573, + "fcm_dpo/beta": 0.0012427914189174771, + "fcm_dpo/delta": 0.09908977150917053, + "fcm_dpo/margin": 244.022705078125, + "fcm_dpo/q_t": 0.4320880174636841, + "grad_norm": 40.19181442260742, + "learning_rate": 5.603696935852426e-08, + "logits/chosen": -0.8845070600509644, + "logits/rejected": -0.8731452226638794, + "logps/chosen": -638.6046142578125, + "logps/ref_chosen": -49.21235656738281, + "logps/ref_rejected": -73.91031646728516, + "logps/rejected": -907.3253173828125, + "loss": 1.181, + "margin_dpo/margin_mean": 244.02273559570312, + "margin_dpo/margin_std": 453.96075439453125, + "step": 549 + }, + { + "KL/chosen_KL_mean": -575.8084716796875, + "KL/mean": -719.7702026367188, + "KL/rejected_KL_mean": -863.7319946289062, + "KL/std": 412.17730712890625, + "epoch": 0.8076358296622613, + "fcm_dpo/beta": 0.0012514062691479921, + "fcm_dpo/delta": 0.04118921235203743, + "fcm_dpo/margin": 287.9236145019531, + "fcm_dpo/q_t": 0.41750797629356384, + "grad_norm": 40.92441940307617, + "learning_rate": 5.5229856368582376e-08, + "logits/chosen": -0.8064329624176025, + "logits/rejected": -0.8260148763656616, + "logps/chosen": -632.6154174804688, + "logps/ref_chosen": -56.80695343017578, + "logps/ref_rejected": -95.12580871582031, + "logps/rejected": -958.8577880859375, + "loss": 1.1311, + "margin_dpo/margin_mean": 287.923583984375, + "margin_dpo/margin_std": 450.417236328125, + "step": 550 + }, + { + "KL/chosen_KL_mean": -510.1761169433594, + "KL/mean": -744.1221923828125, + "KL/rejected_KL_mean": -978.068359375, + "KL/std": 450.1910400390625, + "epoch": 0.8091042584434655, + "fcm_dpo/beta": 0.001218787394464016, + "fcm_dpo/delta": -0.18103429675102234, + "fcm_dpo/margin": 467.89215087890625, + "fcm_dpo/q_t": 0.3692789375782013, + "grad_norm": 53.24525833129883, + "learning_rate": 5.4427875753062734e-08, + "logits/chosen": -0.7796275615692139, + "logits/rejected": -0.8252531886100769, + "logps/chosen": -569.282470703125, + "logps/ref_chosen": -59.10633087158203, + "logps/ref_rejected": -111.67280578613281, + "logps/rejected": -1089.7412109375, + "loss": 0.9655, + "margin_dpo/margin_mean": 467.89215087890625, + "margin_dpo/margin_std": 460.2682800292969, + "step": 551 + }, + { + "KL/chosen_KL_mean": -502.0711975097656, + "KL/mean": -760.1432495117188, + "KL/rejected_KL_mean": -1018.2152099609375, + "KL/std": 528.348876953125, + "epoch": 0.8105726872246696, + "fcm_dpo/beta": 0.001160782529041171, + "fcm_dpo/delta": -0.21574066579341888, + "fcm_dpo/margin": 516.14404296875, + "fcm_dpo/q_t": 0.36910372972488403, + "grad_norm": 41.11149597167969, + "learning_rate": 5.363104864490034e-08, + "logits/chosen": -0.859273374080658, + "logits/rejected": -0.8876142501831055, + "logps/chosen": -564.42578125, + "logps/ref_chosen": -62.35459899902344, + "logps/ref_rejected": -104.56210327148438, + "logps/rejected": -1122.77734375, + "loss": 0.9781, + "margin_dpo/margin_mean": 516.14404296875, + "margin_dpo/margin_std": 595.195068359375, + "step": 552 + }, + { + "KL/chosen_KL_mean": -559.717529296875, + "KL/mean": -693.601806640625, + "KL/rejected_KL_mean": -827.4859619140625, + "KL/std": 416.41571044921875, + "epoch": 0.8120411160058737, + "fcm_dpo/beta": 0.0011716227745637298, + "fcm_dpo/delta": 0.08916747570037842, + "fcm_dpo/margin": 267.76837158203125, + "fcm_dpo/q_t": 0.43077874183654785, + "grad_norm": 29.42908477783203, + "learning_rate": 5.2839396041230415e-08, + "logits/chosen": -0.8399643898010254, + "logits/rejected": -0.8348550200462341, + "logps/chosen": -627.9763793945312, + "logps/ref_chosen": -68.25881958007812, + "logps/ref_rejected": -98.0971450805664, + "logps/rejected": -925.5831298828125, + "loss": 1.1687, + "margin_dpo/margin_mean": 267.7684020996094, + "margin_dpo/margin_std": 478.6548767089844, + "step": 553 + }, + { + "KL/chosen_KL_mean": -586.1414794921875, + "KL/mean": -774.9417724609375, + "KL/rejected_KL_mean": -963.7420654296875, + "KL/std": 468.91375732421875, + "epoch": 0.8135095447870778, + "fcm_dpo/beta": 0.0011788744013756514, + "fcm_dpo/delta": -0.04776027426123619, + "fcm_dpo/margin": 377.6005859375, + "fcm_dpo/q_t": 0.4016305208206177, + "grad_norm": 67.11528778076172, + "learning_rate": 5.205293880283551e-08, + "logits/chosen": -0.8197340369224548, + "logits/rejected": -0.7931340932846069, + "logps/chosen": -654.089111328125, + "logps/ref_chosen": -67.94767761230469, + "logps/ref_rejected": -89.78272247314453, + "logps/rejected": -1053.5247802734375, + "loss": 1.1021, + "margin_dpo/margin_mean": 377.6005859375, + "margin_dpo/margin_std": 588.55615234375, + "step": 554 + }, + { + "KL/chosen_KL_mean": -597.59375, + "KL/mean": -807.906005859375, + "KL/rejected_KL_mean": -1018.2183227539062, + "KL/std": 496.7644348144531, + "epoch": 0.8149779735682819, + "fcm_dpo/beta": 0.0011544254375621676, + "fcm_dpo/delta": -0.0898696631193161, + "fcm_dpo/margin": 420.6246032714844, + "fcm_dpo/q_t": 0.3962913751602173, + "grad_norm": 28.15028953552246, + "learning_rate": 5.127169765359515e-08, + "logits/chosen": -0.897038459777832, + "logits/rejected": -0.9390174150466919, + "logps/chosen": -650.9241943359375, + "logps/ref_chosen": -53.33049011230469, + "logps/ref_rejected": -108.47937774658203, + "logps/rejected": -1126.69775390625, + "loss": 1.0808, + "margin_dpo/margin_mean": 420.6246032714844, + "margin_dpo/margin_std": 651.3489990234375, + "step": 555 + }, + { + "KL/chosen_KL_mean": -573.87255859375, + "KL/mean": -702.5924072265625, + "KL/rejected_KL_mean": -831.312255859375, + "KL/std": 383.98846435546875, + "epoch": 0.8164464023494861, + "fcm_dpo/beta": 0.001165332505479455, + "fcm_dpo/delta": 0.10310876369476318, + "fcm_dpo/margin": 257.43975830078125, + "fcm_dpo/q_t": 0.4312754273414612, + "grad_norm": 32.8856086730957, + "learning_rate": 5.049569317994012e-08, + "logits/chosen": -0.8695886731147766, + "logits/rejected": -0.8604377508163452, + "logps/chosen": -632.5169677734375, + "logps/ref_chosen": -58.64447021484375, + "logps/ref_rejected": -101.34040832519531, + "logps/rejected": -932.6526489257812, + "loss": 1.1598, + "margin_dpo/margin_mean": 257.43975830078125, + "margin_dpo/margin_std": 401.28082275390625, + "step": 556 + }, + { + "KL/chosen_KL_mean": -607.11181640625, + "KL/mean": -808.7969970703125, + "KL/rejected_KL_mean": -1010.482177734375, + "KL/std": 527.2576904296875, + "epoch": 0.8179148311306902, + "fcm_dpo/beta": 0.0011563836596906185, + "fcm_dpo/delta": -0.07004015892744064, + "fcm_dpo/margin": 403.37042236328125, + "fcm_dpo/q_t": 0.3975653350353241, + "grad_norm": 53.122276306152344, + "learning_rate": 4.9724945830310144e-08, + "logits/chosen": -0.895065426826477, + "logits/rejected": -0.9235693216323853, + "logps/chosen": -674.9524536132812, + "logps/ref_chosen": -67.84066009521484, + "logps/ref_rejected": -109.93965911865234, + "logps/rejected": -1120.421875, + "loss": 1.0825, + "margin_dpo/margin_mean": 403.37042236328125, + "margin_dpo/margin_std": 600.696044921875, + "step": 557 + }, + { + "KL/chosen_KL_mean": -559.891845703125, + "KL/mean": -821.7356567382812, + "KL/rejected_KL_mean": -1083.579345703125, + "KL/std": 499.6360168457031, + "epoch": 0.8193832599118943, + "fcm_dpo/beta": 0.0011204106267541647, + "fcm_dpo/delta": -0.19866636395454407, + "fcm_dpo/margin": 523.6875, + "fcm_dpo/q_t": 0.3648674488067627, + "grad_norm": 35.1843376159668, + "learning_rate": 4.8959475914614554e-08, + "logits/chosen": -0.9096598625183105, + "logits/rejected": -0.92439866065979, + "logps/chosen": -622.2601318359375, + "logps/ref_chosen": -62.36824035644531, + "logps/ref_rejected": -102.16102600097656, + "logps/rejected": -1185.740478515625, + "loss": 0.9752, + "margin_dpo/margin_mean": 523.6875, + "margin_dpo/margin_std": 566.728515625, + "step": 558 + }, + { + "KL/chosen_KL_mean": -622.984130859375, + "KL/mean": -833.32373046875, + "KL/rejected_KL_mean": -1043.6634521484375, + "KL/std": 492.6358642578125, + "epoch": 0.8208516886930984, + "fcm_dpo/beta": 0.001100032590329647, + "fcm_dpo/delta": -0.0657808855175972, + "fcm_dpo/margin": 420.67926025390625, + "fcm_dpo/q_t": 0.39390745759010315, + "grad_norm": 35.2745475769043, + "learning_rate": 4.8199303603697614e-08, + "logits/chosen": -1.016822338104248, + "logits/rejected": -1.0179095268249512, + "logps/chosen": -683.7364501953125, + "logps/ref_chosen": -60.752323150634766, + "logps/ref_rejected": -93.44229125976562, + "logps/rejected": -1137.105712890625, + "loss": 1.0476, + "margin_dpo/margin_mean": 420.6792907714844, + "margin_dpo/margin_std": 506.8309326171875, + "step": 559 + }, + { + "KL/chosen_KL_mean": -593.2108154296875, + "KL/mean": -745.4317626953125, + "KL/rejected_KL_mean": -897.6527099609375, + "KL/std": 441.3681945800781, + "epoch": 0.8223201174743024, + "fcm_dpo/beta": 0.001101277768611908, + "fcm_dpo/delta": 0.0667891800403595, + "fcm_dpo/margin": 304.4419250488281, + "fcm_dpo/q_t": 0.4226078391075134, + "grad_norm": 28.90320587158203, + "learning_rate": 4.7444448928806615e-08, + "logits/chosen": -0.8108816146850586, + "logits/rejected": -0.7898497581481934, + "logps/chosen": -651.314697265625, + "logps/ref_chosen": -58.10382080078125, + "logps/ref_rejected": -79.99122619628906, + "logps/rejected": -977.6439208984375, + "loss": 1.146, + "margin_dpo/margin_mean": 304.4419250488281, + "margin_dpo/margin_std": 474.00811767578125, + "step": 560 + }, + { + "KL/chosen_KL_mean": -665.6651611328125, + "KL/mean": -807.514404296875, + "KL/rejected_KL_mean": -949.363525390625, + "KL/std": 453.2119140625, + "epoch": 0.8237885462555066, + "fcm_dpo/beta": 0.0011282196501269937, + "fcm_dpo/delta": 0.08202661573886871, + "fcm_dpo/margin": 283.6983642578125, + "fcm_dpo/q_t": 0.4258641004562378, + "grad_norm": 38.57765579223633, + "learning_rate": 4.669493178106432e-08, + "logits/chosen": -0.9530788660049438, + "logits/rejected": -0.969383716583252, + "logps/chosen": -716.5780639648438, + "logps/ref_chosen": -50.912879943847656, + "logps/ref_rejected": -99.06856536865234, + "logps/rejected": -1048.43212890625, + "loss": 1.1834, + "margin_dpo/margin_mean": 283.6983947753906, + "margin_dpo/margin_std": 554.9716796875, + "step": 561 + }, + { + "KL/chosen_KL_mean": -642.4440307617188, + "KL/mean": -829.6681518554688, + "KL/rejected_KL_mean": -1016.8922119140625, + "KL/std": 508.0831298828125, + "epoch": 0.8252569750367107, + "fcm_dpo/beta": 0.0011184532195329666, + "fcm_dpo/delta": -0.020561814308166504, + "fcm_dpo/margin": 374.4482727050781, + "fcm_dpo/q_t": 0.4057563841342926, + "grad_norm": 35.47517776489258, + "learning_rate": 4.5950771910944596e-08, + "logits/chosen": -0.8892195224761963, + "logits/rejected": -0.89787757396698, + "logps/chosen": -701.908447265625, + "logps/ref_chosen": -59.46440124511719, + "logps/ref_rejected": -96.54266357421875, + "logps/rejected": -1113.4349365234375, + "loss": 1.0976, + "margin_dpo/margin_mean": 374.4482421875, + "margin_dpo/margin_std": 549.26806640625, + "step": 562 + }, + { + "KL/chosen_KL_mean": -701.1400756835938, + "KL/mean": -842.6939697265625, + "KL/rejected_KL_mean": -984.247802734375, + "KL/std": 542.4169921875, + "epoch": 0.8267254038179148, + "fcm_dpo/beta": 0.00114821782335639, + "fcm_dpo/delta": 0.07472698390483856, + "fcm_dpo/margin": 283.1077575683594, + "fcm_dpo/q_t": 0.4208974540233612, + "grad_norm": 55.25715255737305, + "learning_rate": 4.521198892775202e-08, + "logits/chosen": -0.901836633682251, + "logits/rejected": -0.9079591035842896, + "logps/chosen": -761.748291015625, + "logps/ref_chosen": -60.60819625854492, + "logps/ref_rejected": -94.56770324707031, + "logps/rejected": -1078.8155517578125, + "loss": 1.2174, + "margin_dpo/margin_mean": 283.1077575683594, + "margin_dpo/margin_std": 628.2413330078125, + "step": 563 + }, + { + "KL/chosen_KL_mean": -627.564453125, + "KL/mean": -802.5059814453125, + "KL/rejected_KL_mean": -977.4475708007812, + "KL/std": 454.1446533203125, + "epoch": 0.8281938325991189, + "fcm_dpo/beta": 0.001143982750363648, + "fcm_dpo/delta": -0.0003933161497116089, + "fcm_dpo/margin": 349.8831481933594, + "fcm_dpo/q_t": 0.4085201025009155, + "grad_norm": 34.40708541870117, + "learning_rate": 4.447860229910544e-08, + "logits/chosen": -0.9562081098556519, + "logits/rejected": -0.9461531639099121, + "logps/chosen": -701.832763671875, + "logps/ref_chosen": -74.26837921142578, + "logps/ref_rejected": -93.23818969726562, + "logps/rejected": -1070.685791015625, + "loss": 1.0896, + "margin_dpo/margin_mean": 349.8831481933594, + "margin_dpo/margin_std": 450.19439697265625, + "step": 564 + }, + { + "KL/chosen_KL_mean": -659.664794921875, + "KL/mean": -855.8405151367188, + "KL/rejected_KL_mean": -1052.0162353515625, + "KL/std": 532.7481079101562, + "epoch": 0.8296622613803231, + "fcm_dpo/beta": 0.0011324284132570028, + "fcm_dpo/delta": -0.046417634934186935, + "fcm_dpo/margin": 392.35150146484375, + "fcm_dpo/q_t": 0.40353649854660034, + "grad_norm": 34.46331024169922, + "learning_rate": 4.375063135042445e-08, + "logits/chosen": -0.9307489395141602, + "logits/rejected": -0.9408478736877441, + "logps/chosen": -728.6846923828125, + "logps/ref_chosen": -69.0199203491211, + "logps/ref_rejected": -85.7789306640625, + "logps/rejected": -1137.795166015625, + "loss": 1.1014, + "margin_dpo/margin_mean": 392.35150146484375, + "margin_dpo/margin_std": 624.1721801757812, + "step": 565 + }, + { + "KL/chosen_KL_mean": -628.304931640625, + "KL/mean": -842.6285400390625, + "KL/rejected_KL_mean": -1056.9521484375, + "KL/std": 567.1864624023438, + "epoch": 0.8311306901615272, + "fcm_dpo/beta": 0.0011231814278289676, + "fcm_dpo/delta": -0.08581139892339706, + "fcm_dpo/margin": 428.6472473144531, + "fcm_dpo/q_t": 0.39444002509117126, + "grad_norm": 38.54751205444336, + "learning_rate": 4.3028095264420525e-08, + "logits/chosen": -0.9179797768592834, + "logits/rejected": -0.9470702409744263, + "logps/chosen": -694.8502197265625, + "logps/ref_chosen": -66.5453109741211, + "logps/ref_rejected": -103.86932373046875, + "logps/rejected": -1160.821533203125, + "loss": 1.0874, + "margin_dpo/margin_mean": 428.64727783203125, + "margin_dpo/margin_std": 652.5841064453125, + "step": 566 + }, + { + "KL/chosen_KL_mean": -615.6129760742188, + "KL/mean": -772.991943359375, + "KL/rejected_KL_mean": -930.370849609375, + "KL/std": 386.2548828125, + "epoch": 0.8325991189427313, + "fcm_dpo/beta": 0.0011194807011634111, + "fcm_dpo/delta": 0.049378592520952225, + "fcm_dpo/margin": 314.75799560546875, + "fcm_dpo/q_t": 0.4178185760974884, + "grad_norm": 45.093955993652344, + "learning_rate": 4.231101308059165e-08, + "logits/chosen": -0.9833869934082031, + "logits/rejected": -0.9983804225921631, + "logps/chosen": -668.4712524414062, + "logps/ref_chosen": -52.85829544067383, + "logps/ref_rejected": -85.37095642089844, + "logps/rejected": -1015.7418212890625, + "loss": 1.1226, + "margin_dpo/margin_mean": 314.75799560546875, + "margin_dpo/margin_std": 445.0365905761719, + "step": 567 + }, + { + "KL/chosen_KL_mean": -607.3502197265625, + "KL/mean": -825.4920654296875, + "KL/rejected_KL_mean": -1043.6339111328125, + "KL/std": 465.9037780761719, + "epoch": 0.8340675477239354, + "fcm_dpo/beta": 0.0011085269507020712, + "fcm_dpo/delta": -0.08783602714538574, + "fcm_dpo/margin": 436.28375244140625, + "fcm_dpo/q_t": 0.38784247636795044, + "grad_norm": 32.346004486083984, + "learning_rate": 4.1599403694720145e-08, + "logits/chosen": -0.8905673027038574, + "logits/rejected": -0.9232343435287476, + "logps/chosen": -652.5426025390625, + "logps/ref_chosen": -45.1923828125, + "logps/ref_rejected": -89.09236907958984, + "logps/rejected": -1132.726318359375, + "loss": 1.0244, + "margin_dpo/margin_mean": 436.2837219238281, + "margin_dpo/margin_std": 477.65777587890625, + "step": 568 + }, + { + "KL/chosen_KL_mean": -678.6146850585938, + "KL/mean": -856.0438232421875, + "KL/rejected_KL_mean": -1033.472900390625, + "KL/std": 577.4537353515625, + "epoch": 0.8355359765051396, + "fcm_dpo/beta": 0.0011109868064522743, + "fcm_dpo/delta": 0.005078420042991638, + "fcm_dpo/margin": 354.8581237792969, + "fcm_dpo/q_t": 0.4105854630470276, + "grad_norm": 57.44594955444336, + "learning_rate": 4.089328585837512e-08, + "logits/chosen": -0.9483610391616821, + "logits/rejected": -0.9578467011451721, + "logps/chosen": -742.3352661132812, + "logps/ref_chosen": -63.72056198120117, + "logps/ref_rejected": -79.10325622558594, + "logps/rejected": -1112.576171875, + "loss": 1.1429, + "margin_dpo/margin_mean": 354.8581237792969, + "margin_dpo/margin_std": 614.8703002929688, + "step": 569 + }, + { + "KL/chosen_KL_mean": -638.6702880859375, + "KL/mean": -804.5684814453125, + "KL/rejected_KL_mean": -970.466552734375, + "KL/std": 463.08331298828125, + "epoch": 0.8370044052863436, + "fcm_dpo/beta": 0.0011088717728853226, + "fcm_dpo/delta": 0.03331791236996651, + "fcm_dpo/margin": 331.7962646484375, + "fcm_dpo/q_t": 0.41667434573173523, + "grad_norm": 29.472156524658203, + "learning_rate": 4.019267817841834e-08, + "logits/chosen": -1.0172860622406006, + "logits/rejected": -1.0165293216705322, + "logps/chosen": -700.2848510742188, + "logps/ref_chosen": -61.61454391479492, + "logps/ref_rejected": -82.14186096191406, + "logps/rejected": -1052.6083984375, + "loss": 1.1261, + "margin_dpo/margin_mean": 331.7962646484375, + "margin_dpo/margin_std": 504.40728759765625, + "step": 570 + }, + { + "KL/chosen_KL_mean": -635.20947265625, + "KL/mean": -841.0794067382812, + "KL/rejected_KL_mean": -1046.949462890625, + "KL/std": 483.367431640625, + "epoch": 0.8384728340675477, + "fcm_dpo/beta": 0.0010986864799633622, + "fcm_dpo/delta": -0.05524833872914314, + "fcm_dpo/margin": 411.7398376464844, + "fcm_dpo/q_t": 0.3991958796977997, + "grad_norm": 33.479488372802734, + "learning_rate": 3.9497599116513705e-08, + "logits/chosen": -0.9390131235122681, + "logits/rejected": -0.9548516273498535, + "logps/chosen": -688.2635498046875, + "logps/ref_chosen": -53.05406188964844, + "logps/ref_rejected": -91.33682250976562, + "logps/rejected": -1138.2861328125, + "loss": 1.0845, + "margin_dpo/margin_mean": 411.7398681640625, + "margin_dpo/margin_std": 602.9391479492188, + "step": 571 + }, + { + "KL/chosen_KL_mean": -681.075439453125, + "KL/mean": -884.181884765625, + "KL/rejected_KL_mean": -1087.2882080078125, + "KL/std": 541.6312255859375, + "epoch": 0.8399412628487518, + "fcm_dpo/beta": 0.0010886520612984896, + "fcm_dpo/delta": -0.04448999464511871, + "fcm_dpo/margin": 406.21282958984375, + "fcm_dpo/q_t": 0.40421679615974426, + "grad_norm": 34.047908782958984, + "learning_rate": 3.880806698864086e-08, + "logits/chosen": -0.9380755424499512, + "logits/rejected": -0.9658199548721313, + "logps/chosen": -729.5347290039062, + "logps/ref_chosen": -48.45928955078125, + "logps/ref_rejected": -83.55703735351562, + "logps/rejected": -1170.84521484375, + "loss": 1.1046, + "margin_dpo/margin_mean": 406.21282958984375, + "margin_dpo/margin_std": 657.9480590820312, + "step": 572 + }, + { + "KL/chosen_KL_mean": -669.11083984375, + "KL/mean": -855.2178955078125, + "KL/rejected_KL_mean": -1041.324951171875, + "KL/std": 513.5198974609375, + "epoch": 0.8414096916299559, + "fcm_dpo/beta": 0.001089778495952487, + "fcm_dpo/delta": -0.0058921948075294495, + "fcm_dpo/margin": 372.2140808105469, + "fcm_dpo/q_t": 0.40893957018852234, + "grad_norm": 29.2487850189209, + "learning_rate": 3.812409996461275e-08, + "logits/chosen": -0.9847406148910522, + "logits/rejected": -0.9911180734634399, + "logps/chosen": -720.7334594726562, + "logps/ref_chosen": -51.62262725830078, + "logps/ref_rejected": -85.32499694824219, + "logps/rejected": -1126.64990234375, + "loss": 1.0893, + "margin_dpo/margin_mean": 372.2140808105469, + "margin_dpo/margin_std": 510.26092529296875, + "step": 573 + }, + { + "KL/chosen_KL_mean": -611.013916015625, + "KL/mean": -807.5015869140625, + "KL/rejected_KL_mean": -1003.9892578125, + "KL/std": 465.5011901855469, + "epoch": 0.8428781204111601, + "fcm_dpo/beta": 0.0010860915062949061, + "fcm_dpo/delta": -0.028046101331710815, + "fcm_dpo/margin": 392.9753112792969, + "fcm_dpo/q_t": 0.4033794403076172, + "grad_norm": 43.88747787475586, + "learning_rate": 3.74457160675965e-08, + "logits/chosen": -0.9889034032821655, + "logits/rejected": -1.0169628858566284, + "logps/chosen": -662.0584106445312, + "logps/ref_chosen": -51.04446029663086, + "logps/ref_rejected": -92.80640411376953, + "logps/rejected": -1096.795654296875, + "loss": 1.0841, + "margin_dpo/margin_mean": 392.97528076171875, + "margin_dpo/margin_std": 539.8909912109375, + "step": 574 + }, + { + "KL/chosen_KL_mean": -683.813232421875, + "KL/mean": -864.3245849609375, + "KL/rejected_KL_mean": -1044.8359375, + "KL/std": 464.4603271484375, + "epoch": 0.8443465491923642, + "fcm_dpo/beta": 0.0010744205210357904, + "fcm_dpo/delta": 0.0111289843916893, + "fcm_dpo/margin": 361.0226745605469, + "fcm_dpo/q_t": 0.41186636686325073, + "grad_norm": 33.57511901855469, + "learning_rate": 3.677293317363864e-08, + "logits/chosen": -0.8667001724243164, + "logits/rejected": -0.8740522265434265, + "logps/chosen": -755.6033935546875, + "logps/ref_chosen": -71.7901382446289, + "logps/ref_rejected": -95.38619995117188, + "logps/rejected": -1140.22216796875, + "loss": 1.1383, + "margin_dpo/margin_mean": 361.0226745605469, + "margin_dpo/margin_std": 593.4249267578125, + "step": 575 + }, + { + "KL/chosen_KL_mean": -668.3992919921875, + "KL/mean": -807.94384765625, + "KL/rejected_KL_mean": -947.4883422851562, + "KL/std": 447.33709716796875, + "epoch": 0.8458149779735683, + "fcm_dpo/beta": 0.0010960063664242625, + "fcm_dpo/delta": 0.09715408086776733, + "fcm_dpo/margin": 279.0890808105469, + "fcm_dpo/q_t": 0.430780291557312, + "grad_norm": 42.60184097290039, + "learning_rate": 3.6105769011194224e-08, + "logits/chosen": -0.979659914970398, + "logits/rejected": -1.0043472051620483, + "logps/chosen": -722.6622314453125, + "logps/ref_chosen": -54.262962341308594, + "logps/ref_rejected": -100.75428009033203, + "logps/rejected": -1048.24267578125, + "loss": 1.1827, + "margin_dpo/margin_mean": 279.0890808105469, + "margin_dpo/margin_std": 521.076416015625, + "step": 576 + }, + { + "KL/chosen_KL_mean": -573.982666015625, + "KL/mean": -754.96923828125, + "KL/rejected_KL_mean": -935.955810546875, + "KL/std": 477.551025390625, + "epoch": 0.8472834067547724, + "fcm_dpo/beta": 0.0011036575306206942, + "fcm_dpo/delta": 0.0004506707191467285, + "fcm_dpo/margin": 361.97320556640625, + "fcm_dpo/q_t": 0.4099828004837036, + "grad_norm": 30.019325256347656, + "learning_rate": 3.5444241160659304e-08, + "logits/chosen": -0.9628755450248718, + "logits/rejected": -0.957021951675415, + "logps/chosen": -635.892333984375, + "logps/ref_chosen": -61.909706115722656, + "logps/ref_rejected": -84.07069396972656, + "logps/rejected": -1020.0265502929688, + "loss": 1.1096, + "margin_dpo/margin_mean": 361.97320556640625, + "margin_dpo/margin_std": 514.8892211914062, + "step": 577 + }, + { + "KL/chosen_KL_mean": -574.492431640625, + "KL/mean": -777.5902099609375, + "KL/rejected_KL_mean": -980.6880493164062, + "KL/std": 478.97015380859375, + "epoch": 0.8487518355359766, + "fcm_dpo/beta": 0.001092255231924355, + "fcm_dpo/delta": -0.04655010625720024, + "fcm_dpo/margin": 406.195556640625, + "fcm_dpo/q_t": 0.3978912830352783, + "grad_norm": 43.33347702026367, + "learning_rate": 3.478836705390808e-08, + "logits/chosen": -0.8652215003967285, + "logits/rejected": -0.8970048427581787, + "logps/chosen": -623.7561645507812, + "logps/ref_chosen": -49.26368713378906, + "logps/ref_rejected": -83.4362564086914, + "logps/rejected": -1064.124267578125, + "loss": 1.0536, + "margin_dpo/margin_mean": 406.1955261230469, + "margin_dpo/margin_std": 466.63641357421875, + "step": 578 + }, + { + "KL/chosen_KL_mean": -647.93896484375, + "KL/mean": -775.7979125976562, + "KL/rejected_KL_mean": -903.6568603515625, + "KL/std": 477.6488037109375, + "epoch": 0.8502202643171806, + "fcm_dpo/beta": 0.0011125376913696527, + "fcm_dpo/delta": 0.11905627697706223, + "fcm_dpo/margin": 255.71783447265625, + "fcm_dpo/q_t": 0.43392279744148254, + "grad_norm": 68.53885650634766, + "learning_rate": 3.41381639738331e-08, + "logits/chosen": -0.9105473756790161, + "logits/rejected": -0.9102990031242371, + "logps/chosen": -706.8248291015625, + "logps/ref_chosen": -58.88581848144531, + "logps/ref_rejected": -94.78762817382812, + "logps/rejected": -998.4444580078125, + "loss": 1.2003, + "margin_dpo/margin_mean": 255.7178192138672, + "margin_dpo/margin_std": 512.1516723632812, + "step": 579 + }, + { + "KL/chosen_KL_mean": -494.50726318359375, + "KL/mean": -724.7994384765625, + "KL/rejected_KL_mean": -955.0916137695312, + "KL/std": 545.931640625, + "epoch": 0.8516886930983847, + "fcm_dpo/beta": 0.0010995608754456043, + "fcm_dpo/delta": -0.11274391412734985, + "fcm_dpo/margin": 460.5844421386719, + "fcm_dpo/q_t": 0.3904712200164795, + "grad_norm": 38.69807052612305, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": -0.8189841508865356, + "logits/rejected": -0.8483502268791199, + "logps/chosen": -543.2140502929688, + "logps/ref_chosen": -48.70683670043945, + "logps/ref_rejected": -81.7583999633789, + "logps/rejected": -1036.85009765625, + "loss": 1.043, + "margin_dpo/margin_mean": 460.5844421386719, + "margin_dpo/margin_std": 635.3092041015625, + "step": 580 + }, + { + "KL/chosen_KL_mean": -643.0278930664062, + "KL/mean": -802.6070556640625, + "KL/rejected_KL_mean": -962.1861572265625, + "KL/std": 479.75579833984375, + "epoch": 0.8531571218795888, + "fcm_dpo/beta": 0.001105679664760828, + "fcm_dpo/delta": 0.048796892166137695, + "fcm_dpo/margin": 319.1583557128906, + "fcm_dpo/q_t": 0.4213637113571167, + "grad_norm": 30.005510330200195, + "learning_rate": 3.285483927764726e-08, + "logits/chosen": -0.9855085611343384, + "logits/rejected": -0.9959384202957153, + "logps/chosen": -705.250244140625, + "logps/ref_chosen": -62.22235107421875, + "logps/ref_rejected": -91.73568725585938, + "logps/rejected": -1053.921875, + "loss": 1.1458, + "margin_dpo/margin_mean": 319.15838623046875, + "margin_dpo/margin_std": 545.41748046875, + "step": 581 + }, + { + "KL/chosen_KL_mean": -564.5281982421875, + "KL/mean": -749.2686157226562, + "KL/rejected_KL_mean": -934.009033203125, + "KL/std": 440.322021484375, + "epoch": 0.8546255506607929, + "fcm_dpo/beta": 0.0011159980203956366, + "fcm_dpo/delta": -0.014601297676563263, + "fcm_dpo/margin": 369.480712890625, + "fcm_dpo/q_t": 0.4044383764266968, + "grad_norm": 39.41720962524414, + "learning_rate": 3.222175147833556e-08, + "logits/chosen": -0.955707848072052, + "logits/rejected": -0.9739981889724731, + "logps/chosen": -622.7568969726562, + "logps/ref_chosen": -58.228660583496094, + "logps/ref_rejected": -110.06959533691406, + "logps/rejected": -1044.07861328125, + "loss": 1.093, + "margin_dpo/margin_mean": 369.480712890625, + "margin_dpo/margin_std": 481.0858154296875, + "step": 582 + }, + { + "KL/chosen_KL_mean": -668.479248046875, + "KL/mean": -783.5263671875, + "KL/rejected_KL_mean": -898.5736083984375, + "KL/std": 474.0484619140625, + "epoch": 0.856093979441997, + "fcm_dpo/beta": 0.0011011306196451187, + "fcm_dpo/delta": -0.015219791792333126, + "fcm_dpo/margin": 230.09426879882812, + "fcm_dpo/q_t": 0.44363144040107727, + "grad_norm": 43.34550857543945, + "learning_rate": 3.159440233840763e-08, + "logits/chosen": -0.9127052426338196, + "logits/rejected": -0.9093770980834961, + "logps/chosen": -725.3421630859375, + "logps/ref_chosen": -56.86286163330078, + "logps/ref_rejected": -88.4039306640625, + "logps/rejected": -986.9774780273438, + "loss": 1.25, + "margin_dpo/margin_mean": 230.09423828125, + "margin_dpo/margin_std": 576.1717529296875, + "step": 583 + }, + { + "KL/chosen_KL_mean": -553.2639770507812, + "KL/mean": -785.3903198242188, + "KL/rejected_KL_mean": -1017.5167236328125, + "KL/std": 505.50018310546875, + "epoch": 0.8575624082232012, + "fcm_dpo/beta": 0.0010839344467967749, + "fcm_dpo/delta": -0.10864884406328201, + "fcm_dpo/margin": 464.25274658203125, + "fcm_dpo/q_t": 0.38627010583877563, + "grad_norm": 28.635910034179688, + "learning_rate": 3.0972808389096635e-08, + "logits/chosen": -0.924347996711731, + "logits/rejected": -0.9356412887573242, + "logps/chosen": -610.1646728515625, + "logps/ref_chosen": -56.90068054199219, + "logps/ref_rejected": -97.63606262207031, + "logps/rejected": -1115.15283203125, + "loss": 1.0195, + "margin_dpo/margin_mean": 464.25274658203125, + "margin_dpo/margin_std": 526.7293090820312, + "step": 584 + }, + { + "KL/chosen_KL_mean": -675.3355712890625, + "KL/mean": -866.5830078125, + "KL/rejected_KL_mean": -1057.83056640625, + "KL/std": 549.9703369140625, + "epoch": 0.8590308370044053, + "fcm_dpo/beta": 0.0010719253914430737, + "fcm_dpo/delta": -0.010730547830462456, + "fcm_dpo/margin": 382.4949645996094, + "fcm_dpo/q_t": 0.4081985652446747, + "grad_norm": 31.741275787353516, + "learning_rate": 3.035698600998121e-08, + "logits/chosen": -0.964898943901062, + "logits/rejected": -0.9843629002571106, + "logps/chosen": -736.3095703125, + "logps/ref_chosen": -60.973968505859375, + "logps/ref_rejected": -84.16952514648438, + "logps/rejected": -1142.0, + "loss": 1.1303, + "margin_dpo/margin_mean": 382.4949645996094, + "margin_dpo/margin_std": 656.9327392578125, + "step": 585 + }, + { + "KL/chosen_KL_mean": -676.3819580078125, + "KL/mean": -811.5240478515625, + "KL/rejected_KL_mean": -946.6661376953125, + "KL/std": 476.75653076171875, + "epoch": 0.8604992657856094, + "fcm_dpo/beta": 0.0010906466050073504, + "fcm_dpo/delta": 0.10854475200176239, + "fcm_dpo/margin": 270.2841796875, + "fcm_dpo/q_t": 0.43295344710350037, + "grad_norm": 32.98674774169922, + "learning_rate": 2.974695142855388e-08, + "logits/chosen": -0.9334938526153564, + "logits/rejected": -0.9494249820709229, + "logps/chosen": -733.237548828125, + "logps/ref_chosen": -56.85559844970703, + "logps/ref_rejected": -91.80261993408203, + "logps/rejected": -1038.46875, + "loss": 1.1953, + "margin_dpo/margin_mean": 270.2841796875, + "margin_dpo/margin_std": 539.8758544921875, + "step": 586 + }, + { + "KL/chosen_KL_mean": -490.77911376953125, + "KL/mean": -676.01806640625, + "KL/rejected_KL_mean": -861.257080078125, + "KL/std": 499.88653564453125, + "epoch": 0.8619676945668135, + "fcm_dpo/beta": 0.0011002285173162818, + "fcm_dpo/delta": -0.008045274764299393, + "fcm_dpo/margin": 370.47796630859375, + "fcm_dpo/q_t": 0.40609079599380493, + "grad_norm": 28.055667877197266, + "learning_rate": 2.9142720719793122e-08, + "logits/chosen": -0.9820740222930908, + "logits/rejected": -1.0075342655181885, + "logps/chosen": -535.470703125, + "logps/ref_chosen": -44.69159698486328, + "logps/ref_rejected": -82.62385559082031, + "logps/rejected": -943.8809204101562, + "loss": 1.0937, + "margin_dpo/margin_mean": 370.47796630859375, + "margin_dpo/margin_std": 512.2026977539062, + "step": 587 + }, + { + "KL/chosen_KL_mean": -632.4581298828125, + "KL/mean": -804.46728515625, + "KL/rejected_KL_mean": -976.4764404296875, + "KL/std": 447.35491943359375, + "epoch": 0.8634361233480177, + "fcm_dpo/beta": 0.0010953000746667385, + "fcm_dpo/delta": 0.02369820699095726, + "fcm_dpo/margin": 344.018310546875, + "fcm_dpo/q_t": 0.4127538204193115, + "grad_norm": 31.495437622070312, + "learning_rate": 2.8544309805740018e-08, + "logits/chosen": -0.9647231101989746, + "logits/rejected": -0.9858247637748718, + "logps/chosen": -682.7531127929688, + "logps/ref_chosen": -50.29494857788086, + "logps/ref_rejected": -107.36988067626953, + "logps/rejected": -1083.846435546875, + "loss": 1.1092, + "margin_dpo/margin_mean": 344.018310546875, + "margin_dpo/margin_std": 468.7295227050781, + "step": 588 + }, + { + "KL/chosen_KL_mean": -627.8831787109375, + "KL/mean": -838.1376953125, + "KL/rejected_KL_mean": -1048.392333984375, + "KL/std": 478.62567138671875, + "epoch": 0.8649045521292217, + "fcm_dpo/beta": 0.0010934111196547747, + "fcm_dpo/delta": -0.06262395530939102, + "fcm_dpo/margin": 420.50909423828125, + "fcm_dpo/q_t": 0.3944852352142334, + "grad_norm": 33.55838394165039, + "learning_rate": 2.7951734455078786e-08, + "logits/chosen": -0.92295241355896, + "logits/rejected": -0.9320027828216553, + "logps/chosen": -687.8131103515625, + "logps/ref_chosen": -59.929908752441406, + "logps/ref_rejected": -111.65534973144531, + "logps/rejected": -1160.047607421875, + "loss": 1.0472, + "margin_dpo/margin_mean": 420.50909423828125, + "margin_dpo/margin_std": 499.3512878417969, + "step": 589 + }, + { + "KL/chosen_KL_mean": -557.1012573242188, + "KL/mean": -763.3665771484375, + "KL/rejected_KL_mean": -969.6317749023438, + "KL/std": 488.5511474609375, + "epoch": 0.8663729809104258, + "fcm_dpo/beta": 0.0010802859906107187, + "fcm_dpo/delta": -0.04780504107475281, + "fcm_dpo/margin": 412.5305480957031, + "fcm_dpo/q_t": 0.39945223927497864, + "grad_norm": 30.184764862060547, + "learning_rate": 2.736501028272095e-08, + "logits/chosen": -0.905895471572876, + "logits/rejected": -0.9255493879318237, + "logps/chosen": -612.9110717773438, + "logps/ref_chosen": -55.80979537963867, + "logps/ref_rejected": -106.06282043457031, + "logps/rejected": -1075.694580078125, + "loss": 1.0646, + "margin_dpo/margin_mean": 412.5305480957031, + "margin_dpo/margin_std": 534.6390380859375, + "step": 590 + }, + { + "KL/chosen_KL_mean": -611.14794921875, + "KL/mean": -795.2865600585938, + "KL/rejected_KL_mean": -979.4251708984375, + "KL/std": 462.18768310546875, + "epoch": 0.8678414096916299, + "fcm_dpo/beta": 0.0010774084366858006, + "fcm_dpo/delta": 0.0033245673403143883, + "fcm_dpo/margin": 368.2772216796875, + "fcm_dpo/q_t": 0.4083741009235382, + "grad_norm": 42.21658706665039, + "learning_rate": 2.678415274939408e-08, + "logits/chosen": -0.9570738673210144, + "logits/rejected": -0.9459799528121948, + "logps/chosen": -667.3885498046875, + "logps/ref_chosen": -56.24061965942383, + "logps/ref_rejected": -83.78629302978516, + "logps/rejected": -1063.21142578125, + "loss": 1.1034, + "margin_dpo/margin_mean": 368.2772216796875, + "margin_dpo/margin_std": 524.36181640625, + "step": 591 + }, + { + "KL/chosen_KL_mean": -624.6557006835938, + "KL/mean": -784.8875732421875, + "KL/rejected_KL_mean": -945.1195068359375, + "KL/std": 459.7552490234375, + "epoch": 0.869309838472834, + "fcm_dpo/beta": 0.0010868444805964828, + "fcm_dpo/delta": 0.05359634757041931, + "fcm_dpo/margin": 320.46380615234375, + "fcm_dpo/q_t": 0.42158329486846924, + "grad_norm": 32.69625473022461, + "learning_rate": 2.6209177161234442e-08, + "logits/chosen": -0.9273663759231567, + "logits/rejected": -0.9230743646621704, + "logps/chosen": -672.595947265625, + "logps/ref_chosen": -47.94025421142578, + "logps/ref_rejected": -75.73287963867188, + "logps/rejected": -1020.8524169921875, + "loss": 1.18, + "margin_dpo/margin_mean": 320.46380615234375, + "margin_dpo/margin_std": 626.68798828125, + "step": 592 + }, + { + "KL/chosen_KL_mean": -588.4630737304688, + "KL/mean": -720.7249755859375, + "KL/rejected_KL_mean": -852.9869995117188, + "KL/std": 485.0901794433594, + "epoch": 0.8707782672540382, + "fcm_dpo/beta": 0.0011051694164052606, + "fcm_dpo/delta": 0.11104539781808853, + "fcm_dpo/margin": 264.52392578125, + "fcm_dpo/q_t": 0.4346621036529541, + "grad_norm": 51.25044250488281, + "learning_rate": 2.564009866938349e-08, + "logits/chosen": -0.8214367628097534, + "logits/rejected": -0.8108229637145996, + "logps/chosen": -637.15380859375, + "logps/ref_chosen": -48.690757751464844, + "logps/ref_rejected": -60.90800094604492, + "logps/rejected": -913.89501953125, + "loss": 1.2031, + "margin_dpo/margin_mean": 264.52392578125, + "margin_dpo/margin_std": 547.176513671875, + "step": 593 + }, + { + "KL/chosen_KL_mean": -561.3931884765625, + "KL/mean": -733.5811767578125, + "KL/rejected_KL_mean": -905.7691650390625, + "KL/std": 492.49456787109375, + "epoch": 0.8722466960352423, + "fcm_dpo/beta": 0.001125779584981501, + "fcm_dpo/delta": 0.011429011821746826, + "fcm_dpo/margin": 344.37591552734375, + "fcm_dpo/q_t": 0.41177088022232056, + "grad_norm": 41.104610443115234, + "learning_rate": 2.5076932269588708e-08, + "logits/chosen": -0.9229486584663391, + "logits/rejected": -0.912185788154602, + "logps/chosen": -616.3280639648438, + "logps/ref_chosen": -54.93488693237305, + "logps/ref_rejected": -86.09967803955078, + "logps/rejected": -991.8687744140625, + "loss": 1.1215, + "margin_dpo/margin_mean": 344.37591552734375, + "margin_dpo/margin_std": 514.7716674804688, + "step": 594 + }, + { + "KL/chosen_KL_mean": -565.7555541992188, + "KL/mean": -749.776123046875, + "KL/rejected_KL_mean": -933.7967529296875, + "KL/std": 474.48052978515625, + "epoch": 0.8737151248164464, + "fcm_dpo/beta": 0.0011134643573313951, + "fcm_dpo/delta": -0.010329552926123142, + "fcm_dpo/margin": 368.0412292480469, + "fcm_dpo/q_t": 0.40979164838790894, + "grad_norm": 36.07206344604492, + "learning_rate": 2.451969280180849e-08, + "logits/chosen": -0.9135459065437317, + "logits/rejected": -0.9275071620941162, + "logps/chosen": -615.1759643554688, + "logps/ref_chosen": -49.4204216003418, + "logps/ref_rejected": -80.62731170654297, + "logps/rejected": -1014.424072265625, + "loss": 1.0946, + "margin_dpo/margin_mean": 368.0412292480469, + "margin_dpo/margin_std": 528.9100341796875, + "step": 595 + }, + { + "KL/chosen_KL_mean": -627.2183837890625, + "KL/mean": -771.135498046875, + "KL/rejected_KL_mean": -915.052490234375, + "KL/std": 470.12677001953125, + "epoch": 0.8751835535976505, + "fcm_dpo/beta": 0.0011297892779111862, + "fcm_dpo/delta": 0.07716254889965057, + "fcm_dpo/margin": 287.83416748046875, + "fcm_dpo/q_t": 0.42858174443244934, + "grad_norm": 44.115386962890625, + "learning_rate": 2.396839494982103e-08, + "logits/chosen": -0.9362180233001709, + "logits/rejected": -0.9097167253494263, + "logps/chosen": -687.0100708007812, + "logps/ref_chosen": -59.791683197021484, + "logps/ref_rejected": -80.09111785888672, + "logps/rejected": -995.1436767578125, + "loss": 1.1769, + "margin_dpo/margin_mean": 287.83416748046875, + "margin_dpo/margin_std": 555.0716552734375, + "step": 596 + }, + { + "KL/chosen_KL_mean": -596.168701171875, + "KL/mean": -822.4722900390625, + "KL/rejected_KL_mean": -1048.77587890625, + "KL/std": 528.4112548828125, + "epoch": 0.8766519823788547, + "fcm_dpo/beta": 0.0011037664953619242, + "fcm_dpo/delta": -0.1077902540564537, + "fcm_dpo/margin": 452.60723876953125, + "fcm_dpo/q_t": 0.38863760232925415, + "grad_norm": 33.63905334472656, + "learning_rate": 2.3423053240837514e-08, + "logits/chosen": -0.8626628518104553, + "logits/rejected": -0.9061563611030579, + "logps/chosen": -653.429443359375, + "logps/ref_chosen": -57.26078796386719, + "logps/ref_rejected": -100.6937255859375, + "logps/rejected": -1149.4697265625, + "loss": 1.0451, + "margin_dpo/margin_mean": 452.60723876953125, + "margin_dpo/margin_std": 572.915771484375, + "step": 597 + }, + { + "KL/chosen_KL_mean": -605.6923828125, + "KL/mean": -782.9190673828125, + "KL/rejected_KL_mean": -960.1458740234375, + "KL/std": 463.83331298828125, + "epoch": 0.8781204111600588, + "fcm_dpo/beta": 0.0011007774155586958, + "fcm_dpo/delta": 0.008908487856388092, + "fcm_dpo/margin": 354.4535827636719, + "fcm_dpo/q_t": 0.4098372757434845, + "grad_norm": 47.247886657714844, + "learning_rate": 2.2883682045119062e-08, + "logits/chosen": -0.9317420721054077, + "logits/rejected": -0.9370558261871338, + "logps/chosen": -658.2108154296875, + "logps/ref_chosen": -52.51850509643555, + "logps/ref_rejected": -89.44385528564453, + "logps/rejected": -1049.5897216796875, + "loss": 1.1114, + "margin_dpo/margin_mean": 354.4535827636719, + "margin_dpo/margin_std": 501.76495361328125, + "step": 598 + }, + { + "KL/chosen_KL_mean": -596.0595703125, + "KL/mean": -746.37451171875, + "KL/rejected_KL_mean": -896.6893920898438, + "KL/std": 440.1773681640625, + "epoch": 0.8795888399412628, + "fcm_dpo/beta": 0.0011121116112917662, + "fcm_dpo/delta": 0.06703872978687286, + "fcm_dpo/margin": 300.6299133300781, + "fcm_dpo/q_t": 0.4199250042438507, + "grad_norm": 27.749969482421875, + "learning_rate": 2.2350295575598367e-08, + "logits/chosen": -0.8937709331512451, + "logits/rejected": -0.899101972579956, + "logps/chosen": -645.8621826171875, + "logps/ref_chosen": -49.802677154541016, + "logps/ref_rejected": -82.978515625, + "logps/rejected": -979.6679077148438, + "loss": 1.1383, + "margin_dpo/margin_mean": 300.6298828125, + "margin_dpo/margin_std": 414.0669250488281, + "step": 599 + }, + { + "KL/chosen_KL_mean": -645.6766967773438, + "KL/mean": -785.5516357421875, + "KL/rejected_KL_mean": -925.4264526367188, + "KL/std": 447.0907897949219, + "epoch": 0.8810572687224669, + "fcm_dpo/beta": 0.0011385063407942653, + "fcm_dpo/delta": 0.08424638211727142, + "fcm_dpo/margin": 279.7497863769531, + "fcm_dpo/q_t": 0.42901042103767395, + "grad_norm": 34.18629837036133, + "learning_rate": 2.1822907887504932e-08, + "logits/chosen": -0.9779192209243774, + "logits/rejected": -0.9724453687667847, + "logps/chosen": -712.111572265625, + "logps/ref_chosen": -66.43487548828125, + "logps/ref_rejected": -85.45649719238281, + "logps/rejected": -1010.8829345703125, + "loss": 1.177, + "margin_dpo/margin_mean": 279.74981689453125, + "margin_dpo/margin_std": 529.72802734375, + "step": 600 + }, + { + "KL/chosen_KL_mean": -664.976806640625, + "KL/mean": -846.01318359375, + "KL/rejected_KL_mean": -1027.0494384765625, + "KL/std": 486.04486083984375, + "epoch": 0.882525697503671, + "fcm_dpo/beta": 0.0011400362709537148, + "fcm_dpo/delta": -0.01342380978167057, + "fcm_dpo/margin": 362.07269287109375, + "fcm_dpo/q_t": 0.40398281812667847, + "grad_norm": 33.51792907714844, + "learning_rate": 2.1301532877994742e-08, + "logits/chosen": -0.9247018098831177, + "logits/rejected": -0.9378587603569031, + "logps/chosen": -724.1104736328125, + "logps/ref_chosen": -59.13361358642578, + "logps/ref_rejected": -94.69093322753906, + "logps/rejected": -1121.740478515625, + "loss": 1.0822, + "margin_dpo/margin_mean": 362.07269287109375, + "margin_dpo/margin_std": 467.5895690917969, + "step": 601 + }, + { + "KL/chosen_KL_mean": -456.95404052734375, + "KL/mean": -683.4141845703125, + "KL/rejected_KL_mean": -909.8743896484375, + "KL/std": 451.41448974609375, + "epoch": 0.8839941262848752, + "fcm_dpo/beta": 0.0011287988163530827, + "fcm_dpo/delta": -0.11732130497694016, + "fcm_dpo/margin": 452.92034912109375, + "fcm_dpo/q_t": 0.3834123909473419, + "grad_norm": 54.98399353027344, + "learning_rate": 2.0786184285784298e-08, + "logits/chosen": -0.9682570695877075, + "logits/rejected": -0.9984170198440552, + "logps/chosen": -505.54754638671875, + "logps/ref_chosen": -48.59352111816406, + "logps/ref_rejected": -87.6685562133789, + "logps/rejected": -997.54296875, + "loss": 1.0084, + "margin_dpo/margin_mean": 452.92034912109375, + "margin_dpo/margin_std": 480.7337646484375, + "step": 602 + }, + { + "KL/chosen_KL_mean": -533.0413818359375, + "KL/mean": -735.8339233398438, + "KL/rejected_KL_mean": -938.6265258789062, + "KL/std": 480.28125, + "epoch": 0.8854625550660793, + "fcm_dpo/beta": 0.0011060773395001888, + "fcm_dpo/delta": -0.05091490596532822, + "fcm_dpo/margin": 405.5850830078125, + "fcm_dpo/q_t": 0.400734007358551, + "grad_norm": 38.465694427490234, + "learning_rate": 2.0276875690788204e-08, + "logits/chosen": -0.9273467063903809, + "logits/rejected": -0.9231489896774292, + "logps/chosen": -603.4559936523438, + "logps/ref_chosen": -70.41461944580078, + "logps/ref_rejected": -100.32559967041016, + "logps/rejected": -1038.9521484375, + "loss": 1.0805, + "margin_dpo/margin_mean": 405.5850830078125, + "margin_dpo/margin_std": 583.6143798828125, + "step": 603 + }, + { + "KL/chosen_KL_mean": -568.7514038085938, + "KL/mean": -793.9749755859375, + "KL/rejected_KL_mean": -1019.198486328125, + "KL/std": 485.3682861328125, + "epoch": 0.8869309838472834, + "fcm_dpo/beta": 0.0010875534499064088, + "fcm_dpo/delta": -0.09458193182945251, + "fcm_dpo/margin": 450.44708251953125, + "fcm_dpo/q_t": 0.3903900384902954, + "grad_norm": 36.59914016723633, + "learning_rate": 1.977362051376158e-08, + "logits/chosen": -0.905841052532196, + "logits/rejected": -0.9343771934509277, + "logps/chosen": -615.20947265625, + "logps/ref_chosen": -46.45808029174805, + "logps/ref_rejected": -91.8544921875, + "logps/rejected": -1111.052978515625, + "loss": 1.0455, + "margin_dpo/margin_mean": 450.44708251953125, + "margin_dpo/margin_std": 581.6656494140625, + "step": 604 + }, + { + "KL/chosen_KL_mean": -586.8192749023438, + "KL/mean": -758.067626953125, + "KL/rejected_KL_mean": -929.31591796875, + "KL/std": 455.83184814453125, + "epoch": 0.8883994126284875, + "fcm_dpo/beta": 0.0010850478429347277, + "fcm_dpo/delta": 0.029418762773275375, + "fcm_dpo/margin": 342.49664306640625, + "fcm_dpo/q_t": 0.41720741987228394, + "grad_norm": 39.87853240966797, + "learning_rate": 1.9276432015946446e-08, + "logits/chosen": -0.8960555791854858, + "logits/rejected": -0.910882294178009, + "logps/chosen": -653.068603515625, + "logps/ref_chosen": -66.24933624267578, + "logps/ref_rejected": -102.30496978759766, + "logps/rejected": -1031.620849609375, + "loss": 1.1268, + "margin_dpo/margin_mean": 342.49664306640625, + "margin_dpo/margin_std": 546.2432861328125, + "step": 605 + }, + { + "KL/chosen_KL_mean": -597.602783203125, + "KL/mean": -785.0491333007812, + "KL/rejected_KL_mean": -972.4955444335938, + "KL/std": 471.9700012207031, + "epoch": 0.8898678414096917, + "fcm_dpo/beta": 0.0010922958608716726, + "fcm_dpo/delta": -0.01021907851099968, + "fcm_dpo/margin": 374.89276123046875, + "fcm_dpo/q_t": 0.40733814239501953, + "grad_norm": 32.22350311279297, + "learning_rate": 1.8785323298722093e-08, + "logits/chosen": -0.9396861791610718, + "logits/rejected": -0.9551488161087036, + "logps/chosen": -652.421875, + "logps/ref_chosen": -54.819122314453125, + "logps/ref_rejected": -98.37146759033203, + "logps/rejected": -1070.866943359375, + "loss": 1.0912, + "margin_dpo/margin_mean": 374.8927917480469, + "margin_dpo/margin_std": 513.25732421875, + "step": 606 + }, + { + "KL/chosen_KL_mean": -603.7939453125, + "KL/mean": -754.0906982421875, + "KL/rejected_KL_mean": -904.387451171875, + "KL/std": 452.41448974609375, + "epoch": 0.8913362701908958, + "fcm_dpo/beta": 0.0011022321414202452, + "fcm_dpo/delta": 0.07058005034923553, + "fcm_dpo/margin": 300.59344482421875, + "fcm_dpo/q_t": 0.42460712790489197, + "grad_norm": 27.491031646728516, + "learning_rate": 1.8300307303259904e-08, + "logits/chosen": -0.8635603785514832, + "logits/rejected": -0.8497999310493469, + "logps/chosen": -661.8780517578125, + "logps/ref_chosen": -58.08403778076172, + "logps/ref_rejected": -79.777099609375, + "logps/rejected": -984.16455078125, + "loss": 1.151, + "margin_dpo/margin_mean": 300.5934143066406, + "margin_dpo/margin_std": 491.64447021484375, + "step": 607 + }, + { + "KL/chosen_KL_mean": -552.9517211914062, + "KL/mean": -731.7744750976562, + "KL/rejected_KL_mean": -910.5972900390625, + "KL/std": 413.10968017578125, + "epoch": 0.8928046989720999, + "fcm_dpo/beta": 0.0011053085327148438, + "fcm_dpo/delta": 0.0048129381611943245, + "fcm_dpo/margin": 357.64556884765625, + "fcm_dpo/q_t": 0.40791454911231995, + "grad_norm": 29.21363639831543, + "learning_rate": 1.7821396810182437e-08, + "logits/chosen": -0.9508018493652344, + "logits/rejected": -0.9595135450363159, + "logps/chosen": -610.402587890625, + "logps/ref_chosen": -57.450836181640625, + "logps/ref_rejected": -94.77339172363281, + "logps/rejected": -1005.3706665039062, + "loss": 1.081, + "margin_dpo/margin_mean": 357.64556884765625, + "margin_dpo/margin_std": 424.187255859375, + "step": 608 + }, + { + "KL/chosen_KL_mean": -582.7489013671875, + "KL/mean": -807.50830078125, + "KL/rejected_KL_mean": -1032.267822265625, + "KL/std": 577.1176147460938, + "epoch": 0.8942731277533039, + "fcm_dpo/beta": 0.001089900266379118, + "fcm_dpo/delta": -0.09446786344051361, + "fcm_dpo/margin": 449.5188293457031, + "fcm_dpo/q_t": 0.3957676887512207, + "grad_norm": 31.981115341186523, + "learning_rate": 1.7348604439226617e-08, + "logits/chosen": -1.0054192543029785, + "logits/rejected": -1.0256993770599365, + "logps/chosen": -641.5543212890625, + "logps/ref_chosen": -58.805355072021484, + "logps/ref_rejected": -88.81600952148438, + "logps/rejected": -1121.083740234375, + "loss": 1.0637, + "margin_dpo/margin_mean": 449.5188293457031, + "margin_dpo/margin_std": 674.6402587890625, + "step": 609 + }, + { + "KL/chosen_KL_mean": -577.7659912109375, + "KL/mean": -720.6400146484375, + "KL/rejected_KL_mean": -863.5140380859375, + "KL/std": 434.3399658203125, + "epoch": 0.895741556534508, + "fcm_dpo/beta": 0.0010983757674694061, + "fcm_dpo/delta": 0.08884203433990479, + "fcm_dpo/margin": 285.748046875, + "fcm_dpo/q_t": 0.42750847339630127, + "grad_norm": 40.23204040527344, + "learning_rate": 1.6881942648911074e-08, + "logits/chosen": -0.9262121319770813, + "logits/rejected": -0.904991626739502, + "logps/chosen": -643.4610595703125, + "logps/ref_chosen": -65.69503784179688, + "logps/ref_rejected": -83.40538787841797, + "logps/rejected": -946.91943359375, + "loss": 1.1661, + "margin_dpo/margin_mean": 285.7480773925781, + "margin_dpo/margin_std": 495.6571044921875, + "step": 610 + }, + { + "KL/chosen_KL_mean": -595.0731811523438, + "KL/mean": -832.9767456054688, + "KL/rejected_KL_mean": -1070.88037109375, + "KL/std": 559.13427734375, + "epoch": 0.8972099853157122, + "fcm_dpo/beta": 0.0010830024257302284, + "fcm_dpo/delta": -0.12153010070323944, + "fcm_dpo/margin": 475.8071594238281, + "fcm_dpo/q_t": 0.38699936866760254, + "grad_norm": 34.51700973510742, + "learning_rate": 1.6421423736208e-08, + "logits/chosen": -0.9600176215171814, + "logits/rejected": -1.0014249086380005, + "logps/chosen": -647.672607421875, + "logps/ref_chosen": -52.59946823120117, + "logps/ref_rejected": -86.33099365234375, + "logps/rejected": -1157.2113037109375, + "loss": 1.0378, + "margin_dpo/margin_mean": 475.8071594238281, + "margin_dpo/margin_std": 629.8643188476562, + "step": 611 + }, + { + "KL/chosen_KL_mean": -638.7127075195312, + "KL/mean": -837.2454223632812, + "KL/rejected_KL_mean": -1035.778076171875, + "KL/std": 478.2944030761719, + "epoch": 0.8986784140969163, + "fcm_dpo/beta": 0.0010748032946139574, + "fcm_dpo/delta": -0.028072573244571686, + "fcm_dpo/margin": 397.0654296875, + "fcm_dpo/q_t": 0.4017370641231537, + "grad_norm": 29.581735610961914, + "learning_rate": 1.5967059836219042e-08, + "logits/chosen": -0.9273912310600281, + "logits/rejected": -0.9367831945419312, + "logps/chosen": -698.0364379882812, + "logps/ref_chosen": -59.32372283935547, + "logps/ref_rejected": -88.31239318847656, + "logps/rejected": -1124.0904541015625, + "loss": 1.072, + "margin_dpo/margin_mean": 397.0653991699219, + "margin_dpo/margin_std": 500.4986572265625, + "step": 612 + }, + { + "KL/chosen_KL_mean": -573.99462890625, + "KL/mean": -799.8018798828125, + "KL/rejected_KL_mean": -1025.609130859375, + "KL/std": 517.7454833984375, + "epoch": 0.9001468428781204, + "fcm_dpo/beta": 0.00105574750341475, + "fcm_dpo/delta": -0.0807756707072258, + "fcm_dpo/margin": 451.61456298828125, + "fcm_dpo/q_t": 0.3911280035972595, + "grad_norm": 31.94742774963379, + "learning_rate": 1.551886292185553e-08, + "logits/chosen": -0.9322279691696167, + "logits/rejected": -0.983695387840271, + "logps/chosen": -633.724609375, + "logps/ref_chosen": -59.72996520996094, + "logps/ref_rejected": -105.10752868652344, + "logps/rejected": -1130.7166748046875, + "loss": 1.0376, + "margin_dpo/margin_mean": 451.6145324707031, + "margin_dpo/margin_std": 535.5646362304688, + "step": 613 + }, + { + "KL/chosen_KL_mean": -613.453369140625, + "KL/mean": -825.698974609375, + "KL/rejected_KL_mean": -1037.944580078125, + "KL/std": 496.1231689453125, + "epoch": 0.9016152716593245, + "fcm_dpo/beta": 0.0010477991309016943, + "fcm_dpo/delta": -0.04686400666832924, + "fcm_dpo/margin": 424.4912414550781, + "fcm_dpo/q_t": 0.39960259199142456, + "grad_norm": 43.36309814453125, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": -0.8932386636734009, + "logits/rejected": -0.9586418271064758, + "logps/chosen": -666.392333984375, + "logps/ref_chosen": -52.93898010253906, + "logps/ref_rejected": -104.67938232421875, + "logps/rejected": -1142.6240234375, + "loss": 1.0767, + "margin_dpo/margin_mean": 424.4912414550781, + "margin_dpo/margin_std": 584.0821533203125, + "step": 614 + }, + { + "KL/chosen_KL_mean": -608.7545166015625, + "KL/mean": -797.580078125, + "KL/rejected_KL_mean": -986.405517578125, + "KL/std": 550.36865234375, + "epoch": 0.9030837004405287, + "fcm_dpo/beta": 0.001047707162797451, + "fcm_dpo/delta": 0.0041780248284339905, + "fcm_dpo/margin": 377.65106201171875, + "fcm_dpo/q_t": 0.41096800565719604, + "grad_norm": 25.20032501220703, + "learning_rate": 1.4641017128809801e-08, + "logits/chosen": -0.9183000326156616, + "logits/rejected": -0.9365602731704712, + "logps/chosen": -674.5718383789062, + "logps/ref_chosen": -65.81727600097656, + "logps/ref_rejected": -95.17749786376953, + "logps/rejected": -1081.5830078125, + "loss": 1.1209, + "margin_dpo/margin_mean": 377.65106201171875, + "margin_dpo/margin_std": 608.4381713867188, + "step": 615 + }, + { + "KL/chosen_KL_mean": -699.6544189453125, + "KL/mean": -851.59375, + "KL/rejected_KL_mean": -1003.532958984375, + "KL/std": 449.58392333984375, + "epoch": 0.9045521292217328, + "fcm_dpo/beta": 0.001057529472745955, + "fcm_dpo/delta": 0.08115516602993011, + "fcm_dpo/margin": 303.8785400390625, + "fcm_dpo/q_t": 0.42706164717674255, + "grad_norm": 28.30909538269043, + "learning_rate": 1.4211391382180637e-08, + "logits/chosen": -0.9700459241867065, + "logits/rejected": -0.956931471824646, + "logps/chosen": -764.7872924804688, + "logps/ref_chosen": -65.13285827636719, + "logps/ref_rejected": -74.70050048828125, + "logps/rejected": -1078.2335205078125, + "loss": 1.1614, + "margin_dpo/margin_mean": 303.8785400390625, + "margin_dpo/margin_std": 516.1566772460938, + "step": 616 + }, + { + "KL/chosen_KL_mean": -680.9326171875, + "KL/mean": -784.277587890625, + "KL/rejected_KL_mean": -887.62255859375, + "KL/std": 428.1174621582031, + "epoch": 0.9060205580029369, + "fcm_dpo/beta": 0.0010916695464402437, + "fcm_dpo/delta": 0.1779545247554779, + "fcm_dpo/margin": 206.68988037109375, + "fcm_dpo/q_t": 0.44872158765792847, + "grad_norm": 50.34171676635742, + "learning_rate": 1.378797888467345e-08, + "logits/chosen": -0.8789099454879761, + "logits/rejected": -0.8473949432373047, + "logps/chosen": -743.9381713867188, + "logps/ref_chosen": -63.005550384521484, + "logps/ref_rejected": -64.234130859375, + "logps/rejected": -951.856689453125, + "loss": 1.2433, + "margin_dpo/margin_mean": 206.6898956298828, + "margin_dpo/margin_std": 494.8637390136719, + "step": 617 + }, + { + "KL/chosen_KL_mean": -682.5164184570312, + "KL/mean": -914.16064453125, + "KL/rejected_KL_mean": -1145.804931640625, + "KL/std": 565.1162719726562, + "epoch": 0.9074889867841409, + "fcm_dpo/beta": 0.0010877924505621195, + "fcm_dpo/delta": -0.10951869189739227, + "fcm_dpo/margin": 463.28851318359375, + "fcm_dpo/q_t": 0.39071911573410034, + "grad_norm": 43.230098724365234, + "learning_rate": 1.3370790793601371e-08, + "logits/chosen": -0.9272968769073486, + "logits/rejected": -0.9583449363708496, + "logps/chosen": -749.6177978515625, + "logps/ref_chosen": -67.10134887695312, + "logps/ref_rejected": -92.15340423583984, + "logps/rejected": -1237.9583740234375, + "loss": 1.0792, + "margin_dpo/margin_mean": 463.2885437011719, + "margin_dpo/margin_std": 716.175048828125, + "step": 618 + }, + { + "KL/chosen_KL_mean": -674.689697265625, + "KL/mean": -854.6988525390625, + "KL/rejected_KL_mean": -1034.7080078125, + "KL/std": 527.9037475585938, + "epoch": 0.908957415565345, + "fcm_dpo/beta": 0.0010771185625344515, + "fcm_dpo/delta": 0.012712216936051846, + "fcm_dpo/margin": 360.01837158203125, + "fcm_dpo/q_t": 0.4185749888420105, + "grad_norm": 38.901329040527344, + "learning_rate": 1.2959838102258535e-08, + "logits/chosen": -0.9141817092895508, + "logits/rejected": -0.9271951913833618, + "logps/chosen": -730.6679077148438, + "logps/ref_chosen": -55.978233337402344, + "logps/ref_rejected": -93.1854019165039, + "logps/rejected": -1127.8935546875, + "loss": 1.1562, + "margin_dpo/margin_mean": 360.01837158203125, + "margin_dpo/margin_std": 684.8106079101562, + "step": 619 + }, + { + "KL/chosen_KL_mean": -612.038330078125, + "KL/mean": -780.5623779296875, + "KL/rejected_KL_mean": -949.08642578125, + "KL/std": 464.1794128417969, + "epoch": 0.9104258443465492, + "fcm_dpo/beta": 0.0010800679447129369, + "fcm_dpo/delta": 0.03716272860765457, + "fcm_dpo/margin": 337.04815673828125, + "fcm_dpo/q_t": 0.4180569648742676, + "grad_norm": 30.291711807250977, + "learning_rate": 1.2555131639630567e-08, + "logits/chosen": -0.9410725831985474, + "logits/rejected": -0.9461079835891724, + "logps/chosen": -671.8358154296875, + "logps/ref_chosen": -59.79750061035156, + "logps/ref_rejected": -78.41075134277344, + "logps/rejected": -1027.4971923828125, + "loss": 1.1314, + "margin_dpo/margin_mean": 337.04815673828125, + "margin_dpo/margin_std": 529.4715576171875, + "step": 620 + }, + { + "KL/chosen_KL_mean": -583.8446655273438, + "KL/mean": -841.3868408203125, + "KL/rejected_KL_mean": -1098.928955078125, + "KL/std": 564.9022827148438, + "epoch": 0.9118942731277533, + "fcm_dpo/beta": 0.0010678130201995373, + "fcm_dpo/delta": -0.15856234729290009, + "fcm_dpo/margin": 515.0842895507812, + "fcm_dpo/q_t": 0.3761498034000397, + "grad_norm": 56.53019714355469, + "learning_rate": 1.2156682070109086e-08, + "logits/chosen": -0.9774070382118225, + "logits/rejected": -1.0240281820297241, + "logps/chosen": -637.7784423828125, + "logps/ref_chosen": -53.93375778198242, + "logps/ref_rejected": -88.36951446533203, + "logps/rejected": -1187.298583984375, + "loss": 1.0088, + "margin_dpo/margin_mean": 515.0842895507812, + "margin_dpo/margin_std": 613.7774658203125, + "step": 621 + }, + { + "KL/chosen_KL_mean": -600.4814453125, + "KL/mean": -777.9830322265625, + "KL/rejected_KL_mean": -955.4846801757812, + "KL/std": 432.13836669921875, + "epoch": 0.9133627019089574, + "fcm_dpo/beta": 0.0010518047492951155, + "fcm_dpo/delta": 0.02732599899172783, + "fcm_dpo/margin": 355.00323486328125, + "fcm_dpo/q_t": 0.41628405451774597, + "grad_norm": 30.599348068237305, + "learning_rate": 1.1764499893210878e-08, + "logits/chosen": -0.8828511238098145, + "logits/rejected": -0.8755066394805908, + "logps/chosen": -660.767333984375, + "logps/ref_chosen": -60.28582000732422, + "logps/ref_rejected": -85.51873779296875, + "logps/rejected": -1041.00341796875, + "loss": 1.1217, + "margin_dpo/margin_mean": 355.00323486328125, + "margin_dpo/margin_std": 544.4783935546875, + "step": 622 + }, + { + "KL/chosen_KL_mean": -659.3543701171875, + "KL/mean": -799.5592041015625, + "KL/rejected_KL_mean": -939.7640380859375, + "KL/std": 451.401123046875, + "epoch": 0.9148311306901615, + "fcm_dpo/beta": 0.0010745935142040253, + "fcm_dpo/delta": 0.10171453654766083, + "fcm_dpo/margin": 280.4097900390625, + "fcm_dpo/q_t": 0.43300747871398926, + "grad_norm": 38.104915618896484, + "learning_rate": 1.1378595443300998e-08, + "logits/chosen": -0.9893115758895874, + "logits/rejected": -0.9900562763214111, + "logps/chosen": -723.5112915039062, + "logps/ref_chosen": -64.1569595336914, + "logps/ref_rejected": -85.08304595947266, + "logps/rejected": -1024.84716796875, + "loss": 1.1854, + "margin_dpo/margin_mean": 280.4097900390625, + "margin_dpo/margin_std": 543.878173828125, + "step": 623 + }, + { + "KL/chosen_KL_mean": -652.4624633789062, + "KL/mean": -863.5037841796875, + "KL/rejected_KL_mean": -1074.544921875, + "KL/std": 486.3994445800781, + "epoch": 0.9162995594713657, + "fcm_dpo/beta": 0.0010720172431319952, + "fcm_dpo/delta": -0.05493501201272011, + "fcm_dpo/margin": 422.08258056640625, + "fcm_dpo/q_t": 0.3948482275009155, + "grad_norm": 35.71148681640625, + "learning_rate": 1.0998978889320582e-08, + "logits/chosen": -1.014406442642212, + "logits/rejected": -1.0207338333129883, + "logps/chosen": -724.381103515625, + "logps/ref_chosen": -71.91862487792969, + "logps/ref_rejected": -97.13203430175781, + "logps/rejected": -1171.677001953125, + "loss": 1.0571, + "margin_dpo/margin_mean": 422.0825500488281, + "margin_dpo/margin_std": 520.140380859375, + "step": 624 + }, + { + "KL/chosen_KL_mean": -627.500244140625, + "KL/mean": -859.1607055664062, + "KL/rejected_KL_mean": -1090.8212890625, + "KL/std": 523.5878295898438, + "epoch": 0.9177679882525698, + "fcm_dpo/beta": 0.0010553135070949793, + "fcm_dpo/delta": -0.09341917186975479, + "fcm_dpo/margin": 463.32098388671875, + "fcm_dpo/q_t": 0.3881823420524597, + "grad_norm": 56.8843994140625, + "learning_rate": 1.0625660234518913e-08, + "logits/chosen": -0.9432724714279175, + "logits/rejected": -0.9646409749984741, + "logps/chosen": -685.84228515625, + "logps/ref_chosen": -58.342071533203125, + "logps/ref_rejected": -86.09038543701172, + "logps/rejected": -1176.91162109375, + "loss": 1.0156, + "margin_dpo/margin_mean": 463.32098388671875, + "margin_dpo/margin_std": 491.7691955566406, + "step": 625 + }, + { + "KL/chosen_KL_mean": -746.43310546875, + "KL/mean": -880.2539672851562, + "KL/rejected_KL_mean": -1014.0748291015625, + "KL/std": 577.3873901367188, + "epoch": 0.9192364170337739, + "fcm_dpo/beta": 0.0010682584252208471, + "fcm_dpo/delta": 0.11725667864084244, + "fcm_dpo/margin": 267.6417236328125, + "fcm_dpo/q_t": 0.43391209840774536, + "grad_norm": 35.362770080566406, + "learning_rate": 1.0258649316189721e-08, + "logits/chosen": -0.9193681478500366, + "logits/rejected": -0.9120655059814453, + "logps/chosen": -821.5457153320312, + "logps/ref_chosen": -75.11260986328125, + "logps/ref_rejected": -99.188720703125, + "logps/rejected": -1113.2635498046875, + "loss": 1.2164, + "margin_dpo/margin_mean": 267.6416931152344, + "margin_dpo/margin_std": 581.1749267578125, + "step": 626 + }, + { + "KL/chosen_KL_mean": -532.278564453125, + "KL/mean": -807.0740966796875, + "KL/rejected_KL_mean": -1081.86962890625, + "KL/std": 631.8243408203125, + "epoch": 0.920704845814978, + "fcm_dpo/beta": 0.0010512054432183504, + "fcm_dpo/delta": -0.18833430111408234, + "fcm_dpo/margin": 549.591064453125, + "fcm_dpo/q_t": 0.37993723154067993, + "grad_norm": 28.75301742553711, + "learning_rate": 9.897955805412e-09, + "logits/chosen": -0.8789898753166199, + "logits/rejected": -0.9454772472381592, + "logps/chosen": -580.021728515625, + "logps/ref_chosen": -47.74314880371094, + "logps/ref_rejected": -106.75448608398438, + "logps/rejected": -1188.6240234375, + "loss": 1.0129, + "margin_dpo/margin_mean": 549.591064453125, + "margin_dpo/margin_std": 725.5487060546875, + "step": 627 + }, + { + "KL/chosen_KL_mean": -685.268798828125, + "KL/mean": -895.5054931640625, + "KL/rejected_KL_mean": -1105.7421875, + "KL/std": 521.978515625, + "epoch": 0.922173274596182, + "fcm_dpo/beta": 0.0010267079342156649, + "fcm_dpo/delta": -0.033246103674173355, + "fcm_dpo/margin": 420.47332763671875, + "fcm_dpo/q_t": 0.40259799361228943, + "grad_norm": 31.96737289428711, + "learning_rate": 9.543589206795238e-09, + "logits/chosen": -0.993385910987854, + "logits/rejected": -1.0080006122589111, + "logps/chosen": -745.4517822265625, + "logps/ref_chosen": -60.182945251464844, + "logps/ref_rejected": -101.55467224121094, + "logps/rejected": -1207.296875, + "loss": 1.0827, + "margin_dpo/margin_mean": 420.47332763671875, + "margin_dpo/margin_std": 580.8540649414062, + "step": 628 + }, + { + "KL/chosen_KL_mean": -672.7838134765625, + "KL/mean": -860.986572265625, + "KL/rejected_KL_mean": -1049.1893310546875, + "KL/std": 503.16357421875, + "epoch": 0.9236417033773862, + "fcm_dpo/beta": 0.0010284529998898506, + "fcm_dpo/delta": 0.0134055744856596, + "fcm_dpo/margin": 376.40557861328125, + "fcm_dpo/q_t": 0.41000691056251526, + "grad_norm": 39.398704528808594, + "learning_rate": 9.19555885822887e-09, + "logits/chosen": -1.0148801803588867, + "logits/rejected": -1.0346441268920898, + "logps/chosen": -736.997314453125, + "logps/ref_chosen": -64.21354675292969, + "logps/ref_rejected": -91.65367126464844, + "logps/rejected": -1140.843017578125, + "loss": 1.0958, + "margin_dpo/margin_mean": 376.40557861328125, + "margin_dpo/margin_std": 488.128173828125, + "step": 629 + }, + { + "KL/chosen_KL_mean": -619.2073974609375, + "KL/mean": -723.8460693359375, + "KL/rejected_KL_mean": -828.4847412109375, + "KL/std": 491.6463928222656, + "epoch": 0.9251101321585903, + "fcm_dpo/beta": 0.0010610285680741072, + "fcm_dpo/delta": 0.18122641742229462, + "fcm_dpo/margin": 209.27731323242188, + "fcm_dpo/q_t": 0.4533764719963074, + "grad_norm": 50.54798889160156, + "learning_rate": 8.85387393063622e-09, + "logits/chosen": -0.9331322312355042, + "logits/rejected": -0.9073890447616577, + "logps/chosen": -678.4984130859375, + "logps/ref_chosen": -59.29100036621094, + "logps/ref_rejected": -83.59829711914062, + "logps/rejected": -912.0830078125, + "loss": 1.2625, + "margin_dpo/margin_mean": 209.2772979736328, + "margin_dpo/margin_std": 571.025634765625, + "step": 630 + }, + { + "KL/chosen_KL_mean": -727.552978515625, + "KL/mean": -893.5679931640625, + "KL/rejected_KL_mean": -1059.5828857421875, + "KL/std": 512.9461059570312, + "epoch": 0.9265785609397944, + "fcm_dpo/beta": 0.001076185842975974, + "fcm_dpo/delta": 0.04413389414548874, + "fcm_dpo/margin": 332.0299072265625, + "fcm_dpo/q_t": 0.4186234474182129, + "grad_norm": 44.696895599365234, + "learning_rate": 8.518543427732949e-09, + "logits/chosen": -1.0083227157592773, + "logits/rejected": -1.012683391571045, + "logps/chosen": -787.006591796875, + "logps/ref_chosen": -59.45360565185547, + "logps/ref_rejected": -80.95156860351562, + "logps/rejected": -1140.534423828125, + "loss": 1.1549, + "margin_dpo/margin_mean": 332.0299072265625, + "margin_dpo/margin_std": 589.2799072265625, + "step": 631 + }, + { + "KL/chosen_KL_mean": -637.191162109375, + "KL/mean": -817.064697265625, + "KL/rejected_KL_mean": -996.9381713867188, + "KL/std": 446.905029296875, + "epoch": 0.9280469897209985, + "fcm_dpo/beta": 0.00107399164699018, + "fcm_dpo/delta": 0.013784755021333694, + "fcm_dpo/margin": 359.7469787597656, + "fcm_dpo/q_t": 0.4118819236755371, + "grad_norm": 36.90241241455078, + "learning_rate": 8.189576185789637e-09, + "logits/chosen": -0.9714510440826416, + "logits/rejected": -0.9745583534240723, + "logps/chosen": -698.542724609375, + "logps/ref_chosen": -61.35155487060547, + "logps/ref_rejected": -86.16017150878906, + "logps/rejected": -1083.098388671875, + "loss": 1.1202, + "margin_dpo/margin_mean": 359.7469787597656, + "margin_dpo/margin_std": 549.3491821289062, + "step": 632 + }, + { + "KL/chosen_KL_mean": -677.3255615234375, + "KL/mean": -810.3518676757812, + "KL/rejected_KL_mean": -943.378173828125, + "KL/std": 452.0666198730469, + "epoch": 0.9295154185022027, + "fcm_dpo/beta": 0.001101176138035953, + "fcm_dpo/delta": 0.1097092255949974, + "fcm_dpo/margin": 266.0525817871094, + "fcm_dpo/q_t": 0.4321713149547577, + "grad_norm": 35.17088317871094, + "learning_rate": 7.866980873399015e-09, + "logits/chosen": -1.0107479095458984, + "logits/rejected": -1.019244909286499, + "logps/chosen": -734.603759765625, + "logps/ref_chosen": -57.27816390991211, + "logps/ref_rejected": -91.58395385742188, + "logps/rejected": -1034.962158203125, + "loss": 1.1926, + "margin_dpo/margin_mean": 266.0526123046875, + "margin_dpo/margin_std": 517.71240234375, + "step": 633 + }, + { + "KL/chosen_KL_mean": -777.34423828125, + "KL/mean": -895.5999755859375, + "KL/rejected_KL_mean": -1013.855712890625, + "KL/std": 547.462890625, + "epoch": 0.9309838472834068, + "fcm_dpo/beta": 0.0011245100758969784, + "fcm_dpo/delta": 0.13790102303028107, + "fcm_dpo/margin": 236.51150512695312, + "fcm_dpo/q_t": 0.44136252999305725, + "grad_norm": 34.355796813964844, + "learning_rate": 7.550765991247654e-09, + "logits/chosen": -0.9018381834030151, + "logits/rejected": -0.8940595388412476, + "logps/chosen": -843.9631958007812, + "logps/ref_chosen": -66.61896514892578, + "logps/ref_rejected": -107.12564849853516, + "logps/rejected": -1120.9814453125, + "loss": 1.2218, + "margin_dpo/margin_mean": 236.51150512695312, + "margin_dpo/margin_std": 533.4472045898438, + "step": 634 + }, + { + "KL/chosen_KL_mean": -675.8604736328125, + "KL/mean": -839.7138671875, + "KL/rejected_KL_mean": -1003.567138671875, + "KL/std": 590.4039306640625, + "epoch": 0.9324522760646109, + "fcm_dpo/beta": 0.001137568149715662, + "fcm_dpo/delta": 0.028254780918359756, + "fcm_dpo/margin": 327.7066955566406, + "fcm_dpo/q_t": 0.41930854320526123, + "grad_norm": 34.57207107543945, + "learning_rate": 7.240939871891699e-09, + "logits/chosen": -0.9981896877288818, + "logits/rejected": -0.9815343618392944, + "logps/chosen": -749.8160400390625, + "logps/ref_chosen": -73.95551300048828, + "logps/ref_rejected": -82.50045776367188, + "logps/rejected": -1086.067626953125, + "loss": 1.1431, + "margin_dpo/margin_mean": 327.7066650390625, + "margin_dpo/margin_std": 579.1690063476562, + "step": 635 + }, + { + "KL/chosen_KL_mean": -641.6556396484375, + "KL/mean": -837.667724609375, + "KL/rejected_KL_mean": -1033.679931640625, + "KL/std": 567.2379150390625, + "epoch": 0.933920704845815, + "fcm_dpo/beta": 0.0011395248584449291, + "fcm_dpo/delta": -0.049328967928886414, + "fcm_dpo/margin": 392.0242004394531, + "fcm_dpo/q_t": 0.40262505412101746, + "grad_norm": 27.586217880249023, + "learning_rate": 6.937510679537628e-09, + "logits/chosen": -0.9401626586914062, + "logits/rejected": -0.9458721876144409, + "logps/chosen": -701.2845458984375, + "logps/ref_chosen": -59.628910064697266, + "logps/ref_rejected": -81.97883605957031, + "logps/rejected": -1115.65869140625, + "loss": 1.0791, + "margin_dpo/margin_mean": 392.0242004394531, + "margin_dpo/margin_std": 573.56640625, + "step": 636 + }, + { + "KL/chosen_KL_mean": -650.4024658203125, + "KL/mean": -858.9671630859375, + "KL/rejected_KL_mean": -1067.5318603515625, + "KL/std": 553.26611328125, + "epoch": 0.9353891336270191, + "fcm_dpo/beta": 0.0011137896217405796, + "fcm_dpo/delta": -0.06838998943567276, + "fcm_dpo/margin": 417.1293029785156, + "fcm_dpo/q_t": 0.3966093063354492, + "grad_norm": 26.735336303710938, + "learning_rate": 6.640486409826785e-09, + "logits/chosen": -1.0075198411941528, + "logits/rejected": -1.0491002798080444, + "logps/chosen": -700.05517578125, + "logps/ref_chosen": -49.652687072753906, + "logps/ref_rejected": -98.40513610839844, + "logps/rejected": -1165.93701171875, + "loss": 1.0642, + "margin_dpo/margin_mean": 417.1293029785156, + "margin_dpo/margin_std": 567.79345703125, + "step": 637 + }, + { + "KL/chosen_KL_mean": -648.9947509765625, + "KL/mean": -814.8811645507812, + "KL/rejected_KL_mean": -980.767578125, + "KL/std": 530.7965087890625, + "epoch": 0.9368575624082232, + "fcm_dpo/beta": 0.001130024204030633, + "fcm_dpo/delta": 0.023703396320343018, + "fcm_dpo/margin": 331.7728271484375, + "fcm_dpo/q_t": 0.4090859293937683, + "grad_norm": 37.624141693115234, + "learning_rate": 6.349874889624962e-09, + "logits/chosen": -0.8736113905906677, + "logits/rejected": -0.860742449760437, + "logps/chosen": -707.1513671875, + "logps/ref_chosen": -58.156639099121094, + "logps/ref_rejected": -79.3014907836914, + "logps/rejected": -1060.069091796875, + "loss": 1.1595, + "margin_dpo/margin_mean": 331.7728271484375, + "margin_dpo/margin_std": 607.9287109375, + "step": 638 + }, + { + "KL/chosen_KL_mean": -850.44189453125, + "KL/mean": -925.354736328125, + "KL/rejected_KL_mean": -1000.2677001953125, + "KL/std": 479.6988525390625, + "epoch": 0.9383259911894273, + "fcm_dpo/beta": 0.0011593474773690104, + "fcm_dpo/delta": 0.23026630282402039, + "fcm_dpo/margin": 149.82571411132812, + "fcm_dpo/q_t": 0.46169108152389526, + "grad_norm": 100.88923645019531, + "learning_rate": 6.065683776815933e-09, + "logits/chosen": -0.8598443269729614, + "logits/rejected": -0.8015762567520142, + "logps/chosen": -922.76513671875, + "logps/ref_chosen": -72.32319641113281, + "logps/ref_rejected": -74.2749252319336, + "logps/rejected": -1074.5426025390625, + "loss": 1.3238, + "margin_dpo/margin_mean": 149.8257293701172, + "margin_dpo/margin_std": 569.2753295898438, + "step": 639 + }, + { + "KL/chosen_KL_mean": -669.903076171875, + "KL/mean": -923.6558837890625, + "KL/rejected_KL_mean": -1177.40869140625, + "KL/std": 576.5742797851562, + "epoch": 0.9397944199706314, + "fcm_dpo/beta": 0.0011469844030216336, + "fcm_dpo/delta": -0.1929648518562317, + "fcm_dpo/margin": 507.5057067871094, + "fcm_dpo/q_t": 0.3761150538921356, + "grad_norm": 36.260929107666016, + "learning_rate": 5.7879205600998296e-09, + "logits/chosen": -0.8822273015975952, + "logits/rejected": -0.9096044898033142, + "logps/chosen": -726.037353515625, + "logps/ref_chosen": -56.13436508178711, + "logps/ref_rejected": -108.60014343261719, + "logps/rejected": -1286.0087890625, + "loss": 1.0178, + "margin_dpo/margin_mean": 507.50567626953125, + "margin_dpo/margin_std": 679.51318359375, + "step": 640 + }, + { + "KL/chosen_KL_mean": -733.76953125, + "KL/mean": -893.8585205078125, + "KL/rejected_KL_mean": -1053.947509765625, + "KL/std": 490.9017333984375, + "epoch": 0.9412628487518355, + "fcm_dpo/beta": 0.0011330367997288704, + "fcm_dpo/delta": 0.038639139384031296, + "fcm_dpo/margin": 320.1780700683594, + "fcm_dpo/q_t": 0.4209401607513428, + "grad_norm": 39.58655548095703, + "learning_rate": 5.516592558795746e-09, + "logits/chosen": -0.9786254167556763, + "logits/rejected": -0.9939931035041809, + "logps/chosen": -798.7664184570312, + "logps/ref_chosen": -64.99689483642578, + "logps/ref_rejected": -86.99232482910156, + "logps/rejected": -1140.93994140625, + "loss": 1.1682, + "margin_dpo/margin_mean": 320.1781005859375, + "margin_dpo/margin_std": 625.5211181640625, + "step": 641 + }, + { + "KL/chosen_KL_mean": -726.5020751953125, + "KL/mean": -920.57958984375, + "KL/rejected_KL_mean": -1114.6572265625, + "KL/std": 639.9869384765625, + "epoch": 0.9427312775330396, + "fcm_dpo/beta": 0.0011348032858222723, + "fcm_dpo/delta": -0.04255044087767601, + "fcm_dpo/margin": 388.155029296875, + "fcm_dpo/q_t": 0.41343218088150024, + "grad_norm": 42.09096908569336, + "learning_rate": 5.251706922648868e-09, + "logits/chosen": -0.9162915945053101, + "logits/rejected": -0.9479303359985352, + "logps/chosen": -792.1912841796875, + "logps/ref_chosen": -65.68924713134766, + "logps/ref_rejected": -110.24205017089844, + "logps/rejected": -1224.899169921875, + "loss": 1.1468, + "margin_dpo/margin_mean": 388.1550598144531, + "margin_dpo/margin_std": 787.761474609375, + "step": 642 + }, + { + "KL/chosen_KL_mean": -623.6024780273438, + "KL/mean": -785.3634033203125, + "KL/rejected_KL_mean": -947.1244506835938, + "KL/std": 469.08172607421875, + "epoch": 0.9441997063142438, + "fcm_dpo/beta": 0.0011254868004471064, + "fcm_dpo/delta": 0.0364561453461647, + "fcm_dpo/margin": 323.52197265625, + "fcm_dpo/q_t": 0.41738998889923096, + "grad_norm": 43.73933029174805, + "learning_rate": 4.993270631642038e-09, + "logits/chosen": -0.9865999221801758, + "logits/rejected": -0.9877755045890808, + "logps/chosen": -675.552490234375, + "logps/ref_chosen": -51.94999694824219, + "logps/ref_rejected": -87.46833801269531, + "logps/rejected": -1034.5927734375, + "loss": 1.1271, + "margin_dpo/margin_mean": 323.52197265625, + "margin_dpo/margin_std": 478.4233703613281, + "step": 643 + }, + { + "KL/chosen_KL_mean": -632.0460205078125, + "KL/mean": -788.1431884765625, + "KL/rejected_KL_mean": -944.2403564453125, + "KL/std": 557.23095703125, + "epoch": 0.9456681350954479, + "fcm_dpo/beta": 0.0011425323318690062, + "fcm_dpo/delta": 0.044929929077625275, + "fcm_dpo/margin": 312.19439697265625, + "fcm_dpo/q_t": 0.42244064807891846, + "grad_norm": 49.204463958740234, + "learning_rate": 4.741290495811873e-09, + "logits/chosen": -0.9041519165039062, + "logits/rejected": -0.9101707339286804, + "logps/chosen": -691.0636596679688, + "logps/ref_chosen": -59.017662048339844, + "logps/ref_rejected": -87.13668823242188, + "logps/rejected": -1031.3770751953125, + "loss": 1.1822, + "margin_dpo/margin_mean": 312.19439697265625, + "margin_dpo/margin_std": 634.4713745117188, + "step": 644 + }, + { + "KL/chosen_KL_mean": -692.5667724609375, + "KL/mean": -776.70458984375, + "KL/rejected_KL_mean": -860.8424072265625, + "KL/std": 455.5659484863281, + "epoch": 0.947136563876652, + "fcm_dpo/beta": 0.001176076359115541, + "fcm_dpo/delta": 0.2067035287618637, + "fcm_dpo/margin": 168.275634765625, + "fcm_dpo/q_t": 0.458232045173645, + "grad_norm": 80.0379638671875, + "learning_rate": 4.495773155069299e-09, + "logits/chosen": -0.9043595194816589, + "logits/rejected": -0.8900693655014038, + "logps/chosen": -748.4427490234375, + "logps/ref_chosen": -55.87602233886719, + "logps/ref_rejected": -97.78080749511719, + "logps/rejected": -958.6231689453125, + "loss": 1.3254, + "margin_dpo/margin_mean": 168.27561950683594, + "margin_dpo/margin_std": 608.6322021484375, + "step": 645 + }, + { + "KL/chosen_KL_mean": -624.6588134765625, + "KL/mean": -764.05859375, + "KL/rejected_KL_mean": -903.4583740234375, + "KL/std": 412.8096618652344, + "epoch": 0.9486049926578561, + "fcm_dpo/beta": 0.0012013267260044813, + "fcm_dpo/delta": 0.06725561618804932, + "fcm_dpo/margin": 278.79949951171875, + "fcm_dpo/q_t": 0.4242590665817261, + "grad_norm": 43.91450881958008, + "learning_rate": 4.256725079024553e-09, + "logits/chosen": -0.9505150318145752, + "logits/rejected": -0.9369181990623474, + "logps/chosen": -685.9346313476562, + "logps/ref_chosen": -61.275787353515625, + "logps/ref_rejected": -77.50580596923828, + "logps/rejected": -980.964111328125, + "loss": 1.1552, + "margin_dpo/margin_mean": 278.79949951171875, + "margin_dpo/margin_std": 466.7037353515625, + "step": 646 + }, + { + "KL/chosen_KL_mean": -572.581298828125, + "KL/mean": -745.3472900390625, + "KL/rejected_KL_mean": -918.1133422851562, + "KL/std": 485.7171936035156, + "epoch": 0.9500734214390602, + "fcm_dpo/beta": 0.001207266584970057, + "fcm_dpo/delta": -0.017894629389047623, + "fcm_dpo/margin": 345.5320129394531, + "fcm_dpo/q_t": 0.40421944856643677, + "grad_norm": 36.778465270996094, + "learning_rate": 4.024152566816791e-09, + "logits/chosen": -0.8633547425270081, + "logits/rejected": -0.891248345375061, + "logps/chosen": -627.4337158203125, + "logps/ref_chosen": -54.8524169921875, + "logps/ref_rejected": -93.5194091796875, + "logps/rejected": -1011.6327514648438, + "loss": 1.0833, + "margin_dpo/margin_mean": 345.53204345703125, + "margin_dpo/margin_std": 453.4792785644531, + "step": 647 + }, + { + "KL/chosen_KL_mean": -597.9881591796875, + "KL/mean": -854.043212890625, + "KL/rejected_KL_mean": -1110.098388671875, + "KL/std": 566.09912109375, + "epoch": 0.9515418502202643, + "fcm_dpo/beta": 0.0011682284530252218, + "fcm_dpo/delta": -0.2110145092010498, + "fcm_dpo/margin": 512.1102905273438, + "fcm_dpo/q_t": 0.37206360697746277, + "grad_norm": 33.85206985473633, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": -0.9995071887969971, + "logits/rejected": -1.050903558731079, + "logps/chosen": -652.1595458984375, + "logps/ref_chosen": -54.17146682739258, + "logps/ref_rejected": -98.7127914428711, + "logps/rejected": -1208.8111572265625, + "loss": 0.9965, + "margin_dpo/margin_mean": 512.1102294921875, + "margin_dpo/margin_std": 664.9869384765625, + "step": 648 + }, + { + "KL/chosen_KL_mean": -615.007080078125, + "KL/mean": -742.46533203125, + "KL/rejected_KL_mean": -869.923583984375, + "KL/std": 460.30889892578125, + "epoch": 0.9530102790014684, + "fcm_dpo/beta": 0.0011777379550039768, + "fcm_dpo/delta": 0.10234013199806213, + "fcm_dpo/margin": 254.91647338867188, + "fcm_dpo/q_t": 0.4372375011444092, + "grad_norm": 29.467697143554688, + "learning_rate": 3.5784585771215235e-09, + "logits/chosen": -1.0217046737670898, + "logits/rejected": -1.0160858631134033, + "logps/chosen": -677.4874267578125, + "logps/ref_chosen": -62.480350494384766, + "logps/ref_rejected": -80.07717895507812, + "logps/rejected": -950.000732421875, + "loss": 1.2132, + "margin_dpo/margin_mean": 254.91644287109375, + "margin_dpo/margin_std": 566.233642578125, + "step": 649 + }, + { + "KL/chosen_KL_mean": -667.095947265625, + "KL/mean": -870.520263671875, + "KL/rejected_KL_mean": -1073.9447021484375, + "KL/std": 554.6080322265625, + "epoch": 0.9544787077826725, + "fcm_dpo/beta": 0.001163251232355833, + "fcm_dpo/delta": -0.07730758190155029, + "fcm_dpo/margin": 406.8487548828125, + "fcm_dpo/q_t": 0.3968607783317566, + "grad_norm": 42.1073112487793, + "learning_rate": 3.3653488440851253e-09, + "logits/chosen": -0.948840320110321, + "logits/rejected": -0.9683371186256409, + "logps/chosen": -723.188720703125, + "logps/ref_chosen": -56.09281921386719, + "logps/ref_rejected": -98.26483917236328, + "logps/rejected": -1172.20947265625, + "loss": 1.0906, + "margin_dpo/margin_mean": 406.8487548828125, + "margin_dpo/margin_std": 639.1219482421875, + "step": 650 + }, + { + "KL/chosen_KL_mean": -465.42462158203125, + "KL/mean": -709.3658447265625, + "KL/rejected_KL_mean": -953.3070678710938, + "KL/std": 522.031005859375, + "epoch": 0.9559471365638766, + "fcm_dpo/beta": 0.0011323931394144893, + "fcm_dpo/delta": -0.16175681352615356, + "fcm_dpo/margin": 487.8824768066406, + "fcm_dpo/q_t": 0.3747558891773224, + "grad_norm": 32.2318000793457, + "learning_rate": 3.158738163478475e-09, + "logits/chosen": -0.9470324516296387, + "logits/rejected": -0.9983922243118286, + "logps/chosen": -508.8500671386719, + "logps/ref_chosen": -43.42544937133789, + "logps/ref_rejected": -99.95791625976562, + "logps/rejected": -1053.264892578125, + "loss": 0.9846, + "margin_dpo/margin_mean": 487.88250732421875, + "margin_dpo/margin_std": 518.0916137695312, + "step": 651 + }, + { + "KL/chosen_KL_mean": -579.7500610351562, + "KL/mean": -770.1712646484375, + "KL/rejected_KL_mean": -960.592529296875, + "KL/std": 537.4967041015625, + "epoch": 0.9574155653450808, + "fcm_dpo/beta": 0.0011219610460102558, + "fcm_dpo/delta": -0.028511468321084976, + "fcm_dpo/margin": 380.8424377441406, + "fcm_dpo/q_t": 0.4063616394996643, + "grad_norm": 38.36195373535156, + "learning_rate": 2.9586319796851555e-09, + "logits/chosen": -0.9568244218826294, + "logits/rejected": -0.9792181253433228, + "logps/chosen": -642.326904296875, + "logps/ref_chosen": -62.57680892944336, + "logps/ref_rejected": -111.76779174804688, + "logps/rejected": -1072.3603515625, + "loss": 1.1056, + "margin_dpo/margin_mean": 380.8424072265625, + "margin_dpo/margin_std": 606.596435546875, + "step": 652 + }, + { + "KL/chosen_KL_mean": -694.0017700195312, + "KL/mean": -866.1856689453125, + "KL/rejected_KL_mean": -1038.3695068359375, + "KL/std": 565.44384765625, + "epoch": 0.9588839941262849, + "fcm_dpo/beta": 0.0011236886493861675, + "fcm_dpo/delta": 0.013402625918388367, + "fcm_dpo/margin": 344.36773681640625, + "fcm_dpo/q_t": 0.41541993618011475, + "grad_norm": 35.408912658691406, + "learning_rate": 2.7650355656892166e-09, + "logits/chosen": -1.0132495164871216, + "logits/rejected": -1.0341593027114868, + "logps/chosen": -755.11474609375, + "logps/ref_chosen": -61.11295700073242, + "logps/ref_rejected": -103.24960327148438, + "logps/rejected": -1141.619140625, + "loss": 1.1366, + "margin_dpo/margin_mean": 344.36773681640625, + "margin_dpo/margin_std": 593.7517700195312, + "step": 653 + }, + { + "KL/chosen_KL_mean": -636.8229370117188, + "KL/mean": -798.19287109375, + "KL/rejected_KL_mean": -959.562744140625, + "KL/std": 455.2934265136719, + "epoch": 0.960352422907489, + "fcm_dpo/beta": 0.0011275492142885923, + "fcm_dpo/delta": 0.03746868297457695, + "fcm_dpo/margin": 322.73980712890625, + "fcm_dpo/q_t": 0.4194484353065491, + "grad_norm": 34.15508270263672, + "learning_rate": 2.577954022936174e-09, + "logits/chosen": -0.9693796634674072, + "logits/rejected": -0.9811614751815796, + "logps/chosen": -698.5510864257812, + "logps/ref_chosen": -61.7281379699707, + "logps/ref_rejected": -98.7738037109375, + "logps/rejected": -1058.3365478515625, + "loss": 1.1369, + "margin_dpo/margin_mean": 322.73980712890625, + "margin_dpo/margin_std": 528.53564453125, + "step": 654 + }, + { + "KL/chosen_KL_mean": -576.0261840820312, + "KL/mean": -755.9908447265625, + "KL/rejected_KL_mean": -935.9555053710938, + "KL/std": 469.98699951171875, + "epoch": 0.9618208516886931, + "fcm_dpo/beta": 0.0011292758863419294, + "fcm_dpo/delta": -0.006732765585184097, + "fcm_dpo/margin": 359.9292297363281, + "fcm_dpo/q_t": 0.4099391996860504, + "grad_norm": 34.95536804199219, + "learning_rate": 2.397392281198729e-09, + "logits/chosen": -0.9169396162033081, + "logits/rejected": -0.9512918591499329, + "logps/chosen": -625.60302734375, + "logps/ref_chosen": -49.576812744140625, + "logps/ref_rejected": -98.29183197021484, + "logps/rejected": -1034.247314453125, + "loss": 1.1086, + "margin_dpo/margin_mean": 359.9292297363281, + "margin_dpo/margin_std": 552.3767700195312, + "step": 655 + }, + { + "KL/chosen_KL_mean": -630.7530517578125, + "KL/mean": -918.9047241210938, + "KL/rejected_KL_mean": -1207.0562744140625, + "KL/std": 602.3056640625, + "epoch": 0.9632892804698973, + "fcm_dpo/beta": 0.0010918871266767383, + "fcm_dpo/delta": -0.2440999299287796, + "fcm_dpo/margin": 576.30322265625, + "fcm_dpo/q_t": 0.361335813999176, + "grad_norm": 84.34712982177734, + "learning_rate": 2.223355098446622e-09, + "logits/chosen": -0.9227770566940308, + "logits/rejected": -0.9797782897949219, + "logps/chosen": -683.302490234375, + "logps/ref_chosen": -52.54943084716797, + "logps/ref_rejected": -113.67464447021484, + "logps/rejected": -1320.73095703125, + "loss": 0.948, + "margin_dpo/margin_mean": 576.30322265625, + "margin_dpo/margin_std": 611.642822265625, + "step": 656 + }, + { + "KL/chosen_KL_mean": -592.7017822265625, + "KL/mean": -825.6055297851562, + "KL/rejected_KL_mean": -1058.50927734375, + "KL/std": 583.1033325195312, + "epoch": 0.9647577092511013, + "fcm_dpo/beta": 0.0010563830146566033, + "fcm_dpo/delta": -0.0972730815410614, + "fcm_dpo/margin": 465.8076171875, + "fcm_dpo/q_t": 0.3898601531982422, + "grad_norm": 42.103797912597656, + "learning_rate": 2.055847060721566e-09, + "logits/chosen": -0.9886813163757324, + "logits/rejected": -1.0220025777816772, + "logps/chosen": -639.4022827148438, + "logps/ref_chosen": -46.700538635253906, + "logps/ref_rejected": -97.91487121582031, + "logps/rejected": -1156.4241943359375, + "loss": 1.045, + "margin_dpo/margin_mean": 465.8076171875, + "margin_dpo/margin_std": 609.584228515625, + "step": 657 + }, + { + "KL/chosen_KL_mean": -648.281982421875, + "KL/mean": -832.0382690429688, + "KL/rejected_KL_mean": -1015.79443359375, + "KL/std": 454.24188232421875, + "epoch": 0.9662261380323054, + "fcm_dpo/beta": 0.001051408820785582, + "fcm_dpo/delta": 0.013769730925559998, + "fcm_dpo/margin": 367.512451171875, + "fcm_dpo/q_t": 0.4112740457057953, + "grad_norm": 31.373876571655273, + "learning_rate": 1.8948725820160662e-09, + "logits/chosen": -0.9426242709159851, + "logits/rejected": -0.9675799012184143, + "logps/chosen": -709.240234375, + "logps/ref_chosen": -60.95820999145508, + "logps/ref_rejected": -95.93949127197266, + "logps/rejected": -1111.73388671875, + "loss": 1.1061, + "margin_dpo/margin_mean": 367.512451171875, + "margin_dpo/margin_std": 511.381103515625, + "step": 658 + }, + { + "KL/chosen_KL_mean": -570.4702758789062, + "KL/mean": -759.959716796875, + "KL/rejected_KL_mean": -949.4490966796875, + "KL/std": 473.38311767578125, + "epoch": 0.9676945668135095, + "fcm_dpo/beta": 0.0010536739137023687, + "fcm_dpo/delta": 0.0004968568682670593, + "fcm_dpo/margin": 378.97882080078125, + "fcm_dpo/q_t": 0.4098956882953644, + "grad_norm": 42.95841979980469, + "learning_rate": 1.7404359041573723e-09, + "logits/chosen": -0.878132700920105, + "logits/rejected": -0.8574471473693848, + "logps/chosen": -647.2132568359375, + "logps/ref_chosen": -76.74298095703125, + "logps/ref_rejected": -87.4709701538086, + "logps/rejected": -1036.9200439453125, + "loss": 1.0935, + "margin_dpo/margin_mean": 378.97882080078125, + "margin_dpo/margin_std": 513.1124267578125, + "step": 659 + }, + { + "KL/chosen_KL_mean": -608.8428344726562, + "KL/mean": -835.480224609375, + "KL/rejected_KL_mean": -1062.117431640625, + "KL/std": 522.89111328125, + "epoch": 0.9691629955947136, + "fcm_dpo/beta": 0.001046313438564539, + "fcm_dpo/delta": -0.07789793610572815, + "fcm_dpo/margin": 453.274658203125, + "fcm_dpo/q_t": 0.39197492599487305, + "grad_norm": 43.292850494384766, + "learning_rate": 1.592541096695571e-09, + "logits/chosen": -0.9468445777893066, + "logits/rejected": -0.9635279178619385, + "logps/chosen": -667.8907470703125, + "logps/ref_chosen": -59.04788589477539, + "logps/ref_rejected": -75.96005249023438, + "logps/rejected": -1138.07763671875, + "loss": 1.0454, + "margin_dpo/margin_mean": 453.274658203125, + "margin_dpo/margin_std": 558.73583984375, + "step": 660 + }, + { + "KL/chosen_KL_mean": -549.8631591796875, + "KL/mean": -752.0217895507812, + "KL/rejected_KL_mean": -954.1804809570312, + "KL/std": 563.0736083984375, + "epoch": 0.9706314243759178, + "fcm_dpo/beta": 0.0010425238870084286, + "fcm_dpo/delta": -0.022867929190397263, + "fcm_dpo/margin": 404.3173522949219, + "fcm_dpo/q_t": 0.40591317415237427, + "grad_norm": 44.050010681152344, + "learning_rate": 1.4511920567963908e-09, + "logits/chosen": -0.9941369295120239, + "logits/rejected": -1.0034549236297607, + "logps/chosen": -600.537109375, + "logps/ref_chosen": -50.673973083496094, + "logps/ref_rejected": -86.00569152832031, + "logps/rejected": -1040.1861572265625, + "loss": 1.0828, + "margin_dpo/margin_mean": 404.3173522949219, + "margin_dpo/margin_std": 560.7882080078125, + "step": 661 + }, + { + "KL/chosen_KL_mean": -640.8428955078125, + "KL/mean": -805.0003662109375, + "KL/rejected_KL_mean": -969.1578369140625, + "KL/std": 488.666015625, + "epoch": 0.9720998531571219, + "fcm_dpo/beta": 0.0010448182001709938, + "fcm_dpo/delta": 0.05901217460632324, + "fcm_dpo/margin": 328.3148498535156, + "fcm_dpo/q_t": 0.4229253828525543, + "grad_norm": 29.6393985748291, + "learning_rate": 1.3163925091384532e-09, + "logits/chosen": -0.8987467288970947, + "logits/rejected": -0.8944799900054932, + "logps/chosen": -710.10400390625, + "logps/ref_chosen": -69.26106262207031, + "logps/ref_rejected": -89.05593872070312, + "logps/rejected": -1058.2137451171875, + "loss": 1.164, + "margin_dpo/margin_mean": 328.3148498535156, + "margin_dpo/margin_std": 602.18505859375, + "step": 662 + }, + { + "KL/chosen_KL_mean": -611.591796875, + "KL/mean": -805.6707763671875, + "KL/rejected_KL_mean": -999.7498168945312, + "KL/std": 552.761474609375, + "epoch": 0.973568281938326, + "fcm_dpo/beta": 0.00104548130184412, + "fcm_dpo/delta": -0.006134232506155968, + "fcm_dpo/margin": 388.15789794921875, + "fcm_dpo/q_t": 0.4104207158088684, + "grad_norm": 25.260534286499023, + "learning_rate": 1.1881460058152382e-09, + "logits/chosen": -0.9808641076087952, + "logits/rejected": -1.0001773834228516, + "logps/chosen": -676.4707641601562, + "logps/ref_chosen": -64.87890625, + "logps/ref_rejected": -113.92536926269531, + "logps/rejected": -1113.6751708984375, + "loss": 1.1198, + "margin_dpo/margin_mean": 388.15789794921875, + "margin_dpo/margin_std": 645.2306518554688, + "step": 663 + }, + { + "KL/chosen_KL_mean": -636.8848876953125, + "KL/mean": -862.3253173828125, + "KL/rejected_KL_mean": -1087.765869140625, + "KL/std": 568.6349487304688, + "epoch": 0.9750367107195301, + "fcm_dpo/beta": 0.001032976433634758, + "fcm_dpo/delta": -0.06939011812210083, + "fcm_dpo/margin": 450.8810119628906, + "fcm_dpo/q_t": 0.39612704515457153, + "grad_norm": 26.127405166625977, + "learning_rate": 1.066455926241383e-09, + "logits/chosen": -0.9494335651397705, + "logits/rejected": -0.9770565032958984, + "logps/chosen": -697.7733154296875, + "logps/ref_chosen": -60.88847351074219, + "logps/ref_rejected": -105.521728515625, + "logps/rejected": -1193.28759765625, + "loss": 1.0633, + "margin_dpo/margin_mean": 450.8809814453125, + "margin_dpo/margin_std": 610.881103515625, + "step": 664 + }, + { + "KL/chosen_KL_mean": -567.0701904296875, + "KL/mean": -746.9297485351562, + "KL/rejected_KL_mean": -926.7893676757812, + "KL/std": 455.4164733886719, + "epoch": 0.9765051395007343, + "fcm_dpo/beta": 0.0010364458430558443, + "fcm_dpo/delta": 0.028231685981154442, + "fcm_dpo/margin": 359.71923828125, + "fcm_dpo/q_t": 0.4139707386493683, + "grad_norm": 37.42238235473633, + "learning_rate": 9.513254770636137e-10, + "logits/chosen": -1.0132501125335693, + "logits/rejected": -1.0314836502075195, + "logps/chosen": -627.63427734375, + "logps/ref_chosen": -60.56413269042969, + "logps/ref_rejected": -84.80882263183594, + "logps/rejected": -1011.5982055664062, + "loss": 1.1014, + "margin_dpo/margin_mean": 359.71923828125, + "margin_dpo/margin_std": 458.2942199707031, + "step": 665 + }, + { + "KL/chosen_KL_mean": -619.1126708984375, + "KL/mean": -813.4324951171875, + "KL/rejected_KL_mean": -1007.7523193359375, + "KL/std": 487.46881103515625, + "epoch": 0.9779735682819384, + "fcm_dpo/beta": 0.00103611103259027, + "fcm_dpo/delta": -0.0028592273592948914, + "fcm_dpo/margin": 388.6396484375, + "fcm_dpo/q_t": 0.40877360105514526, + "grad_norm": 33.26556396484375, + "learning_rate": 8.427576920763956e-10, + "logits/chosen": -0.8844137191772461, + "logits/rejected": -0.8921805024147034, + "logps/chosen": -683.5326538085938, + "logps/ref_chosen": -64.41996002197266, + "logps/ref_rejected": -95.8916244506836, + "logps/rejected": -1103.64404296875, + "loss": 1.0977, + "margin_dpo/margin_mean": 388.6396484375, + "margin_dpo/margin_std": 539.1551513671875, + "step": 666 + }, + { + "KL/chosen_KL_mean": -677.048828125, + "KL/mean": -896.513671875, + "KL/rejected_KL_mean": -1115.978515625, + "KL/std": 515.116455078125, + "epoch": 0.9794419970631424, + "fcm_dpo/beta": 0.00102834845893085, + "fcm_dpo/delta": -0.05389273539185524, + "fcm_dpo/margin": 438.9295654296875, + "fcm_dpo/q_t": 0.3974354863166809, + "grad_norm": 49.83727264404297, + "learning_rate": 7.407554321417764e-10, + "logits/chosen": -0.8834275007247925, + "logits/rejected": -0.8831124305725098, + "logps/chosen": -746.3258666992188, + "logps/ref_chosen": -69.27702331542969, + "logps/ref_rejected": -87.83549499511719, + "logps/rejected": -1203.81396484375, + "loss": 1.0624, + "margin_dpo/margin_mean": 438.9295349121094, + "margin_dpo/margin_std": 566.962890625, + "step": 667 + }, + { + "KL/chosen_KL_mean": -733.402587890625, + "KL/mean": -894.012939453125, + "KL/rejected_KL_mean": -1054.623291015625, + "KL/std": 560.261962890625, + "epoch": 0.9809104258443465, + "fcm_dpo/beta": 0.0010424241190776229, + "fcm_dpo/delta": 0.06649725884199142, + "fcm_dpo/margin": 321.22064208984375, + "fcm_dpo/q_t": 0.42711547017097473, + "grad_norm": 41.80741500854492, + "learning_rate": 6.453213851142225e-10, + "logits/chosen": -0.9494549036026001, + "logits/rejected": -0.9580808877944946, + "logps/chosen": -806.006591796875, + "logps/ref_chosen": -72.60400390625, + "logps/ref_rejected": -103.73905944824219, + "logps/rejected": -1158.3623046875, + "loss": 1.1829, + "margin_dpo/margin_mean": 321.22064208984375, + "margin_dpo/margin_std": 639.4466552734375, + "step": 668 + }, + { + "KL/chosen_KL_mean": -553.5703735351562, + "KL/mean": -774.89501953125, + "KL/rejected_KL_mean": -996.2195434570312, + "KL/std": 520.9459228515625, + "epoch": 0.9823788546255506, + "fcm_dpo/beta": 0.0010317617561668158, + "fcm_dpo/delta": -0.059400614351034164, + "fcm_dpo/margin": 442.649169921875, + "fcm_dpo/q_t": 0.3953366279602051, + "grad_norm": 26.183456420898438, + "learning_rate": 5.564580657695939e-10, + "logits/chosen": -0.950742244720459, + "logits/rejected": -0.9570592641830444, + "logps/chosen": -599.686767578125, + "logps/ref_chosen": -46.116416931152344, + "logps/ref_rejected": -77.92434692382812, + "logps/rejected": -1074.1439208984375, + "loss": 1.0634, + "margin_dpo/margin_mean": 442.649169921875, + "margin_dpo/margin_std": 582.044921875, + "step": 669 + }, + { + "KL/chosen_KL_mean": -542.683837890625, + "KL/mean": -772.5099487304688, + "KL/rejected_KL_mean": -1002.3360595703125, + "KL/std": 490.7078552246094, + "epoch": 0.9838472834067548, + "fcm_dpo/beta": 0.001021178555674851, + "fcm_dpo/delta": -0.07283845543861389, + "fcm_dpo/margin": 459.6522521972656, + "fcm_dpo/q_t": 0.3936052620410919, + "grad_norm": 26.939546585083008, + "learning_rate": 4.741678157389739e-10, + "logits/chosen": -0.9101635217666626, + "logits/rejected": -0.9309284687042236, + "logps/chosen": -605.0296020507812, + "logps/ref_chosen": -62.34575271606445, + "logps/ref_rejected": -96.9405517578125, + "logps/rejected": -1099.276611328125, + "loss": 1.0577, + "margin_dpo/margin_mean": 459.6522521972656, + "margin_dpo/margin_std": 595.2643432617188, + "step": 670 + }, + { + "KL/chosen_KL_mean": -657.3043823242188, + "KL/mean": -844.4332275390625, + "KL/rejected_KL_mean": -1031.56201171875, + "KL/std": 473.44189453125, + "epoch": 0.9853157121879589, + "fcm_dpo/beta": 0.0010192393092438579, + "fcm_dpo/delta": 0.019037535414099693, + "fcm_dpo/margin": 374.25762939453125, + "fcm_dpo/q_t": 0.41170093417167664, + "grad_norm": 26.634918212890625, + "learning_rate": 3.9845280344705245e-10, + "logits/chosen": -0.9962849617004395, + "logits/rejected": -1.0246731042861938, + "logps/chosen": -705.3045043945312, + "logps/ref_chosen": -48.00010681152344, + "logps/ref_rejected": -83.81932067871094, + "logps/rejected": -1115.38134765625, + "loss": 1.1151, + "margin_dpo/margin_mean": 374.25762939453125, + "margin_dpo/margin_std": 549.691162109375, + "step": 671 + }, + { + "KL/chosen_KL_mean": -719.322998046875, + "KL/mean": -906.4647216796875, + "KL/rejected_KL_mean": -1093.6064453125, + "KL/std": 566.9267578125, + "epoch": 0.986784140969163, + "fcm_dpo/beta": 0.0010189152089878917, + "fcm_dpo/delta": 0.01937013305723667, + "fcm_dpo/margin": 374.28350830078125, + "fcm_dpo/q_t": 0.41550666093826294, + "grad_norm": 35.86020278930664, + "learning_rate": 3.293150240547549e-10, + "logits/chosen": -1.0255699157714844, + "logits/rejected": -1.027112364768982, + "logps/chosen": -777.90625, + "logps/ref_chosen": -58.58328628540039, + "logps/ref_rejected": -93.14015197753906, + "logps/rejected": -1186.74658203125, + "loss": 1.1382, + "margin_dpo/margin_mean": 374.2834777832031, + "margin_dpo/margin_std": 629.8716430664062, + "step": 672 + }, + { + "KL/chosen_KL_mean": -668.7772216796875, + "KL/mean": -847.4249267578125, + "KL/rejected_KL_mean": -1026.072509765625, + "KL/std": 507.55853271484375, + "epoch": 0.9882525697503671, + "fcm_dpo/beta": 0.0010265372693538666, + "fcm_dpo/delta": 0.034482501447200775, + "fcm_dpo/margin": 357.29541015625, + "fcm_dpo/q_t": 0.4174554646015167, + "grad_norm": 31.460580825805664, + "learning_rate": 2.6675629940689504e-10, + "logits/chosen": -0.9649622440338135, + "logits/rejected": -0.968742847442627, + "logps/chosen": -715.5003662109375, + "logps/ref_chosen": -46.72320556640625, + "logps/ref_rejected": -85.29623413085938, + "logps/rejected": -1111.3687744140625, + "loss": 1.127, + "margin_dpo/margin_mean": 357.29541015625, + "margin_dpo/margin_std": 567.166015625, + "step": 673 + }, + { + "KL/chosen_KL_mean": -557.0751953125, + "KL/mean": -779.7994384765625, + "KL/rejected_KL_mean": -1002.5236206054688, + "KL/std": 495.4407958984375, + "epoch": 0.9897209985315712, + "fcm_dpo/beta": 0.001021510106511414, + "fcm_dpo/delta": -0.05762124061584473, + "fcm_dpo/margin": 445.4484558105469, + "fcm_dpo/q_t": 0.39889925718307495, + "grad_norm": 22.074878692626953, + "learning_rate": 2.1077827798404725e-10, + "logits/chosen": -0.8819149136543274, + "logits/rejected": -0.8991641998291016, + "logps/chosen": -602.5206909179688, + "logps/ref_chosen": -45.445526123046875, + "logps/ref_rejected": -70.04593658447266, + "logps/rejected": -1072.569580078125, + "loss": 1.0619, + "margin_dpo/margin_mean": 445.4484558105469, + "margin_dpo/margin_std": 588.2993774414062, + "step": 674 + }, + { + "KL/chosen_KL_mean": -620.5293579101562, + "KL/mean": -859.3121337890625, + "KL/rejected_KL_mean": -1098.0947265625, + "KL/std": 547.046875, + "epoch": 0.9911894273127754, + "fcm_dpo/beta": 0.000996412243694067, + "fcm_dpo/delta": -0.08150520920753479, + "fcm_dpo/margin": 477.5653381347656, + "fcm_dpo/q_t": 0.3942088484764099, + "grad_norm": 24.40850257873535, + "learning_rate": 1.6138243485910863e-10, + "logits/chosen": -0.9287329316139221, + "logits/rejected": -0.9418089389801025, + "logps/chosen": -664.7056884765625, + "logps/ref_chosen": -44.17628479003906, + "logps/ref_rejected": -74.09197998046875, + "logps/rejected": -1172.186767578125, + "loss": 1.0481, + "margin_dpo/margin_mean": 477.56536865234375, + "margin_dpo/margin_std": 582.3136596679688, + "step": 675 + }, + { + "KL/chosen_KL_mean": -674.1378784179688, + "KL/mean": -893.676025390625, + "KL/rejected_KL_mean": -1113.2142333984375, + "KL/std": 516.1575927734375, + "epoch": 0.9926578560939795, + "fcm_dpo/beta": 0.000995003618299961, + "fcm_dpo/delta": -0.038550153374671936, + "fcm_dpo/margin": 439.076416015625, + "fcm_dpo/q_t": 0.3991606831550598, + "grad_norm": 26.64472007751465, + "learning_rate": 1.1857007165852472e-10, + "logits/chosen": -0.8965151906013489, + "logits/rejected": -0.9073841571807861, + "logps/chosen": -745.536376953125, + "logps/ref_chosen": -71.39852905273438, + "logps/ref_rejected": -88.3587646484375, + "logps/rejected": -1201.572998046875, + "loss": 1.0554, + "margin_dpo/margin_mean": 439.076416015625, + "margin_dpo/margin_std": 500.6483154296875, + "step": 676 + }, + { + "KL/chosen_KL_mean": -677.4783935546875, + "KL/mean": -880.2694702148438, + "KL/rejected_KL_mean": -1083.060546875, + "KL/std": 518.0269775390625, + "epoch": 0.9941262848751835, + "fcm_dpo/beta": 0.0009909621439874172, + "fcm_dpo/delta": -0.00202114786952734, + "fcm_dpo/margin": 405.5820617675781, + "fcm_dpo/q_t": 0.4127524495124817, + "grad_norm": 39.86590576171875, + "learning_rate": 8.23423165278725e-11, + "logits/chosen": -0.9709941744804382, + "logits/rejected": -0.962664008140564, + "logps/chosen": -734.005859375, + "logps/ref_chosen": -56.527435302734375, + "logps/ref_rejected": -78.22654724121094, + "logps/rejected": -1161.287109375, + "loss": 1.107, + "margin_dpo/margin_mean": 405.5820617675781, + "margin_dpo/margin_std": 623.3728637695312, + "step": 677 + }, + { + "KL/chosen_KL_mean": -572.25390625, + "KL/mean": -817.163818359375, + "KL/rejected_KL_mean": -1062.07373046875, + "KL/std": 575.556884765625, + "epoch": 0.9955947136563876, + "fcm_dpo/beta": 0.0009832927025854588, + "fcm_dpo/delta": -0.0857529491186142, + "fcm_dpo/margin": 489.81982421875, + "fcm_dpo/q_t": 0.39209699630737305, + "grad_norm": 40.62564468383789, + "learning_rate": 5.270012410216185e-11, + "logits/chosen": -0.9188889265060425, + "logits/rejected": -0.9523910880088806, + "logps/chosen": -618.388427734375, + "logps/ref_chosen": -46.13447570800781, + "logps/ref_rejected": -80.60462951660156, + "logps/rejected": -1142.678466796875, + "loss": 1.0514, + "margin_dpo/margin_mean": 489.81982421875, + "margin_dpo/margin_std": 646.03662109375, + "step": 678 + }, + { + "KL/chosen_KL_mean": -651.777099609375, + "KL/mean": -824.8280029296875, + "KL/rejected_KL_mean": -997.8789672851562, + "KL/std": 468.39227294921875, + "epoch": 0.9970631424375918, + "fcm_dpo/beta": 0.0009836409008130431, + "fcm_dpo/delta": 0.06168115884065628, + "fcm_dpo/margin": 346.101806640625, + "fcm_dpo/q_t": 0.4226842224597931, + "grad_norm": 32.5433349609375, + "learning_rate": 2.9644275480772416e-11, + "logits/chosen": -0.9497741460800171, + "logits/rejected": -0.9401757121086121, + "logps/chosen": -702.072021484375, + "logps/ref_chosen": -50.294921875, + "logps/ref_rejected": -76.59813690185547, + "logps/rejected": -1074.47705078125, + "loss": 1.1381, + "margin_dpo/margin_mean": 346.1018371582031, + "margin_dpo/margin_std": 531.5057983398438, + "step": 679 + }, + { + "KL/chosen_KL_mean": -660.9338989257812, + "KL/mean": -887.5235595703125, + "KL/rejected_KL_mean": -1114.11328125, + "KL/std": 608.47216796875, + "epoch": 0.9985315712187959, + "fcm_dpo/beta": 0.0009757577208802104, + "fcm_dpo/delta": -0.04485994204878807, + "fcm_dpo/margin": 453.1793212890625, + "fcm_dpo/q_t": 0.39889073371887207, + "grad_norm": 42.46123504638672, + "learning_rate": 1.31753782067201e-11, + "logits/chosen": -0.9346154928207397, + "logits/rejected": -0.9627900123596191, + "logps/chosen": -737.849609375, + "logps/ref_chosen": -76.91569519042969, + "logps/ref_rejected": -112.384765625, + "logps/rejected": -1226.498046875, + "loss": 1.0907, + "margin_dpo/margin_mean": 453.1793518066406, + "margin_dpo/margin_std": 672.1544799804688, + "step": 680 + }, + { + "KL/chosen_KL_mean": -638.86279296875, + "KL/mean": -822.6510009765625, + "KL/rejected_KL_mean": -1006.4391479492188, + "KL/std": 489.5135498046875, + "epoch": 1.0, + "fcm_dpo/beta": 0.000990185420960188, + "fcm_dpo/delta": 0.03625159710645676, + "fcm_dpo/margin": 367.5763244628906, + "fcm_dpo/q_t": 0.41684556007385254, + "grad_norm": 29.843454360961914, + "learning_rate": 3.2938662507808745e-12, + "logits/chosen": -0.9915759563446045, + "logits/rejected": -1.0077660083770752, + "logps/chosen": -699.820068359375, + "logps/ref_chosen": -60.957279205322266, + "logps/ref_rejected": -88.55797576904297, + "logps/rejected": -1094.9970703125, + "loss": 1.1249, + "margin_dpo/margin_mean": 367.5762939453125, + "margin_dpo/margin_std": 529.50439453125, + "step": 681 + }, + { + "epoch": 1.0, + "step": 681, + "total_flos": 0.0, + "train_loss": 1.0968060302664355, + "train_runtime": 1742.1859, + "train_samples_per_second": 25.025, + "train_steps_per_second": 0.391 + } + ], + "logging_steps": 1, + "max_steps": 681, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}