From 89b1c23d071d84127654a0fda539d57f5b4a2af3 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 1 Jun 2026 07:20:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.3 Source: Original Platform --- .gitattributes | 36 + README.md | 62 + all_results.json | 9 + config.json | 29 + generation_config.json | 9 + model-00001-of-00007.safetensors | 3 + model-00002-of-00007.safetensors | 3 + model-00003-of-00007.safetensors | 3 + model-00004-of-00007.safetensors | 3 + model-00005-of-00007.safetensors | 3 + model-00006-of-00007.safetensors | 3 + model-00007-of-00007.safetensors | 3 + model.safetensors.index.json | 298 + special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2064 ++++ train_results.json | 9 + trainer_state.json | 15246 +++++++++++++++++++++++++++++ 18 files changed, 17809 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00007.safetensors create mode 100644 model-00002-of-00007.safetensors create mode 100644 model-00003-of-00007.safetensors create mode 100644 model-00004-of-00007.safetensors create mode 100644 model-00005-of-00007.safetensors create mode 100644 model-00006-of-00007.safetensors create mode 100644 model-00007-of-00007.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..a465290 --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-harmless-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.3 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.3 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-harmless-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-harmless-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..ec90f61 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.999244142101285, + "total_flos": 0.0, + "train_loss": 1.1683954546924078, + "train_runtime": 1648.9587, + "train_samples": 42336, + "train_samples_per_second": 25.674, + "train_steps_per_second": 0.401 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..0cb91f0 --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aef274f19ce17e2c139b9cc9bb38cb6f021bfec44b92af080e2f7a561849788 +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..95afc32 --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4188f66c8268276bd71eec96e21b8840ed505ef981c3b056099241b9c928e1dd +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..781dc0f --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1c1c72e72c6430b73f12bf7ff318c4a876eae8b7ed79b983080ebddf2061ef1 +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..0e9ff5f --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:700a5be2b93f787db12eb8d44c05207c4046a9d178c861160a085feaf2b837df +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..9c07a0c --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43294db34951c76c8a753175971e61b525a063e72ba48373a584c1f41dbd3488 +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..8b8cccc --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7270361d622da56139f71d8b634f59a1cee17e1fb96cfb334ed756c5a64058b2 +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..1034a2d --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91e4f9b735c6d029e15816428546cefa05c69e0277f7707072f4098da2fac9cf +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..ec90f61 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.999244142101285, + "total_flos": 0.0, + "train_loss": 1.1683954546924078, + "train_runtime": 1648.9587, + "train_samples": 42336, + "train_samples_per_second": 25.674, + "train_steps_per_second": 0.401 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..714a0d4 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15246 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.999244142101285, + "eval_steps": 200, + "global_step": 661, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.02867889404296875, + "KL/mean": 0.029354453086853027, + "KL/rejected_KL_mean": 0.030029296875, + "KL/std": 0.2071000635623932, + "epoch": 0.0015117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0013532638549804688, + "fcm_dpo/q_t": 0.5000336766242981, + "grad_norm": 28.21967887878418, + "learning_rate": 0.0, + "logits/chosen": 0.13337239623069763, + "logits/rejected": 0.12492949515581131, + "logps/chosen": -64.5841293334961, + "logps/ref_chosen": -64.61280822753906, + "logps/ref_rejected": -64.17195129394531, + "logps/rejected": -64.14192199707031, + "loss": 1.3866, + "margin_dpo/margin_mean": -0.0013527870178222656, + "margin_dpo/margin_std": 0.2561596930027008, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.00289154052734375, + "KL/mean": -0.021616414189338684, + "KL/rejected_KL_mean": -0.04033660888671875, + "KL/std": 0.19624735414981842, + "epoch": 0.0030234315948601664, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.037450045347213745, + "fcm_dpo/q_t": 0.4990639090538025, + "grad_norm": 27.828706741333008, + "learning_rate": 7.462686567164179e-09, + "logits/chosen": 0.09414851665496826, + "logits/rejected": 0.07363267242908478, + "logps/chosen": -56.101890563964844, + "logps/ref_chosen": -56.0989990234375, + "logps/ref_rejected": -66.59971618652344, + "logps/rejected": -66.64006042480469, + "loss": 1.3828, + "margin_dpo/margin_mean": 0.03744968771934509, + "margin_dpo/margin_std": 0.27811938524246216, + "step": 2 + }, + { + "KL/chosen_KL_mean": 0.007659912109375, + "KL/mean": 0.032633185386657715, + "KL/rejected_KL_mean": 0.0576019287109375, + "KL/std": 0.2508915066719055, + "epoch": 0.0045351473922902496, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.04993748664855957, + "fcm_dpo/q_t": 0.501247763633728, + "grad_norm": 31.35702133178711, + "learning_rate": 1.4925373134328357e-08, + "logits/chosen": 0.09931470453739166, + "logits/rejected": 0.06133907288312912, + "logps/chosen": -65.44960021972656, + "logps/ref_chosen": -65.45726013183594, + "logps/ref_rejected": -90.82853698730469, + "logps/rejected": -90.77093505859375, + "loss": 1.3916, + "margin_dpo/margin_mean": -0.04993721842765808, + "margin_dpo/margin_std": 0.33015739917755127, + "step": 3 + }, + { + "KL/chosen_KL_mean": 0.003841400146484375, + "KL/mean": -0.005494982004165649, + "KL/rejected_KL_mean": -0.014835357666015625, + "KL/std": 0.22913160920143127, + "epoch": 0.006046863189720333, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.01867443323135376, + "fcm_dpo/q_t": 0.49953362345695496, + "grad_norm": 34.08740997314453, + "learning_rate": 2.2388059701492534e-08, + "logits/chosen": 0.1126205176115036, + "logits/rejected": 0.09632235765457153, + "logps/chosen": -76.85633850097656, + "logps/ref_chosen": -76.86018371582031, + "logps/ref_rejected": -79.91523742675781, + "logps/rejected": -79.93006896972656, + "loss": 1.3847, + "margin_dpo/margin_mean": 0.018673986196517944, + "margin_dpo/margin_std": 0.3117932677268982, + "step": 4 + }, + { + "KL/chosen_KL_mean": -0.014787673950195312, + "KL/mean": -0.016955822706222534, + "KL/rejected_KL_mean": -0.0191192626953125, + "KL/std": 0.222198024392128, + "epoch": 0.007558578987150416, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.004332542419433594, + "fcm_dpo/q_t": 0.4998916983604431, + "grad_norm": 29.587570190429688, + "learning_rate": 2.9850746268656714e-08, + "logits/chosen": 0.0821952074766159, + "logits/rejected": 0.04323691129684448, + "logps/chosen": -62.98612976074219, + "logps/ref_chosen": -62.97134017944336, + "logps/ref_rejected": -79.9192123413086, + "logps/rejected": -79.9383316040039, + "loss": 1.3861, + "margin_dpo/margin_mean": 0.004333585500717163, + "margin_dpo/margin_std": 0.30388158559799194, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.020517349243164062, + "KL/mean": 0.00726713240146637, + "KL/rejected_KL_mean": 0.03505706787109375, + "KL/std": 0.30582496523857117, + "epoch": 0.009070294784580499, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.05557325482368469, + "fcm_dpo/q_t": 0.5013865232467651, + "grad_norm": 29.830081939697266, + "learning_rate": 3.731343283582089e-08, + "logits/chosen": 0.14537031948566437, + "logits/rejected": 0.1057555228471756, + "logps/chosen": -51.327880859375, + "logps/ref_chosen": -51.30736541748047, + "logps/ref_rejected": -82.77239227294922, + "logps/rejected": -82.73733520507812, + "loss": 1.3923, + "margin_dpo/margin_mean": -0.05557364225387573, + "margin_dpo/margin_std": 0.4328301250934601, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.017831802368164062, + "KL/mean": 0.02575281262397766, + "KL/rejected_KL_mean": 0.033672332763671875, + "KL/std": 0.20926260948181152, + "epoch": 0.010582010582010581, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.01584029197692871, + "fcm_dpo/q_t": 0.5003958940505981, + "grad_norm": 27.20305633544922, + "learning_rate": 4.477611940298507e-08, + "logits/chosen": 0.007591056637465954, + "logits/rejected": -0.035748470574617386, + "logps/chosen": -51.44158172607422, + "logps/ref_chosen": -51.45941162109375, + "logps/ref_rejected": -66.3828125, + "logps/rejected": -66.34913635253906, + "loss": 1.388, + "margin_dpo/margin_mean": -0.015840649604797363, + "margin_dpo/margin_std": 0.22362905740737915, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.002826690673828125, + "KL/mean": -0.014976292848587036, + "KL/rejected_KL_mean": -0.03277397155761719, + "KL/std": 0.2553374171257019, + "epoch": 0.012093726379440665, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03559964895248413, + "fcm_dpo/q_t": 0.49911004304885864, + "grad_norm": 28.653799057006836, + "learning_rate": 5.223880597014925e-08, + "logits/chosen": 0.06839226186275482, + "logits/rejected": 0.04627063870429993, + "logps/chosen": -62.19471740722656, + "logps/ref_chosen": -62.197547912597656, + "logps/ref_rejected": -74.66180419921875, + "logps/rejected": -74.694580078125, + "loss": 1.383, + "margin_dpo/margin_mean": 0.03559872508049011, + "margin_dpo/margin_std": 0.33308953046798706, + "step": 8 + }, + { + "KL/chosen_KL_mean": -0.03289794921875, + "KL/mean": -0.015181422233581543, + "KL/rejected_KL_mean": 0.002529144287109375, + "KL/std": 0.23224589228630066, + "epoch": 0.013605442176870748, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.03543180227279663, + "fcm_dpo/q_t": 0.5008852481842041, + "grad_norm": 31.527856826782227, + "learning_rate": 5.970149253731343e-08, + "logits/chosen": 0.17893418669700623, + "logits/rejected": 0.1187577098608017, + "logps/chosen": -55.662620544433594, + "logps/ref_chosen": -55.629722595214844, + "logps/ref_rejected": -86.21221923828125, + "logps/rejected": -86.2096939086914, + "loss": 1.3901, + "margin_dpo/margin_mean": -0.035430908203125, + "margin_dpo/margin_std": 0.3422740697860718, + "step": 9 + }, + { + "KL/chosen_KL_mean": 0.03826141357421875, + "KL/mean": 0.021607831120491028, + "KL/rejected_KL_mean": 0.00495147705078125, + "KL/std": 0.270057737827301, + "epoch": 0.015117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03330746293067932, + "fcm_dpo/q_t": 0.49916791915893555, + "grad_norm": 29.675212860107422, + "learning_rate": 6.71641791044776e-08, + "logits/chosen": 0.1317688524723053, + "logits/rejected": 0.10075588524341583, + "logps/chosen": -62.652339935302734, + "logps/ref_chosen": -62.69060134887695, + "logps/ref_rejected": -90.610107421875, + "logps/rejected": -90.60516357421875, + "loss": 1.3834, + "margin_dpo/margin_mean": 0.033307403326034546, + "margin_dpo/margin_std": 0.38862723112106323, + "step": 10 + }, + { + "KL/chosen_KL_mean": -0.004878997802734375, + "KL/mean": -0.020426854491233826, + "KL/rejected_KL_mean": -0.035980224609375, + "KL/std": 0.1961967498064041, + "epoch": 0.016628873771730914, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.031100362539291382, + "fcm_dpo/q_t": 0.4992234408855438, + "grad_norm": 29.25577163696289, + "learning_rate": 7.462686567164178e-08, + "logits/chosen": 0.11657217144966125, + "logits/rejected": 0.10957114398479462, + "logps/chosen": -65.7719955444336, + "logps/ref_chosen": -65.76712036132812, + "logps/ref_rejected": -72.4764633178711, + "logps/rejected": -72.51244354248047, + "loss": 1.3834, + "margin_dpo/margin_mean": 0.031100690364837646, + "margin_dpo/margin_std": 0.2950153946876526, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.008037567138671875, + "KL/mean": -0.016360193490982056, + "KL/rejected_KL_mean": -0.02468109130859375, + "KL/std": 0.21698607504367828, + "epoch": 0.018140589569160998, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.016642391681671143, + "fcm_dpo/q_t": 0.49958401918411255, + "grad_norm": 28.2968692779541, + "learning_rate": 8.208955223880596e-08, + "logits/chosen": 0.04076576605439186, + "logits/rejected": 0.024239014834165573, + "logps/chosen": -60.712928771972656, + "logps/ref_chosen": -60.704891204833984, + "logps/ref_rejected": -69.41564178466797, + "logps/rejected": -69.44032287597656, + "loss": 1.3849, + "margin_dpo/margin_mean": 0.016642063856124878, + "margin_dpo/margin_std": 0.3236123323440552, + "step": 12 + }, + { + "KL/chosen_KL_mean": -0.025043487548828125, + "KL/mean": 0.008205235004425049, + "KL/rejected_KL_mean": 0.04146575927734375, + "KL/std": 0.23161034286022186, + "epoch": 0.019652305366591082, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06650584936141968, + "fcm_dpo/q_t": 0.5016615986824036, + "grad_norm": 29.276233673095703, + "learning_rate": 8.955223880597014e-08, + "logits/chosen": 0.10737905651330948, + "logits/rejected": 0.04509834945201874, + "logps/chosen": -49.93429946899414, + "logps/ref_chosen": -49.90925598144531, + "logps/ref_rejected": -92.37818145751953, + "logps/rejected": -92.33671569824219, + "loss": 1.3933, + "margin_dpo/margin_mean": -0.06650650501251221, + "margin_dpo/margin_std": 0.34030449390411377, + "step": 13 + }, + { + "KL/chosen_KL_mean": 0.05045509338378906, + "KL/mean": 0.0025360435247421265, + "KL/rejected_KL_mean": -0.045379638671875, + "KL/std": 0.19650276005268097, + "epoch": 0.021164021164021163, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.095831960439682, + "fcm_dpo/q_t": 0.4976050853729248, + "grad_norm": 29.4014835357666, + "learning_rate": 9.701492537313432e-08, + "logits/chosen": 0.08358614891767502, + "logits/rejected": 0.06596517562866211, + "logps/chosen": -60.568336486816406, + "logps/ref_chosen": -60.61879348754883, + "logps/ref_rejected": -71.79306030273438, + "logps/rejected": -71.83843994140625, + "loss": 1.3769, + "margin_dpo/margin_mean": 0.09583166241645813, + "margin_dpo/margin_std": 0.2745246887207031, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.04099273681640625, + "KL/mean": -0.02316710352897644, + "KL/rejected_KL_mean": -0.005344390869140625, + "KL/std": 0.2433144450187683, + "epoch": 0.022675736961451247, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.03565239906311035, + "fcm_dpo/q_t": 0.5008911490440369, + "grad_norm": 33.39137649536133, + "learning_rate": 1.044776119402985e-07, + "logits/chosen": 0.06936109066009521, + "logits/rejected": 0.02610601857304573, + "logps/chosen": -63.510528564453125, + "logps/ref_chosen": -63.46953582763672, + "logps/ref_rejected": -88.88951110839844, + "logps/rejected": -88.89485931396484, + "loss": 1.3902, + "margin_dpo/margin_mean": -0.03565263748168945, + "margin_dpo/margin_std": 0.3474429249763489, + "step": 15 + }, + { + "KL/chosen_KL_mean": -0.024089813232421875, + "KL/mean": -0.025951489806175232, + "KL/rejected_KL_mean": -0.0278167724609375, + "KL/std": 0.20087632536888123, + "epoch": 0.02418745275888133, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.003727465867996216, + "fcm_dpo/q_t": 0.49990740418434143, + "grad_norm": 26.88129997253418, + "learning_rate": 1.1194029850746268e-07, + "logits/chosen": 0.11440499126911163, + "logits/rejected": 0.07738970220088959, + "logps/chosen": -46.55638885498047, + "logps/ref_chosen": -46.53229904174805, + "logps/ref_rejected": -74.27533721923828, + "logps/rejected": -74.30315399169922, + "loss": 1.3861, + "margin_dpo/margin_mean": 0.003727525472640991, + "margin_dpo/margin_std": 0.2645993232727051, + "step": 16 + }, + { + "KL/chosen_KL_mean": 0.02545928955078125, + "KL/mean": 0.028340786695480347, + "KL/rejected_KL_mean": 0.031219482421875, + "KL/std": 0.23972105979919434, + "epoch": 0.025699168556311415, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.005758881568908691, + "fcm_dpo/q_t": 0.5001437067985535, + "grad_norm": 32.905975341796875, + "learning_rate": 1.1940298507462686e-07, + "logits/chosen": 0.031329765915870667, + "logits/rejected": 0.013444792479276657, + "logps/chosen": -64.05237579345703, + "logps/ref_chosen": -64.07783508300781, + "logps/ref_rejected": -86.40876770019531, + "logps/rejected": -86.37754821777344, + "loss": 1.3871, + "margin_dpo/margin_mean": -0.005759298801422119, + "margin_dpo/margin_std": 0.31801456212997437, + "step": 17 + }, + { + "KL/chosen_KL_mean": 0.02117156982421875, + "KL/mean": -0.0042324066162109375, + "KL/rejected_KL_mean": -0.02964019775390625, + "KL/std": 0.2313615083694458, + "epoch": 0.027210884353741496, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.050807178020477295, + "fcm_dpo/q_t": 0.49873000383377075, + "grad_norm": 27.679706573486328, + "learning_rate": 1.2686567164179106e-07, + "logits/chosen": 0.08802352845668793, + "logits/rejected": 0.042935267090797424, + "logps/chosen": -44.85316467285156, + "logps/ref_chosen": -44.87433624267578, + "logps/ref_rejected": -70.97604370117188, + "logps/rejected": -71.00569152832031, + "loss": 1.3815, + "margin_dpo/margin_mean": 0.05080723762512207, + "margin_dpo/margin_std": 0.3414297103881836, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.05913352966308594, + "KL/mean": 0.02635572850704193, + "KL/rejected_KL_mean": -0.006427764892578125, + "KL/std": 0.2230260968208313, + "epoch": 0.02872260015117158, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06556209921836853, + "fcm_dpo/q_t": 0.49836117029190063, + "grad_norm": 30.85642433166504, + "learning_rate": 1.343283582089552e-07, + "logits/chosen": 0.06814460456371307, + "logits/rejected": 0.054654691368341446, + "logps/chosen": -68.10067749023438, + "logps/ref_chosen": -68.1598129272461, + "logps/ref_rejected": -81.17138671875, + "logps/rejected": -81.17781829833984, + "loss": 1.38, + "margin_dpo/margin_mean": 0.06556195020675659, + "margin_dpo/margin_std": 0.2927999496459961, + "step": 19 + }, + { + "KL/chosen_KL_mean": 0.027414321899414062, + "KL/mean": 0.022625096142292023, + "KL/rejected_KL_mean": 0.0178375244140625, + "KL/std": 0.22135794162750244, + "epoch": 0.030234315948601664, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.009577527642250061, + "fcm_dpo/q_t": 0.4997604191303253, + "grad_norm": 29.147907257080078, + "learning_rate": 1.4179104477611938e-07, + "logits/chosen": 0.12029474228620529, + "logits/rejected": 0.09780436754226685, + "logps/chosen": -53.651145935058594, + "logps/ref_chosen": -53.67856216430664, + "logps/ref_rejected": -74.16911315917969, + "logps/rejected": -74.15127563476562, + "loss": 1.3855, + "margin_dpo/margin_mean": 0.009577080607414246, + "margin_dpo/margin_std": 0.27818232774734497, + "step": 20 + }, + { + "KL/chosen_KL_mean": -0.01338958740234375, + "KL/mean": -0.00590062141418457, + "KL/rejected_KL_mean": 0.0015869140625, + "KL/std": 0.24868792295455933, + "epoch": 0.031746031746031744, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.014971256256103516, + "fcm_dpo/q_t": 0.5003751516342163, + "grad_norm": 29.285295486450195, + "learning_rate": 1.4925373134328355e-07, + "logits/chosen": 0.12290213257074356, + "logits/rejected": 0.09662627428770065, + "logps/chosen": -64.7149429321289, + "logps/ref_chosen": -64.70155334472656, + "logps/ref_rejected": -81.02095031738281, + "logps/rejected": -81.01936340332031, + "loss": 1.3881, + "margin_dpo/margin_mean": -0.014971524477005005, + "margin_dpo/margin_std": 0.3347770571708679, + "step": 21 + }, + { + "KL/chosen_KL_mean": -0.007518768310546875, + "KL/mean": -0.034809038043022156, + "KL/rejected_KL_mean": -0.06209564208984375, + "KL/std": 0.20913906395435333, + "epoch": 0.03325774754346183, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05458232760429382, + "fcm_dpo/q_t": 0.4986357092857361, + "grad_norm": 28.887042999267578, + "learning_rate": 1.5671641791044775e-07, + "logits/chosen": 0.02207999862730503, + "logits/rejected": 0.00017184100579470396, + "logps/chosen": -58.04351043701172, + "logps/ref_chosen": -58.03599166870117, + "logps/ref_rejected": -80.72721862792969, + "logps/rejected": -80.78931427001953, + "loss": 1.381, + "margin_dpo/margin_mean": 0.054582446813583374, + "margin_dpo/margin_std": 0.2853773832321167, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.012264251708984375, + "KL/mean": 0.009754911065101624, + "KL/rejected_KL_mean": 0.00724029541015625, + "KL/std": 0.26380670070648193, + "epoch": 0.03476946334089191, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.005020111799240112, + "fcm_dpo/q_t": 0.4998743534088135, + "grad_norm": 33.045352935791016, + "learning_rate": 1.6417910447761193e-07, + "logits/chosen": 0.12751388549804688, + "logits/rejected": 0.10264043509960175, + "logps/chosen": -66.34382629394531, + "logps/ref_chosen": -66.35608673095703, + "logps/ref_rejected": -93.02769470214844, + "logps/rejected": -93.02044677734375, + "loss": 1.3861, + "margin_dpo/margin_mean": 0.005019426345825195, + "margin_dpo/margin_std": 0.3555549383163452, + "step": 23 + }, + { + "KL/chosen_KL_mean": -0.004886627197265625, + "KL/mean": -0.0031652823090553284, + "KL/rejected_KL_mean": -0.0014438629150390625, + "KL/std": 0.2178821861743927, + "epoch": 0.036281179138321996, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.003436669707298279, + "fcm_dpo/q_t": 0.5000854730606079, + "grad_norm": 26.289295196533203, + "learning_rate": 1.716417910447761e-07, + "logits/chosen": 0.15002357959747314, + "logits/rejected": 0.1167023628950119, + "logps/chosen": -54.46612548828125, + "logps/ref_chosen": -54.461238861083984, + "logps/ref_rejected": -68.33817291259766, + "logps/rejected": -68.33961486816406, + "loss": 1.3869, + "margin_dpo/margin_mean": -0.003437221050262451, + "margin_dpo/margin_std": 0.29839447140693665, + "step": 24 + }, + { + "KL/chosen_KL_mean": -0.03343772888183594, + "KL/mean": -0.024197697639465332, + "KL/rejected_KL_mean": -0.01496124267578125, + "KL/std": 0.23071929812431335, + "epoch": 0.03779289493575208, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.01847773790359497, + "fcm_dpo/q_t": 0.5004626512527466, + "grad_norm": 29.7857666015625, + "learning_rate": 1.7910447761194027e-07, + "logits/chosen": 0.11460703611373901, + "logits/rejected": 0.06244586408138275, + "logps/chosen": -60.03764343261719, + "logps/ref_chosen": -60.00420379638672, + "logps/ref_rejected": -90.47376251220703, + "logps/rejected": -90.48872375488281, + "loss": 1.3884, + "margin_dpo/margin_mean": -0.01847824454307556, + "margin_dpo/margin_std": 0.2980327606201172, + "step": 25 + }, + { + "KL/chosen_KL_mean": -0.00635528564453125, + "KL/mean": -0.007910311222076416, + "KL/rejected_KL_mean": -0.009464263916015625, + "KL/std": 0.1977149397134781, + "epoch": 0.039304610733182165, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.003108978271484375, + "fcm_dpo/q_t": 0.499922513961792, + "grad_norm": 29.830995559692383, + "learning_rate": 1.8656716417910447e-07, + "logits/chosen": 0.0936226099729538, + "logits/rejected": 0.07567183673381805, + "logps/chosen": -56.82550811767578, + "logps/ref_chosen": -56.81915283203125, + "logps/ref_rejected": -77.84333038330078, + "logps/rejected": -77.85279846191406, + "loss": 1.3862, + "margin_dpo/margin_mean": 0.0031086504459381104, + "margin_dpo/margin_std": 0.2862233519554138, + "step": 26 + }, + { + "KL/chosen_KL_mean": -0.016384124755859375, + "KL/mean": -0.016014084219932556, + "KL/rejected_KL_mean": -0.015649795532226562, + "KL/std": 0.2392708659172058, + "epoch": 0.04081632653061224, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.000739365816116333, + "fcm_dpo/q_t": 0.5000181198120117, + "grad_norm": 28.958324432373047, + "learning_rate": 1.9402985074626865e-07, + "logits/chosen": 0.11648187786340714, + "logits/rejected": 0.09103013575077057, + "logps/chosen": -62.893409729003906, + "logps/ref_chosen": -62.87702560424805, + "logps/ref_rejected": -71.34437561035156, + "logps/rejected": -71.36002349853516, + "loss": 1.3867, + "margin_dpo/margin_mean": -0.0007395446300506592, + "margin_dpo/margin_std": 0.36002424359321594, + "step": 27 + }, + { + "KL/chosen_KL_mean": -0.012584686279296875, + "KL/mean": -0.004347294569015503, + "KL/rejected_KL_mean": 0.0038890838623046875, + "KL/std": 0.23575863242149353, + "epoch": 0.042328042328042326, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.016472816467285156, + "fcm_dpo/q_t": 0.5004111528396606, + "grad_norm": 27.684032440185547, + "learning_rate": 2.0149253731343282e-07, + "logits/chosen": 0.07097271829843521, + "logits/rejected": 0.06192881613969803, + "logps/chosen": -59.84596252441406, + "logps/ref_chosen": -59.8333740234375, + "logps/ref_rejected": -70.39804077148438, + "logps/rejected": -70.39414978027344, + "loss": 1.3882, + "margin_dpo/margin_mean": -0.016472458839416504, + "margin_dpo/margin_std": 0.3243771195411682, + "step": 28 + }, + { + "KL/chosen_KL_mean": -0.018489837646484375, + "KL/mean": -0.031423747539520264, + "KL/rejected_KL_mean": -0.04434967041015625, + "KL/std": 0.20472386479377747, + "epoch": 0.04383975812547241, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.025858044624328613, + "fcm_dpo/q_t": 0.4993535876274109, + "grad_norm": 32.789634704589844, + "learning_rate": 2.08955223880597e-07, + "logits/chosen": 0.15825878083705902, + "logits/rejected": 0.1399884819984436, + "logps/chosen": -74.13868713378906, + "logps/ref_chosen": -74.12020111083984, + "logps/ref_rejected": -83.33099365234375, + "logps/rejected": -83.37533569335938, + "loss": 1.3839, + "margin_dpo/margin_mean": 0.025858759880065918, + "margin_dpo/margin_std": 0.2755180895328522, + "step": 29 + }, + { + "KL/chosen_KL_mean": -0.0454254150390625, + "KL/mean": -0.06478771567344666, + "KL/rejected_KL_mean": -0.0841522216796875, + "KL/std": 0.20571255683898926, + "epoch": 0.045351473922902494, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.038724422454833984, + "fcm_dpo/q_t": 0.49903199076652527, + "grad_norm": 30.31739616394043, + "learning_rate": 2.1641791044776117e-07, + "logits/chosen": 0.11089472472667694, + "logits/rejected": 0.05757853388786316, + "logps/chosen": -50.796714782714844, + "logps/ref_chosen": -50.75128936767578, + "logps/ref_rejected": -89.29063415527344, + "logps/rejected": -89.37478637695312, + "loss": 1.3827, + "margin_dpo/margin_mean": 0.038724422454833984, + "margin_dpo/margin_std": 0.30037564039230347, + "step": 30 + }, + { + "KL/chosen_KL_mean": 0.008935928344726562, + "KL/mean": -0.04200010001659393, + "KL/rejected_KL_mean": -0.09293365478515625, + "KL/std": 0.27889418601989746, + "epoch": 0.04686318972033258, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.10187044739723206, + "fcm_dpo/q_t": 0.4974536597728729, + "grad_norm": 34.08384323120117, + "learning_rate": 2.2388059701492537e-07, + "logits/chosen": 0.111326202750206, + "logits/rejected": 0.0648956447839737, + "logps/chosen": -65.32781982421875, + "logps/ref_chosen": -65.33675384521484, + "logps/ref_rejected": -100.76666259765625, + "logps/rejected": -100.8595962524414, + "loss": 1.3766, + "margin_dpo/margin_mean": 0.10187077522277832, + "margin_dpo/margin_std": 0.39277029037475586, + "step": 31 + }, + { + "KL/chosen_KL_mean": -0.01952362060546875, + "KL/mean": -0.024612531065940857, + "KL/rejected_KL_mean": -0.029697418212890625, + "KL/std": 0.22407123446464539, + "epoch": 0.04837490551776266, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.01017645001411438, + "fcm_dpo/q_t": 0.49974536895751953, + "grad_norm": 30.059810638427734, + "learning_rate": 2.3134328358208954e-07, + "logits/chosen": 0.07288508117198944, + "logits/rejected": 0.06522229313850403, + "logps/chosen": -67.20285034179688, + "logps/ref_chosen": -67.18333435058594, + "logps/ref_rejected": -82.80763244628906, + "logps/rejected": -82.83733367919922, + "loss": 1.3856, + "margin_dpo/margin_mean": 0.010176807641983032, + "margin_dpo/margin_std": 0.31959301233291626, + "step": 32 + }, + { + "KL/chosen_KL_mean": -0.026327133178710938, + "KL/mean": -0.03532563894987106, + "KL/rejected_KL_mean": -0.044322967529296875, + "KL/std": 0.258139967918396, + "epoch": 0.049886621315192746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.017992421984672546, + "fcm_dpo/q_t": 0.4995502829551697, + "grad_norm": 31.032793045043945, + "learning_rate": 2.388059701492537e-07, + "logits/chosen": 0.0272968877106905, + "logits/rejected": 0.0015150085091590881, + "logps/chosen": -64.06581115722656, + "logps/ref_chosen": -64.03948211669922, + "logps/ref_rejected": -75.68357849121094, + "logps/rejected": -75.7279052734375, + "loss": 1.3848, + "margin_dpo/margin_mean": 0.0179927796125412, + "margin_dpo/margin_std": 0.3556821942329407, + "step": 33 + }, + { + "KL/chosen_KL_mean": -0.025463104248046875, + "KL/mean": -0.06123238801956177, + "KL/rejected_KL_mean": -0.09700202941894531, + "KL/std": 0.20554988086223602, + "epoch": 0.05139833711262283, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.07154315710067749, + "fcm_dpo/q_t": 0.49821144342422485, + "grad_norm": 28.452770233154297, + "learning_rate": 2.4626865671641786e-07, + "logits/chosen": 0.11309738457202911, + "logits/rejected": 0.08256950974464417, + "logps/chosen": -53.68975830078125, + "logps/ref_chosen": -53.6642951965332, + "logps/ref_rejected": -65.77989959716797, + "logps/rejected": -65.87689971923828, + "loss": 1.3793, + "margin_dpo/margin_mean": 0.07154309749603271, + "margin_dpo/margin_std": 0.269156813621521, + "step": 34 + }, + { + "KL/chosen_KL_mean": -0.09962081909179688, + "KL/mean": -0.09243303537368774, + "KL/rejected_KL_mean": -0.08524322509765625, + "KL/std": 0.23130691051483154, + "epoch": 0.05291005291005291, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.01437610387802124, + "fcm_dpo/q_t": 0.5003600120544434, + "grad_norm": 27.835346221923828, + "learning_rate": 2.537313432835821e-07, + "logits/chosen": 0.05176283419132233, + "logits/rejected": 0.029404601082205772, + "logps/chosen": -61.116485595703125, + "logps/ref_chosen": -61.01686096191406, + "logps/ref_rejected": -72.78598022460938, + "logps/rejected": -72.87123107910156, + "loss": 1.388, + "margin_dpo/margin_mean": -0.014376223087310791, + "margin_dpo/margin_std": 0.30166250467300415, + "step": 35 + }, + { + "KL/chosen_KL_mean": -0.10001564025878906, + "KL/mean": -0.09694293141365051, + "KL/rejected_KL_mean": -0.0938720703125, + "KL/std": 0.2522987723350525, + "epoch": 0.05442176870748299, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.006146609783172607, + "fcm_dpo/q_t": 0.5001544952392578, + "grad_norm": 28.722679138183594, + "learning_rate": 2.611940298507462e-07, + "logits/chosen": 0.08311149477958679, + "logits/rejected": 0.031588129699230194, + "logps/chosen": -50.63737487792969, + "logps/ref_chosen": -50.53736114501953, + "logps/ref_rejected": -78.11678314208984, + "logps/rejected": -78.21065521240234, + "loss": 1.3873, + "margin_dpo/margin_mean": -0.006146460771560669, + "margin_dpo/margin_std": 0.3678821325302124, + "step": 36 + }, + { + "KL/chosen_KL_mean": -0.05405426025390625, + "KL/mean": -0.0969603955745697, + "KL/rejected_KL_mean": -0.13987350463867188, + "KL/std": 0.2613227665424347, + "epoch": 0.055933484504913075, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08581572771072388, + "fcm_dpo/q_t": 0.4978557825088501, + "grad_norm": 37.60136413574219, + "learning_rate": 2.686567164179104e-07, + "logits/chosen": 0.09612583369016647, + "logits/rejected": 0.017332565039396286, + "logps/chosen": -59.608001708984375, + "logps/ref_chosen": -59.55394744873047, + "logps/ref_rejected": -108.27702331542969, + "logps/rejected": -108.41690063476562, + "loss": 1.3781, + "margin_dpo/margin_mean": 0.08581539988517761, + "margin_dpo/margin_std": 0.36218589544296265, + "step": 37 + }, + { + "KL/chosen_KL_mean": -0.07653617858886719, + "KL/mean": -0.09487441182136536, + "KL/rejected_KL_mean": -0.11320877075195312, + "KL/std": 0.23041307926177979, + "epoch": 0.05744520030234316, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03667086362838745, + "fcm_dpo/q_t": 0.499083936214447, + "grad_norm": 29.755857467651367, + "learning_rate": 2.761194029850746e-07, + "logits/chosen": 0.0761660635471344, + "logits/rejected": 0.0621175691485405, + "logps/chosen": -65.8648910522461, + "logps/ref_chosen": -65.78836059570312, + "logps/ref_rejected": -76.1619873046875, + "logps/rejected": -76.27519226074219, + "loss": 1.3829, + "margin_dpo/margin_mean": 0.03667101263999939, + "margin_dpo/margin_std": 0.30828237533569336, + "step": 38 + }, + { + "KL/chosen_KL_mean": -0.1410198211669922, + "KL/mean": -0.12570250034332275, + "KL/rejected_KL_mean": -0.110382080078125, + "KL/std": 0.28514018654823303, + "epoch": 0.05895691609977324, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.030639827251434326, + "fcm_dpo/q_t": 0.5007643699645996, + "grad_norm": 29.475297927856445, + "learning_rate": 2.8358208955223876e-07, + "logits/chosen": 0.17340002954006195, + "logits/rejected": 0.14621543884277344, + "logps/chosen": -57.31782913208008, + "logps/ref_chosen": -57.17681121826172, + "logps/ref_rejected": -79.486328125, + "logps/rejected": -79.59671020507812, + "loss": 1.3897, + "margin_dpo/margin_mean": -0.03063926100730896, + "margin_dpo/margin_std": 0.3815447986125946, + "step": 39 + }, + { + "KL/chosen_KL_mean": -0.135894775390625, + "KL/mean": -0.1299644410610199, + "KL/rejected_KL_mean": -0.1240386962890625, + "KL/std": 0.2603451907634735, + "epoch": 0.06046863189720333, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.01186208426952362, + "fcm_dpo/q_t": 0.5002955198287964, + "grad_norm": 31.529159545898438, + "learning_rate": 2.9104477611940296e-07, + "logits/chosen": 0.11634434759616852, + "logits/rejected": 0.06645578145980835, + "logps/chosen": -61.470062255859375, + "logps/ref_chosen": -61.33416748046875, + "logps/ref_rejected": -79.10697174072266, + "logps/rejected": -79.23101043701172, + "loss": 1.3878, + "margin_dpo/margin_mean": -0.011862263083457947, + "margin_dpo/margin_std": 0.34227511286735535, + "step": 40 + }, + { + "KL/chosen_KL_mean": -0.101654052734375, + "KL/mean": -0.15321721136569977, + "KL/rejected_KL_mean": -0.20477676391601562, + "KL/std": 0.2765595316886902, + "epoch": 0.06198034769463341, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.10311892628669739, + "fcm_dpo/q_t": 0.4974229335784912, + "grad_norm": 30.425931930541992, + "learning_rate": 2.985074626865671e-07, + "logits/chosen": 0.06092709302902222, + "logits/rejected": 0.040676962584257126, + "logps/chosen": -67.64838409423828, + "logps/ref_chosen": -67.5467300415039, + "logps/ref_rejected": -83.87788391113281, + "logps/rejected": -84.0826644897461, + "loss": 1.3763, + "margin_dpo/margin_mean": 0.10311943292617798, + "margin_dpo/margin_std": 0.34685975313186646, + "step": 41 + }, + { + "KL/chosen_KL_mean": -0.1368083953857422, + "KL/mean": -0.14792399108409882, + "KL/rejected_KL_mean": -0.15903854370117188, + "KL/std": 0.2869614362716675, + "epoch": 0.06349206349206349, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.022232264280319214, + "fcm_dpo/q_t": 0.49944519996643066, + "grad_norm": 29.266164779663086, + "learning_rate": 3.059701492537313e-07, + "logits/chosen": 0.07238396257162094, + "logits/rejected": 0.05022376403212547, + "logps/chosen": -61.40166473388672, + "logps/ref_chosen": -61.26485824584961, + "logps/ref_rejected": -76.3629150390625, + "logps/rejected": -76.52195739746094, + "loss": 1.3845, + "margin_dpo/margin_mean": 0.02223220467567444, + "margin_dpo/margin_std": 0.38739311695098877, + "step": 42 + }, + { + "KL/chosen_KL_mean": -0.08211898803710938, + "KL/mean": -0.10783997178077698, + "KL/rejected_KL_mean": -0.1335601806640625, + "KL/std": 0.28459784388542175, + "epoch": 0.06500377928949358, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.051439881324768066, + "fcm_dpo/q_t": 0.49871477484703064, + "grad_norm": 34.59758758544922, + "learning_rate": 3.134328358208955e-07, + "logits/chosen": 0.08342611789703369, + "logits/rejected": 0.07251065224409103, + "logps/chosen": -71.89114379882812, + "logps/ref_chosen": -71.80902862548828, + "logps/ref_rejected": -81.12464141845703, + "logps/rejected": -81.2582015991211, + "loss": 1.3815, + "margin_dpo/margin_mean": 0.051440536975860596, + "margin_dpo/margin_std": 0.35654395818710327, + "step": 43 + }, + { + "KL/chosen_KL_mean": -0.14848709106445312, + "KL/mean": -0.1632252335548401, + "KL/rejected_KL_mean": -0.17796707153320312, + "KL/std": 0.3100023865699768, + "epoch": 0.06651549508692366, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.029478788375854492, + "fcm_dpo/q_t": 0.4992641508579254, + "grad_norm": 32.544593811035156, + "learning_rate": 3.2089552238805965e-07, + "logits/chosen": 0.022504672408103943, + "logits/rejected": -0.006912057753652334, + "logps/chosen": -66.69891357421875, + "logps/ref_chosen": -66.55043029785156, + "logps/ref_rejected": -85.06198120117188, + "logps/rejected": -85.23994445800781, + "loss": 1.3839, + "margin_dpo/margin_mean": 0.029478758573532104, + "margin_dpo/margin_std": 0.44489896297454834, + "step": 44 + }, + { + "KL/chosen_KL_mean": -0.1548919677734375, + "KL/mean": -0.20103472471237183, + "KL/rejected_KL_mean": -0.24718093872070312, + "KL/std": 0.2796245217323303, + "epoch": 0.06802721088435375, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.09229524433612823, + "fcm_dpo/q_t": 0.4976927936077118, + "grad_norm": 31.892377853393555, + "learning_rate": 3.2835820895522385e-07, + "logits/chosen": 0.13807475566864014, + "logits/rejected": 0.08398568630218506, + "logps/chosen": -62.398746490478516, + "logps/ref_chosen": -62.24385452270508, + "logps/ref_rejected": -92.96665954589844, + "logps/rejected": -93.2138442993164, + "loss": 1.3774, + "margin_dpo/margin_mean": 0.09229600429534912, + "margin_dpo/margin_std": 0.3383764326572418, + "step": 45 + }, + { + "KL/chosen_KL_mean": -0.08540916442871094, + "KL/mean": -0.16423317790031433, + "KL/rejected_KL_mean": -0.24305343627929688, + "KL/std": 0.3250676989555359, + "epoch": 0.06953892668178382, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.15764117240905762, + "fcm_dpo/q_t": 0.4960617423057556, + "grad_norm": 31.140756607055664, + "learning_rate": 3.3582089552238805e-07, + "logits/chosen": 0.12817460298538208, + "logits/rejected": 0.0819612368941307, + "logps/chosen": -61.584312438964844, + "logps/ref_chosen": -61.498905181884766, + "logps/ref_rejected": -78.91172790527344, + "logps/rejected": -79.15478515625, + "loss": 1.3711, + "margin_dpo/margin_mean": 0.15764087438583374, + "margin_dpo/margin_std": 0.43353208899497986, + "step": 46 + }, + { + "KL/chosen_KL_mean": -0.10969352722167969, + "KL/mean": -0.1963786482810974, + "KL/rejected_KL_mean": -0.2830657958984375, + "KL/std": 0.2953278422355652, + "epoch": 0.0710506424792139, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.17337125539779663, + "fcm_dpo/q_t": 0.4956666827201843, + "grad_norm": 28.431907653808594, + "learning_rate": 3.432835820895522e-07, + "logits/chosen": 0.0386110283434391, + "logits/rejected": -0.0044650724157691, + "logps/chosen": -51.68804168701172, + "logps/ref_chosen": -51.578346252441406, + "logps/ref_rejected": -68.2215576171875, + "logps/rejected": -68.50462341308594, + "loss": 1.3694, + "margin_dpo/margin_mean": 0.17337137460708618, + "margin_dpo/margin_std": 0.39710623025894165, + "step": 47 + }, + { + "KL/chosen_KL_mean": -0.22099685668945312, + "KL/mean": -0.24017362296581268, + "KL/rejected_KL_mean": -0.2593536376953125, + "KL/std": 0.31508952379226685, + "epoch": 0.07256235827664399, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03835412859916687, + "fcm_dpo/q_t": 0.49904191493988037, + "grad_norm": 26.774696350097656, + "learning_rate": 3.507462686567164e-07, + "logits/chosen": 0.1500009000301361, + "logits/rejected": 0.12023096531629562, + "logps/chosen": -52.0146484375, + "logps/ref_chosen": -51.79365158081055, + "logps/ref_rejected": -64.22503662109375, + "logps/rejected": -64.48439025878906, + "loss": 1.3829, + "margin_dpo/margin_mean": 0.03835412859916687, + "margin_dpo/margin_std": 0.4297327399253845, + "step": 48 + }, + { + "KL/chosen_KL_mean": -0.18651199340820312, + "KL/mean": -0.25922006368637085, + "KL/rejected_KL_mean": -0.3319263458251953, + "KL/std": 0.33801430463790894, + "epoch": 0.07407407407407407, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1454138606786728, + "fcm_dpo/q_t": 0.49636557698249817, + "grad_norm": 27.162975311279297, + "learning_rate": 3.5820895522388055e-07, + "logits/chosen": 0.026051603257656097, + "logits/rejected": 0.004764057695865631, + "logps/chosen": -58.32111358642578, + "logps/ref_chosen": -58.13460159301758, + "logps/ref_rejected": -64.63206481933594, + "logps/rejected": -64.96399688720703, + "loss": 1.3723, + "margin_dpo/margin_mean": 0.14541450142860413, + "margin_dpo/margin_std": 0.43787360191345215, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.30684661865234375, + "KL/mean": -0.3562437891960144, + "KL/rejected_KL_mean": -0.4056396484375, + "KL/std": 0.35121792554855347, + "epoch": 0.07558578987150416, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.09879663586616516, + "fcm_dpo/q_t": 0.497530460357666, + "grad_norm": 28.057424545288086, + "learning_rate": 3.6567164179104475e-07, + "logits/chosen": 0.10507173836231232, + "logits/rejected": 0.07558847963809967, + "logps/chosen": -53.16328430175781, + "logps/ref_chosen": -52.85643768310547, + "logps/ref_rejected": -72.17460632324219, + "logps/rejected": -72.58024597167969, + "loss": 1.3768, + "margin_dpo/margin_mean": 0.09879684448242188, + "margin_dpo/margin_std": 0.38183924555778503, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.2800483703613281, + "KL/mean": -0.3949533998966217, + "KL/rejected_KL_mean": -0.5098648071289062, + "KL/std": 0.39111822843551636, + "epoch": 0.07709750566893424, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.22981137037277222, + "fcm_dpo/q_t": 0.4942595958709717, + "grad_norm": 31.003662109375, + "learning_rate": 3.7313432835820895e-07, + "logits/chosen": 0.059184394776821136, + "logits/rejected": 0.03226791322231293, + "logps/chosen": -63.936492919921875, + "logps/ref_chosen": -63.65644073486328, + "logps/ref_rejected": -86.13229370117188, + "logps/rejected": -86.64216613769531, + "loss": 1.364, + "margin_dpo/margin_mean": 0.22981131076812744, + "margin_dpo/margin_std": 0.4713187515735626, + "step": 51 + }, + { + "KL/chosen_KL_mean": -0.3123207092285156, + "KL/mean": -0.42203959822654724, + "KL/rejected_KL_mean": -0.5317573547363281, + "KL/std": 0.42343559861183167, + "epoch": 0.07860922146636433, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.21943794190883636, + "fcm_dpo/q_t": 0.49452149868011475, + "grad_norm": 32.59117126464844, + "learning_rate": 3.805970149253731e-07, + "logits/chosen": 0.08497665822505951, + "logits/rejected": 0.03561322018504143, + "logps/chosen": -68.15253448486328, + "logps/ref_chosen": -67.8402099609375, + "logps/ref_rejected": -96.97090911865234, + "logps/rejected": -97.50267028808594, + "loss": 1.3652, + "margin_dpo/margin_mean": 0.21943779289722443, + "margin_dpo/margin_std": 0.5385311841964722, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.3397502899169922, + "KL/mean": -0.4259885549545288, + "KL/rejected_KL_mean": -0.5122280120849609, + "KL/std": 0.391630619764328, + "epoch": 0.0801209372637944, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.17248192429542542, + "fcm_dpo/q_t": 0.4956892132759094, + "grad_norm": 27.551605224609375, + "learning_rate": 3.880597014925373e-07, + "logits/chosen": 0.10184863209724426, + "logits/rejected": 0.09087856113910675, + "logps/chosen": -57.21788787841797, + "logps/ref_chosen": -56.87813949584961, + "logps/ref_rejected": -60.75569152832031, + "logps/rejected": -61.267921447753906, + "loss": 1.3698, + "margin_dpo/margin_mean": 0.17248201370239258, + "margin_dpo/margin_std": 0.4895680248737335, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.39040374755859375, + "KL/mean": -0.47420793771743774, + "KL/rejected_KL_mean": -0.5580081939697266, + "KL/std": 0.3961087465286255, + "epoch": 0.08163265306122448, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.16760483384132385, + "fcm_dpo/q_t": 0.4958149492740631, + "grad_norm": 26.449312210083008, + "learning_rate": 3.9552238805970144e-07, + "logits/chosen": 0.07982660830020905, + "logits/rejected": 0.06419865787029266, + "logps/chosen": -47.657325744628906, + "logps/ref_chosen": -47.26692199707031, + "logps/ref_rejected": -62.19426727294922, + "logps/rejected": -62.75227355957031, + "loss": 1.3703, + "margin_dpo/margin_mean": 0.16760462522506714, + "margin_dpo/margin_std": 0.5214763879776001, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.3712482452392578, + "KL/mean": -0.4966890513896942, + "KL/rejected_KL_mean": -0.62213134765625, + "KL/std": 0.4974730610847473, + "epoch": 0.08314436885865457, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2508871555328369, + "fcm_dpo/q_t": 0.493741512298584, + "grad_norm": 31.378108978271484, + "learning_rate": 4.0298507462686564e-07, + "logits/chosen": 0.03751315549015999, + "logits/rejected": -0.03892657905817032, + "logps/chosen": -50.69744110107422, + "logps/ref_chosen": -50.32619094848633, + "logps/ref_rejected": -92.44389343261719, + "logps/rejected": -93.06602478027344, + "loss": 1.3625, + "margin_dpo/margin_mean": 0.2508869171142578, + "margin_dpo/margin_std": 0.6618380546569824, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.41105079650878906, + "KL/mean": -0.4723691940307617, + "KL/rejected_KL_mean": -0.5336799621582031, + "KL/std": 0.5030519962310791, + "epoch": 0.08465608465608465, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.12263241410255432, + "fcm_dpo/q_t": 0.49693045020103455, + "grad_norm": 27.65058708190918, + "learning_rate": 4.1044776119402984e-07, + "logits/chosen": 0.14740391075611115, + "logits/rejected": 0.12471330165863037, + "logps/chosen": -57.17802429199219, + "logps/ref_chosen": -56.766971588134766, + "logps/ref_rejected": -66.30504608154297, + "logps/rejected": -66.83872985839844, + "loss": 1.3752, + "margin_dpo/margin_mean": 0.12263259291648865, + "margin_dpo/margin_std": 0.6547993421554565, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.5095005035400391, + "KL/mean": -0.6674191951751709, + "KL/rejected_KL_mean": -0.8253402709960938, + "KL/std": 0.6238527297973633, + "epoch": 0.08616780045351474, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.315839022397995, + "fcm_dpo/q_t": 0.4921194911003113, + "grad_norm": 30.4954833984375, + "learning_rate": 4.17910447761194e-07, + "logits/chosen": 0.09336856752634048, + "logits/rejected": 0.029725098982453346, + "logps/chosen": -58.277244567871094, + "logps/ref_chosen": -57.76774597167969, + "logps/ref_rejected": -82.75698852539062, + "logps/rejected": -83.58232879638672, + "loss": 1.3562, + "margin_dpo/margin_mean": 0.3158394694328308, + "margin_dpo/margin_std": 0.6920211911201477, + "step": 57 + }, + { + "KL/chosen_KL_mean": -0.5488929748535156, + "KL/mean": -0.6903192400932312, + "KL/rejected_KL_mean": -0.8317451477050781, + "KL/std": 0.6478506326675415, + "epoch": 0.08767951625094482, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.282858282327652, + "fcm_dpo/q_t": 0.49298739433288574, + "grad_norm": 30.036136627197266, + "learning_rate": 4.253731343283582e-07, + "logits/chosen": 0.0658133402466774, + "logits/rejected": 0.05019402876496315, + "logps/chosen": -73.31298065185547, + "logps/ref_chosen": -72.76408386230469, + "logps/ref_rejected": -84.49275207519531, + "logps/rejected": -85.32449340820312, + "loss": 1.3607, + "margin_dpo/margin_mean": 0.28285834193229675, + "margin_dpo/margin_std": 0.9295713305473328, + "step": 58 + }, + { + "KL/chosen_KL_mean": -0.5454654693603516, + "KL/mean": -0.6809777021408081, + "KL/rejected_KL_mean": -0.8164939880371094, + "KL/std": 0.6243307590484619, + "epoch": 0.08919123204837491, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.27102532982826233, + "fcm_dpo/q_t": 0.49324700236320496, + "grad_norm": 26.647247314453125, + "learning_rate": 4.3283582089552234e-07, + "logits/chosen": 0.13273830711841583, + "logits/rejected": 0.0665852427482605, + "logps/chosen": -50.366241455078125, + "logps/ref_chosen": -49.820777893066406, + "logps/ref_rejected": -77.14368438720703, + "logps/rejected": -77.96017456054688, + "loss": 1.3612, + "margin_dpo/margin_mean": 0.2710251212120056, + "margin_dpo/margin_std": 0.7743154168128967, + "step": 59 + }, + { + "KL/chosen_KL_mean": -0.5872421264648438, + "KL/mean": -0.6434541940689087, + "KL/rejected_KL_mean": -0.6996688842773438, + "KL/std": 0.6261066198348999, + "epoch": 0.09070294784580499, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.11242368817329407, + "fcm_dpo/q_t": 0.4971911907196045, + "grad_norm": 29.517229080200195, + "learning_rate": 4.4029850746268654e-07, + "logits/chosen": 0.12953968346118927, + "logits/rejected": 0.12811188399791718, + "logps/chosen": -63.812015533447266, + "logps/ref_chosen": -63.22477340698242, + "logps/ref_rejected": -61.360477447509766, + "logps/rejected": -62.06014633178711, + "loss": 1.3769, + "margin_dpo/margin_mean": 0.11242353916168213, + "margin_dpo/margin_std": 0.850089430809021, + "step": 60 + }, + { + "KL/chosen_KL_mean": -0.7440834045410156, + "KL/mean": -0.7743111848831177, + "KL/rejected_KL_mean": -0.8045387268066406, + "KL/std": 0.6986474990844727, + "epoch": 0.09221466364323508, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06045445799827576, + "fcm_dpo/q_t": 0.4984866976737976, + "grad_norm": 28.10788917541504, + "learning_rate": 4.4776119402985074e-07, + "logits/chosen": 0.14267820119857788, + "logits/rejected": 0.1098393127322197, + "logps/chosen": -49.76087951660156, + "logps/ref_chosen": -49.01679992675781, + "logps/ref_rejected": -74.90817260742188, + "logps/rejected": -75.71270751953125, + "loss": 1.3827, + "margin_dpo/margin_mean": 0.06045445799827576, + "margin_dpo/margin_std": 0.9751712083816528, + "step": 61 + }, + { + "KL/chosen_KL_mean": -0.7253894805908203, + "KL/mean": -0.8400178551673889, + "KL/rejected_KL_mean": -0.9546432495117188, + "KL/std": 0.623093843460083, + "epoch": 0.09372637944066516, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.229254812002182, + "fcm_dpo/q_t": 0.4942789077758789, + "grad_norm": 28.548622131347656, + "learning_rate": 4.552238805970149e-07, + "logits/chosen": 0.13225194811820984, + "logits/rejected": 0.09242310374975204, + "logps/chosen": -63.477256774902344, + "logps/ref_chosen": -62.751869201660156, + "logps/ref_rejected": -78.93360900878906, + "logps/rejected": -79.88825225830078, + "loss": 1.3653, + "margin_dpo/margin_mean": 0.22925478219985962, + "margin_dpo/margin_std": 0.8522534370422363, + "step": 62 + }, + { + "KL/chosen_KL_mean": -0.5486431121826172, + "KL/mean": -0.8511289358139038, + "KL/rejected_KL_mean": -1.1536216735839844, + "KL/std": 0.7331008911132812, + "epoch": 0.09523809523809523, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6049745082855225, + "fcm_dpo/q_t": 0.48493385314941406, + "grad_norm": 31.141754150390625, + "learning_rate": 4.626865671641791e-07, + "logits/chosen": 0.18645809590816498, + "logits/rejected": 0.16159288585186005, + "logps/chosen": -61.06389236450195, + "logps/ref_chosen": -60.51525115966797, + "logps/ref_rejected": -85.11021423339844, + "logps/rejected": -86.26383972167969, + "loss": 1.3289, + "margin_dpo/margin_mean": 0.6049743890762329, + "margin_dpo/margin_std": 0.9378039836883545, + "step": 63 + }, + { + "KL/chosen_KL_mean": -0.8053035736083984, + "KL/mean": -0.84546959400177, + "KL/rejected_KL_mean": -0.8856353759765625, + "KL/std": 0.6875101327896118, + "epoch": 0.09674981103552532, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08033087849617004, + "fcm_dpo/q_t": 0.49798619747161865, + "grad_norm": 26.139707565307617, + "learning_rate": 4.701492537313433e-07, + "logits/chosen": 0.09058545529842377, + "logits/rejected": 0.06569143384695053, + "logps/chosen": -52.012149810791016, + "logps/ref_chosen": -51.20684814453125, + "logps/ref_rejected": -66.93081665039062, + "logps/rejected": -67.81645202636719, + "loss": 1.38, + "margin_dpo/margin_mean": 0.08033090829849243, + "margin_dpo/margin_std": 0.8200113773345947, + "step": 64 + }, + { + "KL/chosen_KL_mean": -0.8628063201904297, + "KL/mean": -1.1255594491958618, + "KL/rejected_KL_mean": -1.3883171081542969, + "KL/std": 0.9253497123718262, + "epoch": 0.0982615268329554, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5255085229873657, + "fcm_dpo/q_t": 0.48692017793655396, + "grad_norm": 30.805511474609375, + "learning_rate": 4.776119402985074e-07, + "logits/chosen": 0.22027479112148285, + "logits/rejected": 0.18991133570671082, + "logps/chosen": -68.15150451660156, + "logps/ref_chosen": -67.2886962890625, + "logps/ref_rejected": -74.44281005859375, + "logps/rejected": -75.83112335205078, + "loss": 1.338, + "margin_dpo/margin_mean": 0.5255081057548523, + "margin_dpo/margin_std": 1.190438985824585, + "step": 65 + }, + { + "KL/chosen_KL_mean": -0.9165191650390625, + "KL/mean": -1.099397897720337, + "KL/rejected_KL_mean": -1.2822761535644531, + "KL/std": 0.8203333616256714, + "epoch": 0.09977324263038549, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3657585382461548, + "fcm_dpo/q_t": 0.4908965229988098, + "grad_norm": 29.26482582092285, + "learning_rate": 4.850746268656717e-07, + "logits/chosen": 0.09936878085136414, + "logits/rejected": 0.07552456855773926, + "logps/chosen": -71.65992736816406, + "logps/ref_chosen": -70.743408203125, + "logps/ref_rejected": -77.26499938964844, + "logps/rejected": -78.54727935791016, + "loss": 1.353, + "margin_dpo/margin_mean": 0.3657584488391876, + "margin_dpo/margin_std": 1.0532437562942505, + "step": 66 + }, + { + "KL/chosen_KL_mean": -0.7958889007568359, + "KL/mean": -0.9142911434173584, + "KL/rejected_KL_mean": -1.0326957702636719, + "KL/std": 0.7134413123130798, + "epoch": 0.10128495842781557, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.23680096864700317, + "fcm_dpo/q_t": 0.49409249424934387, + "grad_norm": 29.31564712524414, + "learning_rate": 4.925373134328357e-07, + "logits/chosen": 0.07396933436393738, + "logits/rejected": 0.01928994059562683, + "logps/chosen": -61.39849090576172, + "logps/ref_chosen": -60.60260009765625, + "logps/ref_rejected": -75.22235870361328, + "logps/rejected": -76.25505065917969, + "loss": 1.3646, + "margin_dpo/margin_mean": 0.23680099844932556, + "margin_dpo/margin_std": 0.8555749654769897, + "step": 67 + }, + { + "KL/chosen_KL_mean": -1.2452201843261719, + "KL/mean": -1.4087040424346924, + "KL/rejected_KL_mean": -1.5721893310546875, + "KL/std": 0.9630727767944336, + "epoch": 0.10279667422524566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3269740343093872, + "fcm_dpo/q_t": 0.49186715483665466, + "grad_norm": 31.29288101196289, + "learning_rate": 5e-07, + "logits/chosen": 0.08105947077274323, + "logits/rejected": 0.05047762021422386, + "logps/chosen": -78.77359008789062, + "logps/ref_chosen": -77.52836608886719, + "logps/ref_rejected": -93.17778015136719, + "logps/rejected": -94.74996948242188, + "loss": 1.358, + "margin_dpo/margin_mean": 0.3269742727279663, + "margin_dpo/margin_std": 1.267851710319519, + "step": 68 + }, + { + "KL/chosen_KL_mean": -1.13970947265625, + "KL/mean": -1.466085433959961, + "KL/rejected_KL_mean": -1.7924575805664062, + "KL/std": 1.0306477546691895, + "epoch": 0.10430839002267574, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6527523994445801, + "fcm_dpo/q_t": 0.4837765097618103, + "grad_norm": 30.93454933166504, + "learning_rate": 4.999965034812934e-07, + "logits/chosen": 0.0892995297908783, + "logits/rejected": 0.04694270342588425, + "logps/chosen": -67.082763671875, + "logps/ref_chosen": -65.94305419921875, + "logps/ref_rejected": -89.7735595703125, + "logps/rejected": -91.5660171508789, + "loss": 1.3259, + "margin_dpo/margin_mean": 0.6527519226074219, + "margin_dpo/margin_std": 1.201098918914795, + "step": 69 + }, + { + "KL/chosen_KL_mean": -1.238992691040039, + "KL/mean": -1.4067550897598267, + "KL/rejected_KL_mean": -1.5745105743408203, + "KL/std": 0.8888180255889893, + "epoch": 0.10582010582010581, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.33551764488220215, + "fcm_dpo/q_t": 0.49164047837257385, + "grad_norm": 28.678653717041016, + "learning_rate": 4.999860140229787e-07, + "logits/chosen": 0.15405255556106567, + "logits/rejected": 0.13066557049751282, + "logps/chosen": -63.19690704345703, + "logps/ref_chosen": -61.95791244506836, + "logps/ref_rejected": -75.80945587158203, + "logps/rejected": -77.38397216796875, + "loss": 1.3565, + "margin_dpo/margin_mean": 0.3355180025100708, + "margin_dpo/margin_std": 1.1736544370651245, + "step": 70 + }, + { + "KL/chosen_KL_mean": -1.4223155975341797, + "KL/mean": -1.555302381515503, + "KL/rejected_KL_mean": -1.6882915496826172, + "KL/std": 1.0254878997802734, + "epoch": 0.1073318216175359, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.26597607135772705, + "fcm_dpo/q_t": 0.49335426092147827, + "grad_norm": 28.391786575317383, + "learning_rate": 4.999685319184688e-07, + "logits/chosen": 0.09895138442516327, + "logits/rejected": 0.08333157747983932, + "logps/chosen": -64.7698974609375, + "logps/ref_chosen": -63.34757995605469, + "logps/ref_rejected": -67.49658203125, + "logps/rejected": -69.18487548828125, + "loss": 1.3653, + "margin_dpo/margin_mean": 0.2659757733345032, + "margin_dpo/margin_std": 1.4624427556991577, + "step": 71 + }, + { + "KL/chosen_KL_mean": -1.1982994079589844, + "KL/mean": -1.54570472240448, + "KL/rejected_KL_mean": -1.8931045532226562, + "KL/std": 1.0516983270645142, + "epoch": 0.10884353741496598, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6948050260543823, + "fcm_dpo/q_t": 0.4827173948287964, + "grad_norm": 30.34983253479004, + "learning_rate": 4.999440576567755e-07, + "logits/chosen": 0.08275315165519714, + "logits/rejected": 0.022004300728440285, + "logps/chosen": -57.05759811401367, + "logps/ref_chosen": -55.85929870605469, + "logps/ref_rejected": -68.45423889160156, + "logps/rejected": -70.34735107421875, + "loss": 1.3223, + "margin_dpo/margin_mean": 0.6948049068450928, + "margin_dpo/margin_std": 1.2801237106323242, + "step": 72 + }, + { + "KL/chosen_KL_mean": -1.8044281005859375, + "KL/mean": -1.856170892715454, + "KL/rejected_KL_mean": -1.9079151153564453, + "KL/std": 1.1962889432907104, + "epoch": 0.11035525321239607, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.10349225997924805, + "fcm_dpo/q_t": 0.49755388498306274, + "grad_norm": 32.140106201171875, + "learning_rate": 4.999125919224965e-07, + "logits/chosen": 0.10440310835838318, + "logits/rejected": 0.09044543653726578, + "logps/chosen": -70.9432373046875, + "logps/ref_chosen": -69.13880920410156, + "logps/ref_rejected": -79.04586791992188, + "logps/rejected": -80.95378112792969, + "loss": 1.3822, + "margin_dpo/margin_mean": 0.10349264740943909, + "margin_dpo/margin_std": 1.502673625946045, + "step": 73 + }, + { + "KL/chosen_KL_mean": -1.2395572662353516, + "KL/mean": -1.5856974124908447, + "KL/rejected_KL_mean": -1.9318351745605469, + "KL/std": 1.0054885149002075, + "epoch": 0.11186696900982615, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6922777891159058, + "fcm_dpo/q_t": 0.48279356956481934, + "grad_norm": 28.01102638244629, + "learning_rate": 4.998741355957963e-07, + "logits/chosen": 0.13102754950523376, + "logits/rejected": 0.07868199050426483, + "logps/chosen": -51.163291931152344, + "logps/ref_chosen": -49.923736572265625, + "logps/ref_rejected": -81.73213958740234, + "logps/rejected": -83.66397094726562, + "loss": 1.3225, + "margin_dpo/margin_mean": 0.6922776699066162, + "margin_dpo/margin_std": 1.2654963731765747, + "step": 74 + }, + { + "KL/chosen_KL_mean": -1.4122562408447266, + "KL/mean": -1.7794251441955566, + "KL/rejected_KL_mean": -2.1465892791748047, + "KL/std": 1.0802136659622192, + "epoch": 0.11337868480725624, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.734336793422699, + "fcm_dpo/q_t": 0.4817506670951843, + "grad_norm": 26.342166900634766, + "learning_rate": 4.998286897523808e-07, + "logits/chosen": 0.10357685387134552, + "logits/rejected": 0.07225304841995239, + "logps/chosen": -47.48101043701172, + "logps/ref_chosen": -46.06875228881836, + "logps/ref_rejected": -66.1181411743164, + "logps/rejected": -68.26473236083984, + "loss": 1.3185, + "margin_dpo/margin_mean": 0.7343367338180542, + "margin_dpo/margin_std": 1.30611252784729, + "step": 75 + }, + { + "KL/chosen_KL_mean": -1.6146888732910156, + "KL/mean": -1.7545742988586426, + "KL/rejected_KL_mean": -1.8944587707519531, + "KL/std": 1.1566370725631714, + "epoch": 0.11489040060468632, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2797727584838867, + "fcm_dpo/q_t": 0.49302801489830017, + "grad_norm": 29.376850128173828, + "learning_rate": 4.997762556634679e-07, + "logits/chosen": 0.13626576960086823, + "logits/rejected": 0.09158352017402649, + "logps/chosen": -55.67743682861328, + "logps/ref_chosen": -54.06275177001953, + "logps/ref_rejected": -74.87464141845703, + "logps/rejected": -76.76910400390625, + "loss": 1.3642, + "margin_dpo/margin_mean": 0.2797728478908539, + "margin_dpo/margin_std": 1.5069594383239746, + "step": 76 + }, + { + "KL/chosen_KL_mean": -1.6369781494140625, + "KL/mean": -2.009381055831909, + "KL/rejected_KL_mean": -2.3817825317382812, + "KL/std": 1.2048678398132324, + "epoch": 0.1164021164021164, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7448074817657471, + "fcm_dpo/q_t": 0.48149538040161133, + "grad_norm": 29.28656768798828, + "learning_rate": 4.99716834795752e-07, + "logits/chosen": 0.1588551104068756, + "logits/rejected": 0.11727854609489441, + "logps/chosen": -54.713069915771484, + "logps/ref_chosen": -53.07609176635742, + "logps/ref_rejected": -74.45601654052734, + "logps/rejected": -76.83779907226562, + "loss": 1.3183, + "margin_dpo/margin_mean": 0.7448074817657471, + "margin_dpo/margin_std": 1.3909995555877686, + "step": 77 + }, + { + "KL/chosen_KL_mean": -1.8279800415039062, + "KL/mean": -2.0209426879882812, + "KL/rejected_KL_mean": -2.2139129638671875, + "KL/std": 1.2771286964416504, + "epoch": 0.11791383219954649, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.38592809438705444, + "fcm_dpo/q_t": 0.490395188331604, + "grad_norm": 29.480390548706055, + "learning_rate": 4.996504288113623e-07, + "logits/chosen": 0.10726994276046753, + "logits/rejected": 0.08700601756572723, + "logps/chosen": -69.55340576171875, + "logps/ref_chosen": -67.72541809082031, + "logps/ref_rejected": -79.03926849365234, + "logps/rejected": -81.25318145751953, + "loss": 1.3534, + "margin_dpo/margin_mean": 0.3859279155731201, + "margin_dpo/margin_std": 1.4653525352478027, + "step": 78 + }, + { + "KL/chosen_KL_mean": -1.7102622985839844, + "KL/mean": -2.2839438915252686, + "KL/rejected_KL_mean": -2.8576278686523438, + "KL/std": 1.5375041961669922, + "epoch": 0.11942554799697656, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1473674774169922, + "fcm_dpo/q_t": 0.4718601703643799, + "grad_norm": 30.79814910888672, + "learning_rate": 4.995770395678171e-07, + "logits/chosen": 0.1881793886423111, + "logits/rejected": 0.12662510573863983, + "logps/chosen": -53.870906829833984, + "logps/ref_chosen": -52.16064453125, + "logps/ref_rejected": -83.31062316894531, + "logps/rejected": -86.16824340820312, + "loss": 1.2847, + "margin_dpo/margin_mean": 1.1473678350448608, + "margin_dpo/margin_std": 1.9917948246002197, + "step": 79 + }, + { + "KL/chosen_KL_mean": -2.041339874267578, + "KL/mean": -2.3201375007629395, + "KL/rejected_KL_mean": -2.5989341735839844, + "KL/std": 1.3568034172058105, + "epoch": 0.12093726379440665, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5575928688049316, + "fcm_dpo/q_t": 0.4862174987792969, + "grad_norm": 28.497337341308594, + "learning_rate": 4.994966691179711e-07, + "logits/chosen": 0.14524391293525696, + "logits/rejected": 0.08540673553943634, + "logps/chosen": -63.451900482177734, + "logps/ref_chosen": -61.410560607910156, + "logps/ref_rejected": -78.66004943847656, + "logps/rejected": -81.25898742675781, + "loss": 1.3409, + "margin_dpo/margin_mean": 0.5575932264328003, + "margin_dpo/margin_std": 1.8942968845367432, + "step": 80 + }, + { + "KL/chosen_KL_mean": -2.136859893798828, + "KL/mean": -2.5757455825805664, + "KL/rejected_KL_mean": -3.0146255493164062, + "KL/std": 1.4461383819580078, + "epoch": 0.12244897959183673, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8777664303779602, + "fcm_dpo/q_t": 0.47836926579475403, + "grad_norm": 28.98223876953125, + "learning_rate": 4.994093197099587e-07, + "logits/chosen": 0.13092893362045288, + "logits/rejected": 0.09700986742973328, + "logps/chosen": -65.94123077392578, + "logps/ref_chosen": -63.80437088012695, + "logps/ref_rejected": -79.3484115600586, + "logps/rejected": -82.363037109375, + "loss": 1.3086, + "margin_dpo/margin_mean": 0.8777658939361572, + "margin_dpo/margin_std": 1.7367991209030151, + "step": 81 + }, + { + "KL/chosen_KL_mean": -1.7457637786865234, + "KL/mean": -2.286520481109619, + "KL/rejected_KL_mean": -2.8272781372070312, + "KL/std": 1.3924403190612793, + "epoch": 0.12396069538926682, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.081512451171875, + "fcm_dpo/q_t": 0.4731561541557312, + "grad_norm": 26.511188507080078, + "learning_rate": 4.993149937871306e-07, + "logits/chosen": 0.10917188227176666, + "logits/rejected": 0.04636561498045921, + "logps/chosen": -50.563655853271484, + "logps/ref_chosen": -48.817893981933594, + "logps/ref_rejected": -70.31497955322266, + "logps/rejected": -73.14225769042969, + "loss": 1.2874, + "margin_dpo/margin_mean": 1.081512212753296, + "margin_dpo/margin_std": 1.579821228981018, + "step": 82 + }, + { + "KL/chosen_KL_mean": -2.0326290130615234, + "KL/mean": -2.6406869888305664, + "KL/rejected_KL_mean": -3.248748779296875, + "KL/std": 1.456777811050415, + "epoch": 0.1254724111866969, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2161188125610352, + "fcm_dpo/q_t": 0.4699665307998657, + "grad_norm": 29.091482162475586, + "learning_rate": 4.992136939879856e-07, + "logits/chosen": 0.22049690783023834, + "logits/rejected": 0.16898763179779053, + "logps/chosen": -59.18340301513672, + "logps/ref_chosen": -57.15077209472656, + "logps/ref_rejected": -75.1710205078125, + "logps/rejected": -78.41976928710938, + "loss": 1.2762, + "margin_dpo/margin_mean": 1.216118335723877, + "margin_dpo/margin_std": 1.7663187980651855, + "step": 83 + }, + { + "KL/chosen_KL_mean": -2.5208396911621094, + "KL/mean": -2.948389768600464, + "KL/rejected_KL_mean": -3.375946044921875, + "KL/std": 1.8345149755477905, + "epoch": 0.12698412698412698, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.855107307434082, + "fcm_dpo/q_t": 0.47885656356811523, + "grad_norm": 30.742326736450195, + "learning_rate": 4.991054231460969e-07, + "logits/chosen": 0.14221562445163727, + "logits/rejected": 0.10221521556377411, + "logps/chosen": -67.29814147949219, + "logps/ref_chosen": -64.77729797363281, + "logps/ref_rejected": -84.71949768066406, + "logps/rejected": -88.09544372558594, + "loss": 1.3119, + "margin_dpo/margin_mean": 0.8551075458526611, + "margin_dpo/margin_std": 1.9248778820037842, + "step": 84 + }, + { + "KL/chosen_KL_mean": -2.2519493103027344, + "KL/mean": -2.8956236839294434, + "KL/rejected_KL_mean": -3.5392990112304688, + "KL/std": 1.7731411457061768, + "epoch": 0.12849584278155707, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2873495817184448, + "fcm_dpo/q_t": 0.468211829662323, + "grad_norm": 26.718862533569336, + "learning_rate": 4.989901842900325e-07, + "logits/chosen": 0.16839201748371124, + "logits/rejected": 0.12411106377840042, + "logps/chosen": -52.50364685058594, + "logps/ref_chosen": -50.25169372558594, + "logps/ref_rejected": -66.55439758300781, + "logps/rejected": -70.09368896484375, + "loss": 1.272, + "margin_dpo/margin_mean": 1.2873497009277344, + "margin_dpo/margin_std": 1.9850984811782837, + "step": 85 + }, + { + "KL/chosen_KL_mean": -2.819011688232422, + "KL/mean": -3.3019821643829346, + "KL/rejected_KL_mean": -3.7849502563476562, + "KL/std": 1.8966737985610962, + "epoch": 0.13000755857898716, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.9659372568130493, + "fcm_dpo/q_t": 0.4761325418949127, + "grad_norm": 27.210853576660156, + "learning_rate": 4.988679806432711e-07, + "logits/chosen": 0.21716086566448212, + "logits/rejected": 0.19776871800422668, + "logps/chosen": -63.54819107055664, + "logps/ref_chosen": -60.72917938232422, + "logps/ref_rejected": -72.30961608886719, + "logps/rejected": -76.09455871582031, + "loss": 1.3025, + "margin_dpo/margin_mean": 0.9659373760223389, + "margin_dpo/margin_std": 2.055220603942871, + "step": 86 + }, + { + "KL/chosen_KL_mean": -2.9924278259277344, + "KL/mean": -3.5514774322509766, + "KL/rejected_KL_mean": -4.110523223876953, + "KL/std": 2.083639144897461, + "epoch": 0.13151927437641722, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1181044578552246, + "fcm_dpo/q_t": 0.47244584560394287, + "grad_norm": 30.3636474609375, + "learning_rate": 4.987388156241114e-07, + "logits/chosen": 0.1789131462574005, + "logits/rejected": 0.11889512836933136, + "logps/chosen": -68.75039672851562, + "logps/ref_chosen": -65.75796508789062, + "logps/ref_rejected": -84.81159973144531, + "logps/rejected": -88.922119140625, + "loss": 1.2933, + "margin_dpo/margin_mean": 1.1181050539016724, + "margin_dpo/margin_std": 2.491665840148926, + "step": 87 + }, + { + "KL/chosen_KL_mean": -3.064952850341797, + "KL/mean": -3.4871764183044434, + "KL/rejected_KL_mean": -3.9093971252441406, + "KL/std": 2.3427746295928955, + "epoch": 0.1330309901738473, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.84444260597229, + "fcm_dpo/q_t": 0.4793391227722168, + "grad_norm": 30.429597854614258, + "learning_rate": 4.986026928455767e-07, + "logits/chosen": 0.24011830985546112, + "logits/rejected": 0.21302275359630585, + "logps/chosen": -65.88897705078125, + "logps/ref_chosen": -62.82402801513672, + "logps/ref_rejected": -74.9607162475586, + "logps/rejected": -78.8701171875, + "loss": 1.327, + "margin_dpo/margin_mean": 0.84444260597229, + "margin_dpo/margin_std": 2.9619760513305664, + "step": 88 + }, + { + "KL/chosen_KL_mean": -2.5539302825927734, + "KL/mean": -3.2578611373901367, + "KL/rejected_KL_mean": -3.9617958068847656, + "KL/std": 2.1723883152008057, + "epoch": 0.1345427059712774, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.4078590869903564, + "fcm_dpo/q_t": 0.4656612277030945, + "grad_norm": 29.212039947509766, + "learning_rate": 4.984596161153135e-07, + "logits/chosen": 0.24471764266490936, + "logits/rejected": 0.16260842978954315, + "logps/chosen": -43.745365142822266, + "logps/ref_chosen": -41.191436767578125, + "logps/ref_rejected": -85.44769287109375, + "logps/rejected": -89.40949249267578, + "loss": 1.2662, + "margin_dpo/margin_mean": 1.4078595638275146, + "margin_dpo/margin_std": 2.449413776397705, + "step": 89 + }, + { + "KL/chosen_KL_mean": -3.128904342651367, + "KL/mean": -3.782576560974121, + "KL/rejected_KL_mean": -4.436252593994141, + "KL/std": 2.440657615661621, + "epoch": 0.1360544217687075, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3073492050170898, + "fcm_dpo/q_t": 0.4679213762283325, + "grad_norm": 29.046615600585938, + "learning_rate": 4.983095894354857e-07, + "logits/chosen": 0.1570890247821808, + "logits/rejected": 0.10190249979496002, + "logps/chosen": -59.71281433105469, + "logps/ref_chosen": -56.58390808105469, + "logps/ref_rejected": -86.86978149414062, + "logps/rejected": -91.3060302734375, + "loss": 1.276, + "margin_dpo/margin_mean": 1.307349443435669, + "margin_dpo/margin_std": 2.514946460723877, + "step": 90 + }, + { + "KL/chosen_KL_mean": -3.1567344665527344, + "KL/mean": -3.703597068786621, + "KL/rejected_KL_mean": -4.250457763671875, + "KL/std": 2.2865095138549805, + "epoch": 0.13756613756613756, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0937232971191406, + "fcm_dpo/q_t": 0.47355741262435913, + "grad_norm": 25.90226936340332, + "learning_rate": 4.98152617002662e-07, + "logits/chosen": 0.15374761819839478, + "logits/rejected": 0.11025101691484451, + "logps/chosen": -55.53907775878906, + "logps/ref_chosen": -52.38234329223633, + "logps/ref_rejected": -72.17642211914062, + "logps/rejected": -76.4268798828125, + "loss": 1.2993, + "margin_dpo/margin_mean": 1.0937230587005615, + "margin_dpo/margin_std": 2.792675018310547, + "step": 91 + }, + { + "KL/chosen_KL_mean": -3.2463722229003906, + "KL/mean": -4.036555290222168, + "KL/rejected_KL_mean": -4.826740264892578, + "KL/std": 2.6802730560302734, + "epoch": 0.13907785336356765, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5803602933883667, + "fcm_dpo/q_t": 0.4618731141090393, + "grad_norm": 27.01513671875, + "learning_rate": 4.979887032076988e-07, + "logits/chosen": 0.22086292505264282, + "logits/rejected": 0.18025824427604675, + "logps/chosen": -56.25507354736328, + "logps/ref_chosen": -53.00870132446289, + "logps/ref_rejected": -79.77812957763672, + "logps/rejected": -84.60487365722656, + "loss": 1.2586, + "margin_dpo/margin_mean": 1.5803604125976562, + "margin_dpo/margin_std": 3.117692470550537, + "step": 92 + }, + { + "KL/chosen_KL_mean": -3.5561141967773438, + "KL/mean": -4.080088138580322, + "KL/rejected_KL_mean": -4.604063034057617, + "KL/std": 2.6413497924804688, + "epoch": 0.14058956916099774, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0479507446289062, + "fcm_dpo/q_t": 0.47497549653053284, + "grad_norm": 24.47339630126953, + "learning_rate": 4.978178526356172e-07, + "logits/chosen": 0.18242627382278442, + "logits/rejected": 0.15430913865566254, + "logps/chosen": -48.463165283203125, + "logps/ref_chosen": -44.90705108642578, + "logps/ref_rejected": -58.7879524230957, + "logps/rejected": -63.39201354980469, + "loss": 1.3097, + "margin_dpo/margin_mean": 1.0479507446289062, + "margin_dpo/margin_std": 3.2303943634033203, + "step": 93 + }, + { + "KL/chosen_KL_mean": -3.2976722717285156, + "KL/mean": -4.257216453552246, + "KL/rejected_KL_mean": -5.216762542724609, + "KL/std": 3.2533302307128906, + "epoch": 0.1421012849584278, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.9190917015075684, + "fcm_dpo/q_t": 0.45400530099868774, + "grad_norm": 27.097318649291992, + "learning_rate": 4.976400700654751e-07, + "logits/chosen": 0.2787092328071594, + "logits/rejected": 0.23703975975513458, + "logps/chosen": -63.235443115234375, + "logps/ref_chosen": -59.93777084350586, + "logps/ref_rejected": -79.3138427734375, + "logps/rejected": -84.53060150146484, + "loss": 1.2433, + "margin_dpo/margin_mean": 1.9190914630889893, + "margin_dpo/margin_std": 4.036993026733398, + "step": 94 + }, + { + "KL/chosen_KL_mean": -4.280851364135742, + "KL/mean": -5.057154655456543, + "KL/rejected_KL_mean": -5.833454132080078, + "KL/std": 2.690345525741577, + "epoch": 0.1436130007558579, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.552598476409912, + "fcm_dpo/q_t": 0.46246445178985596, + "grad_norm": 29.381898880004883, + "learning_rate": 4.974553604702332e-07, + "logits/chosen": 0.1680109202861786, + "logits/rejected": 0.10245092958211899, + "logps/chosen": -64.4493408203125, + "logps/ref_chosen": -60.168487548828125, + "logps/ref_rejected": -90.73665618896484, + "logps/rejected": -96.57011413574219, + "loss": 1.2654, + "margin_dpo/margin_mean": 1.552598237991333, + "margin_dpo/margin_std": 3.3945610523223877, + "step": 95 + }, + { + "KL/chosen_KL_mean": -4.168708801269531, + "KL/mean": -5.101164817810059, + "KL/rejected_KL_mean": -6.033618927001953, + "KL/std": 3.3781328201293945, + "epoch": 0.14512471655328799, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8649134635925293, + "fcm_dpo/q_t": 0.4566536247730255, + "grad_norm": 27.34356117248535, + "learning_rate": 4.972637290166157e-07, + "logits/chosen": 0.20887598395347595, + "logits/rejected": 0.16518929600715637, + "logps/chosen": -64.83748626708984, + "logps/ref_chosen": -60.66877746582031, + "logps/ref_rejected": -88.30673217773438, + "logps/rejected": -94.34034729003906, + "loss": 1.2566, + "margin_dpo/margin_mean": 1.8649135828018188, + "margin_dpo/margin_std": 4.4204630851745605, + "step": 96 + }, + { + "KL/chosen_KL_mean": -5.003776550292969, + "KL/mean": -5.500189304351807, + "KL/rejected_KL_mean": -5.996604919433594, + "KL/std": 3.620288848876953, + "epoch": 0.14663643235071808, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.992835283279419, + "fcm_dpo/q_t": 0.4764229357242584, + "grad_norm": 36.290287017822266, + "learning_rate": 4.970651810649666e-07, + "logits/chosen": 0.15075242519378662, + "logits/rejected": 0.10647913813591003, + "logps/chosen": -70.04789733886719, + "logps/ref_chosen": -65.04412078857422, + "logps/ref_rejected": -78.42092895507812, + "logps/rejected": -84.41753387451172, + "loss": 1.3485, + "margin_dpo/margin_mean": 0.9928349256515503, + "margin_dpo/margin_std": 4.941525459289551, + "step": 97 + }, + { + "KL/chosen_KL_mean": -4.657588958740234, + "KL/mean": -5.016092300415039, + "KL/rejected_KL_mean": -5.374599456787109, + "KL/std": 3.3176703453063965, + "epoch": 0.14814814814814814, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7170079946517944, + "fcm_dpo/q_t": 0.48207545280456543, + "grad_norm": 30.212032318115234, + "learning_rate": 4.968597221690985e-07, + "logits/chosen": 0.2352330982685089, + "logits/rejected": 0.20778122544288635, + "logps/chosen": -60.16082000732422, + "logps/ref_chosen": -55.503231048583984, + "logps/ref_rejected": -72.81553649902344, + "logps/rejected": -78.19013977050781, + "loss": 1.3422, + "margin_dpo/margin_mean": 0.7170075178146362, + "margin_dpo/margin_std": 3.2238073348999023, + "step": 98 + }, + { + "KL/chosen_KL_mean": -4.7126617431640625, + "KL/mean": -5.202777862548828, + "KL/rejected_KL_mean": -5.692897796630859, + "KL/std": 3.4897851943969727, + "epoch": 0.14965986394557823, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.9802349209785461, + "fcm_dpo/q_t": 0.47680675983428955, + "grad_norm": 32.75977325439453, + "learning_rate": 4.966473580761389e-07, + "logits/chosen": 0.26266586780548096, + "logits/rejected": 0.22229918837547302, + "logps/chosen": -63.288299560546875, + "logps/ref_chosen": -58.57563781738281, + "logps/ref_rejected": -78.693603515625, + "logps/rejected": -84.38650512695312, + "loss": 1.3412, + "margin_dpo/margin_mean": 0.9802342057228088, + "margin_dpo/margin_std": 4.541085720062256, + "step": 99 + }, + { + "KL/chosen_KL_mean": -5.3377838134765625, + "KL/mean": -6.065300941467285, + "KL/rejected_KL_mean": -6.792819976806641, + "KL/std": 4.165156841278076, + "epoch": 0.15117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.455038070678711, + "fcm_dpo/q_t": 0.4675733149051666, + "grad_norm": 33.132652282714844, + "learning_rate": 4.964280947263676e-07, + "logits/chosen": 0.24497297406196594, + "logits/rejected": 0.23685091733932495, + "logps/chosen": -84.92121887207031, + "logps/ref_chosen": -79.58343505859375, + "logps/ref_rejected": -92.152587890625, + "logps/rejected": -98.94540405273438, + "loss": 1.3314, + "margin_dpo/margin_mean": 1.4550374746322632, + "margin_dpo/margin_std": 5.805309772491455, + "step": 100 + }, + { + "KL/chosen_KL_mean": -4.292753219604492, + "KL/mean": -5.332766532897949, + "KL/rejected_KL_mean": -6.372776031494141, + "KL/std": 3.6866824626922607, + "epoch": 0.15268329554043839, + "fcm_dpo/beta": 0.10244277864694595, + "fcm_dpo/delta": 0.07949934154748917, + "fcm_dpo/margin": 2.0800251960754395, + "fcm_dpo/q_t": 0.4498664438724518, + "grad_norm": 26.55908203125, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": 0.2531740963459015, + "logits/rejected": 0.21294589340686798, + "logps/chosen": -56.62554168701172, + "logps/ref_chosen": -52.332786560058594, + "logps/ref_rejected": -69.55589294433594, + "logps/rejected": -75.92867279052734, + "loss": 1.2299, + "margin_dpo/margin_mean": 2.0800249576568604, + "margin_dpo/margin_std": 4.159194469451904, + "step": 101 + }, + { + "KL/chosen_KL_mean": -5.356512069702148, + "KL/mean": -5.766068458557129, + "KL/rejected_KL_mean": -6.175632476806641, + "KL/std": 3.7042946815490723, + "epoch": 0.15419501133786848, + "fcm_dpo/beta": 0.10488555580377579, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8191168308258057, + "fcm_dpo/q_t": 0.4826941192150116, + "grad_norm": 34.83320617675781, + "learning_rate": 4.959688949822748e-07, + "logits/chosen": 0.17034727334976196, + "logits/rejected": 0.13114404678344727, + "logps/chosen": -70.09999084472656, + "logps/ref_chosen": -64.74348449707031, + "logps/ref_rejected": -69.06132507324219, + "logps/rejected": -75.2369613647461, + "loss": 1.38, + "margin_dpo/margin_mean": 0.8191161751747131, + "margin_dpo/margin_std": 5.501483917236328, + "step": 102 + }, + { + "KL/chosen_KL_mean": -4.785610198974609, + "KL/mean": -5.598519325256348, + "KL/rejected_KL_mean": -6.411426544189453, + "KL/std": 3.8898379802703857, + "epoch": 0.15570672713529857, + "fcm_dpo/beta": 0.10772866755723953, + "fcm_dpo/delta": 0.08799183368682861, + "fcm_dpo/margin": 1.6258132457733154, + "fcm_dpo/q_t": 0.459196537733078, + "grad_norm": 31.35845184326172, + "learning_rate": 4.957289714327572e-07, + "logits/chosen": 0.26058486104011536, + "logits/rejected": 0.22672635316848755, + "logps/chosen": -68.62225341796875, + "logps/ref_chosen": -63.83664321899414, + "logps/ref_rejected": -79.32362365722656, + "logps/rejected": -85.73505401611328, + "loss": 1.2559, + "margin_dpo/margin_mean": 1.6258125305175781, + "margin_dpo/margin_std": 3.6502134799957275, + "step": 103 + }, + { + "KL/chosen_KL_mean": -4.706455230712891, + "KL/mean": -5.858922004699707, + "KL/rejected_KL_mean": -7.011390686035156, + "KL/std": 4.994261741638184, + "epoch": 0.15721844293272866, + "fcm_dpo/beta": 0.112154021859169, + "fcm_dpo/delta": 0.04702896624803543, + "fcm_dpo/margin": 2.3049371242523193, + "fcm_dpo/q_t": 0.44699156284332275, + "grad_norm": 34.770877838134766, + "learning_rate": 4.954821743156767e-07, + "logits/chosen": 0.24318566918373108, + "logits/rejected": 0.1570195108652115, + "logps/chosen": -65.7056655883789, + "logps/ref_chosen": -60.99920654296875, + "logps/ref_rejected": -98.84645080566406, + "logps/rejected": -105.85784912109375, + "loss": 1.2607, + "margin_dpo/margin_mean": 2.304936408996582, + "margin_dpo/margin_std": 6.377119064331055, + "step": 104 + }, + { + "KL/chosen_KL_mean": -5.352252960205078, + "KL/mean": -6.269779205322266, + "KL/rejected_KL_mean": -7.187309265136719, + "KL/std": 3.987088680267334, + "epoch": 0.15873015873015872, + "fcm_dpo/beta": 0.11373626440763474, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8350532054901123, + "fcm_dpo/q_t": 0.45318853855133057, + "grad_norm": 34.859764099121094, + "learning_rate": 4.952285105344791e-07, + "logits/chosen": 0.20636920630931854, + "logits/rejected": 0.15100428462028503, + "logps/chosen": -76.30252075195312, + "logps/ref_chosen": -70.95027160644531, + "logps/ref_rejected": -87.88340759277344, + "logps/rejected": -95.07072448730469, + "loss": 1.2632, + "margin_dpo/margin_mean": 1.8350532054901123, + "margin_dpo/margin_std": 4.919508457183838, + "step": 105 + }, + { + "KL/chosen_KL_mean": -5.029964447021484, + "KL/mean": -5.7955522537231445, + "KL/rejected_KL_mean": -6.561138153076172, + "KL/std": 3.89530611038208, + "epoch": 0.1602418745275888, + "fcm_dpo/beta": 0.11373626440763474, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5311670303344727, + "fcm_dpo/q_t": 0.4586039185523987, + "grad_norm": 36.046531677246094, + "learning_rate": 4.949679871846857e-07, + "logits/chosen": 0.24574297666549683, + "logits/rejected": 0.2315644919872284, + "logps/chosen": -67.48929595947266, + "logps/ref_chosen": -62.45933151245117, + "logps/ref_rejected": -67.00595092773438, + "logps/rejected": -73.56708526611328, + "loss": 1.3057, + "margin_dpo/margin_mean": 1.5311667919158936, + "margin_dpo/margin_std": 5.281202793121338, + "step": 106 + }, + { + "KL/chosen_KL_mean": -6.49530029296875, + "KL/mean": -6.9094672203063965, + "KL/rejected_KL_mean": -7.323631286621094, + "KL/std": 4.887589454650879, + "epoch": 0.1617535903250189, + "fcm_dpo/beta": 0.11373626440763474, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8283355236053467, + "fcm_dpo/q_t": 0.47569483518600464, + "grad_norm": 48.95326232910156, + "learning_rate": 4.947006115536947e-07, + "logits/chosen": 0.17019329965114594, + "logits/rejected": 0.1465989649295807, + "logps/chosen": -82.33326721191406, + "logps/ref_chosen": -75.83796691894531, + "logps/ref_rejected": -87.74038696289062, + "logps/rejected": -95.06401062011719, + "loss": 1.409, + "margin_dpo/margin_mean": 0.828335702419281, + "margin_dpo/margin_std": 6.10739803314209, + "step": 107 + }, + { + "KL/chosen_KL_mean": -5.463619232177734, + "KL/mean": -6.383543014526367, + "KL/rejected_KL_mean": -7.303462982177734, + "KL/std": 4.103908061981201, + "epoch": 0.16326530612244897, + "fcm_dpo/beta": 0.11373626440763474, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.839846134185791, + "fcm_dpo/q_t": 0.4533940255641937, + "grad_norm": 32.70323944091797, + "learning_rate": 4.944263911205772e-07, + "logits/chosen": 0.19207993149757385, + "logits/rejected": 0.16023018956184387, + "logps/chosen": -73.85685729980469, + "logps/ref_chosen": -68.39323425292969, + "logps/ref_rejected": -83.24267578125, + "logps/rejected": -90.54613494873047, + "loss": 1.2767, + "margin_dpo/margin_mean": 1.8398463726043701, + "margin_dpo/margin_std": 5.364116668701172, + "step": 108 + }, + { + "KL/chosen_KL_mean": -4.823814392089844, + "KL/mean": -6.310299873352051, + "KL/rejected_KL_mean": -7.796791076660156, + "KL/std": 4.97265100479126, + "epoch": 0.16477702191987906, + "fcm_dpo/beta": 0.11705504357814789, + "fcm_dpo/delta": 0.05780094116926193, + "fcm_dpo/margin": 2.9729766845703125, + "fcm_dpo/q_t": 0.426702082157135, + "grad_norm": 32.38801193237305, + "learning_rate": 4.941453335558681e-07, + "logits/chosen": 0.17428874969482422, + "logits/rejected": 0.12089354544878006, + "logps/chosen": -60.351295471191406, + "logps/ref_chosen": -55.52748107910156, + "logps/ref_rejected": -83.55218505859375, + "logps/rejected": -91.34898376464844, + "loss": 1.1535, + "margin_dpo/margin_mean": 2.9729764461517334, + "margin_dpo/margin_std": 5.388824462890625, + "step": 109 + }, + { + "KL/chosen_KL_mean": -6.182544708251953, + "KL/mean": -6.312487602233887, + "KL/rejected_KL_mean": -6.442424774169922, + "KL/std": 3.990682363510132, + "epoch": 0.16628873771730915, + "fcm_dpo/beta": 0.1177498996257782, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2598845958709717, + "fcm_dpo/q_t": 0.4898316264152527, + "grad_norm": 53.71846008300781, + "learning_rate": 4.938574467213517e-07, + "logits/chosen": 0.17680279910564423, + "logits/rejected": 0.18614478409290314, + "logps/chosen": -87.34129333496094, + "logps/ref_chosen": -81.15874481201172, + "logps/ref_rejected": -72.56021118164062, + "logps/rejected": -79.00263977050781, + "loss": 1.4483, + "margin_dpo/margin_mean": 0.25988438725471497, + "margin_dpo/margin_std": 5.361874103546143, + "step": 110 + }, + { + "KL/chosen_KL_mean": -5.583719253540039, + "KL/mean": -6.557001113891602, + "KL/rejected_KL_mean": -7.5302886962890625, + "KL/std": 4.138578414916992, + "epoch": 0.16780045351473924, + "fcm_dpo/beta": 0.1203727126121521, + "fcm_dpo/delta": 0.07264174520969391, + "fcm_dpo/margin": 1.9465702772140503, + "fcm_dpo/q_t": 0.44942060112953186, + "grad_norm": 30.87441062927246, + "learning_rate": 4.935627386698418e-07, + "logits/chosen": 0.2694501280784607, + "logits/rejected": 0.23262867331504822, + "logps/chosen": -57.94270324707031, + "logps/ref_chosen": -52.358985900878906, + "logps/ref_rejected": -77.06150817871094, + "logps/rejected": -84.591796875, + "loss": 1.2512, + "margin_dpo/margin_mean": 1.9465705156326294, + "margin_dpo/margin_std": 5.017286777496338, + "step": 111 + }, + { + "KL/chosen_KL_mean": -5.246620178222656, + "KL/mean": -6.449799537658691, + "KL/rejected_KL_mean": -7.652973175048828, + "KL/std": 4.185157775878906, + "epoch": 0.1693121693121693, + "fcm_dpo/beta": 0.12910687923431396, + "fcm_dpo/delta": 0.09781448543071747, + "fcm_dpo/margin": 2.4063587188720703, + "fcm_dpo/q_t": 0.4301964044570923, + "grad_norm": 40.442134857177734, + "learning_rate": 4.932612176449559e-07, + "logits/chosen": 0.19383251667022705, + "logits/rejected": 0.12825272977352142, + "logps/chosen": -68.26668548583984, + "logps/ref_chosen": -63.02006530761719, + "logps/ref_rejected": -111.36941528320312, + "logps/rejected": -119.02238464355469, + "loss": 1.1838, + "margin_dpo/margin_mean": 2.4063572883605957, + "margin_dpo/margin_std": 4.644060134887695, + "step": 112 + }, + { + "KL/chosen_KL_mean": -5.255073547363281, + "KL/mean": -5.934301376342773, + "KL/rejected_KL_mean": -6.613533020019531, + "KL/std": 3.480978488922119, + "epoch": 0.1708238851095994, + "fcm_dpo/beta": 0.13342835009098053, + "fcm_dpo/delta": 0.07491803169250488, + "fcm_dpo/margin": 1.3584556579589844, + "fcm_dpo/q_t": 0.4581720232963562, + "grad_norm": 49.05155944824219, + "learning_rate": 4.929528920808854e-07, + "logits/chosen": 0.19861599802970886, + "logits/rejected": 0.15952877700328827, + "logps/chosen": -61.06273651123047, + "logps/ref_chosen": -55.80766296386719, + "logps/ref_rejected": -69.84014129638672, + "logps/rejected": -76.45367431640625, + "loss": 1.2973, + "margin_dpo/margin_mean": 1.3584556579589844, + "margin_dpo/margin_std": 4.468038558959961, + "step": 113 + }, + { + "KL/chosen_KL_mean": -4.988502502441406, + "KL/mean": -6.088685035705566, + "KL/rejected_KL_mean": -7.188873291015625, + "KL/std": 3.774844169616699, + "epoch": 0.17233560090702948, + "fcm_dpo/beta": 0.14195749163627625, + "fcm_dpo/delta": 0.09694860130548477, + "fcm_dpo/margin": 2.200364351272583, + "fcm_dpo/q_t": 0.4333820343017578, + "grad_norm": 42.0706901550293, + "learning_rate": 4.92637770602159e-07, + "logits/chosen": 0.24624261260032654, + "logits/rejected": 0.1857389360666275, + "logps/chosen": -71.32127380371094, + "logps/ref_chosen": -66.33277130126953, + "logps/ref_rejected": -71.61489868164062, + "logps/rejected": -78.80377197265625, + "loss": 1.1946, + "margin_dpo/margin_mean": 2.200364828109741, + "margin_dpo/margin_std": 4.570239067077637, + "step": 114 + }, + { + "KL/chosen_KL_mean": -5.369632720947266, + "KL/mean": -6.250866889953613, + "KL/rejected_KL_mean": -7.132101058959961, + "KL/std": 3.4158787727355957, + "epoch": 0.17384731670445955, + "fcm_dpo/beta": 0.14961527287960052, + "fcm_dpo/delta": 0.056850727647542953, + "fcm_dpo/margin": 1.7624667882919312, + "fcm_dpo/q_t": 0.44253766536712646, + "grad_norm": 42.88827133178711, + "learning_rate": 4.923158620234019e-07, + "logits/chosen": 0.23988944292068481, + "logits/rejected": 0.17627336084842682, + "logps/chosen": -61.11867141723633, + "logps/ref_chosen": -55.74903869628906, + "logps/ref_rejected": -79.59849548339844, + "logps/rejected": -86.73059844970703, + "loss": 1.2282, + "margin_dpo/margin_mean": 1.7624670267105103, + "margin_dpo/margin_std": 4.0995025634765625, + "step": 115 + }, + { + "KL/chosen_KL_mean": -4.933021545410156, + "KL/mean": -6.309746742248535, + "KL/rejected_KL_mean": -7.6864776611328125, + "KL/std": 3.5162925720214844, + "epoch": 0.17535903250188964, + "fcm_dpo/beta": 0.15041202306747437, + "fcm_dpo/delta": -0.017097879201173782, + "fcm_dpo/margin": 2.7534589767456055, + "fcm_dpo/q_t": 0.4049130082130432, + "grad_norm": 35.12335968017578, + "learning_rate": 4.91987175349089e-07, + "logits/chosen": 0.2185758650302887, + "logits/rejected": 0.15106023848056793, + "logps/chosen": -54.298187255859375, + "logps/ref_chosen": -49.36516571044922, + "logps/ref_rejected": -72.84671020507812, + "logps/rejected": -80.53318786621094, + "loss": 1.0934, + "margin_dpo/margin_mean": 2.7534589767456055, + "margin_dpo/margin_std": 3.808335542678833, + "step": 116 + }, + { + "KL/chosen_KL_mean": -4.931730270385742, + "KL/mean": -5.858213901519775, + "KL/rejected_KL_mean": -6.784702301025391, + "KL/std": 3.3140063285827637, + "epoch": 0.17687074829931973, + "fcm_dpo/beta": 0.14422379434108734, + "fcm_dpo/delta": -0.04407034069299698, + "fcm_dpo/margin": 1.8529614210128784, + "fcm_dpo/q_t": 0.43684980273246765, + "grad_norm": 40.910606384277344, + "learning_rate": 4.916517197732933e-07, + "logits/chosen": 0.2307669222354889, + "logits/rejected": 0.19402286410331726, + "logps/chosen": -62.64262771606445, + "logps/ref_chosen": -57.710899353027344, + "logps/ref_rejected": -69.77253723144531, + "logps/rejected": -76.55723571777344, + "loss": 1.2279, + "margin_dpo/margin_mean": 1.8529608249664307, + "margin_dpo/margin_std": 4.091181755065918, + "step": 117 + }, + { + "KL/chosen_KL_mean": -4.755271911621094, + "KL/mean": -5.991068363189697, + "KL/rejected_KL_mean": -7.226863861083984, + "KL/std": 3.593959093093872, + "epoch": 0.17838246409674982, + "fcm_dpo/beta": 0.14586882293224335, + "fcm_dpo/delta": 0.0441410131752491, + "fcm_dpo/margin": 2.471588611602783, + "fcm_dpo/q_t": 0.4175887703895569, + "grad_norm": 38.68694305419922, + "learning_rate": 4.913095046794281e-07, + "logits/chosen": 0.30474239587783813, + "logits/rejected": 0.2626001238822937, + "logps/chosen": -57.23516845703125, + "logps/ref_chosen": -52.479896545410156, + "logps/ref_rejected": -81.359130859375, + "logps/rejected": -88.58599090576172, + "loss": 1.1265, + "margin_dpo/margin_mean": 2.471587896347046, + "margin_dpo/margin_std": 3.647130012512207, + "step": 118 + }, + { + "KL/chosen_KL_mean": -6.017473220825195, + "KL/mean": -6.920298099517822, + "KL/rejected_KL_mean": -7.823123931884766, + "KL/std": 4.139496803283691, + "epoch": 0.17989417989417988, + "fcm_dpo/beta": 0.14982125163078308, + "fcm_dpo/delta": 0.038419194519519806, + "fcm_dpo/margin": 1.8056480884552002, + "fcm_dpo/q_t": 0.44211965799331665, + "grad_norm": 44.770748138427734, + "learning_rate": 4.909605396399855e-07, + "logits/chosen": 0.20789092779159546, + "logits/rejected": 0.1685742735862732, + "logps/chosen": -67.3751449584961, + "logps/ref_chosen": -61.35767364501953, + "logps/ref_rejected": -75.71510314941406, + "logps/rejected": -83.53822326660156, + "loss": 1.2659, + "margin_dpo/margin_mean": 1.8056479692459106, + "margin_dpo/margin_std": 5.146032333374023, + "step": 119 + }, + { + "KL/chosen_KL_mean": -4.956226348876953, + "KL/mean": -6.255353927612305, + "KL/rejected_KL_mean": -7.5544891357421875, + "KL/std": 3.479482650756836, + "epoch": 0.18140589569160998, + "fcm_dpo/beta": 0.15262466669082642, + "fcm_dpo/delta": 0.0036748070269823074, + "fcm_dpo/margin": 2.5982601642608643, + "fcm_dpo/q_t": 0.4099358320236206, + "grad_norm": 40.47877502441406, + "learning_rate": 4.906048344162676e-07, + "logits/chosen": 0.19425508379936218, + "logits/rejected": 0.13444660604000092, + "logps/chosen": -64.8637924194336, + "logps/ref_chosen": -59.907569885253906, + "logps/ref_rejected": -79.6910629272461, + "logps/rejected": -87.24555206298828, + "loss": 1.0971, + "margin_dpo/margin_mean": 2.5982608795166016, + "margin_dpo/margin_std": 3.5266871452331543, + "step": 120 + }, + { + "KL/chosen_KL_mean": -5.358709335327148, + "KL/mean": -6.739768028259277, + "KL/rejected_KL_mean": -8.120830535888672, + "KL/std": 3.996863842010498, + "epoch": 0.18291761148904007, + "fcm_dpo/beta": 0.1501387357711792, + "fcm_dpo/delta": -0.016977794468402863, + "fcm_dpo/margin": 2.7621190547943115, + "fcm_dpo/q_t": 0.4117959439754486, + "grad_norm": 37.1208610534668, + "learning_rate": 4.902423989581143e-07, + "logits/chosen": 0.30443626642227173, + "logits/rejected": 0.2181735634803772, + "logps/chosen": -61.024749755859375, + "logps/ref_chosen": -55.66604232788086, + "logps/ref_rejected": -101.56233978271484, + "logps/rejected": -109.68316650390625, + "loss": 1.1077, + "margin_dpo/margin_mean": 2.7621188163757324, + "margin_dpo/margin_std": 4.442630290985107, + "step": 121 + }, + { + "KL/chosen_KL_mean": -5.777549743652344, + "KL/mean": -7.061952114105225, + "KL/rejected_KL_mean": -8.346351623535156, + "KL/std": 4.270412445068359, + "epoch": 0.18442932728647016, + "fcm_dpo/beta": 0.15251445770263672, + "fcm_dpo/delta": 0.008429568260908127, + "fcm_dpo/margin": 2.5688040256500244, + "fcm_dpo/q_t": 0.42014241218566895, + "grad_norm": 46.91085433959961, + "learning_rate": 4.898732434036243e-07, + "logits/chosen": 0.2145451307296753, + "logits/rejected": 0.1792818009853363, + "logps/chosen": -69.11192321777344, + "logps/ref_chosen": -63.334373474121094, + "logps/ref_rejected": -73.67523193359375, + "logps/rejected": -82.02159118652344, + "loss": 1.1596, + "margin_dpo/margin_mean": 2.568803548812866, + "margin_dpo/margin_std": 5.07568359375, + "step": 122 + }, + { + "KL/chosen_KL_mean": -5.459968566894531, + "KL/mean": -6.718021392822266, + "KL/rejected_KL_mean": -7.97607421875, + "KL/std": 3.7030413150787354, + "epoch": 0.18594104308390022, + "fcm_dpo/beta": 0.1532004177570343, + "fcm_dpo/delta": 0.01600516401231289, + "fcm_dpo/margin": 2.5161049365997314, + "fcm_dpo/q_t": 0.41378509998321533, + "grad_norm": 38.989585876464844, + "learning_rate": 4.894973780788722e-07, + "logits/chosen": 0.23768547177314758, + "logits/rejected": 0.19618701934814453, + "logps/chosen": -62.35871124267578, + "logps/ref_chosen": -56.89874267578125, + "logps/ref_rejected": -78.97028350830078, + "logps/rejected": -86.94635772705078, + "loss": 1.123, + "margin_dpo/margin_mean": 2.5161049365997314, + "margin_dpo/margin_std": 3.9700427055358887, + "step": 123 + }, + { + "KL/chosen_KL_mean": -5.782169342041016, + "KL/mean": -7.465737342834473, + "KL/rejected_KL_mean": -9.149307250976562, + "KL/std": 4.042940616607666, + "epoch": 0.1874527588813303, + "fcm_dpo/beta": 0.14460201561450958, + "fcm_dpo/delta": -0.10314086824655533, + "fcm_dpo/margin": 3.367136240005493, + "fcm_dpo/q_t": 0.3885384202003479, + "grad_norm": 36.7084846496582, + "learning_rate": 4.89114813497619e-07, + "logits/chosen": 0.28056734800338745, + "logits/rejected": 0.21720805764198303, + "logps/chosen": -62.89825439453125, + "logps/ref_chosen": -57.116085052490234, + "logps/ref_rejected": -87.93074035644531, + "logps/rejected": -97.08004760742188, + "loss": 1.0417, + "margin_dpo/margin_mean": 3.3671364784240723, + "margin_dpo/margin_std": 4.220187187194824, + "step": 124 + }, + { + "KL/chosen_KL_mean": -6.223566055297852, + "KL/mean": -7.6594648361206055, + "KL/rejected_KL_mean": -9.095359802246094, + "KL/std": 4.320698261260986, + "epoch": 0.1889644746787604, + "fcm_dpo/beta": 0.1405811309814453, + "fcm_dpo/delta": -0.006451927125453949, + "fcm_dpo/margin": 2.8717947006225586, + "fcm_dpo/q_t": 0.410498708486557, + "grad_norm": 38.985382080078125, + "learning_rate": 4.887255603610184e-07, + "logits/chosen": 0.29075124859809875, + "logits/rejected": 0.22918814420700073, + "logps/chosen": -71.92974853515625, + "logps/ref_chosen": -65.7061767578125, + "logps/ref_rejected": -91.72711944580078, + "logps/rejected": -100.82247924804688, + "loss": 1.1143, + "margin_dpo/margin_mean": 2.8717949390411377, + "margin_dpo/margin_std": 4.4547529220581055, + "step": 125 + }, + { + "KL/chosen_KL_mean": -5.821783065795898, + "KL/mean": -6.677826404571533, + "KL/rejected_KL_mean": -7.53387451171875, + "KL/std": 4.0686540603637695, + "epoch": 0.19047619047619047, + "fcm_dpo/beta": 0.15313303470611572, + "fcm_dpo/delta": 0.14905323088169098, + "fcm_dpo/margin": 1.7120908498764038, + "fcm_dpo/q_t": 0.4439869523048401, + "grad_norm": 42.53828430175781, + "learning_rate": 4.883296295573176e-07, + "logits/chosen": 0.09436048567295074, + "logits/rejected": 0.08861444890499115, + "logps/chosen": -73.99787139892578, + "logps/ref_chosen": -68.17608642578125, + "logps/ref_rejected": -65.1175537109375, + "logps/rejected": -72.65142822265625, + "loss": 1.2429, + "margin_dpo/margin_mean": 1.7120912075042725, + "margin_dpo/margin_std": 4.371842384338379, + "step": 126 + }, + { + "KL/chosen_KL_mean": -5.998382568359375, + "KL/mean": -7.5352582931518555, + "KL/rejected_KL_mean": -9.072135925292969, + "KL/std": 3.6161179542541504, + "epoch": 0.19198790627362056, + "fcm_dpo/beta": 0.1484401971101761, + "fcm_dpo/delta": -0.06965281069278717, + "fcm_dpo/margin": 3.0737504959106445, + "fcm_dpo/q_t": 0.3940262198448181, + "grad_norm": 37.3619270324707, + "learning_rate": 4.87927032161552e-07, + "logits/chosen": 0.22243531048297882, + "logits/rejected": 0.1914488971233368, + "logps/chosen": -67.87861633300781, + "logps/ref_chosen": -61.88023376464844, + "logps/ref_rejected": -68.46012878417969, + "logps/rejected": -77.53225708007812, + "loss": 1.0581, + "margin_dpo/margin_mean": 3.0737502574920654, + "margin_dpo/margin_std": 3.76737117767334, + "step": 127 + }, + { + "KL/chosen_KL_mean": -6.474908828735352, + "KL/mean": -8.036764144897461, + "KL/rejected_KL_mean": -9.598621368408203, + "KL/std": 4.666970252990723, + "epoch": 0.19349962207105065, + "fcm_dpo/beta": 0.14405301213264465, + "fcm_dpo/delta": -0.05977921932935715, + "fcm_dpo/margin": 3.123711109161377, + "fcm_dpo/q_t": 0.4026954770088196, + "grad_norm": 37.78749084472656, + "learning_rate": 4.875177794352363e-07, + "logits/chosen": 0.2339283525943756, + "logits/rejected": 0.17583772540092468, + "logps/chosen": -73.18389129638672, + "logps/ref_chosen": -66.708984375, + "logps/ref_rejected": -94.97969055175781, + "logps/rejected": -104.57830810546875, + "loss": 1.1297, + "margin_dpo/margin_mean": 3.123711585998535, + "margin_dpo/margin_std": 5.568070411682129, + "step": 128 + }, + { + "KL/chosen_KL_mean": -7.183927536010742, + "KL/mean": -8.192344665527344, + "KL/rejected_KL_mean": -9.200759887695312, + "KL/std": 4.225727558135986, + "epoch": 0.19501133786848074, + "fcm_dpo/beta": 0.15166246891021729, + "fcm_dpo/delta": 0.10337048768997192, + "fcm_dpo/margin": 2.016836404800415, + "fcm_dpo/q_t": 0.43475955724716187, + "grad_norm": 50.885433197021484, + "learning_rate": 4.871018828260491e-07, + "logits/chosen": 0.2141018807888031, + "logits/rejected": 0.20807525515556335, + "logps/chosen": -72.52275085449219, + "logps/ref_chosen": -65.33882904052734, + "logps/ref_rejected": -68.06109619140625, + "logps/rejected": -77.26185607910156, + "loss": 1.2322, + "margin_dpo/margin_mean": 2.0168368816375732, + "margin_dpo/margin_std": 4.955423355102539, + "step": 129 + }, + { + "KL/chosen_KL_mean": -6.767240524291992, + "KL/mean": -8.247897148132324, + "KL/rejected_KL_mean": -9.72854995727539, + "KL/std": 4.21795129776001, + "epoch": 0.1965230536659108, + "fcm_dpo/beta": 0.1474171131849289, + "fcm_dpo/delta": -0.04654637351632118, + "fcm_dpo/margin": 2.9613113403320312, + "fcm_dpo/q_t": 0.39949163794517517, + "grad_norm": 43.239646911621094, + "learning_rate": 4.866793539675126e-07, + "logits/chosen": 0.1729985773563385, + "logits/rejected": 0.12204181402921677, + "logps/chosen": -65.42798614501953, + "logps/ref_chosen": -58.660743713378906, + "logps/ref_rejected": -79.24510192871094, + "logps/rejected": -88.97364807128906, + "loss": 1.0837, + "margin_dpo/margin_mean": 2.9613113403320312, + "margin_dpo/margin_std": 4.048201084136963, + "step": 130 + }, + { + "KL/chosen_KL_mean": -6.502410888671875, + "KL/mean": -8.361289978027344, + "KL/rejected_KL_mean": -10.220169067382812, + "KL/std": 4.88187313079834, + "epoch": 0.1980347694633409, + "fcm_dpo/beta": 0.1393418312072754, + "fcm_dpo/delta": -0.14160066843032837, + "fcm_dpo/margin": 3.717766761779785, + "fcm_dpo/q_t": 0.3856920897960663, + "grad_norm": 34.58555603027344, + "learning_rate": 4.86250204678667e-07, + "logits/chosen": 0.19449803233146667, + "logits/rejected": 0.12917035818099976, + "logps/chosen": -59.016944885253906, + "logps/ref_chosen": -52.51453399658203, + "logps/ref_rejected": -85.18299865722656, + "logps/rejected": -95.40316772460938, + "loss": 1.0653, + "margin_dpo/margin_mean": 3.7177672386169434, + "margin_dpo/margin_std": 5.4574785232543945, + "step": 131 + }, + { + "KL/chosen_KL_mean": -7.104595184326172, + "KL/mean": -8.693756103515625, + "KL/rejected_KL_mean": -10.282918930053711, + "KL/std": 4.658677101135254, + "epoch": 0.19954648526077098, + "fcm_dpo/beta": 0.1348930299282074, + "fcm_dpo/delta": -0.033193182200193405, + "fcm_dpo/margin": 3.1783313751220703, + "fcm_dpo/q_t": 0.4000505208969116, + "grad_norm": 36.00300979614258, + "learning_rate": 4.858144469637408e-07, + "logits/chosen": 0.26692995429039, + "logits/rejected": 0.23427237570285797, + "logps/chosen": -72.78972625732422, + "logps/ref_chosen": -65.68513488769531, + "logps/ref_rejected": -69.54120635986328, + "logps/rejected": -79.82412719726562, + "loss": 1.0949, + "margin_dpo/margin_mean": 3.1783313751220703, + "margin_dpo/margin_std": 4.635673522949219, + "step": 132 + }, + { + "KL/chosen_KL_mean": -7.397819519042969, + "KL/mean": -8.735570907592773, + "KL/rejected_KL_mean": -10.073318481445312, + "KL/std": 4.610377311706543, + "epoch": 0.20105820105820105, + "fcm_dpo/beta": 0.13705599308013916, + "fcm_dpo/delta": 0.037375159561634064, + "fcm_dpo/margin": 2.675502061843872, + "fcm_dpo/q_t": 0.41829919815063477, + "grad_norm": 41.43574905395508, + "learning_rate": 4.853720930118138e-07, + "logits/chosen": 0.20174159109592438, + "logits/rejected": 0.1917198747396469, + "logps/chosen": -70.99593353271484, + "logps/ref_chosen": -63.598114013671875, + "logps/ref_rejected": -73.72798156738281, + "logps/rejected": -83.80130004882812, + "loss": 1.1554, + "margin_dpo/margin_mean": 2.675502300262451, + "margin_dpo/margin_std": 4.851496696472168, + "step": 133 + }, + { + "KL/chosen_KL_mean": -6.886777877807617, + "KL/mean": -9.030449867248535, + "KL/rejected_KL_mean": -11.174121856689453, + "KL/std": 5.263517379760742, + "epoch": 0.20256991685563114, + "fcm_dpo/beta": 0.12709717452526093, + "fcm_dpo/delta": -0.17415767908096313, + "fcm_dpo/margin": 4.287341117858887, + "fcm_dpo/q_t": 0.3749292492866516, + "grad_norm": 30.688871383666992, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": 0.29332664608955383, + "logits/rejected": 0.23596280813217163, + "logps/chosen": -60.68135070800781, + "logps/ref_chosen": -53.79457092285156, + "logps/ref_rejected": -74.16741943359375, + "logps/rejected": -85.34153747558594, + "loss": 1.0112, + "margin_dpo/margin_mean": 4.287341594696045, + "margin_dpo/margin_std": 5.231392860412598, + "step": 134 + }, + { + "KL/chosen_KL_mean": -6.964086532592773, + "KL/mean": -8.33128547668457, + "KL/rejected_KL_mean": -9.698490142822266, + "KL/std": 4.9238457679748535, + "epoch": 0.20408163265306123, + "fcm_dpo/beta": 0.12767961621284485, + "fcm_dpo/delta": 0.05650739371776581, + "fcm_dpo/margin": 2.73439884185791, + "fcm_dpo/q_t": 0.42211827635765076, + "grad_norm": 31.442127227783203, + "learning_rate": 4.844676460754862e-07, + "logits/chosen": 0.22777020931243896, + "logits/rejected": 0.19519740343093872, + "logps/chosen": -56.4051628112793, + "logps/ref_chosen": -49.441078186035156, + "logps/ref_rejected": -65.96878051757812, + "logps/rejected": -75.66726684570312, + "loss": 1.1643, + "margin_dpo/margin_mean": 2.734398603439331, + "margin_dpo/margin_std": 5.071935653686523, + "step": 135 + }, + { + "KL/chosen_KL_mean": -8.580795288085938, + "KL/mean": -10.33868408203125, + "KL/rejected_KL_mean": -12.096576690673828, + "KL/std": 5.4477667808532715, + "epoch": 0.20559334845049132, + "fcm_dpo/beta": 0.1240832507610321, + "fcm_dpo/delta": -0.043839938938617706, + "fcm_dpo/margin": 3.5157880783081055, + "fcm_dpo/q_t": 0.40707147121429443, + "grad_norm": 37.98371505737305, + "learning_rate": 4.840055783904106e-07, + "logits/chosen": 0.23468932509422302, + "logits/rejected": 0.1596575379371643, + "logps/chosen": -75.34005737304688, + "logps/ref_chosen": -66.75926208496094, + "logps/ref_rejected": -94.61787414550781, + "logps/rejected": -106.71444702148438, + "loss": 1.1626, + "margin_dpo/margin_mean": 3.515789031982422, + "margin_dpo/margin_std": 6.921201705932617, + "step": 136 + }, + { + "KL/chosen_KL_mean": -8.011341094970703, + "KL/mean": -9.820709228515625, + "KL/rejected_KL_mean": -11.630073547363281, + "KL/std": 5.151564598083496, + "epoch": 0.20710506424792138, + "fcm_dpo/beta": 0.12334209680557251, + "fcm_dpo/delta": -0.05368944630026817, + "fcm_dpo/margin": 3.6187355518341064, + "fcm_dpo/q_t": 0.4003620445728302, + "grad_norm": 30.845598220825195, + "learning_rate": 4.835369650662767e-07, + "logits/chosen": 0.25939062237739563, + "logits/rejected": 0.22949111461639404, + "logps/chosen": -64.79513549804688, + "logps/ref_chosen": -56.78379821777344, + "logps/ref_rejected": -69.89952087402344, + "logps/rejected": -81.52959442138672, + "loss": 1.0943, + "margin_dpo/margin_mean": 3.6187360286712646, + "margin_dpo/margin_std": 5.483453750610352, + "step": 137 + }, + { + "KL/chosen_KL_mean": -9.439451217651367, + "KL/mean": -10.996476173400879, + "KL/rejected_KL_mean": -12.55350112915039, + "KL/std": 5.402915000915527, + "epoch": 0.20861678004535147, + "fcm_dpo/beta": 0.12223243713378906, + "fcm_dpo/delta": 0.021854639053344727, + "fcm_dpo/margin": 3.1140499114990234, + "fcm_dpo/q_t": 0.4149552583694458, + "grad_norm": 32.88602066040039, + "learning_rate": 4.830618192112065e-07, + "logits/chosen": 0.26229894161224365, + "logits/rejected": 0.22495831549167633, + "logps/chosen": -68.20545959472656, + "logps/ref_chosen": -58.766014099121094, + "logps/ref_rejected": -68.12371826171875, + "logps/rejected": -80.67721557617188, + "loss": 1.1688, + "margin_dpo/margin_mean": 3.1140503883361816, + "margin_dpo/margin_std": 6.036888599395752, + "step": 138 + }, + { + "KL/chosen_KL_mean": -9.852619171142578, + "KL/mean": -11.74112319946289, + "KL/rejected_KL_mean": -13.629623413085938, + "KL/std": 5.306262969970703, + "epoch": 0.21012849584278157, + "fcm_dpo/beta": 0.12008741497993469, + "fcm_dpo/delta": -0.06189559027552605, + "fcm_dpo/margin": 3.7770047187805176, + "fcm_dpo/q_t": 0.3965948224067688, + "grad_norm": 34.94479751586914, + "learning_rate": 4.825801541160509e-07, + "logits/chosen": 0.21446341276168823, + "logits/rejected": 0.18413154780864716, + "logps/chosen": -81.07820892333984, + "logps/ref_chosen": -71.2255859375, + "logps/ref_rejected": -82.1834716796875, + "logps/rejected": -95.81309509277344, + "loss": 1.0784, + "margin_dpo/margin_mean": 3.7770049571990967, + "margin_dpo/margin_std": 5.333680629730225, + "step": 139 + }, + { + "KL/chosen_KL_mean": -9.090442657470703, + "KL/mean": -11.611061096191406, + "KL/rejected_KL_mean": -14.13167953491211, + "KL/std": 7.010415077209473, + "epoch": 0.21164021164021163, + "fcm_dpo/beta": 0.1083516851067543, + "fcm_dpo/delta": -0.1772288680076599, + "fcm_dpo/margin": 5.04123592376709, + "fcm_dpo/q_t": 0.3767069876194, + "grad_norm": 32.844200134277344, + "learning_rate": 4.820919832540181e-07, + "logits/chosen": 0.2197141796350479, + "logits/rejected": 0.17169150710105896, + "logps/chosen": -72.36810302734375, + "logps/ref_chosen": -63.27766418457031, + "logps/ref_rejected": -83.30647277832031, + "logps/rejected": -97.43815612792969, + "loss": 1.0681, + "margin_dpo/margin_mean": 5.041234970092773, + "margin_dpo/margin_std": 7.689390182495117, + "step": 140 + }, + { + "KL/chosen_KL_mean": -9.954669952392578, + "KL/mean": -12.317865371704102, + "KL/rejected_KL_mean": -14.681060791015625, + "KL/std": 5.889538764953613, + "epoch": 0.21315192743764172, + "fcm_dpo/beta": 0.09989713877439499, + "fcm_dpo/delta": -0.08965474367141724, + "fcm_dpo/margin": 4.726386070251465, + "fcm_dpo/q_t": 0.3953888714313507, + "grad_norm": 28.874492645263672, + "learning_rate": 4.815973202802966e-07, + "logits/chosen": 0.25134333968162537, + "logits/rejected": 0.20534633100032806, + "logps/chosen": -71.721435546875, + "logps/ref_chosen": -61.76676940917969, + "logps/ref_rejected": -88.60601806640625, + "logps/rejected": -103.28707885742188, + "loss": 1.0964, + "margin_dpo/margin_mean": 4.726386070251465, + "margin_dpo/margin_std": 7.404045104980469, + "step": 141 + }, + { + "KL/chosen_KL_mean": -9.856754302978516, + "KL/mean": -11.797260284423828, + "KL/rejected_KL_mean": -13.73776626586914, + "KL/std": 5.778664588928223, + "epoch": 0.2146636432350718, + "fcm_dpo/beta": 0.10087822377681732, + "fcm_dpo/delta": 0.009604006074368954, + "fcm_dpo/margin": 3.8810181617736816, + "fcm_dpo/q_t": 0.41101446747779846, + "grad_norm": 27.98453140258789, + "learning_rate": 4.810961790316729e-07, + "logits/chosen": 0.255909264087677, + "logits/rejected": 0.23032045364379883, + "logps/chosen": -75.13153076171875, + "logps/ref_chosen": -65.2747802734375, + "logps/ref_rejected": -81.1378173828125, + "logps/rejected": -94.87557983398438, + "loss": 1.1152, + "margin_dpo/margin_mean": 3.8810176849365234, + "margin_dpo/margin_std": 5.909215927124023, + "step": 142 + }, + { + "KL/chosen_KL_mean": -10.505781173706055, + "KL/mean": -12.216259002685547, + "KL/rejected_KL_mean": -13.92673110961914, + "KL/std": 6.674481391906738, + "epoch": 0.2161753590325019, + "fcm_dpo/beta": 0.10392451286315918, + "fcm_dpo/delta": 0.04947058483958244, + "fcm_dpo/margin": 3.4209537506103516, + "fcm_dpo/q_t": 0.4193747937679291, + "grad_norm": 37.11439895629883, + "learning_rate": 4.805885735261454e-07, + "logits/chosen": 0.2696152329444885, + "logits/rejected": 0.25197485089302063, + "logps/chosen": -73.12361145019531, + "logps/ref_chosen": -62.617828369140625, + "logps/ref_rejected": -70.39239501953125, + "logps/rejected": -84.31912231445312, + "loss": 1.2105, + "margin_dpo/margin_mean": 3.4209539890289307, + "margin_dpo/margin_std": 7.7674102783203125, + "step": 143 + }, + { + "KL/chosen_KL_mean": -10.830635070800781, + "KL/mean": -12.981417655944824, + "KL/rejected_KL_mean": -15.132190704345703, + "KL/std": 6.750263214111328, + "epoch": 0.21768707482993196, + "fcm_dpo/beta": 0.1010802835226059, + "fcm_dpo/delta": -0.04181172698736191, + "fcm_dpo/margin": 4.301558971405029, + "fcm_dpo/q_t": 0.40543052554130554, + "grad_norm": 29.94562530517578, + "learning_rate": 4.800745179625307e-07, + "logits/chosen": 0.23211714625358582, + "logits/rejected": 0.20083823800086975, + "logps/chosen": -71.6333236694336, + "logps/ref_chosen": -60.80268859863281, + "logps/ref_rejected": -79.07284545898438, + "logps/rejected": -94.20503234863281, + "loss": 1.1339, + "margin_dpo/margin_mean": 4.301558494567871, + "margin_dpo/margin_std": 7.674080848693848, + "step": 144 + }, + { + "KL/chosen_KL_mean": -10.08212661743164, + "KL/mean": -12.36398696899414, + "KL/rejected_KL_mean": -14.645843505859375, + "KL/std": 6.890389442443848, + "epoch": 0.21919879062736206, + "fcm_dpo/beta": 0.09865110367536545, + "fcm_dpo/delta": -0.05853622406721115, + "fcm_dpo/margin": 4.563719749450684, + "fcm_dpo/q_t": 0.39978501200675964, + "grad_norm": 32.027320861816406, + "learning_rate": 4.795540267200686e-07, + "logits/chosen": 0.20314472913742065, + "logits/rejected": 0.21950089931488037, + "logps/chosen": -84.69358825683594, + "logps/ref_chosen": -74.61146545410156, + "logps/ref_rejected": -83.24461364746094, + "logps/rejected": -97.89045715332031, + "loss": 1.1284, + "margin_dpo/margin_mean": 4.563719749450684, + "margin_dpo/margin_std": 8.117656707763672, + "step": 145 + }, + { + "KL/chosen_KL_mean": -9.851871490478516, + "KL/mean": -12.232227325439453, + "KL/rejected_KL_mean": -14.612586975097656, + "KL/std": 6.808660507202148, + "epoch": 0.22071050642479215, + "fcm_dpo/beta": 0.09611174464225769, + "fcm_dpo/delta": -0.06661855429410934, + "fcm_dpo/margin": 4.760718822479248, + "fcm_dpo/q_t": 0.3982863426208496, + "grad_norm": 27.760150909423828, + "learning_rate": 4.790271143580173e-07, + "logits/chosen": 0.22264136373996735, + "logits/rejected": 0.2081959843635559, + "logps/chosen": -67.69285583496094, + "logps/ref_chosen": -57.84098434448242, + "logps/ref_rejected": -67.47422790527344, + "logps/rejected": -82.0868148803711, + "loss": 1.0995, + "margin_dpo/margin_mean": 4.76071834564209, + "margin_dpo/margin_std": 7.512134075164795, + "step": 146 + }, + { + "KL/chosen_KL_mean": -11.579130172729492, + "KL/mean": -13.471564292907715, + "KL/rejected_KL_mean": -15.364002227783203, + "KL/std": 7.3946404457092285, + "epoch": 0.2222222222222222, + "fcm_dpo/beta": 0.09622933715581894, + "fcm_dpo/delta": 0.040078941732645035, + "fcm_dpo/margin": 3.78487491607666, + "fcm_dpo/q_t": 0.4198254346847534, + "grad_norm": 34.06290817260742, + "learning_rate": 4.784937956152489e-07, + "logits/chosen": 0.24301037192344666, + "logits/rejected": 0.1984240710735321, + "logps/chosen": -78.3926010131836, + "logps/ref_chosen": -66.81346893310547, + "logps/ref_rejected": -81.1796875, + "logps/rejected": -96.54369354248047, + "loss": 1.1853, + "margin_dpo/margin_mean": 3.784874439239502, + "margin_dpo/margin_std": 7.882870674133301, + "step": 147 + }, + { + "KL/chosen_KL_mean": -10.643348693847656, + "KL/mean": -13.073385238647461, + "KL/rejected_KL_mean": -15.50341796875, + "KL/std": 6.8529863357543945, + "epoch": 0.2237339380196523, + "fcm_dpo/beta": 0.09473659843206406, + "fcm_dpo/delta": -0.07015595585107803, + "fcm_dpo/margin": 4.860075950622559, + "fcm_dpo/q_t": 0.3961476683616638, + "grad_norm": 23.567968368530273, + "learning_rate": 4.779540854098347e-07, + "logits/chosen": 0.36633625626564026, + "logits/rejected": 0.2913614511489868, + "logps/chosen": -59.33110427856445, + "logps/ref_chosen": -48.6877555847168, + "logps/ref_rejected": -67.50503540039062, + "logps/rejected": -83.00845336914062, + "loss": 1.1018, + "margin_dpo/margin_mean": 4.860075950622559, + "margin_dpo/margin_std": 7.616449356079102, + "step": 148 + }, + { + "KL/chosen_KL_mean": -12.942892074584961, + "KL/mean": -15.660242080688477, + "KL/rejected_KL_mean": -18.37759017944336, + "KL/std": 7.375458717346191, + "epoch": 0.2252456538170824, + "fcm_dpo/beta": 0.08893537521362305, + "fcm_dpo/delta": -0.09736001491546631, + "fcm_dpo/margin": 5.434703350067139, + "fcm_dpo/q_t": 0.3881411850452423, + "grad_norm": 24.1125431060791, + "learning_rate": 4.774079988386296e-07, + "logits/chosen": 0.21307995915412903, + "logits/rejected": 0.16604451835155487, + "logps/chosen": -68.086669921875, + "logps/ref_chosen": -55.143775939941406, + "logps/ref_rejected": -64.79888916015625, + "logps/rejected": -83.17648315429688, + "loss": 1.0482, + "margin_dpo/margin_mean": 5.4347028732299805, + "margin_dpo/margin_std": 6.959212303161621, + "step": 149 + }, + { + "KL/chosen_KL_mean": -10.891637802124023, + "KL/mean": -14.504773139953613, + "KL/rejected_KL_mean": -18.117908477783203, + "KL/std": 8.100428581237793, + "epoch": 0.22675736961451248, + "fcm_dpo/beta": 0.07920362055301666, + "fcm_dpo/delta": -0.21006450057029724, + "fcm_dpo/margin": 7.226279258728027, + "fcm_dpo/q_t": 0.36848869919776917, + "grad_norm": 21.305072784423828, + "learning_rate": 4.768555511768486e-07, + "logits/chosen": 0.24731308221817017, + "logits/rejected": 0.202421173453331, + "logps/chosen": -78.36238098144531, + "logps/ref_chosen": -67.47074890136719, + "logps/ref_rejected": -89.21170806884766, + "logps/rejected": -107.32962036132812, + "loss": 0.9942, + "margin_dpo/margin_mean": 7.226279258728027, + "margin_dpo/margin_std": 8.23183822631836, + "step": 150 + }, + { + "KL/chosen_KL_mean": -9.562496185302734, + "KL/mean": -13.620951652526855, + "KL/rejected_KL_mean": -17.679405212402344, + "KL/std": 8.622806549072266, + "epoch": 0.22826908541194255, + "fcm_dpo/beta": 0.07037020474672318, + "fcm_dpo/delta": -0.2068619728088379, + "fcm_dpo/margin": 8.116912841796875, + "fcm_dpo/q_t": 0.3663896322250366, + "grad_norm": 19.14907455444336, + "learning_rate": 4.762967578776406e-07, + "logits/chosen": 0.2455185353755951, + "logits/rejected": 0.1907639503479004, + "logps/chosen": -62.02204132080078, + "logps/ref_chosen": -52.45954132080078, + "logps/ref_rejected": -79.0630111694336, + "logps/rejected": -96.74241638183594, + "loss": 0.9791, + "margin_dpo/margin_mean": 8.116911888122559, + "margin_dpo/margin_std": 8.724565505981445, + "step": 151 + }, + { + "KL/chosen_KL_mean": -12.31747817993164, + "KL/mean": -14.943049430847168, + "KL/rejected_KL_mean": -17.568626403808594, + "KL/std": 8.21334171295166, + "epoch": 0.22978080120937264, + "fcm_dpo/beta": 0.07021472603082657, + "fcm_dpo/delta": 0.0327349528670311, + "fcm_dpo/margin": 5.2511467933654785, + "fcm_dpo/q_t": 0.4175514876842499, + "grad_norm": 21.101016998291016, + "learning_rate": 4.757316345716553e-07, + "logits/chosen": 0.3428304195404053, + "logits/rejected": 0.2872798442840576, + "logps/chosen": -68.87130737304688, + "logps/ref_chosen": -56.5538330078125, + "logps/ref_rejected": -76.55074310302734, + "logps/rejected": -94.11936950683594, + "loss": 1.1541, + "margin_dpo/margin_mean": 5.25114631652832, + "margin_dpo/margin_std": 9.311445236206055, + "step": 152 + }, + { + "KL/chosen_KL_mean": -11.60400390625, + "KL/mean": -14.734918594360352, + "KL/rejected_KL_mean": -17.865833282470703, + "KL/std": 8.638544082641602, + "epoch": 0.23129251700680273, + "fcm_dpo/beta": 0.06840331852436066, + "fcm_dpo/delta": -0.03252699226140976, + "fcm_dpo/margin": 6.261826515197754, + "fcm_dpo/q_t": 0.4008117616176605, + "grad_norm": 20.02750587463379, + "learning_rate": 4.751601970666064e-07, + "logits/chosen": 0.24029231071472168, + "logits/rejected": 0.203395813703537, + "logps/chosen": -79.61090087890625, + "logps/ref_chosen": -68.00689697265625, + "logps/ref_rejected": -74.83482360839844, + "logps/rejected": -92.70065307617188, + "loss": 1.0756, + "margin_dpo/margin_mean": 6.261826515197754, + "margin_dpo/margin_std": 8.131772994995117, + "step": 153 + }, + { + "KL/chosen_KL_mean": -12.812858581542969, + "KL/mean": -15.129095077514648, + "KL/rejected_KL_mean": -17.445331573486328, + "KL/std": 8.995925903320312, + "epoch": 0.2328042328042328, + "fcm_dpo/beta": 0.0704408809542656, + "fcm_dpo/delta": 0.08183015882968903, + "fcm_dpo/margin": 4.63247537612915, + "fcm_dpo/q_t": 0.42660561203956604, + "grad_norm": 20.98277473449707, + "learning_rate": 4.745824613468292e-07, + "logits/chosen": 0.3132394552230835, + "logits/rejected": 0.3103998899459839, + "logps/chosen": -72.035400390625, + "logps/ref_chosen": -59.222537994384766, + "logps/ref_rejected": -64.19131469726562, + "logps/rejected": -81.63665008544922, + "loss": 1.1868, + "margin_dpo/margin_mean": 4.63247537612915, + "margin_dpo/margin_std": 9.253047943115234, + "step": 154 + }, + { + "KL/chosen_KL_mean": -13.649024963378906, + "KL/mean": -16.94133949279785, + "KL/rejected_KL_mean": -20.233646392822266, + "KL/std": 8.983402252197266, + "epoch": 0.23431594860166288, + "fcm_dpo/beta": 0.06851907074451447, + "fcm_dpo/delta": -0.06188790500164032, + "fcm_dpo/margin": 6.584621906280518, + "fcm_dpo/q_t": 0.39981919527053833, + "grad_norm": 20.74521255493164, + "learning_rate": 4.7399844357283393e-07, + "logits/chosen": 0.3174290060997009, + "logits/rejected": 0.29693371057510376, + "logps/chosen": -82.10372161865234, + "logps/ref_chosen": -68.45469665527344, + "logps/ref_rejected": -77.91763305664062, + "logps/rejected": -98.15128326416016, + "loss": 1.1242, + "margin_dpo/margin_mean": 6.584621429443359, + "margin_dpo/margin_std": 11.35929012298584, + "step": 155 + }, + { + "KL/chosen_KL_mean": -13.674962997436523, + "KL/mean": -17.565982818603516, + "KL/rejected_KL_mean": -21.45700454711914, + "KL/std": 10.152885437011719, + "epoch": 0.23582766439909297, + "fcm_dpo/beta": 0.06530015915632248, + "fcm_dpo/delta": -0.12741731107234955, + "fcm_dpo/margin": 7.782038688659668, + "fcm_dpo/q_t": 0.38315021991729736, + "grad_norm": 20.189838409423828, + "learning_rate": 4.7340816008085305e-07, + "logits/chosen": 0.26459452509880066, + "logits/rejected": 0.21891286969184875, + "logps/chosen": -80.94456481933594, + "logps/ref_chosen": -67.26959991455078, + "logps/ref_rejected": -86.95914459228516, + "logps/rejected": -108.41615295410156, + "loss": 1.0288, + "margin_dpo/margin_mean": 7.782039642333984, + "margin_dpo/margin_std": 9.416492462158203, + "step": 156 + }, + { + "KL/chosen_KL_mean": -12.285833358764648, + "KL/mean": -15.452082633972168, + "KL/rejected_KL_mean": -18.618331909179688, + "KL/std": 9.885778427124023, + "epoch": 0.23733938019652306, + "fcm_dpo/beta": 0.06017298251390457, + "fcm_dpo/delta": -0.09840401262044907, + "fcm_dpo/margin": 6.332496643066406, + "fcm_dpo/q_t": 0.40991830825805664, + "grad_norm": 18.193811416625977, + "learning_rate": 4.728116273823847e-07, + "logits/chosen": 0.2879556715488434, + "logits/rejected": 0.26909253001213074, + "logps/chosen": -67.05870819091797, + "logps/ref_chosen": -54.77287292480469, + "logps/ref_rejected": -63.87866973876953, + "logps/rejected": -82.49700164794922, + "loss": 1.1274, + "margin_dpo/margin_mean": 6.332496166229248, + "margin_dpo/margin_std": 9.572912216186523, + "step": 157 + }, + { + "KL/chosen_KL_mean": -13.837478637695312, + "KL/mean": -17.254711151123047, + "KL/rejected_KL_mean": -20.671947479248047, + "KL/std": 10.21955680847168, + "epoch": 0.23885109599395313, + "fcm_dpo/beta": 0.060004714876413345, + "fcm_dpo/delta": -0.011527525261044502, + "fcm_dpo/margin": 6.8344645500183105, + "fcm_dpo/q_t": 0.4079589247703552, + "grad_norm": 18.990568161010742, + "learning_rate": 4.7220886216373085e-07, + "logits/chosen": 0.30475467443466187, + "logits/rejected": 0.266677588224411, + "logps/chosen": -78.76019287109375, + "logps/ref_chosen": -64.92271423339844, + "logps/ref_rejected": -82.23789978027344, + "logps/rejected": -102.90984344482422, + "loss": 1.116, + "margin_dpo/margin_mean": 6.8344645500183105, + "margin_dpo/margin_std": 10.858131408691406, + "step": 158 + }, + { + "KL/chosen_KL_mean": -14.046161651611328, + "KL/mean": -17.997652053833008, + "KL/rejected_KL_mean": -21.949138641357422, + "KL/std": 11.05732250213623, + "epoch": 0.24036281179138322, + "fcm_dpo/beta": 0.05865692347288132, + "fcm_dpo/delta": -0.07415425777435303, + "fcm_dpo/margin": 7.902973651885986, + "fcm_dpo/q_t": 0.39414629340171814, + "grad_norm": 19.708227157592773, + "learning_rate": 4.715998812855304e-07, + "logits/chosen": 0.33380645513534546, + "logits/rejected": 0.29338324069976807, + "logps/chosen": -71.09315490722656, + "logps/ref_chosen": -57.046993255615234, + "logps/ref_rejected": -73.32441711425781, + "logps/rejected": -95.2735595703125, + "loss": 1.0867, + "margin_dpo/margin_mean": 7.902973175048828, + "margin_dpo/margin_std": 11.675771713256836, + "step": 159 + }, + { + "KL/chosen_KL_mean": -15.937501907348633, + "KL/mean": -19.319564819335938, + "KL/rejected_KL_mean": -22.701629638671875, + "KL/std": 10.381500244140625, + "epoch": 0.2418745275888133, + "fcm_dpo/beta": 0.05758850276470184, + "fcm_dpo/delta": 0.011783316731452942, + "fcm_dpo/margin": 6.764126777648926, + "fcm_dpo/q_t": 0.4151209890842438, + "grad_norm": 18.15164566040039, + "learning_rate": 4.7098470178228755e-07, + "logits/chosen": 0.19479069113731384, + "logits/rejected": 0.14844146370887756, + "logps/chosen": -65.74441528320312, + "logps/ref_chosen": -49.806915283203125, + "logps/ref_rejected": -68.3370132446289, + "logps/rejected": -91.03864288330078, + "loss": 1.1425, + "margin_dpo/margin_mean": 6.764126300811768, + "margin_dpo/margin_std": 11.992984771728516, + "step": 160 + }, + { + "KL/chosen_KL_mean": -15.949237823486328, + "KL/mean": -19.665210723876953, + "KL/rejected_KL_mean": -23.381183624267578, + "KL/std": 10.27374267578125, + "epoch": 0.24338624338624337, + "fcm_dpo/beta": 0.05700618401169777, + "fcm_dpo/delta": -0.02708788402378559, + "fcm_dpo/margin": 7.431947708129883, + "fcm_dpo/q_t": 0.4038432836532593, + "grad_norm": 17.391952514648438, + "learning_rate": 4.703633408618955e-07, + "logits/chosen": 0.34192147850990295, + "logits/rejected": 0.3018055856227875, + "logps/chosen": -68.44972229003906, + "logps/ref_chosen": -52.50048828125, + "logps/ref_rejected": -66.04540252685547, + "logps/rejected": -89.42658996582031, + "loss": 1.109, + "margin_dpo/margin_mean": 7.431947708129883, + "margin_dpo/margin_std": 11.519304275512695, + "step": 161 + }, + { + "KL/chosen_KL_mean": -16.880916595458984, + "KL/mean": -21.874000549316406, + "KL/rejected_KL_mean": -26.867088317871094, + "KL/std": 11.709345817565918, + "epoch": 0.24489795918367346, + "fcm_dpo/beta": 0.052794016897678375, + "fcm_dpo/delta": -0.1518443524837494, + "fcm_dpo/margin": 9.98617172241211, + "fcm_dpo/q_t": 0.376575231552124, + "grad_norm": 18.568201065063477, + "learning_rate": 4.697358159051549e-07, + "logits/chosen": 0.3624524474143982, + "logits/rejected": 0.31142184138298035, + "logps/chosen": -86.35011291503906, + "logps/ref_chosen": -69.46919250488281, + "logps/ref_rejected": -92.00952911376953, + "logps/rejected": -118.87661743164062, + "loss": 1.0213, + "margin_dpo/margin_mean": 9.98617172241211, + "margin_dpo/margin_std": 12.183841705322266, + "step": 162 + }, + { + "KL/chosen_KL_mean": -14.589275360107422, + "KL/mean": -19.729949951171875, + "KL/rejected_KL_mean": -24.870624542236328, + "KL/std": 11.731868743896484, + "epoch": 0.24640967498110355, + "fcm_dpo/beta": 0.05017222464084625, + "fcm_dpo/delta": -0.13923859596252441, + "fcm_dpo/margin": 10.28134822845459, + "fcm_dpo/q_t": 0.3796467185020447, + "grad_norm": 16.747222900390625, + "learning_rate": 4.691021444652876e-07, + "logits/chosen": 0.30114084482192993, + "logits/rejected": 0.25208836793899536, + "logps/chosen": -65.20310974121094, + "logps/ref_chosen": -50.613834381103516, + "logps/ref_rejected": -74.62033081054688, + "logps/rejected": -99.49095153808594, + "loss": 1.0379, + "margin_dpo/margin_mean": 10.281349182128906, + "margin_dpo/margin_std": 12.120843887329102, + "step": 163 + }, + { + "KL/chosen_KL_mean": -15.791715621948242, + "KL/mean": -20.810779571533203, + "KL/rejected_KL_mean": -25.829849243164062, + "KL/std": 12.496322631835938, + "epoch": 0.24792139077853365, + "fcm_dpo/beta": 0.04662519693374634, + "fcm_dpo/delta": -0.07952342927455902, + "fcm_dpo/margin": 10.038135528564453, + "fcm_dpo/q_t": 0.3925841450691223, + "grad_norm": 14.853240013122559, + "learning_rate": 4.6846234426744624e-07, + "logits/chosen": 0.29247328639030457, + "logits/rejected": 0.22746919095516205, + "logps/chosen": -70.63983154296875, + "logps/ref_chosen": -54.848114013671875, + "logps/ref_rejected": -79.0630111694336, + "logps/rejected": -104.89286041259766, + "loss": 1.0734, + "margin_dpo/margin_mean": 10.038135528564453, + "margin_dpo/margin_std": 13.581424713134766, + "step": 164 + }, + { + "KL/chosen_KL_mean": -17.847217559814453, + "KL/mean": -22.394338607788086, + "KL/rejected_KL_mean": -26.94145965576172, + "KL/std": 12.855323791503906, + "epoch": 0.2494331065759637, + "fcm_dpo/beta": 0.04560963064432144, + "fcm_dpo/delta": -0.01796562224626541, + "fcm_dpo/margin": 9.09423828125, + "fcm_dpo/q_t": 0.404633492231369, + "grad_norm": 15.527134895324707, + "learning_rate": 4.678164332082175e-07, + "logits/chosen": 0.3545019328594208, + "logits/rejected": 0.29826444387435913, + "logps/chosen": -68.93643188476562, + "logps/ref_chosen": -51.089210510253906, + "logps/ref_rejected": -71.23370361328125, + "logps/rejected": -98.17515563964844, + "loss": 1.1163, + "margin_dpo/margin_mean": 9.09423828125, + "margin_dpo/margin_std": 13.98188591003418, + "step": 165 + }, + { + "KL/chosen_KL_mean": -17.026296615600586, + "KL/mean": -20.65865707397461, + "KL/rejected_KL_mean": -24.29102325439453, + "KL/std": 13.004007339477539, + "epoch": 0.2509448223733938, + "fcm_dpo/beta": 0.04612819850444794, + "fcm_dpo/delta": 0.07232268154621124, + "fcm_dpo/margin": 7.2647247314453125, + "fcm_dpo/q_t": 0.42566025257110596, + "grad_norm": 18.30398941040039, + "learning_rate": 4.6716442935512214e-07, + "logits/chosen": 0.3076387047767639, + "logits/rejected": 0.2137855738401413, + "logps/chosen": -80.21710205078125, + "logps/ref_chosen": -63.19081115722656, + "logps/ref_rejected": -93.8402099609375, + "logps/rejected": -118.13123321533203, + "loss": 1.157, + "margin_dpo/margin_mean": 7.264724254608154, + "margin_dpo/margin_std": 12.455879211425781, + "step": 166 + }, + { + "KL/chosen_KL_mean": -14.038339614868164, + "KL/mean": -19.74565887451172, + "KL/rejected_KL_mean": -25.452983856201172, + "KL/std": 13.267440795898438, + "epoch": 0.25245653817082386, + "fcm_dpo/beta": 0.04366353154182434, + "fcm_dpo/delta": -0.12011955678462982, + "fcm_dpo/margin": 11.414639472961426, + "fcm_dpo/q_t": 0.3832022547721863, + "grad_norm": 15.69991683959961, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": 0.30109137296676636, + "logits/rejected": 0.2652606666088104, + "logps/chosen": -72.96261596679688, + "logps/ref_chosen": -58.92427062988281, + "logps/ref_rejected": -72.97377014160156, + "logps/rejected": -98.4267578125, + "loss": 1.0246, + "margin_dpo/margin_mean": 11.414640426635742, + "margin_dpo/margin_std": 13.03073501586914, + "step": 167 + }, + { + "KL/chosen_KL_mean": -16.865724563598633, + "KL/mean": -21.457231521606445, + "KL/rejected_KL_mean": -26.048744201660156, + "KL/std": 14.453012466430664, + "epoch": 0.25396825396825395, + "fcm_dpo/beta": 0.043425120413303375, + "fcm_dpo/delta": 0.0012646140530705452, + "fcm_dpo/margin": 9.183013916015625, + "fcm_dpo/q_t": 0.41033321619033813, + "grad_norm": 16.60943603515625, + "learning_rate": 4.6584221638904767e-07, + "logits/chosen": 0.310530424118042, + "logits/rejected": 0.27260318398475647, + "logps/chosen": -82.51710510253906, + "logps/ref_chosen": -65.65138244628906, + "logps/ref_rejected": -79.71418762207031, + "logps/rejected": -105.76293182373047, + "loss": 1.1069, + "margin_dpo/margin_mean": 9.183013916015625, + "margin_dpo/margin_std": 13.43674087524414, + "step": 168 + }, + { + "KL/chosen_KL_mean": -13.360231399536133, + "KL/mean": -18.598499298095703, + "KL/rejected_KL_mean": -23.836761474609375, + "KL/std": 14.224285125732422, + "epoch": 0.25547996976568405, + "fcm_dpo/beta": 0.04259002208709717, + "fcm_dpo/delta": -0.0533658042550087, + "fcm_dpo/margin": 10.476530075073242, + "fcm_dpo/q_t": 0.40058034658432007, + "grad_norm": 17.076465606689453, + "learning_rate": 4.651720442612075e-07, + "logits/chosen": 0.38942497968673706, + "logits/rejected": 0.3538346290588379, + "logps/chosen": -74.78610229492188, + "logps/ref_chosen": -61.425865173339844, + "logps/ref_rejected": -76.09590148925781, + "logps/rejected": -99.93266296386719, + "loss": 1.107, + "margin_dpo/margin_mean": 10.476531028747559, + "margin_dpo/margin_std": 16.978721618652344, + "step": 169 + }, + { + "KL/chosen_KL_mean": -12.65814208984375, + "KL/mean": -17.938709259033203, + "KL/rejected_KL_mean": -23.219276428222656, + "KL/std": 14.04989242553711, + "epoch": 0.25699168556311414, + "fcm_dpo/beta": 0.041654251515865326, + "fcm_dpo/delta": -0.04596564173698425, + "fcm_dpo/margin": 10.561136245727539, + "fcm_dpo/q_t": 0.40057674050331116, + "grad_norm": 14.404908180236816, + "learning_rate": 4.6449585330874425e-07, + "logits/chosen": 0.3427773118019104, + "logits/rejected": 0.34061866998672485, + "logps/chosen": -69.31133270263672, + "logps/ref_chosen": -56.65319061279297, + "logps/ref_rejected": -63.45965576171875, + "logps/rejected": -86.6789321899414, + "loss": 1.1222, + "margin_dpo/margin_mean": 10.561136245727539, + "margin_dpo/margin_std": 17.518104553222656, + "step": 170 + }, + { + "KL/chosen_KL_mean": -14.985160827636719, + "KL/mean": -20.769207000732422, + "KL/rejected_KL_mean": -26.55325698852539, + "KL/std": 16.237634658813477, + "epoch": 0.2585034013605442, + "fcm_dpo/beta": 0.038949161767959595, + "fcm_dpo/delta": -0.06474211066961288, + "fcm_dpo/margin": 11.56810188293457, + "fcm_dpo/q_t": 0.39703309535980225, + "grad_norm": 14.443869590759277, + "learning_rate": 4.6381366244617224e-07, + "logits/chosen": 0.3847231864929199, + "logits/rejected": 0.3326500654220581, + "logps/chosen": -78.71992492675781, + "logps/ref_chosen": -63.73476028442383, + "logps/ref_rejected": -78.50328063964844, + "logps/rejected": -105.05653381347656, + "loss": 1.1113, + "margin_dpo/margin_mean": 11.56810188293457, + "margin_dpo/margin_std": 18.460580825805664, + "step": 171 + }, + { + "KL/chosen_KL_mean": -16.810169219970703, + "KL/mean": -22.31606674194336, + "KL/rejected_KL_mean": -27.82196044921875, + "KL/std": 16.634769439697266, + "epoch": 0.2600151171579743, + "fcm_dpo/beta": 0.0396546795964241, + "fcm_dpo/delta": -0.04476618766784668, + "fcm_dpo/margin": 11.011789321899414, + "fcm_dpo/q_t": 0.40239837765693665, + "grad_norm": 16.143247604370117, + "learning_rate": 4.631254907558365e-07, + "logits/chosen": 0.3681456744670868, + "logits/rejected": 0.3081226348876953, + "logps/chosen": -69.01193237304688, + "logps/ref_chosen": -52.201759338378906, + "logps/ref_rejected": -82.85285949707031, + "logps/rejected": -110.67481994628906, + "loss": 1.1085, + "margin_dpo/margin_mean": 11.011789321899414, + "margin_dpo/margin_std": 17.142223358154297, + "step": 172 + }, + { + "KL/chosen_KL_mean": -15.272012710571289, + "KL/mean": -21.283023834228516, + "KL/rejected_KL_mean": -27.29404067993164, + "KL/std": 17.057445526123047, + "epoch": 0.2615268329554044, + "fcm_dpo/beta": 0.03659302368760109, + "fcm_dpo/delta": -0.052808038890361786, + "fcm_dpo/margin": 12.02202320098877, + "fcm_dpo/q_t": 0.4022437334060669, + "grad_norm": 14.902726173400879, + "learning_rate": 4.624313574873786e-07, + "logits/chosen": 0.39478111267089844, + "logits/rejected": 0.30247700214385986, + "logps/chosen": -70.70674133300781, + "logps/ref_chosen": -55.434722900390625, + "logps/ref_rejected": -77.81967163085938, + "logps/rejected": -105.11371612548828, + "loss": 1.1372, + "margin_dpo/margin_mean": 12.022025108337402, + "margin_dpo/margin_std": 20.99781036376953, + "step": 173 + }, + { + "KL/chosen_KL_mean": -17.438180923461914, + "KL/mean": -24.091327667236328, + "KL/rejected_KL_mean": -30.744476318359375, + "KL/std": 17.439682006835938, + "epoch": 0.26303854875283444, + "fcm_dpo/beta": 0.03558962047100067, + "fcm_dpo/delta": -0.08698572218418121, + "fcm_dpo/margin": 13.306293487548828, + "fcm_dpo/q_t": 0.3912256360054016, + "grad_norm": 15.202779769897461, + "learning_rate": 4.61731282057198e-07, + "logits/chosen": 0.3748180866241455, + "logits/rejected": 0.305186927318573, + "logps/chosen": -74.61013793945312, + "logps/ref_chosen": -57.17195129394531, + "logps/ref_rejected": -85.47578430175781, + "logps/rejected": -116.22026062011719, + "loss": 1.0715, + "margin_dpo/margin_mean": 13.306294441223145, + "margin_dpo/margin_std": 18.859642028808594, + "step": 174 + }, + { + "KL/chosen_KL_mean": -14.990503311157227, + "KL/mean": -22.452695846557617, + "KL/rejected_KL_mean": -29.914894104003906, + "KL/std": 17.86532211303711, + "epoch": 0.26455026455026454, + "fcm_dpo/beta": 0.03374076634645462, + "fcm_dpo/delta": -0.12154931575059891, + "fcm_dpo/margin": 14.924393653869629, + "fcm_dpo/q_t": 0.38714203238487244, + "grad_norm": 14.585382461547852, + "learning_rate": 4.6102528404790965e-07, + "logits/chosen": 0.4304041266441345, + "logits/rejected": 0.3987458050251007, + "logps/chosen": -82.6561279296875, + "logps/ref_chosen": -67.6656265258789, + "logps/ref_rejected": -84.36766815185547, + "logps/rejected": -114.28256225585938, + "loss": 1.0598, + "margin_dpo/margin_mean": 14.924394607543945, + "margin_dpo/margin_std": 20.989219665527344, + "step": 175 + }, + { + "KL/chosen_KL_mean": -19.427631378173828, + "KL/mean": -24.25865936279297, + "KL/rejected_KL_mean": -29.08969497680664, + "KL/std": 19.192419052124023, + "epoch": 0.2660619803476946, + "fcm_dpo/beta": 0.033753879368305206, + "fcm_dpo/delta": 0.08193635195493698, + "fcm_dpo/margin": 9.66206169128418, + "fcm_dpo/q_t": 0.4294675588607788, + "grad_norm": 17.79058265686035, + "learning_rate": 4.603133832077953e-07, + "logits/chosen": 0.3631625771522522, + "logits/rejected": 0.33612459897994995, + "logps/chosen": -97.28639221191406, + "logps/ref_chosen": -77.8587646484375, + "logps/ref_rejected": -81.08732604980469, + "logps/rejected": -110.17701721191406, + "loss": 1.2166, + "margin_dpo/margin_mean": 9.66206169128418, + "margin_dpo/margin_std": 21.81412696838379, + "step": 176 + }, + { + "KL/chosen_KL_mean": -14.337238311767578, + "KL/mean": -23.852760314941406, + "KL/rejected_KL_mean": -33.3682861328125, + "KL/std": 19.595916748046875, + "epoch": 0.2675736961451247, + "fcm_dpo/beta": 0.03083290532231331, + "fcm_dpo/delta": -0.2311885952949524, + "fcm_dpo/margin": 19.031044006347656, + "fcm_dpo/q_t": 0.360975980758667, + "grad_norm": 17.56546974182129, + "learning_rate": 4.5959559945025183e-07, + "logits/chosen": 0.50090491771698, + "logits/rejected": 0.4008740186691284, + "logps/chosen": -69.55763244628906, + "logps/ref_chosen": -55.22039794921875, + "logps/ref_rejected": -92.54973602294922, + "logps/rejected": -125.91802215576172, + "loss": 0.9693, + "margin_dpo/margin_mean": 19.031042098999023, + "margin_dpo/margin_std": 20.025856018066406, + "step": 177 + }, + { + "KL/chosen_KL_mean": -17.92155647277832, + "KL/mean": -23.787391662597656, + "KL/rejected_KL_mean": -29.653221130371094, + "KL/std": 18.697711944580078, + "epoch": 0.2690854119425548, + "fcm_dpo/beta": 0.02928192727267742, + "fcm_dpo/delta": -0.04489610344171524, + "fcm_dpo/margin": 11.731661796569824, + "fcm_dpo/q_t": 0.4197039306163788, + "grad_norm": 17.016468048095703, + "learning_rate": 4.588719528532341e-07, + "logits/chosen": 0.3574860692024231, + "logits/rejected": 0.3063201308250427, + "logps/chosen": -78.73204803466797, + "logps/ref_chosen": -60.81049346923828, + "logps/ref_rejected": -81.12973022460938, + "logps/rejected": -110.78294372558594, + "loss": 1.1482, + "margin_dpo/margin_mean": 11.731661796569824, + "margin_dpo/margin_std": 18.86032485961914, + "step": 178 + }, + { + "KL/chosen_KL_mean": -18.39486312866211, + "KL/mean": -25.84204864501953, + "KL/rejected_KL_mean": -33.28923797607422, + "KL/std": 20.058395385742188, + "epoch": 0.2705971277399849, + "fcm_dpo/beta": 0.02877836301922798, + "fcm_dpo/delta": -0.03301222249865532, + "fcm_dpo/margin": 14.894369125366211, + "fcm_dpo/q_t": 0.4027270972728729, + "grad_norm": 14.578384399414062, + "learning_rate": 4.581424636586928e-07, + "logits/chosen": 0.39680802822113037, + "logits/rejected": 0.38149362802505493, + "logps/chosen": -84.06657409667969, + "logps/ref_chosen": -65.67171478271484, + "logps/ref_rejected": -75.32586669921875, + "logps/rejected": -108.61510467529297, + "loss": 1.1094, + "margin_dpo/margin_mean": 14.894369125366211, + "margin_dpo/margin_std": 23.633628845214844, + "step": 179 + }, + { + "KL/chosen_KL_mean": -14.09145736694336, + "KL/mean": -20.09218978881836, + "KL/rejected_KL_mean": -26.092912673950195, + "KL/std": 19.371274948120117, + "epoch": 0.272108843537415, + "fcm_dpo/beta": 0.029514048248529434, + "fcm_dpo/delta": 0.050767071545124054, + "fcm_dpo/margin": 12.001461029052734, + "fcm_dpo/q_t": 0.4218587279319763, + "grad_norm": 13.016547203063965, + "learning_rate": 4.5740715227200897e-07, + "logits/chosen": 0.24703602492809296, + "logits/rejected": 0.228146493434906, + "logps/chosen": -70.77426147460938, + "logps/ref_chosen": -56.68280792236328, + "logps/ref_rejected": -64.94414520263672, + "logps/rejected": -91.03706359863281, + "loss": 1.1642, + "margin_dpo/margin_mean": 12.001461029052734, + "margin_dpo/margin_std": 22.415281295776367, + "step": 180 + }, + { + "KL/chosen_KL_mean": -12.105340957641602, + "KL/mean": -20.832984924316406, + "KL/rejected_KL_mean": -29.560623168945312, + "KL/std": 20.69134521484375, + "epoch": 0.273620559334845, + "fcm_dpo/beta": 0.028023969382047653, + "fcm_dpo/delta": -0.10585962235927582, + "fcm_dpo/margin": 17.45528793334961, + "fcm_dpo/q_t": 0.38700011372566223, + "grad_norm": 15.768712997436523, + "learning_rate": 4.566660392614228e-07, + "logits/chosen": 0.42835646867752075, + "logits/rejected": 0.38607752323150635, + "logps/chosen": -72.88139343261719, + "logps/ref_chosen": -60.77604675292969, + "logps/ref_rejected": -83.98361206054688, + "logps/rejected": -113.54423522949219, + "loss": 1.031, + "margin_dpo/margin_mean": 17.45528793334961, + "margin_dpo/margin_std": 20.554719924926758, + "step": 181 + }, + { + "KL/chosen_KL_mean": -19.097064971923828, + "KL/mean": -28.098819732666016, + "KL/rejected_KL_mean": -37.1005744934082, + "KL/std": 22.797513961791992, + "epoch": 0.2751322751322751, + "fcm_dpo/beta": 0.026667404919862747, + "fcm_dpo/delta": -0.0935642197728157, + "fcm_dpo/margin": 18.003517150878906, + "fcm_dpo/q_t": 0.39077216386795044, + "grad_norm": 16.841510772705078, + "learning_rate": 4.5591914535745817e-07, + "logits/chosen": 0.4146467447280884, + "logits/rejected": 0.32692649960517883, + "logps/chosen": -79.35084533691406, + "logps/ref_chosen": -60.2537841796875, + "logps/ref_rejected": -89.7706298828125, + "logps/rejected": -126.87120056152344, + "loss": 1.071, + "margin_dpo/margin_mean": 18.003517150878906, + "margin_dpo/margin_std": 25.671062469482422, + "step": 182 + }, + { + "KL/chosen_KL_mean": -23.710100173950195, + "KL/mean": -27.463716506958008, + "KL/rejected_KL_mean": -31.217327117919922, + "KL/std": 21.911300659179688, + "epoch": 0.2766439909297052, + "fcm_dpo/beta": 0.027505898848176003, + "fcm_dpo/delta": 0.07738558948040009, + "fcm_dpo/margin": 7.507230281829834, + "fcm_dpo/q_t": 0.4531930983066559, + "grad_norm": 14.247156143188477, + "learning_rate": 4.551664914523433e-07, + "logits/chosen": 0.40092331171035767, + "logits/rejected": 0.3776063919067383, + "logps/chosen": -85.47151947021484, + "logps/ref_chosen": -61.76142120361328, + "logps/ref_rejected": -72.54627990722656, + "logps/rejected": -103.76361083984375, + "loss": 1.2818, + "margin_dpo/margin_mean": 7.507230281829834, + "margin_dpo/margin_std": 22.548179626464844, + "step": 183 + }, + { + "KL/chosen_KL_mean": -13.734045028686523, + "KL/mean": -20.633419036865234, + "KL/rejected_KL_mean": -27.532791137695312, + "KL/std": 19.101581573486328, + "epoch": 0.2781557067271353, + "fcm_dpo/beta": 0.02745695784687996, + "fcm_dpo/delta": 0.02308075502514839, + "fcm_dpo/margin": 13.798742294311523, + "fcm_dpo/q_t": 0.4148487448692322, + "grad_norm": 16.434070587158203, + "learning_rate": 4.544080985994258e-07, + "logits/chosen": 0.4927828907966614, + "logits/rejected": 0.42545321583747864, + "logps/chosen": -60.57476806640625, + "logps/ref_chosen": -46.840721130371094, + "logps/ref_rejected": -69.3609390258789, + "logps/rejected": -96.89373016357422, + "loss": 1.1172, + "margin_dpo/margin_mean": 13.798742294311523, + "margin_dpo/margin_std": 20.220142364501953, + "step": 184 + }, + { + "KL/chosen_KL_mean": -18.83686065673828, + "KL/mean": -27.118553161621094, + "KL/rejected_KL_mean": -35.40023422241211, + "KL/std": 23.780193328857422, + "epoch": 0.2796674225245654, + "fcm_dpo/beta": 0.026584237813949585, + "fcm_dpo/delta": -0.05228573456406593, + "fcm_dpo/margin": 16.563377380371094, + "fcm_dpo/q_t": 0.40145543217658997, + "grad_norm": 14.18293571472168, + "learning_rate": 4.5364398801258394e-07, + "logits/chosen": 0.43396925926208496, + "logits/rejected": 0.38681963086128235, + "logps/chosen": -71.15800476074219, + "logps/ref_chosen": -52.32114028930664, + "logps/ref_rejected": -68.3885726928711, + "logps/rejected": -103.78880310058594, + "loss": 1.1334, + "margin_dpo/margin_mean": 16.56337547302246, + "margin_dpo/margin_std": 28.58876609802246, + "step": 185 + }, + { + "KL/chosen_KL_mean": -16.70131492614746, + "KL/mean": -25.68690299987793, + "KL/rejected_KL_mean": -34.67249298095703, + "KL/std": 23.681175231933594, + "epoch": 0.2811791383219955, + "fcm_dpo/beta": 0.025996234267950058, + "fcm_dpo/delta": -0.07862785458564758, + "fcm_dpo/margin": 17.971176147460938, + "fcm_dpo/q_t": 0.3956528306007385, + "grad_norm": 17.511899948120117, + "learning_rate": 4.5287418106563354e-07, + "logits/chosen": 0.3419583737850189, + "logits/rejected": 0.299105703830719, + "logps/chosen": -84.12144470214844, + "logps/ref_chosen": -67.42012786865234, + "logps/ref_rejected": -82.50968933105469, + "logps/rejected": -117.18218231201172, + "loss": 1.0992, + "margin_dpo/margin_mean": 17.971176147460938, + "margin_dpo/margin_std": 28.787887573242188, + "step": 186 + }, + { + "KL/chosen_KL_mean": -22.477245330810547, + "KL/mean": -30.472248077392578, + "KL/rejected_KL_mean": -38.467254638671875, + "KL/std": 25.28232765197754, + "epoch": 0.28269085411942557, + "fcm_dpo/beta": 0.02547437697649002, + "fcm_dpo/delta": -0.008900739252567291, + "fcm_dpo/margin": 15.990015983581543, + "fcm_dpo/q_t": 0.4118683338165283, + "grad_norm": 16.354949951171875, + "learning_rate": 4.520986992917297e-07, + "logits/chosen": 0.4215141534805298, + "logits/rejected": 0.3655751347541809, + "logps/chosen": -98.00273895263672, + "logps/ref_chosen": -75.52549743652344, + "logps/ref_rejected": -94.76289367675781, + "logps/rejected": -133.2301483154297, + "loss": 1.1527, + "margin_dpo/margin_mean": 15.990015983581543, + "margin_dpo/margin_std": 29.899837493896484, + "step": 187 + }, + { + "KL/chosen_KL_mean": -20.62126922607422, + "KL/mean": -30.150714874267578, + "KL/rejected_KL_mean": -39.68015670776367, + "KL/std": 26.517009735107422, + "epoch": 0.2842025699168556, + "fcm_dpo/beta": 0.024722332134842873, + "fcm_dpo/delta": -0.08292864263057709, + "fcm_dpo/margin": 19.058883666992188, + "fcm_dpo/q_t": 0.3959978222846985, + "grad_norm": 13.136139869689941, + "learning_rate": 4.5131756438276466e-07, + "logits/chosen": 0.42119163274765015, + "logits/rejected": 0.37826281785964966, + "logps/chosen": -92.14459991455078, + "logps/ref_chosen": -71.52333068847656, + "logps/ref_rejected": -78.29949951171875, + "logps/rejected": -117.97966003417969, + "loss": 1.0819, + "margin_dpo/margin_mean": 19.058883666992188, + "margin_dpo/margin_std": 28.6160831451416, + "step": 188 + }, + { + "KL/chosen_KL_mean": -20.660493850708008, + "KL/mean": -29.015668869018555, + "KL/rejected_KL_mean": -37.37084197998047, + "KL/std": 26.431987762451172, + "epoch": 0.2857142857142857, + "fcm_dpo/beta": 0.023861508816480637, + "fcm_dpo/delta": -0.0019575804471969604, + "fcm_dpo/margin": 16.710351943969727, + "fcm_dpo/q_t": 0.41077572107315063, + "grad_norm": 15.193591117858887, + "learning_rate": 4.5053079818876096e-07, + "logits/chosen": 0.47629261016845703, + "logits/rejected": 0.492626428604126, + "logps/chosen": -92.83676147460938, + "logps/ref_chosen": -72.17626953125, + "logps/ref_rejected": -75.26313781738281, + "logps/rejected": -112.63397216796875, + "loss": 1.1338, + "margin_dpo/margin_mean": 16.71035385131836, + "margin_dpo/margin_std": 27.63460350036621, + "step": 189 + }, + { + "KL/chosen_KL_mean": -19.903602600097656, + "KL/mean": -32.08153533935547, + "KL/rejected_KL_mean": -44.25946807861328, + "KL/std": 27.69138526916504, + "epoch": 0.2872260015117158, + "fcm_dpo/beta": 0.02243766561150551, + "fcm_dpo/delta": -0.17649930715560913, + "fcm_dpo/margin": 24.355867385864258, + "fcm_dpo/q_t": 0.3752169609069824, + "grad_norm": 19.759334564208984, + "learning_rate": 4.4973842271726024e-07, + "logits/chosen": 0.5364293456077576, + "logits/rejected": 0.3797008693218231, + "logps/chosen": -74.52787780761719, + "logps/ref_chosen": -54.624271392822266, + "logps/ref_rejected": -101.47068786621094, + "logps/rejected": -145.73016357421875, + "loss": 1.0183, + "margin_dpo/margin_mean": 24.355865478515625, + "margin_dpo/margin_std": 30.53769302368164, + "step": 190 + }, + { + "KL/chosen_KL_mean": -29.63574981689453, + "KL/mean": -39.05760192871094, + "KL/rejected_KL_mean": -48.479454040527344, + "KL/std": 29.330245971679688, + "epoch": 0.2887377173091459, + "fcm_dpo/beta": 0.021781757473945618, + "fcm_dpo/delta": -0.01194111630320549, + "fcm_dpo/margin": 18.843711853027344, + "fcm_dpo/q_t": 0.4067850112915039, + "grad_norm": 15.226569175720215, + "learning_rate": 4.48940460132708e-07, + "logits/chosen": 0.5249963998794556, + "logits/rejected": 0.49379801750183105, + "logps/chosen": -102.56826782226562, + "logps/ref_chosen": -72.93251037597656, + "logps/ref_rejected": -89.95103454589844, + "logps/rejected": -138.43048095703125, + "loss": 1.1252, + "margin_dpo/margin_mean": 18.843713760375977, + "margin_dpo/margin_std": 31.241641998291016, + "step": 191 + }, + { + "KL/chosen_KL_mean": -33.76609802246094, + "KL/mean": -39.7779426574707, + "KL/rejected_KL_mean": -45.78978729248047, + "KL/std": 30.88973617553711, + "epoch": 0.29024943310657597, + "fcm_dpo/beta": 0.02324771136045456, + "fcm_dpo/delta": 0.1299854815006256, + "fcm_dpo/margin": 12.023693084716797, + "fcm_dpo/q_t": 0.43884414434432983, + "grad_norm": 15.96609878540039, + "learning_rate": 4.481369327558329e-07, + "logits/chosen": 0.5415316820144653, + "logits/rejected": 0.5124877691268921, + "logps/chosen": -87.76721954345703, + "logps/ref_chosen": -54.001121520996094, + "logps/ref_rejected": -63.531551361083984, + "logps/rejected": -109.32133483886719, + "loss": 1.2218, + "margin_dpo/margin_mean": 12.02369213104248, + "margin_dpo/margin_std": 27.488590240478516, + "step": 192 + }, + { + "KL/chosen_KL_mean": -26.880212783813477, + "KL/mean": -38.29176330566406, + "KL/rejected_KL_mean": -49.70331573486328, + "KL/std": 28.740842819213867, + "epoch": 0.29176114890400606, + "fcm_dpo/beta": 0.02194758504629135, + "fcm_dpo/delta": -0.12258221954107285, + "fcm_dpo/margin": 22.823104858398438, + "fcm_dpo/q_t": 0.38504743576049805, + "grad_norm": 13.015534400939941, + "learning_rate": 4.47327863063023e-07, + "logits/chosen": 0.41978591680526733, + "logits/rejected": 0.39997708797454834, + "logps/chosen": -83.62948608398438, + "logps/ref_chosen": -56.74927520751953, + "logps/ref_rejected": -58.80629348754883, + "logps/rejected": -108.50961303710938, + "loss": 1.0454, + "margin_dpo/margin_mean": 22.82310676574707, + "margin_dpo/margin_std": 29.807849884033203, + "step": 193 + }, + { + "KL/chosen_KL_mean": -30.15066909790039, + "KL/mean": -39.23329162597656, + "KL/rejected_KL_mean": -48.315921783447266, + "KL/std": 30.352039337158203, + "epoch": 0.29327286470143615, + "fcm_dpo/beta": 0.02143619954586029, + "fcm_dpo/delta": 0.00822427123785019, + "fcm_dpo/margin": 18.165246963500977, + "fcm_dpo/q_t": 0.4135337471961975, + "grad_norm": 17.924869537353516, + "learning_rate": 4.4651327368569684e-07, + "logits/chosen": 0.526753842830658, + "logits/rejected": 0.4955962300300598, + "logps/chosen": -86.80010986328125, + "logps/ref_chosen": -56.64944076538086, + "logps/ref_rejected": -69.98954772949219, + "logps/rejected": -118.30546569824219, + "loss": 1.1742, + "margin_dpo/margin_mean": 18.165246963500977, + "margin_dpo/margin_std": 35.39740753173828, + "step": 194 + }, + { + "KL/chosen_KL_mean": -35.72456741333008, + "KL/mean": -48.061038970947266, + "KL/rejected_KL_mean": -60.39750671386719, + "KL/std": 33.23961639404297, + "epoch": 0.2947845804988662, + "fcm_dpo/beta": 0.020506273955106735, + "fcm_dpo/delta": -0.1278218924999237, + "fcm_dpo/margin": 24.672945022583008, + "fcm_dpo/q_t": 0.38505733013153076, + "grad_norm": 16.904264450073242, + "learning_rate": 4.4569318740967043e-07, + "logits/chosen": 0.43730270862579346, + "logits/rejected": 0.44750988483428955, + "logps/chosen": -106.13434600830078, + "logps/ref_chosen": -70.40977478027344, + "logps/ref_rejected": -74.39448547363281, + "logps/rejected": -134.7919921875, + "loss": 1.0634, + "margin_dpo/margin_mean": 24.67294692993164, + "margin_dpo/margin_std": 35.26850128173828, + "step": 195 + }, + { + "KL/chosen_KL_mean": -36.346405029296875, + "KL/mean": -45.410797119140625, + "KL/rejected_KL_mean": -54.47519302368164, + "KL/std": 34.48020935058594, + "epoch": 0.2962962962962963, + "fcm_dpo/beta": 0.020830024033784866, + "fcm_dpo/delta": 0.02376365289092064, + "fcm_dpo/margin": 18.128787994384766, + "fcm_dpo/q_t": 0.414675235748291, + "grad_norm": 18.091453552246094, + "learning_rate": 4.448676271745197e-07, + "logits/chosen": 0.5153293013572693, + "logits/rejected": 0.46972033381462097, + "logps/chosen": -95.573974609375, + "logps/ref_chosen": -59.227577209472656, + "logps/ref_rejected": -83.54757690429688, + "logps/rejected": -138.02276611328125, + "loss": 1.1365, + "margin_dpo/margin_mean": 18.128787994384766, + "margin_dpo/margin_std": 29.96463966369629, + "step": 196 + }, + { + "KL/chosen_KL_mean": -32.234886169433594, + "KL/mean": -43.992149353027344, + "KL/rejected_KL_mean": -55.74940872192383, + "KL/std": 32.462039947509766, + "epoch": 0.29780801209372637, + "fcm_dpo/beta": 0.020198438316583633, + "fcm_dpo/delta": -0.08790335804224014, + "fcm_dpo/margin": 23.51451873779297, + "fcm_dpo/q_t": 0.3938829302787781, + "grad_norm": 17.229829788208008, + "learning_rate": 4.440366160729392e-07, + "logits/chosen": 0.593604564666748, + "logits/rejected": 0.5391672849655151, + "logps/chosen": -83.76402282714844, + "logps/ref_chosen": -51.52912902832031, + "logps/ref_rejected": -73.70631408691406, + "logps/rejected": -129.45571899414062, + "loss": 1.1271, + "margin_dpo/margin_mean": 23.51451873779297, + "margin_dpo/margin_std": 40.65414047241211, + "step": 197 + }, + { + "KL/chosen_KL_mean": -35.11063003540039, + "KL/mean": -47.747779846191406, + "KL/rejected_KL_mean": -60.38494110107422, + "KL/std": 33.210731506347656, + "epoch": 0.29931972789115646, + "fcm_dpo/beta": 0.018990688025951385, + "fcm_dpo/delta": -0.09321387112140656, + "fcm_dpo/margin": 25.274303436279297, + "fcm_dpo/q_t": 0.3890335261821747, + "grad_norm": 15.211456298828125, + "learning_rate": 4.432001773500957e-07, + "logits/chosen": 0.5949510931968689, + "logits/rejected": 0.5511294603347778, + "logps/chosen": -94.893310546875, + "logps/ref_chosen": -59.78268051147461, + "logps/ref_rejected": -72.24533081054688, + "logps/rejected": -132.63026428222656, + "loss": 1.0603, + "margin_dpo/margin_mean": 25.274303436279297, + "margin_dpo/margin_std": 33.733585357666016, + "step": 198 + }, + { + "KL/chosen_KL_mean": -42.495819091796875, + "KL/mean": -52.347877502441406, + "KL/rejected_KL_mean": -62.19993209838867, + "KL/std": 33.97826385498047, + "epoch": 0.30083144368858655, + "fcm_dpo/beta": 0.0181651059538126, + "fcm_dpo/delta": -0.06439946591854095, + "fcm_dpo/margin": 19.704111099243164, + "fcm_dpo/q_t": 0.41771239042282104, + "grad_norm": 13.64396858215332, + "learning_rate": 4.4235833440297856e-07, + "logits/chosen": 0.5503775477409363, + "logits/rejected": 0.4570651054382324, + "logps/chosen": -98.8825912475586, + "logps/ref_chosen": -56.38677215576172, + "logps/ref_rejected": -74.56779479980469, + "logps/rejected": -136.76773071289062, + "loss": 1.1815, + "margin_dpo/margin_mean": 19.704111099243164, + "margin_dpo/margin_std": 38.327781677246094, + "step": 199 + }, + { + "KL/chosen_KL_mean": -34.580101013183594, + "KL/mean": -48.731632232666016, + "KL/rejected_KL_mean": -62.88316345214844, + "KL/std": 35.259788513183594, + "epoch": 0.30234315948601664, + "fcm_dpo/beta": 0.016897382214665413, + "fcm_dpo/delta": -0.09347662329673767, + "fcm_dpo/margin": 28.303054809570312, + "fcm_dpo/q_t": 0.39045828580856323, + "grad_norm": 12.926334381103516, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": 0.5941851139068604, + "logits/rejected": 0.512370228767395, + "logps/chosen": -92.40442657470703, + "logps/ref_chosen": -57.82432556152344, + "logps/ref_rejected": -89.28246307373047, + "logps/rejected": -152.16561889648438, + "loss": 1.0802, + "margin_dpo/margin_mean": 28.303054809570312, + "margin_dpo/margin_std": 41.966915130615234, + "step": 200 + }, + { + "KL/chosen_KL_mean": -47.82463836669922, + "KL/mean": -58.91985321044922, + "KL/rejected_KL_mean": -70.01507568359375, + "KL/std": 37.099159240722656, + "epoch": 0.30385487528344673, + "fcm_dpo/beta": 0.016795823350548744, + "fcm_dpo/delta": 0.029392898082733154, + "fcm_dpo/margin": 22.190425872802734, + "fcm_dpo/q_t": 0.41658544540405273, + "grad_norm": 15.126965522766113, + "learning_rate": 4.4065853017905953e-07, + "logits/chosen": 0.6271176338195801, + "logits/rejected": 0.5756611824035645, + "logps/chosen": -106.82440185546875, + "logps/ref_chosen": -58.999759674072266, + "logps/ref_rejected": -84.67575073242188, + "logps/rejected": -154.69082641601562, + "loss": 1.1508, + "margin_dpo/margin_mean": 22.190425872802734, + "margin_dpo/margin_std": 38.777530670166016, + "step": 201 + }, + { + "KL/chosen_KL_mean": -40.08701705932617, + "KL/mean": -54.63224792480469, + "KL/rejected_KL_mean": -69.17748260498047, + "KL/std": 35.3343505859375, + "epoch": 0.30536659108087677, + "fcm_dpo/beta": 0.01654578559100628, + "fcm_dpo/delta": -0.09474054723978043, + "fcm_dpo/margin": 29.090471267700195, + "fcm_dpo/q_t": 0.38745981454849243, + "grad_norm": 13.597728729248047, + "learning_rate": 4.3980061644943575e-07, + "logits/chosen": 0.5242820978164673, + "logits/rejected": 0.444463849067688, + "logps/chosen": -87.74766540527344, + "logps/ref_chosen": -47.660648345947266, + "logps/ref_rejected": -73.63249969482422, + "logps/rejected": -142.8099822998047, + "loss": 1.0619, + "margin_dpo/margin_mean": 29.090471267700195, + "margin_dpo/margin_std": 39.005897521972656, + "step": 202 + }, + { + "KL/chosen_KL_mean": -50.41577911376953, + "KL/mean": -63.223907470703125, + "KL/rejected_KL_mean": -76.03202819824219, + "KL/std": 36.80195999145508, + "epoch": 0.30687830687830686, + "fcm_dpo/beta": 0.016147522255778313, + "fcm_dpo/delta": -0.015619541518390179, + "fcm_dpo/margin": 25.616247177124023, + "fcm_dpo/q_t": 0.4067385792732239, + "grad_norm": 16.166540145874023, + "learning_rate": 4.3893739358856455e-07, + "logits/chosen": 0.5744930505752563, + "logits/rejected": 0.4879806637763977, + "logps/chosen": -112.74131774902344, + "logps/ref_chosen": -62.32553482055664, + "logps/ref_rejected": -99.37226104736328, + "logps/rejected": -175.404296875, + "loss": 1.115, + "margin_dpo/margin_mean": 25.616249084472656, + "margin_dpo/margin_std": 40.74905014038086, + "step": 203 + }, + { + "KL/chosen_KL_mean": -45.187042236328125, + "KL/mean": -58.972225189208984, + "KL/rejected_KL_mean": -72.75740051269531, + "KL/std": 37.236968994140625, + "epoch": 0.30839002267573695, + "fcm_dpo/beta": 0.015411125496029854, + "fcm_dpo/delta": -0.0340307280421257, + "fcm_dpo/margin": 27.570358276367188, + "fcm_dpo/q_t": 0.40242862701416016, + "grad_norm": 14.934419631958008, + "learning_rate": 4.380688857426449e-07, + "logits/chosen": 0.562065839767456, + "logits/rejected": 0.48749643564224243, + "logps/chosen": -95.81636047363281, + "logps/ref_chosen": -50.62931823730469, + "logps/ref_rejected": -66.60475158691406, + "logps/rejected": -139.36215209960938, + "loss": 1.0957, + "margin_dpo/margin_mean": 27.570356369018555, + "margin_dpo/margin_std": 38.67369079589844, + "step": 204 + }, + { + "KL/chosen_KL_mean": -52.95893859863281, + "KL/mean": -66.04808044433594, + "KL/rejected_KL_mean": -79.13722229003906, + "KL/std": 39.503875732421875, + "epoch": 0.30990173847316704, + "fcm_dpo/beta": 0.015607166104018688, + "fcm_dpo/delta": -0.009851815178990364, + "fcm_dpo/margin": 26.17829704284668, + "fcm_dpo/q_t": 0.40857863426208496, + "grad_norm": 16.72960090637207, + "learning_rate": 4.3719511720570814e-07, + "logits/chosen": 0.5846419334411621, + "logits/rejected": 0.5103527903556824, + "logps/chosen": -123.31511688232422, + "logps/ref_chosen": -70.3561782836914, + "logps/ref_rejected": -93.39848327636719, + "logps/rejected": -172.53570556640625, + "loss": 1.1293, + "margin_dpo/margin_mean": 26.178298950195312, + "margin_dpo/margin_std": 44.114990234375, + "step": 205 + }, + { + "KL/chosen_KL_mean": -55.539981842041016, + "KL/mean": -64.34222412109375, + "KL/rejected_KL_mean": -73.14447021484375, + "KL/std": 38.425743103027344, + "epoch": 0.31141345427059713, + "fcm_dpo/beta": 0.015734657645225525, + "fcm_dpo/delta": 0.022806979715824127, + "fcm_dpo/margin": 17.60448455810547, + "fcm_dpo/q_t": 0.4400056302547455, + "grad_norm": 14.786349296569824, + "learning_rate": 4.363161124189387e-07, + "logits/chosen": 0.6014150381088257, + "logits/rejected": 0.5844457149505615, + "logps/chosen": -123.18546295166016, + "logps/ref_chosen": -67.64547729492188, + "logps/ref_rejected": -79.89584350585938, + "logps/rejected": -153.04031372070312, + "loss": 1.2576, + "margin_dpo/margin_mean": 17.6044864654541, + "margin_dpo/margin_std": 47.01768493652344, + "step": 206 + }, + { + "KL/chosen_KL_mean": -63.995445251464844, + "KL/mean": -76.38787841796875, + "KL/rejected_KL_mean": -88.78030395507812, + "KL/std": 39.89057159423828, + "epoch": 0.3129251700680272, + "fcm_dpo/beta": 0.015928978100419044, + "fcm_dpo/delta": 0.005825323052704334, + "fcm_dpo/margin": 24.784862518310547, + "fcm_dpo/q_t": 0.41365471482276917, + "grad_norm": 13.180200576782227, + "learning_rate": 4.3543189596998986e-07, + "logits/chosen": 0.5777188539505005, + "logits/rejected": 0.5047867894172668, + "logps/chosen": -131.65963745117188, + "logps/ref_chosen": -67.66419219970703, + "logps/ref_rejected": -85.10249328613281, + "logps/rejected": -173.88279724121094, + "loss": 1.1359, + "margin_dpo/margin_mean": 24.784862518310547, + "margin_dpo/margin_std": 42.67274475097656, + "step": 207 + }, + { + "KL/chosen_KL_mean": -54.84553527832031, + "KL/mean": -61.52077865600586, + "KL/rejected_KL_mean": -68.1960220336914, + "KL/std": 38.0513916015625, + "epoch": 0.3144368858654573, + "fcm_dpo/beta": 0.01662435196340084, + "fcm_dpo/delta": 0.0744815543293953, + "fcm_dpo/margin": 13.350489616394043, + "fcm_dpo/q_t": 0.4523673355579376, + "grad_norm": 21.40291976928711, + "learning_rate": 4.3454249259229664e-07, + "logits/chosen": 0.5781328678131104, + "logits/rejected": 0.5493608713150024, + "logps/chosen": -112.5772476196289, + "logps/ref_chosen": -57.731712341308594, + "logps/ref_rejected": -74.19276428222656, + "logps/rejected": -142.3887939453125, + "loss": 1.3069, + "margin_dpo/margin_mean": 13.350488662719727, + "margin_dpo/margin_std": 44.465492248535156, + "step": 208 + }, + { + "KL/chosen_KL_mean": -54.61763000488281, + "KL/mean": -70.42623138427734, + "KL/rejected_KL_mean": -86.23484802246094, + "KL/std": 42.82319641113281, + "epoch": 0.31594860166288735, + "fcm_dpo/beta": 0.01570359617471695, + "fcm_dpo/delta": -0.1145251914858818, + "fcm_dpo/margin": 31.617210388183594, + "fcm_dpo/q_t": 0.3877779245376587, + "grad_norm": 15.687070846557617, + "learning_rate": 4.336479271643833e-07, + "logits/chosen": 0.5210272073745728, + "logits/rejected": 0.46536776423454285, + "logps/chosen": -123.16770935058594, + "logps/ref_chosen": -68.55007934570312, + "logps/ref_rejected": -87.90541076660156, + "logps/rejected": -174.1402587890625, + "loss": 1.0937, + "margin_dpo/margin_mean": 31.617210388183594, + "margin_dpo/margin_std": 50.869163513183594, + "step": 209 + }, + { + "KL/chosen_KL_mean": -53.08406066894531, + "KL/mean": -69.13988494873047, + "KL/rejected_KL_mean": -85.19570922851562, + "KL/std": 43.93097686767578, + "epoch": 0.31746031746031744, + "fcm_dpo/beta": 0.01513850037008524, + "fcm_dpo/delta": -0.10149059444665909, + "fcm_dpo/margin": 32.111637115478516, + "fcm_dpo/q_t": 0.39089614152908325, + "grad_norm": 15.86282730102539, + "learning_rate": 4.327482247091679e-07, + "logits/chosen": 0.6385220289230347, + "logits/rejected": 0.5306141376495361, + "logps/chosen": -110.35234069824219, + "logps/ref_chosen": -57.268272399902344, + "logps/ref_rejected": -85.72807312011719, + "logps/rejected": -170.9237823486328, + "loss": 1.0724, + "margin_dpo/margin_mean": 32.111637115478516, + "margin_dpo/margin_std": 45.491607666015625, + "step": 210 + }, + { + "KL/chosen_KL_mean": -48.97825622558594, + "KL/mean": -64.92655944824219, + "KL/rejected_KL_mean": -80.8748779296875, + "KL/std": 40.996124267578125, + "epoch": 0.31897203325774753, + "fcm_dpo/beta": 0.014405088499188423, + "fcm_dpo/delta": -0.06979866325855255, + "fcm_dpo/margin": 31.896615982055664, + "fcm_dpo/q_t": 0.39662906527519226, + "grad_norm": 17.855039596557617, + "learning_rate": 4.3184341039326217e-07, + "logits/chosen": 0.6159626245498657, + "logits/rejected": 0.5108903050422668, + "logps/chosen": -102.61896514892578, + "logps/ref_chosen": -53.640708923339844, + "logps/ref_rejected": -93.0387954711914, + "logps/rejected": -173.91366577148438, + "loss": 1.0677, + "margin_dpo/margin_mean": 31.896615982055664, + "margin_dpo/margin_std": 42.558563232421875, + "step": 211 + }, + { + "KL/chosen_KL_mean": -55.17749786376953, + "KL/mean": -70.99906158447266, + "KL/rejected_KL_mean": -86.82063293457031, + "KL/std": 43.388309478759766, + "epoch": 0.3204837490551776, + "fcm_dpo/beta": 0.013755107298493385, + "fcm_dpo/delta": -0.040556252002716064, + "fcm_dpo/margin": 31.643131256103516, + "fcm_dpo/q_t": 0.3995632529258728, + "grad_norm": 10.621484756469727, + "learning_rate": 4.309335095262675e-07, + "logits/chosen": 0.6260430812835693, + "logits/rejected": 0.5469928979873657, + "logps/chosen": -112.54423522949219, + "logps/ref_chosen": -57.36674499511719, + "logps/ref_rejected": -79.89643096923828, + "logps/rejected": -166.71707153320312, + "loss": 1.096, + "margin_dpo/margin_mean": 31.643129348754883, + "margin_dpo/margin_std": 46.78819274902344, + "step": 212 + }, + { + "KL/chosen_KL_mean": -44.155555725097656, + "KL/mean": -62.20250701904297, + "KL/rejected_KL_mean": -80.24946594238281, + "KL/std": 43.83998107910156, + "epoch": 0.3219954648526077, + "fcm_dpo/beta": 0.013000023551285267, + "fcm_dpo/delta": -0.08304151147603989, + "fcm_dpo/margin": 36.09390640258789, + "fcm_dpo/q_t": 0.3916971981525421, + "grad_norm": 11.67632007598877, + "learning_rate": 4.3001854756006724e-07, + "logits/chosen": 0.5951361656188965, + "logits/rejected": 0.5706344842910767, + "logps/chosen": -109.37667083740234, + "logps/ref_chosen": -65.22111511230469, + "logps/ref_rejected": -80.1810302734375, + "logps/rejected": -160.43048095703125, + "loss": 1.0649, + "margin_dpo/margin_mean": 36.09390640258789, + "margin_dpo/margin_std": 48.79711151123047, + "step": 213 + }, + { + "KL/chosen_KL_mean": -52.43152618408203, + "KL/mean": -69.8787841796875, + "KL/rejected_KL_mean": -87.3260269165039, + "KL/std": 44.159698486328125, + "epoch": 0.3235071806500378, + "fcm_dpo/beta": 0.012891286984086037, + "fcm_dpo/delta": -0.058638766407966614, + "fcm_dpo/margin": 34.89451217651367, + "fcm_dpo/q_t": 0.39819642901420593, + "grad_norm": 13.59187126159668, + "learning_rate": 4.290985500881143e-07, + "logits/chosen": 0.47652411460876465, + "logits/rejected": 0.46216946840286255, + "logps/chosen": -113.7238540649414, + "logps/ref_chosen": -61.292327880859375, + "logps/ref_rejected": -67.69841003417969, + "logps/rejected": -155.02444458007812, + "loss": 1.0924, + "margin_dpo/margin_mean": 34.89451599121094, + "margin_dpo/margin_std": 51.43964385986328, + "step": 214 + }, + { + "KL/chosen_KL_mean": -57.65421676635742, + "KL/mean": -75.66905212402344, + "KL/rejected_KL_mean": -93.68387603759766, + "KL/std": 45.626731872558594, + "epoch": 0.3250188964474679, + "fcm_dpo/beta": 0.012519944459199905, + "fcm_dpo/delta": -0.061683304607868195, + "fcm_dpo/margin": 36.02965545654297, + "fcm_dpo/q_t": 0.3975563645362854, + "grad_norm": 12.812163352966309, + "learning_rate": 4.281735428447157e-07, + "logits/chosen": 0.49123620986938477, + "logits/rejected": 0.3736293911933899, + "logps/chosen": -121.52335357666016, + "logps/ref_chosen": -63.869136810302734, + "logps/ref_rejected": -98.7657241821289, + "logps/rejected": -192.44960021972656, + "loss": 1.099, + "margin_dpo/margin_mean": 36.02965545654297, + "margin_dpo/margin_std": 53.505271911621094, + "step": 215 + }, + { + "KL/chosen_KL_mean": -48.55633544921875, + "KL/mean": -68.02696228027344, + "KL/rejected_KL_mean": -87.49758911132812, + "KL/std": 44.254615783691406, + "epoch": 0.32653061224489793, + "fcm_dpo/beta": 0.011713217943906784, + "fcm_dpo/delta": -0.06535829603672028, + "fcm_dpo/margin": 38.941246032714844, + "fcm_dpo/q_t": 0.3921303153038025, + "grad_norm": 22.394311904907227, + "learning_rate": 4.2724355170431247e-07, + "logits/chosen": 0.5988740921020508, + "logits/rejected": 0.5082088708877563, + "logps/chosen": -116.38128662109375, + "logps/ref_chosen": -67.824951171875, + "logps/ref_rejected": -96.40231323242188, + "logps/rejected": -183.89990234375, + "loss": 1.0456, + "margin_dpo/margin_mean": 38.941246032714844, + "margin_dpo/margin_std": 44.615882873535156, + "step": 216 + }, + { + "KL/chosen_KL_mean": -52.889949798583984, + "KL/mean": -73.1640625, + "KL/rejected_KL_mean": -93.43817901611328, + "KL/std": 45.83582305908203, + "epoch": 0.328042328042328, + "fcm_dpo/beta": 0.01119733415544033, + "fcm_dpo/delta": -0.06392455846071243, + "fcm_dpo/margin": 40.548240661621094, + "fcm_dpo/q_t": 0.39370107650756836, + "grad_norm": 10.61658000946045, + "learning_rate": 4.26308602680756e-07, + "logits/chosen": 0.5777255892753601, + "logits/rejected": 0.4716952443122864, + "logps/chosen": -113.39494323730469, + "logps/ref_chosen": -60.5049934387207, + "logps/ref_rejected": -84.26618194580078, + "logps/rejected": -177.70436096191406, + "loss": 1.0503, + "margin_dpo/margin_mean": 40.548240661621094, + "margin_dpo/margin_std": 47.78947448730469, + "step": 217 + }, + { + "KL/chosen_KL_mean": -58.71846008300781, + "KL/mean": -70.38279724121094, + "KL/rejected_KL_mean": -82.04712677001953, + "KL/std": 45.208160400390625, + "epoch": 0.3295540438397581, + "fcm_dpo/beta": 0.011268625035881996, + "fcm_dpo/delta": 0.02180871181190014, + "fcm_dpo/margin": 23.328662872314453, + "fcm_dpo/q_t": 0.4390827715396881, + "grad_norm": 11.695416450500488, + "learning_rate": 4.253687219265803e-07, + "logits/chosen": 0.4690364599227905, + "logits/rejected": 0.472156822681427, + "logps/chosen": -129.31277465820312, + "logps/ref_chosen": -70.59431457519531, + "logps/ref_rejected": -73.89038848876953, + "logps/rejected": -155.93751525878906, + "loss": 1.234, + "margin_dpo/margin_mean": 23.32866096496582, + "margin_dpo/margin_std": 54.49486541748047, + "step": 218 + }, + { + "KL/chosen_KL_mean": -54.63685989379883, + "KL/mean": -67.47407531738281, + "KL/rejected_KL_mean": -80.31128692626953, + "KL/std": 43.43921661376953, + "epoch": 0.3310657596371882, + "fcm_dpo/beta": 0.011822700500488281, + "fcm_dpo/delta": 0.10650709271430969, + "fcm_dpo/margin": 25.674427032470703, + "fcm_dpo/q_t": 0.4315260052680969, + "grad_norm": 11.611766815185547, + "learning_rate": 4.2442393573227043e-07, + "logits/chosen": 0.5588555335998535, + "logits/rejected": 0.5147054195404053, + "logps/chosen": -115.1278076171875, + "logps/ref_chosen": -60.490943908691406, + "logps/ref_rejected": -75.85001373291016, + "logps/rejected": -156.1613006591797, + "loss": 1.1746, + "margin_dpo/margin_mean": 25.674427032470703, + "margin_dpo/margin_std": 45.024803161621094, + "step": 219 + }, + { + "KL/chosen_KL_mean": -46.359901428222656, + "KL/mean": -61.93693923950195, + "KL/rejected_KL_mean": -77.51397705078125, + "KL/std": 46.67603302001953, + "epoch": 0.3325774754346183, + "fcm_dpo/beta": 0.012025123462080956, + "fcm_dpo/delta": 0.02687015011906624, + "fcm_dpo/margin": 31.15407943725586, + "fcm_dpo/q_t": 0.4164389967918396, + "grad_norm": 13.288846969604492, + "learning_rate": 4.234742705255272e-07, + "logits/chosen": 0.6493104100227356, + "logits/rejected": 0.5782603025436401, + "logps/chosen": -91.37329864501953, + "logps/ref_chosen": -45.013397216796875, + "logps/ref_rejected": -70.49369812011719, + "logps/rejected": -148.0076904296875, + "loss": 1.1525, + "margin_dpo/margin_mean": 31.154081344604492, + "margin_dpo/margin_std": 54.201744079589844, + "step": 220 + }, + { + "KL/chosen_KL_mean": -45.98303985595703, + "KL/mean": -63.76835632324219, + "KL/rejected_KL_mean": -81.55365753173828, + "KL/std": 46.344482421875, + "epoch": 0.3340891912320484, + "fcm_dpo/beta": 0.01212066039443016, + "fcm_dpo/delta": -0.03572237864136696, + "fcm_dpo/margin": 35.570621490478516, + "fcm_dpo/q_t": 0.4028802216053009, + "grad_norm": 13.347899436950684, + "learning_rate": 4.22519752870528e-07, + "logits/chosen": 0.577314019203186, + "logits/rejected": 0.494738906621933, + "logps/chosen": -105.0788803100586, + "logps/ref_chosen": -59.09584045410156, + "logps/ref_rejected": -88.64388275146484, + "logps/rejected": -170.19754028320312, + "loss": 1.0937, + "margin_dpo/margin_mean": 35.570621490478516, + "margin_dpo/margin_std": 52.52630615234375, + "step": 221 + }, + { + "KL/chosen_KL_mean": -46.059627532958984, + "KL/mean": -69.68561553955078, + "KL/rejected_KL_mean": -93.31161499023438, + "KL/std": 52.56859588623047, + "epoch": 0.3356009070294785, + "fcm_dpo/beta": 0.011211092583835125, + "fcm_dpo/delta": -0.15475650131702423, + "fcm_dpo/margin": 47.251983642578125, + "fcm_dpo/q_t": 0.37726062536239624, + "grad_norm": 11.579093933105469, + "learning_rate": 4.2156040946718343e-07, + "logits/chosen": 0.6488215327262878, + "logits/rejected": 0.5492867827415466, + "logps/chosen": -102.05731964111328, + "logps/ref_chosen": -55.9976921081543, + "logps/ref_rejected": -111.94727325439453, + "logps/rejected": -205.25888061523438, + "loss": 1.0177, + "margin_dpo/margin_mean": 47.251983642578125, + "margin_dpo/margin_std": 56.59059143066406, + "step": 222 + }, + { + "KL/chosen_KL_mean": -55.13484191894531, + "KL/mean": -73.27153015136719, + "KL/rejected_KL_mean": -91.40821838378906, + "KL/std": 49.953399658203125, + "epoch": 0.3371126228269085, + "fcm_dpo/beta": 0.01093536801636219, + "fcm_dpo/delta": 0.003693356178700924, + "fcm_dpo/margin": 36.27337646484375, + "fcm_dpo/q_t": 0.4097467362880707, + "grad_norm": 13.193449974060059, + "learning_rate": 4.2059626715039065e-07, + "logits/chosen": 0.6461388468742371, + "logits/rejected": 0.5862003564834595, + "logps/chosen": -115.02626037597656, + "logps/ref_chosen": -59.891422271728516, + "logps/ref_rejected": -86.28954315185547, + "logps/rejected": -177.69775390625, + "loss": 1.1034, + "margin_dpo/margin_mean": 36.27337646484375, + "margin_dpo/margin_std": 51.730384826660156, + "step": 223 + }, + { + "KL/chosen_KL_mean": -64.09883117675781, + "KL/mean": -74.00580596923828, + "KL/rejected_KL_mean": -83.91278076171875, + "KL/std": 49.710262298583984, + "epoch": 0.3386243386243386, + "fcm_dpo/beta": 0.011504280380904675, + "fcm_dpo/delta": 0.07703431695699692, + "fcm_dpo/margin": 19.813947677612305, + "fcm_dpo/q_t": 0.449110746383667, + "grad_norm": 16.505056381225586, + "learning_rate": 4.1962735288928304e-07, + "logits/chosen": 0.6655477285385132, + "logits/rejected": 0.6418548822402954, + "logps/chosen": -128.14346313476562, + "logps/ref_chosen": -64.04463195800781, + "logps/ref_rejected": -75.05450439453125, + "logps/rejected": -158.96728515625, + "loss": 1.2611, + "margin_dpo/margin_mean": 19.813949584960938, + "margin_dpo/margin_std": 53.515785217285156, + "step": 224 + }, + { + "KL/chosen_KL_mean": -58.37687683105469, + "KL/mean": -79.15052795410156, + "KL/rejected_KL_mean": -99.92417907714844, + "KL/std": 54.812477111816406, + "epoch": 0.3401360544217687, + "fcm_dpo/beta": 0.010968768037855625, + "fcm_dpo/delta": -0.06804075092077255, + "fcm_dpo/margin": 41.54730987548828, + "fcm_dpo/q_t": 0.3937387466430664, + "grad_norm": 11.77187728881836, + "learning_rate": 4.186536937864752e-07, + "logits/chosen": 0.606415867805481, + "logits/rejected": 0.4917227327823639, + "logps/chosen": -124.47270202636719, + "logps/ref_chosen": -66.0958251953125, + "logps/ref_rejected": -97.68675231933594, + "logps/rejected": -197.61093139648438, + "loss": 1.0544, + "margin_dpo/margin_mean": 41.54731369018555, + "margin_dpo/margin_std": 49.95249938964844, + "step": 225 + }, + { + "KL/chosen_KL_mean": -56.503387451171875, + "KL/mean": -72.67691802978516, + "KL/rejected_KL_mean": -88.85043334960938, + "KL/std": 49.93806076049805, + "epoch": 0.3416477702191988, + "fcm_dpo/beta": 0.011389894410967827, + "fcm_dpo/delta": 0.03310755267739296, + "fcm_dpo/margin": 32.347049713134766, + "fcm_dpo/q_t": 0.41660165786743164, + "grad_norm": 13.261937141418457, + "learning_rate": 4.176753170773052e-07, + "logits/chosen": 0.6644654273986816, + "logits/rejected": 0.6175069808959961, + "logps/chosen": -107.92025756835938, + "logps/ref_chosen": -51.4168701171875, + "logps/ref_rejected": -66.30068969726562, + "logps/rejected": -155.151123046875, + "loss": 1.1674, + "margin_dpo/margin_mean": 32.3470458984375, + "margin_dpo/margin_std": 60.95067596435547, + "step": 226 + }, + { + "KL/chosen_KL_mean": -60.69549560546875, + "KL/mean": -77.11978149414062, + "KL/rejected_KL_mean": -93.5440673828125, + "KL/std": 54.52500915527344, + "epoch": 0.3431594860166289, + "fcm_dpo/beta": 0.01134055107831955, + "fcm_dpo/delta": 0.03080243244767189, + "fcm_dpo/margin": 32.84856414794922, + "fcm_dpo/q_t": 0.41694962978363037, + "grad_norm": 13.040262222290039, + "learning_rate": 4.166922501290729e-07, + "logits/chosen": 0.6646544933319092, + "logits/rejected": 0.6282267570495605, + "logps/chosen": -118.68527221679688, + "logps/ref_chosen": -57.989776611328125, + "logps/ref_rejected": -75.05464172363281, + "logps/rejected": -168.5987091064453, + "loss": 1.1623, + "margin_dpo/margin_mean": 32.84856414794922, + "margin_dpo/margin_std": 61.44432830810547, + "step": 227 + }, + { + "KL/chosen_KL_mean": -64.08232116699219, + "KL/mean": -83.22323608398438, + "KL/rejected_KL_mean": -102.36415100097656, + "KL/std": 51.782081604003906, + "epoch": 0.34467120181405897, + "fcm_dpo/beta": 0.011403821408748627, + "fcm_dpo/delta": -0.04262223467230797, + "fcm_dpo/margin": 38.281822204589844, + "fcm_dpo/q_t": 0.3999175429344177, + "grad_norm": 15.122424125671387, + "learning_rate": 4.1570452044027405e-07, + "logits/chosen": 0.684807538986206, + "logits/rejected": 0.6012556552886963, + "logps/chosen": -119.64169311523438, + "logps/ref_chosen": -55.55936813354492, + "logps/ref_rejected": -77.02364349365234, + "logps/rejected": -179.38778686523438, + "loss": 1.0908, + "margin_dpo/margin_mean": 38.281822204589844, + "margin_dpo/margin_std": 54.99129867553711, + "step": 228 + }, + { + "KL/chosen_KL_mean": -54.697181701660156, + "KL/mean": -71.32321166992188, + "KL/rejected_KL_mean": -87.9492416381836, + "KL/std": 50.878578186035156, + "epoch": 0.34618291761148906, + "fcm_dpo/beta": 0.011315664276480675, + "fcm_dpo/delta": 0.02675747312605381, + "fcm_dpo/margin": 33.25206756591797, + "fcm_dpo/q_t": 0.4166818857192993, + "grad_norm": 16.81686019897461, + "learning_rate": 4.147121556398312e-07, + "logits/chosen": 0.7582691311836243, + "logits/rejected": 0.6779066324234009, + "logps/chosen": -105.49183654785156, + "logps/ref_chosen": -50.79466247558594, + "logps/ref_rejected": -78.4474105834961, + "logps/rejected": -166.3966522216797, + "loss": 1.1589, + "margin_dpo/margin_mean": 33.25206756591797, + "margin_dpo/margin_std": 61.93717956542969, + "step": 229 + }, + { + "KL/chosen_KL_mean": -59.01070022583008, + "KL/mean": -77.44501495361328, + "KL/rejected_KL_mean": -95.87932586669922, + "KL/std": 54.955116271972656, + "epoch": 0.3476946334089191, + "fcm_dpo/beta": 0.011499742977321148, + "fcm_dpo/delta": -0.031111925840377808, + "fcm_dpo/margin": 36.868629455566406, + "fcm_dpo/q_t": 0.403672993183136, + "grad_norm": 13.358758926391602, + "learning_rate": 4.137151834863213e-07, + "logits/chosen": 0.6462364196777344, + "logits/rejected": 0.648872971534729, + "logps/chosen": -115.73992919921875, + "logps/ref_chosen": -56.729225158691406, + "logps/ref_rejected": -62.99180603027344, + "logps/rejected": -158.87112426757812, + "loss": 1.1114, + "margin_dpo/margin_mean": 36.86862564086914, + "margin_dpo/margin_std": 56.367958068847656, + "step": 230 + }, + { + "KL/chosen_KL_mean": -65.8025131225586, + "KL/mean": -93.48104858398438, + "KL/rejected_KL_mean": -121.15958404541016, + "KL/std": 53.62238693237305, + "epoch": 0.3492063492063492, + "fcm_dpo/beta": 0.010107467882335186, + "fcm_dpo/delta": -0.1956898272037506, + "fcm_dpo/margin": 55.35707092285156, + "fcm_dpo/q_t": 0.36493778228759766, + "grad_norm": 12.845860481262207, + "learning_rate": 4.1271363186719835e-07, + "logits/chosen": 0.6005295515060425, + "logits/rejected": 0.5976369976997375, + "logps/chosen": -138.3996124267578, + "logps/ref_chosen": -72.59709930419922, + "logps/ref_rejected": -86.2322998046875, + "logps/rejected": -207.39187622070312, + "loss": 0.9845, + "margin_dpo/margin_mean": 55.35707092285156, + "margin_dpo/margin_std": 58.276145935058594, + "step": 231 + }, + { + "KL/chosen_KL_mean": -66.58941650390625, + "KL/mean": -85.17105102539062, + "KL/rejected_KL_mean": -103.752685546875, + "KL/std": 59.081756591796875, + "epoch": 0.3507180650037793, + "fcm_dpo/beta": 0.010086476802825928, + "fcm_dpo/delta": 0.028100861236453056, + "fcm_dpo/margin": 37.163265228271484, + "fcm_dpo/q_t": 0.4179150462150574, + "grad_norm": 12.528980255126953, + "learning_rate": 4.1170752879801436e-07, + "logits/chosen": 0.6124403476715088, + "logits/rejected": 0.5794280767440796, + "logps/chosen": -134.70794677734375, + "logps/ref_chosen": -68.1185302734375, + "logps/ref_rejected": -83.79415893554688, + "logps/rejected": -187.54684448242188, + "loss": 1.1554, + "margin_dpo/margin_mean": 37.163265228271484, + "margin_dpo/margin_std": 67.85264587402344, + "step": 232 + }, + { + "KL/chosen_KL_mean": -76.2654800415039, + "KL/mean": -89.84437561035156, + "KL/rejected_KL_mean": -103.42329406738281, + "KL/std": 53.858177185058594, + "epoch": 0.35222978080120937, + "fcm_dpo/beta": 0.009955944493412971, + "fcm_dpo/delta": -0.02221892587840557, + "fcm_dpo/margin": 27.157814025878906, + "fcm_dpo/q_t": 0.4383409917354584, + "grad_norm": 12.91989803314209, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": 0.6682592630386353, + "logits/rejected": 0.6073845028877258, + "logps/chosen": -131.33563232421875, + "logps/ref_chosen": -55.070152282714844, + "logps/ref_rejected": -66.61845397949219, + "logps/rejected": -170.041748046875, + "loss": 1.2163, + "margin_dpo/margin_mean": 27.157814025878906, + "margin_dpo/margin_std": 56.61811828613281, + "step": 233 + }, + { + "KL/chosen_KL_mean": -67.56138610839844, + "KL/mean": -80.51182556152344, + "KL/rejected_KL_mean": -93.46226501464844, + "KL/std": 52.28334426879883, + "epoch": 0.35374149659863946, + "fcm_dpo/beta": 0.009842677041888237, + "fcm_dpo/delta": -0.03836081176996231, + "fcm_dpo/margin": 25.900876998901367, + "fcm_dpo/q_t": 0.44125378131866455, + "grad_norm": 18.20611000061035, + "learning_rate": 4.09681781007452e-07, + "logits/chosen": 0.621496319770813, + "logits/rejected": 0.6191729307174683, + "logps/chosen": -123.48728942871094, + "logps/ref_chosen": -55.92589569091797, + "logps/ref_rejected": -51.11608123779297, + "logps/rejected": -144.57833862304688, + "loss": 1.2379, + "margin_dpo/margin_mean": 25.90087890625, + "margin_dpo/margin_std": 57.52040100097656, + "step": 234 + }, + { + "KL/chosen_KL_mean": -62.01856231689453, + "KL/mean": -85.31228637695312, + "KL/rejected_KL_mean": -108.60600280761719, + "KL/std": 57.6728630065918, + "epoch": 0.35525321239606955, + "fcm_dpo/beta": 0.009540073573589325, + "fcm_dpo/delta": -0.051198702305555344, + "fcm_dpo/margin": 46.587440490722656, + "fcm_dpo/q_t": 0.39602601528167725, + "grad_norm": 14.494193077087402, + "learning_rate": 4.08662192950594e-07, + "logits/chosen": 0.6710847616195679, + "logits/rejected": 0.6589834690093994, + "logps/chosen": -126.55828857421875, + "logps/ref_chosen": -64.53972625732422, + "logps/ref_rejected": -77.69151306152344, + "logps/rejected": -186.29751586914062, + "loss": 1.0562, + "margin_dpo/margin_mean": 46.58743667602539, + "margin_dpo/margin_std": 55.38169860839844, + "step": 235 + }, + { + "KL/chosen_KL_mean": -89.08969116210938, + "KL/mean": -106.5599365234375, + "KL/rejected_KL_mean": -124.0301742553711, + "KL/std": 55.50977325439453, + "epoch": 0.35676492819349964, + "fcm_dpo/beta": 0.009686114266514778, + "fcm_dpo/delta": 0.06868328154087067, + "fcm_dpo/margin": 34.94047546386719, + "fcm_dpo/q_t": 0.42309021949768066, + "grad_norm": 11.22431468963623, + "learning_rate": 4.076381667711306e-07, + "logits/chosen": 0.6049457788467407, + "logits/rejected": 0.5927116870880127, + "logps/chosen": -160.2444305419922, + "logps/ref_chosen": -71.15473937988281, + "logps/ref_rejected": -84.88541412353516, + "logps/rejected": -208.91558837890625, + "loss": 1.1686, + "margin_dpo/margin_mean": 34.94047546386719, + "margin_dpo/margin_std": 64.02879333496094, + "step": 236 + }, + { + "KL/chosen_KL_mean": -74.93379211425781, + "KL/mean": -96.43376159667969, + "KL/rejected_KL_mean": -117.93374633789062, + "KL/std": 55.61524963378906, + "epoch": 0.35827664399092973, + "fcm_dpo/beta": 0.009799846448004246, + "fcm_dpo/delta": -0.024707447737455368, + "fcm_dpo/margin": 42.99995422363281, + "fcm_dpo/q_t": 0.402961790561676, + "grad_norm": 15.107486724853516, + "learning_rate": 4.066097311132753e-07, + "logits/chosen": 0.6670963168144226, + "logits/rejected": 0.6589772701263428, + "logps/chosen": -151.0758056640625, + "logps/ref_chosen": -76.14201354980469, + "logps/ref_rejected": -80.88479614257812, + "logps/rejected": -198.81854248046875, + "loss": 1.1068, + "margin_dpo/margin_mean": 42.99995422363281, + "margin_dpo/margin_std": 64.99576568603516, + "step": 237 + }, + { + "KL/chosen_KL_mean": -66.69758605957031, + "KL/mean": -87.274658203125, + "KL/rejected_KL_mean": -107.85171508789062, + "KL/std": 57.005157470703125, + "epoch": 0.35978835978835977, + "fcm_dpo/beta": 0.009592464193701744, + "fcm_dpo/delta": 0.004950430244207382, + "fcm_dpo/margin": 41.15412902832031, + "fcm_dpo/q_t": 0.4092939496040344, + "grad_norm": 15.823814392089844, + "learning_rate": 4.0557691474458414e-07, + "logits/chosen": 0.6299471855163574, + "logits/rejected": 0.6191462278366089, + "logps/chosen": -135.5824432373047, + "logps/ref_chosen": -68.88484954833984, + "logps/ref_rejected": -75.8946304321289, + "logps/rejected": -183.746337890625, + "loss": 1.1119, + "margin_dpo/margin_mean": 41.15412139892578, + "margin_dpo/margin_std": 60.12267303466797, + "step": 238 + }, + { + "KL/chosen_KL_mean": -80.4689712524414, + "KL/mean": -102.43445587158203, + "KL/rejected_KL_mean": -124.39994049072266, + "KL/std": 62.442771911621094, + "epoch": 0.36130007558578986, + "fcm_dpo/beta": 0.009715170599520206, + "fcm_dpo/delta": -0.03162968158721924, + "fcm_dpo/margin": 43.93098068237305, + "fcm_dpo/q_t": 0.40267249941825867, + "grad_norm": 18.650232315063477, + "learning_rate": 4.045397465551513e-07, + "logits/chosen": 0.7951195240020752, + "logits/rejected": 0.6444146633148193, + "logps/chosen": -137.2407989501953, + "logps/ref_chosen": -56.771827697753906, + "logps/ref_rejected": -116.23050689697266, + "logps/rejected": -240.6304473876953, + "loss": 1.1093, + "margin_dpo/margin_mean": 43.93098449707031, + "margin_dpo/margin_std": 67.45944213867188, + "step": 239 + }, + { + "KL/chosen_KL_mean": -78.93304443359375, + "KL/mean": -105.90691375732422, + "KL/rejected_KL_mean": -132.8807830810547, + "KL/std": 62.66727828979492, + "epoch": 0.36281179138321995, + "fcm_dpo/beta": 0.009080219082534313, + "fcm_dpo/delta": -0.10564997047185898, + "fcm_dpo/margin": 53.94773483276367, + "fcm_dpo/q_t": 0.38679730892181396, + "grad_norm": 12.00704574584961, + "learning_rate": 4.0349825555680045e-07, + "logits/chosen": 0.68144690990448, + "logits/rejected": 0.5815201997756958, + "logps/chosen": -132.2871551513672, + "logps/ref_chosen": -53.35411071777344, + "logps/ref_rejected": -80.12019348144531, + "logps/rejected": -213.0009765625, + "loss": 1.0426, + "margin_dpo/margin_mean": 53.947731018066406, + "margin_dpo/margin_std": 67.94525909423828, + "step": 240 + }, + { + "KL/chosen_KL_mean": -74.51988983154297, + "KL/mean": -92.31866455078125, + "KL/rejected_KL_mean": -110.117431640625, + "KL/std": 59.715797424316406, + "epoch": 0.36432350718065004, + "fcm_dpo/beta": 0.009323697537183762, + "fcm_dpo/delta": 0.07455047965049744, + "fcm_dpo/margin": 35.59754943847656, + "fcm_dpo/q_t": 0.42614781856536865, + "grad_norm": 17.535839080810547, + "learning_rate": 4.0245247088227377e-07, + "logits/chosen": 0.6315206289291382, + "logits/rejected": 0.5956459641456604, + "logps/chosen": -146.41529846191406, + "logps/ref_chosen": -71.89541625976562, + "logps/ref_rejected": -83.03492736816406, + "logps/rejected": -193.15235900878906, + "loss": 1.1761, + "margin_dpo/margin_mean": 35.5975456237793, + "margin_dpo/margin_std": 67.8523941040039, + "step": 241 + }, + { + "KL/chosen_KL_mean": -76.13401794433594, + "KL/mean": -98.4712905883789, + "KL/rejected_KL_mean": -120.80856323242188, + "KL/std": 61.277618408203125, + "epoch": 0.36583522297808013, + "fcm_dpo/beta": 0.009163142181932926, + "fcm_dpo/delta": -0.012581542134284973, + "fcm_dpo/margin": 44.67455291748047, + "fcm_dpo/q_t": 0.40708714723587036, + "grad_norm": 11.770535469055176, + "learning_rate": 4.0140242178441665e-07, + "logits/chosen": 0.6444741487503052, + "logits/rejected": 0.624873161315918, + "logps/chosen": -134.0614471435547, + "logps/ref_chosen": -57.927433013916016, + "logps/ref_rejected": -67.838623046875, + "logps/rejected": -188.64718627929688, + "loss": 1.1051, + "margin_dpo/margin_mean": 44.67455291748047, + "margin_dpo/margin_std": 65.82492065429688, + "step": 242 + }, + { + "KL/chosen_KL_mean": -75.29219055175781, + "KL/mean": -95.34475708007812, + "KL/rejected_KL_mean": -115.39732360839844, + "KL/std": 60.416358947753906, + "epoch": 0.3673469387755102, + "fcm_dpo/beta": 0.009415372274816036, + "fcm_dpo/delta": 0.025139713659882545, + "fcm_dpo/margin": 40.10512924194336, + "fcm_dpo/q_t": 0.41358304023742676, + "grad_norm": 16.480270385742188, + "learning_rate": 4.003481376353596e-07, + "logits/chosen": 0.6188076734542847, + "logits/rejected": 0.6329231262207031, + "logps/chosen": -149.56887817382812, + "logps/ref_chosen": -74.27667236328125, + "logps/ref_rejected": -73.24340057373047, + "logps/rejected": -188.64071655273438, + "loss": 1.1272, + "margin_dpo/margin_mean": 40.105133056640625, + "margin_dpo/margin_std": 62.97103500366211, + "step": 243 + }, + { + "KL/chosen_KL_mean": -68.44050598144531, + "KL/mean": -94.90727996826172, + "KL/rejected_KL_mean": -121.3740463256836, + "KL/std": 60.30860900878906, + "epoch": 0.3688586545729403, + "fcm_dpo/beta": 0.009139430709183216, + "fcm_dpo/delta": -0.09782852977514267, + "fcm_dpo/margin": 52.93354797363281, + "fcm_dpo/q_t": 0.38723719120025635, + "grad_norm": 11.722265243530273, + "learning_rate": 3.9928964792569654e-07, + "logits/chosen": 0.6751892566680908, + "logits/rejected": 0.586534321308136, + "logps/chosen": -121.80440521240234, + "logps/ref_chosen": -53.36390686035156, + "logps/ref_rejected": -71.10276794433594, + "logps/rejected": -192.47682189941406, + "loss": 1.0407, + "margin_dpo/margin_mean": 52.93354797363281, + "margin_dpo/margin_std": 63.211524963378906, + "step": 244 + }, + { + "KL/chosen_KL_mean": -77.30024719238281, + "KL/mean": -107.91534423828125, + "KL/rejected_KL_mean": -138.53045654296875, + "KL/std": 62.56925964355469, + "epoch": 0.37037037037037035, + "fcm_dpo/beta": 0.008342870511114597, + "fcm_dpo/delta": -0.13193881511688232, + "fcm_dpo/margin": 61.230194091796875, + "fcm_dpo/q_t": 0.37750697135925293, + "grad_norm": 13.34937572479248, + "learning_rate": 3.982269822636601e-07, + "logits/chosen": 0.7413665056228638, + "logits/rejected": 0.7142044305801392, + "logps/chosen": -148.49534606933594, + "logps/ref_chosen": -71.19510650634766, + "logps/ref_rejected": -80.76235961914062, + "logps/rejected": -219.29281616210938, + "loss": 1.0067, + "margin_dpo/margin_mean": 61.230194091796875, + "margin_dpo/margin_std": 64.25137329101562, + "step": 245 + }, + { + "KL/chosen_KL_mean": -91.80043029785156, + "KL/mean": -113.60960388183594, + "KL/rejected_KL_mean": -135.4187774658203, + "KL/std": 62.362632751464844, + "epoch": 0.37188208616780044, + "fcm_dpo/beta": 0.008444206789135933, + "fcm_dpo/delta": 0.03405901789665222, + "fcm_dpo/margin": 43.61833953857422, + "fcm_dpo/q_t": 0.41692936420440674, + "grad_norm": 12.155149459838867, + "learning_rate": 3.971601703742932e-07, + "logits/chosen": 0.7366214990615845, + "logits/rejected": 0.6703383922576904, + "logps/chosen": -163.42147827148438, + "logps/ref_chosen": -71.62104797363281, + "logps/ref_rejected": -94.03392028808594, + "logps/rejected": -229.45269775390625, + "loss": 1.1571, + "margin_dpo/margin_mean": 43.61833953857422, + "margin_dpo/margin_std": 78.9627914428711, + "step": 246 + }, + { + "KL/chosen_KL_mean": -94.80854034423828, + "KL/mean": -109.75330352783203, + "KL/rejected_KL_mean": -124.69806671142578, + "KL/std": 63.64940643310547, + "epoch": 0.37339380196523053, + "fcm_dpo/beta": 0.008503669872879982, + "fcm_dpo/delta": 0.047118451446294785, + "fcm_dpo/margin": 29.8895206451416, + "fcm_dpo/q_t": 0.4427517056465149, + "grad_norm": 15.709403991699219, + "learning_rate": 3.960892420986177e-07, + "logits/chosen": 0.6784308552742004, + "logits/rejected": 0.6688522100448608, + "logps/chosen": -174.83108520507812, + "logps/ref_chosen": -80.02254486083984, + "logps/ref_rejected": -89.22705841064453, + "logps/rejected": -213.9251251220703, + "loss": 1.2434, + "margin_dpo/margin_mean": 29.88951873779297, + "margin_dpo/margin_std": 74.12858581542969, + "step": 247 + }, + { + "KL/chosen_KL_mean": -80.26335144042969, + "KL/mean": -104.7964096069336, + "KL/rejected_KL_mean": -129.3294677734375, + "KL/std": 65.8143310546875, + "epoch": 0.3749055177626606, + "fcm_dpo/beta": 0.008583104237914085, + "fcm_dpo/delta": -0.024302110075950623, + "fcm_dpo/margin": 49.06610107421875, + "fcm_dpo/q_t": 0.4045664370059967, + "grad_norm": 16.8564395904541, + "learning_rate": 3.9501422739279953e-07, + "logits/chosen": 0.6709645986557007, + "logits/rejected": 0.7285152673721313, + "logps/chosen": -145.6413116455078, + "logps/ref_chosen": -65.37796020507812, + "logps/ref_rejected": -61.365787506103516, + "logps/rejected": -190.69525146484375, + "loss": 1.1079, + "margin_dpo/margin_mean": 49.06610107421875, + "margin_dpo/margin_std": 75.50920104980469, + "step": 248 + }, + { + "KL/chosen_KL_mean": -101.28555297851562, + "KL/mean": -109.88075256347656, + "KL/rejected_KL_mean": -118.47596740722656, + "KL/std": 61.5130500793457, + "epoch": 0.3764172335600907, + "fcm_dpo/beta": 0.008645986206829548, + "fcm_dpo/delta": 0.05666162446141243, + "fcm_dpo/margin": 17.190412521362305, + "fcm_dpo/q_t": 0.4651719331741333, + "grad_norm": 16.34800910949707, + "learning_rate": 3.9393515632731094e-07, + "logits/chosen": 0.6691962480545044, + "logits/rejected": 0.7120274305343628, + "logps/chosen": -175.8870086669922, + "logps/ref_chosen": -74.60145568847656, + "logps/ref_rejected": -63.79338455200195, + "logps/rejected": -182.26934814453125, + "loss": 1.3502, + "margin_dpo/margin_mean": 17.190414428710938, + "margin_dpo/margin_std": 75.83460998535156, + "step": 249 + }, + { + "KL/chosen_KL_mean": -88.96881103515625, + "KL/mean": -113.87507629394531, + "KL/rejected_KL_mean": -138.7813720703125, + "KL/std": 69.089599609375, + "epoch": 0.3779289493575208, + "fcm_dpo/beta": 0.008802786469459534, + "fcm_dpo/delta": -0.046542391180992126, + "fcm_dpo/margin": 49.81255340576172, + "fcm_dpo/q_t": 0.39933088421821594, + "grad_norm": 14.993885040283203, + "learning_rate": 3.9285205908608934e-07, + "logits/chosen": 0.7950924634933472, + "logits/rejected": 0.7543875575065613, + "logps/chosen": -150.90701293945312, + "logps/ref_chosen": -61.938209533691406, + "logps/ref_rejected": -72.21602630615234, + "logps/rejected": -210.9973907470703, + "loss": 1.0931, + "margin_dpo/margin_mean": 49.81255340576172, + "margin_dpo/margin_std": 71.54025268554688, + "step": 250 + }, + { + "KL/chosen_KL_mean": -96.157470703125, + "KL/mean": -113.1865234375, + "KL/rejected_KL_mean": -130.21556091308594, + "KL/std": 60.91024398803711, + "epoch": 0.3794406651549509, + "fcm_dpo/beta": 0.00891035981476307, + "fcm_dpo/delta": 0.10659044981002808, + "fcm_dpo/margin": 34.05808639526367, + "fcm_dpo/q_t": 0.43218693137168884, + "grad_norm": 23.815366744995117, + "learning_rate": 3.9176496596569265e-07, + "logits/chosen": 0.7540764808654785, + "logits/rejected": 0.7111141681671143, + "logps/chosen": -163.01441955566406, + "logps/ref_chosen": -66.85694885253906, + "logps/ref_rejected": -84.83396911621094, + "logps/rejected": -215.04953002929688, + "loss": 1.1989, + "margin_dpo/margin_mean": 34.05808639526367, + "margin_dpo/margin_std": 69.84614562988281, + "step": 251 + }, + { + "KL/chosen_KL_mean": -90.72431945800781, + "KL/mean": -103.80723571777344, + "KL/rejected_KL_mean": -116.89015197753906, + "KL/std": 63.69288635253906, + "epoch": 0.38095238095238093, + "fcm_dpo/beta": 0.0090938750654459, + "fcm_dpo/delta": -0.007932256907224655, + "fcm_dpo/margin": 26.16583824157715, + "fcm_dpo/q_t": 0.4449775218963623, + "grad_norm": 20.581787109375, + "learning_rate": 3.9067390737445254e-07, + "logits/chosen": 0.6826932430267334, + "logits/rejected": 0.6261128187179565, + "logps/chosen": -146.9482421875, + "logps/ref_chosen": -56.22393035888672, + "logps/ref_rejected": -77.1136245727539, + "logps/rejected": -194.0037841796875, + "loss": 1.2792, + "margin_dpo/margin_mean": 26.16583824157715, + "margin_dpo/margin_std": 74.60096740722656, + "step": 252 + }, + { + "KL/chosen_KL_mean": -89.88412475585938, + "KL/mean": -108.26229095458984, + "KL/rejected_KL_mean": -126.64045715332031, + "KL/std": 60.83412170410156, + "epoch": 0.382464096749811, + "fcm_dpo/beta": 0.008921799249947071, + "fcm_dpo/delta": -0.027868332341313362, + "fcm_dpo/margin": 36.7563362121582, + "fcm_dpo/q_t": 0.4237293004989624, + "grad_norm": 16.071880340576172, + "learning_rate": 3.8957891383162304e-07, + "logits/chosen": 0.7912516593933105, + "logits/rejected": 0.7534704208374023, + "logps/chosen": -142.09414672851562, + "logps/ref_chosen": -52.21001434326172, + "logps/ref_rejected": -58.75764846801758, + "logps/rejected": -185.39810180664062, + "loss": 1.1652, + "margin_dpo/margin_mean": 36.7563362121582, + "margin_dpo/margin_std": 63.523983001708984, + "step": 253 + }, + { + "KL/chosen_KL_mean": -98.28326416015625, + "KL/mean": -117.90261840820312, + "KL/rejected_KL_mean": -137.52197265625, + "KL/std": 67.03953552246094, + "epoch": 0.3839758125472411, + "fcm_dpo/beta": 0.009032230824232101, + "fcm_dpo/delta": 0.05078422278165817, + "fcm_dpo/margin": 39.23871612548828, + "fcm_dpo/q_t": 0.42154332995414734, + "grad_norm": 13.182265281677246, + "learning_rate": 3.884800159665276e-07, + "logits/chosen": 0.70362389087677, + "logits/rejected": 0.6483662128448486, + "logps/chosen": -163.91958618164062, + "logps/ref_chosen": -65.63632202148438, + "logps/ref_rejected": -82.34425354003906, + "logps/rejected": -219.86622619628906, + "loss": 1.1509, + "margin_dpo/margin_mean": 39.23871612548828, + "margin_dpo/margin_std": 67.09379577636719, + "step": 254 + }, + { + "KL/chosen_KL_mean": -87.982666015625, + "KL/mean": -110.94923400878906, + "KL/rejected_KL_mean": -133.91580200195312, + "KL/std": 64.83241271972656, + "epoch": 0.3854875283446712, + "fcm_dpo/beta": 0.009070008993148804, + "fcm_dpo/delta": -0.019309692084789276, + "fcm_dpo/margin": 45.93316650390625, + "fcm_dpo/q_t": 0.4051594138145447, + "grad_norm": 17.457691192626953, + "learning_rate": 3.873772445177015e-07, + "logits/chosen": 0.6465994119644165, + "logits/rejected": 0.6145304441452026, + "logps/chosen": -155.8937530517578, + "logps/ref_chosen": -67.91108703613281, + "logps/ref_rejected": -83.89114379882812, + "logps/rejected": -217.8069610595703, + "loss": 1.1121, + "margin_dpo/margin_mean": 45.933170318603516, + "margin_dpo/margin_std": 72.16119384765625, + "step": 255 + }, + { + "KL/chosen_KL_mean": -106.05748748779297, + "KL/mean": -128.17782592773438, + "KL/rejected_KL_mean": -150.2981719970703, + "KL/std": 61.828369140625, + "epoch": 0.3869992441421013, + "fcm_dpo/beta": 0.009000243619084358, + "fcm_dpo/delta": 0.0012618564069271088, + "fcm_dpo/margin": 44.24068069458008, + "fcm_dpo/q_t": 0.4116033911705017, + "grad_norm": 15.866179466247559, + "learning_rate": 3.862706303320329e-07, + "logits/chosen": 0.6873655319213867, + "logits/rejected": 0.6170613765716553, + "logps/chosen": -169.55746459960938, + "logps/ref_chosen": -63.49998474121094, + "logps/ref_rejected": -90.77104187011719, + "logps/rejected": -241.0692138671875, + "loss": 1.1498, + "margin_dpo/margin_mean": 44.240684509277344, + "margin_dpo/margin_std": 80.80990600585938, + "step": 256 + }, + { + "KL/chosen_KL_mean": -99.76148986816406, + "KL/mean": -128.57794189453125, + "KL/rejected_KL_mean": -157.39439392089844, + "KL/std": 69.79254913330078, + "epoch": 0.3885109599395314, + "fcm_dpo/beta": 0.00856287032365799, + "fcm_dpo/delta": -0.11203688383102417, + "fcm_dpo/margin": 57.63291931152344, + "fcm_dpo/q_t": 0.38817721605300903, + "grad_norm": 11.6730318069458, + "learning_rate": 3.851602043638994e-07, + "logits/chosen": 0.6632556915283203, + "logits/rejected": 0.5904806852340698, + "logps/chosen": -170.36215209960938, + "logps/ref_chosen": -70.60064697265625, + "logps/ref_rejected": -108.58313751220703, + "logps/rejected": -265.9775390625, + "loss": 1.0577, + "margin_dpo/margin_mean": 57.63291931152344, + "margin_dpo/margin_std": 79.96395111083984, + "step": 257 + }, + { + "KL/chosen_KL_mean": -102.39521026611328, + "KL/mean": -123.94535827636719, + "KL/rejected_KL_mean": -145.49549865722656, + "KL/std": 61.29582214355469, + "epoch": 0.3900226757369615, + "fcm_dpo/beta": 0.008610175922513008, + "fcm_dpo/delta": 0.03255663812160492, + "fcm_dpo/margin": 43.10028839111328, + "fcm_dpo/q_t": 0.4138616621494293, + "grad_norm": 14.99991512298584, + "learning_rate": 3.840459976743023e-07, + "logits/chosen": 0.7334133386611938, + "logits/rejected": 0.6779052019119263, + "logps/chosen": -161.64938354492188, + "logps/ref_chosen": -59.25416564941406, + "logps/ref_rejected": -85.58709716796875, + "logps/rejected": -231.0825958251953, + "loss": 1.12, + "margin_dpo/margin_mean": 43.10028839111328, + "margin_dpo/margin_std": 63.61687469482422, + "step": 258 + }, + { + "KL/chosen_KL_mean": -90.74569702148438, + "KL/mean": -121.82440185546875, + "KL/rejected_KL_mean": -152.9031219482422, + "KL/std": 67.67817687988281, + "epoch": 0.3915343915343915, + "fcm_dpo/beta": 0.008102967403829098, + "fcm_dpo/delta": -0.12510867416858673, + "fcm_dpo/margin": 62.157413482666016, + "fcm_dpo/q_t": 0.3824307322502136, + "grad_norm": 12.970193862915039, + "learning_rate": 3.8292804142999796e-07, + "logits/chosen": 0.6587529182434082, + "logits/rejected": 0.549065113067627, + "logps/chosen": -156.18057250976562, + "logps/ref_chosen": -65.43487548828125, + "logps/ref_rejected": -95.41731262207031, + "logps/rejected": -248.3204345703125, + "loss": 1.0524, + "margin_dpo/margin_mean": 62.157413482666016, + "margin_dpo/margin_std": 83.17296600341797, + "step": 259 + }, + { + "KL/chosen_KL_mean": -91.60452270507812, + "KL/mean": -115.55876922607422, + "KL/rejected_KL_mean": -139.5130157470703, + "KL/std": 61.73158264160156, + "epoch": 0.3930461073318216, + "fcm_dpo/beta": 0.008017702028155327, + "fcm_dpo/delta": 0.017222033813595772, + "fcm_dpo/margin": 47.908504486083984, + "fcm_dpo/q_t": 0.41276681423187256, + "grad_norm": 18.296205520629883, + "learning_rate": 3.818063669026256e-07, + "logits/chosen": 0.7184348702430725, + "logits/rejected": 0.6148696541786194, + "logps/chosen": -140.69410705566406, + "logps/ref_chosen": -49.08958435058594, + "logps/ref_rejected": -79.01708221435547, + "logps/rejected": -218.53009033203125, + "loss": 1.1336, + "margin_dpo/margin_mean": 47.90850067138672, + "margin_dpo/margin_std": 78.28831481933594, + "step": 260 + }, + { + "KL/chosen_KL_mean": -91.9481201171875, + "KL/mean": -112.35318756103516, + "KL/rejected_KL_mean": -132.7582550048828, + "KL/std": 60.71597671508789, + "epoch": 0.3945578231292517, + "fcm_dpo/beta": 0.008456382900476456, + "fcm_dpo/delta": 0.05926769599318504, + "fcm_dpo/margin": 40.810142517089844, + "fcm_dpo/q_t": 0.42231160402297974, + "grad_norm": 16.93952178955078, + "learning_rate": 3.806810054678331e-07, + "logits/chosen": 0.5660909414291382, + "logits/rejected": 0.6089234352111816, + "logps/chosen": -162.8205108642578, + "logps/ref_chosen": -70.87239074707031, + "logps/ref_rejected": -65.01522064208984, + "logps/rejected": -197.77346801757812, + "loss": 1.1557, + "margin_dpo/margin_mean": 40.810142517089844, + "margin_dpo/margin_std": 70.20431518554688, + "step": 261 + }, + { + "KL/chosen_KL_mean": -99.01957702636719, + "KL/mean": -119.05585479736328, + "KL/rejected_KL_mean": -139.09213256835938, + "KL/std": 61.233192443847656, + "epoch": 0.3960695389266818, + "fcm_dpo/beta": 0.008709060028195381, + "fcm_dpo/delta": 0.056011371314525604, + "fcm_dpo/margin": 40.072566986083984, + "fcm_dpo/q_t": 0.42116984724998474, + "grad_norm": 18.52885627746582, + "learning_rate": 3.7955198860439887e-07, + "logits/chosen": 0.7286216020584106, + "logits/rejected": 0.6594671607017517, + "logps/chosen": -166.89019775390625, + "logps/ref_chosen": -67.8706283569336, + "logps/ref_rejected": -88.7205810546875, + "logps/rejected": -227.81271362304688, + "loss": 1.1519, + "margin_dpo/margin_mean": 40.07257080078125, + "margin_dpo/margin_std": 68.45884704589844, + "step": 262 + }, + { + "KL/chosen_KL_mean": -93.94923400878906, + "KL/mean": -115.14928436279297, + "KL/rejected_KL_mean": -136.34933471679688, + "KL/std": 62.44976806640625, + "epoch": 0.3975812547241119, + "fcm_dpo/beta": 0.008855888620018959, + "fcm_dpo/delta": 0.027387959882616997, + "fcm_dpo/margin": 42.40009307861328, + "fcm_dpo/q_t": 0.41403600573539734, + "grad_norm": 16.228689193725586, + "learning_rate": 3.784193478933516e-07, + "logits/chosen": 0.6711243391036987, + "logits/rejected": 0.5571412444114685, + "logps/chosen": -149.14381408691406, + "logps/ref_chosen": -55.194583892822266, + "logps/ref_rejected": -80.54048156738281, + "logps/rejected": -216.88983154296875, + "loss": 1.1333, + "margin_dpo/margin_mean": 42.40009307861328, + "margin_dpo/margin_std": 68.54437255859375, + "step": 263 + }, + { + "KL/chosen_KL_mean": -95.62120819091797, + "KL/mean": -118.658203125, + "KL/rejected_KL_mean": -141.6951904296875, + "KL/std": 69.25521087646484, + "epoch": 0.39909297052154197, + "fcm_dpo/beta": 0.00880684144794941, + "fcm_dpo/delta": -0.0066429730504751205, + "fcm_dpo/margin": 46.07396697998047, + "fcm_dpo/q_t": 0.4083606004714966, + "grad_norm": 14.93736743927002, + "learning_rate": 3.7728311501708674e-07, + "logits/chosen": 0.5669023990631104, + "logits/rejected": 0.5313173532485962, + "logps/chosen": -178.79190063476562, + "logps/ref_chosen": -83.17068481445312, + "logps/ref_rejected": -88.33625793457031, + "logps/rejected": -230.0314483642578, + "loss": 1.1267, + "margin_dpo/margin_mean": 46.07396697998047, + "margin_dpo/margin_std": 75.87529754638672, + "step": 264 + }, + { + "KL/chosen_KL_mean": -91.21208190917969, + "KL/mean": -114.84857177734375, + "KL/rejected_KL_mean": -138.48507690429688, + "KL/std": 64.7674560546875, + "epoch": 0.40060468631897206, + "fcm_dpo/beta": 0.008796760812401772, + "fcm_dpo/delta": -0.018197163939476013, + "fcm_dpo/margin": 47.27299499511719, + "fcm_dpo/q_t": 0.40623968839645386, + "grad_norm": 12.336769104003906, + "learning_rate": 3.7614332175848027e-07, + "logits/chosen": 0.769055962562561, + "logits/rejected": 0.7016372680664062, + "logps/chosen": -142.87493896484375, + "logps/ref_chosen": -51.66284942626953, + "logps/ref_rejected": -67.1720962524414, + "logps/rejected": -205.65716552734375, + "loss": 1.138, + "margin_dpo/margin_mean": 47.27299499511719, + "margin_dpo/margin_std": 82.93624114990234, + "step": 265 + }, + { + "KL/chosen_KL_mean": -86.67529296875, + "KL/mean": -110.70747375488281, + "KL/rejected_KL_mean": -134.73963928222656, + "KL/std": 65.62783813476562, + "epoch": 0.4021164021164021, + "fcm_dpo/beta": 0.008618027903139591, + "fcm_dpo/delta": -0.016524948179721832, + "fcm_dpo/margin": 48.064353942871094, + "fcm_dpo/q_t": 0.4059688448905945, + "grad_norm": 22.678979873657227, + "learning_rate": 3.75e-07, + "logits/chosen": 0.6709597110748291, + "logits/rejected": 0.5944674015045166, + "logps/chosen": -144.12579345703125, + "logps/ref_chosen": -57.45049285888672, + "logps/ref_rejected": -77.60826110839844, + "logps/rejected": -212.347900390625, + "loss": 1.1195, + "margin_dpo/margin_mean": 48.064353942871094, + "margin_dpo/margin_std": 77.88729858398438, + "step": 266 + }, + { + "KL/chosen_KL_mean": -79.54592895507812, + "KL/mean": -98.55870056152344, + "KL/rejected_KL_mean": -117.57147216796875, + "KL/std": 63.595970153808594, + "epoch": 0.4036281179138322, + "fcm_dpo/beta": 0.008466208353638649, + "fcm_dpo/delta": -0.06876867264509201, + "fcm_dpo/margin": 38.025535583496094, + "fcm_dpo/q_t": 0.42545759677886963, + "grad_norm": 15.666740417480469, + "learning_rate": 3.738531817228131e-07, + "logits/chosen": 0.7178699970245361, + "logits/rejected": 0.6993035078048706, + "logps/chosen": -134.58128356933594, + "logps/ref_chosen": -55.03535079956055, + "logps/ref_rejected": -66.0953369140625, + "logps/rejected": -183.66680908203125, + "loss": 1.2011, + "margin_dpo/margin_mean": 38.02553939819336, + "margin_dpo/margin_std": 74.72657775878906, + "step": 267 + }, + { + "KL/chosen_KL_mean": -76.53323364257812, + "KL/mean": -100.77192687988281, + "KL/rejected_KL_mean": -125.0106201171875, + "KL/std": 66.77596282958984, + "epoch": 0.4051398337112623, + "fcm_dpo/beta": 0.008353805169463158, + "fcm_dpo/delta": -0.0063354093581438065, + "fcm_dpo/margin": 48.47738265991211, + "fcm_dpo/q_t": 0.40715789794921875, + "grad_norm": 15.87929916381836, + "learning_rate": 3.7270289900589204e-07, + "logits/chosen": 0.5625072717666626, + "logits/rejected": 0.5523948669433594, + "logps/chosen": -141.60498046875, + "logps/ref_chosen": -65.07174682617188, + "logps/ref_rejected": -71.42485809326172, + "logps/rejected": -196.43548583984375, + "loss": 1.0939, + "margin_dpo/margin_mean": 48.477378845214844, + "margin_dpo/margin_std": 65.1787109375, + "step": 268 + }, + { + "KL/chosen_KL_mean": -91.21495056152344, + "KL/mean": -115.9734115600586, + "KL/rejected_KL_mean": -140.7318878173828, + "KL/std": 69.93099212646484, + "epoch": 0.40665154950869237, + "fcm_dpo/beta": 0.008321639150381088, + "fcm_dpo/delta": -0.014926902949810028, + "fcm_dpo/margin": 49.516929626464844, + "fcm_dpo/q_t": 0.4059554636478424, + "grad_norm": 14.74932861328125, + "learning_rate": 3.7154918402511714e-07, + "logits/chosen": 0.7675411701202393, + "logits/rejected": 0.7207101583480835, + "logps/chosen": -158.35116577148438, + "logps/ref_chosen": -67.1362075805664, + "logps/ref_rejected": -82.55778503417969, + "logps/rejected": -223.2896728515625, + "loss": 1.0967, + "margin_dpo/margin_mean": 49.51692199707031, + "margin_dpo/margin_std": 69.16948699951172, + "step": 269 + }, + { + "KL/chosen_KL_mean": -89.67121124267578, + "KL/mean": -112.38426971435547, + "KL/rejected_KL_mean": -135.09732055664062, + "KL/std": 68.95745849609375, + "epoch": 0.40816326530612246, + "fcm_dpo/beta": 0.008279960602521896, + "fcm_dpo/delta": 0.026922225952148438, + "fcm_dpo/margin": 45.42612075805664, + "fcm_dpo/q_t": 0.41440480947494507, + "grad_norm": 16.14615821838379, + "learning_rate": 3.7039206905237656e-07, + "logits/chosen": 0.7438903450965881, + "logits/rejected": 0.6575706005096436, + "logps/chosen": -156.3599090576172, + "logps/ref_chosen": -66.6886978149414, + "logps/ref_rejected": -85.16129302978516, + "logps/rejected": -220.2586212158203, + "loss": 1.1353, + "margin_dpo/margin_mean": 45.426116943359375, + "margin_dpo/margin_std": 75.07006072998047, + "step": 270 + }, + { + "KL/chosen_KL_mean": -91.91019439697266, + "KL/mean": -108.07470703125, + "KL/rejected_KL_mean": -124.23921966552734, + "KL/std": 74.5631103515625, + "epoch": 0.40967498110355255, + "fcm_dpo/beta": 0.00850964430719614, + "fcm_dpo/delta": 0.0376083105802536, + "fcm_dpo/margin": 32.32902145385742, + "fcm_dpo/q_t": 0.4424757659435272, + "grad_norm": 19.207263946533203, + "learning_rate": 3.692315864546635e-07, + "logits/chosen": 0.7038223743438721, + "logits/rejected": 0.6382116079330444, + "logps/chosen": -164.3177490234375, + "logps/ref_chosen": -72.40754699707031, + "logps/ref_rejected": -92.06311798095703, + "logps/rejected": -216.30233764648438, + "loss": 1.2645, + "margin_dpo/margin_mean": 32.32902145385742, + "margin_dpo/margin_std": 89.17990112304688, + "step": 271 + }, + { + "KL/chosen_KL_mean": -75.82728576660156, + "KL/mean": -107.36683654785156, + "KL/rejected_KL_mean": -138.9063720703125, + "KL/std": 63.622032165527344, + "epoch": 0.41118669690098264, + "fcm_dpo/beta": 0.008081506937742233, + "fcm_dpo/delta": -0.12915097177028656, + "fcm_dpo/margin": 63.079078674316406, + "fcm_dpo/q_t": 0.3778059780597687, + "grad_norm": 15.495028495788574, + "learning_rate": 3.6806776869317067e-07, + "logits/chosen": 0.7020183801651001, + "logits/rejected": 0.7239981293678284, + "logps/chosen": -142.42868041992188, + "logps/ref_chosen": -66.60140228271484, + "logps/ref_rejected": -67.74340057373047, + "logps/rejected": -206.64976501464844, + "loss": 0.9983, + "margin_dpo/margin_mean": 63.079078674316406, + "margin_dpo/margin_std": 61.04596710205078, + "step": 272 + }, + { + "KL/chosen_KL_mean": -97.05889892578125, + "KL/mean": -120.1033706665039, + "KL/rejected_KL_mean": -143.1478271484375, + "KL/std": 70.52786254882812, + "epoch": 0.4126984126984127, + "fcm_dpo/beta": 0.007941139861941338, + "fcm_dpo/delta": 0.03800379857420921, + "fcm_dpo/margin": 46.088951110839844, + "fcm_dpo/q_t": 0.4187663197517395, + "grad_norm": 19.712482452392578, + "learning_rate": 3.669006483223828e-07, + "logits/chosen": 0.722759485244751, + "logits/rejected": 0.645821750164032, + "logps/chosen": -154.41378784179688, + "logps/ref_chosen": -57.35487747192383, + "logps/ref_rejected": -84.17168426513672, + "logps/rejected": -227.31951904296875, + "loss": 1.1726, + "margin_dpo/margin_mean": 46.088951110839844, + "margin_dpo/margin_std": 88.80404663085938, + "step": 273 + }, + { + "KL/chosen_KL_mean": -95.72166442871094, + "KL/mean": -120.80400085449219, + "KL/rejected_KL_mean": -145.88633728027344, + "KL/std": 67.72232055664062, + "epoch": 0.41421012849584277, + "fcm_dpo/beta": 0.008074736222624779, + "fcm_dpo/delta": -0.005904029123485088, + "fcm_dpo/margin": 50.16468048095703, + "fcm_dpo/q_t": 0.4069592356681824, + "grad_norm": 13.46408748626709, + "learning_rate": 3.657302579891656e-07, + "logits/chosen": 0.6265521049499512, + "logits/rejected": 0.609045147895813, + "logps/chosen": -155.3631591796875, + "logps/ref_chosen": -59.64149475097656, + "logps/ref_rejected": -68.29348754882812, + "logps/rejected": -214.17982482910156, + "loss": 1.12, + "margin_dpo/margin_mean": 50.16468048095703, + "margin_dpo/margin_std": 79.97119140625, + "step": 274 + }, + { + "KL/chosen_KL_mean": -93.24505615234375, + "KL/mean": -117.6572265625, + "KL/rejected_KL_mean": -142.06939697265625, + "KL/std": 64.98899841308594, + "epoch": 0.41572184429327286, + "fcm_dpo/beta": 0.008090265095233917, + "fcm_dpo/delta": 0.005442675203084946, + "fcm_dpo/margin": 48.82433319091797, + "fcm_dpo/q_t": 0.40933775901794434, + "grad_norm": 17.434755325317383, + "learning_rate": 3.645566304318526e-07, + "logits/chosen": 0.692554235458374, + "logits/rejected": 0.6077108383178711, + "logps/chosen": -146.51170349121094, + "logps/ref_chosen": -53.26664352416992, + "logps/ref_rejected": -73.84062194824219, + "logps/rejected": -215.91000366210938, + "loss": 1.0988, + "margin_dpo/margin_mean": 48.82433319091797, + "margin_dpo/margin_std": 66.89923858642578, + "step": 275 + }, + { + "KL/chosen_KL_mean": -86.27903747558594, + "KL/mean": -110.57057189941406, + "KL/rejected_KL_mean": -134.8621063232422, + "KL/std": 60.111572265625, + "epoch": 0.41723356009070295, + "fcm_dpo/beta": 0.00810546800494194, + "fcm_dpo/delta": 0.006946409121155739, + "fcm_dpo/margin": 48.58306121826172, + "fcm_dpo/q_t": 0.4088294506072998, + "grad_norm": 17.899261474609375, + "learning_rate": 3.633797984793294e-07, + "logits/chosen": 0.6355469226837158, + "logits/rejected": 0.6031860113143921, + "logps/chosen": -139.29983520507812, + "logps/ref_chosen": -53.02079772949219, + "logps/ref_rejected": -61.56678771972656, + "logps/rejected": -196.42889404296875, + "loss": 1.1036, + "margin_dpo/margin_mean": 48.58306121826172, + "margin_dpo/margin_std": 68.3792724609375, + "step": 276 + }, + { + "KL/chosen_KL_mean": -100.11457824707031, + "KL/mean": -111.95452880859375, + "KL/rejected_KL_mean": -123.79446411132812, + "KL/std": 69.6145248413086, + "epoch": 0.41874527588813304, + "fcm_dpo/beta": 0.008502400480210781, + "fcm_dpo/delta": 0.08324266970157623, + "fcm_dpo/margin": 23.679885864257812, + "fcm_dpo/q_t": 0.4552352726459503, + "grad_norm": 21.53533935546875, + "learning_rate": 3.6219979505011555e-07, + "logits/chosen": 0.743852972984314, + "logits/rejected": 0.7702978253364563, + "logps/chosen": -171.54757690429688, + "logps/ref_chosen": -71.43299102783203, + "logps/ref_rejected": -67.65852355957031, + "logps/rejected": -191.4530029296875, + "loss": 1.3012, + "margin_dpo/margin_mean": 23.679885864257812, + "margin_dpo/margin_std": 78.18290710449219, + "step": 277 + }, + { + "KL/chosen_KL_mean": -106.93940734863281, + "KL/mean": -130.02792358398438, + "KL/rejected_KL_mean": -153.11642456054688, + "KL/std": 70.84649658203125, + "epoch": 0.42025699168556313, + "fcm_dpo/beta": 0.0086892768740654, + "fcm_dpo/delta": -0.005315154790878296, + "fcm_dpo/margin": 46.17700958251953, + "fcm_dpo/q_t": 0.4069432318210602, + "grad_norm": 23.12691307067871, + "learning_rate": 3.6101665315144353e-07, + "logits/chosen": 0.6467447876930237, + "logits/rejected": 0.5822904109954834, + "logps/chosen": -174.0501708984375, + "logps/ref_chosen": -67.11076354980469, + "logps/ref_rejected": -88.74851989746094, + "logps/rejected": -241.8649444580078, + "loss": 1.1301, + "margin_dpo/margin_mean": 46.17700958251953, + "margin_dpo/margin_std": 73.78169250488281, + "step": 278 + }, + { + "KL/chosen_KL_mean": -83.75779724121094, + "KL/mean": -115.92961883544922, + "KL/rejected_KL_mean": -148.10145568847656, + "KL/std": 68.06074523925781, + "epoch": 0.4217687074829932, + "fcm_dpo/beta": 0.008041076362133026, + "fcm_dpo/delta": -0.1383289098739624, + "fcm_dpo/margin": 64.34365844726562, + "fcm_dpo/q_t": 0.376503050327301, + "grad_norm": 16.200817108154297, + "learning_rate": 3.5983040587833563e-07, + "logits/chosen": 0.6431260108947754, + "logits/rejected": 0.6045100092887878, + "logps/chosen": -138.25527954101562, + "logps/ref_chosen": -54.49748611450195, + "logps/ref_rejected": -70.42373657226562, + "logps/rejected": -218.52517700195312, + "loss": 1.0068, + "margin_dpo/margin_mean": 64.34365844726562, + "margin_dpo/margin_std": 67.10409545898438, + "step": 279 + }, + { + "KL/chosen_KL_mean": -89.94000244140625, + "KL/mean": -123.85331726074219, + "KL/rejected_KL_mean": -157.76663208007812, + "KL/std": 71.78616333007812, + "epoch": 0.42328042328042326, + "fcm_dpo/beta": 0.007400077767670155, + "fcm_dpo/delta": -0.12002548575401306, + "fcm_dpo/margin": 67.82662963867188, + "fcm_dpo/q_t": 0.3807293772697449, + "grad_norm": 11.438913345336914, + "learning_rate": 3.586410864126781e-07, + "logits/chosen": 0.7191169261932373, + "logits/rejected": 0.6715154647827148, + "logps/chosen": -150.37281799316406, + "logps/ref_chosen": -60.43281173706055, + "logps/ref_rejected": -78.39051818847656, + "logps/rejected": -236.15716552734375, + "loss": 1.0056, + "margin_dpo/margin_mean": 67.82662963867188, + "margin_dpo/margin_std": 68.43364715576172, + "step": 280 + }, + { + "KL/chosen_KL_mean": -92.57947540283203, + "KL/mean": -119.57850646972656, + "KL/rejected_KL_mean": -146.57754516601562, + "KL/std": 70.05896759033203, + "epoch": 0.42479213907785335, + "fcm_dpo/beta": 0.007345996797084808, + "fcm_dpo/delta": 0.0029397867619991302, + "fcm_dpo/margin": 53.998077392578125, + "fcm_dpo/q_t": 0.4082247316837311, + "grad_norm": 12.51746940612793, + "learning_rate": 3.574487280222929e-07, + "logits/chosen": 0.6676753163337708, + "logits/rejected": 0.6956182718276978, + "logps/chosen": -152.861572265625, + "logps/ref_chosen": -60.2820930480957, + "logps/ref_rejected": -62.04009246826172, + "logps/rejected": -208.61764526367188, + "loss": 1.1074, + "margin_dpo/margin_mean": 53.998077392578125, + "margin_dpo/margin_std": 77.513427734375, + "step": 281 + }, + { + "KL/chosen_KL_mean": -101.51087951660156, + "KL/mean": -125.53376770019531, + "KL/rejected_KL_mean": -149.55665588378906, + "KL/std": 70.40267944335938, + "epoch": 0.42630385487528344, + "fcm_dpo/beta": 0.007150539197027683, + "fcm_dpo/delta": -0.056217122822999954, + "fcm_dpo/margin": 48.0457763671875, + "fcm_dpo/q_t": 0.4206792414188385, + "grad_norm": 17.20449447631836, + "learning_rate": 3.562533640600075e-07, + "logits/chosen": 0.5914024114608765, + "logits/rejected": 0.549197256565094, + "logps/chosen": -162.13479614257812, + "logps/ref_chosen": -60.623924255371094, + "logps/ref_rejected": -68.67400360107422, + "logps/rejected": -218.23065185546875, + "loss": 1.1632, + "margin_dpo/margin_mean": 48.045772552490234, + "margin_dpo/margin_std": 82.70968627929688, + "step": 282 + }, + { + "KL/chosen_KL_mean": -98.51730346679688, + "KL/mean": -124.37229919433594, + "KL/rejected_KL_mean": -150.22727966308594, + "KL/std": 74.53094482421875, + "epoch": 0.42781557067271353, + "fcm_dpo/beta": 0.007138520944863558, + "fcm_dpo/delta": 0.034711360931396484, + "fcm_dpo/margin": 51.70996856689453, + "fcm_dpo/q_t": 0.416533887386322, + "grad_norm": 13.802699089050293, + "learning_rate": 3.550550279627215e-07, + "logits/chosen": 0.6436029672622681, + "logits/rejected": 0.5408251881599426, + "logps/chosen": -166.16505432128906, + "logps/ref_chosen": -67.64775085449219, + "logps/ref_rejected": -99.96835327148438, + "logps/rejected": -250.19564819335938, + "loss": 1.1355, + "margin_dpo/margin_mean": 51.70996856689453, + "margin_dpo/margin_std": 84.02940368652344, + "step": 283 + }, + { + "KL/chosen_KL_mean": -94.60420227050781, + "KL/mean": -119.55040740966797, + "KL/rejected_KL_mean": -144.49659729003906, + "KL/std": 73.55546569824219, + "epoch": 0.4293272864701436, + "fcm_dpo/beta": 0.007238998077809811, + "fcm_dpo/delta": 0.04319431632757187, + "fcm_dpo/margin": 49.892398834228516, + "fcm_dpo/q_t": 0.41724467277526855, + "grad_norm": 12.608574867248535, + "learning_rate": 3.5385375325047163e-07, + "logits/chosen": 0.7289709448814392, + "logits/rejected": 0.6605273485183716, + "logps/chosen": -151.57162475585938, + "logps/ref_chosen": -56.96742630004883, + "logps/ref_rejected": -86.36236572265625, + "logps/rejected": -230.85897827148438, + "loss": 1.1231, + "margin_dpo/margin_mean": 49.892398834228516, + "margin_dpo/margin_std": 71.17445373535156, + "step": 284 + }, + { + "KL/chosen_KL_mean": -108.02169799804688, + "KL/mean": -129.18577575683594, + "KL/rejected_KL_mean": -150.34986877441406, + "KL/std": 66.75422668457031, + "epoch": 0.4308390022675737, + "fcm_dpo/beta": 0.007741519715636969, + "fcm_dpo/delta": 0.07731801271438599, + "fcm_dpo/margin": 42.32817077636719, + "fcm_dpo/q_t": 0.4266880750656128, + "grad_norm": 18.31220245361328, + "learning_rate": 3.5264957352549375e-07, + "logits/chosen": 0.6713681221008301, + "logits/rejected": 0.6448211669921875, + "logps/chosen": -179.67779541015625, + "logps/ref_chosen": -71.65611267089844, + "logps/ref_rejected": -81.63829803466797, + "logps/rejected": -231.9881591796875, + "loss": 1.1678, + "margin_dpo/margin_mean": 42.32816696166992, + "margin_dpo/margin_std": 75.15558624267578, + "step": 285 + }, + { + "KL/chosen_KL_mean": -103.962890625, + "KL/mean": -138.55560302734375, + "KL/rejected_KL_mean": -173.14833068847656, + "KL/std": 73.11457824707031, + "epoch": 0.4323507180650038, + "fcm_dpo/beta": 0.007291554007679224, + "fcm_dpo/delta": -0.12348268926143646, + "fcm_dpo/margin": 69.1854476928711, + "fcm_dpo/q_t": 0.38189804553985596, + "grad_norm": 12.251094818115234, + "learning_rate": 3.514425224712835e-07, + "logits/chosen": 0.6184149384498596, + "logits/rejected": 0.5270093083381653, + "logps/chosen": -165.04241943359375, + "logps/ref_chosen": -61.07952117919922, + "logps/ref_rejected": -91.28128051757812, + "logps/rejected": -264.42962646484375, + "loss": 1.0169, + "margin_dpo/margin_mean": 69.1854476928711, + "margin_dpo/margin_std": 77.75883483886719, + "step": 286 + }, + { + "KL/chosen_KL_mean": -86.13887023925781, + "KL/mean": -118.71305847167969, + "KL/rejected_KL_mean": -151.28726196289062, + "KL/std": 72.01513671875, + "epoch": 0.43386243386243384, + "fcm_dpo/beta": 0.006978640798479319, + "fcm_dpo/delta": -0.06320708990097046, + "fcm_dpo/margin": 65.14839172363281, + "fcm_dpo/q_t": 0.3935144543647766, + "grad_norm": 11.138228416442871, + "learning_rate": 3.502326338516534e-07, + "logits/chosen": 0.7138686180114746, + "logits/rejected": 0.6729492545127869, + "logps/chosen": -132.17465209960938, + "logps/ref_chosen": -46.035789489746094, + "logps/ref_rejected": -59.95293426513672, + "logps/rejected": -211.24020385742188, + "loss": 1.0475, + "margin_dpo/margin_mean": 65.14839172363281, + "margin_dpo/margin_std": 75.49003601074219, + "step": 287 + }, + { + "KL/chosen_KL_mean": -108.60914611816406, + "KL/mean": -133.51132202148438, + "KL/rejected_KL_mean": -158.41348266601562, + "KL/std": 69.94869232177734, + "epoch": 0.43537414965986393, + "fcm_dpo/beta": 0.007091499399393797, + "fcm_dpo/delta": 0.05196710675954819, + "fcm_dpo/margin": 49.80432891845703, + "fcm_dpo/q_t": 0.4193596839904785, + "grad_norm": 13.565356254577637, + "learning_rate": 3.490199415097892e-07, + "logits/chosen": 0.5789628028869629, + "logits/rejected": 0.5190857648849487, + "logps/chosen": -174.0, + "logps/ref_chosen": -65.3908462524414, + "logps/ref_rejected": -88.53607940673828, + "logps/rejected": -246.94955444335938, + "loss": 1.1339, + "margin_dpo/margin_mean": 49.80432891845703, + "margin_dpo/margin_std": 76.44319915771484, + "step": 288 + }, + { + "KL/chosen_KL_mean": -109.88246154785156, + "KL/mean": -133.86720275878906, + "KL/rejected_KL_mean": -157.85194396972656, + "KL/std": 71.7451171875, + "epoch": 0.436885865457294, + "fcm_dpo/beta": 0.0073424819856882095, + "fcm_dpo/delta": 0.05256052315235138, + "fcm_dpo/margin": 47.9694938659668, + "fcm_dpo/q_t": 0.42131608724594116, + "grad_norm": 11.81661319732666, + "learning_rate": 3.4780447936730247e-07, + "logits/chosen": 0.7805777788162231, + "logits/rejected": 0.7439557313919067, + "logps/chosen": -164.47608947753906, + "logps/ref_chosen": -54.5936279296875, + "logps/ref_rejected": -67.20855712890625, + "logps/rejected": -225.0605010986328, + "loss": 1.1489, + "margin_dpo/margin_mean": 47.9694938659668, + "margin_dpo/margin_std": 80.595703125, + "step": 289 + }, + { + "KL/chosen_KL_mean": -122.74056243896484, + "KL/mean": -149.30039978027344, + "KL/rejected_KL_mean": -175.86026000976562, + "KL/std": 70.5626220703125, + "epoch": 0.4383975812547241, + "fcm_dpo/beta": 0.007285839878022671, + "fcm_dpo/delta": 0.01345156878232956, + "fcm_dpo/margin": 53.11968994140625, + "fcm_dpo/q_t": 0.4115050137042999, + "grad_norm": 14.872110366821289, + "learning_rate": 3.465862814232821e-07, + "logits/chosen": 0.7720860242843628, + "logits/rejected": 0.694909930229187, + "logps/chosen": -184.12513732910156, + "logps/ref_chosen": -61.38457489013672, + "logps/ref_rejected": -91.92778015136719, + "logps/rejected": -267.78802490234375, + "loss": 1.1188, + "margin_dpo/margin_mean": 53.11968994140625, + "margin_dpo/margin_std": 80.01852416992188, + "step": 290 + }, + { + "KL/chosen_KL_mean": -111.4900894165039, + "KL/mean": -144.89549255371094, + "KL/rejected_KL_mean": -178.30091857910156, + "KL/std": 73.21499633789062, + "epoch": 0.4399092970521542, + "fcm_dpo/beta": 0.007250026799738407, + "fcm_dpo/delta": -0.10002140700817108, + "fcm_dpo/margin": 66.81082153320312, + "fcm_dpo/q_t": 0.3905301094055176, + "grad_norm": 14.139022827148438, + "learning_rate": 3.4536538175334343e-07, + "logits/chosen": 0.802875280380249, + "logits/rejected": 0.7305212020874023, + "logps/chosen": -162.35311889648438, + "logps/ref_chosen": -50.863037109375, + "logps/ref_rejected": -82.20868682861328, + "logps/rejected": -260.50958251953125, + "loss": 1.0554, + "margin_dpo/margin_mean": 66.8108139038086, + "margin_dpo/margin_std": 86.83169555664062, + "step": 291 + }, + { + "KL/chosen_KL_mean": -114.57568359375, + "KL/mean": -136.93133544921875, + "KL/rejected_KL_mean": -159.2869873046875, + "KL/std": 76.05390930175781, + "epoch": 0.4414210128495843, + "fcm_dpo/beta": 0.007219928316771984, + "fcm_dpo/delta": 0.08566851168870926, + "fcm_dpo/margin": 44.71131134033203, + "fcm_dpo/q_t": 0.4291321933269501, + "grad_norm": 16.09522247314453, + "learning_rate": 3.4414181450867465e-07, + "logits/chosen": 0.7329747676849365, + "logits/rejected": 0.6827974319458008, + "logps/chosen": -178.924560546875, + "logps/ref_chosen": -64.34888458251953, + "logps/ref_rejected": -72.86434173583984, + "logps/rejected": -232.15133666992188, + "loss": 1.195, + "margin_dpo/margin_mean": 44.71131134033203, + "margin_dpo/margin_std": 93.11201477050781, + "step": 292 + }, + { + "KL/chosen_KL_mean": -108.29093933105469, + "KL/mean": -143.0504608154297, + "KL/rejected_KL_mean": -177.80999755859375, + "KL/std": 77.0187759399414, + "epoch": 0.4429327286470144, + "fcm_dpo/beta": 0.007083355449140072, + "fcm_dpo/delta": -0.10814375430345535, + "fcm_dpo/margin": 69.51905822753906, + "fcm_dpo/q_t": 0.387387752532959, + "grad_norm": 10.112757682800293, + "learning_rate": 3.4291561391508185e-07, + "logits/chosen": 0.827242374420166, + "logits/rejected": 0.7373911142349243, + "logps/chosen": -163.160400390625, + "logps/ref_chosen": -54.869468688964844, + "logps/ref_rejected": -81.858642578125, + "logps/rejected": -259.66864013671875, + "loss": 1.0591, + "margin_dpo/margin_mean": 69.51905822753906, + "margin_dpo/margin_std": 93.7623291015625, + "step": 293 + }, + { + "KL/chosen_KL_mean": -117.05465698242188, + "KL/mean": -140.6841583251953, + "KL/rejected_KL_mean": -164.31365966796875, + "KL/std": 78.87942504882812, + "epoch": 0.4444444444444444, + "fcm_dpo/beta": 0.007031150162220001, + "fcm_dpo/delta": 0.07512365281581879, + "fcm_dpo/margin": 47.259010314941406, + "fcm_dpo/q_t": 0.4250057339668274, + "grad_norm": 11.902660369873047, + "learning_rate": 3.4168681427203153e-07, + "logits/chosen": 0.7192553877830505, + "logits/rejected": 0.6725665330886841, + "logps/chosen": -173.72555541992188, + "logps/ref_chosen": -56.670902252197266, + "logps/ref_rejected": -70.32819366455078, + "logps/rejected": -234.64186096191406, + "loss": 1.1411, + "margin_dpo/margin_mean": 47.259010314941406, + "margin_dpo/margin_std": 70.30636596679688, + "step": 294 + }, + { + "KL/chosen_KL_mean": -119.32664489746094, + "KL/mean": -141.36685180664062, + "KL/rejected_KL_mean": -163.40704345703125, + "KL/std": 73.06025695800781, + "epoch": 0.4459561602418745, + "fcm_dpo/beta": 0.007442053407430649, + "fcm_dpo/delta": 0.07993526011705399, + "fcm_dpo/margin": 44.08039855957031, + "fcm_dpo/q_t": 0.42684632539749146, + "grad_norm": 16.47260856628418, + "learning_rate": 3.4045544995169125e-07, + "logits/chosen": 0.712509036064148, + "logits/rejected": 0.6072986721992493, + "logps/chosen": -169.7275390625, + "logps/ref_chosen": -50.40088653564453, + "logps/ref_rejected": -83.43521881103516, + "logps/rejected": -246.84225463867188, + "loss": 1.1656, + "margin_dpo/margin_mean": 44.08039855957031, + "margin_dpo/margin_std": 78.5947265625, + "step": 295 + }, + { + "KL/chosen_KL_mean": -124.35566711425781, + "KL/mean": -153.10382080078125, + "KL/rejected_KL_mean": -181.85195922851562, + "KL/std": 80.76585388183594, + "epoch": 0.4474678760393046, + "fcm_dpo/beta": 0.00737424660474062, + "fcm_dpo/delta": -0.02867615595459938, + "fcm_dpo/margin": 57.496299743652344, + "fcm_dpo/q_t": 0.4039427638053894, + "grad_norm": 12.168935775756836, + "learning_rate": 3.392215553979679e-07, + "logits/chosen": 0.6344171762466431, + "logits/rejected": 0.5863820910453796, + "logps/chosen": -193.50601196289062, + "logps/ref_chosen": -69.15034484863281, + "logps/ref_rejected": -89.60166931152344, + "logps/rejected": -271.45361328125, + "loss": 1.1178, + "margin_dpo/margin_mean": 57.496299743652344, + "margin_dpo/margin_std": 93.3548355102539, + "step": 296 + }, + { + "KL/chosen_KL_mean": -126.96737670898438, + "KL/mean": -155.42323303222656, + "KL/rejected_KL_mean": -183.87908935546875, + "KL/std": 70.00838470458984, + "epoch": 0.4489795918367347, + "fcm_dpo/beta": 0.0074002379551529884, + "fcm_dpo/delta": -0.024462923407554626, + "fcm_dpo/margin": 56.91170883178711, + "fcm_dpo/q_t": 0.4019927382469177, + "grad_norm": 12.931771278381348, + "learning_rate": 3.3798516512554485e-07, + "logits/chosen": 0.6887728571891785, + "logits/rejected": 0.6372237801551819, + "logps/chosen": -184.98367309570312, + "logps/ref_chosen": -58.01630401611328, + "logps/ref_rejected": -69.95780944824219, + "logps/rejected": -253.8369140625, + "loss": 1.0781, + "margin_dpo/margin_mean": 56.911705017089844, + "margin_dpo/margin_std": 72.6642074584961, + "step": 297 + }, + { + "KL/chosen_KL_mean": -124.369384765625, + "KL/mean": -145.10752868652344, + "KL/rejected_KL_mean": -165.845703125, + "KL/std": 74.36654663085938, + "epoch": 0.4504913076341648, + "fcm_dpo/beta": 0.007596236653625965, + "fcm_dpo/delta": 0.09406104683876038, + "fcm_dpo/margin": 41.47632598876953, + "fcm_dpo/q_t": 0.43052124977111816, + "grad_norm": 13.027968406677246, + "learning_rate": 3.367463137189156e-07, + "logits/chosen": 0.8152356147766113, + "logits/rejected": 0.7590856552124023, + "logps/chosen": -180.5386962890625, + "logps/ref_chosen": -56.1693115234375, + "logps/ref_rejected": -68.55052185058594, + "logps/rejected": -234.39620971679688, + "loss": 1.1905, + "margin_dpo/margin_mean": 41.47632598876953, + "margin_dpo/margin_std": 82.53489685058594, + "step": 298 + }, + { + "KL/chosen_KL_mean": -128.62130737304688, + "KL/mean": -147.03543090820312, + "KL/rejected_KL_mean": -165.44956970214844, + "KL/std": 71.73980712890625, + "epoch": 0.4520030234315949, + "fcm_dpo/beta": 0.00786143820732832, + "fcm_dpo/delta": 0.026317991316318512, + "fcm_dpo/margin": 36.82827377319336, + "fcm_dpo/q_t": 0.43662887811660767, + "grad_norm": 21.02098274230957, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": 0.631700873374939, + "logits/rejected": 0.5993541479110718, + "logps/chosen": -190.93911743164062, + "logps/ref_chosen": -62.31780242919922, + "logps/ref_rejected": -72.60028839111328, + "logps/rejected": -238.04986572265625, + "loss": 1.2412, + "margin_dpo/margin_mean": 36.828269958496094, + "margin_dpo/margin_std": 91.29080963134766, + "step": 299 + }, + { + "KL/chosen_KL_mean": -127.76425170898438, + "KL/mean": -151.45286560058594, + "KL/rejected_KL_mean": -175.1414794921875, + "KL/std": 74.65655517578125, + "epoch": 0.45351473922902497, + "fcm_dpo/beta": 0.007995443418622017, + "fcm_dpo/delta": 0.023457201197743416, + "fcm_dpo/margin": 47.377227783203125, + "fcm_dpo/q_t": 0.4160943329334259, + "grad_norm": 15.658246040344238, + "learning_rate": 3.3426136618426043e-07, + "logits/chosen": 0.7396022081375122, + "logits/rejected": 0.6713778972625732, + "logps/chosen": -188.1458282470703, + "logps/ref_chosen": -60.38157653808594, + "logps/ref_rejected": -75.45442199707031, + "logps/rejected": -250.59591674804688, + "loss": 1.152, + "margin_dpo/margin_mean": 47.377227783203125, + "margin_dpo/margin_std": 84.77381896972656, + "step": 300 + }, + { + "KL/chosen_KL_mean": -127.62648010253906, + "KL/mean": -148.23974609375, + "KL/rejected_KL_mean": -168.85302734375, + "KL/std": 72.34912109375, + "epoch": 0.455026455026455, + "fcm_dpo/beta": 0.00814204104244709, + "fcm_dpo/delta": 0.07148119807243347, + "fcm_dpo/margin": 41.22654724121094, + "fcm_dpo/q_t": 0.426349401473999, + "grad_norm": 13.319229125976562, + "learning_rate": 3.3301533956555885e-07, + "logits/chosen": 0.7542002201080322, + "logits/rejected": 0.7245285511016846, + "logps/chosen": -180.47738647460938, + "logps/ref_chosen": -52.85089111328125, + "logps/ref_rejected": -69.97584533691406, + "logps/rejected": -238.82888793945312, + "loss": 1.1898, + "margin_dpo/margin_mean": 41.22654724121094, + "margin_dpo/margin_std": 84.22346496582031, + "step": 301 + }, + { + "KL/chosen_KL_mean": -134.59283447265625, + "KL/mean": -150.45022583007812, + "KL/rejected_KL_mean": -166.30764770507812, + "KL/std": 76.41629791259766, + "epoch": 0.4565381708238851, + "fcm_dpo/beta": 0.00883575901389122, + "fcm_dpo/delta": 0.1305047869682312, + "fcm_dpo/margin": 31.714813232421875, + "fcm_dpo/q_t": 0.4405342936515808, + "grad_norm": 17.557220458984375, + "learning_rate": 3.317669908293554e-07, + "logits/chosen": 0.5931792259216309, + "logits/rejected": 0.5361766815185547, + "logps/chosen": -201.55934143066406, + "logps/ref_chosen": -66.96650695800781, + "logps/ref_rejected": -88.09510803222656, + "logps/rejected": -254.4027557373047, + "loss": 1.2561, + "margin_dpo/margin_mean": 31.714813232421875, + "margin_dpo/margin_std": 85.427001953125, + "step": 302 + }, + { + "KL/chosen_KL_mean": -116.82827758789062, + "KL/mean": -148.2735137939453, + "KL/rejected_KL_mean": -179.71875, + "KL/std": 78.29273223876953, + "epoch": 0.4580498866213152, + "fcm_dpo/beta": 0.00839284434914589, + "fcm_dpo/delta": -0.15248456597328186, + "fcm_dpo/margin": 62.89048767089844, + "fcm_dpo/q_t": 0.38311582803726196, + "grad_norm": 13.024530410766602, + "learning_rate": 3.3051635489464793e-07, + "logits/chosen": 0.6601454019546509, + "logits/rejected": 0.5881760120391846, + "logps/chosen": -178.94979858398438, + "logps/ref_chosen": -62.12152862548828, + "logps/ref_rejected": -90.31204223632812, + "logps/rejected": -270.0307922363281, + "loss": 1.0848, + "margin_dpo/margin_mean": 62.89048767089844, + "margin_dpo/margin_std": 98.73794555664062, + "step": 303 + }, + { + "KL/chosen_KL_mean": -109.92771911621094, + "KL/mean": -142.41702270507812, + "KL/rejected_KL_mean": -174.90631103515625, + "KL/std": 81.21237182617188, + "epoch": 0.4595616024187453, + "fcm_dpo/beta": 0.007727502845227718, + "fcm_dpo/delta": -0.12161770462989807, + "fcm_dpo/margin": 64.97858428955078, + "fcm_dpo/q_t": 0.38245490193367004, + "grad_norm": 12.52614688873291, + "learning_rate": 3.292634667444117e-07, + "logits/chosen": 0.6955829858779907, + "logits/rejected": 0.6446535587310791, + "logps/chosen": -170.622802734375, + "logps/ref_chosen": -60.695091247558594, + "logps/ref_rejected": -78.2525405883789, + "logps/rejected": -253.1588592529297, + "loss": 1.0246, + "margin_dpo/margin_mean": 64.97859191894531, + "margin_dpo/margin_std": 75.37732696533203, + "step": 304 + }, + { + "KL/chosen_KL_mean": -125.00523376464844, + "KL/mean": -144.45281982421875, + "KL/rejected_KL_mean": -163.90040588378906, + "KL/std": 72.6624755859375, + "epoch": 0.46107331821617537, + "fcm_dpo/beta": 0.0076552326790988445, + "fcm_dpo/delta": 0.004358620848506689, + "fcm_dpo/margin": 38.89517593383789, + "fcm_dpo/q_t": 0.43163472414016724, + "grad_norm": 17.9072208404541, + "learning_rate": 3.280083614246217e-07, + "logits/chosen": 0.6535234451293945, + "logits/rejected": 0.6887332797050476, + "logps/chosen": -197.70437622070312, + "logps/ref_chosen": -72.69914245605469, + "logps/ref_rejected": -65.65670776367188, + "logps/rejected": -229.55712890625, + "loss": 1.2238, + "margin_dpo/margin_mean": 38.89517593383789, + "margin_dpo/margin_std": 89.01888275146484, + "step": 305 + }, + { + "KL/chosen_KL_mean": -114.75829315185547, + "KL/mean": -139.0925750732422, + "KL/rejected_KL_mean": -163.42686462402344, + "KL/std": 70.30106353759766, + "epoch": 0.46258503401360546, + "fcm_dpo/beta": 0.007610576692968607, + "fcm_dpo/delta": 0.031120628118515015, + "fcm_dpo/margin": 48.66856384277344, + "fcm_dpo/q_t": 0.4150772988796234, + "grad_norm": 11.311169624328613, + "learning_rate": 3.267510740432719e-07, + "logits/chosen": 0.7671083211898804, + "logits/rejected": 0.663329005241394, + "logps/chosen": -168.72882080078125, + "logps/ref_chosen": -53.97052764892578, + "logps/ref_rejected": -71.02423095703125, + "logps/rejected": -234.4510955810547, + "loss": 1.123, + "margin_dpo/margin_mean": 48.66856384277344, + "margin_dpo/margin_std": 69.87991333007812, + "step": 306 + }, + { + "KL/chosen_KL_mean": -110.68402099609375, + "KL/mean": -122.84514617919922, + "KL/rejected_KL_mean": -135.00628662109375, + "KL/std": 71.01148986816406, + "epoch": 0.46409674981103555, + "fcm_dpo/beta": 0.007799516897648573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 24.322265625, + "fcm_dpo/q_t": 0.45799070596694946, + "grad_norm": 18.40856170654297, + "learning_rate": 3.2549163976939285e-07, + "logits/chosen": 0.7283176183700562, + "logits/rejected": 0.6819779872894287, + "logps/chosen": -168.09713745117188, + "logps/ref_chosen": -57.413108825683594, + "logps/ref_rejected": -68.68010711669922, + "logps/rejected": -203.68638610839844, + "loss": 1.327, + "margin_dpo/margin_mean": 24.322263717651367, + "margin_dpo/margin_std": 91.626708984375, + "step": 307 + }, + { + "KL/chosen_KL_mean": -111.70747375488281, + "KL/mean": -135.7213592529297, + "KL/rejected_KL_mean": -159.7352294921875, + "KL/std": 73.46157836914062, + "epoch": 0.4656084656084656, + "fcm_dpo/beta": 0.007773532532155514, + "fcm_dpo/delta": 0.02862391620874405, + "fcm_dpo/margin": 48.02775573730469, + "fcm_dpo/q_t": 0.41647428274154663, + "grad_norm": 11.686368942260742, + "learning_rate": 3.2423009383206874e-07, + "logits/chosen": 0.688902735710144, + "logits/rejected": 0.6754565834999084, + "logps/chosen": -178.3062744140625, + "logps/ref_chosen": -66.59879302978516, + "logps/ref_rejected": -74.337158203125, + "logps/rejected": -234.0723876953125, + "loss": 1.1581, + "margin_dpo/margin_mean": 48.02775573730469, + "margin_dpo/margin_std": 87.01692199707031, + "step": 308 + }, + { + "KL/chosen_KL_mean": -125.97828674316406, + "KL/mean": -151.03265380859375, + "KL/rejected_KL_mean": -176.0870361328125, + "KL/std": 71.87471771240234, + "epoch": 0.4671201814058957, + "fcm_dpo/beta": 0.007821241393685341, + "fcm_dpo/delta": 0.007066238671541214, + "fcm_dpo/margin": 50.10874557495117, + "fcm_dpo/q_t": 0.4118044972419739, + "grad_norm": 13.115415573120117, + "learning_rate": 3.229664715194511e-07, + "logits/chosen": 0.7336107492446899, + "logits/rejected": 0.6807036399841309, + "logps/chosen": -191.37303161621094, + "logps/ref_chosen": -65.39474487304688, + "logps/ref_rejected": -75.70930480957031, + "logps/rejected": -251.7963409423828, + "loss": 1.1404, + "margin_dpo/margin_mean": 50.108741760253906, + "margin_dpo/margin_std": 84.24324035644531, + "step": 309 + }, + { + "KL/chosen_KL_mean": -134.229248046875, + "KL/mean": -147.745849609375, + "KL/rejected_KL_mean": -161.26243591308594, + "KL/std": 66.22117614746094, + "epoch": 0.46863189720332576, + "fcm_dpo/beta": 0.008301754482090473, + "fcm_dpo/delta": 0.06831841170787811, + "fcm_dpo/margin": 27.033180236816406, + "fcm_dpo/q_t": 0.4524414539337158, + "grad_norm": 15.16258716583252, + "learning_rate": 3.2170080817777257e-07, + "logits/chosen": 0.7237935066223145, + "logits/rejected": 0.7082376480102539, + "logps/chosen": -208.89752197265625, + "logps/ref_chosen": -74.66827392578125, + "logps/ref_rejected": -80.5689697265625, + "logps/rejected": -241.83140563964844, + "loss": 1.2791, + "margin_dpo/margin_mean": 27.033180236816406, + "margin_dpo/margin_std": 80.27223205566406, + "step": 310 + }, + { + "KL/chosen_KL_mean": -104.70181274414062, + "KL/mean": -132.61630249023438, + "KL/rejected_KL_mean": -160.53082275390625, + "KL/std": 85.0822982788086, + "epoch": 0.47014361300075586, + "fcm_dpo/beta": 0.007963906042277813, + "fcm_dpo/delta": -0.054659221321344376, + "fcm_dpo/margin": 55.829002380371094, + "fcm_dpo/q_t": 0.40370649099349976, + "grad_norm": 13.94991683959961, + "learning_rate": 3.204331392103574e-07, + "logits/chosen": 0.6307729482650757, + "logits/rejected": 0.4885410666465759, + "logps/chosen": -164.43984985351562, + "logps/ref_chosen": -59.738033294677734, + "logps/ref_rejected": -93.60757446289062, + "logps/rejected": -254.13839721679688, + "loss": 1.1068, + "margin_dpo/margin_mean": 55.829002380371094, + "margin_dpo/margin_std": 91.33370971679688, + "step": 311 + }, + { + "KL/chosen_KL_mean": -108.8864974975586, + "KL/mean": -138.23374938964844, + "KL/rejected_KL_mean": -167.58099365234375, + "KL/std": 72.34786987304688, + "epoch": 0.47165532879818595, + "fcm_dpo/beta": 0.007836501114070415, + "fcm_dpo/delta": -0.06939233839511871, + "fcm_dpo/margin": 58.694496154785156, + "fcm_dpo/q_t": 0.3929941952228546, + "grad_norm": 11.712054252624512, + "learning_rate": 3.1916350007663176e-07, + "logits/chosen": 0.7151386737823486, + "logits/rejected": 0.6277899742126465, + "logps/chosen": -162.70294189453125, + "logps/ref_chosen": -53.816436767578125, + "logps/ref_rejected": -68.6575698852539, + "logps/rejected": -236.23855590820312, + "loss": 1.0515, + "margin_dpo/margin_mean": 58.694496154785156, + "margin_dpo/margin_std": 71.33744812011719, + "step": 312 + }, + { + "KL/chosen_KL_mean": -114.5650634765625, + "KL/mean": -130.61492919921875, + "KL/rejected_KL_mean": -146.664794921875, + "KL/std": 71.21220397949219, + "epoch": 0.47316704459561604, + "fcm_dpo/beta": 0.007850449532270432, + "fcm_dpo/delta": 0.06122079864144325, + "fcm_dpo/margin": 32.0997314453125, + "fcm_dpo/q_t": 0.4438740611076355, + "grad_norm": 13.880863189697266, + "learning_rate": 3.178919262911314e-07, + "logits/chosen": 0.7805558443069458, + "logits/rejected": 0.7599529027938843, + "logps/chosen": -174.52243041992188, + "logps/ref_chosen": -59.957359313964844, + "logps/ref_rejected": -69.31729888916016, + "logps/rejected": -215.98208618164062, + "loss": 1.2399, + "margin_dpo/margin_mean": 32.0997314453125, + "margin_dpo/margin_std": 78.52413177490234, + "step": 313 + }, + { + "KL/chosen_KL_mean": -107.243408203125, + "KL/mean": -136.32138061523438, + "KL/rejected_KL_mean": -165.3993682861328, + "KL/std": 67.65750885009766, + "epoch": 0.47467876039304613, + "fcm_dpo/beta": 0.007725432515144348, + "fcm_dpo/delta": -0.05801774933934212, + "fcm_dpo/margin": 58.155967712402344, + "fcm_dpo/q_t": 0.3972868025302887, + "grad_norm": 12.526936531066895, + "learning_rate": 3.166184534225087e-07, + "logits/chosen": 0.6884793043136597, + "logits/rejected": 0.7177518606185913, + "logps/chosen": -177.51156616210938, + "logps/ref_chosen": -70.26815795898438, + "logps/ref_rejected": -69.23971557617188, + "logps/rejected": -234.6390838623047, + "loss": 1.0679, + "margin_dpo/margin_mean": 58.15597152709961, + "margin_dpo/margin_std": 75.71588897705078, + "step": 314 + }, + { + "KL/chosen_KL_mean": -113.2779541015625, + "KL/mean": -135.77932739257812, + "KL/rejected_KL_mean": -158.28070068359375, + "KL/std": 69.8665771484375, + "epoch": 0.47619047619047616, + "fcm_dpo/beta": 0.007983740419149399, + "fcm_dpo/delta": 0.04358711838722229, + "fcm_dpo/margin": 45.00275802612305, + "fcm_dpo/q_t": 0.41818535327911377, + "grad_norm": 12.93481731414795, + "learning_rate": 3.1534311709253723e-07, + "logits/chosen": 0.6272811889648438, + "logits/rejected": 0.5963453054428101, + "logps/chosen": -181.07264709472656, + "logps/ref_chosen": -67.79469299316406, + "logps/ref_rejected": -74.55148315429688, + "logps/rejected": -232.83218383789062, + "loss": 1.1472, + "margin_dpo/margin_mean": 45.00275802612305, + "margin_dpo/margin_std": 75.59745025634766, + "step": 315 + }, + { + "KL/chosen_KL_mean": -105.72105407714844, + "KL/mean": -138.439697265625, + "KL/rejected_KL_mean": -171.1583251953125, + "KL/std": 74.41236877441406, + "epoch": 0.47770219198790626, + "fcm_dpo/beta": 0.007744115777313709, + "fcm_dpo/delta": -0.12915290892124176, + "fcm_dpo/margin": 65.43728637695312, + "fcm_dpo/q_t": 0.38332080841064453, + "grad_norm": 13.909781455993652, + "learning_rate": 3.1406595297511564e-07, + "logits/chosen": 0.5902273058891296, + "logits/rejected": 0.45409518480300903, + "logps/chosen": -161.00953674316406, + "logps/ref_chosen": -55.288482666015625, + "logps/ref_rejected": -96.15723419189453, + "logps/rejected": -267.3155517578125, + "loss": 1.0384, + "margin_dpo/margin_mean": 65.43728637695312, + "margin_dpo/margin_std": 76.73016357421875, + "step": 316 + }, + { + "KL/chosen_KL_mean": -95.98619079589844, + "KL/mean": -126.41956329345703, + "KL/rejected_KL_mean": -156.85293579101562, + "KL/std": 66.47119140625, + "epoch": 0.47921390778533635, + "fcm_dpo/beta": 0.007124145515263081, + "fcm_dpo/delta": -0.04006510600447655, + "fcm_dpo/margin": 60.866737365722656, + "fcm_dpo/q_t": 0.3974044919013977, + "grad_norm": 13.468562126159668, + "learning_rate": 3.1278699679526975e-07, + "logits/chosen": 0.715498685836792, + "logits/rejected": 0.6702585816383362, + "logps/chosen": -150.56756591796875, + "logps/ref_chosen": -54.58137512207031, + "logps/ref_rejected": -72.77232360839844, + "logps/rejected": -229.62525939941406, + "loss": 1.0662, + "margin_dpo/margin_mean": 60.866737365722656, + "margin_dpo/margin_std": 74.26116180419922, + "step": 317 + }, + { + "KL/chosen_KL_mean": -106.12240600585938, + "KL/mean": -128.9796905517578, + "KL/rejected_KL_mean": -151.8369598388672, + "KL/std": 72.66458129882812, + "epoch": 0.48072562358276644, + "fcm_dpo/beta": 0.007349280174821615, + "fcm_dpo/delta": 0.07138577103614807, + "fcm_dpo/margin": 45.71455764770508, + "fcm_dpo/q_t": 0.42840999364852905, + "grad_norm": 13.374271392822266, + "learning_rate": 3.1150628432815336e-07, + "logits/chosen": 0.6954464912414551, + "logits/rejected": 0.6276102662086487, + "logps/chosen": -159.01065063476562, + "logps/ref_chosen": -52.88822937011719, + "logps/ref_rejected": -80.63988494873047, + "logps/rejected": -232.4768524169922, + "loss": 1.2033, + "margin_dpo/margin_mean": 45.71455764770508, + "margin_dpo/margin_std": 100.32778930664062, + "step": 318 + }, + { + "KL/chosen_KL_mean": -105.37348937988281, + "KL/mean": -137.54539489746094, + "KL/rejected_KL_mean": -169.71728515625, + "KL/std": 77.01676177978516, + "epoch": 0.48223733938019653, + "fcm_dpo/beta": 0.007210618816316128, + "fcm_dpo/delta": -0.07460720092058182, + "fcm_dpo/margin": 64.34378814697266, + "fcm_dpo/q_t": 0.3947829604148865, + "grad_norm": 10.253146171569824, + "learning_rate": 3.1022385139804707e-07, + "logits/chosen": 0.6536087989807129, + "logits/rejected": 0.6382144689559937, + "logps/chosen": -169.73681640625, + "logps/ref_chosen": -64.36333465576172, + "logps/ref_rejected": -79.47296142578125, + "logps/rejected": -249.19024658203125, + "loss": 1.0683, + "margin_dpo/margin_mean": 64.34378814697266, + "margin_dpo/margin_std": 89.17861938476562, + "step": 319 + }, + { + "KL/chosen_KL_mean": -96.51083374023438, + "KL/mean": -121.93170166015625, + "KL/rejected_KL_mean": -147.35256958007812, + "KL/std": 78.21331787109375, + "epoch": 0.4837490551776266, + "fcm_dpo/beta": 0.006960249040275812, + "fcm_dpo/delta": -0.08929912000894547, + "fcm_dpo/margin": 50.84172821044922, + "fcm_dpo/q_t": 0.41984930634498596, + "grad_norm": 11.877212524414062, + "learning_rate": 3.0893973387735683e-07, + "logits/chosen": 0.6033248901367188, + "logits/rejected": 0.5647040605545044, + "logps/chosen": -146.069580078125, + "logps/ref_chosen": -49.558746337890625, + "logps/ref_rejected": -71.23444366455078, + "logps/rejected": -218.58700561523438, + "loss": 1.1531, + "margin_dpo/margin_mean": 50.84172439575195, + "margin_dpo/margin_std": 83.79927062988281, + "step": 320 + }, + { + "KL/chosen_KL_mean": -100.57034301757812, + "KL/mean": -128.36460876464844, + "KL/rejected_KL_mean": -156.15890502929688, + "KL/std": 66.17784881591797, + "epoch": 0.4852607709750567, + "fcm_dpo/beta": 0.006733224727213383, + "fcm_dpo/delta": 0.02710144966840744, + "fcm_dpo/margin": 55.58854675292969, + "fcm_dpo/q_t": 0.41282814741134644, + "grad_norm": 15.027644157409668, + "learning_rate": 3.0765396768561004e-07, + "logits/chosen": 0.6975396275520325, + "logits/rejected": 0.6782538890838623, + "logps/chosen": -152.65560913085938, + "logps/ref_chosen": -52.08526611328125, + "logps/ref_rejected": -55.58674621582031, + "logps/rejected": -211.74563598632812, + "loss": 1.1195, + "margin_dpo/margin_mean": 55.58854675292969, + "margin_dpo/margin_std": 78.52020263671875, + "step": 321 + }, + { + "KL/chosen_KL_mean": -112.34437561035156, + "KL/mean": -147.16134643554688, + "KL/rejected_KL_mean": -181.97833251953125, + "KL/std": 75.60289001464844, + "epoch": 0.48677248677248675, + "fcm_dpo/beta": 0.006711442023515701, + "fcm_dpo/delta": -0.07818324863910675, + "fcm_dpo/margin": 69.63396453857422, + "fcm_dpo/q_t": 0.3887701630592346, + "grad_norm": 10.670733451843262, + "learning_rate": 3.063665887884511e-07, + "logits/chosen": 0.7195281982421875, + "logits/rejected": 0.6374897956848145, + "logps/chosen": -159.74847412109375, + "logps/ref_chosen": -47.404109954833984, + "logps/ref_rejected": -73.4260025024414, + "logps/rejected": -255.40432739257812, + "loss": 1.031, + "margin_dpo/margin_mean": 69.63395690917969, + "margin_dpo/margin_std": 72.7001953125, + "step": 322 + }, + { + "KL/chosen_KL_mean": -116.77687072753906, + "KL/mean": -138.69090270996094, + "KL/rejected_KL_mean": -160.60494995117188, + "KL/std": 72.23223876953125, + "epoch": 0.48828420256991684, + "fcm_dpo/beta": 0.006896693259477615, + "fcm_dpo/delta": 0.10736851394176483, + "fcm_dpo/margin": 43.82807159423828, + "fcm_dpo/q_t": 0.4345787763595581, + "grad_norm": 12.62841510772705, + "learning_rate": 3.0507763319663517e-07, + "logits/chosen": 0.6039708852767944, + "logits/rejected": 0.5336369276046753, + "logps/chosen": -186.78317260742188, + "logps/ref_chosen": -70.00630187988281, + "logps/ref_rejected": -86.96690368652344, + "logps/rejected": -247.57183837890625, + "loss": 1.2031, + "margin_dpo/margin_mean": 43.82807159423828, + "margin_dpo/margin_std": 92.93321228027344, + "step": 323 + }, + { + "KL/chosen_KL_mean": -102.77244567871094, + "KL/mean": -132.74996948242188, + "KL/rejected_KL_mean": -162.7274932861328, + "KL/std": 84.26118469238281, + "epoch": 0.4897959183673469, + "fcm_dpo/beta": 0.006869921460747719, + "fcm_dpo/delta": -0.014991983771324158, + "fcm_dpo/margin": 59.95502853393555, + "fcm_dpo/q_t": 0.4040718674659729, + "grad_norm": 13.32012939453125, + "learning_rate": 3.0378713696502097e-07, + "logits/chosen": 0.7294822931289673, + "logits/rejected": 0.6708847284317017, + "logps/chosen": -158.66127014160156, + "logps/ref_chosen": -55.88882064819336, + "logps/ref_rejected": -75.23088073730469, + "logps/rejected": -237.9583740234375, + "loss": 1.0831, + "margin_dpo/margin_mean": 59.95503234863281, + "margin_dpo/margin_std": 75.76171875, + "step": 324 + }, + { + "KL/chosen_KL_mean": -124.10344696044922, + "KL/mean": -154.2080078125, + "KL/rejected_KL_mean": -184.31256103515625, + "KL/std": 82.18153381347656, + "epoch": 0.491307634164777, + "fcm_dpo/beta": 0.006824733689427376, + "fcm_dpo/delta": -0.013523155823349953, + "fcm_dpo/margin": 60.20912170410156, + "fcm_dpo/q_t": 0.4077095687389374, + "grad_norm": 11.118515014648438, + "learning_rate": 3.0249513619156206e-07, + "logits/chosen": 0.6534192562103271, + "logits/rejected": 0.5929680466651917, + "logps/chosen": -188.25045776367188, + "logps/ref_chosen": -64.14701843261719, + "logps/ref_rejected": -79.91143798828125, + "logps/rejected": -264.2239990234375, + "loss": 1.1226, + "margin_dpo/margin_mean": 60.20912170410156, + "margin_dpo/margin_std": 100.43673706054688, + "step": 325 + }, + { + "KL/chosen_KL_mean": -145.35369873046875, + "KL/mean": -156.6713409423828, + "KL/rejected_KL_mean": -167.98898315429688, + "KL/std": 74.04638671875, + "epoch": 0.4928193499622071, + "fcm_dpo/beta": 0.0068876477889716625, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 22.635299682617188, + "fcm_dpo/q_t": 0.46425288915634155, + "grad_norm": 12.687813758850098, + "learning_rate": 3.012016670162977e-07, + "logits/chosen": 0.6178878545761108, + "logits/rejected": 0.6271042823791504, + "logps/chosen": -220.885009765625, + "logps/ref_chosen": -75.53131103515625, + "logps/ref_rejected": -76.5898666381836, + "logps/rejected": -244.578857421875, + "loss": 1.3161, + "margin_dpo/margin_mean": 22.635299682617188, + "margin_dpo/margin_std": 83.47059631347656, + "step": 326 + }, + { + "KL/chosen_KL_mean": -128.65380859375, + "KL/mean": -151.80908203125, + "KL/rejected_KL_mean": -174.96434020996094, + "KL/std": 78.66702270507812, + "epoch": 0.4943310657596372, + "fcm_dpo/beta": 0.007172505371272564, + "fcm_dpo/delta": 0.07436676323413849, + "fcm_dpo/margin": 46.310516357421875, + "fcm_dpo/q_t": 0.425973117351532, + "grad_norm": 12.866050720214844, + "learning_rate": 2.99906765620341e-07, + "logits/chosen": 0.5928927063941956, + "logits/rejected": 0.5698734521865845, + "logps/chosen": -197.990966796875, + "logps/ref_chosen": -69.33717346191406, + "logps/ref_rejected": -73.37751770019531, + "logps/rejected": -248.34185791015625, + "loss": 1.1804, + "margin_dpo/margin_mean": 46.310516357421875, + "margin_dpo/margin_std": 90.25209045410156, + "step": 327 + }, + { + "KL/chosen_KL_mean": -113.55990600585938, + "KL/mean": -142.6298828125, + "KL/rejected_KL_mean": -171.69985961914062, + "KL/std": 80.41529846191406, + "epoch": 0.4958427815570673, + "fcm_dpo/beta": 0.007189217023551464, + "fcm_dpo/delta": -0.02078922465443611, + "fcm_dpo/margin": 58.13993453979492, + "fcm_dpo/q_t": 0.4041920304298401, + "grad_norm": 13.29574966430664, + "learning_rate": 2.9861046822486766e-07, + "logits/chosen": 0.5948680639266968, + "logits/rejected": 0.5532065033912659, + "logps/chosen": -175.26614379882812, + "logps/ref_chosen": -61.70623016357422, + "logps/ref_rejected": -83.73808288574219, + "logps/rejected": -255.43792724609375, + "loss": 1.0833, + "margin_dpo/margin_mean": 58.13993835449219, + "margin_dpo/margin_std": 76.74394226074219, + "step": 328 + }, + { + "KL/chosen_KL_mean": -127.44103240966797, + "KL/mean": -156.460205078125, + "KL/rejected_KL_mean": -185.47938537597656, + "KL/std": 85.77304077148438, + "epoch": 0.4973544973544973, + "fcm_dpo/beta": 0.007048935629427433, + "fcm_dpo/delta": -0.010564148426055908, + "fcm_dpo/margin": 58.038368225097656, + "fcm_dpo/q_t": 0.40908321738243103, + "grad_norm": 14.72163200378418, + "learning_rate": 2.9731281109010253e-07, + "logits/chosen": 0.7163276672363281, + "logits/rejected": 0.6566041111946106, + "logps/chosen": -191.939453125, + "logps/ref_chosen": -64.4984130859375, + "logps/ref_rejected": -83.6591796875, + "logps/rejected": -269.1385498046875, + "loss": 1.1121, + "margin_dpo/margin_mean": 58.038368225097656, + "margin_dpo/margin_std": 90.64898681640625, + "step": 329 + }, + { + "KL/chosen_KL_mean": -111.15478515625, + "KL/mean": -139.06466674804688, + "KL/rejected_KL_mean": -166.97454833984375, + "KL/std": 77.32144165039062, + "epoch": 0.4988662131519274, + "fcm_dpo/beta": 0.0071370261721313, + "fcm_dpo/delta": 0.001220574602484703, + "fcm_dpo/margin": 55.819766998291016, + "fcm_dpo/q_t": 0.41049909591674805, + "grad_norm": 12.301667213439941, + "learning_rate": 2.9601383051430505e-07, + "logits/chosen": 0.6728509664535522, + "logits/rejected": 0.6075490713119507, + "logps/chosen": -165.9594268798828, + "logps/ref_chosen": -54.80464172363281, + "logps/ref_rejected": -75.3194351196289, + "logps/rejected": -242.2939910888672, + "loss": 1.1461, + "margin_dpo/margin_mean": 55.81976318359375, + "margin_dpo/margin_std": 99.13479614257812, + "step": 330 + }, + { + "KL/chosen_KL_mean": -118.90031433105469, + "KL/mean": -155.21522521972656, + "KL/rejected_KL_mean": -191.53013610839844, + "KL/std": 81.10183715820312, + "epoch": 0.5003779289493575, + "fcm_dpo/beta": 0.006745649501681328, + "fcm_dpo/delta": -0.105572909116745, + "fcm_dpo/margin": 72.62981414794922, + "fcm_dpo/q_t": 0.38496410846710205, + "grad_norm": 12.032770156860352, + "learning_rate": 2.947135628327544e-07, + "logits/chosen": 0.7782831192016602, + "logits/rejected": 0.7540770769119263, + "logps/chosen": -178.14288330078125, + "logps/ref_chosen": -59.242584228515625, + "logps/ref_rejected": -69.87483215332031, + "logps/rejected": -261.40496826171875, + "loss": 1.0371, + "margin_dpo/margin_mean": 72.62981414794922, + "margin_dpo/margin_std": 88.2421875, + "step": 331 + }, + { + "KL/chosen_KL_mean": -124.0113754272461, + "KL/mean": -153.82049560546875, + "KL/rejected_KL_mean": -183.62960815429688, + "KL/std": 78.55375671386719, + "epoch": 0.5018896447467877, + "fcm_dpo/beta": 0.006745035294443369, + "fcm_dpo/delta": -0.00457579642534256, + "fcm_dpo/margin": 59.618255615234375, + "fcm_dpo/q_t": 0.4085049033164978, + "grad_norm": 11.804621696472168, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": 0.6241295337677002, + "logits/rejected": 0.5831087827682495, + "logps/chosen": -191.12112426757812, + "logps/ref_chosen": -67.10975646972656, + "logps/ref_rejected": -77.11839294433594, + "logps/rejected": -260.74798583984375, + "loss": 1.1011, + "margin_dpo/margin_mean": 59.618255615234375, + "margin_dpo/margin_std": 83.41954803466797, + "step": 332 + }, + { + "KL/chosen_KL_mean": -129.91983032226562, + "KL/mean": -164.92306518554688, + "KL/rejected_KL_mean": -199.92630004882812, + "KL/std": 81.36688232421875, + "epoch": 0.5034013605442177, + "fcm_dpo/beta": 0.0064663151279091835, + "fcm_dpo/delta": -0.060843661427497864, + "fcm_dpo/margin": 70.00645446777344, + "fcm_dpo/q_t": 0.39520421624183655, + "grad_norm": 10.76503849029541, + "learning_rate": 2.921093116725076e-07, + "logits/chosen": 0.6815329790115356, + "logits/rejected": 0.604261040687561, + "logps/chosen": -188.30096435546875, + "logps/ref_chosen": -58.381134033203125, + "logps/ref_rejected": -85.02839660644531, + "logps/rejected": -284.9547119140625, + "loss": 1.0549, + "margin_dpo/margin_mean": 70.00645446777344, + "margin_dpo/margin_std": 85.70564270019531, + "step": 333 + }, + { + "KL/chosen_KL_mean": -125.50909423828125, + "KL/mean": -150.45120239257812, + "KL/rejected_KL_mean": -175.393310546875, + "KL/std": 80.46173858642578, + "epoch": 0.5049130763416477, + "fcm_dpo/beta": 0.006630011834204197, + "fcm_dpo/delta": 0.07646586000919342, + "fcm_dpo/margin": 49.88423538208008, + "fcm_dpo/q_t": 0.42625001072883606, + "grad_norm": 12.834829330444336, + "learning_rate": 2.9080540104031484e-07, + "logits/chosen": 0.685641348361969, + "logits/rejected": 0.6379861831665039, + "logps/chosen": -192.40109252929688, + "logps/ref_chosen": -66.89199829101562, + "logps/ref_rejected": -91.83695220947266, + "logps/rejected": -267.23028564453125, + "loss": 1.1679, + "margin_dpo/margin_mean": 49.88423156738281, + "margin_dpo/margin_std": 90.79962158203125, + "step": 334 + }, + { + "KL/chosen_KL_mean": -126.83473205566406, + "KL/mean": -153.43295288085938, + "KL/rejected_KL_mean": -180.03115844726562, + "KL/std": 80.2386474609375, + "epoch": 0.5064247921390779, + "fcm_dpo/beta": 0.00688770180568099, + "fcm_dpo/delta": 0.03577427193522453, + "fcm_dpo/margin": 53.196434020996094, + "fcm_dpo/q_t": 0.41868066787719727, + "grad_norm": 14.178523063659668, + "learning_rate": 2.895003489933375e-07, + "logits/chosen": 0.7071677446365356, + "logits/rejected": 0.6692987084388733, + "logps/chosen": -188.34918212890625, + "logps/ref_chosen": -61.51445770263672, + "logps/ref_rejected": -75.68916320800781, + "logps/rejected": -255.7203369140625, + "loss": 1.1555, + "margin_dpo/margin_mean": 53.196434020996094, + "margin_dpo/margin_std": 95.25994110107422, + "step": 335 + }, + { + "KL/chosen_KL_mean": -140.4132537841797, + "KL/mean": -169.2587890625, + "KL/rejected_KL_mean": -198.10433959960938, + "KL/std": 85.74678039550781, + "epoch": 0.5079365079365079, + "fcm_dpo/beta": 0.006724413949996233, + "fcm_dpo/delta": 0.010933250188827515, + "fcm_dpo/margin": 57.69108200073242, + "fcm_dpo/q_t": 0.4159916043281555, + "grad_norm": 13.194255828857422, + "learning_rate": 2.8819419203668675e-07, + "logits/chosen": 0.6335151195526123, + "logits/rejected": 0.6091711521148682, + "logps/chosen": -209.26332092285156, + "logps/ref_chosen": -68.85006713867188, + "logps/ref_rejected": -92.99603271484375, + "logps/rejected": -291.100341796875, + "loss": 1.1379, + "margin_dpo/margin_mean": 57.69108581542969, + "margin_dpo/margin_std": 97.63800811767578, + "step": 336 + }, + { + "KL/chosen_KL_mean": -148.26010131835938, + "KL/mean": -171.10504150390625, + "KL/rejected_KL_mean": -193.94998168945312, + "KL/std": 85.65023803710938, + "epoch": 0.509448223733938, + "fcm_dpo/beta": 0.007140236906707287, + "fcm_dpo/delta": 0.08157092332839966, + "fcm_dpo/margin": 45.68986511230469, + "fcm_dpo/q_t": 0.4279956817626953, + "grad_norm": 11.6563138961792, + "learning_rate": 2.8688696670638053e-07, + "logits/chosen": 0.5941582918167114, + "logits/rejected": 0.5655093193054199, + "logps/chosen": -221.44793701171875, + "logps/ref_chosen": -73.18783569335938, + "logps/ref_rejected": -86.89118957519531, + "logps/rejected": -280.8411865234375, + "loss": 1.1877, + "margin_dpo/margin_mean": 45.68986511230469, + "margin_dpo/margin_std": 92.1810302734375, + "step": 337 + }, + { + "KL/chosen_KL_mean": -145.2806854248047, + "KL/mean": -168.95672607421875, + "KL/rejected_KL_mean": -192.6327667236328, + "KL/std": 78.41549682617188, + "epoch": 0.5109599395313681, + "fcm_dpo/beta": 0.007375883869826794, + "fcm_dpo/delta": 0.05678681656718254, + "fcm_dpo/margin": 47.35206604003906, + "fcm_dpo/q_t": 0.4219357669353485, + "grad_norm": 11.649815559387207, + "learning_rate": 2.8557870956832133e-07, + "logits/chosen": 0.6714822053909302, + "logits/rejected": 0.6458303928375244, + "logps/chosen": -209.22030639648438, + "logps/ref_chosen": -63.939613342285156, + "logps/ref_rejected": -75.34243774414062, + "logps/rejected": -267.9752197265625, + "loss": 1.1663, + "margin_dpo/margin_mean": 47.35206604003906, + "margin_dpo/margin_std": 87.88858032226562, + "step": 338 + }, + { + "KL/chosen_KL_mean": -118.4639892578125, + "KL/mean": -147.70608520507812, + "KL/rejected_KL_mean": -176.9481964111328, + "KL/std": 79.511962890625, + "epoch": 0.5124716553287982, + "fcm_dpo/beta": 0.007303786464035511, + "fcm_dpo/delta": -0.031887464225292206, + "fcm_dpo/margin": 58.484214782714844, + "fcm_dpo/q_t": 0.4025455117225647, + "grad_norm": 14.439207077026367, + "learning_rate": 2.842694572172736e-07, + "logits/chosen": 0.8139640688896179, + "logits/rejected": 0.7292660474777222, + "logps/chosen": -164.01312255859375, + "logps/ref_chosen": -45.54913330078125, + "logps/ref_rejected": -67.0482177734375, + "logps/rejected": -243.9964141845703, + "loss": 1.0965, + "margin_dpo/margin_mean": 58.484214782714844, + "margin_dpo/margin_std": 85.29948425292969, + "step": 339 + }, + { + "KL/chosen_KL_mean": -133.04364013671875, + "KL/mean": -160.40573120117188, + "KL/rejected_KL_mean": -187.767822265625, + "KL/std": 85.34622192382812, + "epoch": 0.5139833711262283, + "fcm_dpo/beta": 0.007357806898653507, + "fcm_dpo/delta": -0.0032272376120090485, + "fcm_dpo/margin": 54.72416687011719, + "fcm_dpo/q_t": 0.41151127219200134, + "grad_norm": 12.648419380187988, + "learning_rate": 2.8295924627584004e-07, + "logits/chosen": 0.6942380666732788, + "logits/rejected": 0.6745313405990601, + "logps/chosen": -187.04928588867188, + "logps/ref_chosen": -54.00564956665039, + "logps/ref_rejected": -61.314430236816406, + "logps/rejected": -249.08224487304688, + "loss": 1.152, + "margin_dpo/margin_mean": 54.72416687011719, + "margin_dpo/margin_std": 100.8072738647461, + "step": 340 + }, + { + "KL/chosen_KL_mean": -135.74366760253906, + "KL/mean": -167.9922637939453, + "KL/rejected_KL_mean": -200.2408447265625, + "KL/std": 85.87152099609375, + "epoch": 0.5154950869236583, + "fcm_dpo/beta": 0.006678248755633831, + "fcm_dpo/delta": -0.15111427009105682, + "fcm_dpo/margin": 64.4971923828125, + "fcm_dpo/q_t": 0.39761853218078613, + "grad_norm": 13.96065902709961, + "learning_rate": 2.816481133934373e-07, + "logits/chosen": 0.7185596823692322, + "logits/rejected": 0.6717268824577332, + "logps/chosen": -199.13876342773438, + "logps/ref_chosen": -63.39509582519531, + "logps/ref_rejected": -76.20973205566406, + "logps/rejected": -276.4505920410156, + "loss": 1.0987, + "margin_dpo/margin_mean": 64.4971923828125, + "margin_dpo/margin_std": 91.53335571289062, + "step": 341 + }, + { + "KL/chosen_KL_mean": -133.71408081054688, + "KL/mean": -164.31256103515625, + "KL/rejected_KL_mean": -194.91104125976562, + "KL/std": 91.55609130859375, + "epoch": 0.5170068027210885, + "fcm_dpo/beta": 0.006609394680708647, + "fcm_dpo/delta": -0.00558522529900074, + "fcm_dpo/margin": 61.19696807861328, + "fcm_dpo/q_t": 0.4092737138271332, + "grad_norm": 10.6194486618042, + "learning_rate": 2.8033609524527046e-07, + "logits/chosen": 0.7518157362937927, + "logits/rejected": 0.7110329270362854, + "logps/chosen": -186.76190185546875, + "logps/ref_chosen": -53.047813415527344, + "logps/ref_rejected": -68.2854232788086, + "logps/rejected": -263.19647216796875, + "loss": 1.1123, + "margin_dpo/margin_mean": 61.19697189331055, + "margin_dpo/margin_std": 93.84374237060547, + "step": 342 + }, + { + "KL/chosen_KL_mean": -129.63284301757812, + "KL/mean": -152.19546508789062, + "KL/rejected_KL_mean": -174.75808715820312, + "KL/std": 84.81193542480469, + "epoch": 0.5185185185185185, + "fcm_dpo/beta": 0.006589554250240326, + "fcm_dpo/delta": -0.016698164865374565, + "fcm_dpo/margin": 45.125267028808594, + "fcm_dpo/q_t": 0.43187424540519714, + "grad_norm": 10.550444602966309, + "learning_rate": 2.7902322853130753e-07, + "logits/chosen": 0.5749995708465576, + "logits/rejected": 0.5654809474945068, + "logps/chosen": -200.21136474609375, + "logps/ref_chosen": -70.57852935791016, + "logps/ref_rejected": -84.73873901367188, + "logps/rejected": -259.496826171875, + "loss": 1.194, + "margin_dpo/margin_mean": 45.125267028808594, + "margin_dpo/margin_std": 88.6053466796875, + "step": 343 + }, + { + "KL/chosen_KL_mean": -140.68621826171875, + "KL/mean": -173.6033172607422, + "KL/rejected_KL_mean": -206.52041625976562, + "KL/std": 85.86953735351562, + "epoch": 0.5200302343159486, + "fcm_dpo/beta": 0.006495362147688866, + "fcm_dpo/delta": -0.031697697937488556, + "fcm_dpo/margin": 65.8342056274414, + "fcm_dpo/q_t": 0.403338760137558, + "grad_norm": 12.14207935333252, + "learning_rate": 2.7770954997525274e-07, + "logits/chosen": 0.7135556936264038, + "logits/rejected": 0.6433905959129333, + "logps/chosen": -196.49722290039062, + "logps/ref_chosen": -55.811004638671875, + "logps/ref_rejected": -84.77637481689453, + "logps/rejected": -291.29681396484375, + "loss": 1.1007, + "margin_dpo/margin_mean": 65.8342056274414, + "margin_dpo/margin_std": 99.62645721435547, + "step": 344 + }, + { + "KL/chosen_KL_mean": -116.20820617675781, + "KL/mean": -145.49966430664062, + "KL/rejected_KL_mean": -174.7911376953125, + "KL/std": 78.99835205078125, + "epoch": 0.5215419501133787, + "fcm_dpo/beta": 0.006551923230290413, + "fcm_dpo/delta": 0.01797226071357727, + "fcm_dpo/margin": 58.582942962646484, + "fcm_dpo/q_t": 0.41312161087989807, + "grad_norm": 11.81797981262207, + "learning_rate": 2.7639509632351927e-07, + "logits/chosen": 0.7470645308494568, + "logits/rejected": 0.7046425342559814, + "logps/chosen": -173.99429321289062, + "logps/ref_chosen": -57.78609848022461, + "logps/ref_rejected": -78.91847229003906, + "logps/rejected": -253.70960998535156, + "loss": 1.1272, + "margin_dpo/margin_mean": 58.58293914794922, + "margin_dpo/margin_std": 93.53245544433594, + "step": 345 + }, + { + "KL/chosen_KL_mean": -122.06107330322266, + "KL/mean": -158.17715454101562, + "KL/rejected_KL_mean": -194.29324340820312, + "KL/std": 93.81214904785156, + "epoch": 0.5230536659108088, + "fcm_dpo/beta": 0.0063993064686656, + "fcm_dpo/delta": -0.0722820833325386, + "fcm_dpo/margin": 72.23216247558594, + "fcm_dpo/q_t": 0.3927100896835327, + "grad_norm": 13.295743942260742, + "learning_rate": 2.7507990434420123e-07, + "logits/chosen": 0.7267513275146484, + "logits/rejected": 0.6444227695465088, + "logps/chosen": -178.34619140625, + "logps/ref_chosen": -56.285125732421875, + "logps/ref_rejected": -91.15303039550781, + "logps/rejected": -285.4462890625, + "loss": 1.0698, + "margin_dpo/margin_mean": 72.23216247558594, + "margin_dpo/margin_std": 97.0244140625, + "step": 346 + }, + { + "KL/chosen_KL_mean": -135.72100830078125, + "KL/mean": -162.5164794921875, + "KL/rejected_KL_mean": -189.3119354248047, + "KL/std": 86.25690460205078, + "epoch": 0.5245653817082389, + "fcm_dpo/beta": 0.006409412249922752, + "fcm_dpo/delta": 0.06314625591039658, + "fcm_dpo/margin": 53.5909309387207, + "fcm_dpo/q_t": 0.4225504994392395, + "grad_norm": 12.886883735656738, + "learning_rate": 2.737640108260456e-07, + "logits/chosen": 0.8315824270248413, + "logits/rejected": 0.784061074256897, + "logps/chosen": -189.22055053710938, + "logps/ref_chosen": -53.499542236328125, + "logps/ref_rejected": -72.52565002441406, + "logps/rejected": -261.83758544921875, + "loss": 1.1462, + "margin_dpo/margin_mean": 53.5909309387207, + "margin_dpo/margin_std": 87.14447784423828, + "step": 347 + }, + { + "KL/chosen_KL_mean": -122.0595932006836, + "KL/mean": -153.3162841796875, + "KL/rejected_KL_mean": -184.57296752929688, + "KL/std": 81.57878112792969, + "epoch": 0.5260770975056689, + "fcm_dpo/beta": 0.006419610697776079, + "fcm_dpo/delta": -0.002443864941596985, + "fcm_dpo/margin": 62.51338195800781, + "fcm_dpo/q_t": 0.41006332635879517, + "grad_norm": 12.771003723144531, + "learning_rate": 2.724474525774229e-07, + "logits/chosen": 0.803961992263794, + "logits/rejected": 0.7731932401657104, + "logps/chosen": -172.846435546875, + "logps/ref_chosen": -50.78684997558594, + "logps/ref_rejected": -68.63732147216797, + "logps/rejected": -253.2102813720703, + "loss": 1.114, + "margin_dpo/margin_mean": 62.51338195800781, + "margin_dpo/margin_std": 96.11529541015625, + "step": 348 + }, + { + "KL/chosen_KL_mean": -124.03947448730469, + "KL/mean": -155.91339111328125, + "KL/rejected_KL_mean": -187.78729248046875, + "KL/std": 91.26483154296875, + "epoch": 0.527588813303099, + "fcm_dpo/beta": 0.006526275537908077, + "fcm_dpo/delta": -0.019439280033111572, + "fcm_dpo/margin": 63.74781036376953, + "fcm_dpo/q_t": 0.4069031774997711, + "grad_norm": 11.912677764892578, + "learning_rate": 2.711302664252973e-07, + "logits/chosen": 0.7378636598587036, + "logits/rejected": 0.6440693736076355, + "logps/chosen": -177.36448669433594, + "logps/ref_chosen": -53.325008392333984, + "logps/ref_rejected": -83.21236419677734, + "logps/rejected": -270.9996643066406, + "loss": 1.1076, + "margin_dpo/margin_mean": 63.7478141784668, + "margin_dpo/margin_std": 96.80409240722656, + "step": 349 + }, + { + "KL/chosen_KL_mean": -126.06622314453125, + "KL/mean": -168.70594787597656, + "KL/rejected_KL_mean": -211.34567260742188, + "KL/std": 97.71580505371094, + "epoch": 0.5291005291005291, + "fcm_dpo/beta": 0.006007745862007141, + "fcm_dpo/delta": -0.13404223322868347, + "fcm_dpo/margin": 85.27944946289062, + "fcm_dpo/q_t": 0.3789331316947937, + "grad_norm": 13.077025413513184, + "learning_rate": 2.698124892141971e-07, + "logits/chosen": 0.7190489768981934, + "logits/rejected": 0.6495992541313171, + "logps/chosen": -187.69198608398438, + "logps/ref_chosen": -61.625770568847656, + "logps/ref_rejected": -87.63627624511719, + "logps/rejected": -298.98193359375, + "loss": 1.0091, + "margin_dpo/margin_mean": 85.27944946289062, + "margin_dpo/margin_std": 92.19660186767578, + "step": 350 + }, + { + "KL/chosen_KL_mean": -122.51304626464844, + "KL/mean": -152.106201171875, + "KL/rejected_KL_mean": -181.69935607910156, + "KL/std": 81.26626586914062, + "epoch": 0.5306122448979592, + "fcm_dpo/beta": 0.006009809672832489, + "fcm_dpo/delta": 0.0496024414896965, + "fcm_dpo/margin": 59.186309814453125, + "fcm_dpo/q_t": 0.4185904562473297, + "grad_norm": 11.393095970153809, + "learning_rate": 2.6849415780518357e-07, + "logits/chosen": 0.6613746881484985, + "logits/rejected": 0.5912742018699646, + "logps/chosen": -178.76937866210938, + "logps/ref_chosen": -56.2563362121582, + "logps/ref_rejected": -79.11589813232422, + "logps/rejected": -260.81524658203125, + "loss": 1.1556, + "margin_dpo/margin_mean": 59.186309814453125, + "margin_dpo/margin_std": 103.63017272949219, + "step": 351 + }, + { + "KL/chosen_KL_mean": -124.99972534179688, + "KL/mean": -158.94674682617188, + "KL/rejected_KL_mean": -192.89376831054688, + "KL/std": 92.21725463867188, + "epoch": 0.5321239606953893, + "fcm_dpo/beta": 0.006002703681588173, + "fcm_dpo/delta": -0.009663868695497513, + "fcm_dpo/margin": 67.89405822753906, + "fcm_dpo/q_t": 0.40806788206100464, + "grad_norm": 11.126773834228516, + "learning_rate": 2.6717530907482024e-07, + "logits/chosen": 0.7351091504096985, + "logits/rejected": 0.6848005056381226, + "logps/chosen": -188.05166625976562, + "logps/ref_chosen": -63.05195236206055, + "logps/ref_rejected": -85.52035522460938, + "logps/rejected": -278.41412353515625, + "loss": 1.1028, + "margin_dpo/margin_mean": 67.89405822753906, + "margin_dpo/margin_std": 99.7062759399414, + "step": 352 + }, + { + "KL/chosen_KL_mean": -121.88029479980469, + "KL/mean": -152.44581604003906, + "KL/rejected_KL_mean": -183.01132202148438, + "KL/std": 83.91139221191406, + "epoch": 0.5336356764928194, + "fcm_dpo/beta": 0.006154675967991352, + "fcm_dpo/delta": 0.026616372168064117, + "fcm_dpo/margin": 61.13102340698242, + "fcm_dpo/q_t": 0.4135306775569916, + "grad_norm": 10.312121391296387, + "learning_rate": 2.658559799141411e-07, + "logits/chosen": 0.7169567942619324, + "logits/rejected": 0.7259862422943115, + "logps/chosen": -190.88949584960938, + "logps/ref_chosen": -69.00918579101562, + "logps/ref_rejected": -72.65840148925781, + "logps/rejected": -255.6697235107422, + "loss": 1.1138, + "margin_dpo/margin_mean": 61.131019592285156, + "margin_dpo/margin_std": 87.21917724609375, + "step": 353 + }, + { + "KL/chosen_KL_mean": -121.11980438232422, + "KL/mean": -157.54824829101562, + "KL/rejected_KL_mean": -193.97665405273438, + "KL/std": 78.74341583251953, + "epoch": 0.5351473922902494, + "fcm_dpo/beta": 0.005997260566800833, + "fcm_dpo/delta": -0.04329075664281845, + "fcm_dpo/margin": 72.85685729980469, + "fcm_dpo/q_t": 0.39849966764450073, + "grad_norm": 12.67549991607666, + "learning_rate": 2.6453620722761895e-07, + "logits/chosen": 0.8099727630615234, + "logits/rejected": 0.6850665807723999, + "logps/chosen": -160.90814208984375, + "logps/ref_chosen": -39.78833770751953, + "logps/ref_rejected": -69.56885528564453, + "logps/rejected": -263.5455322265625, + "loss": 1.0772, + "margin_dpo/margin_mean": 72.85685729980469, + "margin_dpo/margin_std": 97.290283203125, + "step": 354 + }, + { + "KL/chosen_KL_mean": -125.32882690429688, + "KL/mean": -166.8375244140625, + "KL/rejected_KL_mean": -208.34622192382812, + "KL/std": 94.46980285644531, + "epoch": 0.5366591080876795, + "fcm_dpo/beta": 0.005797875113785267, + "fcm_dpo/delta": -0.09473682194948196, + "fcm_dpo/margin": 83.01737213134766, + "fcm_dpo/q_t": 0.38854479789733887, + "grad_norm": 13.447663307189941, + "learning_rate": 2.632160279321328e-07, + "logits/chosen": 0.7900065183639526, + "logits/rejected": 0.6645567417144775, + "logps/chosen": -171.58421325683594, + "logps/ref_chosen": -46.25537872314453, + "logps/ref_rejected": -78.20236206054688, + "logps/rejected": -286.548583984375, + "loss": 1.0509, + "margin_dpo/margin_mean": 83.01736450195312, + "margin_dpo/margin_std": 104.88021850585938, + "step": 355 + }, + { + "KL/chosen_KL_mean": -119.88313293457031, + "KL/mean": -152.29454040527344, + "KL/rejected_KL_mean": -184.70596313476562, + "KL/std": 87.65028381347656, + "epoch": 0.5381708238851096, + "fcm_dpo/beta": 0.005688229110091925, + "fcm_dpo/delta": 0.03451262786984444, + "fcm_dpo/margin": 64.82282257080078, + "fcm_dpo/q_t": 0.41692763566970825, + "grad_norm": 11.188521385192871, + "learning_rate": 2.618954789559356e-07, + "logits/chosen": 0.7430191040039062, + "logits/rejected": 0.6651791334152222, + "logps/chosen": -167.78929138183594, + "logps/ref_chosen": -47.906158447265625, + "logps/ref_rejected": -74.29397583007812, + "logps/rejected": -258.99993896484375, + "loss": 1.1454, + "margin_dpo/margin_mean": 64.82282257080078, + "margin_dpo/margin_std": 109.8090591430664, + "step": 356 + }, + { + "KL/chosen_KL_mean": -143.23452758789062, + "KL/mean": -169.093017578125, + "KL/rejected_KL_mean": -194.9515380859375, + "KL/std": 87.9165267944336, + "epoch": 0.5396825396825397, + "fcm_dpo/beta": 0.005629734601825476, + "fcm_dpo/delta": -0.04662557691335678, + "fcm_dpo/margin": 51.71701431274414, + "fcm_dpo/q_t": 0.4305192530155182, + "grad_norm": 11.694966316223145, + "learning_rate": 2.6057459723762076e-07, + "logits/chosen": 0.752831220626831, + "logits/rejected": 0.7355064153671265, + "logps/chosen": -205.86952209472656, + "logps/ref_chosen": -62.63500213623047, + "logps/ref_rejected": -65.11399841308594, + "logps/rejected": -260.0655517578125, + "loss": 1.1955, + "margin_dpo/margin_mean": 51.717018127441406, + "margin_dpo/margin_std": 97.05239868164062, + "step": 357 + }, + { + "KL/chosen_KL_mean": -142.22438049316406, + "KL/mean": -176.40380859375, + "KL/rejected_KL_mean": -210.58322143554688, + "KL/std": 94.13438415527344, + "epoch": 0.5411942554799698, + "fcm_dpo/beta": 0.00548651535063982, + "fcm_dpo/delta": -0.08703271299600601, + "fcm_dpo/margin": 68.35882568359375, + "fcm_dpo/q_t": 0.41342803835868835, + "grad_norm": 15.619354248046875, + "learning_rate": 2.5925341972508954e-07, + "logits/chosen": 0.6815810203552246, + "logits/rejected": 0.6956534385681152, + "logps/chosen": -209.43399047851562, + "logps/ref_chosen": -67.20960998535156, + "logps/ref_rejected": -69.34715270996094, + "logps/rejected": -279.93035888671875, + "loss": 1.134, + "margin_dpo/margin_mean": 68.35882568359375, + "margin_dpo/margin_std": 105.12598419189453, + "step": 358 + }, + { + "KL/chosen_KL_mean": -158.29672241210938, + "KL/mean": -176.74636840820312, + "KL/rejected_KL_mean": -195.1959991455078, + "KL/std": 92.31513977050781, + "epoch": 0.5427059712773998, + "fcm_dpo/beta": 0.005430396646261215, + "fcm_dpo/delta": 0.05346956476569176, + "fcm_dpo/margin": 36.89925765991211, + "fcm_dpo/q_t": 0.45445123314857483, + "grad_norm": 13.617073059082031, + "learning_rate": 2.579319833745169e-07, + "logits/chosen": 0.7321456074714661, + "logits/rejected": 0.6983045339584351, + "logps/chosen": -220.822509765625, + "logps/ref_chosen": -62.52578353881836, + "logps/ref_rejected": -76.63114929199219, + "logps/rejected": -271.8271484375, + "loss": 1.2704, + "margin_dpo/margin_mean": 36.89925765991211, + "margin_dpo/margin_std": 100.25183868408203, + "step": 359 + }, + { + "KL/chosen_KL_mean": -151.61859130859375, + "KL/mean": -184.96478271484375, + "KL/rejected_KL_mean": -218.31097412109375, + "KL/std": 98.37626647949219, + "epoch": 0.54421768707483, + "fcm_dpo/beta": 0.005576558876782656, + "fcm_dpo/delta": 0.031639836728572845, + "fcm_dpo/margin": 66.6923599243164, + "fcm_dpo/q_t": 0.415699303150177, + "grad_norm": 11.561031341552734, + "learning_rate": 2.5661032514931834e-07, + "logits/chosen": 0.6751697063446045, + "logits/rejected": 0.5904595255851746, + "logps/chosen": -215.1063232421875, + "logps/ref_chosen": -63.48772048950195, + "logps/ref_rejected": -90.6891098022461, + "logps/rejected": -309.00006103515625, + "loss": 1.1178, + "margin_dpo/margin_mean": 66.6923599243164, + "margin_dpo/margin_std": 97.05094909667969, + "step": 360 + }, + { + "KL/chosen_KL_mean": -143.3883056640625, + "KL/mean": -180.1334228515625, + "KL/rejected_KL_mean": -216.8785400390625, + "KL/std": 92.39273071289062, + "epoch": 0.54572940287226, + "fcm_dpo/beta": 0.005615275353193283, + "fcm_dpo/delta": -0.014561126939952374, + "fcm_dpo/margin": 73.49020385742188, + "fcm_dpo/q_t": 0.40417200326919556, + "grad_norm": 10.36706256866455, + "learning_rate": 2.552884820191154e-07, + "logits/chosen": 0.8067169785499573, + "logits/rejected": 0.7659634351730347, + "logps/chosen": -201.30545043945312, + "logps/ref_chosen": -57.917144775390625, + "logps/ref_rejected": -72.39089965820312, + "logps/rejected": -289.2694396972656, + "loss": 1.0856, + "margin_dpo/margin_mean": 73.49020385742188, + "margin_dpo/margin_std": 96.46673583984375, + "step": 361 + }, + { + "KL/chosen_KL_mean": -144.8900146484375, + "KL/mean": -188.41073608398438, + "KL/rejected_KL_mean": -231.93145751953125, + "KL/std": 106.30439758300781, + "epoch": 0.54724111866969, + "fcm_dpo/beta": 0.005439079366624355, + "fcm_dpo/delta": -0.08564796298742294, + "fcm_dpo/margin": 87.04143524169922, + "fcm_dpo/q_t": 0.39451199769973755, + "grad_norm": 11.566519737243652, + "learning_rate": 2.53966490958702e-07, + "logits/chosen": 0.8194698095321655, + "logits/rejected": 0.7212804555892944, + "logps/chosen": -208.33348083496094, + "logps/ref_chosen": -63.4434700012207, + "logps/ref_rejected": -103.45516967773438, + "logps/rejected": -335.3866271972656, + "loss": 1.0693, + "margin_dpo/margin_mean": 87.04142761230469, + "margin_dpo/margin_std": 124.4408187866211, + "step": 362 + }, + { + "KL/chosen_KL_mean": -147.78335571289062, + "KL/mean": -186.64004516601562, + "KL/rejected_KL_mean": -225.4967041015625, + "KL/std": 92.55686950683594, + "epoch": 0.5487528344671202, + "fcm_dpo/beta": 0.005266488064080477, + "fcm_dpo/delta": -0.010586944408714771, + "fcm_dpo/margin": 77.71334075927734, + "fcm_dpo/q_t": 0.40681716799736023, + "grad_norm": 13.70695686340332, + "learning_rate": 2.526443889470099e-07, + "logits/chosen": 0.7966769933700562, + "logits/rejected": 0.6717593669891357, + "logps/chosen": -196.43519592285156, + "logps/ref_chosen": -48.65182876586914, + "logps/ref_rejected": -88.65904235839844, + "logps/rejected": -314.15576171875, + "loss": 1.0935, + "margin_dpo/margin_mean": 77.71334075927734, + "margin_dpo/margin_std": 108.69964599609375, + "step": 363 + }, + { + "KL/chosen_KL_mean": -131.58168029785156, + "KL/mean": -174.65626525878906, + "KL/rejected_KL_mean": -217.7308349609375, + "KL/std": 102.02590942382812, + "epoch": 0.5502645502645502, + "fcm_dpo/beta": 0.0051279449835419655, + "fcm_dpo/delta": -0.04850374162197113, + "fcm_dpo/margin": 86.14913940429688, + "fcm_dpo/q_t": 0.4003210961818695, + "grad_norm": 9.81286334991455, + "learning_rate": 2.513222129660744e-07, + "logits/chosen": 0.6942867040634155, + "logits/rejected": 0.6157143115997314, + "logps/chosen": -189.4527587890625, + "logps/ref_chosen": -57.87107467651367, + "logps/ref_rejected": -80.95503234863281, + "logps/rejected": -298.68585205078125, + "loss": 1.0928, + "margin_dpo/margin_mean": 86.14913940429688, + "margin_dpo/margin_std": 129.07220458984375, + "step": 364 + }, + { + "KL/chosen_KL_mean": -125.02583312988281, + "KL/mean": -164.640380859375, + "KL/rejected_KL_mean": -204.25491333007812, + "KL/std": 108.65450286865234, + "epoch": 0.5517762660619804, + "fcm_dpo/beta": 0.0050535742193460464, + "fcm_dpo/delta": -0.0012763142585754395, + "fcm_dpo/margin": 79.22908782958984, + "fcm_dpo/q_t": 0.4074895977973938, + "grad_norm": 10.674773216247559, + "learning_rate": 2.5e-07, + "logits/chosen": 0.7654601335525513, + "logits/rejected": 0.7636349201202393, + "logps/chosen": -189.968017578125, + "logps/ref_chosen": -64.94217681884766, + "logps/ref_rejected": -74.8599853515625, + "logps/rejected": -279.1148986816406, + "loss": 1.0785, + "margin_dpo/margin_mean": 79.22908782958984, + "margin_dpo/margin_std": 95.32018280029297, + "step": 365 + }, + { + "KL/chosen_KL_mean": -124.21640014648438, + "KL/mean": -156.70164489746094, + "KL/rejected_KL_mean": -189.18690490722656, + "KL/std": 95.22203063964844, + "epoch": 0.5532879818594104, + "fcm_dpo/beta": 0.005297908093780279, + "fcm_dpo/delta": 0.06093678995966911, + "fcm_dpo/margin": 64.97052001953125, + "fcm_dpo/q_t": 0.4231628477573395, + "grad_norm": 13.27842903137207, + "learning_rate": 2.486777870339255e-07, + "logits/chosen": 0.7186771631240845, + "logits/rejected": 0.7142035365104675, + "logps/chosen": -179.38238525390625, + "logps/ref_chosen": -55.16598129272461, + "logps/ref_rejected": -65.26121520996094, + "logps/rejected": -254.4481201171875, + "loss": 1.1684, + "margin_dpo/margin_mean": 64.97052001953125, + "margin_dpo/margin_std": 120.9197769165039, + "step": 366 + }, + { + "KL/chosen_KL_mean": -142.8965606689453, + "KL/mean": -175.44778442382812, + "KL/rejected_KL_mean": -207.9990234375, + "KL/std": 94.457275390625, + "epoch": 0.5547996976568406, + "fcm_dpo/beta": 0.005413766950368881, + "fcm_dpo/delta": 0.05324382334947586, + "fcm_dpo/margin": 65.10247802734375, + "fcm_dpo/q_t": 0.4184330105781555, + "grad_norm": 11.966232299804688, + "learning_rate": 2.4735561105299014e-07, + "logits/chosen": 0.754651665687561, + "logits/rejected": 0.658037543296814, + "logps/chosen": -198.90701293945312, + "logps/ref_chosen": -56.01046371459961, + "logps/ref_rejected": -77.31010437011719, + "logps/rejected": -285.30914306640625, + "loss": 1.1341, + "margin_dpo/margin_mean": 65.10247802734375, + "margin_dpo/margin_std": 99.43478393554688, + "step": 367 + }, + { + "KL/chosen_KL_mean": -154.50941467285156, + "KL/mean": -183.74929809570312, + "KL/rejected_KL_mean": -212.98915100097656, + "KL/std": 93.31791687011719, + "epoch": 0.5563114134542706, + "fcm_dpo/beta": 0.0056096892803907394, + "fcm_dpo/delta": 0.07988474518060684, + "fcm_dpo/margin": 58.47974395751953, + "fcm_dpo/q_t": 0.42577698826789856, + "grad_norm": 13.028302192687988, + "learning_rate": 2.46033509041298e-07, + "logits/chosen": 0.5896161794662476, + "logits/rejected": 0.592335045337677, + "logps/chosen": -229.3386993408203, + "logps/ref_chosen": -74.82927703857422, + "logps/ref_rejected": -76.11680603027344, + "logps/rejected": -289.10595703125, + "loss": 1.1633, + "margin_dpo/margin_mean": 58.47974395751953, + "margin_dpo/margin_std": 100.59825134277344, + "step": 368 + }, + { + "KL/chosen_KL_mean": -141.26634216308594, + "KL/mean": -168.32205200195312, + "KL/rejected_KL_mean": -195.3777618408203, + "KL/std": 93.77578735351562, + "epoch": 0.5578231292517006, + "fcm_dpo/beta": 0.0060209897346794605, + "fcm_dpo/delta": 0.0800437182188034, + "fcm_dpo/margin": 54.111427307128906, + "fcm_dpo/q_t": 0.4274851679801941, + "grad_norm": 13.085291862487793, + "learning_rate": 2.447115179808846e-07, + "logits/chosen": 0.7499760389328003, + "logits/rejected": 0.704288125038147, + "logps/chosen": -199.59255981445312, + "logps/ref_chosen": -58.32621765136719, + "logps/ref_rejected": -80.92183685302734, + "logps/rejected": -276.29962158203125, + "loss": 1.1727, + "margin_dpo/margin_mean": 54.111427307128906, + "margin_dpo/margin_std": 99.9386978149414, + "step": 369 + }, + { + "KL/chosen_KL_mean": -133.82888793945312, + "KL/mean": -173.8616943359375, + "KL/rejected_KL_mean": -213.89450073242188, + "KL/std": 97.72712707519531, + "epoch": 0.5593348450491308, + "fcm_dpo/beta": 0.005765399895608425, + "fcm_dpo/delta": -0.07319603860378265, + "fcm_dpo/margin": 80.06562805175781, + "fcm_dpo/q_t": 0.39234304428100586, + "grad_norm": 12.59056568145752, + "learning_rate": 2.4338967485068164e-07, + "logits/chosen": 0.8812678456306458, + "logits/rejected": 0.8152058124542236, + "logps/chosen": -186.71258544921875, + "logps/ref_chosen": -52.88372039794922, + "logps/ref_rejected": -79.43692016601562, + "logps/rejected": -293.3314208984375, + "loss": 1.0779, + "margin_dpo/margin_mean": 80.06562805175781, + "margin_dpo/margin_std": 113.96977233886719, + "step": 370 + }, + { + "KL/chosen_KL_mean": -131.85989379882812, + "KL/mean": -166.550048828125, + "KL/rejected_KL_mean": -201.240234375, + "KL/std": 91.61387634277344, + "epoch": 0.5608465608465608, + "fcm_dpo/beta": 0.005842489190399647, + "fcm_dpo/delta": -0.008122660219669342, + "fcm_dpo/margin": 69.38032531738281, + "fcm_dpo/q_t": 0.4071889817714691, + "grad_norm": 15.300116539001465, + "learning_rate": 2.420680166254831e-07, + "logits/chosen": 0.8666425347328186, + "logits/rejected": 0.8389255404472351, + "logps/chosen": -181.0841064453125, + "logps/ref_chosen": -49.224212646484375, + "logps/ref_rejected": -63.348472595214844, + "logps/rejected": -264.58868408203125, + "loss": 1.1081, + "margin_dpo/margin_mean": 69.38032531738281, + "margin_dpo/margin_std": 101.50696563720703, + "step": 371 + }, + { + "KL/chosen_KL_mean": -139.83580017089844, + "KL/mean": -160.21075439453125, + "KL/rejected_KL_mean": -180.585693359375, + "KL/std": 92.19517517089844, + "epoch": 0.562358276643991, + "fcm_dpo/beta": 0.005775434896349907, + "fcm_dpo/delta": 0.02449907548725605, + "fcm_dpo/margin": 40.7498779296875, + "fcm_dpo/q_t": 0.448069304227829, + "grad_norm": 13.797256469726562, + "learning_rate": 2.4074658027491044e-07, + "logits/chosen": 0.7692815065383911, + "logits/rejected": 0.6794453859329224, + "logps/chosen": -192.1053466796875, + "logps/ref_chosen": -52.269554138183594, + "logps/ref_rejected": -72.99522399902344, + "logps/rejected": -253.58090209960938, + "loss": 1.2809, + "margin_dpo/margin_mean": 40.749881744384766, + "margin_dpo/margin_std": 119.3675537109375, + "step": 372 + }, + { + "KL/chosen_KL_mean": -158.91616821289062, + "KL/mean": -186.7388916015625, + "KL/rejected_KL_mean": -214.56161499023438, + "KL/std": 99.78253173828125, + "epoch": 0.563869992441421, + "fcm_dpo/beta": 0.006023020949214697, + "fcm_dpo/delta": 0.07183150947093964, + "fcm_dpo/margin": 55.64544677734375, + "fcm_dpo/q_t": 0.42674756050109863, + "grad_norm": 13.5969820022583, + "learning_rate": 2.394254027623792e-07, + "logits/chosen": 0.787378191947937, + "logits/rejected": 0.7218998670578003, + "logps/chosen": -220.0291748046875, + "logps/ref_chosen": -61.112998962402344, + "logps/ref_rejected": -76.24851989746094, + "logps/rejected": -290.81011962890625, + "loss": 1.2072, + "margin_dpo/margin_mean": 55.64544677734375, + "margin_dpo/margin_std": 123.12017822265625, + "step": 373 + }, + { + "KL/chosen_KL_mean": -134.72891235351562, + "KL/mean": -178.64892578125, + "KL/rejected_KL_mean": -222.56893920898438, + "KL/std": 102.30409240722656, + "epoch": 0.5653817082388511, + "fcm_dpo/beta": 0.005733366124331951, + "fcm_dpo/delta": -0.12269183248281479, + "fcm_dpo/margin": 87.84001159667969, + "fcm_dpo/q_t": 0.38412588834762573, + "grad_norm": 14.229789733886719, + "learning_rate": 2.381045210440644e-07, + "logits/chosen": 0.6749398708343506, + "logits/rejected": 0.6806223392486572, + "logps/chosen": -207.3981170654297, + "logps/ref_chosen": -72.66920471191406, + "logps/ref_rejected": -76.83158874511719, + "logps/rejected": -299.4005126953125, + "loss": 1.0639, + "margin_dpo/margin_mean": 87.84001159667969, + "margin_dpo/margin_std": 126.37348175048828, + "step": 374 + }, + { + "KL/chosen_KL_mean": -133.24624633789062, + "KL/mean": -164.95437622070312, + "KL/rejected_KL_mean": -196.66249084472656, + "KL/std": 110.367431640625, + "epoch": 0.5668934240362812, + "fcm_dpo/beta": 0.005722872447222471, + "fcm_dpo/delta": 0.04165318235754967, + "fcm_dpo/margin": 63.41625213623047, + "fcm_dpo/q_t": 0.4203999638557434, + "grad_norm": 15.683096885681152, + "learning_rate": 2.3678397206786715e-07, + "logits/chosen": 0.7528005838394165, + "logits/rejected": 0.7014273405075073, + "logps/chosen": -190.92953491210938, + "logps/ref_chosen": -57.68330383300781, + "logps/ref_rejected": -79.34097290039062, + "logps/rejected": -276.00347900390625, + "loss": 1.1684, + "margin_dpo/margin_mean": 63.4162483215332, + "margin_dpo/margin_std": 121.38080596923828, + "step": 375 + }, + { + "KL/chosen_KL_mean": -141.19493103027344, + "KL/mean": -181.21041870117188, + "KL/rejected_KL_mean": -221.2259521484375, + "KL/std": 112.07803344726562, + "epoch": 0.5684051398337112, + "fcm_dpo/beta": 0.005588100291788578, + "fcm_dpo/delta": -0.05586222559213638, + "fcm_dpo/margin": 80.03101348876953, + "fcm_dpo/q_t": 0.4027999937534332, + "grad_norm": 12.794271469116211, + "learning_rate": 2.3546379277238103e-07, + "logits/chosen": 0.8139768838882446, + "logits/rejected": 0.7517552971839905, + "logps/chosen": -192.86898803710938, + "logps/ref_chosen": -51.674072265625, + "logps/ref_rejected": -75.69713592529297, + "logps/rejected": -296.923095703125, + "loss": 1.112, + "margin_dpo/margin_mean": 80.03101348876953, + "margin_dpo/margin_std": 133.2711639404297, + "step": 376 + }, + { + "KL/chosen_KL_mean": -141.94168090820312, + "KL/mean": -169.37771606445312, + "KL/rejected_KL_mean": -196.8137664794922, + "KL/std": 87.98646545410156, + "epoch": 0.5699168556311414, + "fcm_dpo/beta": 0.005861810874193907, + "fcm_dpo/delta": 0.08542998135089874, + "fcm_dpo/margin": 54.872093200683594, + "fcm_dpo/q_t": 0.42690926790237427, + "grad_norm": 14.541651725769043, + "learning_rate": 2.3414402008585886e-07, + "logits/chosen": 0.7738527059555054, + "logits/rejected": 0.7508142590522766, + "logps/chosen": -188.12020874023438, + "logps/ref_chosen": -46.17853546142578, + "logps/ref_rejected": -57.756500244140625, + "logps/rejected": -254.5702667236328, + "loss": 1.1706, + "margin_dpo/margin_mean": 54.872093200683594, + "margin_dpo/margin_std": 98.96908569335938, + "step": 377 + }, + { + "KL/chosen_KL_mean": -139.3751678466797, + "KL/mean": -166.8651123046875, + "KL/rejected_KL_mean": -194.35507202148438, + "KL/std": 103.14115142822266, + "epoch": 0.5714285714285714, + "fcm_dpo/beta": 0.006156137678772211, + "fcm_dpo/delta": 0.06586823612451553, + "fcm_dpo/margin": 54.979896545410156, + "fcm_dpo/q_t": 0.4245694875717163, + "grad_norm": 11.343628883361816, + "learning_rate": 2.3282469092517977e-07, + "logits/chosen": 0.8040895462036133, + "logits/rejected": 0.7606953382492065, + "logps/chosen": -198.59405517578125, + "logps/ref_chosen": -59.21887969970703, + "logps/ref_rejected": -71.24818420410156, + "logps/rejected": -265.603271484375, + "loss": 1.1734, + "margin_dpo/margin_mean": 54.97990036010742, + "margin_dpo/margin_std": 101.69361877441406, + "step": 378 + }, + { + "KL/chosen_KL_mean": -141.87744140625, + "KL/mean": -181.7213897705078, + "KL/rejected_KL_mean": -221.56533813476562, + "KL/std": 107.65896606445312, + "epoch": 0.5729402872260015, + "fcm_dpo/beta": 0.005909925326704979, + "fcm_dpo/delta": -0.08261381089687347, + "fcm_dpo/margin": 79.68791961669922, + "fcm_dpo/q_t": 0.39472633600234985, + "grad_norm": 14.828629493713379, + "learning_rate": 2.3150584219481643e-07, + "logits/chosen": 0.7641682624816895, + "logits/rejected": 0.6950868368148804, + "logps/chosen": -218.19403076171875, + "logps/ref_chosen": -76.31658935546875, + "logps/ref_rejected": -104.26200103759766, + "logps/rejected": -325.82733154296875, + "loss": 1.0766, + "margin_dpo/margin_mean": 79.68791961669922, + "margin_dpo/margin_std": 117.0067138671875, + "step": 379 + }, + { + "KL/chosen_KL_mean": -118.77130889892578, + "KL/mean": -161.55316162109375, + "KL/rejected_KL_mean": -204.33502197265625, + "KL/std": 95.21235656738281, + "epoch": 0.5744520030234316, + "fcm_dpo/beta": 0.005602892488241196, + "fcm_dpo/delta": -0.09268143028020859, + "fcm_dpo/margin": 85.5636978149414, + "fcm_dpo/q_t": 0.388763964176178, + "grad_norm": 13.451189041137695, + "learning_rate": 2.3018751078580283e-07, + "logits/chosen": 0.7501556873321533, + "logits/rejected": 0.7167741060256958, + "logps/chosen": -180.05447387695312, + "logps/ref_chosen": -61.283164978027344, + "logps/ref_rejected": -72.38892364501953, + "logps/rejected": -276.72393798828125, + "loss": 1.0568, + "margin_dpo/margin_mean": 85.5636978149414, + "margin_dpo/margin_std": 113.08406066894531, + "step": 380 + }, + { + "KL/chosen_KL_mean": -145.89193725585938, + "KL/mean": -162.467041015625, + "KL/rejected_KL_mean": -179.04214477539062, + "KL/std": 100.26214599609375, + "epoch": 0.5759637188208617, + "fcm_dpo/beta": 0.005632633343338966, + "fcm_dpo/delta": 0.06954426318407059, + "fcm_dpo/margin": 33.150203704833984, + "fcm_dpo/q_t": 0.45884740352630615, + "grad_norm": 11.799565315246582, + "learning_rate": 2.288697335747027e-07, + "logits/chosen": 0.725066065788269, + "logits/rejected": 0.7106317281723022, + "logps/chosen": -204.10592651367188, + "logps/ref_chosen": -58.2139892578125, + "logps/ref_rejected": -60.78669357299805, + "logps/rejected": -239.82882690429688, + "loss": 1.3176, + "margin_dpo/margin_mean": 33.15020751953125, + "margin_dpo/margin_std": 121.16805267333984, + "step": 381 + }, + { + "KL/chosen_KL_mean": -146.5373077392578, + "KL/mean": -177.61614990234375, + "KL/rejected_KL_mean": -208.69497680664062, + "KL/std": 100.65556335449219, + "epoch": 0.5774754346182918, + "fcm_dpo/beta": 0.005918251350522041, + "fcm_dpo/delta": 0.0343366302549839, + "fcm_dpo/margin": 62.157691955566406, + "fcm_dpo/q_t": 0.4175347685813904, + "grad_norm": 14.061190605163574, + "learning_rate": 2.2755254742257706e-07, + "logits/chosen": 0.7362748384475708, + "logits/rejected": 0.6886606216430664, + "logps/chosen": -208.3626251220703, + "logps/ref_chosen": -61.82532501220703, + "logps/ref_rejected": -83.0452880859375, + "logps/rejected": -291.7402648925781, + "loss": 1.1333, + "margin_dpo/margin_mean": 62.157691955566406, + "margin_dpo/margin_std": 98.62821960449219, + "step": 382 + }, + { + "KL/chosen_KL_mean": -151.32742309570312, + "KL/mean": -177.73983764648438, + "KL/rejected_KL_mean": -204.15228271484375, + "KL/std": 99.92002868652344, + "epoch": 0.5789871504157218, + "fcm_dpo/beta": 0.006043318659067154, + "fcm_dpo/delta": 0.08946429938077927, + "fcm_dpo/margin": 52.82486343383789, + "fcm_dpo/q_t": 0.43128010630607605, + "grad_norm": 16.85250473022461, + "learning_rate": 2.2623598917395436e-07, + "logits/chosen": 0.6481820344924927, + "logits/rejected": 0.6832894086837769, + "logps/chosen": -231.89068603515625, + "logps/ref_chosen": -80.56326293945312, + "logps/ref_rejected": -74.62922668457031, + "logps/rejected": -278.781494140625, + "loss": 1.2263, + "margin_dpo/margin_mean": 52.824859619140625, + "margin_dpo/margin_std": 125.01660919189453, + "step": 383 + }, + { + "KL/chosen_KL_mean": -137.10409545898438, + "KL/mean": -170.72140502929688, + "KL/rejected_KL_mean": -204.33868408203125, + "KL/std": 95.64372253417969, + "epoch": 0.5804988662131519, + "fcm_dpo/beta": 0.006174423731863499, + "fcm_dpo/delta": -0.01735561341047287, + "fcm_dpo/margin": 67.234619140625, + "fcm_dpo/q_t": 0.403439462184906, + "grad_norm": 15.312804222106934, + "learning_rate": 2.2492009565579875e-07, + "logits/chosen": 0.7630481123924255, + "logits/rejected": 0.7314999103546143, + "logps/chosen": -202.57925415039062, + "logps/ref_chosen": -65.47514343261719, + "logps/ref_rejected": -79.67378234863281, + "logps/rejected": -284.0124816894531, + "loss": 1.1058, + "margin_dpo/margin_mean": 67.234619140625, + "margin_dpo/margin_std": 101.94595336914062, + "step": 384 + }, + { + "KL/chosen_KL_mean": -140.1160888671875, + "KL/mean": -182.65090942382812, + "KL/rejected_KL_mean": -225.18572998046875, + "KL/std": 100.97268676757812, + "epoch": 0.582010582010582, + "fcm_dpo/beta": 0.005884962156414986, + "fcm_dpo/delta": -0.11797355115413666, + "fcm_dpo/margin": 85.06967163085938, + "fcm_dpo/q_t": 0.3843558132648468, + "grad_norm": 11.957921028137207, + "learning_rate": 2.2360490367648084e-07, + "logits/chosen": 0.701429009437561, + "logits/rejected": 0.6622118949890137, + "logps/chosen": -206.17259216308594, + "logps/ref_chosen": -66.0565185546875, + "logps/ref_rejected": -86.68023681640625, + "logps/rejected": -311.865966796875, + "loss": 1.0349, + "margin_dpo/margin_mean": 85.06966400146484, + "margin_dpo/margin_std": 103.32591247558594, + "step": 385 + }, + { + "KL/chosen_KL_mean": -161.50335693359375, + "KL/mean": -189.95013427734375, + "KL/rejected_KL_mean": -218.39691162109375, + "KL/std": 107.01244354248047, + "epoch": 0.5835222978080121, + "fcm_dpo/beta": 0.005871026776731014, + "fcm_dpo/delta": 0.07351464033126831, + "fcm_dpo/margin": 56.8935661315918, + "fcm_dpo/q_t": 0.42427319288253784, + "grad_norm": 13.5010347366333, + "learning_rate": 2.2229045002474724e-07, + "logits/chosen": 0.6531039476394653, + "logits/rejected": 0.6064084768295288, + "logps/chosen": -237.12701416015625, + "logps/ref_chosen": -75.6236572265625, + "logps/ref_rejected": -92.62330627441406, + "logps/rejected": -311.02020263671875, + "loss": 1.1754, + "margin_dpo/margin_mean": 56.89356994628906, + "margin_dpo/margin_std": 107.92727661132812, + "step": 386 + }, + { + "KL/chosen_KL_mean": -141.63909912109375, + "KL/mean": -181.66258239746094, + "KL/rejected_KL_mean": -221.68606567382812, + "KL/std": 96.4151382446289, + "epoch": 0.5850340136054422, + "fcm_dpo/beta": 0.005757711827754974, + "fcm_dpo/delta": -0.07114600390195847, + "fcm_dpo/margin": 80.04695129394531, + "fcm_dpo/q_t": 0.3940368890762329, + "grad_norm": 13.86131763458252, + "learning_rate": 2.209767714686924e-07, + "logits/chosen": 0.7648110389709473, + "logits/rejected": 0.6623188257217407, + "logps/chosen": -188.86080932617188, + "logps/ref_chosen": -47.22170639038086, + "logps/ref_rejected": -87.338134765625, + "logps/rejected": -309.0242004394531, + "loss": 1.0566, + "margin_dpo/margin_mean": 80.04695129394531, + "margin_dpo/margin_std": 101.74493408203125, + "step": 387 + }, + { + "KL/chosen_KL_mean": -148.16046142578125, + "KL/mean": -172.68748474121094, + "KL/rejected_KL_mean": -197.21450805664062, + "KL/std": 103.10355377197266, + "epoch": 0.5865457294028723, + "fcm_dpo/beta": 0.0057308124378323555, + "fcm_dpo/delta": 0.003550410969182849, + "fcm_dpo/margin": 49.054039001464844, + "fcm_dpo/q_t": 0.4384087324142456, + "grad_norm": 12.839620590209961, + "learning_rate": 2.1966390475472954e-07, + "logits/chosen": 0.7350201606750488, + "logits/rejected": 0.7327404022216797, + "logps/chosen": -222.73992919921875, + "logps/ref_chosen": -74.5794677734375, + "logps/ref_rejected": -79.92558288574219, + "logps/rejected": -277.14007568359375, + "loss": 1.2403, + "margin_dpo/margin_mean": 49.054039001464844, + "margin_dpo/margin_std": 121.98270416259766, + "step": 388 + }, + { + "KL/chosen_KL_mean": -138.47047424316406, + "KL/mean": -178.14364624023438, + "KL/rejected_KL_mean": -217.81683349609375, + "KL/std": 97.10962677001953, + "epoch": 0.5880574452003023, + "fcm_dpo/beta": 0.005619604140520096, + "fcm_dpo/delta": -0.05288499593734741, + "fcm_dpo/margin": 79.34635925292969, + "fcm_dpo/q_t": 0.39804351329803467, + "grad_norm": 21.324504852294922, + "learning_rate": 2.1835188660656265e-07, + "logits/chosen": 0.7512508630752563, + "logits/rejected": 0.7208691835403442, + "logps/chosen": -200.0948486328125, + "logps/ref_chosen": -61.624366760253906, + "logps/ref_rejected": -76.50978088378906, + "logps/rejected": -294.32659912109375, + "loss": 1.0743, + "margin_dpo/margin_mean": 79.34635925292969, + "margin_dpo/margin_std": 107.70132446289062, + "step": 389 + }, + { + "KL/chosen_KL_mean": -123.1337661743164, + "KL/mean": -153.21640014648438, + "KL/rejected_KL_mean": -183.29904174804688, + "KL/std": 92.85203552246094, + "epoch": 0.5895691609977324, + "fcm_dpo/beta": 0.00567449489608407, + "fcm_dpo/delta": 0.06538356095552444, + "fcm_dpo/margin": 60.165260314941406, + "fcm_dpo/q_t": 0.4235107898712158, + "grad_norm": 11.731263160705566, + "learning_rate": 2.170407537241599e-07, + "logits/chosen": 0.8297513723373413, + "logits/rejected": 0.7623568773269653, + "logps/chosen": -169.00563049316406, + "logps/ref_chosen": -45.871864318847656, + "logps/ref_rejected": -61.305999755859375, + "logps/rejected": -244.60504150390625, + "loss": 1.15, + "margin_dpo/margin_mean": 60.16526412963867, + "margin_dpo/margin_std": 99.79165649414062, + "step": 390 + }, + { + "KL/chosen_KL_mean": -138.01197814941406, + "KL/mean": -175.88485717773438, + "KL/rejected_KL_mean": -213.7576904296875, + "KL/std": 97.07452392578125, + "epoch": 0.5910808767951625, + "fcm_dpo/beta": 0.005652905907481909, + "fcm_dpo/delta": -0.03288257122039795, + "fcm_dpo/margin": 75.74574279785156, + "fcm_dpo/q_t": 0.40263625979423523, + "grad_norm": 12.69221305847168, + "learning_rate": 2.1573054278272636e-07, + "logits/chosen": 0.7633911967277527, + "logits/rejected": 0.7019753456115723, + "logps/chosen": -196.198974609375, + "logps/ref_chosen": -58.18701171875, + "logps/ref_rejected": -83.63442993164062, + "logps/rejected": -297.39215087890625, + "loss": 1.1052, + "margin_dpo/margin_mean": 75.74574279785156, + "margin_dpo/margin_std": 116.3956298828125, + "step": 391 + }, + { + "KL/chosen_KL_mean": -127.93291473388672, + "KL/mean": -166.52969360351562, + "KL/rejected_KL_mean": -205.12648010253906, + "KL/std": 105.89525604248047, + "epoch": 0.5925925925925926, + "fcm_dpo/beta": 0.005653767846524715, + "fcm_dpo/delta": -0.043347496539354324, + "fcm_dpo/margin": 77.19355773925781, + "fcm_dpo/q_t": 0.4022483229637146, + "grad_norm": 11.86667537689209, + "learning_rate": 2.1442129043167873e-07, + "logits/chosen": 0.7835770845413208, + "logits/rejected": 0.7315959930419922, + "logps/chosen": -197.6774444580078, + "logps/ref_chosen": -69.7445297241211, + "logps/ref_rejected": -94.05877685546875, + "logps/rejected": -299.18524169921875, + "loss": 1.1083, + "margin_dpo/margin_mean": 77.19355773925781, + "margin_dpo/margin_std": 123.01547241210938, + "step": 392 + }, + { + "KL/chosen_KL_mean": -142.97479248046875, + "KL/mean": -184.1305694580078, + "KL/rejected_KL_mean": -225.28634643554688, + "KL/std": 102.22776794433594, + "epoch": 0.5941043083900227, + "fcm_dpo/beta": 0.005305946804583073, + "fcm_dpo/delta": -0.04566541686654091, + "fcm_dpo/margin": 82.31155395507812, + "fcm_dpo/q_t": 0.3981964588165283, + "grad_norm": 10.274748802185059, + "learning_rate": 2.131130332936195e-07, + "logits/chosen": 0.7935898303985596, + "logits/rejected": 0.7587308883666992, + "logps/chosen": -195.3096923828125, + "logps/ref_chosen": -52.33489990234375, + "logps/ref_rejected": -74.33809661865234, + "logps/rejected": -299.62445068359375, + "loss": 1.0668, + "margin_dpo/margin_mean": 82.3115463256836, + "margin_dpo/margin_std": 100.75532531738281, + "step": 393 + }, + { + "KL/chosen_KL_mean": -136.1352996826172, + "KL/mean": -171.53225708007812, + "KL/rejected_KL_mean": -206.92921447753906, + "KL/std": 95.84933471679688, + "epoch": 0.5956160241874527, + "fcm_dpo/beta": 0.005464646033942699, + "fcm_dpo/delta": 0.013672813773155212, + "fcm_dpo/margin": 70.79390716552734, + "fcm_dpo/q_t": 0.41006964445114136, + "grad_norm": 12.026927947998047, + "learning_rate": 2.1180580796331323e-07, + "logits/chosen": 0.7781930565834045, + "logits/rejected": 0.751598596572876, + "logps/chosen": -196.81143188476562, + "logps/ref_chosen": -60.6761360168457, + "logps/ref_rejected": -71.36074829101562, + "logps/rejected": -278.28997802734375, + "loss": 1.1026, + "margin_dpo/margin_mean": 70.79390716552734, + "margin_dpo/margin_std": 94.3880615234375, + "step": 394 + }, + { + "KL/chosen_KL_mean": -137.5908660888672, + "KL/mean": -171.83412170410156, + "KL/rejected_KL_mean": -206.07737731933594, + "KL/std": 98.70809173583984, + "epoch": 0.5971277399848829, + "fcm_dpo/beta": 0.005507326684892178, + "fcm_dpo/delta": 0.02526915818452835, + "fcm_dpo/margin": 68.48651885986328, + "fcm_dpo/q_t": 0.41558602452278137, + "grad_norm": 11.91160774230957, + "learning_rate": 2.104996510066625e-07, + "logits/chosen": 0.8012561798095703, + "logits/rejected": 0.7142231464385986, + "logps/chosen": -188.1951904296875, + "logps/ref_chosen": -50.60432434082031, + "logps/ref_rejected": -77.08731079101562, + "logps/rejected": -283.1646728515625, + "loss": 1.1236, + "margin_dpo/margin_mean": 68.48651885986328, + "margin_dpo/margin_std": 106.47164916992188, + "step": 395 + }, + { + "KL/chosen_KL_mean": -133.67884826660156, + "KL/mean": -168.15289306640625, + "KL/rejected_KL_mean": -202.62692260742188, + "KL/std": 106.60101318359375, + "epoch": 0.5986394557823129, + "fcm_dpo/beta": 0.0052660866640508175, + "fcm_dpo/delta": -0.07049454748630524, + "fcm_dpo/margin": 68.94806671142578, + "fcm_dpo/q_t": 0.4145070016384125, + "grad_norm": 10.632168769836426, + "learning_rate": 2.0919459895968517e-07, + "logits/chosen": 0.7917169332504272, + "logits/rejected": 0.7078319787979126, + "logps/chosen": -185.0384521484375, + "logps/ref_chosen": -51.35961151123047, + "logps/ref_rejected": -79.89360046386719, + "logps/rejected": -282.5205078125, + "loss": 1.1199, + "margin_dpo/margin_mean": 68.94806671142578, + "margin_dpo/margin_std": 95.077392578125, + "step": 396 + }, + { + "KL/chosen_KL_mean": -154.8973388671875, + "KL/mean": -174.40435791015625, + "KL/rejected_KL_mean": -193.911376953125, + "KL/std": 99.12435150146484, + "epoch": 0.600151171579743, + "fcm_dpo/beta": 0.0052660866640508175, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 39.0140266418457, + "fcm_dpo/q_t": 0.45285657048225403, + "grad_norm": 11.291228294372559, + "learning_rate": 2.078906883274924e-07, + "logits/chosen": 0.7051912546157837, + "logits/rejected": 0.6572903990745544, + "logps/chosen": -221.35357666015625, + "logps/ref_chosen": -66.45622253417969, + "logps/ref_rejected": -85.74736785888672, + "logps/rejected": -279.65875244140625, + "loss": 1.2758, + "margin_dpo/margin_mean": 39.01403045654297, + "margin_dpo/margin_std": 111.57695770263672, + "step": 397 + }, + { + "KL/chosen_KL_mean": -129.05348205566406, + "KL/mean": -174.99749755859375, + "KL/rejected_KL_mean": -220.94149780273438, + "KL/std": 103.12016296386719, + "epoch": 0.6016628873771731, + "fcm_dpo/beta": 0.0050222016870975494, + "fcm_dpo/delta": -0.0740644708275795, + "fcm_dpo/margin": 91.88800048828125, + "fcm_dpo/q_t": 0.39427825808525085, + "grad_norm": 9.868289947509766, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": 0.7551954984664917, + "logits/rejected": 0.6983195543289185, + "logps/chosen": -178.2977294921875, + "logps/ref_chosen": -49.244239807128906, + "logps/ref_rejected": -75.18949127197266, + "logps/rejected": -296.1309814453125, + "loss": 1.0491, + "margin_dpo/margin_mean": 91.88800048828125, + "margin_dpo/margin_std": 112.2702865600586, + "step": 398 + }, + { + "KL/chosen_KL_mean": -154.25454711914062, + "KL/mean": -207.419921875, + "KL/rejected_KL_mean": -260.58526611328125, + "KL/std": 118.88225555419922, + "epoch": 0.6031746031746031, + "fcm_dpo/beta": 0.004701526835560799, + "fcm_dpo/delta": -0.12128760665655136, + "fcm_dpo/margin": 106.33074188232422, + "fcm_dpo/q_t": 0.3860124349594116, + "grad_norm": 10.134847640991211, + "learning_rate": 2.052864371672457e-07, + "logits/chosen": 0.7207819223403931, + "logits/rejected": 0.591735303401947, + "logps/chosen": -222.56134033203125, + "logps/ref_chosen": -68.30679321289062, + "logps/ref_rejected": -113.2708511352539, + "logps/rejected": -373.85614013671875, + "loss": 1.0369, + "margin_dpo/margin_mean": 106.33074188232422, + "margin_dpo/margin_std": 135.85537719726562, + "step": 399 + }, + { + "KL/chosen_KL_mean": -171.22833251953125, + "KL/mean": -200.75521850585938, + "KL/rejected_KL_mean": -230.28211975097656, + "KL/std": 106.50505065917969, + "epoch": 0.6046863189720333, + "fcm_dpo/beta": 0.004656361881643534, + "fcm_dpo/delta": -0.00973660871386528, + "fcm_dpo/margin": 59.05378341674805, + "fcm_dpo/q_t": 0.43434858322143555, + "grad_norm": 13.978921890258789, + "learning_rate": 2.0398616948569493e-07, + "logits/chosen": 0.7625016570091248, + "logits/rejected": 0.7079190611839294, + "logps/chosen": -242.85482788085938, + "logps/ref_chosen": -71.62649536132812, + "logps/ref_rejected": -90.98765563964844, + "logps/rejected": -321.269775390625, + "loss": 1.188, + "margin_dpo/margin_mean": 59.05378341674805, + "margin_dpo/margin_std": 100.4428939819336, + "step": 400 + }, + { + "KL/chosen_KL_mean": -128.79498291015625, + "KL/mean": -171.2772674560547, + "KL/rejected_KL_mean": -213.75953674316406, + "KL/std": 112.90093994140625, + "epoch": 0.6061980347694633, + "fcm_dpo/beta": 0.004625148139894009, + "fcm_dpo/delta": 0.007325233891606331, + "fcm_dpo/margin": 84.96454620361328, + "fcm_dpo/q_t": 0.4098255932331085, + "grad_norm": 7.507087230682373, + "learning_rate": 2.0268718890989752e-07, + "logits/chosen": 0.8112884759902954, + "logits/rejected": 0.7324414253234863, + "logps/chosen": -182.51995849609375, + "logps/ref_chosen": -53.72495651245117, + "logps/ref_rejected": -75.06304931640625, + "logps/rejected": -288.82257080078125, + "loss": 1.0941, + "margin_dpo/margin_mean": 84.96454620361328, + "margin_dpo/margin_std": 110.92296600341797, + "step": 401 + }, + { + "KL/chosen_KL_mean": -143.20652770996094, + "KL/mean": -174.81996154785156, + "KL/rejected_KL_mean": -206.43341064453125, + "KL/std": 94.28620910644531, + "epoch": 0.6077097505668935, + "fcm_dpo/beta": 0.004691399168223143, + "fcm_dpo/delta": 0.005168864969164133, + "fcm_dpo/margin": 63.226871490478516, + "fcm_dpo/q_t": 0.43117454648017883, + "grad_norm": 11.448514938354492, + "learning_rate": 2.013895317751323e-07, + "logits/chosen": 0.7634487152099609, + "logits/rejected": 0.7546102404594421, + "logps/chosen": -205.0804443359375, + "logps/ref_chosen": -61.873931884765625, + "logps/ref_rejected": -66.15198516845703, + "logps/rejected": -272.58538818359375, + "loss": 1.1843, + "margin_dpo/margin_mean": 63.22686767578125, + "margin_dpo/margin_std": 115.47822570800781, + "step": 402 + }, + { + "KL/chosen_KL_mean": -149.43191528320312, + "KL/mean": -191.5021209716797, + "KL/rejected_KL_mean": -233.57232666015625, + "KL/std": 111.39325714111328, + "epoch": 0.6092214663643235, + "fcm_dpo/beta": 0.004674067720770836, + "fcm_dpo/delta": 0.007327292114496231, + "fcm_dpo/margin": 84.14042663574219, + "fcm_dpo/q_t": 0.4105861485004425, + "grad_norm": 8.619983673095703, + "learning_rate": 2.0009323437965898e-07, + "logits/chosen": 0.8772906064987183, + "logits/rejected": 0.7967815399169922, + "logps/chosen": -200.75341796875, + "logps/ref_chosen": -51.321502685546875, + "logps/ref_rejected": -86.54010772705078, + "logps/rejected": -320.1124267578125, + "loss": 1.1015, + "margin_dpo/margin_mean": 84.14042663574219, + "margin_dpo/margin_std": 116.0869140625, + "step": 403 + }, + { + "KL/chosen_KL_mean": -143.56373596191406, + "KL/mean": -187.47364807128906, + "KL/rejected_KL_mean": -231.38356018066406, + "KL/std": 115.00575256347656, + "epoch": 0.6107331821617535, + "fcm_dpo/beta": 0.0046137175522744656, + "fcm_dpo/delta": -0.00824352353811264, + "fcm_dpo/margin": 87.81982421875, + "fcm_dpo/q_t": 0.40820419788360596, + "grad_norm": 12.866887092590332, + "learning_rate": 1.9879833298370237e-07, + "logits/chosen": 0.7568644285202026, + "logits/rejected": 0.6709632873535156, + "logps/chosen": -205.82662963867188, + "logps/ref_chosen": -62.26288604736328, + "logps/ref_rejected": -95.19029998779297, + "logps/rejected": -326.5738525390625, + "loss": 1.1079, + "margin_dpo/margin_mean": 87.81982421875, + "margin_dpo/margin_std": 128.9308624267578, + "step": 404 + }, + { + "KL/chosen_KL_mean": -144.8743896484375, + "KL/mean": -178.5138397216797, + "KL/rejected_KL_mean": -212.15328979492188, + "KL/std": 99.92108154296875, + "epoch": 0.6122448979591837, + "fcm_dpo/beta": 0.004627191927284002, + "fcm_dpo/delta": -0.022051792591810226, + "fcm_dpo/margin": 67.27889251708984, + "fcm_dpo/q_t": 0.4273110032081604, + "grad_norm": 9.795719146728516, + "learning_rate": 1.975048638084379e-07, + "logits/chosen": 0.8541995286941528, + "logits/rejected": 0.8144976496696472, + "logps/chosen": -195.458740234375, + "logps/ref_chosen": -50.5843391418457, + "logps/ref_rejected": -65.43156433105469, + "logps/rejected": -277.5848388671875, + "loss": 1.1557, + "margin_dpo/margin_mean": 67.27889251708984, + "margin_dpo/margin_std": 100.67938232421875, + "step": 405 + }, + { + "KL/chosen_KL_mean": -136.11376953125, + "KL/mean": -183.8330078125, + "KL/rejected_KL_mean": -231.55224609375, + "KL/std": 109.36830139160156, + "epoch": 0.6137566137566137, + "fcm_dpo/beta": 0.004584764130413532, + "fcm_dpo/delta": -0.04349678382277489, + "fcm_dpo/margin": 95.4384536743164, + "fcm_dpo/q_t": 0.398017942905426, + "grad_norm": 9.676344871520996, + "learning_rate": 1.9621286303497914e-07, + "logits/chosen": 0.8280668258666992, + "logits/rejected": 0.685680627822876, + "logps/chosen": -185.109375, + "logps/ref_chosen": -48.99560546875, + "logps/ref_rejected": -92.47774505615234, + "logps/rejected": -324.02996826171875, + "loss": 1.0703, + "margin_dpo/margin_mean": 95.43846130371094, + "margin_dpo/margin_std": 120.8671875, + "step": 406 + }, + { + "KL/chosen_KL_mean": -180.22271728515625, + "KL/mean": -216.6717529296875, + "KL/rejected_KL_mean": -253.1208038330078, + "KL/std": 127.8733139038086, + "epoch": 0.6152683295540439, + "fcm_dpo/beta": 0.004674847237765789, + "fcm_dpo/delta": 0.06505510210990906, + "fcm_dpo/margin": 72.8980941772461, + "fcm_dpo/q_t": 0.4223581552505493, + "grad_norm": 11.925191879272461, + "learning_rate": 1.9492236680336483e-07, + "logits/chosen": 0.6953166723251343, + "logits/rejected": 0.6430000066757202, + "logps/chosen": -269.623291015625, + "logps/ref_chosen": -89.40056610107422, + "logps/ref_rejected": -99.28775024414062, + "logps/rejected": -352.4085693359375, + "loss": 1.1603, + "margin_dpo/margin_mean": 72.89810180664062, + "margin_dpo/margin_std": 128.1045684814453, + "step": 407 + }, + { + "KL/chosen_KL_mean": -133.22731018066406, + "KL/mean": -183.61326599121094, + "KL/rejected_KL_mean": -233.99923706054688, + "KL/std": 101.46388244628906, + "epoch": 0.6167800453514739, + "fcm_dpo/beta": 0.004601156804710627, + "fcm_dpo/delta": -0.07428466528654099, + "fcm_dpo/margin": 100.77191162109375, + "fcm_dpo/q_t": 0.3899993896484375, + "grad_norm": 9.173323631286621, + "learning_rate": 1.9363341121154895e-07, + "logits/chosen": 0.7866462469100952, + "logits/rejected": 0.7229700088500977, + "logps/chosen": -187.9312286376953, + "logps/ref_chosen": -54.70391845703125, + "logps/ref_rejected": -73.98648834228516, + "logps/rejected": -307.9857177734375, + "loss": 1.0333, + "margin_dpo/margin_mean": 100.77191925048828, + "margin_dpo/margin_std": 104.93372344970703, + "step": 408 + }, + { + "KL/chosen_KL_mean": -165.34664916992188, + "KL/mean": -187.67367553710938, + "KL/rejected_KL_mean": -210.000732421875, + "KL/std": 87.25897216796875, + "epoch": 0.618291761148904, + "fcm_dpo/beta": 0.004743535537272692, + "fcm_dpo/delta": 0.09411386400461197, + "fcm_dpo/margin": 44.654083251953125, + "fcm_dpo/q_t": 0.45126640796661377, + "grad_norm": 11.334348678588867, + "learning_rate": 1.9234603231438994e-07, + "logits/chosen": 0.798064649105072, + "logits/rejected": 0.807654857635498, + "logps/chosen": -227.46487426757812, + "logps/ref_chosen": -62.11822509765625, + "logps/ref_rejected": -61.933509826660156, + "logps/rejected": -271.9342346191406, + "loss": 1.2371, + "margin_dpo/margin_mean": 44.654083251953125, + "margin_dpo/margin_std": 96.73648071289062, + "step": 409 + }, + { + "KL/chosen_KL_mean": -156.24102783203125, + "KL/mean": -196.73635864257812, + "KL/rejected_KL_mean": -237.231689453125, + "KL/std": 102.72321319580078, + "epoch": 0.6198034769463341, + "fcm_dpo/beta": 0.0046883877366781235, + "fcm_dpo/delta": 0.020214948803186417, + "fcm_dpo/margin": 80.99066162109375, + "fcm_dpo/q_t": 0.4119493365287781, + "grad_norm": 10.344640731811523, + "learning_rate": 1.9106026612264315e-07, + "logits/chosen": 0.7926585674285889, + "logits/rejected": 0.7750309705734253, + "logps/chosen": -218.043701171875, + "logps/ref_chosen": -61.80266189575195, + "logps/ref_rejected": -76.60002136230469, + "logps/rejected": -313.83172607421875, + "loss": 1.1039, + "margin_dpo/margin_mean": 80.99066162109375, + "margin_dpo/margin_std": 103.93037414550781, + "step": 410 + }, + { + "KL/chosen_KL_mean": -162.68948364257812, + "KL/mean": -203.23043823242188, + "KL/rejected_KL_mean": -243.77142333984375, + "KL/std": 108.2699966430664, + "epoch": 0.6213151927437641, + "fcm_dpo/beta": 0.004831541329622269, + "fcm_dpo/delta": 0.009294688701629639, + "fcm_dpo/margin": 81.08193969726562, + "fcm_dpo/q_t": 0.4101915955543518, + "grad_norm": 9.14470386505127, + "learning_rate": 1.8977614860195296e-07, + "logits/chosen": 0.8241767287254333, + "logits/rejected": 0.779667317867279, + "logps/chosen": -217.13487243652344, + "logps/ref_chosen": -54.44539260864258, + "logps/ref_rejected": -74.5650863647461, + "logps/rejected": -318.3365173339844, + "loss": 1.1031, + "margin_dpo/margin_mean": 81.08193969726562, + "margin_dpo/margin_std": 114.11431884765625, + "step": 411 + }, + { + "KL/chosen_KL_mean": -166.61082458496094, + "KL/mean": -202.292236328125, + "KL/rejected_KL_mean": -237.9736328125, + "KL/std": 91.02214813232422, + "epoch": 0.6228269085411943, + "fcm_dpo/beta": 0.004908284172415733, + "fcm_dpo/delta": 0.05554809793829918, + "fcm_dpo/margin": 71.36279296875, + "fcm_dpo/q_t": 0.41911643743515015, + "grad_norm": 10.918268203735352, + "learning_rate": 1.8849371567184662e-07, + "logits/chosen": 0.8318126797676086, + "logits/rejected": 0.7821172475814819, + "logps/chosen": -221.85891723632812, + "logps/ref_chosen": -55.248085021972656, + "logps/ref_rejected": -68.96623229980469, + "logps/rejected": -306.9398498535156, + "loss": 1.1214, + "margin_dpo/margin_mean": 71.36279296875, + "margin_dpo/margin_std": 96.66009521484375, + "step": 412 + }, + { + "KL/chosen_KL_mean": -187.35760498046875, + "KL/mean": -221.8975830078125, + "KL/rejected_KL_mean": -256.4375305175781, + "KL/std": 111.54348754882812, + "epoch": 0.6243386243386243, + "fcm_dpo/beta": 0.00513922143727541, + "fcm_dpo/delta": 0.04962531104683876, + "fcm_dpo/margin": 69.0799560546875, + "fcm_dpo/q_t": 0.42187249660491943, + "grad_norm": 13.688892364501953, + "learning_rate": 1.872130032047302e-07, + "logits/chosen": 0.6731942296028137, + "logits/rejected": 0.6520799398422241, + "logps/chosen": -256.0783386230469, + "logps/ref_chosen": -68.72074890136719, + "logps/ref_rejected": -78.76539611816406, + "logps/rejected": -335.20294189453125, + "loss": 1.1735, + "margin_dpo/margin_mean": 69.0799560546875, + "margin_dpo/margin_std": 134.26339721679688, + "step": 413 + }, + { + "KL/chosen_KL_mean": -164.39401245117188, + "KL/mean": -205.43978881835938, + "KL/rejected_KL_mean": -246.48556518554688, + "KL/std": 119.57435607910156, + "epoch": 0.6258503401360545, + "fcm_dpo/beta": 0.005096019245684147, + "fcm_dpo/delta": -0.020970303565263748, + "fcm_dpo/margin": 82.09153747558594, + "fcm_dpo/q_t": 0.40284112095832825, + "grad_norm": 10.116288185119629, + "learning_rate": 1.8593404702488436e-07, + "logits/chosen": 0.8319721221923828, + "logits/rejected": 0.7853256464004517, + "logps/chosen": -218.5322265625, + "logps/ref_chosen": -54.138214111328125, + "logps/ref_rejected": -74.65741729736328, + "logps/rejected": -321.1429748535156, + "loss": 1.0866, + "margin_dpo/margin_mean": 82.09153747558594, + "margin_dpo/margin_std": 109.47962951660156, + "step": 414 + }, + { + "KL/chosen_KL_mean": -164.3896484375, + "KL/mean": -195.80303955078125, + "KL/rejected_KL_mean": -227.21646118164062, + "KL/std": 107.44061279296875, + "epoch": 0.6273620559334845, + "fcm_dpo/beta": 0.0052430275827646255, + "fcm_dpo/delta": 0.07850854098796844, + "fcm_dpo/margin": 62.826805114746094, + "fcm_dpo/q_t": 0.4264346957206726, + "grad_norm": 11.805853843688965, + "learning_rate": 1.846568829074628e-07, + "logits/chosen": 0.812616765499115, + "logits/rejected": 0.801384449005127, + "logps/chosen": -220.30819702148438, + "logps/ref_chosen": -55.91856002807617, + "logps/ref_rejected": -61.747703552246094, + "logps/rejected": -288.96417236328125, + "loss": 1.1706, + "margin_dpo/margin_mean": 62.826805114746094, + "margin_dpo/margin_std": 115.88273620605469, + "step": 415 + }, + { + "KL/chosen_KL_mean": -174.27407836914062, + "KL/mean": -204.4618682861328, + "KL/rejected_KL_mean": -234.649658203125, + "KL/std": 114.86009216308594, + "epoch": 0.6288737717309146, + "fcm_dpo/beta": 0.005179405678063631, + "fcm_dpo/delta": -0.09270616620779037, + "fcm_dpo/margin": 60.37559509277344, + "fcm_dpo/q_t": 0.4285816550254822, + "grad_norm": 12.966651916503906, + "learning_rate": 1.8338154657749128e-07, + "logits/chosen": 0.8164198398590088, + "logits/rejected": 0.7761438488960266, + "logps/chosen": -228.99716186523438, + "logps/ref_chosen": -54.72308349609375, + "logps/ref_rejected": -69.17388916015625, + "logps/rejected": -303.82354736328125, + "loss": 1.1915, + "margin_dpo/margin_mean": 60.37559509277344, + "margin_dpo/margin_std": 108.85138702392578, + "step": 416 + }, + { + "KL/chosen_KL_mean": -181.67933654785156, + "KL/mean": -218.7706298828125, + "KL/rejected_KL_mean": -255.86190795898438, + "KL/std": 106.55653381347656, + "epoch": 0.6303854875283447, + "fcm_dpo/beta": 0.005155562423169613, + "fcm_dpo/delta": 0.017802808433771133, + "fcm_dpo/margin": 74.18257904052734, + "fcm_dpo/q_t": 0.4107144773006439, + "grad_norm": 12.372243881225586, + "learning_rate": 1.8210807370886849e-07, + "logits/chosen": 0.9078543782234192, + "logits/rejected": 0.8568928837776184, + "logps/chosen": -238.47059631347656, + "logps/ref_chosen": -56.791259765625, + "logps/ref_rejected": -68.7791748046875, + "logps/rejected": -324.64111328125, + "loss": 1.1455, + "margin_dpo/margin_mean": 74.18257904052734, + "margin_dpo/margin_std": 129.5694122314453, + "step": 417 + }, + { + "KL/chosen_KL_mean": -196.171875, + "KL/mean": -226.3531036376953, + "KL/rejected_KL_mean": -256.53436279296875, + "KL/std": 122.66242980957031, + "epoch": 0.6318972033257747, + "fcm_dpo/beta": 0.004903099965304136, + "fcm_dpo/delta": -0.06217623129487038, + "fcm_dpo/margin": 60.362483978271484, + "fcm_dpo/q_t": 0.4314631521701813, + "grad_norm": 12.916627883911133, + "learning_rate": 1.8083649992336825e-07, + "logits/chosen": 0.8412420153617859, + "logits/rejected": 0.8448699712753296, + "logps/chosen": -265.27984619140625, + "logps/ref_chosen": -69.10798645019531, + "logps/ref_rejected": -75.09132385253906, + "logps/rejected": -331.62567138671875, + "loss": 1.2087, + "margin_dpo/margin_mean": 60.362483978271484, + "margin_dpo/margin_std": 124.63722229003906, + "step": 418 + }, + { + "KL/chosen_KL_mean": -157.86094665527344, + "KL/mean": -203.56805419921875, + "KL/rejected_KL_mean": -249.27517700195312, + "KL/std": 108.47840881347656, + "epoch": 0.6334089191232048, + "fcm_dpo/beta": 0.004795808810740709, + "fcm_dpo/delta": -0.04438748210668564, + "fcm_dpo/margin": 91.41424560546875, + "fcm_dpo/q_t": 0.40094202756881714, + "grad_norm": 11.021547317504883, + "learning_rate": 1.7956686078964255e-07, + "logits/chosen": 0.7512973546981812, + "logits/rejected": 0.717224657535553, + "logps/chosen": -216.03271484375, + "logps/ref_chosen": -58.1717643737793, + "logps/ref_rejected": -71.67066955566406, + "logps/rejected": -320.94586181640625, + "loss": 1.0782, + "margin_dpo/margin_mean": 91.41425323486328, + "margin_dpo/margin_std": 126.16435241699219, + "step": 419 + }, + { + "KL/chosen_KL_mean": -189.92071533203125, + "KL/mean": -213.03028869628906, + "KL/rejected_KL_mean": -236.13986206054688, + "KL/std": 111.73857116699219, + "epoch": 0.6349206349206349, + "fcm_dpo/beta": 0.004948785062879324, + "fcm_dpo/delta": 0.05984485149383545, + "fcm_dpo/margin": 46.219139099121094, + "fcm_dpo/q_t": 0.4502519965171814, + "grad_norm": 11.667963981628418, + "learning_rate": 1.782991918222275e-07, + "logits/chosen": 0.8240588903427124, + "logits/rejected": 0.7886266708374023, + "logps/chosen": -246.97422790527344, + "logps/ref_chosen": -57.05351257324219, + "logps/ref_rejected": -62.670982360839844, + "logps/rejected": -298.81085205078125, + "loss": 1.2697, + "margin_dpo/margin_mean": 46.219139099121094, + "margin_dpo/margin_std": 130.39532470703125, + "step": 420 + }, + { + "KL/chosen_KL_mean": -178.27980041503906, + "KL/mean": -212.07435607910156, + "KL/rejected_KL_mean": -245.86886596679688, + "KL/std": 111.47808837890625, + "epoch": 0.636432350718065, + "fcm_dpo/beta": 0.004891910124570131, + "fcm_dpo/delta": -0.01926536299288273, + "fcm_dpo/margin": 67.58908081054688, + "fcm_dpo/q_t": 0.42542341351509094, + "grad_norm": 12.780967712402344, + "learning_rate": 1.7703352848054887e-07, + "logits/chosen": 0.7950316071510315, + "logits/rejected": 0.7450087070465088, + "logps/chosen": -235.60305786132812, + "logps/ref_chosen": -57.32324981689453, + "logps/ref_rejected": -75.33782958984375, + "logps/rejected": -321.20672607421875, + "loss": 1.2105, + "margin_dpo/margin_mean": 67.5890884399414, + "margin_dpo/margin_std": 150.59071350097656, + "step": 421 + }, + { + "KL/chosen_KL_mean": -158.81130981445312, + "KL/mean": -200.82315063476562, + "KL/rejected_KL_mean": -242.8350067138672, + "KL/std": 111.04067993164062, + "epoch": 0.6379440665154951, + "fcm_dpo/beta": 0.004911776166409254, + "fcm_dpo/delta": -0.015146823599934578, + "fcm_dpo/margin": 84.02371215820312, + "fcm_dpo/q_t": 0.40485405921936035, + "grad_norm": 12.647407531738281, + "learning_rate": 1.7576990616793137e-07, + "logits/chosen": 0.7745881080627441, + "logits/rejected": 0.775098443031311, + "logps/chosen": -225.86888122558594, + "logps/ref_chosen": -67.05757141113281, + "logps/ref_rejected": -72.12803649902344, + "logps/rejected": -314.9630432128906, + "loss": 1.0853, + "margin_dpo/margin_mean": 84.0237045288086, + "margin_dpo/margin_std": 109.76282501220703, + "step": 422 + }, + { + "KL/chosen_KL_mean": -155.72247314453125, + "KL/mean": -203.5447998046875, + "KL/rejected_KL_mean": -251.36709594726562, + "KL/std": 113.12423706054688, + "epoch": 0.6394557823129252, + "fcm_dpo/beta": 0.004685261286795139, + "fcm_dpo/delta": -0.05671604722738266, + "fcm_dpo/margin": 95.64463806152344, + "fcm_dpo/q_t": 0.39703303575515747, + "grad_norm": 11.059937477111816, + "learning_rate": 1.745083602306071e-07, + "logits/chosen": 0.8790537118911743, + "logits/rejected": 0.8249520063400269, + "logps/chosen": -209.78414916992188, + "logps/ref_chosen": -54.06167221069336, + "logps/ref_rejected": -76.64092254638672, + "logps/rejected": -328.0080261230469, + "loss": 1.0619, + "margin_dpo/margin_mean": 95.6446304321289, + "margin_dpo/margin_std": 121.1661605834961, + "step": 423 + }, + { + "KL/chosen_KL_mean": -170.83416748046875, + "KL/mean": -215.72669982910156, + "KL/rejected_KL_mean": -260.6192321777344, + "KL/std": 103.92689514160156, + "epoch": 0.6409674981103552, + "fcm_dpo/beta": 0.004581287503242493, + "fcm_dpo/delta": -0.014987409114837646, + "fcm_dpo/margin": 89.7850570678711, + "fcm_dpo/q_t": 0.40572795271873474, + "grad_norm": 14.250228881835938, + "learning_rate": 1.7324892595672804e-07, + "logits/chosen": 0.7274860143661499, + "logits/rejected": 0.6937150359153748, + "logps/chosen": -224.4430389404297, + "logps/ref_chosen": -53.60887145996094, + "logps/ref_rejected": -79.2139892578125, + "logps/rejected": -339.8332214355469, + "loss": 1.0895, + "margin_dpo/margin_mean": 89.7850570678711, + "margin_dpo/margin_std": 120.62132263183594, + "step": 424 + }, + { + "KL/chosen_KL_mean": -167.8055419921875, + "KL/mean": -202.077392578125, + "KL/rejected_KL_mean": -236.3492431640625, + "KL/std": 100.29512023925781, + "epoch": 0.6424792139077853, + "fcm_dpo/beta": 0.004820922389626503, + "fcm_dpo/delta": 0.07685823738574982, + "fcm_dpo/margin": 68.54373168945312, + "fcm_dpo/q_t": 0.4257993698120117, + "grad_norm": 11.25399398803711, + "learning_rate": 1.7199163857537824e-07, + "logits/chosen": 0.8642111420631409, + "logits/rejected": 0.843815803527832, + "logps/chosen": -226.22021484375, + "logps/ref_chosen": -58.41468048095703, + "logps/ref_rejected": -66.59054565429688, + "logps/rejected": -302.9397888183594, + "loss": 1.157, + "margin_dpo/margin_mean": 68.54371643066406, + "margin_dpo/margin_std": 115.27527618408203, + "step": 425 + }, + { + "KL/chosen_KL_mean": -201.40029907226562, + "KL/mean": -221.83224487304688, + "KL/rejected_KL_mean": -242.26416015625, + "KL/std": 111.92045593261719, + "epoch": 0.6439909297052154, + "fcm_dpo/beta": 0.004862668924033642, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 40.86387634277344, + "fcm_dpo/q_t": 0.4547138810157776, + "grad_norm": 15.746712684631348, + "learning_rate": 1.7073653325558828e-07, + "logits/chosen": 0.7816729545593262, + "logits/rejected": 0.7827064990997314, + "logps/chosen": -273.1085205078125, + "logps/ref_chosen": -71.70822143554688, + "logps/ref_rejected": -73.57725524902344, + "logps/rejected": -315.8414306640625, + "loss": 1.3073, + "margin_dpo/margin_mean": 40.86387634277344, + "margin_dpo/margin_std": 142.01010131835938, + "step": 426 + }, + { + "KL/chosen_KL_mean": -189.36972045898438, + "KL/mean": -227.52943420410156, + "KL/rejected_KL_mean": -265.6891784667969, + "KL/std": 122.76222229003906, + "epoch": 0.6455026455026455, + "fcm_dpo/beta": 0.004909333307296038, + "fcm_dpo/delta": 0.028550054877996445, + "fcm_dpo/margin": 76.31942749023438, + "fcm_dpo/q_t": 0.417738676071167, + "grad_norm": 12.455328941345215, + "learning_rate": 1.6948364510535218e-07, + "logits/chosen": 0.8194973468780518, + "logits/rejected": 0.7630200982093811, + "logps/chosen": -248.0124969482422, + "logps/ref_chosen": -58.64276885986328, + "logps/ref_rejected": -86.25437927246094, + "logps/rejected": -351.94354248046875, + "loss": 1.1438, + "margin_dpo/margin_mean": 76.31942749023438, + "margin_dpo/margin_std": 133.41810607910156, + "step": 427 + }, + { + "KL/chosen_KL_mean": -184.95486450195312, + "KL/mean": -222.78594970703125, + "KL/rejected_KL_mean": -260.6170349121094, + "KL/std": 124.32109832763672, + "epoch": 0.6470143613000756, + "fcm_dpo/beta": 0.005052429623901844, + "fcm_dpo/delta": 0.018595866858959198, + "fcm_dpo/margin": 75.66215515136719, + "fcm_dpo/q_t": 0.4149612486362457, + "grad_norm": 11.38692569732666, + "learning_rate": 1.6823300917064458e-07, + "logits/chosen": 0.8025504946708679, + "logits/rejected": 0.7721965909004211, + "logps/chosen": -251.55091857910156, + "logps/ref_chosen": -66.5960464477539, + "logps/ref_rejected": -82.3941650390625, + "logps/rejected": -343.01116943359375, + "loss": 1.1482, + "margin_dpo/margin_mean": 75.66214752197266, + "margin_dpo/margin_std": 133.01632690429688, + "step": 428 + }, + { + "KL/chosen_KL_mean": -187.84649658203125, + "KL/mean": -216.41619873046875, + "KL/rejected_KL_mean": -244.9859161376953, + "KL/std": 100.31537628173828, + "epoch": 0.6485260770975056, + "fcm_dpo/beta": 0.005307496525347233, + "fcm_dpo/delta": 0.10380184650421143, + "fcm_dpo/margin": 57.139427185058594, + "fcm_dpo/q_t": 0.43025392293930054, + "grad_norm": 12.65349006652832, + "learning_rate": 1.669846604344412e-07, + "logits/chosen": 0.7885690927505493, + "logits/rejected": 0.8087594509124756, + "logps/chosen": -244.856201171875, + "logps/ref_chosen": -57.00970458984375, + "logps/ref_rejected": -59.86549377441406, + "logps/rejected": -304.8514099121094, + "loss": 1.1977, + "margin_dpo/margin_mean": 57.139427185058594, + "margin_dpo/margin_std": 115.14334106445312, + "step": 429 + }, + { + "KL/chosen_KL_mean": -165.86953735351562, + "KL/mean": -214.21212768554688, + "KL/rejected_KL_mean": -262.5546875, + "KL/std": 101.71453094482422, + "epoch": 0.6500377928949358, + "fcm_dpo/beta": 0.005062808282673359, + "fcm_dpo/delta": -0.1054583340883255, + "fcm_dpo/margin": 96.68514251708984, + "fcm_dpo/q_t": 0.3835219144821167, + "grad_norm": 13.401054382324219, + "learning_rate": 1.6573863381573954e-07, + "logits/chosen": 0.7247724533081055, + "logits/rejected": 0.7243069410324097, + "logps/chosen": -225.4327392578125, + "logps/ref_chosen": -59.563194274902344, + "logps/ref_rejected": -70.52289581298828, + "logps/rejected": -333.0776062011719, + "loss": 1.0434, + "margin_dpo/margin_mean": 96.68515014648438, + "margin_dpo/margin_std": 119.87529754638672, + "step": 430 + }, + { + "KL/chosen_KL_mean": -164.5930633544922, + "KL/mean": -201.393310546875, + "KL/rejected_KL_mean": -238.1935577392578, + "KL/std": 106.98062896728516, + "epoch": 0.6515495086923658, + "fcm_dpo/beta": 0.0050440579652786255, + "fcm_dpo/delta": 0.032343633472919464, + "fcm_dpo/margin": 73.6004867553711, + "fcm_dpo/q_t": 0.41668662428855896, + "grad_norm": 12.450831413269043, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": 0.790619969367981, + "logits/rejected": 0.7420604825019836, + "logps/chosen": -214.79339599609375, + "logps/ref_chosen": -50.20032501220703, + "logps/ref_rejected": -77.81680297851562, + "logps/rejected": -316.0103759765625, + "loss": 1.1385, + "margin_dpo/margin_mean": 73.60047912597656, + "margin_dpo/margin_std": 121.77957916259766, + "step": 431 + }, + { + "KL/chosen_KL_mean": -174.41119384765625, + "KL/mean": -212.10067749023438, + "KL/rejected_KL_mean": -249.7901611328125, + "KL/std": 106.63932037353516, + "epoch": 0.6530612244897959, + "fcm_dpo/beta": 0.005112434271723032, + "fcm_dpo/delta": 0.016498606652021408, + "fcm_dpo/margin": 75.37895202636719, + "fcm_dpo/q_t": 0.4119753837585449, + "grad_norm": 13.068375587463379, + "learning_rate": 1.632536862810844e-07, + "logits/chosen": 0.8498221039772034, + "logits/rejected": 0.8017172813415527, + "logps/chosen": -236.07394409179688, + "logps/ref_chosen": -61.662757873535156, + "logps/ref_rejected": -83.94496154785156, + "logps/rejected": -333.735107421875, + "loss": 1.1373, + "margin_dpo/margin_mean": 75.37895202636719, + "margin_dpo/margin_std": 127.255615234375, + "step": 432 + }, + { + "KL/chosen_KL_mean": -171.37161254882812, + "KL/mean": -213.88760375976562, + "KL/rejected_KL_mean": -256.4035949707031, + "KL/std": 99.43716430664062, + "epoch": 0.654572940287226, + "fcm_dpo/beta": 0.00509435310959816, + "fcm_dpo/delta": -0.03820203244686127, + "fcm_dpo/margin": 85.03195190429688, + "fcm_dpo/q_t": 0.3992968797683716, + "grad_norm": 13.277294158935547, + "learning_rate": 1.6201483487445515e-07, + "logits/chosen": 0.8910690546035767, + "logits/rejected": 0.8961876034736633, + "logps/chosen": -235.10079956054688, + "logps/ref_chosen": -63.72917938232422, + "logps/ref_rejected": -65.8391342163086, + "logps/rejected": -322.24273681640625, + "loss": 1.079, + "margin_dpo/margin_mean": 85.03195190429688, + "margin_dpo/margin_std": 113.61282348632812, + "step": 433 + }, + { + "KL/chosen_KL_mean": -146.40029907226562, + "KL/mean": -194.85731506347656, + "KL/rejected_KL_mean": -243.3143310546875, + "KL/std": 110.77456665039062, + "epoch": 0.656084656084656, + "fcm_dpo/beta": 0.004744514357298613, + "fcm_dpo/delta": -0.07580451667308807, + "fcm_dpo/margin": 96.91404724121094, + "fcm_dpo/q_t": 0.3922083377838135, + "grad_norm": 11.460370063781738, + "learning_rate": 1.6077844460203204e-07, + "logits/chosen": 0.9136182069778442, + "logits/rejected": 0.8583601713180542, + "logps/chosen": -194.3736114501953, + "logps/ref_chosen": -47.97331619262695, + "logps/ref_rejected": -72.51132202148438, + "logps/rejected": -315.8256530761719, + "loss": 1.0742, + "margin_dpo/margin_mean": 96.91405487060547, + "margin_dpo/margin_std": 131.41989135742188, + "step": 434 + }, + { + "KL/chosen_KL_mean": -177.25384521484375, + "KL/mean": -212.98779296875, + "KL/rejected_KL_mean": -248.72177124023438, + "KL/std": 106.70513916015625, + "epoch": 0.6575963718820862, + "fcm_dpo/beta": 0.004973133094608784, + "fcm_dpo/delta": 0.04807582497596741, + "fcm_dpo/margin": 71.46792602539062, + "fcm_dpo/q_t": 0.4196769893169403, + "grad_norm": 13.298686027526855, + "learning_rate": 1.5954455004830878e-07, + "logits/chosen": 0.9066946506500244, + "logits/rejected": 0.8741989135742188, + "logps/chosen": -234.3140869140625, + "logps/ref_chosen": -57.06024932861328, + "logps/ref_rejected": -71.69146728515625, + "logps/rejected": -320.4132385253906, + "loss": 1.1529, + "margin_dpo/margin_mean": 71.46792602539062, + "margin_dpo/margin_std": 123.43423461914062, + "step": 435 + }, + { + "KL/chosen_KL_mean": -178.68710327148438, + "KL/mean": -208.48751831054688, + "KL/rejected_KL_mean": -238.28793334960938, + "KL/std": 103.73971557617188, + "epoch": 0.6591080876795162, + "fcm_dpo/beta": 0.005142658948898315, + "fcm_dpo/delta": 0.1033397912979126, + "fcm_dpo/margin": 59.600830078125, + "fcm_dpo/q_t": 0.4327392578125, + "grad_norm": 13.91799545288086, + "learning_rate": 1.5831318572796847e-07, + "logits/chosen": 0.8306369781494141, + "logits/rejected": 0.7831582427024841, + "logps/chosen": -234.84515380859375, + "logps/ref_chosen": -56.158050537109375, + "logps/ref_rejected": -67.63787841796875, + "logps/rejected": -305.9258117675781, + "loss": 1.206, + "margin_dpo/margin_mean": 59.600830078125, + "margin_dpo/margin_std": 127.89569091796875, + "step": 436 + }, + { + "KL/chosen_KL_mean": -183.9375457763672, + "KL/mean": -221.9730224609375, + "KL/rejected_KL_mean": -260.00848388671875, + "KL/std": 115.573486328125, + "epoch": 0.6606198034769464, + "fcm_dpo/beta": 0.005009764805436134, + "fcm_dpo/delta": -0.082071453332901, + "fcm_dpo/margin": 76.07093811035156, + "fcm_dpo/q_t": 0.41367313265800476, + "grad_norm": 15.81583023071289, + "learning_rate": 1.5708438608491815e-07, + "logits/chosen": 0.8169673085212708, + "logits/rejected": 0.7099220752716064, + "logps/chosen": -240.92333984375, + "logps/ref_chosen": -56.98578643798828, + "logps/ref_rejected": -85.61524963378906, + "logps/rejected": -345.62371826171875, + "loss": 1.1729, + "margin_dpo/margin_mean": 76.07093811035156, + "margin_dpo/margin_std": 148.16189575195312, + "step": 437 + }, + { + "KL/chosen_KL_mean": -159.0219268798828, + "KL/mean": -206.44488525390625, + "KL/rejected_KL_mean": -253.86782836914062, + "KL/std": 118.10974884033203, + "epoch": 0.6621315192743764, + "fcm_dpo/beta": 0.004896242171525955, + "fcm_dpo/delta": -0.07475844770669937, + "fcm_dpo/margin": 94.84590148925781, + "fcm_dpo/q_t": 0.3970157504081726, + "grad_norm": 12.760135650634766, + "learning_rate": 1.558581854913253e-07, + "logits/chosen": 0.8870106935501099, + "logits/rejected": 0.8304316997528076, + "logps/chosen": -200.29971313476562, + "logps/ref_chosen": -41.27777862548828, + "logps/ref_rejected": -65.33840942382812, + "logps/rejected": -319.20623779296875, + "loss": 1.0676, + "margin_dpo/margin_mean": 94.84590148925781, + "margin_dpo/margin_std": 133.10060119628906, + "step": 438 + }, + { + "KL/chosen_KL_mean": -186.03314208984375, + "KL/mean": -229.74700927734375, + "KL/rejected_KL_mean": -273.4609069824219, + "KL/std": 121.18110656738281, + "epoch": 0.6636432350718064, + "fcm_dpo/beta": 0.004845023155212402, + "fcm_dpo/delta": -0.03054000437259674, + "fcm_dpo/margin": 87.42778015136719, + "fcm_dpo/q_t": 0.40231576561927795, + "grad_norm": 11.937276840209961, + "learning_rate": 1.5463461824665658e-07, + "logits/chosen": 0.726641833782196, + "logits/rejected": 0.7027250528335571, + "logps/chosen": -267.4508056640625, + "logps/ref_chosen": -81.41764831542969, + "logps/ref_rejected": -94.72309875488281, + "logps/rejected": -368.1839904785156, + "loss": 1.0944, + "margin_dpo/margin_mean": 87.42778015136719, + "margin_dpo/margin_std": 121.04769897460938, + "step": 439 + }, + { + "KL/chosen_KL_mean": -155.7847900390625, + "KL/mean": -199.2237548828125, + "KL/rejected_KL_mean": -242.66273498535156, + "KL/std": 106.59927368164062, + "epoch": 0.6651549508692366, + "fcm_dpo/beta": 0.00472570164129138, + "fcm_dpo/delta": -0.012669751420617104, + "fcm_dpo/margin": 86.87796020507812, + "fcm_dpo/q_t": 0.40606674551963806, + "grad_norm": 15.137947082519531, + "learning_rate": 1.534137185767178e-07, + "logits/chosen": 0.8190924525260925, + "logits/rejected": 0.7351869344711304, + "logps/chosen": -198.32296752929688, + "logps/ref_chosen": -42.538185119628906, + "logps/ref_rejected": -69.78813934326172, + "logps/rejected": -312.45086669921875, + "loss": 1.0963, + "margin_dpo/margin_mean": 86.87796783447266, + "margin_dpo/margin_std": 120.90702819824219, + "step": 440 + }, + { + "KL/chosen_KL_mean": -163.71417236328125, + "KL/mean": -209.40249633789062, + "KL/rejected_KL_mean": -255.09083557128906, + "KL/std": 117.68836212158203, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.004568706266582012, + "fcm_dpo/delta": -0.021269341930747032, + "fcm_dpo/margin": 91.37667083740234, + "fcm_dpo/q_t": 0.4020621180534363, + "grad_norm": 12.57343864440918, + "learning_rate": 1.521955206326976e-07, + "logits/chosen": 0.7851335406303406, + "logits/rejected": 0.7083240747451782, + "logps/chosen": -221.30738830566406, + "logps/ref_chosen": -57.593223571777344, + "logps/ref_rejected": -84.82878875732422, + "logps/rejected": -339.91961669921875, + "loss": 1.0718, + "margin_dpo/margin_mean": 91.37667083740234, + "margin_dpo/margin_std": 108.32664489746094, + "step": 441 + }, + { + "KL/chosen_KL_mean": -194.39462280273438, + "KL/mean": -241.25390625, + "KL/rejected_KL_mean": -288.11322021484375, + "KL/std": 117.97537231445312, + "epoch": 0.6681783824640968, + "fcm_dpo/beta": 0.004501115530729294, + "fcm_dpo/delta": -0.026340365409851074, + "fcm_dpo/margin": 93.71859741210938, + "fcm_dpo/q_t": 0.40267208218574524, + "grad_norm": 14.261401176452637, + "learning_rate": 1.5098005849021078e-07, + "logits/chosen": 0.790300726890564, + "logits/rejected": 0.7574805021286011, + "logps/chosen": -261.8558349609375, + "logps/ref_chosen": -67.46121978759766, + "logps/ref_rejected": -89.0693588256836, + "logps/rejected": -377.18255615234375, + "loss": 1.0786, + "margin_dpo/margin_mean": 93.7186050415039, + "margin_dpo/margin_std": 120.49788665771484, + "step": 442 + }, + { + "KL/chosen_KL_mean": -160.43263244628906, + "KL/mean": -216.71011352539062, + "KL/rejected_KL_mean": -272.98760986328125, + "KL/std": 124.23033905029297, + "epoch": 0.6696900982615268, + "fcm_dpo/beta": 0.004319561645388603, + "fcm_dpo/delta": -0.10150419175624847, + "fcm_dpo/margin": 112.55494689941406, + "fcm_dpo/q_t": 0.3878478407859802, + "grad_norm": 11.641592025756836, + "learning_rate": 1.4976736614834662e-07, + "logits/chosen": 0.8116977214813232, + "logits/rejected": 0.7602815628051758, + "logps/chosen": -215.22872924804688, + "logps/ref_chosen": -54.79610061645508, + "logps/ref_rejected": -77.80781555175781, + "logps/rejected": -350.79541015625, + "loss": 1.0382, + "margin_dpo/margin_mean": 112.5549545288086, + "margin_dpo/margin_std": 139.6777801513672, + "step": 443 + }, + { + "KL/chosen_KL_mean": -191.93292236328125, + "KL/mean": -221.1522674560547, + "KL/rejected_KL_mean": -250.37164306640625, + "KL/std": 127.5732421875, + "epoch": 0.671201814058957, + "fcm_dpo/beta": 0.00436423160135746, + "fcm_dpo/delta": 0.03717103973031044, + "fcm_dpo/margin": 58.438720703125, + "fcm_dpo/q_t": 0.4444841146469116, + "grad_norm": 12.10918140411377, + "learning_rate": 1.4855747752871654e-07, + "logits/chosen": 0.8365426063537598, + "logits/rejected": 0.7632496356964111, + "logps/chosen": -250.68197631835938, + "logps/ref_chosen": -58.749061584472656, + "logps/ref_rejected": -86.87396240234375, + "logps/rejected": -337.24560546875, + "loss": 1.2394, + "margin_dpo/margin_mean": 58.438720703125, + "margin_dpo/margin_std": 142.1253662109375, + "step": 444 + }, + { + "KL/chosen_KL_mean": -183.2960968017578, + "KL/mean": -231.47491455078125, + "KL/rejected_KL_mean": -279.6537170410156, + "KL/std": 110.73534393310547, + "epoch": 0.672713529856387, + "fcm_dpo/beta": 0.004318728111684322, + "fcm_dpo/delta": -0.01852697879076004, + "fcm_dpo/margin": 96.35762023925781, + "fcm_dpo/q_t": 0.4018581509590149, + "grad_norm": 10.044200897216797, + "learning_rate": 1.473504264745062e-07, + "logits/chosen": 0.7895313501358032, + "logits/rejected": 0.7898776531219482, + "logps/chosen": -244.21353149414062, + "logps/ref_chosen": -60.91743850708008, + "logps/ref_rejected": -71.5637435913086, + "logps/rejected": -351.21746826171875, + "loss": 1.0725, + "margin_dpo/margin_mean": 96.35762023925781, + "margin_dpo/margin_std": 115.74655151367188, + "step": 445 + }, + { + "KL/chosen_KL_mean": -164.505126953125, + "KL/mean": -212.84637451171875, + "KL/rejected_KL_mean": -261.1876525878906, + "KL/std": 113.10897827148438, + "epoch": 0.674225245653817, + "fcm_dpo/beta": 0.004369608126580715, + "fcm_dpo/delta": -0.0294049009680748, + "fcm_dpo/margin": 96.6825180053711, + "fcm_dpo/q_t": 0.4011649489402771, + "grad_norm": 11.11204719543457, + "learning_rate": 1.461462467495284e-07, + "logits/chosen": 0.8308959007263184, + "logits/rejected": 0.7669998407363892, + "logps/chosen": -213.3043670654297, + "logps/ref_chosen": -48.79924774169922, + "logps/ref_rejected": -71.8719482421875, + "logps/rejected": -333.0596008300781, + "loss": 1.0718, + "margin_dpo/margin_mean": 96.6825180053711, + "margin_dpo/margin_std": 111.4892349243164, + "step": 446 + }, + { + "KL/chosen_KL_mean": -160.591796875, + "KL/mean": -217.41326904296875, + "KL/rejected_KL_mean": -274.2347412109375, + "KL/std": 113.17485046386719, + "epoch": 0.6757369614512472, + "fcm_dpo/beta": 0.004061791114509106, + "fcm_dpo/delta": -0.07315800338983536, + "fcm_dpo/margin": 113.64291381835938, + "fcm_dpo/q_t": 0.3898429274559021, + "grad_norm": 9.85269546508789, + "learning_rate": 1.4494497203727843e-07, + "logits/chosen": 0.8119666576385498, + "logits/rejected": 0.7234156131744385, + "logps/chosen": -214.27452087402344, + "logps/ref_chosen": -53.682716369628906, + "logps/ref_rejected": -88.17315673828125, + "logps/rejected": -362.40789794921875, + "loss": 1.0415, + "margin_dpo/margin_mean": 113.6429214477539, + "margin_dpo/margin_std": 126.60255432128906, + "step": 447 + }, + { + "KL/chosen_KL_mean": -177.05801391601562, + "KL/mean": -221.44229125976562, + "KL/rejected_KL_mean": -265.82659912109375, + "KL/std": 117.35690307617188, + "epoch": 0.6772486772486772, + "fcm_dpo/beta": 0.0041312845423817635, + "fcm_dpo/delta": 0.03732322156429291, + "fcm_dpo/margin": 88.76856231689453, + "fcm_dpo/q_t": 0.41569164395332336, + "grad_norm": 8.617594718933105, + "learning_rate": 1.4374663593999256e-07, + "logits/chosen": 0.8559887409210205, + "logits/rejected": 0.8163323402404785, + "logps/chosen": -230.80926513671875, + "logps/ref_chosen": -53.75125503540039, + "logps/ref_rejected": -77.17623901367188, + "logps/rejected": -343.0028076171875, + "loss": 1.1175, + "margin_dpo/margin_mean": 88.76856231689453, + "margin_dpo/margin_std": 127.51617431640625, + "step": 448 + }, + { + "KL/chosen_KL_mean": -211.01304626464844, + "KL/mean": -233.7032470703125, + "KL/rejected_KL_mean": -256.3934326171875, + "KL/std": 123.69744873046875, + "epoch": 0.6787603930461074, + "fcm_dpo/beta": 0.004320129752159119, + "fcm_dpo/delta": 0.06683868914842606, + "fcm_dpo/margin": 45.38038635253906, + "fcm_dpo/q_t": 0.45501431822776794, + "grad_norm": 17.40116310119629, + "learning_rate": 1.4255127197770707e-07, + "logits/chosen": 0.6914898157119751, + "logits/rejected": 0.6926765441894531, + "logps/chosen": -286.8404235839844, + "logps/ref_chosen": -75.82737731933594, + "logps/ref_rejected": -82.20687866210938, + "logps/rejected": -338.6003112792969, + "loss": 1.2687, + "margin_dpo/margin_mean": 45.38037872314453, + "margin_dpo/margin_std": 122.3283920288086, + "step": 449 + }, + { + "KL/chosen_KL_mean": -170.02328491210938, + "KL/mean": -208.6387176513672, + "KL/rejected_KL_mean": -247.25418090820312, + "KL/std": 121.925537109375, + "epoch": 0.6802721088435374, + "fcm_dpo/beta": 0.0044110435992479324, + "fcm_dpo/delta": 0.06613278388977051, + "fcm_dpo/margin": 77.23091888427734, + "fcm_dpo/q_t": 0.4259797930717468, + "grad_norm": 12.112007141113281, + "learning_rate": 1.4135891358732205e-07, + "logits/chosen": 0.9353618621826172, + "logits/rejected": 0.8390960097312927, + "logps/chosen": -217.1389923095703, + "logps/ref_chosen": -47.11572265625, + "logps/ref_rejected": -78.7546615600586, + "logps/rejected": -326.00885009765625, + "loss": 1.1643, + "margin_dpo/margin_mean": 77.23091888427734, + "margin_dpo/margin_std": 139.64755249023438, + "step": 450 + }, + { + "KL/chosen_KL_mean": -172.9991912841797, + "KL/mean": -207.0963134765625, + "KL/rejected_KL_mean": -241.19345092773438, + "KL/std": 114.673095703125, + "epoch": 0.6817838246409675, + "fcm_dpo/beta": 0.004698569420725107, + "fcm_dpo/delta": 0.08733482658863068, + "fcm_dpo/margin": 68.19424438476562, + "fcm_dpo/q_t": 0.4278087615966797, + "grad_norm": 12.366849899291992, + "learning_rate": 1.4016959412166437e-07, + "logits/chosen": 0.7637462615966797, + "logits/rejected": 0.7285028696060181, + "logps/chosen": -236.34963989257812, + "logps/ref_chosen": -63.350440979003906, + "logps/ref_rejected": -76.28530883789062, + "logps/rejected": -317.478759765625, + "loss": 1.1642, + "margin_dpo/margin_mean": 68.19424438476562, + "margin_dpo/margin_std": 118.42993927001953, + "step": 451 + }, + { + "KL/chosen_KL_mean": -173.6109161376953, + "KL/mean": -212.7416229248047, + "KL/rejected_KL_mean": -251.87232971191406, + "KL/std": 110.82177734375, + "epoch": 0.6832955404383976, + "fcm_dpo/beta": 0.004780272487550974, + "fcm_dpo/delta": 0.02917136438190937, + "fcm_dpo/margin": 78.26138305664062, + "fcm_dpo/q_t": 0.41561344265937805, + "grad_norm": 13.4918851852417, + "learning_rate": 1.3898334684855645e-07, + "logits/chosen": 0.7915092706680298, + "logits/rejected": 0.723894476890564, + "logps/chosen": -229.19674682617188, + "logps/ref_chosen": -55.58583450317383, + "logps/ref_rejected": -77.68738555908203, + "logps/rejected": -329.5596923828125, + "loss": 1.1423, + "margin_dpo/margin_mean": 78.26138305664062, + "margin_dpo/margin_std": 133.20782470703125, + "step": 452 + }, + { + "KL/chosen_KL_mean": -175.6853485107422, + "KL/mean": -211.41357421875, + "KL/rejected_KL_mean": -247.1417999267578, + "KL/std": 114.35888671875, + "epoch": 0.6848072562358276, + "fcm_dpo/beta": 0.004915791098028421, + "fcm_dpo/delta": 0.05457682907581329, + "fcm_dpo/margin": 71.45646667480469, + "fcm_dpo/q_t": 0.42137593030929565, + "grad_norm": 15.09415340423584, + "learning_rate": 1.3780020494988445e-07, + "logits/chosen": 0.7863320708274841, + "logits/rejected": 0.772057294845581, + "logps/chosen": -237.4635467529297, + "logps/ref_chosen": -61.778202056884766, + "logps/ref_rejected": -71.51403045654297, + "logps/rejected": -318.65582275390625, + "loss": 1.1581, + "margin_dpo/margin_mean": 71.45646667480469, + "margin_dpo/margin_std": 126.16264343261719, + "step": 453 + }, + { + "KL/chosen_KL_mean": -158.27056884765625, + "KL/mean": -200.40567016601562, + "KL/rejected_KL_mean": -242.54075622558594, + "KL/std": 111.83387756347656, + "epoch": 0.6863189720332578, + "fcm_dpo/beta": 0.0049323514103889465, + "fcm_dpo/delta": -0.01793980970978737, + "fcm_dpo/margin": 84.27018737792969, + "fcm_dpo/q_t": 0.4046657383441925, + "grad_norm": 11.837895393371582, + "learning_rate": 1.366202015206706e-07, + "logits/chosen": 0.8560349345207214, + "logits/rejected": 0.8259134292602539, + "logps/chosen": -209.86572265625, + "logps/ref_chosen": -51.59515380859375, + "logps/ref_rejected": -63.96732711791992, + "logps/rejected": -306.5080871582031, + "loss": 1.1011, + "margin_dpo/margin_mean": 84.27018737792969, + "margin_dpo/margin_std": 126.40806579589844, + "step": 454 + }, + { + "KL/chosen_KL_mean": -186.71917724609375, + "KL/mean": -226.54803466796875, + "KL/rejected_KL_mean": -266.37689208984375, + "KL/std": 115.4573745727539, + "epoch": 0.6878306878306878, + "fcm_dpo/beta": 0.004945638123899698, + "fcm_dpo/delta": 0.006844331510365009, + "fcm_dpo/margin": 79.65769958496094, + "fcm_dpo/q_t": 0.41296789050102234, + "grad_norm": 13.294573783874512, + "learning_rate": 1.354433695681474e-07, + "logits/chosen": 0.7359009385108948, + "logits/rejected": 0.717836320400238, + "logps/chosen": -257.3708801269531, + "logps/ref_chosen": -70.65170288085938, + "logps/ref_rejected": -77.44276428222656, + "logps/rejected": -343.81964111328125, + "loss": 1.1178, + "margin_dpo/margin_mean": 79.65769958496094, + "margin_dpo/margin_std": 125.02285766601562, + "step": 455 + }, + { + "KL/chosen_KL_mean": -177.96502685546875, + "KL/mean": -217.13229370117188, + "KL/rejected_KL_mean": -256.299560546875, + "KL/std": 114.41992950439453, + "epoch": 0.6893424036281179, + "fcm_dpo/beta": 0.0049400972202420235, + "fcm_dpo/delta": 0.014393320307135582, + "fcm_dpo/margin": 78.33451843261719, + "fcm_dpo/q_t": 0.41560494899749756, + "grad_norm": 17.193586349487305, + "learning_rate": 1.3426974201083439e-07, + "logits/chosen": 0.7663091421127319, + "logits/rejected": 0.7156355381011963, + "logps/chosen": -234.36331176757812, + "logps/ref_chosen": -56.398284912109375, + "logps/ref_rejected": -82.61642456054688, + "logps/rejected": -338.91595458984375, + "loss": 1.1366, + "margin_dpo/margin_mean": 78.33451843261719, + "margin_dpo/margin_std": 135.641357421875, + "step": 456 + }, + { + "KL/chosen_KL_mean": -173.41329956054688, + "KL/mean": -213.361572265625, + "KL/rejected_KL_mean": -253.3098602294922, + "KL/std": 117.31248474121094, + "epoch": 0.690854119425548, + "fcm_dpo/beta": 0.004963235929608345, + "fcm_dpo/delta": 0.003583187237381935, + "fcm_dpo/margin": 79.89659118652344, + "fcm_dpo/q_t": 0.4083296060562134, + "grad_norm": 14.671182632446289, + "learning_rate": 1.3309935167761717e-07, + "logits/chosen": 0.9071280360221863, + "logits/rejected": 0.8378027677536011, + "logps/chosen": -218.1338653564453, + "logps/ref_chosen": -44.72057342529297, + "logps/ref_rejected": -68.1158676147461, + "logps/rejected": -321.42572021484375, + "loss": 1.1027, + "margin_dpo/margin_mean": 79.8965835571289, + "margin_dpo/margin_std": 113.3616943359375, + "step": 457 + }, + { + "KL/chosen_KL_mean": -168.78573608398438, + "KL/mean": -213.8314208984375, + "KL/rejected_KL_mean": -258.87713623046875, + "KL/std": 119.34392547607422, + "epoch": 0.6923658352229781, + "fcm_dpo/beta": 0.004918850027024746, + "fcm_dpo/delta": -0.049693845212459564, + "fcm_dpo/margin": 90.09138488769531, + "fcm_dpo/q_t": 0.3987731337547302, + "grad_norm": 12.683366775512695, + "learning_rate": 1.3193223130682936e-07, + "logits/chosen": 0.8713840246200562, + "logits/rejected": 0.7701141834259033, + "logps/chosen": -218.7914276123047, + "logps/ref_chosen": -50.00569152832031, + "logps/ref_rejected": -87.50015258789062, + "logps/rejected": -346.37725830078125, + "loss": 1.0915, + "margin_dpo/margin_mean": 90.09138488769531, + "margin_dpo/margin_std": 134.1453399658203, + "step": 458 + }, + { + "KL/chosen_KL_mean": -167.32614135742188, + "KL/mean": -218.5425262451172, + "KL/rejected_KL_mean": -269.7589111328125, + "KL/std": 142.56143188476562, + "epoch": 0.6938775510204082, + "fcm_dpo/beta": 0.004762580618262291, + "fcm_dpo/delta": -0.10504220426082611, + "fcm_dpo/margin": 102.4327392578125, + "fcm_dpo/q_t": 0.3850099444389343, + "grad_norm": 11.739134788513184, + "learning_rate": 1.3076841354533658e-07, + "logits/chosen": 0.868898868560791, + "logits/rejected": 0.8451403975486755, + "logps/chosen": -232.7041015625, + "logps/ref_chosen": -65.37794494628906, + "logps/ref_rejected": -88.19244384765625, + "logps/rejected": -357.95135498046875, + "loss": 1.0621, + "margin_dpo/margin_mean": 102.43274688720703, + "margin_dpo/margin_std": 134.36404418945312, + "step": 459 + }, + { + "KL/chosen_KL_mean": -177.44586181640625, + "KL/mean": -228.05621337890625, + "KL/rejected_KL_mean": -278.66656494140625, + "KL/std": 122.83148193359375, + "epoch": 0.6953892668178382, + "fcm_dpo/beta": 0.0044008782133460045, + "fcm_dpo/delta": -0.05401448905467987, + "fcm_dpo/margin": 101.220703125, + "fcm_dpo/q_t": 0.3985062539577484, + "grad_norm": 10.761886596679688, + "learning_rate": 1.2960793094762345e-07, + "logits/chosen": 0.8326910734176636, + "logits/rejected": 0.7360147833824158, + "logps/chosen": -242.00753784179688, + "logps/ref_chosen": -64.5616683959961, + "logps/ref_rejected": -88.67890167236328, + "logps/rejected": -367.345458984375, + "loss": 1.0705, + "margin_dpo/margin_mean": 101.220703125, + "margin_dpo/margin_std": 134.64776611328125, + "step": 460 + }, + { + "KL/chosen_KL_mean": -146.41845703125, + "KL/mean": -197.54617309570312, + "KL/rejected_KL_mean": -248.6739044189453, + "KL/std": 113.24662780761719, + "epoch": 0.6969009826152683, + "fcm_dpo/beta": 0.004217734094709158, + "fcm_dpo/delta": -0.04221995174884796, + "fcm_dpo/margin": 102.25545501708984, + "fcm_dpo/q_t": 0.3992024064064026, + "grad_norm": 10.860950469970703, + "learning_rate": 1.2845081597488286e-07, + "logits/chosen": 0.897905707359314, + "logits/rejected": 0.8329803943634033, + "logps/chosen": -195.89637756347656, + "logps/ref_chosen": -49.4779167175293, + "logps/ref_rejected": -72.65262603759766, + "logps/rejected": -321.3265380859375, + "loss": 1.0704, + "margin_dpo/margin_mean": 102.25545501708984, + "margin_dpo/margin_std": 122.31402587890625, + "step": 461 + }, + { + "KL/chosen_KL_mean": -162.05267333984375, + "KL/mean": -212.8834228515625, + "KL/rejected_KL_mean": -263.71417236328125, + "KL/std": 105.76100158691406, + "epoch": 0.6984126984126984, + "fcm_dpo/beta": 0.004214158747345209, + "fcm_dpo/delta": -0.0330844447016716, + "fcm_dpo/margin": 101.66149139404297, + "fcm_dpo/q_t": 0.39864546060562134, + "grad_norm": 10.956618309020996, + "learning_rate": 1.27297100994108e-07, + "logits/chosen": 0.8604691028594971, + "logits/rejected": 0.8234196901321411, + "logps/chosen": -222.54779052734375, + "logps/ref_chosen": -60.4951171875, + "logps/ref_rejected": -74.82136535644531, + "logps/rejected": -338.5355224609375, + "loss": 1.062, + "margin_dpo/margin_mean": 101.66149139404297, + "margin_dpo/margin_std": 117.94256591796875, + "step": 462 + }, + { + "KL/chosen_KL_mean": -197.11163330078125, + "KL/mean": -225.55178833007812, + "KL/rejected_KL_mean": -253.991943359375, + "KL/std": 109.48463439941406, + "epoch": 0.6999244142101285, + "fcm_dpo/beta": 0.004320027772337198, + "fcm_dpo/delta": 0.039919111877679825, + "fcm_dpo/margin": 56.880313873291016, + "fcm_dpo/q_t": 0.44253993034362793, + "grad_norm": 17.169588088989258, + "learning_rate": 1.2614681827718695e-07, + "logits/chosen": 0.8233051896095276, + "logits/rejected": 0.8192650675773621, + "logps/chosen": -264.7967529296875, + "logps/ref_chosen": -67.68511962890625, + "logps/ref_rejected": -71.32196044921875, + "logps/rejected": -325.31390380859375, + "loss": 1.2257, + "margin_dpo/margin_mean": 56.88031005859375, + "margin_dpo/margin_std": 123.54747009277344, + "step": 463 + }, + { + "KL/chosen_KL_mean": -176.97918701171875, + "KL/mean": -224.4793701171875, + "KL/rejected_KL_mean": -271.97955322265625, + "KL/std": 127.88569641113281, + "epoch": 0.7014361300075586, + "fcm_dpo/beta": 0.0043390789069235325, + "fcm_dpo/delta": -0.014604151248931885, + "fcm_dpo/margin": 95.00037384033203, + "fcm_dpo/q_t": 0.40515702962875366, + "grad_norm": 10.686640739440918, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": 0.8410533666610718, + "logits/rejected": 0.8190803527832031, + "logps/chosen": -236.14483642578125, + "logps/ref_chosen": -59.16564178466797, + "logps/ref_rejected": -69.56146240234375, + "logps/rejected": -341.541015625, + "loss": 1.108, + "margin_dpo/margin_mean": 95.0003662109375, + "margin_dpo/margin_std": 141.3034210205078, + "step": 464 + }, + { + "KL/chosen_KL_mean": -184.62344360351562, + "KL/mean": -226.74331665039062, + "KL/rejected_KL_mean": -268.8631896972656, + "KL/std": 118.66059875488281, + "epoch": 0.7029478458049887, + "fcm_dpo/beta": 0.004308546893298626, + "fcm_dpo/delta": 0.04102543368935585, + "fcm_dpo/margin": 84.23976135253906, + "fcm_dpo/q_t": 0.4176803529262543, + "grad_norm": 11.88809871673584, + "learning_rate": 1.238566782415197e-07, + "logits/chosen": 0.8855515718460083, + "logits/rejected": 0.836800217628479, + "logps/chosen": -243.13711547851562, + "logps/ref_chosen": -58.513671875, + "logps/ref_rejected": -84.31745910644531, + "logps/rejected": -353.1806640625, + "loss": 1.1367, + "margin_dpo/margin_mean": 84.23976135253906, + "margin_dpo/margin_std": 132.88246154785156, + "step": 465 + }, + { + "KL/chosen_KL_mean": -203.14230346679688, + "KL/mean": -222.7254638671875, + "KL/rejected_KL_mean": -242.3086395263672, + "KL/std": 127.04100799560547, + "epoch": 0.7044595616024187, + "fcm_dpo/beta": 0.004587305244058371, + "fcm_dpo/delta": 0.07363003492355347, + "fcm_dpo/margin": 39.16636276245117, + "fcm_dpo/q_t": 0.45767632126808167, + "grad_norm": 16.998245239257812, + "learning_rate": 1.2271688498291334e-07, + "logits/chosen": 0.8211065530776978, + "logits/rejected": 0.8255276083946228, + "logps/chosen": -276.4081115722656, + "logps/ref_chosen": -73.26580810546875, + "logps/ref_rejected": -74.83621215820312, + "logps/rejected": -317.14483642578125, + "loss": 1.3027, + "margin_dpo/margin_mean": 39.166358947753906, + "margin_dpo/margin_std": 131.44540405273438, + "step": 466 + }, + { + "KL/chosen_KL_mean": -176.02005004882812, + "KL/mean": -217.30421447753906, + "KL/rejected_KL_mean": -258.58837890625, + "KL/std": 126.46763610839844, + "epoch": 0.7059712773998488, + "fcm_dpo/beta": 0.004604835994541645, + "fcm_dpo/delta": 0.022210007533431053, + "fcm_dpo/margin": 82.56832885742188, + "fcm_dpo/q_t": 0.41688525676727295, + "grad_norm": 10.904779434204102, + "learning_rate": 1.2158065210664848e-07, + "logits/chosen": 0.8689050674438477, + "logits/rejected": 0.7562901973724365, + "logps/chosen": -223.59951782226562, + "logps/ref_chosen": -47.57947540283203, + "logps/ref_rejected": -78.68522644042969, + "logps/rejected": -337.27362060546875, + "loss": 1.1317, + "margin_dpo/margin_mean": 82.56832885742188, + "margin_dpo/margin_std": 138.4664306640625, + "step": 467 + }, + { + "KL/chosen_KL_mean": -171.49169921875, + "KL/mean": -218.56295776367188, + "KL/rejected_KL_mean": -265.6342468261719, + "KL/std": 121.87532806396484, + "epoch": 0.7074829931972789, + "fcm_dpo/beta": 0.004571876488626003, + "fcm_dpo/delta": -0.03497558832168579, + "fcm_dpo/margin": 94.14254760742188, + "fcm_dpo/q_t": 0.39979416131973267, + "grad_norm": 13.04074764251709, + "learning_rate": 1.204480113956011e-07, + "logits/chosen": 0.798638105392456, + "logits/rejected": 0.7876749038696289, + "logps/chosen": -235.4194793701172, + "logps/ref_chosen": -63.92778778076172, + "logps/ref_rejected": -76.51626586914062, + "logps/rejected": -342.1505126953125, + "loss": 1.101, + "margin_dpo/margin_mean": 94.14254760742188, + "margin_dpo/margin_std": 142.54898071289062, + "step": 468 + }, + { + "KL/chosen_KL_mean": -172.53428649902344, + "KL/mean": -215.07339477539062, + "KL/rejected_KL_mean": -257.6125183105469, + "KL/std": 118.53016662597656, + "epoch": 0.708994708994709, + "fcm_dpo/beta": 0.0045113954693078995, + "fcm_dpo/delta": 0.016616512089967728, + "fcm_dpo/margin": 85.07823181152344, + "fcm_dpo/q_t": 0.4123454689979553, + "grad_norm": 13.645792961120605, + "learning_rate": 1.1931899453216697e-07, + "logits/chosen": 0.8709956407546997, + "logits/rejected": 0.855722188949585, + "logps/chosen": -231.59246826171875, + "logps/ref_chosen": -59.05818176269531, + "logps/ref_rejected": -75.67672729492188, + "logps/rejected": -333.28924560546875, + "loss": 1.1028, + "margin_dpo/margin_mean": 85.0782241821289, + "margin_dpo/margin_std": 114.00239562988281, + "step": 469 + }, + { + "KL/chosen_KL_mean": -160.6546630859375, + "KL/mean": -205.504150390625, + "KL/rejected_KL_mean": -250.3536376953125, + "KL/std": 114.10494995117188, + "epoch": 0.7105064247921391, + "fcm_dpo/beta": 0.004605274647474289, + "fcm_dpo/delta": -0.015301994979381561, + "fcm_dpo/margin": 89.69898986816406, + "fcm_dpo/q_t": 0.4052748680114746, + "grad_norm": 11.65384578704834, + "learning_rate": 1.1819363309737438e-07, + "logits/chosen": 0.8430336713790894, + "logits/rejected": 0.7940839529037476, + "logps/chosen": -208.5220947265625, + "logps/ref_chosen": -47.86743927001953, + "logps/ref_rejected": -65.96859741210938, + "logps/rejected": -316.32220458984375, + "loss": 1.0995, + "margin_dpo/margin_mean": 89.69898986816406, + "margin_dpo/margin_std": 129.3060760498047, + "step": 470 + }, + { + "KL/chosen_KL_mean": -158.75082397460938, + "KL/mean": -207.02197265625, + "KL/rejected_KL_mean": -255.29310607910156, + "KL/std": 111.86181640625, + "epoch": 0.7120181405895691, + "fcm_dpo/beta": 0.004474523477256298, + "fcm_dpo/delta": -0.03683408349752426, + "fcm_dpo/margin": 96.54228210449219, + "fcm_dpo/q_t": 0.3983868360519409, + "grad_norm": 12.488258361816406, + "learning_rate": 1.1707195857000215e-07, + "logits/chosen": 0.8283665180206299, + "logits/rejected": 0.7850099802017212, + "logps/chosen": -216.52867126464844, + "logps/ref_chosen": -57.777854919433594, + "logps/ref_rejected": -73.81172180175781, + "logps/rejected": -329.1048278808594, + "loss": 1.0719, + "margin_dpo/margin_mean": 96.54228210449219, + "margin_dpo/margin_std": 121.92526245117188, + "step": 471 + }, + { + "KL/chosen_KL_mean": -159.306884765625, + "KL/mean": -197.11712646484375, + "KL/rejected_KL_mean": -234.9273681640625, + "KL/std": 117.94495391845703, + "epoch": 0.7135298563869993, + "fcm_dpo/beta": 0.004620303399860859, + "fcm_dpo/delta": 0.054885368794202805, + "fcm_dpo/margin": 75.62045288085938, + "fcm_dpo/q_t": 0.4230712652206421, + "grad_norm": 12.124162673950195, + "learning_rate": 1.1595400232569768e-07, + "logits/chosen": 0.8315516710281372, + "logits/rejected": 0.7926985025405884, + "logps/chosen": -215.21556091308594, + "logps/ref_chosen": -55.908668518066406, + "logps/ref_rejected": -74.70294189453125, + "logps/rejected": -309.63031005859375, + "loss": 1.1772, + "margin_dpo/margin_mean": 75.62045288085938, + "margin_dpo/margin_std": 149.9696044921875, + "step": 472 + }, + { + "KL/chosen_KL_mean": -163.22857666015625, + "KL/mean": -215.5937042236328, + "KL/rejected_KL_mean": -267.95880126953125, + "KL/std": 143.45526123046875, + "epoch": 0.7150415721844293, + "fcm_dpo/beta": 0.004440706688910723, + "fcm_dpo/delta": -0.0759398564696312, + "fcm_dpo/margin": 104.73027038574219, + "fcm_dpo/q_t": 0.3982590436935425, + "grad_norm": 13.992444038391113, + "learning_rate": 1.1483979563610069e-07, + "logits/chosen": 0.9021942615509033, + "logits/rejected": 0.809353232383728, + "logps/chosen": -217.3894500732422, + "logps/ref_chosen": -54.16088104248047, + "logps/ref_rejected": -92.76789855957031, + "logps/rejected": -360.7267150878906, + "loss": 1.096, + "margin_dpo/margin_mean": 104.73027038574219, + "margin_dpo/margin_std": 166.75250244140625, + "step": 473 + }, + { + "KL/chosen_KL_mean": -165.42330932617188, + "KL/mean": -207.98435974121094, + "KL/rejected_KL_mean": -250.54539489746094, + "KL/std": 122.40956115722656, + "epoch": 0.7165532879818595, + "fcm_dpo/beta": 0.004457107279449701, + "fcm_dpo/delta": 0.02313932217657566, + "fcm_dpo/margin": 85.1220703125, + "fcm_dpo/q_t": 0.41587191820144653, + "grad_norm": 17.439899444580078, + "learning_rate": 1.1372936966796709e-07, + "logits/chosen": 0.9221115112304688, + "logits/rejected": 0.8645721673965454, + "logps/chosen": -212.10902404785156, + "logps/ref_chosen": -46.685707092285156, + "logps/ref_rejected": -71.44731903076172, + "logps/rejected": -321.9927062988281, + "loss": 1.1342, + "margin_dpo/margin_mean": 85.1220703125, + "margin_dpo/margin_std": 141.8304443359375, + "step": 474 + }, + { + "KL/chosen_KL_mean": -164.73910522460938, + "KL/mean": -222.14610290527344, + "KL/rejected_KL_mean": -279.5531005859375, + "KL/std": 123.50313568115234, + "epoch": 0.7180650037792895, + "fcm_dpo/beta": 0.004225746262818575, + "fcm_dpo/delta": -0.10140831023454666, + "fcm_dpo/margin": 114.81401062011719, + "fcm_dpo/q_t": 0.3880518078804016, + "grad_norm": 8.140756607055664, + "learning_rate": 1.126227554822985e-07, + "logits/chosen": 0.8333104848861694, + "logits/rejected": 0.7912643551826477, + "logps/chosen": -223.22640991210938, + "logps/ref_chosen": -58.4873046875, + "logps/ref_rejected": -87.00187683105469, + "logps/rejected": -366.55499267578125, + "loss": 1.0399, + "margin_dpo/margin_mean": 114.81401062011719, + "margin_dpo/margin_std": 140.57183837890625, + "step": 475 + }, + { + "KL/chosen_KL_mean": -198.56451416015625, + "KL/mean": -232.54180908203125, + "KL/rejected_KL_mean": -266.51910400390625, + "KL/std": 132.65438842773438, + "epoch": 0.7195767195767195, + "fcm_dpo/beta": 0.004227438010275364, + "fcm_dpo/delta": 0.02281678467988968, + "fcm_dpo/margin": 67.95459747314453, + "fcm_dpo/q_t": 0.43554818630218506, + "grad_norm": 13.2657470703125, + "learning_rate": 1.1151998403347243e-07, + "logits/chosen": 0.7603304386138916, + "logits/rejected": 0.764335036277771, + "logps/chosen": -273.9461364746094, + "logps/ref_chosen": -75.38162231445312, + "logps/ref_rejected": -76.99822235107422, + "logps/rejected": -343.517333984375, + "loss": 1.2024, + "margin_dpo/margin_mean": 67.95459747314453, + "margin_dpo/margin_std": 140.07485961914062, + "step": 476 + }, + { + "KL/chosen_KL_mean": -194.00865173339844, + "KL/mean": -236.6685791015625, + "KL/rejected_KL_mean": -279.3285217285156, + "KL/std": 133.24874877929688, + "epoch": 0.7210884353741497, + "fcm_dpo/beta": 0.004316464997828007, + "fcm_dpo/delta": 0.035700492560863495, + "fcm_dpo/margin": 85.31986999511719, + "fcm_dpo/q_t": 0.4191203713417053, + "grad_norm": 12.513143539428711, + "learning_rate": 1.1042108616837692e-07, + "logits/chosen": 0.8567191362380981, + "logits/rejected": 0.8166630268096924, + "logps/chosen": -255.08203125, + "logps/ref_chosen": -61.073387145996094, + "logps/ref_rejected": -81.34375, + "logps/rejected": -360.6722717285156, + "loss": 1.1779, + "margin_dpo/margin_mean": 85.31986236572266, + "margin_dpo/margin_std": 170.67071533203125, + "step": 477 + }, + { + "KL/chosen_KL_mean": -171.2495574951172, + "KL/mean": -202.14273071289062, + "KL/rejected_KL_mean": -233.03590393066406, + "KL/std": 106.54711151123047, + "epoch": 0.7226001511715797, + "fcm_dpo/beta": 0.004573522135615349, + "fcm_dpo/delta": 0.1288546621799469, + "fcm_dpo/margin": 61.78632736206055, + "fcm_dpo/q_t": 0.4368034899234772, + "grad_norm": 14.673408508300781, + "learning_rate": 1.0932609262554746e-07, + "logits/chosen": 0.8044695854187012, + "logits/rejected": 0.8161039352416992, + "logps/chosen": -228.4168701171875, + "logps/ref_chosen": -57.16731643676758, + "logps/ref_rejected": -53.30917739868164, + "logps/rejected": -286.3450927734375, + "loss": 1.2093, + "margin_dpo/margin_mean": 61.78633117675781, + "margin_dpo/margin_std": 129.24046325683594, + "step": 478 + }, + { + "KL/chosen_KL_mean": -177.13055419921875, + "KL/mean": -204.7392578125, + "KL/rejected_KL_mean": -232.34796142578125, + "KL/std": 110.43617248535156, + "epoch": 0.7241118669690099, + "fcm_dpo/beta": 0.004763299599289894, + "fcm_dpo/delta": 0.022982783615589142, + "fcm_dpo/margin": 55.2174072265625, + "fcm_dpo/q_t": 0.44107237458229065, + "grad_norm": 14.663484573364258, + "learning_rate": 1.0823503403430734e-07, + "logits/chosen": 0.8042382001876831, + "logits/rejected": 0.778433084487915, + "logps/chosen": -236.0438690185547, + "logps/ref_chosen": -58.91331481933594, + "logps/ref_rejected": -63.7403450012207, + "logps/rejected": -296.08831787109375, + "loss": 1.2346, + "margin_dpo/margin_mean": 55.2174072265625, + "margin_dpo/margin_std": 131.55545043945312, + "step": 479 + }, + { + "KL/chosen_KL_mean": -184.4405975341797, + "KL/mean": -227.0792236328125, + "KL/rejected_KL_mean": -269.71783447265625, + "KL/std": 115.04316711425781, + "epoch": 0.7256235827664399, + "fcm_dpo/beta": 0.004858708009123802, + "fcm_dpo/delta": -0.02168682962656021, + "fcm_dpo/margin": 85.27723693847656, + "fcm_dpo/q_t": 0.40630629658699036, + "grad_norm": 15.03400707244873, + "learning_rate": 1.0714794091391072e-07, + "logits/chosen": 0.7868234515190125, + "logits/rejected": 0.7834052443504333, + "logps/chosen": -247.2412109375, + "logps/ref_chosen": -62.80061340332031, + "logps/ref_rejected": -67.58859252929688, + "logps/rejected": -337.30645751953125, + "loss": 1.1233, + "margin_dpo/margin_mean": 85.27723693847656, + "margin_dpo/margin_std": 134.57638549804688, + "step": 480 + }, + { + "KL/chosen_KL_mean": -182.68031311035156, + "KL/mean": -222.99514770507812, + "KL/rejected_KL_mean": -263.3099670410156, + "KL/std": 121.83908081054688, + "epoch": 0.72713529856387, + "fcm_dpo/beta": 0.004709561821073294, + "fcm_dpo/delta": 0.022568443790078163, + "fcm_dpo/margin": 80.62967681884766, + "fcm_dpo/q_t": 0.4156482219696045, + "grad_norm": 12.423121452331543, + "learning_rate": 1.0606484367268906e-07, + "logits/chosen": 0.8025602102279663, + "logits/rejected": 0.8090294599533081, + "logps/chosen": -247.966796875, + "logps/ref_chosen": -65.28649139404297, + "logps/ref_rejected": -70.78668212890625, + "logps/rejected": -334.0966491699219, + "loss": 1.132, + "margin_dpo/margin_mean": 80.62968444824219, + "margin_dpo/margin_std": 133.42701721191406, + "step": 481 + }, + { + "KL/chosen_KL_mean": -201.8643798828125, + "KL/mean": -246.51490783691406, + "KL/rejected_KL_mean": -291.16546630859375, + "KL/std": 135.26222229003906, + "epoch": 0.7286470143613001, + "fcm_dpo/beta": 0.004750003106892109, + "fcm_dpo/delta": -0.02802390605211258, + "fcm_dpo/margin": 89.30106353759766, + "fcm_dpo/q_t": 0.41019535064697266, + "grad_norm": 15.123866081237793, + "learning_rate": 1.0498577260720048e-07, + "logits/chosen": 0.7807713747024536, + "logits/rejected": 0.6487196087837219, + "logps/chosen": -262.77056884765625, + "logps/ref_chosen": -60.906185150146484, + "logps/ref_rejected": -103.44656372070312, + "logps/rejected": -394.61199951171875, + "loss": 1.1465, + "margin_dpo/margin_mean": 89.30105590820312, + "margin_dpo/margin_std": 167.93521118164062, + "step": 482 + }, + { + "KL/chosen_KL_mean": -169.6586151123047, + "KL/mean": -220.8936309814453, + "KL/rejected_KL_mean": -272.128662109375, + "KL/std": 117.83233642578125, + "epoch": 0.7301587301587301, + "fcm_dpo/beta": 0.004558105953037739, + "fcm_dpo/delta": -0.07779208570718765, + "fcm_dpo/margin": 102.47002410888672, + "fcm_dpo/q_t": 0.39235061407089233, + "grad_norm": 13.63161849975586, + "learning_rate": 1.0391075790138232e-07, + "logits/chosen": 0.8855106830596924, + "logits/rejected": 0.7963603734970093, + "logps/chosen": -222.85061645507812, + "logps/ref_chosen": -53.192012786865234, + "logps/ref_rejected": -81.83927154541016, + "logps/rejected": -353.9678955078125, + "loss": 1.0606, + "margin_dpo/margin_mean": 102.47001647949219, + "margin_dpo/margin_std": 134.73834228515625, + "step": 483 + }, + { + "KL/chosen_KL_mean": -176.22354125976562, + "KL/mean": -208.54843139648438, + "KL/rejected_KL_mean": -240.87332153320312, + "KL/std": 116.13772583007812, + "epoch": 0.7316704459561603, + "fcm_dpo/beta": 0.004734124056994915, + "fcm_dpo/delta": 0.10121459513902664, + "fcm_dpo/margin": 64.64977264404297, + "fcm_dpo/q_t": 0.4296380281448364, + "grad_norm": 16.20368003845215, + "learning_rate": 1.0283982962570681e-07, + "logits/chosen": 0.8983777761459351, + "logits/rejected": 0.8746373057365417, + "logps/chosen": -233.9929962158203, + "logps/ref_chosen": -57.76945877075195, + "logps/ref_rejected": -71.6829833984375, + "logps/rejected": -312.5563049316406, + "loss": 1.159, + "margin_dpo/margin_mean": 64.64977264404297, + "margin_dpo/margin_std": 101.06908416748047, + "step": 484 + }, + { + "KL/chosen_KL_mean": -182.181396484375, + "KL/mean": -218.22222900390625, + "KL/rejected_KL_mean": -254.26309204101562, + "KL/std": 114.957763671875, + "epoch": 0.7331821617535903, + "fcm_dpo/beta": 0.004620219115167856, + "fcm_dpo/delta": -0.04711120203137398, + "fcm_dpo/margin": 72.08170318603516, + "fcm_dpo/q_t": 0.4228460192680359, + "grad_norm": 13.281914710998535, + "learning_rate": 1.0177301773633992e-07, + "logits/chosen": 0.9076398611068726, + "logits/rejected": 0.8928012847900391, + "logps/chosen": -238.81724548339844, + "logps/ref_chosen": -56.63584899902344, + "logps/ref_rejected": -70.85614013671875, + "logps/rejected": -325.1192321777344, + "loss": 1.1514, + "margin_dpo/margin_mean": 72.08171081542969, + "margin_dpo/margin_std": 115.27439880371094, + "step": 485 + }, + { + "KL/chosen_KL_mean": -207.69192504882812, + "KL/mean": -243.96141052246094, + "KL/rejected_KL_mean": -280.23089599609375, + "KL/std": 137.62863159179688, + "epoch": 0.7346938775510204, + "fcm_dpo/beta": 0.00475216843187809, + "fcm_dpo/delta": 0.0615973100066185, + "fcm_dpo/margin": 72.53897857666016, + "fcm_dpo/q_t": 0.42720019817352295, + "grad_norm": 11.114177703857422, + "learning_rate": 1.007103520743035e-07, + "logits/chosen": 0.8727377653121948, + "logits/rejected": 0.7796794176101685, + "logps/chosen": -264.0389404296875, + "logps/ref_chosen": -56.347023010253906, + "logps/ref_rejected": -85.97221374511719, + "logps/rejected": -366.203125, + "loss": 1.2038, + "margin_dpo/margin_mean": 72.53898620605469, + "margin_dpo/margin_std": 161.3694610595703, + "step": 486 + }, + { + "KL/chosen_KL_mean": -187.3951416015625, + "KL/mean": -228.60194396972656, + "KL/rejected_KL_mean": -269.80877685546875, + "KL/std": 126.41487121582031, + "epoch": 0.7362055933484505, + "fcm_dpo/beta": 0.004806404933333397, + "fcm_dpo/delta": 0.004405636340379715, + "fcm_dpo/margin": 82.41361999511719, + "fcm_dpo/q_t": 0.4161032438278198, + "grad_norm": 13.856200218200684, + "learning_rate": 9.965186236464046e-08, + "logits/chosen": 0.9156618118286133, + "logits/rejected": 0.8678491115570068, + "logps/chosen": -248.01235961914062, + "logps/ref_chosen": -60.617218017578125, + "logps/ref_rejected": -82.50975036621094, + "logps/rejected": -352.3184814453125, + "loss": 1.1419, + "margin_dpo/margin_mean": 82.41361236572266, + "margin_dpo/margin_std": 151.87709045410156, + "step": 487 + }, + { + "KL/chosen_KL_mean": -175.20208740234375, + "KL/mean": -221.89056396484375, + "KL/rejected_KL_mean": -268.57904052734375, + "KL/std": 118.11080932617188, + "epoch": 0.7377173091458806, + "fcm_dpo/beta": 0.004773295484483242, + "fcm_dpo/delta": -0.05380316078662872, + "fcm_dpo/margin": 93.37692260742188, + "fcm_dpo/q_t": 0.39976638555526733, + "grad_norm": 14.908041000366211, + "learning_rate": 9.859757821558337e-08, + "logits/chosen": 0.8313355445861816, + "logits/rejected": 0.792324423789978, + "logps/chosen": -238.31114196777344, + "logps/ref_chosen": -63.10905075073242, + "logps/ref_rejected": -82.49348449707031, + "logps/rejected": -351.072509765625, + "loss": 1.0806, + "margin_dpo/margin_mean": 93.3769302368164, + "margin_dpo/margin_std": 130.49966430664062, + "step": 488 + }, + { + "KL/chosen_KL_mean": -204.91387939453125, + "KL/mean": -233.1622314453125, + "KL/rejected_KL_mean": -261.41058349609375, + "KL/std": 130.28018188476562, + "epoch": 0.7392290249433107, + "fcm_dpo/beta": 0.0049452874809503555, + "fcm_dpo/delta": 0.13130411505699158, + "fcm_dpo/margin": 56.49671936035156, + "fcm_dpo/q_t": 0.4389078915119171, + "grad_norm": 13.527437210083008, + "learning_rate": 9.754752911772615e-08, + "logits/chosen": 0.8165801763534546, + "logits/rejected": 0.780007004737854, + "logps/chosen": -269.90283203125, + "logps/ref_chosen": -64.98896026611328, + "logps/ref_rejected": -84.39607238769531, + "logps/rejected": -345.806640625, + "loss": 1.2361, + "margin_dpo/margin_mean": 56.49671936035156, + "margin_dpo/margin_std": 137.77053833007812, + "step": 489 + }, + { + "KL/chosen_KL_mean": -184.7069091796875, + "KL/mean": -218.8941192626953, + "KL/rejected_KL_mean": -253.08131408691406, + "KL/std": 124.6688232421875, + "epoch": 0.7407407407407407, + "fcm_dpo/beta": 0.005108346696943045, + "fcm_dpo/delta": 0.05644252896308899, + "fcm_dpo/margin": 68.3744125366211, + "fcm_dpo/q_t": 0.4248546361923218, + "grad_norm": 12.403369903564453, + "learning_rate": 9.650174444319956e-08, + "logits/chosen": 0.8969284892082214, + "logits/rejected": 0.8864274024963379, + "logps/chosen": -246.61566162109375, + "logps/ref_chosen": -61.90874481201172, + "logps/ref_rejected": -70.58566284179688, + "logps/rejected": -323.6669921875, + "loss": 1.2101, + "margin_dpo/margin_mean": 68.3744125366211, + "margin_dpo/margin_std": 154.39389038085938, + "step": 490 + }, + { + "KL/chosen_KL_mean": -178.02613830566406, + "KL/mean": -212.85516357421875, + "KL/rejected_KL_mean": -247.6842041015625, + "KL/std": 112.87677001953125, + "epoch": 0.7422524565381708, + "fcm_dpo/beta": 0.005029967520385981, + "fcm_dpo/delta": -0.05832719802856445, + "fcm_dpo/margin": 69.65803527832031, + "fcm_dpo/q_t": 0.41990724205970764, + "grad_norm": 10.252985000610352, + "learning_rate": 9.546025344484868e-08, + "logits/chosen": 0.7992293834686279, + "logits/rejected": 0.745780348777771, + "logps/chosen": -233.50184631347656, + "logps/ref_chosen": -55.47570037841797, + "logps/ref_rejected": -78.70318603515625, + "logps/rejected": -326.38739013671875, + "loss": 1.1552, + "margin_dpo/margin_mean": 69.65803527832031, + "margin_dpo/margin_std": 120.66893005371094, + "step": 491 + }, + { + "KL/chosen_KL_mean": -211.10516357421875, + "KL/mean": -243.54579162597656, + "KL/rejected_KL_mean": -275.98638916015625, + "KL/std": 131.36009216308594, + "epoch": 0.7437641723356009, + "fcm_dpo/beta": 0.005244538653641939, + "fcm_dpo/delta": 0.06441338360309601, + "fcm_dpo/margin": 64.88124084472656, + "fcm_dpo/q_t": 0.4269523024559021, + "grad_norm": 15.789043426513672, + "learning_rate": 9.442308525541589e-08, + "logits/chosen": 0.8516812920570374, + "logits/rejected": 0.7955601215362549, + "logps/chosen": -278.39154052734375, + "logps/ref_chosen": -67.28638458251953, + "logps/ref_rejected": -82.78628540039062, + "logps/rejected": -358.772705078125, + "loss": 1.2185, + "margin_dpo/margin_mean": 64.88124084472656, + "margin_dpo/margin_std": 148.1044921875, + "step": 492 + }, + { + "KL/chosen_KL_mean": -176.27764892578125, + "KL/mean": -222.05245971679688, + "KL/rejected_KL_mean": -267.8272705078125, + "KL/std": 127.64932250976562, + "epoch": 0.745275888133031, + "fcm_dpo/beta": 0.005175878759473562, + "fcm_dpo/delta": -0.08929616212844849, + "fcm_dpo/margin": 91.54963684082031, + "fcm_dpo/q_t": 0.3954174220561981, + "grad_norm": 15.408149719238281, + "learning_rate": 9.339026888672468e-08, + "logits/chosen": 0.8074703812599182, + "logits/rejected": 0.7508785128593445, + "logps/chosen": -232.20513916015625, + "logps/ref_chosen": -55.92750549316406, + "logps/ref_rejected": -79.12149810791016, + "logps/rejected": -346.9487609863281, + "loss": 1.0922, + "margin_dpo/margin_mean": 91.54963684082031, + "margin_dpo/margin_std": 139.12684631347656, + "step": 493 + }, + { + "KL/chosen_KL_mean": -181.6630096435547, + "KL/mean": -223.94744873046875, + "KL/rejected_KL_mean": -266.23193359375, + "KL/std": 128.83995056152344, + "epoch": 0.7467876039304611, + "fcm_dpo/beta": 0.004889290779829025, + "fcm_dpo/delta": -0.015794701874256134, + "fcm_dpo/margin": 84.56891632080078, + "fcm_dpo/q_t": 0.4098696708679199, + "grad_norm": 14.83998966217041, + "learning_rate": 9.236183322886945e-08, + "logits/chosen": 0.7640923857688904, + "logits/rejected": 0.7272478342056274, + "logps/chosen": -249.61712646484375, + "logps/ref_chosen": -67.95410919189453, + "logps/ref_rejected": -90.50865173339844, + "logps/rejected": -356.7405700683594, + "loss": 1.1521, + "margin_dpo/margin_mean": 84.56890869140625, + "margin_dpo/margin_std": 161.0829620361328, + "step": 494 + }, + { + "KL/chosen_KL_mean": -177.00015258789062, + "KL/mean": -209.45477294921875, + "KL/rejected_KL_mean": -241.90940856933594, + "KL/std": 125.46236419677734, + "epoch": 0.7482993197278912, + "fcm_dpo/beta": 0.004900210537016392, + "fcm_dpo/delta": -0.005768109112977982, + "fcm_dpo/margin": 64.90924835205078, + "fcm_dpo/q_t": 0.4349347949028015, + "grad_norm": 16.218822479248047, + "learning_rate": 9.133780704940594e-08, + "logits/chosen": 0.8546461462974548, + "logits/rejected": 0.8084453344345093, + "logps/chosen": -229.6256103515625, + "logps/ref_chosen": -52.62546157836914, + "logps/ref_rejected": -72.06781005859375, + "logps/rejected": -313.97723388671875, + "loss": 1.2211, + "margin_dpo/margin_mean": 64.90924835205078, + "margin_dpo/margin_std": 156.70840454101562, + "step": 495 + }, + { + "KL/chosen_KL_mean": -191.11280822753906, + "KL/mean": -238.58676147460938, + "KL/rejected_KL_mean": -286.0606994628906, + "KL/std": 138.51930236816406, + "epoch": 0.7498110355253212, + "fcm_dpo/beta": 0.004659120924770832, + "fcm_dpo/delta": -0.054637569934129715, + "fcm_dpo/margin": 94.94786834716797, + "fcm_dpo/q_t": 0.40979450941085815, + "grad_norm": 11.95200252532959, + "learning_rate": 9.031821899254797e-08, + "logits/chosen": 0.8415661454200745, + "logits/rejected": 0.7441533803939819, + "logps/chosen": -248.71014404296875, + "logps/ref_chosen": -57.597320556640625, + "logps/ref_rejected": -94.36127471923828, + "logps/rejected": -380.4219665527344, + "loss": 1.1355, + "margin_dpo/margin_mean": 94.94786834716797, + "margin_dpo/margin_std": 179.43215942382812, + "step": 496 + }, + { + "KL/chosen_KL_mean": -197.67959594726562, + "KL/mean": -241.09596252441406, + "KL/rejected_KL_mean": -284.5123291015625, + "KL/std": 124.52940368652344, + "epoch": 0.7513227513227513, + "fcm_dpo/beta": 0.004687286913394928, + "fcm_dpo/delta": -0.008316351100802422, + "fcm_dpo/margin": 86.83271789550781, + "fcm_dpo/q_t": 0.406663715839386, + "grad_norm": 13.023819923400879, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": 0.853161096572876, + "logits/rejected": 0.8242659568786621, + "logps/chosen": -270.46954345703125, + "logps/ref_chosen": -72.78994750976562, + "logps/ref_rejected": -89.48483276367188, + "logps/rejected": -373.99713134765625, + "loss": 1.116, + "margin_dpo/margin_mean": 86.83271789550781, + "margin_dpo/margin_std": 136.6311492919922, + "step": 497 + }, + { + "KL/chosen_KL_mean": -175.44488525390625, + "KL/mean": -222.20950317382812, + "KL/rejected_KL_mean": -268.97412109375, + "KL/std": 117.231201171875, + "epoch": 0.7528344671201814, + "fcm_dpo/beta": 0.00461503816768527, + "fcm_dpo/delta": -0.036642152816057205, + "fcm_dpo/margin": 93.52923583984375, + "fcm_dpo/q_t": 0.39883777499198914, + "grad_norm": 13.756954193115234, + "learning_rate": 8.829247120198563e-08, + "logits/chosen": 0.8330202102661133, + "logits/rejected": 0.8214312791824341, + "logps/chosen": -243.81060791015625, + "logps/ref_chosen": -68.36572265625, + "logps/ref_rejected": -71.28846740722656, + "logps/rejected": -340.2626037597656, + "loss": 1.0839, + "margin_dpo/margin_mean": 93.52923583984375, + "margin_dpo/margin_std": 129.33668518066406, + "step": 498 + }, + { + "KL/chosen_KL_mean": -179.2896728515625, + "KL/mean": -229.2539825439453, + "KL/rejected_KL_mean": -279.2182922363281, + "KL/std": 132.14337158203125, + "epoch": 0.7543461829176115, + "fcm_dpo/beta": 0.004479332827031612, + "fcm_dpo/delta": -0.055337462574243546, + "fcm_dpo/margin": 99.9286117553711, + "fcm_dpo/q_t": 0.40216517448425293, + "grad_norm": 19.308687210083008, + "learning_rate": 8.728636813280163e-08, + "logits/chosen": 0.8368034362792969, + "logits/rejected": 0.7836736440658569, + "logps/chosen": -241.19850158691406, + "logps/ref_chosen": -61.90882873535156, + "logps/ref_rejected": -91.9411392211914, + "logps/rejected": -371.159423828125, + "loss": 1.1267, + "margin_dpo/margin_mean": 99.92861938476562, + "margin_dpo/margin_std": 174.06784057617188, + "step": 499 + }, + { + "KL/chosen_KL_mean": -188.59982299804688, + "KL/mean": -225.63168334960938, + "KL/rejected_KL_mean": -262.66351318359375, + "KL/std": 115.21099853515625, + "epoch": 0.7558578987150416, + "fcm_dpo/beta": 0.00457107275724411, + "fcm_dpo/delta": 0.06856130063533783, + "fcm_dpo/margin": 74.0636978149414, + "fcm_dpo/q_t": 0.42126649618148804, + "grad_norm": 14.0758695602417, + "learning_rate": 8.628481651367875e-08, + "logits/chosen": 0.8072834014892578, + "logits/rejected": 0.8198223114013672, + "logps/chosen": -258.8256530761719, + "logps/ref_chosen": -70.225830078125, + "logps/ref_rejected": -71.72203063964844, + "logps/rejected": -334.38555908203125, + "loss": 1.1835, + "margin_dpo/margin_mean": 74.0636978149414, + "margin_dpo/margin_std": 146.86541748046875, + "step": 500 + }, + { + "KL/chosen_KL_mean": -187.39126586914062, + "KL/mean": -222.43211364746094, + "KL/rejected_KL_mean": -257.47296142578125, + "KL/std": 122.28995513916016, + "epoch": 0.7573696145124716, + "fcm_dpo/beta": 0.004816969856619835, + "fcm_dpo/delta": 0.06878426671028137, + "fcm_dpo/margin": 70.08174133300781, + "fcm_dpo/q_t": 0.42239677906036377, + "grad_norm": 11.63576889038086, + "learning_rate": 8.528784436016878e-08, + "logits/chosen": 0.833807110786438, + "logits/rejected": 0.8403280973434448, + "logps/chosen": -251.9900665283203, + "logps/ref_chosen": -64.59880828857422, + "logps/ref_rejected": -70.59329223632812, + "logps/rejected": -328.0662841796875, + "loss": 1.132, + "margin_dpo/margin_mean": 70.08173370361328, + "margin_dpo/margin_std": 99.08871459960938, + "step": 501 + }, + { + "KL/chosen_KL_mean": -186.89634704589844, + "KL/mean": -225.32411193847656, + "KL/rejected_KL_mean": -263.75189208984375, + "KL/std": 133.69595336914062, + "epoch": 0.7588813303099018, + "fcm_dpo/beta": 0.004946129396557808, + "fcm_dpo/delta": 0.021091684699058533, + "fcm_dpo/margin": 76.85551452636719, + "fcm_dpo/q_t": 0.41427573561668396, + "grad_norm": 16.61263656616211, + "learning_rate": 8.4295479559726e-08, + "logits/chosen": 0.8703242540359497, + "logits/rejected": 0.8291156888008118, + "logps/chosen": -252.36297607421875, + "logps/ref_chosen": -65.46662902832031, + "logps/ref_rejected": -90.22233581542969, + "logps/rejected": -353.9742126464844, + "loss": 1.1302, + "margin_dpo/margin_mean": 76.85551452636719, + "margin_dpo/margin_std": 123.59288024902344, + "step": 502 + }, + { + "KL/chosen_KL_mean": -165.91510009765625, + "KL/mean": -204.1113739013672, + "KL/rejected_KL_mean": -242.3076171875, + "KL/std": 110.12651062011719, + "epoch": 0.7603930461073318, + "fcm_dpo/beta": 0.004937716759741306, + "fcm_dpo/delta": 0.02568972110748291, + "fcm_dpo/margin": 76.39253997802734, + "fcm_dpo/q_t": 0.41301050782203674, + "grad_norm": 12.290018081665039, + "learning_rate": 8.330774987092712e-08, + "logits/chosen": 0.8349542617797852, + "logits/rejected": 0.8423024415969849, + "logps/chosen": -217.74986267089844, + "logps/ref_chosen": -51.83476257324219, + "logps/ref_rejected": -57.62522506713867, + "logps/rejected": -299.932861328125, + "loss": 1.1346, + "margin_dpo/margin_mean": 76.39253997802734, + "margin_dpo/margin_std": 124.44419860839844, + "step": 503 + }, + { + "KL/chosen_KL_mean": -174.50894165039062, + "KL/mean": -224.36419677734375, + "KL/rejected_KL_mean": -274.2194519042969, + "KL/std": 112.46851348876953, + "epoch": 0.7619047619047619, + "fcm_dpo/beta": 0.0048708608373999596, + "fcm_dpo/delta": -0.10153862833976746, + "fcm_dpo/margin": 99.71051025390625, + "fcm_dpo/q_t": 0.3862113058567047, + "grad_norm": 11.172093391418457, + "learning_rate": 8.232468292269479e-08, + "logits/chosen": 0.8066504001617432, + "logits/rejected": 0.7978000044822693, + "logps/chosen": -243.16014099121094, + "logps/ref_chosen": -68.65119934082031, + "logps/ref_rejected": -77.91394805908203, + "logps/rejected": -352.1334228515625, + "loss": 1.0413, + "margin_dpo/margin_mean": 99.71051025390625, + "margin_dpo/margin_std": 116.26593017578125, + "step": 504 + }, + { + "KL/chosen_KL_mean": -184.58160400390625, + "KL/mean": -219.14266967773438, + "KL/rejected_KL_mean": -253.7037353515625, + "KL/std": 134.02723693847656, + "epoch": 0.763416477702192, + "fcm_dpo/beta": 0.004590325523167849, + "fcm_dpo/delta": -0.03364703059196472, + "fcm_dpo/margin": 69.12216186523438, + "fcm_dpo/q_t": 0.4291911721229553, + "grad_norm": 13.124581336975098, + "learning_rate": 8.134630621352483e-08, + "logits/chosen": 0.8404581546783447, + "logits/rejected": 0.8074555397033691, + "logps/chosen": -244.5804443359375, + "logps/ref_chosen": -59.99884796142578, + "logps/ref_rejected": -76.88048553466797, + "logps/rejected": -330.584228515625, + "loss": 1.2016, + "margin_dpo/margin_mean": 69.12216186523438, + "margin_dpo/margin_std": 145.76710510253906, + "step": 505 + }, + { + "KL/chosen_KL_mean": -185.93222045898438, + "KL/mean": -222.56521606445312, + "KL/rejected_KL_mean": -259.1982116699219, + "KL/std": 118.15262603759766, + "epoch": 0.764928193499622, + "fcm_dpo/beta": 0.00469227135181427, + "fcm_dpo/delta": 0.06280513852834702, + "fcm_dpo/margin": 73.26598358154297, + "fcm_dpo/q_t": 0.42215216159820557, + "grad_norm": 12.451192855834961, + "learning_rate": 8.037264711071698e-08, + "logits/chosen": 0.8122513890266418, + "logits/rejected": 0.8038109540939331, + "logps/chosen": -256.0035400390625, + "logps/ref_chosen": -70.07130432128906, + "logps/ref_rejected": -82.03775024414062, + "logps/rejected": -341.2359619140625, + "loss": 1.1876, + "margin_dpo/margin_mean": 73.2659912109375, + "margin_dpo/margin_std": 149.51535034179688, + "step": 506 + }, + { + "KL/chosen_KL_mean": -195.21424865722656, + "KL/mean": -238.96438598632812, + "KL/rejected_KL_mean": -282.71453857421875, + "KL/std": 134.91099548339844, + "epoch": 0.7664399092970522, + "fcm_dpo/beta": 0.004667514935135841, + "fcm_dpo/delta": -0.011594630777835846, + "fcm_dpo/margin": 87.50030517578125, + "fcm_dpo/q_t": 0.41374701261520386, + "grad_norm": 13.628171920776367, + "learning_rate": 7.940373284960933e-08, + "logits/chosen": 0.852962076663971, + "logits/rejected": 0.8136993646621704, + "logps/chosen": -267.2212829589844, + "logps/ref_chosen": -72.00703430175781, + "logps/ref_rejected": -93.94987487792969, + "logps/rejected": -376.6644287109375, + "loss": 1.1472, + "margin_dpo/margin_mean": 87.50030517578125, + "margin_dpo/margin_std": 159.53916931152344, + "step": 507 + }, + { + "KL/chosen_KL_mean": -180.30194091796875, + "KL/mean": -231.80496215820312, + "KL/rejected_KL_mean": -283.3079833984375, + "KL/std": 141.48910522460938, + "epoch": 0.7679516250944822, + "fcm_dpo/beta": 0.004642902873456478, + "fcm_dpo/delta": -0.0925033912062645, + "fcm_dpo/margin": 103.00605773925781, + "fcm_dpo/q_t": 0.3941517472267151, + "grad_norm": 15.925118446350098, + "learning_rate": 7.843959053281663e-08, + "logits/chosen": 0.8314013481140137, + "logits/rejected": 0.7309713959693909, + "logps/chosen": -240.5218505859375, + "logps/ref_chosen": -60.21992492675781, + "logps/ref_rejected": -95.9200668334961, + "logps/rejected": -379.2280578613281, + "loss": 1.0892, + "margin_dpo/margin_mean": 103.00605773925781, + "margin_dpo/margin_std": 155.87930297851562, + "step": 508 + }, + { + "KL/chosen_KL_mean": -190.84176635742188, + "KL/mean": -227.35723876953125, + "KL/rejected_KL_mean": -263.8727111816406, + "KL/std": 123.76275634765625, + "epoch": 0.7694633408919124, + "fcm_dpo/beta": 0.004593648016452789, + "fcm_dpo/delta": 0.07191221415996552, + "fcm_dpo/margin": 73.03093719482422, + "fcm_dpo/q_t": 0.42444539070129395, + "grad_norm": 15.941633224487305, + "learning_rate": 7.748024712947204e-08, + "logits/chosen": 0.8122048377990723, + "logits/rejected": 0.7954943776130676, + "logps/chosen": -257.1119384765625, + "logps/ref_chosen": -66.27017211914062, + "logps/ref_rejected": -71.73065185546875, + "logps/rejected": -335.60333251953125, + "loss": 1.1895, + "margin_dpo/margin_mean": 73.03093719482422, + "margin_dpo/margin_std": 148.45220947265625, + "step": 509 + }, + { + "KL/chosen_KL_mean": -189.88351440429688, + "KL/mean": -238.0792999267578, + "KL/rejected_KL_mean": -286.27508544921875, + "KL/std": 139.2322540283203, + "epoch": 0.7709750566893424, + "fcm_dpo/beta": 0.004529876634478569, + "fcm_dpo/delta": -0.04380728304386139, + "fcm_dpo/margin": 96.39158630371094, + "fcm_dpo/q_t": 0.40820297598838806, + "grad_norm": 14.016731262207031, + "learning_rate": 7.652572947447272e-08, + "logits/chosen": 0.9248769283294678, + "logits/rejected": 0.8472373485565186, + "logps/chosen": -243.4283905029297, + "logps/ref_chosen": -53.54487609863281, + "logps/ref_rejected": -91.36648559570312, + "logps/rejected": -377.6415710449219, + "loss": 1.1419, + "margin_dpo/margin_mean": 96.39158630371094, + "margin_dpo/margin_std": 180.8556365966797, + "step": 510 + }, + { + "KL/chosen_KL_mean": -177.66510009765625, + "KL/mean": -231.65658569335938, + "KL/rejected_KL_mean": -285.6480712890625, + "KL/std": 121.31975555419922, + "epoch": 0.7724867724867724, + "fcm_dpo/beta": 0.004389429464936256, + "fcm_dpo/delta": -0.08617927134037018, + "fcm_dpo/margin": 107.98297119140625, + "fcm_dpo/q_t": 0.38881558179855347, + "grad_norm": 14.997942924499512, + "learning_rate": 7.557606426772961e-08, + "logits/chosen": 0.8680216670036316, + "logits/rejected": 0.8205208778381348, + "logps/chosen": -233.50949096679688, + "logps/ref_chosen": -55.844383239746094, + "logps/ref_rejected": -86.49819946289062, + "logps/rejected": -372.1462707519531, + "loss": 1.0434, + "margin_dpo/margin_mean": 107.98297119140625, + "margin_dpo/margin_std": 127.46925354003906, + "step": 511 + }, + { + "KL/chosen_KL_mean": -185.30795288085938, + "KL/mean": -217.4866485595703, + "KL/rejected_KL_mean": -249.66534423828125, + "KL/std": 113.47346496582031, + "epoch": 0.7739984882842026, + "fcm_dpo/beta": 0.004385386127978563, + "fcm_dpo/delta": 0.023245226591825485, + "fcm_dpo/margin": 64.3573989868164, + "fcm_dpo/q_t": 0.43406805396080017, + "grad_norm": 19.79949378967285, + "learning_rate": 7.463127807341966e-08, + "logits/chosen": 0.7447441816329956, + "logits/rejected": 0.7479703426361084, + "logps/chosen": -246.96099853515625, + "logps/ref_chosen": -61.653038024902344, + "logps/ref_rejected": -72.83148193359375, + "logps/rejected": -322.496826171875, + "loss": 1.2125, + "margin_dpo/margin_mean": 64.3573989868164, + "margin_dpo/margin_std": 137.24069213867188, + "step": 512 + }, + { + "KL/chosen_KL_mean": -166.38235473632812, + "KL/mean": -207.5938720703125, + "KL/rejected_KL_mean": -248.80538940429688, + "KL/std": 113.1106948852539, + "epoch": 0.7755102040816326, + "fcm_dpo/beta": 0.00444423221051693, + "fcm_dpo/delta": 0.03789909556508064, + "fcm_dpo/margin": 82.42303466796875, + "fcm_dpo/q_t": 0.4171144366264343, + "grad_norm": 10.346599578857422, + "learning_rate": 7.369139731924401e-08, + "logits/chosen": 0.9161933660507202, + "logits/rejected": 0.8713808655738831, + "logps/chosen": -217.23492431640625, + "logps/ref_chosen": -50.85256576538086, + "logps/ref_rejected": -69.21754455566406, + "logps/rejected": -318.02294921875, + "loss": 1.1225, + "margin_dpo/margin_mean": 82.42303466796875, + "margin_dpo/margin_std": 122.04438018798828, + "step": 513 + }, + { + "KL/chosen_KL_mean": -187.51318359375, + "KL/mean": -235.22531127929688, + "KL/rejected_KL_mean": -282.93743896484375, + "KL/std": 125.52702331542969, + "epoch": 0.7770219198790628, + "fcm_dpo/beta": 0.004477824550122023, + "fcm_dpo/delta": -0.03195594251155853, + "fcm_dpo/margin": 95.42424011230469, + "fcm_dpo/q_t": 0.40270817279815674, + "grad_norm": 15.198481559753418, + "learning_rate": 7.275644829568747e-08, + "logits/chosen": 0.8606992959976196, + "logits/rejected": 0.837975263595581, + "logps/chosen": -256.89813232421875, + "logps/ref_chosen": -69.38493347167969, + "logps/ref_rejected": -83.32447814941406, + "logps/rejected": -366.26190185546875, + "loss": 1.0998, + "margin_dpo/margin_mean": 95.42424774169922, + "margin_dpo/margin_std": 142.98733520507812, + "step": 514 + }, + { + "KL/chosen_KL_mean": -194.24365234375, + "KL/mean": -232.44851684570312, + "KL/rejected_KL_mean": -270.65338134765625, + "KL/std": 117.28639221191406, + "epoch": 0.7785336356764928, + "fcm_dpo/beta": 0.004507084377110004, + "fcm_dpo/delta": 0.062156371772289276, + "fcm_dpo/margin": 76.40972900390625, + "fcm_dpo/q_t": 0.4212613105773926, + "grad_norm": 12.018317222595215, + "learning_rate": 7.182645715528435e-08, + "logits/chosen": 0.8571234345436096, + "logits/rejected": 0.7873523235321045, + "logps/chosen": -247.93069458007812, + "logps/ref_chosen": -53.687034606933594, + "logps/ref_rejected": -83.59614562988281, + "logps/rejected": -354.24951171875, + "loss": 1.1574, + "margin_dpo/margin_mean": 76.40972900390625, + "margin_dpo/margin_std": 133.1450653076172, + "step": 515 + }, + { + "KL/chosen_KL_mean": -168.87564086914062, + "KL/mean": -206.1563262939453, + "KL/rejected_KL_mean": -243.43699645996094, + "KL/std": 115.80433654785156, + "epoch": 0.780045351473923, + "fcm_dpo/beta": 0.004723469726741314, + "fcm_dpo/delta": 0.05215982347726822, + "fcm_dpo/margin": 74.56133270263672, + "fcm_dpo/q_t": 0.41921383142471313, + "grad_norm": 15.109283447265625, + "learning_rate": 7.090144991188568e-08, + "logits/chosen": 0.8263188600540161, + "logits/rejected": 0.7975659370422363, + "logps/chosen": -225.77737426757812, + "logps/ref_chosen": -56.9017219543457, + "logps/ref_rejected": -67.83477783203125, + "logps/rejected": -311.27178955078125, + "loss": 1.1633, + "margin_dpo/margin_mean": 74.56134033203125, + "margin_dpo/margin_std": 136.02789306640625, + "step": 516 + }, + { + "KL/chosen_KL_mean": -201.57992553710938, + "KL/mean": -228.9405517578125, + "KL/rejected_KL_mean": -256.3011779785156, + "KL/std": 123.93075561523438, + "epoch": 0.781557067271353, + "fcm_dpo/beta": 0.004781331401318312, + "fcm_dpo/delta": 0.04728348180651665, + "fcm_dpo/margin": 54.72124481201172, + "fcm_dpo/q_t": 0.4420250654220581, + "grad_norm": 13.745712280273438, + "learning_rate": 6.998145243993284e-08, + "logits/chosen": 0.8844262361526489, + "logits/rejected": 0.8897521495819092, + "logps/chosen": -263.3550720214844, + "logps/ref_chosen": -61.775142669677734, + "logps/ref_rejected": -62.88270950317383, + "logps/rejected": -319.18389892578125, + "loss": 1.2287, + "margin_dpo/margin_mean": 54.72125244140625, + "margin_dpo/margin_std": 127.82027435302734, + "step": 517 + }, + { + "KL/chosen_KL_mean": -173.06509399414062, + "KL/mean": -213.97628784179688, + "KL/rejected_KL_mean": -254.88751220703125, + "KL/std": 121.97323608398438, + "epoch": 0.783068783068783, + "fcm_dpo/beta": 0.004860128741711378, + "fcm_dpo/delta": 0.0026370445266366005, + "fcm_dpo/margin": 81.82240295410156, + "fcm_dpo/q_t": 0.41280555725097656, + "grad_norm": 13.351316452026367, + "learning_rate": 6.906649047373245e-08, + "logits/chosen": 0.8314996957778931, + "logits/rejected": 0.799196720123291, + "logps/chosen": -235.09033203125, + "logps/ref_chosen": -62.02523422241211, + "logps/ref_rejected": -79.06085205078125, + "logps/rejected": -333.9483642578125, + "loss": 1.13, + "margin_dpo/margin_mean": 81.82240295410156, + "margin_dpo/margin_std": 138.0740966796875, + "step": 518 + }, + { + "KL/chosen_KL_mean": -204.05413818359375, + "KL/mean": -227.598388671875, + "KL/rejected_KL_mean": -251.1426239013672, + "KL/std": 121.79615020751953, + "epoch": 0.7845804988662132, + "fcm_dpo/beta": 0.00496459798887372, + "fcm_dpo/delta": 0.07236956804990768, + "fcm_dpo/margin": 47.088470458984375, + "fcm_dpo/q_t": 0.44760170578956604, + "grad_norm": 19.102073669433594, + "learning_rate": 6.815658960673781e-08, + "logits/chosen": 0.8614056706428528, + "logits/rejected": 0.8238772749900818, + "logps/chosen": -265.6605224609375, + "logps/ref_chosen": -61.60636901855469, + "logps/ref_rejected": -74.50727844238281, + "logps/rejected": -325.64990234375, + "loss": 1.3117, + "margin_dpo/margin_mean": 47.088478088378906, + "margin_dpo/margin_std": 159.3134765625, + "step": 519 + }, + { + "KL/chosen_KL_mean": -189.42742919921875, + "KL/mean": -226.57359313964844, + "KL/rejected_KL_mean": -263.71978759765625, + "KL/std": 123.97309112548828, + "epoch": 0.7860922146636432, + "fcm_dpo/beta": 0.005040885414928198, + "fcm_dpo/delta": 0.02684013918042183, + "fcm_dpo/margin": 74.29235076904297, + "fcm_dpo/q_t": 0.4172334671020508, + "grad_norm": 16.04875373840332, + "learning_rate": 6.725177529083209e-08, + "logits/chosen": 0.9040594100952148, + "logits/rejected": 0.8583179116249084, + "logps/chosen": -252.30084228515625, + "logps/ref_chosen": -62.87343215942383, + "logps/ref_rejected": -76.505615234375, + "logps/rejected": -340.2253723144531, + "loss": 1.1434, + "margin_dpo/margin_mean": 74.29235076904297, + "margin_dpo/margin_std": 126.32000732421875, + "step": 520 + }, + { + "KL/chosen_KL_mean": -187.01785278320312, + "KL/mean": -239.00657653808594, + "KL/rejected_KL_mean": -290.9953308105469, + "KL/std": 124.35540008544922, + "epoch": 0.7876039304610734, + "fcm_dpo/beta": 0.00489568617194891, + "fcm_dpo/delta": -0.1282760053873062, + "fcm_dpo/margin": 103.97747802734375, + "fcm_dpo/q_t": 0.3860055208206177, + "grad_norm": 11.902304649353027, + "learning_rate": 6.63520728356167e-08, + "logits/chosen": 0.7507075071334839, + "logits/rejected": 0.691694974899292, + "logps/chosen": -251.22451782226562, + "logps/ref_chosen": -64.20668029785156, + "logps/ref_rejected": -92.28083038330078, + "logps/rejected": -383.2761535644531, + "loss": 1.0477, + "margin_dpo/margin_mean": 103.97747802734375, + "margin_dpo/margin_std": 142.90042114257812, + "step": 521 + }, + { + "KL/chosen_KL_mean": -189.99844360351562, + "KL/mean": -219.89682006835938, + "KL/rejected_KL_mean": -249.79522705078125, + "KL/std": 121.7298583984375, + "epoch": 0.7891156462585034, + "fcm_dpo/beta": 0.004985436797142029, + "fcm_dpo/delta": 0.1123264878988266, + "fcm_dpo/margin": 59.796783447265625, + "fcm_dpo/q_t": 0.43511706590652466, + "grad_norm": 14.604763984680176, + "learning_rate": 6.545750740770336e-08, + "logits/chosen": 0.8380166292190552, + "logits/rejected": 0.8317359089851379, + "logps/chosen": -248.3681640625, + "logps/ref_chosen": -58.369720458984375, + "logps/ref_rejected": -68.79248046875, + "logps/rejected": -318.58770751953125, + "loss": 1.2558, + "margin_dpo/margin_mean": 59.796783447265625, + "margin_dpo/margin_std": 157.91360473632812, + "step": 522 + }, + { + "KL/chosen_KL_mean": -195.73423767089844, + "KL/mean": -234.60708618164062, + "KL/rejected_KL_mean": -273.47991943359375, + "KL/std": 125.57742309570312, + "epoch": 0.7906273620559335, + "fcm_dpo/beta": 0.005116622429341078, + "fcm_dpo/delta": 0.002492782659828663, + "fcm_dpo/margin": 77.7457046508789, + "fcm_dpo/q_t": 0.4087350368499756, + "grad_norm": 18.966442108154297, + "learning_rate": 6.456810403001012e-08, + "logits/chosen": 0.8302779793739319, + "logits/rejected": 0.728390634059906, + "logps/chosen": -261.4474792480469, + "logps/ref_chosen": -65.71324157714844, + "logps/ref_rejected": -91.98896789550781, + "logps/rejected": -365.4689025878906, + "loss": 1.1459, + "margin_dpo/margin_mean": 77.74569702148438, + "margin_dpo/margin_std": 139.64659118652344, + "step": 523 + }, + { + "KL/chosen_KL_mean": -173.21641540527344, + "KL/mean": -211.4966278076172, + "KL/rejected_KL_mean": -249.77685546875, + "KL/std": 121.24716186523438, + "epoch": 0.7921390778533636, + "fcm_dpo/beta": 0.005164645612239838, + "fcm_dpo/delta": 0.004730384796857834, + "fcm_dpo/margin": 76.56043243408203, + "fcm_dpo/q_t": 0.4102930724620819, + "grad_norm": 15.195032119750977, + "learning_rate": 6.368388758106134e-08, + "logits/chosen": 0.7700381875038147, + "logits/rejected": 0.7530121803283691, + "logps/chosen": -249.56765747070312, + "logps/ref_chosen": -76.35124969482422, + "logps/ref_rejected": -89.96072387695312, + "logps/rejected": -339.7375793457031, + "loss": 1.1218, + "margin_dpo/margin_mean": 76.5604248046875, + "margin_dpo/margin_std": 122.09478759765625, + "step": 524 + }, + { + "KL/chosen_KL_mean": -193.4317626953125, + "KL/mean": -225.12921142578125, + "KL/rejected_KL_mean": -256.82666015625, + "KL/std": 117.6169204711914, + "epoch": 0.7936507936507936, + "fcm_dpo/beta": 0.0052588703110814095, + "fcm_dpo/delta": 0.07416895776987076, + "fcm_dpo/margin": 63.3948974609375, + "fcm_dpo/q_t": 0.42921310663223267, + "grad_norm": 19.115665435791016, + "learning_rate": 6.280488279429185e-08, + "logits/chosen": 0.7093310952186584, + "logits/rejected": 0.7094483971595764, + "logps/chosen": -268.92755126953125, + "logps/ref_chosen": -75.49578857421875, + "logps/ref_rejected": -84.04852294921875, + "logps/rejected": -340.87518310546875, + "loss": 1.2039, + "margin_dpo/margin_mean": 63.394893646240234, + "margin_dpo/margin_std": 140.00247192382812, + "step": 525 + }, + { + "KL/chosen_KL_mean": -198.68052673339844, + "KL/mean": -229.0758819580078, + "KL/rejected_KL_mean": -259.47125244140625, + "KL/std": 122.75657653808594, + "epoch": 0.7951625094482238, + "fcm_dpo/beta": 0.005194986704736948, + "fcm_dpo/delta": -0.052471913397312164, + "fcm_dpo/margin": 60.79069900512695, + "fcm_dpo/q_t": 0.42842093110084534, + "grad_norm": 14.487711906433105, + "learning_rate": 6.193111425735515e-08, + "logits/chosen": 0.8522295951843262, + "logits/rejected": 0.7942419648170471, + "logps/chosen": -259.9729309082031, + "logps/ref_chosen": -61.29241943359375, + "logps/ref_rejected": -82.47763061523438, + "logps/rejected": -341.9488525390625, + "loss": 1.1995, + "margin_dpo/margin_mean": 60.79069900512695, + "margin_dpo/margin_std": 125.36583709716797, + "step": 526 + }, + { + "KL/chosen_KL_mean": -207.10394287109375, + "KL/mean": -233.09361267089844, + "KL/rejected_KL_mean": -259.0832824707031, + "KL/std": 121.70289611816406, + "epoch": 0.7966742252456538, + "fcm_dpo/beta": 0.005217268131673336, + "fcm_dpo/delta": 0.014235904440283775, + "fcm_dpo/margin": 51.979347229003906, + "fcm_dpo/q_t": 0.44098371267318726, + "grad_norm": 15.067662239074707, + "learning_rate": 6.106260641143546e-08, + "logits/chosen": 0.9322826266288757, + "logits/rejected": 0.8576953411102295, + "logps/chosen": -268.5765686035156, + "logps/ref_chosen": -61.472625732421875, + "logps/ref_rejected": -90.52831268310547, + "logps/rejected": -349.6116027832031, + "loss": 1.2546, + "margin_dpo/margin_mean": 51.979347229003906, + "margin_dpo/margin_std": 137.49215698242188, + "step": 527 + }, + { + "KL/chosen_KL_mean": -183.95230102539062, + "KL/mean": -214.3380584716797, + "KL/rejected_KL_mean": -244.72381591796875, + "KL/std": 114.09736633300781, + "epoch": 0.7981859410430839, + "fcm_dpo/beta": 0.005392709746956825, + "fcm_dpo/delta": 0.08037621527910233, + "fcm_dpo/margin": 60.771522521972656, + "fcm_dpo/q_t": 0.4303974509239197, + "grad_norm": 17.312894821166992, + "learning_rate": 6.019938355056422e-08, + "logits/chosen": 0.8145349621772766, + "logits/rejected": 0.7549326419830322, + "logps/chosen": -242.7443084716797, + "logps/ref_chosen": -58.792015075683594, + "logps/ref_rejected": -71.82516479492188, + "logps/rejected": -316.5489807128906, + "loss": 1.2365, + "margin_dpo/margin_mean": 60.771522521972656, + "margin_dpo/margin_std": 148.80494689941406, + "step": 528 + }, + { + "KL/chosen_KL_mean": -178.44883728027344, + "KL/mean": -234.34661865234375, + "KL/rejected_KL_mean": -290.2444152832031, + "KL/std": 112.45913696289062, + "epoch": 0.799697656840514, + "fcm_dpo/beta": 0.005035213194787502, + "fcm_dpo/delta": -0.1965881735086441, + "fcm_dpo/margin": 111.79557800292969, + "fcm_dpo/q_t": 0.3671392798423767, + "grad_norm": 15.271794319152832, + "learning_rate": 5.934146982094049e-08, + "logits/chosen": 0.7759064435958862, + "logits/rejected": 0.7353440523147583, + "logps/chosen": -233.51980590820312, + "logps/ref_chosen": -55.070960998535156, + "logps/ref_rejected": -75.44007873535156, + "logps/rejected": -365.68450927734375, + "loss": 0.9889, + "margin_dpo/margin_mean": 111.79557800292969, + "margin_dpo/margin_std": 122.36137390136719, + "step": 529 + }, + { + "KL/chosen_KL_mean": -176.23171997070312, + "KL/mean": -214.2334747314453, + "KL/rejected_KL_mean": -252.23526000976562, + "KL/std": 122.73760986328125, + "epoch": 0.8012093726379441, + "fcm_dpo/beta": 0.004925016779452562, + "fcm_dpo/delta": 0.028898127377033234, + "fcm_dpo/margin": 76.00357055664062, + "fcm_dpo/q_t": 0.4183453917503357, + "grad_norm": 16.061771392822266, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": 0.8739749193191528, + "logits/rejected": 0.836665153503418, + "logps/chosen": -232.97552490234375, + "logps/ref_chosen": -56.743812561035156, + "logps/ref_rejected": -76.6692123413086, + "logps/rejected": -328.90447998046875, + "loss": 1.1416, + "margin_dpo/margin_mean": 76.00357055664062, + "margin_dpo/margin_std": 129.30343627929688, + "step": 530 + }, + { + "KL/chosen_KL_mean": -176.384521484375, + "KL/mean": -215.5994873046875, + "KL/rejected_KL_mean": -254.81443786621094, + "KL/std": 120.86744689941406, + "epoch": 0.8027210884353742, + "fcm_dpo/beta": 0.0049905176274478436, + "fcm_dpo/delta": 0.009740887209773064, + "fcm_dpo/margin": 78.42990112304688, + "fcm_dpo/q_t": 0.41260451078414917, + "grad_norm": 13.00410270690918, + "learning_rate": 5.7641665597021435e-08, + "logits/chosen": 0.8380810618400574, + "logits/rejected": 0.7674177885055542, + "logps/chosen": -227.5009765625, + "logps/ref_chosen": -51.116455078125, + "logps/ref_rejected": -79.52884674072266, + "logps/rejected": -334.34326171875, + "loss": 1.1312, + "margin_dpo/margin_mean": 78.42990112304688, + "margin_dpo/margin_std": 130.77659606933594, + "step": 531 + }, + { + "KL/chosen_KL_mean": -206.09271240234375, + "KL/mean": -245.57955932617188, + "KL/rejected_KL_mean": -285.06646728515625, + "KL/std": 119.67216491699219, + "epoch": 0.8042328042328042, + "fcm_dpo/beta": 0.005016631446778774, + "fcm_dpo/delta": 0.004318161401897669, + "fcm_dpo/margin": 78.97373962402344, + "fcm_dpo/q_t": 0.4110661447048187, + "grad_norm": 14.44060230255127, + "learning_rate": 5.679982264990424e-08, + "logits/chosen": 0.8174869418144226, + "logits/rejected": 0.774748682975769, + "logps/chosen": -264.3726501464844, + "logps/ref_chosen": -58.279945373535156, + "logps/ref_rejected": -78.05426788330078, + "logps/rejected": -363.1207275390625, + "loss": 1.1321, + "margin_dpo/margin_mean": 78.97373962402344, + "margin_dpo/margin_std": 132.41571044921875, + "step": 532 + }, + { + "KL/chosen_KL_mean": -148.63015747070312, + "KL/mean": -195.14443969726562, + "KL/rejected_KL_mean": -241.65870666503906, + "KL/std": 127.76698303222656, + "epoch": 0.8057445200302343, + "fcm_dpo/beta": 0.004902138374745846, + "fcm_dpo/delta": -0.06478457897901535, + "fcm_dpo/margin": 93.02857208251953, + "fcm_dpo/q_t": 0.39741477370262146, + "grad_norm": 14.071173667907715, + "learning_rate": 5.596338392706076e-08, + "logits/chosen": 0.9038737416267395, + "logits/rejected": 0.8549951314926147, + "logps/chosen": -205.04815673828125, + "logps/ref_chosen": -56.41801071166992, + "logps/ref_rejected": -73.89324951171875, + "logps/rejected": -315.55194091796875, + "loss": 1.0814, + "margin_dpo/margin_mean": 93.02857971191406, + "margin_dpo/margin_std": 134.64431762695312, + "step": 533 + }, + { + "KL/chosen_KL_mean": -185.22732543945312, + "KL/mean": -220.9058837890625, + "KL/rejected_KL_mean": -256.58441162109375, + "KL/std": 115.06756591796875, + "epoch": 0.8072562358276644, + "fcm_dpo/beta": 0.004894358105957508, + "fcm_dpo/delta": 0.0565091148018837, + "fcm_dpo/margin": 71.35710144042969, + "fcm_dpo/q_t": 0.423191636800766, + "grad_norm": 13.81146240234375, + "learning_rate": 5.513237282548033e-08, + "logits/chosen": 0.816210150718689, + "logits/rejected": 0.7883972525596619, + "logps/chosen": -245.97601318359375, + "logps/ref_chosen": -60.748687744140625, + "logps/ref_rejected": -73.8623046875, + "logps/rejected": -330.44671630859375, + "loss": 1.1783, + "margin_dpo/margin_mean": 71.35710144042969, + "margin_dpo/margin_std": 140.3214111328125, + "step": 534 + }, + { + "KL/chosen_KL_mean": -196.07479858398438, + "KL/mean": -230.17123413085938, + "KL/rejected_KL_mean": -264.26763916015625, + "KL/std": 123.80078887939453, + "epoch": 0.8087679516250945, + "fcm_dpo/beta": 0.005145275965332985, + "fcm_dpo/delta": 0.054210808128118515, + "fcm_dpo/margin": 68.1928482055664, + "fcm_dpo/q_t": 0.4239533543586731, + "grad_norm": 15.887540817260742, + "learning_rate": 5.430681259032957e-08, + "logits/chosen": 0.7688905000686646, + "logits/rejected": 0.7201669812202454, + "logps/chosen": -257.71221923828125, + "logps/ref_chosen": -61.637413024902344, + "logps/ref_rejected": -80.93138885498047, + "logps/rejected": -345.19903564453125, + "loss": 1.1858, + "margin_dpo/margin_mean": 68.1928482055664, + "margin_dpo/margin_std": 138.06951904296875, + "step": 535 + }, + { + "KL/chosen_KL_mean": -169.49693298339844, + "KL/mean": -222.1195068359375, + "KL/rejected_KL_mean": -274.7420959472656, + "KL/std": 123.30923461914062, + "epoch": 0.8102796674225246, + "fcm_dpo/beta": 0.004823877010494471, + "fcm_dpo/delta": -0.12901368737220764, + "fcm_dpo/margin": 105.24514770507812, + "fcm_dpo/q_t": 0.3823744058609009, + "grad_norm": 10.788253784179688, + "learning_rate": 5.3486726314303175e-08, + "logits/chosen": 0.8770267963409424, + "logits/rejected": 0.8094578981399536, + "logps/chosen": -221.3859100341797, + "logps/ref_chosen": -51.88897705078125, + "logps/ref_rejected": -73.34864044189453, + "logps/rejected": -348.0907287597656, + "loss": 1.0211, + "margin_dpo/margin_mean": 105.24514770507812, + "margin_dpo/margin_std": 122.67505645751953, + "step": 536 + }, + { + "KL/chosen_KL_mean": -192.197509765625, + "KL/mean": -238.49530029296875, + "KL/rejected_KL_mean": -284.7930603027344, + "KL/std": 133.47552490234375, + "epoch": 0.8117913832199547, + "fcm_dpo/beta": 0.004660390317440033, + "fcm_dpo/delta": -0.03684820234775543, + "fcm_dpo/margin": 92.59556579589844, + "fcm_dpo/q_t": 0.4039597511291504, + "grad_norm": 13.52630615234375, + "learning_rate": 5.267213693697695e-08, + "logits/chosen": 0.9239789843559265, + "logits/rejected": 0.8453102707862854, + "logps/chosen": -246.44613647460938, + "logps/ref_chosen": -54.248619079589844, + "logps/ref_rejected": -94.94343566894531, + "logps/rejected": -379.73651123046875, + "loss": 1.1087, + "margin_dpo/margin_mean": 92.59556579589844, + "margin_dpo/margin_std": 146.90977478027344, + "step": 537 + }, + { + "KL/chosen_KL_mean": -195.64366149902344, + "KL/mean": -239.24957275390625, + "KL/rejected_KL_mean": -282.8554992675781, + "KL/std": 131.49520874023438, + "epoch": 0.8133030990173847, + "fcm_dpo/beta": 0.004660367965698242, + "fcm_dpo/delta": -0.00734717445448041, + "fcm_dpo/margin": 87.21183776855469, + "fcm_dpo/q_t": 0.407853901386261, + "grad_norm": 12.92613697052002, + "learning_rate": 5.1863067244167144e-08, + "logits/chosen": 0.8298152685165405, + "logits/rejected": 0.8131669759750366, + "logps/chosen": -265.7371826171875, + "logps/ref_chosen": -70.09353637695312, + "logps/ref_rejected": -79.49833679199219, + "logps/rejected": -362.35382080078125, + "loss": 1.1085, + "margin_dpo/margin_mean": 87.21183776855469, + "margin_dpo/margin_std": 133.052001953125, + "step": 538 + }, + { + "KL/chosen_KL_mean": -199.86004638671875, + "KL/mean": -234.01394653320312, + "KL/rejected_KL_mean": -268.1678466796875, + "KL/std": 120.2940902709961, + "epoch": 0.8148148148148148, + "fcm_dpo/beta": 0.004803318530321121, + "fcm_dpo/delta": 0.0798788070678711, + "fcm_dpo/margin": 68.30778503417969, + "fcm_dpo/q_t": 0.43010759353637695, + "grad_norm": 14.349104881286621, + "learning_rate": 5.105953986729195e-08, + "logits/chosen": 0.8133977651596069, + "logits/rejected": 0.7414849400520325, + "logps/chosen": -261.791748046875, + "logps/ref_chosen": -61.93169403076172, + "logps/ref_rejected": -84.08946228027344, + "logps/rejected": -352.25732421875, + "loss": 1.1762, + "margin_dpo/margin_mean": 68.30778503417969, + "margin_dpo/margin_std": 132.4424591064453, + "step": 539 + }, + { + "KL/chosen_KL_mean": -183.73110961914062, + "KL/mean": -237.5785369873047, + "KL/rejected_KL_mean": -291.42596435546875, + "KL/std": 140.208251953125, + "epoch": 0.8163265306122449, + "fcm_dpo/beta": 0.004663660190999508, + "fcm_dpo/delta": -0.11989345401525497, + "fcm_dpo/margin": 107.69487762451172, + "fcm_dpo/q_t": 0.3844473659992218, + "grad_norm": 12.575929641723633, + "learning_rate": 5.026157728273966e-08, + "logits/chosen": 0.8718093633651733, + "logits/rejected": 0.7981296181678772, + "logps/chosen": -246.4353485107422, + "logps/ref_chosen": -62.704254150390625, + "logps/ref_rejected": -95.63597106933594, + "logps/rejected": -387.06195068359375, + "loss": 1.0385, + "margin_dpo/margin_mean": 107.69487762451172, + "margin_dpo/margin_std": 134.9456787109375, + "step": 540 + }, + { + "KL/chosen_KL_mean": -179.12994384765625, + "KL/mean": -221.66107177734375, + "KL/rejected_KL_mean": -264.1922302246094, + "KL/std": 119.99224853515625, + "epoch": 0.817838246409675, + "fcm_dpo/beta": 0.004524096846580505, + "fcm_dpo/delta": 0.016618637368083, + "fcm_dpo/margin": 85.06229400634766, + "fcm_dpo/q_t": 0.41030341386795044, + "grad_norm": 11.988428115844727, + "learning_rate": 4.9469201811239035e-08, + "logits/chosen": 0.8793637752532959, + "logits/rejected": 0.9092428684234619, + "logps/chosen": -241.61077880859375, + "logps/ref_chosen": -62.48084259033203, + "logps/ref_rejected": -57.55541229248047, + "logps/rejected": -321.74761962890625, + "loss": 1.1052, + "margin_dpo/margin_mean": 85.06229400634766, + "margin_dpo/margin_std": 115.75038146972656, + "step": 541 + }, + { + "KL/chosen_KL_mean": -155.74496459960938, + "KL/mean": -203.70249938964844, + "KL/rejected_KL_mean": -251.66004943847656, + "KL/std": 112.92686462402344, + "epoch": 0.8193499622071051, + "fcm_dpo/beta": 0.004484962671995163, + "fcm_dpo/delta": -0.03508245199918747, + "fcm_dpo/margin": 95.91510772705078, + "fcm_dpo/q_t": 0.40033650398254395, + "grad_norm": 11.718820571899414, + "learning_rate": 4.868243561723534e-08, + "logits/chosen": 0.9004342555999756, + "logits/rejected": 0.8623563051223755, + "logps/chosen": -205.19984436035156, + "logps/ref_chosen": -49.454891204833984, + "logps/ref_rejected": -65.33275604248047, + "logps/rejected": -316.9927978515625, + "loss": 1.0898, + "margin_dpo/margin_mean": 95.91511535644531, + "margin_dpo/margin_std": 136.8139190673828, + "step": 542 + }, + { + "KL/chosen_KL_mean": -170.90618896484375, + "KL/mean": -218.6575164794922, + "KL/rejected_KL_mean": -266.4088134765625, + "KL/std": 120.44759368896484, + "epoch": 0.8208616780045351, + "fcm_dpo/beta": 0.0044562616385519505, + "fcm_dpo/delta": -0.029431413859128952, + "fcm_dpo/margin": 95.50263977050781, + "fcm_dpo/q_t": 0.40224945545196533, + "grad_norm": 10.733229637145996, + "learning_rate": 4.790130070827028e-08, + "logits/chosen": 0.8563024997711182, + "logits/rejected": 0.7859128713607788, + "logps/chosen": -222.00704956054688, + "logps/ref_chosen": -51.100860595703125, + "logps/ref_rejected": -76.06130981445312, + "logps/rejected": -342.47015380859375, + "loss": 1.0909, + "margin_dpo/margin_mean": 95.50263977050781, + "margin_dpo/margin_std": 133.79779052734375, + "step": 543 + }, + { + "KL/chosen_KL_mean": -183.34523010253906, + "KL/mean": -237.28253173828125, + "KL/rejected_KL_mean": -291.21978759765625, + "KL/std": 133.13616943359375, + "epoch": 0.8223733938019653, + "fcm_dpo/beta": 0.004260823130607605, + "fcm_dpo/delta": -0.06947409361600876, + "fcm_dpo/margin": 107.87458038330078, + "fcm_dpo/q_t": 0.39583128690719604, + "grad_norm": 13.217146873474121, + "learning_rate": 4.7125818934366454e-08, + "logits/chosen": 0.8630880117416382, + "logits/rejected": 0.7988213300704956, + "logps/chosen": -243.62246704101562, + "logps/ref_chosen": -60.2772331237793, + "logps/ref_rejected": -88.40553283691406, + "logps/rejected": -379.6253356933594, + "loss": 1.0805, + "margin_dpo/margin_mean": 107.87457275390625, + "margin_dpo/margin_std": 156.6991424560547, + "step": 544 + }, + { + "KL/chosen_KL_mean": -197.96246337890625, + "KL/mean": -227.96221923828125, + "KL/rejected_KL_mean": -257.9619445800781, + "KL/std": 122.77239990234375, + "epoch": 0.8238851095993953, + "fcm_dpo/beta": 0.0045236810110509396, + "fcm_dpo/delta": 0.13886934518814087, + "fcm_dpo/margin": 59.999481201171875, + "fcm_dpo/q_t": 0.4393478035926819, + "grad_norm": 13.92799186706543, + "learning_rate": 4.635601198741607e-08, + "logits/chosen": 0.8469263315200806, + "logits/rejected": 0.7915807366371155, + "logps/chosen": -259.57769775390625, + "logps/ref_chosen": -61.61524963378906, + "logps/ref_rejected": -78.71266174316406, + "logps/rejected": -336.67462158203125, + "loss": 1.2181, + "margin_dpo/margin_mean": 59.99948501586914, + "margin_dpo/margin_std": 132.14329528808594, + "step": 545 + }, + { + "KL/chosen_KL_mean": -189.22254943847656, + "KL/mean": -221.83804321289062, + "KL/rejected_KL_mean": -254.45355224609375, + "KL/std": 116.31465911865234, + "epoch": 0.8253968253968254, + "fcm_dpo/beta": 0.004792468622326851, + "fcm_dpo/delta": 0.09647992998361588, + "fcm_dpo/margin": 65.2309799194336, + "fcm_dpo/q_t": 0.4293987452983856, + "grad_norm": 18.117361068725586, + "learning_rate": 4.559190140057428e-08, + "logits/chosen": 0.957390308380127, + "logits/rejected": 0.95533686876297, + "logps/chosen": -248.5358123779297, + "logps/ref_chosen": -59.313262939453125, + "logps/ref_rejected": -64.73631286621094, + "logps/rejected": -319.1898498535156, + "loss": 1.209, + "margin_dpo/margin_mean": 65.23097229003906, + "margin_dpo/margin_std": 142.85653686523438, + "step": 546 + }, + { + "KL/chosen_KL_mean": -165.288330078125, + "KL/mean": -215.5997772216797, + "KL/rejected_KL_mean": -265.9112243652344, + "KL/std": 121.85418701171875, + "epoch": 0.8269085411942555, + "fcm_dpo/beta": 0.004634900949895382, + "fcm_dpo/delta": -0.07973036170005798, + "fcm_dpo/margin": 100.62291717529297, + "fcm_dpo/q_t": 0.3919479250907898, + "grad_norm": 12.286870002746582, + "learning_rate": 4.483350854765672e-08, + "logits/chosen": 0.7986713647842407, + "logits/rejected": 0.7496638298034668, + "logps/chosen": -220.26507568359375, + "logps/ref_chosen": -54.97674560546875, + "logps/ref_rejected": -75.35922241210938, + "logps/rejected": -341.27044677734375, + "loss": 1.0699, + "margin_dpo/margin_mean": 100.6229248046875, + "margin_dpo/margin_std": 139.28582763671875, + "step": 547 + }, + { + "KL/chosen_KL_mean": -196.86660766601562, + "KL/mean": -226.04827880859375, + "KL/rejected_KL_mean": -255.2299346923828, + "KL/std": 120.6140365600586, + "epoch": 0.8284202569916855, + "fcm_dpo/beta": 0.004941076040267944, + "fcm_dpo/delta": 0.12047231942415237, + "fcm_dpo/margin": 58.363319396972656, + "fcm_dpo/q_t": 0.43659114837646484, + "grad_norm": 16.297290802001953, + "learning_rate": 4.4080854642541826e-08, + "logits/chosen": 0.8177128434181213, + "logits/rejected": 0.7706683278083801, + "logps/chosen": -260.0772705078125, + "logps/ref_chosen": -63.21067428588867, + "logps/ref_rejected": -81.23347473144531, + "logps/rejected": -336.4634094238281, + "loss": 1.2194, + "margin_dpo/margin_mean": 58.36331558227539, + "margin_dpo/margin_std": 132.25997924804688, + "step": 548 + }, + { + "KL/chosen_KL_mean": -194.89645385742188, + "KL/mean": -231.360595703125, + "KL/rejected_KL_mean": -267.8247375488281, + "KL/std": 134.2811737060547, + "epoch": 0.8299319727891157, + "fcm_dpo/beta": 0.005007403437048197, + "fcm_dpo/delta": 0.03835400193929672, + "fcm_dpo/margin": 72.92826080322266, + "fcm_dpo/q_t": 0.4223543405532837, + "grad_norm": 18.85927963256836, + "learning_rate": 4.333396073857723e-08, + "logits/chosen": 0.9243029952049255, + "logits/rejected": 0.8670613765716553, + "logps/chosen": -259.16998291015625, + "logps/ref_chosen": -64.27351379394531, + "logps/ref_rejected": -92.31663513183594, + "logps/rejected": -360.141357421875, + "loss": 1.199, + "margin_dpo/margin_mean": 72.92826080322266, + "margin_dpo/margin_std": 159.06471252441406, + "step": 549 + }, + { + "KL/chosen_KL_mean": -193.56687927246094, + "KL/mean": -220.8571319580078, + "KL/rejected_KL_mean": -248.14736938476562, + "KL/std": 117.43412017822266, + "epoch": 0.8314436885865457, + "fcm_dpo/beta": 0.005119511857628822, + "fcm_dpo/delta": 0.004839625675231218, + "fcm_dpo/margin": 54.58049392700195, + "fcm_dpo/q_t": 0.43776458501815796, + "grad_norm": 15.661591529846191, + "learning_rate": 4.259284772799099e-08, + "logits/chosen": 0.8754514455795288, + "logits/rejected": 0.8566372990608215, + "logps/chosen": -249.7973175048828, + "logps/ref_chosen": -56.230438232421875, + "logps/ref_rejected": -62.59788513183594, + "logps/rejected": -310.7452697753906, + "loss": 1.2314, + "margin_dpo/margin_mean": 54.58049392700195, + "margin_dpo/margin_std": 128.6197509765625, + "step": 550 + }, + { + "KL/chosen_KL_mean": -201.9175262451172, + "KL/mean": -235.73777770996094, + "KL/rejected_KL_mean": -269.55804443359375, + "KL/std": 127.38159942626953, + "epoch": 0.8329554043839759, + "fcm_dpo/beta": 0.0052591776475310326, + "fcm_dpo/delta": 0.0491047166287899, + "fcm_dpo/margin": 67.64051055908203, + "fcm_dpo/q_t": 0.42484039068222046, + "grad_norm": 14.79029369354248, + "learning_rate": 4.1857536341307176e-08, + "logits/chosen": 0.8869715929031372, + "logits/rejected": 0.8607189655303955, + "logps/chosen": -269.66473388671875, + "logps/ref_chosen": -67.74720764160156, + "logps/ref_rejected": -87.04285430908203, + "logps/rejected": -356.60089111328125, + "loss": 1.1578, + "margin_dpo/margin_mean": 67.64051818847656, + "margin_dpo/margin_std": 126.78274536132812, + "step": 551 + }, + { + "KL/chosen_KL_mean": -186.17919921875, + "KL/mean": -225.56101989746094, + "KL/rejected_KL_mean": -264.94281005859375, + "KL/std": 130.92601013183594, + "epoch": 0.8344671201814059, + "fcm_dpo/beta": 0.005316844675689936, + "fcm_dpo/delta": -0.023466050624847412, + "fcm_dpo/margin": 78.76361083984375, + "fcm_dpo/q_t": 0.4041834771633148, + "grad_norm": 18.497468948364258, + "learning_rate": 4.112804714676593e-08, + "logits/chosen": 0.851292073726654, + "logits/rejected": 0.8060963153839111, + "logps/chosen": -249.10546875, + "logps/ref_chosen": -62.92625427246094, + "logps/ref_rejected": -82.98365783691406, + "logps/rejected": -347.92645263671875, + "loss": 1.1122, + "margin_dpo/margin_mean": 78.76361083984375, + "margin_dpo/margin_std": 122.3577651977539, + "step": 552 + }, + { + "KL/chosen_KL_mean": -201.0130615234375, + "KL/mean": -239.95428466796875, + "KL/rejected_KL_mean": -278.8955078125, + "KL/std": 127.23973083496094, + "epoch": 0.8359788359788359, + "fcm_dpo/beta": 0.005192439071834087, + "fcm_dpo/delta": -0.0050092036835849285, + "fcm_dpo/margin": 77.88245391845703, + "fcm_dpo/q_t": 0.41371631622314453, + "grad_norm": 14.231207847595215, + "learning_rate": 4.0404400549748144e-08, + "logits/chosen": 0.838943362236023, + "logits/rejected": 0.7557328939437866, + "logps/chosen": -257.0515441894531, + "logps/ref_chosen": -56.038490295410156, + "logps/ref_rejected": -84.48454284667969, + "logps/rejected": -363.3800354003906, + "loss": 1.158, + "margin_dpo/margin_mean": 77.8824462890625, + "margin_dpo/margin_std": 150.22286987304688, + "step": 553 + }, + { + "KL/chosen_KL_mean": -182.27084350585938, + "KL/mean": -227.0911865234375, + "KL/rejected_KL_mean": -271.91156005859375, + "KL/std": 118.93246459960938, + "epoch": 0.8374905517762661, + "fcm_dpo/beta": 0.0050364104099571705, + "fcm_dpo/delta": -0.05972806736826897, + "fcm_dpo/margin": 89.64070129394531, + "fcm_dpo/q_t": 0.39440518617630005, + "grad_norm": 14.413543701171875, + "learning_rate": 3.968661679220467e-08, + "logits/chosen": 0.8107480406761169, + "logits/rejected": 0.8082457780838013, + "logps/chosen": -246.80142211914062, + "logps/ref_chosen": -64.53059387207031, + "logps/ref_rejected": -71.2155990600586, + "logps/rejected": -343.12713623046875, + "loss": 1.0842, + "margin_dpo/margin_mean": 89.64070129394531, + "margin_dpo/margin_std": 128.74749755859375, + "step": 554 + }, + { + "KL/chosen_KL_mean": -196.64315795898438, + "KL/mean": -230.64242553710938, + "KL/rejected_KL_mean": -264.6416931152344, + "KL/std": 117.89613342285156, + "epoch": 0.8390022675736961, + "fcm_dpo/beta": 0.005218766164034605, + "fcm_dpo/delta": 0.04598980396986008, + "fcm_dpo/margin": 67.9985580444336, + "fcm_dpo/q_t": 0.4178510904312134, + "grad_norm": 15.634132385253906, + "learning_rate": 3.89747159520904e-08, + "logits/chosen": 0.8837528228759766, + "logits/rejected": 0.8687556982040405, + "logps/chosen": -263.2950744628906, + "logps/ref_chosen": -66.65191650390625, + "logps/ref_rejected": -68.6667251586914, + "logps/rejected": -333.30841064453125, + "loss": 1.1975, + "margin_dpo/margin_mean": 67.9985580444336, + "margin_dpo/margin_std": 140.62289428710938, + "step": 555 + }, + { + "KL/chosen_KL_mean": -193.26910400390625, + "KL/mean": -228.2431640625, + "KL/rejected_KL_mean": -263.21722412109375, + "KL/std": 122.90667724609375, + "epoch": 0.8405139833711263, + "fcm_dpo/beta": 0.005200220271945, + "fcm_dpo/delta": 0.040658943355083466, + "fcm_dpo/margin": 69.94815063476562, + "fcm_dpo/q_t": 0.42058825492858887, + "grad_norm": 14.569718360900879, + "learning_rate": 3.826871794280192e-08, + "logits/chosen": 0.8939093351364136, + "logits/rejected": 0.8580869436264038, + "logps/chosen": -246.10147094726562, + "logps/ref_chosen": -52.832366943359375, + "logps/ref_rejected": -64.49044036865234, + "logps/rejected": -327.7076721191406, + "loss": 1.173, + "margin_dpo/margin_mean": 69.94815826416016, + "margin_dpo/margin_std": 135.2369842529297, + "step": 556 + }, + { + "KL/chosen_KL_mean": -191.77627563476562, + "KL/mean": -241.3262939453125, + "KL/rejected_KL_mean": -290.87628173828125, + "KL/std": 122.63204193115234, + "epoch": 0.8420256991685563, + "fcm_dpo/beta": 0.004956225864589214, + "fcm_dpo/delta": -0.10954894870519638, + "fcm_dpo/margin": 99.10000610351562, + "fcm_dpo/q_t": 0.38677504658699036, + "grad_norm": 12.53800106048584, + "learning_rate": 3.756864251262143e-08, + "logits/chosen": 0.9151580929756165, + "logits/rejected": 0.8559887409210205, + "logps/chosen": -246.81227111816406, + "logps/ref_chosen": -55.03598403930664, + "logps/ref_rejected": -75.80644989013672, + "logps/rejected": -366.6827392578125, + "loss": 1.0388, + "margin_dpo/margin_mean": 99.10000610351562, + "margin_dpo/margin_std": 120.31071472167969, + "step": 557 + }, + { + "KL/chosen_KL_mean": -180.4897003173828, + "KL/mean": -234.2262420654297, + "KL/rejected_KL_mean": -287.9627685546875, + "KL/std": 130.8162078857422, + "epoch": 0.8435374149659864, + "fcm_dpo/beta": 0.00461628008633852, + "fcm_dpo/delta": -0.11640501767396927, + "fcm_dpo/margin": 107.4730453491211, + "fcm_dpo/q_t": 0.38629502058029175, + "grad_norm": 10.66162395477295, + "learning_rate": 3.687450924416341e-08, + "logits/chosen": 0.8702242970466614, + "logits/rejected": 0.8275398015975952, + "logps/chosen": -243.716064453125, + "logps/ref_chosen": -63.226348876953125, + "logps/ref_rejected": -91.46881866455078, + "logps/rejected": -379.43157958984375, + "loss": 1.0436, + "margin_dpo/margin_mean": 107.4730453491211, + "margin_dpo/margin_std": 138.13763427734375, + "step": 558 + }, + { + "KL/chosen_KL_mean": -184.31173706054688, + "KL/mean": -231.5626678466797, + "KL/rejected_KL_mean": -278.8135986328125, + "KL/std": 130.44276428222656, + "epoch": 0.8450491307634165, + "fcm_dpo/beta": 0.004464943893253803, + "fcm_dpo/delta": -0.027830161154270172, + "fcm_dpo/margin": 94.5018310546875, + "fcm_dpo/q_t": 0.408642053604126, + "grad_norm": 11.145119667053223, + "learning_rate": 3.6186337553827743e-08, + "logits/chosen": 0.8411096334457397, + "logits/rejected": 0.7925565242767334, + "logps/chosen": -245.83338928222656, + "logps/ref_chosen": -61.521644592285156, + "logps/ref_rejected": -82.83859252929688, + "logps/rejected": -361.65216064453125, + "loss": 1.1223, + "margin_dpo/margin_mean": 94.5018310546875, + "margin_dpo/margin_std": 158.6476287841797, + "step": 559 + }, + { + "KL/chosen_KL_mean": -199.8177490234375, + "KL/mean": -244.57350158691406, + "KL/rejected_KL_mean": -289.3292541503906, + "KL/std": 123.44962310791016, + "epoch": 0.8465608465608465, + "fcm_dpo/beta": 0.004556549713015556, + "fcm_dpo/delta": -0.009785711765289307, + "fcm_dpo/margin": 89.5114974975586, + "fcm_dpo/q_t": 0.4058662950992584, + "grad_norm": 16.259973526000977, + "learning_rate": 3.550414669125573e-08, + "logits/chosen": 0.8841683864593506, + "logits/rejected": 0.8540163040161133, + "logps/chosen": -260.458984375, + "logps/ref_chosen": -60.64122009277344, + "logps/ref_rejected": -78.75474548339844, + "logps/rejected": -368.083984375, + "loss": 1.0881, + "margin_dpo/margin_mean": 89.5114974975586, + "margin_dpo/margin_std": 116.04913330078125, + "step": 560 + }, + { + "KL/chosen_KL_mean": -187.08404541015625, + "KL/mean": -225.16224670410156, + "KL/rejected_KL_mean": -263.2404479980469, + "KL/std": 119.03514099121094, + "epoch": 0.8480725623582767, + "fcm_dpo/beta": 0.004581731744110584, + "fcm_dpo/delta": 0.05710749328136444, + "fcm_dpo/margin": 76.15641784667969, + "fcm_dpo/q_t": 0.42259570956230164, + "grad_norm": 11.56666374206543, + "learning_rate": 3.482795573879241e-08, + "logits/chosen": 0.8361924886703491, + "logits/rejected": 0.8081060647964478, + "logps/chosen": -249.5826416015625, + "logps/ref_chosen": -62.49859619140625, + "logps/ref_rejected": -78.72064208984375, + "logps/rejected": -341.9610900878906, + "loss": 1.1554, + "margin_dpo/margin_mean": 76.15641784667969, + "margin_dpo/margin_std": 131.91342163085938, + "step": 561 + }, + { + "KL/chosen_KL_mean": -197.9278564453125, + "KL/mean": -250.18325805664062, + "KL/rejected_KL_mean": -302.4386901855469, + "KL/std": 145.4912109375, + "epoch": 0.8495842781557067, + "fcm_dpo/beta": 0.004399956203997135, + "fcm_dpo/delta": -0.07504507899284363, + "fcm_dpo/margin": 104.51081848144531, + "fcm_dpo/q_t": 0.3958006501197815, + "grad_norm": 14.884467124938965, + "learning_rate": 3.415778361095226e-08, + "logits/chosen": 0.8424015045166016, + "logits/rejected": 0.820836067199707, + "logps/chosen": -272.7095947265625, + "logps/ref_chosen": -74.78173828125, + "logps/ref_rejected": -92.63499450683594, + "logps/rejected": -395.07366943359375, + "loss": 1.0671, + "margin_dpo/margin_mean": 104.51081848144531, + "margin_dpo/margin_std": 139.0022430419922, + "step": 562 + }, + { + "KL/chosen_KL_mean": -164.1915283203125, + "KL/mean": -207.3955535888672, + "KL/rejected_KL_mean": -250.5995635986328, + "KL/std": 106.54350280761719, + "epoch": 0.8510959939531368, + "fcm_dpo/beta": 0.004485203884541988, + "fcm_dpo/delta": 0.01402625348418951, + "fcm_dpo/margin": 86.40805053710938, + "fcm_dpo/q_t": 0.41050225496292114, + "grad_norm": 18.291183471679688, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": 0.9199409484863281, + "logits/rejected": 0.8749498128890991, + "logps/chosen": -214.39004516601562, + "logps/ref_chosen": -50.19850158691406, + "logps/ref_rejected": -66.76687622070312, + "logps/rejected": -317.366455078125, + "loss": 1.1242, + "margin_dpo/margin_mean": 86.40804290771484, + "margin_dpo/margin_std": 135.92982482910156, + "step": 563 + }, + { + "KL/chosen_KL_mean": -165.2862091064453, + "KL/mean": -219.43699645996094, + "KL/rejected_KL_mean": -273.5877685546875, + "KL/std": 115.89987182617188, + "epoch": 0.8526077097505669, + "fcm_dpo/beta": 0.004321890417486429, + "fcm_dpo/delta": -0.07944278419017792, + "fcm_dpo/margin": 108.30156707763672, + "fcm_dpo/q_t": 0.3904789388179779, + "grad_norm": 11.61489486694336, + "learning_rate": 3.283557064487785e-08, + "logits/chosen": 0.8549599051475525, + "logits/rejected": 0.8323065042495728, + "logps/chosen": -221.02706909179688, + "logps/ref_chosen": -55.7408447265625, + "logps/ref_rejected": -74.82323455810547, + "logps/rejected": -348.4110107421875, + "loss": 1.0597, + "margin_dpo/margin_mean": 108.30156707763672, + "margin_dpo/margin_std": 141.85757446289062, + "step": 564 + }, + { + "KL/chosen_KL_mean": -198.53341674804688, + "KL/mean": -242.3411865234375, + "KL/rejected_KL_mean": -286.14898681640625, + "KL/std": 123.26646423339844, + "epoch": 0.854119425547997, + "fcm_dpo/beta": 0.004352842457592487, + "fcm_dpo/delta": 0.02035663276910782, + "fcm_dpo/margin": 87.61554718017578, + "fcm_dpo/q_t": 0.4115191698074341, + "grad_norm": 13.795531272888184, + "learning_rate": 3.218356679178252e-08, + "logits/chosen": 0.886420488357544, + "logits/rejected": 0.8450945615768433, + "logps/chosen": -256.87078857421875, + "logps/ref_chosen": -58.33738327026367, + "logps/ref_rejected": -78.31776428222656, + "logps/rejected": -364.46673583984375, + "loss": 1.1088, + "margin_dpo/margin_mean": 87.61554718017578, + "margin_dpo/margin_std": 122.73006439208984, + "step": 565 + }, + { + "KL/chosen_KL_mean": -189.62576293945312, + "KL/mean": -228.21856689453125, + "KL/rejected_KL_mean": -266.81134033203125, + "KL/std": 130.59353637695312, + "epoch": 0.8556311413454271, + "fcm_dpo/beta": 0.004506157711148262, + "fcm_dpo/delta": 0.055811285972595215, + "fcm_dpo/margin": 77.18556213378906, + "fcm_dpo/q_t": 0.4241322875022888, + "grad_norm": 12.119365692138672, + "learning_rate": 3.1537655732553764e-08, + "logits/chosen": 0.8291478157043457, + "logits/rejected": 0.8274587392807007, + "logps/chosen": -260.8495178222656, + "logps/ref_chosen": -71.22373962402344, + "logps/ref_rejected": -71.11601257324219, + "logps/rejected": -337.9273681640625, + "loss": 1.1956, + "margin_dpo/margin_mean": 77.18555450439453, + "margin_dpo/margin_std": 164.42337036132812, + "step": 566 + }, + { + "KL/chosen_KL_mean": -179.55332946777344, + "KL/mean": -228.03167724609375, + "KL/rejected_KL_mean": -276.5100402832031, + "KL/std": 120.30104064941406, + "epoch": 0.8571428571428571, + "fcm_dpo/beta": 0.004341046325862408, + "fcm_dpo/delta": -0.02707264944911003, + "fcm_dpo/margin": 96.95671844482422, + "fcm_dpo/q_t": 0.4018208384513855, + "grad_norm": 10.208983421325684, + "learning_rate": 3.089785553471233e-08, + "logits/chosen": 0.8846025466918945, + "logits/rejected": 0.8115559816360474, + "logps/chosen": -232.22259521484375, + "logps/ref_chosen": -52.669273376464844, + "logps/ref_rejected": -74.34785461425781, + "logps/rejected": -350.85791015625, + "loss": 1.0904, + "margin_dpo/margin_mean": 96.95671081542969, + "margin_dpo/margin_std": 131.9867401123047, + "step": 567 + }, + { + "KL/chosen_KL_mean": -167.61322021484375, + "KL/mean": -225.8235626220703, + "KL/rejected_KL_mean": -284.0339050292969, + "KL/std": 128.29510498046875, + "epoch": 0.8586545729402872, + "fcm_dpo/beta": 0.0042817965149879456, + "fcm_dpo/delta": -0.1166883111000061, + "fcm_dpo/margin": 116.42066955566406, + "fcm_dpo/q_t": 0.3852314352989197, + "grad_norm": 13.811408042907715, + "learning_rate": 3.026418409484513e-08, + "logits/chosen": 0.8656144738197327, + "logits/rejected": 0.7998012900352478, + "logps/chosen": -219.79122924804688, + "logps/ref_chosen": -52.178001403808594, + "logps/ref_rejected": -85.8277587890625, + "logps/rejected": -369.8616638183594, + "loss": 1.0303, + "margin_dpo/margin_mean": 116.42066955566406, + "margin_dpo/margin_std": 132.6590576171875, + "step": 568 + }, + { + "KL/chosen_KL_mean": -197.28277587890625, + "KL/mean": -225.33621215820312, + "KL/rejected_KL_mean": -253.38963317871094, + "KL/std": 129.16079711914062, + "epoch": 0.8601662887377173, + "fcm_dpo/beta": 0.004134716931730509, + "fcm_dpo/delta": 0.010899978689849377, + "fcm_dpo/margin": 56.106849670410156, + "fcm_dpo/q_t": 0.4470366835594177, + "grad_norm": 13.65986442565918, + "learning_rate": 2.963665913810451e-08, + "logits/chosen": 0.7979574203491211, + "logits/rejected": 0.7730275392532349, + "logps/chosen": -259.9320373535156, + "logps/ref_chosen": -62.649261474609375, + "logps/ref_rejected": -75.4298324584961, + "logps/rejected": -328.8194580078125, + "loss": 1.2424, + "margin_dpo/margin_mean": 56.106842041015625, + "margin_dpo/margin_std": 131.40196228027344, + "step": 569 + }, + { + "KL/chosen_KL_mean": -172.97483825683594, + "KL/mean": -227.10557556152344, + "KL/rejected_KL_mean": -281.23626708984375, + "KL/std": 119.91664123535156, + "epoch": 0.8616780045351474, + "fcm_dpo/beta": 0.004098374396562576, + "fcm_dpo/delta": -0.05100153386592865, + "fcm_dpo/margin": 108.261474609375, + "fcm_dpo/q_t": 0.39809536933898926, + "grad_norm": 11.868734359741211, + "learning_rate": 2.9015298217712453e-08, + "logits/chosen": 0.8359570503234863, + "logits/rejected": 0.7631780505180359, + "logps/chosen": -223.01663208007812, + "logps/ref_chosen": -50.04179382324219, + "logps/ref_rejected": -78.27146911621094, + "logps/rejected": -359.50775146484375, + "loss": 1.0673, + "margin_dpo/margin_mean": 108.261474609375, + "margin_dpo/margin_std": 139.59825134277344, + "step": 570 + }, + { + "KL/chosen_KL_mean": -183.54083251953125, + "KL/mean": -215.38575744628906, + "KL/rejected_KL_mean": -247.2306671142578, + "KL/std": 116.32156372070312, + "epoch": 0.8631897203325775, + "fcm_dpo/beta": 0.004064176231622696, + "fcm_dpo/delta": 0.044348590075969696, + "fcm_dpo/margin": 63.68983840942383, + "fcm_dpo/q_t": 0.44058936834335327, + "grad_norm": 11.032413482666016, + "learning_rate": 2.840011871446962e-08, + "logits/chosen": 0.8487415909767151, + "logits/rejected": 0.8350812196731567, + "logps/chosen": -237.19764709472656, + "logps/ref_chosen": -53.65681457519531, + "logps/ref_rejected": -66.13298034667969, + "logps/rejected": -313.3636474609375, + "loss": 1.2164, + "margin_dpo/margin_mean": 63.689842224121094, + "margin_dpo/margin_std": 133.0120849609375, + "step": 571 + }, + { + "KL/chosen_KL_mean": -191.580078125, + "KL/mean": -224.85147094726562, + "KL/rejected_KL_mean": -258.12286376953125, + "KL/std": 121.44296264648438, + "epoch": 0.8647014361300076, + "fcm_dpo/beta": 0.004147428087890148, + "fcm_dpo/delta": 0.023455314338207245, + "fcm_dpo/margin": 66.54278564453125, + "fcm_dpo/q_t": 0.4355035126209259, + "grad_norm": 13.11365795135498, + "learning_rate": 2.7791137836269158e-08, + "logits/chosen": 0.8420843482017517, + "logits/rejected": 0.8802669048309326, + "logps/chosen": -266.39801025390625, + "logps/ref_chosen": -74.81792449951172, + "logps/ref_rejected": -65.88681030273438, + "logps/rejected": -324.0096435546875, + "loss": 1.1947, + "margin_dpo/margin_mean": 66.54278564453125, + "margin_dpo/margin_std": 124.58303833007812, + "step": 572 + }, + { + "KL/chosen_KL_mean": -204.505126953125, + "KL/mean": -246.52902221679688, + "KL/rejected_KL_mean": -288.55291748046875, + "KL/std": 137.63925170898438, + "epoch": 0.8662131519274376, + "fcm_dpo/beta": 0.004270592704415321, + "fcm_dpo/delta": 0.04590342938899994, + "fcm_dpo/margin": 84.04781341552734, + "fcm_dpo/q_t": 0.4227134585380554, + "grad_norm": 12.49295425415039, + "learning_rate": 2.718837261761528e-08, + "logits/chosen": 0.8661466836929321, + "logits/rejected": 0.8327881097793579, + "logps/chosen": -273.23077392578125, + "logps/ref_chosen": -68.72564697265625, + "logps/ref_rejected": -88.16201782226562, + "logps/rejected": -376.7149658203125, + "loss": 1.1742, + "margin_dpo/margin_mean": 84.04782104492188, + "margin_dpo/margin_std": 166.25271606445312, + "step": 573 + }, + { + "KL/chosen_KL_mean": -182.09056091308594, + "KL/mean": -235.65748596191406, + "KL/rejected_KL_mean": -289.22442626953125, + "KL/std": 127.6211166381836, + "epoch": 0.8677248677248677, + "fcm_dpo/beta": 0.004246646538376808, + "fcm_dpo/delta": -0.06463881582021713, + "fcm_dpo/margin": 107.13386535644531, + "fcm_dpo/q_t": 0.3947995901107788, + "grad_norm": 10.736037254333496, + "learning_rate": 2.659183991914696e-08, + "logits/chosen": 0.8799293041229248, + "logits/rejected": 0.8327153921127319, + "logps/chosen": -238.40396118164062, + "logps/ref_chosen": -56.31340026855469, + "logps/ref_rejected": -83.91553497314453, + "logps/rejected": -373.13995361328125, + "loss": 1.0468, + "margin_dpo/margin_mean": 107.13386535644531, + "margin_dpo/margin_std": 122.67759704589844, + "step": 574 + }, + { + "KL/chosen_KL_mean": -188.4454345703125, + "KL/mean": -226.3487091064453, + "KL/rejected_KL_mean": -264.2519836425781, + "KL/std": 127.46905517578125, + "epoch": 0.8692365835222978, + "fcm_dpo/beta": 0.004053239710628986, + "fcm_dpo/delta": -0.031237700954079628, + "fcm_dpo/margin": 75.80654907226562, + "fcm_dpo/q_t": 0.43161332607269287, + "grad_norm": 11.321064949035645, + "learning_rate": 2.600155642716606e-08, + "logits/chosen": 0.8765904903411865, + "logits/rejected": 0.8120791912078857, + "logps/chosen": -253.02957153320312, + "logps/ref_chosen": -64.5841293334961, + "logps/ref_rejected": -93.47034454345703, + "logps/rejected": -357.72235107421875, + "loss": 1.1997, + "margin_dpo/margin_mean": 75.80654907226562, + "margin_dpo/margin_std": 154.2758331298828, + "step": 575 + }, + { + "KL/chosen_KL_mean": -165.71563720703125, + "KL/mean": -223.3694610595703, + "KL/rejected_KL_mean": -281.02325439453125, + "KL/std": 128.47235107421875, + "epoch": 0.8707482993197279, + "fcm_dpo/beta": 0.0038730669766664505, + "fcm_dpo/delta": -0.05786684527993202, + "fcm_dpo/margin": 115.30767822265625, + "fcm_dpo/q_t": 0.3955625295639038, + "grad_norm": 10.995619773864746, + "learning_rate": 2.5417538653170754e-08, + "logits/chosen": 0.8378143310546875, + "logits/rejected": 0.7612361907958984, + "logps/chosen": -218.99615478515625, + "logps/ref_chosen": -53.28052520751953, + "logps/ref_rejected": -84.2000503540039, + "logps/rejected": -365.22332763671875, + "loss": 1.0621, + "margin_dpo/margin_mean": 115.30767822265625, + "margin_dpo/margin_std": 140.68199157714844, + "step": 576 + }, + { + "KL/chosen_KL_mean": -185.73255920410156, + "KL/mean": -220.54190063476562, + "KL/rejected_KL_mean": -255.35122680664062, + "KL/std": 123.33712768554688, + "epoch": 0.872260015117158, + "fcm_dpo/beta": 0.004132484085857868, + "fcm_dpo/delta": 0.12298332154750824, + "fcm_dpo/margin": 69.61865234375, + "fcm_dpo/q_t": 0.43443581461906433, + "grad_norm": 12.665448188781738, + "learning_rate": 2.4839802933393607e-08, + "logits/chosen": 0.8501470685005188, + "logits/rejected": 0.8491848707199097, + "logps/chosen": -248.0572509765625, + "logps/ref_chosen": -62.32468795776367, + "logps/ref_rejected": -67.300537109375, + "logps/rejected": -322.6517333984375, + "loss": 1.1952, + "margin_dpo/margin_mean": 69.61865234375, + "margin_dpo/margin_std": 131.3208770751953, + "step": 577 + }, + { + "KL/chosen_KL_mean": -177.81094360351562, + "KL/mean": -212.05685424804688, + "KL/rejected_KL_mean": -246.3027801513672, + "KL/std": 130.85694885253906, + "epoch": 0.873771730914588, + "fcm_dpo/beta": 0.004423568025231361, + "fcm_dpo/delta": 0.10660433769226074, + "fcm_dpo/margin": 68.4918441772461, + "fcm_dpo/q_t": 0.43448588252067566, + "grad_norm": 13.477178573608398, + "learning_rate": 2.4268365428344733e-08, + "logits/chosen": 0.8857970237731934, + "logits/rejected": 0.8791966438293457, + "logps/chosen": -234.46652221679688, + "logps/ref_chosen": -56.65557861328125, + "logps/ref_rejected": -68.21835327148438, + "logps/rejected": -314.5211181640625, + "loss": 1.21, + "margin_dpo/margin_mean": 68.49185180664062, + "margin_dpo/margin_std": 150.7880096435547, + "step": 578 + }, + { + "KL/chosen_KL_mean": -182.95477294921875, + "KL/mean": -233.42739868164062, + "KL/rejected_KL_mean": -283.9000244140625, + "KL/std": 118.61689758300781, + "epoch": 0.8752834467120182, + "fcm_dpo/beta": 0.004369863774627447, + "fcm_dpo/delta": -0.04803343117237091, + "fcm_dpo/margin": 100.94525146484375, + "fcm_dpo/q_t": 0.3958134353160858, + "grad_norm": 10.97381591796875, + "learning_rate": 2.3703242122359357e-08, + "logits/chosen": 0.805631697177887, + "logits/rejected": 0.7936950922012329, + "logps/chosen": -239.76443481445312, + "logps/ref_chosen": -56.809661865234375, + "logps/ref_rejected": -68.09613037109375, + "logps/rejected": -351.99615478515625, + "loss": 1.0605, + "margin_dpo/margin_mean": 100.94525146484375, + "margin_dpo/margin_std": 122.35321807861328, + "step": 579 + }, + { + "KL/chosen_KL_mean": -186.7950439453125, + "KL/mean": -232.87738037109375, + "KL/rejected_KL_mean": -278.959716796875, + "KL/std": 143.08419799804688, + "epoch": 0.8767951625094482, + "fcm_dpo/beta": 0.004418280906975269, + "fcm_dpo/delta": -0.009930390864610672, + "fcm_dpo/margin": 92.1646728515625, + "fcm_dpo/q_t": 0.41188204288482666, + "grad_norm": 13.299028396606445, + "learning_rate": 2.3144448823151392e-08, + "logits/chosen": 0.8283756375312805, + "logits/rejected": 0.7941699624061584, + "logps/chosen": -244.4951629638672, + "logps/ref_chosen": -57.70011520385742, + "logps/ref_rejected": -77.90664672851562, + "logps/rejected": -356.8663635253906, + "loss": 1.1349, + "margin_dpo/margin_mean": 92.16466522216797, + "margin_dpo/margin_std": 160.25291442871094, + "step": 580 + }, + { + "KL/chosen_KL_mean": -203.46536254882812, + "KL/mean": -243.61395263671875, + "KL/rejected_KL_mean": -283.7625732421875, + "KL/std": 123.7155532836914, + "epoch": 0.8783068783068783, + "fcm_dpo/beta": 0.004408898763358593, + "fcm_dpo/delta": 0.0514557883143425, + "fcm_dpo/margin": 80.29718017578125, + "fcm_dpo/q_t": 0.42192864418029785, + "grad_norm": 11.928445816040039, + "learning_rate": 2.259200116137039e-08, + "logits/chosen": 0.8557339906692505, + "logits/rejected": 0.8073595762252808, + "logps/chosen": -262.7977294921875, + "logps/ref_chosen": -59.332359313964844, + "logps/ref_rejected": -83.64482116699219, + "logps/rejected": -367.4073791503906, + "loss": 1.1666, + "margin_dpo/margin_mean": 80.29718017578125, + "margin_dpo/margin_std": 150.23489379882812, + "step": 581 + }, + { + "KL/chosen_KL_mean": -186.45236206054688, + "KL/mean": -223.76060485839844, + "KL/rejected_KL_mean": -261.06884765625, + "KL/std": 116.26499938964844, + "epoch": 0.8798185941043084, + "fcm_dpo/beta": 0.004633273929357529, + "fcm_dpo/delta": 0.05958259850740433, + "fcm_dpo/margin": 74.61648559570312, + "fcm_dpo/q_t": 0.423893004655838, + "grad_norm": 11.573064804077148, + "learning_rate": 2.204591459016525e-08, + "logits/chosen": 0.8565012812614441, + "logits/rejected": 0.8867921829223633, + "logps/chosen": -250.61521911621094, + "logps/ref_chosen": -64.16285705566406, + "logps/ref_rejected": -58.632896423339844, + "logps/rejected": -319.70172119140625, + "loss": 1.1645, + "margin_dpo/margin_mean": 74.61648559570312, + "margin_dpo/margin_std": 135.59817504882812, + "step": 582 + }, + { + "KL/chosen_KL_mean": -183.05465698242188, + "KL/mean": -235.2905731201172, + "KL/rejected_KL_mean": -287.5264892578125, + "KL/std": 138.89993286132812, + "epoch": 0.8813303099017384, + "fcm_dpo/beta": 0.004453700035810471, + "fcm_dpo/delta": -0.07676573097705841, + "fcm_dpo/margin": 104.47183990478516, + "fcm_dpo/q_t": 0.39537179470062256, + "grad_norm": 13.700339317321777, + "learning_rate": 2.1506204384751064e-08, + "logits/chosen": 0.9295327067375183, + "logits/rejected": 0.8535386323928833, + "logps/chosen": -234.9270477294922, + "logps/ref_chosen": -51.87239456176758, + "logps/ref_rejected": -83.86331176757812, + "logps/rejected": -371.3898010253906, + "loss": 1.0891, + "margin_dpo/margin_mean": 104.47184753417969, + "margin_dpo/margin_std": 159.20358276367188, + "step": 583 + }, + { + "KL/chosen_KL_mean": -168.95220947265625, + "KL/mean": -212.09420776367188, + "KL/rejected_KL_mean": -255.2362060546875, + "KL/std": 120.96197509765625, + "epoch": 0.8828420256991686, + "fcm_dpo/beta": 0.00443212129175663, + "fcm_dpo/delta": 0.019545778632164, + "fcm_dpo/margin": 86.28401184082031, + "fcm_dpo/q_t": 0.4162449240684509, + "grad_norm": 13.947019577026367, + "learning_rate": 2.09728856419826e-08, + "logits/chosen": 0.9570223093032837, + "logits/rejected": 0.8717998266220093, + "logps/chosen": -215.52359008789062, + "logps/ref_chosen": -46.571388244628906, + "logps/ref_rejected": -80.67969512939453, + "logps/rejected": -335.9158935546875, + "loss": 1.1478, + "margin_dpo/margin_mean": 86.28401184082031, + "margin_dpo/margin_std": 154.09945678710938, + "step": 584 + }, + { + "KL/chosen_KL_mean": -197.41429138183594, + "KL/mean": -228.14865112304688, + "KL/rejected_KL_mean": -258.88299560546875, + "KL/std": 134.65402221679688, + "epoch": 0.8843537414965986, + "fcm_dpo/beta": 0.004502854309976101, + "fcm_dpo/delta": 0.015462912619113922, + "fcm_dpo/margin": 61.46870422363281, + "fcm_dpo/q_t": 0.4372748136520386, + "grad_norm": 13.030404090881348, + "learning_rate": 2.044597327993153e-08, + "logits/chosen": 0.8678371906280518, + "logits/rejected": 0.8205997943878174, + "logps/chosen": -255.538818359375, + "logps/ref_chosen": -58.124534606933594, + "logps/ref_rejected": -79.00538635253906, + "logps/rejected": -337.88836669921875, + "loss": 1.212, + "margin_dpo/margin_mean": 61.46870422363281, + "margin_dpo/margin_std": 131.03814697265625, + "step": 585 + }, + { + "KL/chosen_KL_mean": -179.77459716796875, + "KL/mean": -218.6102294921875, + "KL/rejected_KL_mean": -257.44586181640625, + "KL/std": 107.11343383789062, + "epoch": 0.8858654572940288, + "fcm_dpo/beta": 0.004600149113684893, + "fcm_dpo/delta": 0.04789983481168747, + "fcm_dpo/margin": 77.6712646484375, + "fcm_dpo/q_t": 0.4177996516227722, + "grad_norm": 11.622013092041016, + "learning_rate": 1.9925482037469187e-08, + "logits/chosen": 0.8768935203552246, + "logits/rejected": 0.8456597328186035, + "logps/chosen": -233.876220703125, + "logps/ref_chosen": -54.10163879394531, + "logps/ref_rejected": -63.72113037109375, + "logps/rejected": -321.1669921875, + "loss": 1.1115, + "margin_dpo/margin_mean": 77.6712646484375, + "margin_dpo/margin_std": 100.46601104736328, + "step": 586 + }, + { + "KL/chosen_KL_mean": -193.71388244628906, + "KL/mean": -231.7980194091797, + "KL/rejected_KL_mean": -269.88214111328125, + "KL/std": 131.6758270263672, + "epoch": 0.8873771730914588, + "fcm_dpo/beta": 0.004727904684841633, + "fcm_dpo/delta": 0.04477377235889435, + "fcm_dpo/margin": 76.16827392578125, + "fcm_dpo/q_t": 0.41351163387298584, + "grad_norm": 16.236108779907227, + "learning_rate": 1.9411426473854687e-08, + "logits/chosen": 0.9039092063903809, + "logits/rejected": 0.9038503766059875, + "logps/chosen": -257.1310729980469, + "logps/ref_chosen": -63.41719436645508, + "logps/ref_rejected": -63.47003936767578, + "logps/rejected": -333.3522033691406, + "loss": 1.2072, + "margin_dpo/margin_mean": 76.16828155517578, + "margin_dpo/margin_std": 177.19003295898438, + "step": 587 + }, + { + "KL/chosen_KL_mean": -197.713134765625, + "KL/mean": -240.94564819335938, + "KL/rejected_KL_mean": -284.1781921386719, + "KL/std": 131.87918090820312, + "epoch": 0.8888888888888888, + "fcm_dpo/beta": 0.004706019535660744, + "fcm_dpo/delta": -0.009034395217895508, + "fcm_dpo/margin": 86.46505737304688, + "fcm_dpo/q_t": 0.410349041223526, + "grad_norm": 16.405550003051758, + "learning_rate": 1.890382096832699e-08, + "logits/chosen": 0.9087653756141663, + "logits/rejected": 0.8741371631622314, + "logps/chosen": -259.9141845703125, + "logps/ref_chosen": -62.20103454589844, + "logps/ref_rejected": -82.10249328613281, + "logps/rejected": -366.28070068359375, + "loss": 1.1565, + "margin_dpo/margin_mean": 86.46505737304688, + "margin_dpo/margin_std": 164.84034729003906, + "step": 588 + }, + { + "KL/chosen_KL_mean": -183.16661071777344, + "KL/mean": -228.384765625, + "KL/rejected_KL_mean": -273.6029357910156, + "KL/std": 123.86094665527344, + "epoch": 0.890400604686319, + "fcm_dpo/beta": 0.0047372374683618546, + "fcm_dpo/delta": -0.033031828701496124, + "fcm_dpo/margin": 90.43630981445312, + "fcm_dpo/q_t": 0.4000184237957001, + "grad_norm": 11.501230239868164, + "learning_rate": 1.840267971970344e-08, + "logits/chosen": 0.8425379991531372, + "logits/rejected": 0.8174772262573242, + "logps/chosen": -239.88021850585938, + "logps/ref_chosen": -56.71361541748047, + "logps/ref_rejected": -76.7366943359375, + "logps/rejected": -350.339599609375, + "loss": 1.0785, + "margin_dpo/margin_mean": 90.43630981445312, + "margin_dpo/margin_std": 118.53308868408203, + "step": 589 + }, + { + "KL/chosen_KL_mean": -199.31167602539062, + "KL/mean": -239.75074768066406, + "KL/rejected_KL_mean": -280.1898193359375, + "KL/std": 112.27293395996094, + "epoch": 0.891912320483749, + "fcm_dpo/beta": 0.00475288974121213, + "fcm_dpo/delta": 0.016453173011541367, + "fcm_dpo/margin": 80.87815856933594, + "fcm_dpo/q_t": 0.41211652755737305, + "grad_norm": 16.781558990478516, + "learning_rate": 1.7908016745981856e-08, + "logits/chosen": 0.8056505918502808, + "logits/rejected": 0.7747730016708374, + "logps/chosen": -265.82550048828125, + "logps/ref_chosen": -66.5138168334961, + "logps/ref_rejected": -85.70820617675781, + "logps/rejected": -365.89801025390625, + "loss": 1.1167, + "margin_dpo/margin_mean": 80.87814331054688, + "margin_dpo/margin_std": 119.72245788574219, + "step": 590 + }, + { + "KL/chosen_KL_mean": -168.68548583984375, + "KL/mean": -223.0603485107422, + "KL/rejected_KL_mean": -277.43524169921875, + "KL/std": 136.9473876953125, + "epoch": 0.8934240362811792, + "fcm_dpo/beta": 0.004390774294734001, + "fcm_dpo/delta": -0.09836189448833466, + "fcm_dpo/margin": 108.74974060058594, + "fcm_dpo/q_t": 0.3937166929244995, + "grad_norm": 13.011321067810059, + "learning_rate": 1.7419845883949098e-08, + "logits/chosen": 0.9268543124198914, + "logits/rejected": 0.8832913637161255, + "logps/chosen": -229.38265991210938, + "logps/ref_chosen": -60.697181701660156, + "logps/ref_rejected": -86.12278747558594, + "logps/rejected": -363.5580139160156, + "loss": 1.0917, + "margin_dpo/margin_mean": 108.7497329711914, + "margin_dpo/margin_std": 168.9647216796875, + "step": 591 + }, + { + "KL/chosen_KL_mean": -185.19931030273438, + "KL/mean": -225.90817260742188, + "KL/rejected_KL_mean": -266.6170349121094, + "KL/std": 114.66996765136719, + "epoch": 0.8949357520786092, + "fcm_dpo/beta": 0.004453813657164574, + "fcm_dpo/delta": 0.04094696044921875, + "fcm_dpo/margin": 81.41773986816406, + "fcm_dpo/q_t": 0.4200698137283325, + "grad_norm": 14.020329475402832, + "learning_rate": 1.6938180788793556e-08, + "logits/chosen": 0.9028749465942383, + "logits/rejected": 0.8090362548828125, + "logps/chosen": -236.4366455078125, + "logps/ref_chosen": -51.237327575683594, + "logps/ref_rejected": -81.60242462158203, + "logps/rejected": -348.219482421875, + "loss": 1.137, + "margin_dpo/margin_mean": 81.41773986816406, + "margin_dpo/margin_std": 130.4501190185547, + "step": 592 + }, + { + "KL/chosen_KL_mean": -151.83436584472656, + "KL/mean": -198.61888122558594, + "KL/rejected_KL_mean": -245.40338134765625, + "KL/std": 116.2724380493164, + "epoch": 0.8964474678760394, + "fcm_dpo/beta": 0.004477858543395996, + "fcm_dpo/delta": -0.022135818377137184, + "fcm_dpo/margin": 93.56902313232422, + "fcm_dpo/q_t": 0.40188807249069214, + "grad_norm": 15.795121192932129, + "learning_rate": 1.6463034933723336e-08, + "logits/chosen": 0.8935588598251343, + "logits/rejected": 0.8158241510391235, + "logps/chosen": -193.91436767578125, + "logps/ref_chosen": -42.08000183105469, + "logps/ref_rejected": -68.47499084472656, + "logps/rejected": -313.87835693359375, + "loss": 1.0948, + "margin_dpo/margin_mean": 93.56902313232422, + "margin_dpo/margin_std": 133.15293884277344, + "step": 593 + }, + { + "KL/chosen_KL_mean": -195.93930053710938, + "KL/mean": -232.91555786132812, + "KL/rejected_KL_mean": -269.891845703125, + "KL/std": 117.66835021972656, + "epoch": 0.8979591836734694, + "fcm_dpo/beta": 0.004622071050107479, + "fcm_dpo/delta": 0.06481346487998962, + "fcm_dpo/margin": 73.95257568359375, + "fcm_dpo/q_t": 0.4218658208847046, + "grad_norm": 12.912012100219727, + "learning_rate": 1.5994421609589385e-08, + "logits/chosen": 0.7965784668922424, + "logits/rejected": 0.7883630990982056, + "logps/chosen": -259.59796142578125, + "logps/ref_chosen": -63.658668518066406, + "logps/ref_rejected": -70.35597229003906, + "logps/rejected": -340.2478332519531, + "loss": 1.1477, + "margin_dpo/margin_mean": 73.95257568359375, + "margin_dpo/margin_std": 120.45957946777344, + "step": 594 + }, + { + "KL/chosen_KL_mean": -178.886962890625, + "KL/mean": -230.2701416015625, + "KL/rejected_KL_mean": -281.6533203125, + "KL/std": 126.58238220214844, + "epoch": 0.8994708994708994, + "fcm_dpo/beta": 0.004551096819341183, + "fcm_dpo/delta": -0.07861794531345367, + "fcm_dpo/margin": 102.76634216308594, + "fcm_dpo/q_t": 0.39468061923980713, + "grad_norm": 10.457369804382324, + "learning_rate": 1.553235392451377e-08, + "logits/chosen": 0.8931225538253784, + "logits/rejected": 0.8216215372085571, + "logps/chosen": -235.105712890625, + "logps/ref_chosen": -56.21875762939453, + "logps/ref_rejected": -83.95773315429688, + "logps/rejected": -365.6110534667969, + "loss": 1.0852, + "margin_dpo/margin_mean": 102.76634979248047, + "margin_dpo/margin_std": 154.8719940185547, + "step": 595 + }, + { + "KL/chosen_KL_mean": -209.03253173828125, + "KL/mean": -227.578369140625, + "KL/rejected_KL_mean": -246.12420654296875, + "KL/std": 113.8258056640625, + "epoch": 0.9009826152683296, + "fcm_dpo/beta": 0.004681180696934462, + "fcm_dpo/delta": 0.08331310749053955, + "fcm_dpo/margin": 37.091651916503906, + "fcm_dpo/q_t": 0.4618571698665619, + "grad_norm": 15.247252464294434, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": 0.7848924398422241, + "logits/rejected": 0.8063439726829529, + "logps/chosen": -277.513427734375, + "logps/ref_chosen": -68.48088073730469, + "logps/ref_rejected": -61.732967376708984, + "logps/rejected": -307.857177734375, + "loss": 1.2958, + "margin_dpo/margin_mean": 37.09164810180664, + "margin_dpo/margin_std": 118.8961181640625, + "step": 596 + }, + { + "KL/chosen_KL_mean": -160.05877685546875, + "KL/mean": -198.61622619628906, + "KL/rejected_KL_mean": -237.17367553710938, + "KL/std": 100.68923950195312, + "epoch": 0.9024943310657596, + "fcm_dpo/beta": 0.004797500092536211, + "fcm_dpo/delta": 0.032740212976932526, + "fcm_dpo/margin": 77.11491394042969, + "fcm_dpo/q_t": 0.41552621126174927, + "grad_norm": 11.356983184814453, + "learning_rate": 1.4627906988186111e-08, + "logits/chosen": 0.8747761845588684, + "logits/rejected": 0.8632344007492065, + "logps/chosen": -208.91627502441406, + "logps/ref_chosen": -48.85750961303711, + "logps/ref_rejected": -55.068084716796875, + "logps/rejected": -292.24176025390625, + "loss": 1.1215, + "margin_dpo/margin_mean": 77.11491394042969, + "margin_dpo/margin_std": 113.88622283935547, + "step": 597 + }, + { + "KL/chosen_KL_mean": -207.90806579589844, + "KL/mean": -234.0879364013672, + "KL/rejected_KL_mean": -260.267822265625, + "KL/std": 122.74574279785156, + "epoch": 0.9040060468631897, + "fcm_dpo/beta": 0.004856814630329609, + "fcm_dpo/delta": 0.056810446083545685, + "fcm_dpo/margin": 52.3597412109375, + "fcm_dpo/q_t": 0.44433027505874634, + "grad_norm": 12.908487319946289, + "learning_rate": 1.4185553036259095e-08, + "logits/chosen": 0.896747350692749, + "logits/rejected": 0.8310115337371826, + "logps/chosen": -266.79522705078125, + "logps/ref_chosen": -58.88715362548828, + "logps/ref_rejected": -81.43145751953125, + "logps/rejected": -341.69927978515625, + "loss": 1.2418, + "margin_dpo/margin_mean": 52.35974884033203, + "margin_dpo/margin_std": 130.03427124023438, + "step": 598 + }, + { + "KL/chosen_KL_mean": -208.6796875, + "KL/mean": -237.74996948242188, + "KL/rejected_KL_mean": -266.8202819824219, + "KL/std": 116.68116760253906, + "epoch": 0.9055177626606198, + "fcm_dpo/beta": 0.004964320454746485, + "fcm_dpo/delta": 0.016610831022262573, + "fcm_dpo/margin": 58.14057922363281, + "fcm_dpo/q_t": 0.43591034412384033, + "grad_norm": 16.04824447631836, + "learning_rate": 1.3749795321332885e-08, + "logits/chosen": 0.910493016242981, + "logits/rejected": 0.8786226511001587, + "logps/chosen": -266.286865234375, + "logps/ref_chosen": -57.60719299316406, + "logps/ref_rejected": -71.80469512939453, + "logps/rejected": -338.6249694824219, + "loss": 1.2245, + "margin_dpo/margin_mean": 58.14057922363281, + "margin_dpo/margin_std": 135.10581970214844, + "step": 599 + }, + { + "KL/chosen_KL_mean": -190.2998046875, + "KL/mean": -226.334228515625, + "KL/rejected_KL_mean": -262.36865234375, + "KL/std": 129.5638885498047, + "epoch": 0.9070294784580499, + "fcm_dpo/beta": 0.004913763143122196, + "fcm_dpo/delta": -0.05108189955353737, + "fcm_dpo/margin": 72.0688247680664, + "fcm_dpo/q_t": 0.4227983355522156, + "grad_norm": 16.685800552368164, + "learning_rate": 1.3320646032487393e-08, + "logits/chosen": 0.8785399198532104, + "logits/rejected": 0.8318706750869751, + "logps/chosen": -248.7421112060547, + "logps/ref_chosen": -58.44231414794922, + "logps/ref_rejected": -83.64639282226562, + "logps/rejected": -346.0150146484375, + "loss": 1.1762, + "margin_dpo/margin_mean": 72.0688247680664, + "margin_dpo/margin_std": 140.14089965820312, + "step": 600 + }, + { + "KL/chosen_KL_mean": -171.17523193359375, + "KL/mean": -220.55459594726562, + "KL/rejected_KL_mean": -269.9339599609375, + "KL/std": 136.19314575195312, + "epoch": 0.90854119425548, + "fcm_dpo/beta": 0.0046309540048241615, + "fcm_dpo/delta": -0.0687466561794281, + "fcm_dpo/margin": 98.75874328613281, + "fcm_dpo/q_t": 0.39738285541534424, + "grad_norm": 11.404427528381348, + "learning_rate": 1.2898117173950868e-08, + "logits/chosen": 0.8668086528778076, + "logits/rejected": 0.8014013171195984, + "logps/chosen": -226.76954650878906, + "logps/ref_chosen": -55.59432601928711, + "logps/ref_rejected": -83.68630981445312, + "logps/rejected": -353.62030029296875, + "loss": 1.0981, + "margin_dpo/margin_mean": 98.75874328613281, + "margin_dpo/margin_std": 154.93240356445312, + "step": 601 + }, + { + "KL/chosen_KL_mean": -160.80662536621094, + "KL/mean": -204.2389373779297, + "KL/rejected_KL_mean": -247.67123413085938, + "KL/std": 116.13655853271484, + "epoch": 0.91005291005291, + "fcm_dpo/beta": 0.004675906151533127, + "fcm_dpo/delta": -0.007718130946159363, + "fcm_dpo/margin": 86.8646011352539, + "fcm_dpo/q_t": 0.4061970114707947, + "grad_norm": 13.002616882324219, + "learning_rate": 1.2482220564763667e-08, + "logits/chosen": 0.8623030185699463, + "logits/rejected": 0.8387429118156433, + "logps/chosen": -217.15582275390625, + "logps/ref_chosen": -56.349185943603516, + "logps/ref_rejected": -71.9959716796875, + "logps/rejected": -319.6672058105469, + "loss": 1.0989, + "margin_dpo/margin_mean": 86.8646011352539, + "margin_dpo/margin_std": 123.03114318847656, + "step": 602 + }, + { + "KL/chosen_KL_mean": -177.90234375, + "KL/mean": -222.01742553710938, + "KL/rejected_KL_mean": -266.13250732421875, + "KL/std": 118.25772094726562, + "epoch": 0.9115646258503401, + "fcm_dpo/beta": 0.0045766388066112995, + "fcm_dpo/delta": -0.004777451977133751, + "fcm_dpo/margin": 88.23017883300781, + "fcm_dpo/q_t": 0.40897947549819946, + "grad_norm": 11.585609436035156, + "learning_rate": 1.2072967838448051e-08, + "logits/chosen": 0.8506954908370972, + "logits/rejected": 0.8117414712905884, + "logps/chosen": -231.0707244873047, + "logps/ref_chosen": -53.16838836669922, + "logps/ref_rejected": -73.8604736328125, + "logps/rejected": -339.99298095703125, + "loss": 1.1146, + "margin_dpo/margin_mean": 88.23017883300781, + "margin_dpo/margin_std": 137.55128479003906, + "step": 603 + }, + { + "KL/chosen_KL_mean": -174.488037109375, + "KL/mean": -210.52664184570312, + "KL/rejected_KL_mean": -246.56524658203125, + "KL/std": 110.95604705810547, + "epoch": 0.9130763416477702, + "fcm_dpo/beta": 0.004770040512084961, + "fcm_dpo/delta": 0.06184397637844086, + "fcm_dpo/margin": 72.07720947265625, + "fcm_dpo/q_t": 0.42401158809661865, + "grad_norm": 13.443038940429688, + "learning_rate": 1.1670370442682459e-08, + "logits/chosen": 0.8126691579818726, + "logits/rejected": 0.8226346969604492, + "logps/chosen": -247.13746643066406, + "logps/ref_chosen": -72.64942169189453, + "logps/ref_rejected": -69.8792724609375, + "logps/rejected": -316.44451904296875, + "loss": 1.1842, + "margin_dpo/margin_mean": 72.07720947265625, + "margin_dpo/margin_std": 146.4234619140625, + "step": 604 + }, + { + "KL/chosen_KL_mean": -195.28396606445312, + "KL/mean": -235.2552490234375, + "KL/rejected_KL_mean": -275.22650146484375, + "KL/std": 115.78852844238281, + "epoch": 0.9145880574452003, + "fcm_dpo/beta": 0.00478768814355135, + "fcm_dpo/delta": 0.01924796961247921, + "fcm_dpo/margin": 79.94253540039062, + "fcm_dpo/q_t": 0.4138326644897461, + "grad_norm": 14.64280891418457, + "learning_rate": 1.1274439638981532e-08, + "logits/chosen": 0.9054049253463745, + "logits/rejected": 0.8625191450119019, + "logps/chosen": -256.8968200683594, + "logps/ref_chosen": -61.61284637451172, + "logps/ref_rejected": -79.34398651123047, + "logps/rejected": -354.57049560546875, + "loss": 1.142, + "margin_dpo/margin_mean": 79.94253540039062, + "margin_dpo/margin_std": 137.07847595214844, + "step": 605 + }, + { + "KL/chosen_KL_mean": -170.9767608642578, + "KL/mean": -221.29107666015625, + "KL/rejected_KL_mean": -271.6053466796875, + "KL/std": 122.00587463378906, + "epoch": 0.9160997732426304, + "fcm_dpo/beta": 0.004710886627435684, + "fcm_dpo/delta": -0.08624030649662018, + "fcm_dpo/margin": 100.62860870361328, + "fcm_dpo/q_t": 0.3922450542449951, + "grad_norm": 14.318273544311523, + "learning_rate": 1.0885186502381016e-08, + "logits/chosen": 0.8514528274536133, + "logits/rejected": 0.7999155521392822, + "logps/chosen": -225.4409942626953, + "logps/ref_chosen": -54.46424102783203, + "logps/ref_rejected": -79.62708282470703, + "logps/rejected": -351.2324523925781, + "loss": 1.0643, + "margin_dpo/margin_mean": 100.62861633300781, + "margin_dpo/margin_std": 136.915283203125, + "step": 606 + }, + { + "KL/chosen_KL_mean": -189.23269653320312, + "KL/mean": -231.669921875, + "KL/rejected_KL_mean": -274.10711669921875, + "KL/std": 117.58612060546875, + "epoch": 0.9176114890400605, + "fcm_dpo/beta": 0.004336424637585878, + "fcm_dpo/delta": -0.09618955850601196, + "fcm_dpo/margin": 84.87444305419922, + "fcm_dpo/q_t": 0.4125783443450928, + "grad_norm": 11.543998718261719, + "learning_rate": 1.0502621921127774e-08, + "logits/chosen": 0.8325244188308716, + "logits/rejected": 0.8169835805892944, + "logps/chosen": -252.09356689453125, + "logps/ref_chosen": -62.86086654663086, + "logps/ref_rejected": -72.5501937866211, + "logps/rejected": -346.6573486328125, + "loss": 1.1358, + "margin_dpo/margin_mean": 84.87443542480469, + "margin_dpo/margin_std": 131.11700439453125, + "step": 607 + }, + { + "KL/chosen_KL_mean": -191.9585723876953, + "KL/mean": -239.46746826171875, + "KL/rejected_KL_mean": -286.976318359375, + "KL/std": 132.16444396972656, + "epoch": 0.9191232048374905, + "fcm_dpo/beta": 0.00433045718818903, + "fcm_dpo/delta": -0.013165525160729885, + "fcm_dpo/margin": 95.01773834228516, + "fcm_dpo/q_t": 0.4065626263618469, + "grad_norm": 12.041932106018066, + "learning_rate": 1.0126756596375685e-08, + "logits/chosen": 0.8287484645843506, + "logits/rejected": 0.7658513784408569, + "logps/chosen": -255.13929748535156, + "logps/ref_chosen": -63.18071746826172, + "logps/ref_rejected": -99.15888214111328, + "logps/rejected": -386.13519287109375, + "loss": 1.0962, + "margin_dpo/margin_mean": 95.01774597167969, + "margin_dpo/margin_std": 135.38153076171875, + "step": 608 + }, + { + "KL/chosen_KL_mean": -171.56765747070312, + "KL/mean": -219.2465057373047, + "KL/rejected_KL_mean": -266.9253845214844, + "KL/std": 113.94578552246094, + "epoch": 0.9206349206349206, + "fcm_dpo/beta": 0.004231642931699753, + "fcm_dpo/delta": -0.005523219704627991, + "fcm_dpo/margin": 95.35771942138672, + "fcm_dpo/q_t": 0.4046892821788788, + "grad_norm": 11.678070068359375, + "learning_rate": 9.757601041885694e-09, + "logits/chosen": 0.9319273829460144, + "logits/rejected": 0.9010778665542603, + "logps/chosen": -220.1908721923828, + "logps/ref_chosen": -48.62322235107422, + "logps/ref_rejected": -68.28271484375, + "logps/rejected": -335.2080993652344, + "loss": 1.0812, + "margin_dpo/margin_mean": 95.35771942138672, + "margin_dpo/margin_std": 111.58070373535156, + "step": 609 + }, + { + "KL/chosen_KL_mean": -191.14016723632812, + "KL/mean": -233.82876586914062, + "KL/rejected_KL_mean": -276.51739501953125, + "KL/std": 129.33859252929688, + "epoch": 0.9221466364323507, + "fcm_dpo/beta": 0.004307628609240055, + "fcm_dpo/delta": 0.03571552410721779, + "fcm_dpo/margin": 85.37721252441406, + "fcm_dpo/q_t": 0.41646891832351685, + "grad_norm": 13.54400634765625, + "learning_rate": 9.395165583732379e-09, + "logits/chosen": 0.86832594871521, + "logits/rejected": 0.8651347160339355, + "logps/chosen": -263.8052978515625, + "logps/ref_chosen": -72.66513061523438, + "logps/ref_rejected": -87.15310668945312, + "logps/rejected": -363.67047119140625, + "loss": 1.1555, + "margin_dpo/margin_mean": 85.37721252441406, + "margin_dpo/margin_std": 151.74803161621094, + "step": 610 + }, + { + "KL/chosen_KL_mean": -172.24514770507812, + "KL/mean": -208.3623504638672, + "KL/rejected_KL_mean": -244.47955322265625, + "KL/std": 112.20661926269531, + "epoch": 0.9236583522297808, + "fcm_dpo/beta": 0.004518900997936726, + "fcm_dpo/delta": 0.08180320262908936, + "fcm_dpo/margin": 72.23439025878906, + "fcm_dpo/q_t": 0.4254700541496277, + "grad_norm": 12.482162475585938, + "learning_rate": 9.03946036001449e-09, + "logits/chosen": 0.8952933549880981, + "logits/rejected": 0.8567260503768921, + "logps/chosen": -220.55374145507812, + "logps/ref_chosen": -48.30857849121094, + "logps/ref_rejected": -70.6141128540039, + "logps/rejected": -315.0936584472656, + "loss": 1.149, + "margin_dpo/margin_mean": 72.23438262939453, + "margin_dpo/margin_std": 112.78076171875, + "step": 611 + }, + { + "KL/chosen_KL_mean": -189.6739044189453, + "KL/mean": -245.22344970703125, + "KL/rejected_KL_mean": -300.77301025390625, + "KL/std": 119.71464538574219, + "epoch": 0.9251700680272109, + "fcm_dpo/beta": 0.004395745228976011, + "fcm_dpo/delta": -0.10357952862977982, + "fcm_dpo/margin": 111.09910583496094, + "fcm_dpo/q_t": 0.3854334354400635, + "grad_norm": 11.041399955749512, + "learning_rate": 8.690495320571839e-09, + "logits/chosen": 0.7935118675231934, + "logits/rejected": 0.735627293586731, + "logps/chosen": -250.90545654296875, + "logps/ref_chosen": -61.23155975341797, + "logps/ref_rejected": -94.37979888916016, + "logps/rejected": -395.15283203125, + "loss": 1.0329, + "margin_dpo/margin_mean": 111.09910583496094, + "margin_dpo/margin_std": 131.01829528808594, + "step": 612 + }, + { + "KL/chosen_KL_mean": -166.52000427246094, + "KL/mean": -213.49661254882812, + "KL/rejected_KL_mean": -260.47320556640625, + "KL/std": 132.3287811279297, + "epoch": 0.926681783824641, + "fcm_dpo/beta": 0.004340745974332094, + "fcm_dpo/delta": -0.009288674220442772, + "fcm_dpo/margin": 93.95318603515625, + "fcm_dpo/q_t": 0.4047100841999054, + "grad_norm": 10.03150749206543, + "learning_rate": 8.348280226706722e-09, + "logits/chosen": 0.8340524435043335, + "logits/rejected": 0.8326983451843262, + "logps/chosen": -220.50311279296875, + "logps/ref_chosen": -53.98310852050781, + "logps/ref_rejected": -58.32208251953125, + "logps/rejected": -318.7952880859375, + "loss": 1.0976, + "margin_dpo/margin_mean": 93.95318603515625, + "margin_dpo/margin_std": 131.71676635742188, + "step": 613 + }, + { + "KL/chosen_KL_mean": -189.92941284179688, + "KL/mean": -233.43902587890625, + "KL/rejected_KL_mean": -276.9486389160156, + "KL/std": 110.43850708007812, + "epoch": 0.9281934996220711, + "fcm_dpo/beta": 0.004416479729115963, + "fcm_dpo/delta": 0.014550581574440002, + "fcm_dpo/margin": 87.01921844482422, + "fcm_dpo/q_t": 0.41044163703918457, + "grad_norm": 13.81963062286377, + "learning_rate": 8.012824650910937e-09, + "logits/chosen": 0.8715497851371765, + "logits/rejected": 0.8666037321090698, + "logps/chosen": -250.17245483398438, + "logps/ref_chosen": -60.24303436279297, + "logps/ref_rejected": -72.26258850097656, + "logps/rejected": -349.21124267578125, + "loss": 1.1157, + "margin_dpo/margin_mean": 87.01921844482422, + "margin_dpo/margin_std": 125.52960205078125, + "step": 614 + }, + { + "KL/chosen_KL_mean": -195.23587036132812, + "KL/mean": -247.610107421875, + "KL/rejected_KL_mean": -299.9843444824219, + "KL/std": 148.3843994140625, + "epoch": 0.9297052154195011, + "fcm_dpo/beta": 0.004148758016526699, + "fcm_dpo/delta": -0.04528382048010826, + "fcm_dpo/margin": 104.74848937988281, + "fcm_dpo/q_t": 0.40149933099746704, + "grad_norm": 11.946541786193848, + "learning_rate": 7.684137976598088e-09, + "logits/chosen": 0.8352917432785034, + "logits/rejected": 0.7855337858200073, + "logps/chosen": -267.3305358886719, + "logps/ref_chosen": -72.09467315673828, + "logps/ref_rejected": -104.02980041503906, + "logps/rejected": -404.01416015625, + "loss": 1.1023, + "margin_dpo/margin_mean": 104.74848937988281, + "margin_dpo/margin_std": 158.98904418945312, + "step": 615 + }, + { + "KL/chosen_KL_mean": -187.78256225585938, + "KL/mean": -227.31333923339844, + "KL/rejected_KL_mean": -266.8441162109375, + "KL/std": 136.93423461914062, + "epoch": 0.9312169312169312, + "fcm_dpo/beta": 0.004337611608207226, + "fcm_dpo/delta": 0.06366471946239471, + "fcm_dpo/margin": 79.06153869628906, + "fcm_dpo/q_t": 0.4219726324081421, + "grad_norm": 11.41348648071289, + "learning_rate": 7.36222939784098e-09, + "logits/chosen": 0.8961197137832642, + "logits/rejected": 0.8402373194694519, + "logps/chosen": -246.31329345703125, + "logps/ref_chosen": -58.530723571777344, + "logps/ref_rejected": -75.48025512695312, + "logps/rejected": -342.3243713378906, + "loss": 1.1435, + "margin_dpo/margin_mean": 79.06153869628906, + "margin_dpo/margin_std": 126.18814086914062, + "step": 616 + }, + { + "KL/chosen_KL_mean": -195.61953735351562, + "KL/mean": -232.73971557617188, + "KL/rejected_KL_mean": -269.85992431640625, + "KL/std": 123.03448486328125, + "epoch": 0.9327286470143613, + "fcm_dpo/beta": 0.004304808564484119, + "fcm_dpo/delta": -0.031089257448911667, + "fcm_dpo/margin": 74.24037170410156, + "fcm_dpo/q_t": 0.4274696111679077, + "grad_norm": 15.603584289550781, + "learning_rate": 7.047107919114586e-09, + "logits/chosen": 0.8908584713935852, + "logits/rejected": 0.8490594029426575, + "logps/chosen": -253.2282257080078, + "logps/ref_chosen": -57.608673095703125, + "logps/ref_rejected": -81.22109985351562, + "logps/rejected": -351.0810241699219, + "loss": 1.1746, + "margin_dpo/margin_mean": 74.24037170410156, + "margin_dpo/margin_std": 133.52430725097656, + "step": 617 + }, + { + "KL/chosen_KL_mean": -182.47540283203125, + "KL/mean": -227.52099609375, + "KL/rejected_KL_mean": -272.5666198730469, + "KL/std": 136.0541229248047, + "epoch": 0.9342403628117913, + "fcm_dpo/beta": 0.004296266473829746, + "fcm_dpo/delta": 0.014270158484578133, + "fcm_dpo/margin": 90.0912094116211, + "fcm_dpo/q_t": 0.4141734838485718, + "grad_norm": 15.708011627197266, + "learning_rate": 6.738782355044048e-09, + "logits/chosen": 0.8595304489135742, + "logits/rejected": 0.7856009006500244, + "logps/chosen": -239.17135620117188, + "logps/ref_chosen": -56.69594192504883, + "logps/ref_rejected": -85.92362976074219, + "logps/rejected": -358.490234375, + "loss": 1.1172, + "margin_dpo/margin_mean": 90.09120178222656, + "margin_dpo/margin_std": 138.923583984375, + "step": 618 + }, + { + "KL/chosen_KL_mean": -174.67626953125, + "KL/mean": -222.02593994140625, + "KL/rejected_KL_mean": -269.3756408691406, + "KL/std": 125.88426971435547, + "epoch": 0.9357520786092215, + "fcm_dpo/beta": 0.004293524194508791, + "fcm_dpo/delta": -0.007992902770638466, + "fcm_dpo/margin": 94.6993637084961, + "fcm_dpo/q_t": 0.40730637311935425, + "grad_norm": 11.522953987121582, + "learning_rate": 6.437261330158206e-09, + "logits/chosen": 0.9194344282150269, + "logits/rejected": 0.8575088977813721, + "logps/chosen": -228.73468017578125, + "logps/ref_chosen": -54.05841827392578, + "logps/ref_rejected": -83.55493927001953, + "logps/rejected": -352.9305725097656, + "loss": 1.103, + "margin_dpo/margin_mean": 94.6993637084961, + "margin_dpo/margin_std": 138.9624481201172, + "step": 619 + }, + { + "KL/chosen_KL_mean": -182.78765869140625, + "KL/mean": -217.02841186523438, + "KL/rejected_KL_mean": -251.26914978027344, + "KL/std": 116.54869842529297, + "epoch": 0.9372637944066515, + "fcm_dpo/beta": 0.004260548390448093, + "fcm_dpo/delta": -0.04734001308679581, + "fcm_dpo/margin": 68.48147583007812, + "fcm_dpo/q_t": 0.43087059259414673, + "grad_norm": 12.573121070861816, + "learning_rate": 6.142553278648238e-09, + "logits/chosen": 0.8781119585037231, + "logits/rejected": 0.8820315599441528, + "logps/chosen": -246.15737915039062, + "logps/ref_chosen": -63.36971664428711, + "logps/ref_rejected": -65.68269348144531, + "logps/rejected": -316.95184326171875, + "loss": 1.1914, + "margin_dpo/margin_mean": 68.48147583007812, + "margin_dpo/margin_std": 121.66095733642578, + "step": 620 + }, + { + "KL/chosen_KL_mean": -175.8046875, + "KL/mean": -221.00616455078125, + "KL/rejected_KL_mean": -266.2076416015625, + "KL/std": 130.67002868652344, + "epoch": 0.9387755102040817, + "fcm_dpo/beta": 0.004282790701836348, + "fcm_dpo/delta": 0.012993309646844864, + "fcm_dpo/margin": 90.40296936035156, + "fcm_dpo/q_t": 0.415774405002594, + "grad_norm": 12.253860473632812, + "learning_rate": 5.854666444131934e-09, + "logits/chosen": 0.9052057862281799, + "logits/rejected": 0.8152543902397156, + "logps/chosen": -228.12591552734375, + "logps/ref_chosen": -52.321224212646484, + "logps/ref_rejected": -88.09001159667969, + "logps/rejected": -354.29766845703125, + "loss": 1.1415, + "margin_dpo/margin_mean": 90.4029769897461, + "margin_dpo/margin_std": 156.630859375, + "step": 621 + }, + { + "KL/chosen_KL_mean": -193.3104248046875, + "KL/mean": -235.54830932617188, + "KL/rejected_KL_mean": -277.78619384765625, + "KL/std": 133.7432861328125, + "epoch": 0.9402872260015117, + "fcm_dpo/beta": 0.004352467134594917, + "fcm_dpo/delta": 0.034792713820934296, + "fcm_dpo/margin": 84.47576904296875, + "fcm_dpo/q_t": 0.41774168610572815, + "grad_norm": 13.547627449035645, + "learning_rate": 5.573608879422875e-09, + "logits/chosen": 0.8366628885269165, + "logits/rejected": 0.797753095626831, + "logps/chosen": -253.17588806152344, + "logps/ref_chosen": -59.86545944213867, + "logps/ref_rejected": -81.86668395996094, + "logps/rejected": -359.65289306640625, + "loss": 1.1378, + "margin_dpo/margin_mean": 84.47576904296875, + "margin_dpo/margin_std": 138.1892852783203, + "step": 622 + }, + { + "KL/chosen_KL_mean": -206.13430786132812, + "KL/mean": -251.16110229492188, + "KL/rejected_KL_mean": -296.18792724609375, + "KL/std": 141.82073974609375, + "epoch": 0.9417989417989417, + "fcm_dpo/beta": 0.004277583211660385, + "fcm_dpo/delta": 0.014890506863594055, + "fcm_dpo/margin": 90.05360412597656, + "fcm_dpo/q_t": 0.4128887951374054, + "grad_norm": 12.97014045715332, + "learning_rate": 5.299388446305342e-09, + "logits/chosen": 0.8766790628433228, + "logits/rejected": 0.8372782468795776, + "logps/chosen": -273.50274658203125, + "logps/ref_chosen": -67.36846160888672, + "logps/ref_rejected": -82.02733612060547, + "logps/rejected": -378.21527099609375, + "loss": 1.1335, + "margin_dpo/margin_mean": 90.05360412597656, + "margin_dpo/margin_std": 147.62298583984375, + "step": 623 + }, + { + "KL/chosen_KL_mean": -175.81561279296875, + "KL/mean": -221.06178283691406, + "KL/rejected_KL_mean": -266.30792236328125, + "KL/std": 129.63758850097656, + "epoch": 0.9433106575963719, + "fcm_dpo/beta": 0.0043399459682404995, + "fcm_dpo/delta": 0.007810791954398155, + "fcm_dpo/margin": 90.49232482910156, + "fcm_dpo/q_t": 0.41192591190338135, + "grad_norm": 14.457840919494629, + "learning_rate": 5.03201281531429e-09, + "logits/chosen": 0.8671579957008362, + "logits/rejected": 0.7905720472335815, + "logps/chosen": -226.8421630859375, + "logps/ref_chosen": -51.02655029296875, + "logps/ref_rejected": -76.49203491210938, + "logps/rejected": -342.79998779296875, + "loss": 1.117, + "margin_dpo/margin_mean": 90.49234008789062, + "margin_dpo/margin_std": 139.15968322753906, + "step": 624 + }, + { + "KL/chosen_KL_mean": -175.61386108398438, + "KL/mean": -212.56109619140625, + "KL/rejected_KL_mean": -249.50831604003906, + "KL/std": 123.15960693359375, + "epoch": 0.9448223733938019, + "fcm_dpo/beta": 0.004533303901553154, + "fcm_dpo/delta": 0.07208952307701111, + "fcm_dpo/margin": 73.89444732666016, + "fcm_dpo/q_t": 0.427746057510376, + "grad_norm": 12.597397804260254, + "learning_rate": 4.7714894655209174e-09, + "logits/chosen": 0.9103964567184448, + "logits/rejected": 0.836378276348114, + "logps/chosen": -229.82147216796875, + "logps/ref_chosen": -54.20761489868164, + "logps/ref_rejected": -84.93669128417969, + "logps/rejected": -334.44500732421875, + "loss": 1.1816, + "margin_dpo/margin_mean": 73.89444732666016, + "margin_dpo/margin_std": 147.33132934570312, + "step": 625 + }, + { + "KL/chosen_KL_mean": -165.85633850097656, + "KL/mean": -216.77398681640625, + "KL/rejected_KL_mean": -267.691650390625, + "KL/std": 134.21945190429688, + "epoch": 0.9463340891912321, + "fcm_dpo/beta": 0.004439115524291992, + "fcm_dpo/delta": -0.06053989380598068, + "fcm_dpo/margin": 101.8353271484375, + "fcm_dpo/q_t": 0.40118056535720825, + "grad_norm": 13.96564769744873, + "learning_rate": 4.517825684323323e-09, + "logits/chosen": 0.9465058445930481, + "logits/rejected": 0.8354614973068237, + "logps/chosen": -210.91836547851562, + "logps/ref_chosen": -45.06201934814453, + "logps/ref_rejected": -89.66368103027344, + "logps/rejected": -357.3553466796875, + "loss": 1.0962, + "margin_dpo/margin_mean": 101.8353271484375, + "margin_dpo/margin_std": 162.71804809570312, + "step": 626 + }, + { + "KL/chosen_KL_mean": -190.51968383789062, + "KL/mean": -244.76681518554688, + "KL/rejected_KL_mean": -299.0139465332031, + "KL/std": 139.9709930419922, + "epoch": 0.9478458049886621, + "fcm_dpo/beta": 0.00424289982765913, + "fcm_dpo/delta": -0.07100832462310791, + "fcm_dpo/margin": 108.49424743652344, + "fcm_dpo/q_t": 0.39188235998153687, + "grad_norm": 12.912845611572266, + "learning_rate": 4.271028567242818e-09, + "logits/chosen": 0.8388135433197021, + "logits/rejected": 0.7388994693756104, + "logps/chosen": -249.31072998046875, + "logps/ref_chosen": -58.791053771972656, + "logps/ref_rejected": -94.90802001953125, + "logps/rejected": -393.9219665527344, + "loss": 1.0535, + "margin_dpo/margin_mean": 108.49424743652344, + "margin_dpo/margin_std": 133.7566375732422, + "step": 627 + }, + { + "KL/chosen_KL_mean": -177.43270874023438, + "KL/mean": -227.32901000976562, + "KL/rejected_KL_mean": -277.22528076171875, + "KL/std": 129.86306762695312, + "epoch": 0.9493575207860923, + "fcm_dpo/beta": 0.004253713879734278, + "fcm_dpo/delta": -0.030077699571847916, + "fcm_dpo/margin": 99.7926025390625, + "fcm_dpo/q_t": 0.4006233811378479, + "grad_norm": 13.415197372436523, + "learning_rate": 4.0311050177251895e-09, + "logits/chosen": 0.8698492646217346, + "logits/rejected": 0.838459849357605, + "logps/chosen": -230.2362823486328, + "logps/ref_chosen": -52.80357360839844, + "logps/ref_rejected": -76.49468994140625, + "logps/rejected": -353.719970703125, + "loss": 1.1146, + "margin_dpo/margin_mean": 99.7926025390625, + "margin_dpo/margin_std": 148.1082763671875, + "step": 628 + }, + { + "KL/chosen_KL_mean": -189.38441467285156, + "KL/mean": -225.45184326171875, + "KL/rejected_KL_mean": -261.51922607421875, + "KL/std": 118.73931884765625, + "epoch": 0.9508692365835223, + "fcm_dpo/beta": 0.004356414079666138, + "fcm_dpo/delta": 0.09377999603748322, + "fcm_dpo/margin": 72.13484191894531, + "fcm_dpo/q_t": 0.43087315559387207, + "grad_norm": 11.81552505493164, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": 0.8537815809249878, + "logits/rejected": 0.857978343963623, + "logps/chosen": -260.1019287109375, + "logps/ref_chosen": -70.71749877929688, + "logps/ref_rejected": -78.96273803710938, + "logps/rejected": -340.48199462890625, + "loss": 1.1703, + "margin_dpo/margin_mean": 72.13484191894531, + "margin_dpo/margin_std": 128.88232421875, + "step": 629 + }, + { + "KL/chosen_KL_mean": -175.94125366210938, + "KL/mean": -225.40545654296875, + "KL/rejected_KL_mean": -274.8696594238281, + "KL/std": 126.93406677246094, + "epoch": 0.9523809523809523, + "fcm_dpo/beta": 0.004350777715444565, + "fcm_dpo/delta": -0.03495955839753151, + "fcm_dpo/margin": 98.92840576171875, + "fcm_dpo/q_t": 0.4007263779640198, + "grad_norm": 10.979743003845215, + "learning_rate": 3.5719052736323806e-09, + "logits/chosen": 0.8208729028701782, + "logits/rejected": 0.7930917739868164, + "logps/chosen": -232.14266967773438, + "logps/ref_chosen": -56.201412200927734, + "logps/ref_rejected": -74.69807434082031, + "logps/rejected": -349.5677490234375, + "loss": 1.0891, + "margin_dpo/margin_mean": 98.92840576171875, + "margin_dpo/margin_std": 140.695556640625, + "step": 630 + }, + { + "KL/chosen_KL_mean": -166.6142120361328, + "KL/mean": -226.4366455078125, + "KL/rejected_KL_mean": -286.25909423828125, + "KL/std": 137.6940460205078, + "epoch": 0.9538926681783825, + "fcm_dpo/beta": 0.00401288690045476, + "fcm_dpo/delta": -0.10077112913131714, + "fcm_dpo/margin": 119.64486694335938, + "fcm_dpo/q_t": 0.3887532353401184, + "grad_norm": 11.224316596984863, + "learning_rate": 3.352641923861144e-09, + "logits/chosen": 0.9621773362159729, + "logits/rejected": 0.8688410520553589, + "logps/chosen": -225.434814453125, + "logps/ref_chosen": -58.82059860229492, + "logps/ref_rejected": -96.51437377929688, + "logps/rejected": -382.7734680175781, + "loss": 1.052, + "margin_dpo/margin_mean": 119.64486694335938, + "margin_dpo/margin_std": 151.0715789794922, + "step": 631 + }, + { + "KL/chosen_KL_mean": -169.34906005859375, + "KL/mean": -222.4647216796875, + "KL/rejected_KL_mean": -275.58038330078125, + "KL/std": 118.47867584228516, + "epoch": 0.9554043839758125, + "fcm_dpo/beta": 0.004019321873784065, + "fcm_dpo/delta": -0.031023576855659485, + "fcm_dpo/margin": 106.2313232421875, + "fcm_dpo/q_t": 0.39956796169281006, + "grad_norm": 11.955537796020508, + "learning_rate": 3.140277830901428e-09, + "logits/chosen": 0.8837977051734924, + "logits/rejected": 0.8737013339996338, + "logps/chosen": -228.13510131835938, + "logps/ref_chosen": -58.786048889160156, + "logps/ref_rejected": -67.21923828125, + "logps/rejected": -342.79962158203125, + "loss": 1.0674, + "margin_dpo/margin_mean": 106.2313232421875, + "margin_dpo/margin_std": 127.79652404785156, + "step": 632 + }, + { + "KL/chosen_KL_mean": -165.90658569335938, + "KL/mean": -208.74703979492188, + "KL/rejected_KL_mean": -251.58749389648438, + "KL/std": 116.66265869140625, + "epoch": 0.9569160997732427, + "fcm_dpo/beta": 0.004116720985621214, + "fcm_dpo/delta": 0.05110543221235275, + "fcm_dpo/margin": 85.68087768554688, + "fcm_dpo/q_t": 0.42053651809692383, + "grad_norm": 12.592107772827148, + "learning_rate": 2.9348189350335007e-09, + "logits/chosen": 0.8748718500137329, + "logits/rejected": 0.8350610733032227, + "logps/chosen": -218.0367889404297, + "logps/ref_chosen": -52.13019561767578, + "logps/ref_rejected": -67.23016357421875, + "logps/rejected": -318.817626953125, + "loss": 1.1355, + "margin_dpo/margin_mean": 85.68087768554688, + "margin_dpo/margin_std": 131.61073303222656, + "step": 633 + }, + { + "KL/chosen_KL_mean": -198.66600036621094, + "KL/mean": -213.45736694335938, + "KL/rejected_KL_mean": -228.24871826171875, + "KL/std": 122.29270935058594, + "epoch": 0.9584278155706727, + "fcm_dpo/beta": 0.004098186735063791, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 29.582706451416016, + "fcm_dpo/q_t": 0.47323864698410034, + "grad_norm": 14.698355674743652, + "learning_rate": 2.736270983384276e-09, + "logits/chosen": 0.9170514941215515, + "logits/rejected": 0.9342811703681946, + "logps/chosen": -259.64581298828125, + "logps/ref_chosen": -60.97979736328125, + "logps/ref_rejected": -58.50825119018555, + "logps/rejected": -286.7569580078125, + "loss": 1.3477, + "margin_dpo/margin_mean": 29.582706451416016, + "margin_dpo/margin_std": 139.38327026367188, + "step": 634 + }, + { + "KL/chosen_KL_mean": -215.3644256591797, + "KL/mean": -250.3736572265625, + "KL/rejected_KL_mean": -285.38287353515625, + "KL/std": 116.93312072753906, + "epoch": 0.9599395313681028, + "fcm_dpo/beta": 0.004337867721915245, + "fcm_dpo/delta": 0.10398625582456589, + "fcm_dpo/margin": 70.0184555053711, + "fcm_dpo/q_t": 0.43173298239707947, + "grad_norm": 11.96898078918457, + "learning_rate": 2.5446395297668287e-09, + "logits/chosen": 0.7966681718826294, + "logits/rejected": 0.7563410997390747, + "logps/chosen": -281.3374328613281, + "logps/ref_chosen": -65.9730224609375, + "logps/ref_rejected": -85.61317443847656, + "logps/rejected": -370.99603271484375, + "loss": 1.1911, + "margin_dpo/margin_mean": 70.0184555053711, + "margin_dpo/margin_std": 137.1169891357422, + "step": 635 + }, + { + "KL/chosen_KL_mean": -167.98959350585938, + "KL/mean": -215.552490234375, + "KL/rejected_KL_mean": -263.1153564453125, + "KL/std": 120.42471313476562, + "epoch": 0.9614512471655329, + "fcm_dpo/beta": 0.004312177188694477, + "fcm_dpo/delta": -0.011981801129877567, + "fcm_dpo/margin": 95.12577819824219, + "fcm_dpo/q_t": 0.4046896696090698, + "grad_norm": 9.827645301818848, + "learning_rate": 2.359929934524829e-09, + "logits/chosen": 0.8474336862564087, + "logits/rejected": 0.7673357129096985, + "logps/chosen": -217.1297607421875, + "logps/ref_chosen": -49.140167236328125, + "logps/ref_rejected": -81.26971435546875, + "logps/rejected": -344.38507080078125, + "loss": 1.0884, + "margin_dpo/margin_mean": 95.12577819824219, + "margin_dpo/margin_std": 126.56785583496094, + "step": 636 + }, + { + "KL/chosen_KL_mean": -226.68218994140625, + "KL/mean": -262.2201232910156, + "KL/rejected_KL_mean": -297.758056640625, + "KL/std": 124.49876403808594, + "epoch": 0.9629629629629629, + "fcm_dpo/beta": 0.0045396797358989716, + "fcm_dpo/delta": 0.08410540223121643, + "fcm_dpo/margin": 71.07585144042969, + "fcm_dpo/q_t": 0.4289219081401825, + "grad_norm": 13.477401733398438, + "learning_rate": 2.1821473643827137e-09, + "logits/chosen": 0.8291600942611694, + "logits/rejected": 0.7868282794952393, + "logps/chosen": -300.3787841796875, + "logps/ref_chosen": -73.69658660888672, + "logps/ref_rejected": -83.01487731933594, + "logps/rejected": -380.77294921875, + "loss": 1.1968, + "margin_dpo/margin_mean": 71.07585144042969, + "margin_dpo/margin_std": 150.5892333984375, + "step": 637 + }, + { + "KL/chosen_KL_mean": -196.17276000976562, + "KL/mean": -238.7854461669922, + "KL/rejected_KL_mean": -281.39813232421875, + "KL/std": 120.97940063476562, + "epoch": 0.9644746787603931, + "fcm_dpo/beta": 0.0045569646172225475, + "fcm_dpo/delta": 0.013032622635364532, + "fcm_dpo/margin": 85.22538757324219, + "fcm_dpo/q_t": 0.4132801294326782, + "grad_norm": 13.217300415039062, + "learning_rate": 2.0112967923011646e-09, + "logits/chosen": 0.8621759414672852, + "logits/rejected": 0.8292132616043091, + "logps/chosen": -258.954345703125, + "logps/ref_chosen": -62.78158187866211, + "logps/ref_rejected": -85.40478515625, + "logps/rejected": -366.80291748046875, + "loss": 1.1234, + "margin_dpo/margin_mean": 85.22538757324219, + "margin_dpo/margin_std": 135.74893188476562, + "step": 638 + }, + { + "KL/chosen_KL_mean": -175.19105529785156, + "KL/mean": -225.06832885742188, + "KL/rejected_KL_mean": -274.94561767578125, + "KL/std": 131.4615478515625, + "epoch": 0.9659863945578231, + "fcm_dpo/beta": 0.004495006985962391, + "fcm_dpo/delta": -0.05581279098987579, + "fcm_dpo/margin": 99.75454711914062, + "fcm_dpo/q_t": 0.39895889163017273, + "grad_norm": 15.222760200500488, + "learning_rate": 1.847382997337943e-09, + "logits/chosen": 0.8577284812927246, + "logits/rejected": 0.7885833978652954, + "logps/chosen": -228.9576416015625, + "logps/ref_chosen": -53.76658630371094, + "logps/ref_rejected": -72.30009460449219, + "logps/rejected": -347.2456970214844, + "loss": 1.0828, + "margin_dpo/margin_mean": 99.75454711914062, + "margin_dpo/margin_std": 143.67074584960938, + "step": 639 + }, + { + "KL/chosen_KL_mean": -182.3665771484375, + "KL/mean": -225.64895629882812, + "KL/rejected_KL_mean": -268.9313049316406, + "KL/std": 123.47915649414062, + "epoch": 0.9674981103552532, + "fcm_dpo/beta": 0.004439163953065872, + "fcm_dpo/delta": 0.017474712803959846, + "fcm_dpo/margin": 86.56473541259766, + "fcm_dpo/q_t": 0.4128446578979492, + "grad_norm": 13.212712287902832, + "learning_rate": 1.690410564514244e-09, + "logits/chosen": 0.9096243381500244, + "logits/rejected": 0.8597659468650818, + "logps/chosen": -233.78436279296875, + "logps/ref_chosen": -51.41777801513672, + "logps/ref_rejected": -77.27879333496094, + "logps/rejected": -346.2101135253906, + "loss": 1.1238, + "margin_dpo/margin_mean": 86.56473541259766, + "margin_dpo/margin_std": 135.56790161132812, + "step": 640 + }, + { + "KL/chosen_KL_mean": -191.44329833984375, + "KL/mean": -231.0288543701172, + "KL/rejected_KL_mean": -270.6143798828125, + "KL/std": 114.08880615234375, + "epoch": 0.9690098261526833, + "fcm_dpo/beta": 0.004546813666820526, + "fcm_dpo/delta": 0.04486197605729103, + "fcm_dpo/margin": 79.17110443115234, + "fcm_dpo/q_t": 0.41722530126571655, + "grad_norm": 13.189962387084961, + "learning_rate": 1.5403838846864692e-09, + "logits/chosen": 0.8492909073829651, + "logits/rejected": 0.8432836532592773, + "logps/chosen": -262.49798583984375, + "logps/ref_chosen": -71.0546646118164, + "logps/ref_rejected": -82.2440185546875, + "logps/rejected": -352.8583984375, + "loss": 1.1164, + "margin_dpo/margin_mean": 79.17110443115234, + "margin_dpo/margin_std": 108.83973693847656, + "step": 641 + }, + { + "KL/chosen_KL_mean": -200.26123046875, + "KL/mean": -229.521728515625, + "KL/rejected_KL_mean": -258.7822265625, + "KL/std": 112.85456085205078, + "epoch": 0.9705215419501134, + "fcm_dpo/beta": 0.004678209777921438, + "fcm_dpo/delta": 0.02495141327381134, + "fcm_dpo/margin": 58.52101135253906, + "fcm_dpo/q_t": 0.43944495916366577, + "grad_norm": 15.50776481628418, + "learning_rate": 1.3973071544233218e-09, + "logits/chosen": 0.8077861070632935, + "logits/rejected": 0.8297359943389893, + "logps/chosen": -269.19049072265625, + "logps/ref_chosen": -68.92927551269531, + "logps/ref_rejected": -70.85682678222656, + "logps/rejected": -329.6390686035156, + "loss": 1.2261, + "margin_dpo/margin_mean": 58.521018981933594, + "margin_dpo/margin_std": 135.40769958496094, + "step": 642 + }, + { + "KL/chosen_KL_mean": -182.4813232421875, + "KL/mean": -230.08364868164062, + "KL/rejected_KL_mean": -277.68597412109375, + "KL/std": 124.2804946899414, + "epoch": 0.9720332577475435, + "fcm_dpo/beta": 0.004592161625623703, + "fcm_dpo/delta": -0.04280085489153862, + "fcm_dpo/margin": 95.20464324951172, + "fcm_dpo/q_t": 0.401271790266037, + "grad_norm": 18.248327255249023, + "learning_rate": 1.261184375888541e-09, + "logits/chosen": 0.8441503047943115, + "logits/rejected": 0.778213381767273, + "logps/chosen": -247.79037475585938, + "logps/ref_chosen": -65.30903625488281, + "logps/ref_rejected": -83.61613464355469, + "logps/rejected": -361.3021240234375, + "loss": 1.0961, + "margin_dpo/margin_mean": 95.20463562011719, + "margin_dpo/margin_std": 142.61578369140625, + "step": 643 + }, + { + "KL/chosen_KL_mean": -173.4718017578125, + "KL/mean": -202.7007293701172, + "KL/rejected_KL_mean": -231.92965698242188, + "KL/std": 106.54061889648438, + "epoch": 0.9735449735449735, + "fcm_dpo/beta": 0.004578079096972942, + "fcm_dpo/delta": 0.013454647734761238, + "fcm_dpo/margin": 58.45787048339844, + "fcm_dpo/q_t": 0.43763917684555054, + "grad_norm": 11.429241180419922, + "learning_rate": 1.1320193567288527e-09, + "logits/chosen": 0.939828634262085, + "logits/rejected": 0.912724494934082, + "logps/chosen": -224.47439575195312, + "logps/ref_chosen": -51.002601623535156, + "logps/ref_rejected": -64.46372985839844, + "logps/rejected": -296.39337158203125, + "loss": 1.2201, + "margin_dpo/margin_mean": 58.45787048339844, + "margin_dpo/margin_std": 126.97639465332031, + "step": 644 + }, + { + "KL/chosen_KL_mean": -190.52029418945312, + "KL/mean": -227.52386474609375, + "KL/rejected_KL_mean": -264.5274353027344, + "KL/std": 117.05931091308594, + "epoch": 0.9750566893424036, + "fcm_dpo/beta": 0.004725134931504726, + "fcm_dpo/delta": 0.05598610267043114, + "fcm_dpo/margin": 74.00714111328125, + "fcm_dpo/q_t": 0.4198160469532013, + "grad_norm": 16.172887802124023, + "learning_rate": 1.0098157099674987e-09, + "logits/chosen": 0.868065357208252, + "logits/rejected": 0.8554788827896118, + "logps/chosen": -251.48370361328125, + "logps/ref_chosen": -60.963409423828125, + "logps/ref_rejected": -69.73353576660156, + "logps/rejected": -334.260986328125, + "loss": 1.1443, + "margin_dpo/margin_mean": 74.00714111328125, + "margin_dpo/margin_std": 120.89301300048828, + "step": 645 + }, + { + "KL/chosen_KL_mean": -205.87545776367188, + "KL/mean": -243.1065673828125, + "KL/rejected_KL_mean": -280.3376770019531, + "KL/std": 141.6141357421875, + "epoch": 0.9765684051398337, + "fcm_dpo/beta": 0.004856896586716175, + "fcm_dpo/delta": 0.04285380616784096, + "fcm_dpo/margin": 74.46222686767578, + "fcm_dpo/q_t": 0.4203829765319824, + "grad_norm": 12.586745262145996, + "learning_rate": 8.945768539031783e-10, + "logits/chosen": 0.8746490478515625, + "logits/rejected": 0.8254890441894531, + "logps/chosen": -268.16552734375, + "logps/ref_chosen": -62.290069580078125, + "logps/ref_rejected": -85.54812622070312, + "logps/rejected": -365.88580322265625, + "loss": 1.1618, + "margin_dpo/margin_mean": 74.46221923828125, + "margin_dpo/margin_std": 138.90460205078125, + "step": 646 + }, + { + "KL/chosen_KL_mean": -203.10272216796875, + "KL/mean": -261.1683654785156, + "KL/rejected_KL_mean": -319.2340087890625, + "KL/std": 138.07496643066406, + "epoch": 0.9780801209372638, + "fcm_dpo/beta": 0.004566292278468609, + "fcm_dpo/delta": -0.15473519265651703, + "fcm_dpo/margin": 116.13129425048828, + "fcm_dpo/q_t": 0.3759966492652893, + "grad_norm": 14.314106941223145, + "learning_rate": 7.863060120144316e-10, + "logits/chosen": 0.8773380517959595, + "logits/rejected": 0.7988033294677734, + "logps/chosen": -270.61859130859375, + "logps/ref_chosen": -67.515869140625, + "logps/ref_rejected": -101.50871276855469, + "logps/rejected": -420.7427062988281, + "loss": 1.0135, + "margin_dpo/margin_mean": 116.13129425048828, + "margin_dpo/margin_std": 136.10638427734375, + "step": 647 + }, + { + "KL/chosen_KL_mean": -204.65838623046875, + "KL/mean": -243.85040283203125, + "KL/rejected_KL_mean": -283.04241943359375, + "KL/std": 119.53384399414062, + "epoch": 0.9795918367346939, + "fcm_dpo/beta": 0.004495234228670597, + "fcm_dpo/delta": 0.05285690724849701, + "fcm_dpo/margin": 78.38400268554688, + "fcm_dpo/q_t": 0.422693133354187, + "grad_norm": 12.690834999084473, + "learning_rate": 6.850062128694045e-10, + "logits/chosen": 0.8503772020339966, + "logits/rejected": 0.8010015487670898, + "logps/chosen": -269.25433349609375, + "logps/ref_chosen": -64.59593963623047, + "logps/ref_rejected": -83.384033203125, + "logps/rejected": -366.42645263671875, + "loss": 1.1861, + "margin_dpo/margin_mean": 78.38400268554688, + "margin_dpo/margin_std": 156.04074096679688, + "step": 648 + }, + { + "KL/chosen_KL_mean": -184.55648803710938, + "KL/mean": -225.07196044921875, + "KL/rejected_KL_mean": -265.5874328613281, + "KL/std": 115.69061279296875, + "epoch": 0.981103552532124, + "fcm_dpo/beta": 0.0046140593476593494, + "fcm_dpo/delta": 0.0292498916387558, + "fcm_dpo/margin": 81.03094482421875, + "fcm_dpo/q_t": 0.4144395589828491, + "grad_norm": 17.35785484313965, + "learning_rate": 5.906802900412788e-10, + "logits/chosen": 0.9027397632598877, + "logits/rejected": 0.850858211517334, + "logps/chosen": -233.8661346435547, + "logps/ref_chosen": -49.30964660644531, + "logps/ref_rejected": -73.73710632324219, + "logps/rejected": -339.32452392578125, + "loss": 1.1432, + "margin_dpo/margin_mean": 81.03094482421875, + "margin_dpo/margin_std": 137.96710205078125, + "step": 649 + }, + { + "KL/chosen_KL_mean": -178.10488891601562, + "KL/mean": -222.58074951171875, + "KL/rejected_KL_mean": -267.05657958984375, + "KL/std": 120.7571029663086, + "epoch": 0.982615268329554, + "fcm_dpo/beta": 0.004608334973454475, + "fcm_dpo/delta": -0.011857246980071068, + "fcm_dpo/margin": 88.95169067382812, + "fcm_dpo/q_t": 0.40797871351242065, + "grad_norm": 13.712696075439453, + "learning_rate": 5.033308820289184e-10, + "logits/chosen": 0.9457135200500488, + "logits/rejected": 0.896229088306427, + "logps/chosen": -233.16815185546875, + "logps/ref_chosen": -55.06325912475586, + "logps/ref_rejected": -77.39610290527344, + "logps/rejected": -344.45269775390625, + "loss": 1.1262, + "margin_dpo/margin_mean": 88.95169067382812, + "margin_dpo/margin_std": 148.16856384277344, + "step": 650 + }, + { + "KL/chosen_KL_mean": -176.51058959960938, + "KL/mean": -214.7784881591797, + "KL/rejected_KL_mean": -253.04638671875, + "KL/std": 118.41923522949219, + "epoch": 0.9841269841269841, + "fcm_dpo/beta": 0.004761428106576204, + "fcm_dpo/delta": 0.03895041346549988, + "fcm_dpo/margin": 76.53578186035156, + "fcm_dpo/q_t": 0.4195203185081482, + "grad_norm": 13.958356857299805, + "learning_rate": 4.2296043218295606e-10, + "logits/chosen": 0.9171409606933594, + "logits/rejected": 0.8589807748794556, + "logps/chosen": -230.5757598876953, + "logps/ref_chosen": -54.065162658691406, + "logps/ref_rejected": -77.79080200195312, + "logps/rejected": -330.8371887207031, + "loss": 1.1358, + "margin_dpo/margin_mean": 76.53578186035156, + "margin_dpo/margin_std": 123.71870422363281, + "step": 651 + }, + { + "KL/chosen_KL_mean": -209.03729248046875, + "KL/mean": -242.5619659423828, + "KL/rejected_KL_mean": -276.086669921875, + "KL/std": 132.26441955566406, + "epoch": 0.9856386999244142, + "fcm_dpo/beta": 0.004862497560679913, + "fcm_dpo/delta": 0.0819351002573967, + "fcm_dpo/margin": 67.04937744140625, + "fcm_dpo/q_t": 0.4319889545440674, + "grad_norm": 16.18764877319336, + "learning_rate": 3.4957118863768176e-10, + "logits/chosen": 0.8910359144210815, + "logits/rejected": 0.8544408082962036, + "logps/chosen": -272.6775817871094, + "logps/ref_chosen": -63.64030456542969, + "logps/ref_rejected": -78.86882019042969, + "logps/rejected": -354.95550537109375, + "loss": 1.2244, + "margin_dpo/margin_mean": 67.04937744140625, + "margin_dpo/margin_std": 160.25506591796875, + "step": 652 + }, + { + "KL/chosen_KL_mean": -185.3717803955078, + "KL/mean": -227.00796508789062, + "KL/rejected_KL_mean": -268.6441955566406, + "KL/std": 115.03123474121094, + "epoch": 0.9871504157218443, + "fcm_dpo/beta": 0.0050035035237669945, + "fcm_dpo/delta": -0.01987006887793541, + "fcm_dpo/margin": 83.27240753173828, + "fcm_dpo/q_t": 0.4065983295440674, + "grad_norm": 15.927544593811035, + "learning_rate": 2.831652042480093e-10, + "logits/chosen": 0.8695135116577148, + "logits/rejected": 0.8330869674682617, + "logps/chosen": -247.04014587402344, + "logps/ref_chosen": -61.668373107910156, + "logps/ref_rejected": -73.83012390136719, + "logps/rejected": -342.47430419921875, + "loss": 1.1174, + "margin_dpo/margin_mean": 83.27241516113281, + "margin_dpo/margin_std": 133.44725036621094, + "step": 653 + }, + { + "KL/chosen_KL_mean": -192.90228271484375, + "KL/mean": -233.95965576171875, + "KL/rejected_KL_mean": -275.01702880859375, + "KL/std": 139.22682189941406, + "epoch": 0.9886621315192744, + "fcm_dpo/beta": 0.0046582408249378204, + "fcm_dpo/delta": -0.09315278381109238, + "fcm_dpo/margin": 82.11475372314453, + "fcm_dpo/q_t": 0.41140812635421753, + "grad_norm": 13.904915809631348, + "learning_rate": 2.2374433653205016e-10, + "logits/chosen": 0.8664817810058594, + "logits/rejected": 0.7882392406463623, + "logps/chosen": -250.47055053710938, + "logps/ref_chosen": -57.568267822265625, + "logps/ref_rejected": -87.74789428710938, + "logps/rejected": -362.7649230957031, + "loss": 1.1324, + "margin_dpo/margin_mean": 82.11475372314453, + "margin_dpo/margin_std": 129.8606719970703, + "step": 654 + }, + { + "KL/chosen_KL_mean": -152.3089599609375, + "KL/mean": -211.10946655273438, + "KL/rejected_KL_mean": -269.90997314453125, + "KL/std": 120.35342407226562, + "epoch": 0.9901738473167044, + "fcm_dpo/beta": 0.004261736758053303, + "fcm_dpo/delta": -0.1305772364139557, + "fcm_dpo/margin": 117.60102081298828, + "fcm_dpo/q_t": 0.38035333156585693, + "grad_norm": 11.05696964263916, + "learning_rate": 1.7131024761923852e-10, + "logits/chosen": 0.8656521439552307, + "logits/rejected": 0.7972902655601501, + "logps/chosen": -204.45611572265625, + "logps/ref_chosen": -52.14714813232422, + "logps/ref_rejected": -80.85014343261719, + "logps/rejected": -350.7601318359375, + "loss": 1.0168, + "margin_dpo/margin_mean": 117.60101318359375, + "margin_dpo/margin_std": 123.00428771972656, + "step": 655 + }, + { + "KL/chosen_KL_mean": -204.72833251953125, + "KL/mean": -246.2228546142578, + "KL/rejected_KL_mean": -287.7173767089844, + "KL/std": 123.21992492675781, + "epoch": 0.9916855631141346, + "fcm_dpo/beta": 0.004393207840621471, + "fcm_dpo/delta": 0.0396343432366848, + "fcm_dpo/margin": 82.98904418945312, + "fcm_dpo/q_t": 0.4176909923553467, + "grad_norm": 11.27653694152832, + "learning_rate": 1.2586440420372934e-10, + "logits/chosen": 0.8115413188934326, + "logits/rejected": 0.7813726663589478, + "logps/chosen": -277.98504638671875, + "logps/ref_chosen": -73.25672912597656, + "logps/ref_rejected": -85.35127258300781, + "logps/rejected": -373.06866455078125, + "loss": 1.138, + "margin_dpo/margin_mean": 82.98904418945312, + "margin_dpo/margin_std": 135.81607055664062, + "step": 656 + }, + { + "KL/chosen_KL_mean": -176.87982177734375, + "KL/mean": -229.69300842285156, + "KL/rejected_KL_mean": -282.5062255859375, + "KL/std": 135.0663604736328, + "epoch": 0.9931972789115646, + "fcm_dpo/beta": 0.0043113818392157555, + "fcm_dpo/delta": -0.06402204930782318, + "fcm_dpo/margin": 105.62638854980469, + "fcm_dpo/q_t": 0.3964206874370575, + "grad_norm": 11.030098915100098, + "learning_rate": 8.740807750345913e-11, + "logits/chosen": 0.9459801912307739, + "logits/rejected": 0.8843368291854858, + "logps/chosen": -226.60321044921875, + "logps/ref_chosen": -49.72339630126953, + "logps/ref_rejected": -75.1568603515625, + "logps/rejected": -357.6630859375, + "loss": 1.0776, + "margin_dpo/margin_mean": 105.62638854980469, + "margin_dpo/margin_std": 149.3883056640625, + "step": 657 + }, + { + "KL/chosen_KL_mean": -190.58401489257812, + "KL/mean": -233.33816528320312, + "KL/rejected_KL_mean": -276.09234619140625, + "KL/std": 143.47869873046875, + "epoch": 0.9947089947089947, + "fcm_dpo/beta": 0.004380069673061371, + "fcm_dpo/delta": 0.025314655154943466, + "fcm_dpo/margin": 85.5083236694336, + "fcm_dpo/q_t": 0.41635990142822266, + "grad_norm": 10.85285758972168, + "learning_rate": 5.594234322453539e-11, + "logits/chosen": 0.8922824859619141, + "logits/rejected": 0.8569316864013672, + "logps/chosen": -253.63035583496094, + "logps/ref_chosen": -63.04634094238281, + "logps/ref_rejected": -83.44963073730469, + "logps/rejected": -359.5419616699219, + "loss": 1.1715, + "margin_dpo/margin_mean": 85.50831604003906, + "margin_dpo/margin_std": 166.18760681152344, + "step": 658 + }, + { + "KL/chosen_KL_mean": -192.74554443359375, + "KL/mean": -224.75161743164062, + "KL/rejected_KL_mean": -256.7576904296875, + "KL/std": 123.44963073730469, + "epoch": 0.9962207105064248, + "fcm_dpo/beta": 0.004360191989690065, + "fcm_dpo/delta": 0.01944735273718834, + "fcm_dpo/margin": 64.01215362548828, + "fcm_dpo/q_t": 0.4362792372703552, + "grad_norm": 14.053616523742676, + "learning_rate": 3.146808153123293e-11, + "logits/chosen": 0.9274283051490784, + "logits/rejected": 0.8760404586791992, + "logps/chosen": -247.82574462890625, + "logps/ref_chosen": -55.0802001953125, + "logps/ref_rejected": -71.91049194335938, + "logps/rejected": -328.6681823730469, + "loss": 1.2169, + "margin_dpo/margin_mean": 64.01214599609375, + "margin_dpo/margin_std": 140.3470458984375, + "step": 659 + }, + { + "KL/chosen_KL_mean": -180.63677978515625, + "KL/mean": -235.10302734375, + "KL/rejected_KL_mean": -289.56927490234375, + "KL/std": 133.57896423339844, + "epoch": 0.9977324263038548, + "fcm_dpo/beta": 0.004226707387715578, + "fcm_dpo/delta": -0.07013650238513947, + "fcm_dpo/margin": 108.93248748779297, + "fcm_dpo/q_t": 0.3952319025993347, + "grad_norm": 13.667136192321777, + "learning_rate": 1.3985977021235829e-11, + "logits/chosen": 0.9608404636383057, + "logits/rejected": 0.9127137660980225, + "logps/chosen": -235.16268920898438, + "logps/ref_chosen": -54.525917053222656, + "logps/ref_rejected": -81.23604583740234, + "logps/rejected": -370.8052978515625, + "loss": 1.0608, + "margin_dpo/margin_mean": 108.93248748779297, + "margin_dpo/margin_std": 145.68505859375, + "step": 660 + }, + { + "KL/chosen_KL_mean": -210.68487548828125, + "KL/mean": -243.6689910888672, + "KL/rejected_KL_mean": -276.653076171875, + "KL/std": 131.54515075683594, + "epoch": 0.999244142101285, + "fcm_dpo/beta": 0.004441337194293737, + "fcm_dpo/delta": 0.1156606376171112, + "fcm_dpo/margin": 65.96818542480469, + "fcm_dpo/q_t": 0.4353044927120209, + "grad_norm": 11.607975959777832, + "learning_rate": 3.4965187065971735e-12, + "logits/chosen": 0.8547237515449524, + "logits/rejected": 0.7953514456748962, + "logps/chosen": -271.0575256347656, + "logps/ref_chosen": -60.37263870239258, + "logps/ref_rejected": -77.42874145507812, + "logps/rejected": -354.08184814453125, + "loss": 1.2196, + "margin_dpo/margin_mean": 65.96818542480469, + "margin_dpo/margin_std": 151.36160278320312, + "step": 661 + }, + { + "epoch": 0.999244142101285, + "step": 661, + "total_flos": 0.0, + "train_loss": 1.1683954546924078, + "train_runtime": 1648.9587, + "train_samples_per_second": 25.674, + "train_steps_per_second": 0.401 + } + ], + "logging_steps": 1, + "max_steps": 661, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}