From 02bf1f76cb73516ac4cce846aeaa15a266e2be8f Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 12 May 2026 14:46:33 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-8 Source: Original Platform --- .gitattributes | 36 + README.md | 62 + all_results.json | 9 + config.json | 29 + generation_config.json | 9 + model-00001-of-00007.safetensors | 3 + model-00002-of-00007.safetensors | 3 + model-00003-of-00007.safetensors | 3 + model-00004-of-00007.safetensors | 3 + model-00005-of-00007.safetensors | 3 + model-00006-of-00007.safetensors | 3 + model-00007-of-00007.safetensors | 3 + model.safetensors.index.json | 298 + special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2064 ++++ train_results.json | 9 + trainer_state.json | 15706 +++++++++++++++++++++++++++++ 18 files changed, 18269 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00007.safetensors create mode 100644 model-00002-of-00007.safetensors create mode 100644 model-00003-of-00007.safetensors create mode 100644 model-00004-of-00007.safetensors create mode 100644 model-00005-of-00007.safetensors create mode 100644 model-00006-of-00007.safetensors create mode 100644 model-00007-of-00007.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..95faa24 --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-helpful-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-8 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-8 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-helpful-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-helpful-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..a99a81e --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.356521034380763, + "train_runtime": 1742.8549, + "train_samples": 43598, + "train_samples_per_second": 25.015, + "train_steps_per_second": 0.391 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..a014f48 --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a781c91c61a94b75819d73bcaad0614e8af05611371fa61bdb3309148aa5dbca +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..4ab047e --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09249db4ccf4450701a24a537af481ea01b4f02b64c1718d8ee772945e13748d +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..82cc177 --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a1b090cb80055210706f9825b994d59e61dd4194cd71bdd350c7f62a33da88c +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..5204694 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:285520d832bd6e43c5c113b4ff36815953aa5a872d73f60e231c78f98bb5697b +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..01c17d5 --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c161355b964dc4b5fe9bb853b218b26ab82391bdc60e1e6b1480406c71771d99 +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..9af216a --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f8fca57d81d93bedafef6b33edcb574c718a3ab3e7341266121967c9f61903f +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..997900b --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:972ced9d54a4249524d6c34ee9b7e4a63630b69684c57d79dd808cb2af26d31a +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..a99a81e --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.356521034380763, + "train_runtime": 1742.8549, + "train_samples": 43598, + "train_samples_per_second": 25.015, + "train_steps_per_second": 0.391 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..407a3fe --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 200, + "global_step": 681, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.00527191162109375, + "KL/mean": 0.016706019639968872, + "KL/rejected_KL_mean": 0.028141021728515625, + "KL/std": 0.272699236869812, + "epoch": 0.0014684287812041115, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02287006378173828, + "fcm_dpo/q_t": 0.5005706548690796, + "grad_norm": 83.5267562866211, + "learning_rate": 0.0, + "logits/chosen": -0.4974287748336792, + "logits/rejected": -0.43299180269241333, + "logps/chosen": -50.1435661315918, + "logps/ref_chosen": -50.14883804321289, + "logps/ref_rejected": -74.1280517578125, + "logps/rejected": -74.09991455078125, + "loss": 1.389, + "margin_dpo/margin_mean": -0.02287048101425171, + "margin_dpo/margin_std": 0.41920793056488037, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.03498649597167969, + "KL/mean": -0.00212840735912323, + "KL/rejected_KL_mean": 0.030735015869140625, + "KL/std": 0.24797174334526062, + "epoch": 0.002936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06572261452674866, + "fcm_dpo/q_t": 0.5016425848007202, + "grad_norm": 72.22103118896484, + "learning_rate": 7.246376811594203e-09, + "logits/chosen": -0.49536412954330444, + "logits/rejected": -0.4594460427761078, + "logps/chosen": -52.65568923950195, + "logps/ref_chosen": -52.620704650878906, + "logps/ref_rejected": -75.30413818359375, + "logps/rejected": -75.27340698242188, + "loss": 1.3932, + "margin_dpo/margin_mean": -0.06572240591049194, + "margin_dpo/margin_std": 0.35048407316207886, + "step": 2 + }, + { + "KL/chosen_KL_mean": 0.016141891479492188, + "KL/mean": 0.009041428565979004, + "KL/rejected_KL_mean": 0.001941680908203125, + "KL/std": 0.2510850429534912, + "epoch": 0.004405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.014200329780578613, + "fcm_dpo/q_t": 0.4996456503868103, + "grad_norm": 70.8984375, + "learning_rate": 1.4492753623188406e-08, + "logits/chosen": -0.4816073179244995, + "logits/rejected": -0.44214364886283875, + "logps/chosen": -60.9654541015625, + "logps/ref_chosen": -60.981597900390625, + "logps/ref_rejected": -68.67259216308594, + "logps/rejected": -68.670654296875, + "loss": 1.3852, + "margin_dpo/margin_mean": 0.01420101523399353, + "margin_dpo/margin_std": 0.37771791219711304, + "step": 3 + }, + { + "KL/chosen_KL_mean": -0.038990020751953125, + "KL/mean": -0.02172142267227173, + "KL/rejected_KL_mean": -0.00445556640625, + "KL/std": 0.22829246520996094, + "epoch": 0.005873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.034536004066467285, + "fcm_dpo/q_t": 0.500863790512085, + "grad_norm": 72.12220764160156, + "learning_rate": 2.1739130434782606e-08, + "logits/chosen": -0.46893513202667236, + "logits/rejected": -0.44124215841293335, + "logps/chosen": -56.80670166015625, + "logps/ref_chosen": -56.7677116394043, + "logps/ref_rejected": -86.64710998535156, + "logps/rejected": -86.65156555175781, + "loss": 1.3901, + "margin_dpo/margin_mean": -0.03453657031059265, + "margin_dpo/margin_std": 0.3669918179512024, + "step": 4 + }, + { + "KL/chosen_KL_mean": 0.02498626708984375, + "KL/mean": 0.023038864135742188, + "KL/rejected_KL_mean": 0.021091461181640625, + "KL/std": 0.25224569439888, + "epoch": 0.007342143906020558, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0038943886756896973, + "fcm_dpo/q_t": 0.4999024569988251, + "grad_norm": 90.13763427734375, + "learning_rate": 2.898550724637681e-08, + "logits/chosen": -0.5147161483764648, + "logits/rejected": -0.47098612785339355, + "logps/chosen": -53.834388732910156, + "logps/ref_chosen": -53.859375, + "logps/ref_rejected": -84.14918518066406, + "logps/rejected": -84.12809753417969, + "loss": 1.3862, + "margin_dpo/margin_mean": 0.0038947463035583496, + "margin_dpo/margin_std": 0.3366306722164154, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.008636474609375, + "KL/mean": -0.022702336311340332, + "KL/rejected_KL_mean": -0.036769866943359375, + "KL/std": 0.2681416869163513, + "epoch": 0.00881057268722467, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.028134524822235107, + "fcm_dpo/q_t": 0.49929672479629517, + "grad_norm": 91.89168548583984, + "learning_rate": 3.6231884057971014e-08, + "logits/chosen": -0.5082045793533325, + "logits/rejected": -0.4662151038646698, + "logps/chosen": -63.01612091064453, + "logps/ref_chosen": -63.007484436035156, + "logps/ref_rejected": -92.64534759521484, + "logps/rejected": -92.68211364746094, + "loss": 1.3838, + "margin_dpo/margin_mean": 0.02813410758972168, + "margin_dpo/margin_std": 0.35160428285598755, + "step": 6 + }, + { + "KL/chosen_KL_mean": -0.013416290283203125, + "KL/mean": 0.02394537627696991, + "KL/rejected_KL_mean": 0.061305999755859375, + "KL/std": 0.2824476361274719, + "epoch": 0.010279001468428781, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.07472285628318787, + "fcm_dpo/q_t": 0.5018666982650757, + "grad_norm": 82.56436157226562, + "learning_rate": 4.347826086956521e-08, + "logits/chosen": -0.5052725076675415, + "logits/rejected": -0.47143226861953735, + "logps/chosen": -57.788238525390625, + "logps/ref_chosen": -57.774818420410156, + "logps/ref_rejected": -103.92059326171875, + "logps/rejected": -103.85928344726562, + "loss": 1.3943, + "margin_dpo/margin_mean": -0.07472258806228638, + "margin_dpo/margin_std": 0.43447619676589966, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.012325286865234375, + "KL/mean": 0.027680113911628723, + "KL/rejected_KL_mean": 0.04303741455078125, + "KL/std": 0.3071158230304718, + "epoch": 0.011747430249632892, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.030712276697158813, + "fcm_dpo/q_t": 0.5007674694061279, + "grad_norm": 78.390869140625, + "learning_rate": 5.0724637681159424e-08, + "logits/chosen": -0.49178242683410645, + "logits/rejected": -0.4640960395336151, + "logps/chosen": -58.703712463378906, + "logps/ref_chosen": -58.716033935546875, + "logps/ref_rejected": -79.3114242553711, + "logps/rejected": -79.26838684082031, + "loss": 1.3898, + "margin_dpo/margin_mean": -0.03071233630180359, + "margin_dpo/margin_std": 0.37911269068717957, + "step": 8 + }, + { + "KL/chosen_KL_mean": 0.020702362060546875, + "KL/mean": 0.0013383924961090088, + "KL/rejected_KL_mean": -0.01802825927734375, + "KL/std": 0.2934737205505371, + "epoch": 0.013215859030837005, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03873598575592041, + "fcm_dpo/q_t": 0.49903160333633423, + "grad_norm": 85.00302124023438, + "learning_rate": 5.797101449275362e-08, + "logits/chosen": -0.5062781572341919, + "logits/rejected": -0.4602203369140625, + "logps/chosen": -69.84614562988281, + "logps/ref_chosen": -69.8668441772461, + "logps/ref_rejected": -99.6026611328125, + "logps/rejected": -99.62069702148438, + "loss": 1.3828, + "margin_dpo/margin_mean": 0.03873533010482788, + "margin_dpo/margin_std": 0.39692068099975586, + "step": 9 + }, + { + "KL/chosen_KL_mean": 0.013669967651367188, + "KL/mean": -0.007347121834754944, + "KL/rejected_KL_mean": -0.028362274169921875, + "KL/std": 0.2732405662536621, + "epoch": 0.014684287812041116, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.042034536600112915, + "fcm_dpo/q_t": 0.49894949793815613, + "grad_norm": 70.61676025390625, + "learning_rate": 6.521739130434782e-08, + "logits/chosen": -0.5029403567314148, + "logits/rejected": -0.46021097898483276, + "logps/chosen": -48.344017028808594, + "logps/ref_chosen": -48.35768508911133, + "logps/ref_rejected": -80.37206268310547, + "logps/rejected": -80.40042114257812, + "loss": 1.3825, + "margin_dpo/margin_mean": 0.04203528165817261, + "margin_dpo/margin_std": 0.3880940079689026, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.007905960083007812, + "KL/mean": -0.01761634647846222, + "KL/rejected_KL_mean": -0.0431365966796875, + "KL/std": 0.2690313160419464, + "epoch": 0.016152716593245228, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05104079842567444, + "fcm_dpo/q_t": 0.4987240731716156, + "grad_norm": 68.2398681640625, + "learning_rate": 7.246376811594203e-08, + "logits/chosen": -0.4520139694213867, + "logits/rejected": -0.4266074299812317, + "logps/chosen": -53.008949279785156, + "logps/ref_chosen": -53.01685333251953, + "logps/ref_rejected": -87.78038024902344, + "logps/rejected": -87.82351684570312, + "loss": 1.3816, + "margin_dpo/margin_mean": 0.05104094743728638, + "margin_dpo/margin_std": 0.3854028582572937, + "step": 11 + }, + { + "KL/chosen_KL_mean": 0.0002536773681640625, + "KL/mean": -0.010517977178096771, + "KL/rejected_KL_mean": -0.0212860107421875, + "KL/std": 0.2928847372531891, + "epoch": 0.01762114537444934, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.021543309092521667, + "fcm_dpo/q_t": 0.49946165084838867, + "grad_norm": 89.65792846679688, + "learning_rate": 7.971014492753623e-08, + "logits/chosen": -0.5362281799316406, + "logits/rejected": -0.49995797872543335, + "logps/chosen": -61.80518341064453, + "logps/ref_chosen": -61.80543518066406, + "logps/ref_rejected": -104.8582763671875, + "logps/rejected": -104.87956237792969, + "loss": 1.3846, + "margin_dpo/margin_mean": 0.021542653441429138, + "margin_dpo/margin_std": 0.40630000829696655, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.0028934478759765625, + "KL/mean": -0.017609775066375732, + "KL/rejected_KL_mean": -0.03810882568359375, + "KL/std": 0.3167712390422821, + "epoch": 0.01908957415565345, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04100680351257324, + "fcm_dpo/q_t": 0.4989779591560364, + "grad_norm": 79.11839294433594, + "learning_rate": 8.695652173913042e-08, + "logits/chosen": -0.4737343192100525, + "logits/rejected": -0.44504889845848083, + "logps/chosen": -64.25746154785156, + "logps/ref_chosen": -64.2603530883789, + "logps/ref_rejected": -87.20307922363281, + "logps/rejected": -87.24119567871094, + "loss": 1.3827, + "margin_dpo/margin_mean": 0.0410064160823822, + "margin_dpo/margin_std": 0.4179210662841797, + "step": 13 + }, + { + "KL/chosen_KL_mean": -0.03240966796875, + "KL/mean": -0.003829583525657654, + "KL/rejected_KL_mean": 0.024745941162109375, + "KL/std": 0.2639964520931244, + "epoch": 0.020558002936857563, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.05715218186378479, + "fcm_dpo/q_t": 0.5014278888702393, + "grad_norm": 85.87844848632812, + "learning_rate": 9.420289855072464e-08, + "logits/chosen": -0.4652435779571533, + "logits/rejected": -0.4254474639892578, + "logps/chosen": -58.14262008666992, + "logps/ref_chosen": -58.11021041870117, + "logps/ref_rejected": -104.04708099365234, + "logps/rejected": -104.0223388671875, + "loss": 1.3924, + "margin_dpo/margin_mean": -0.0571514368057251, + "margin_dpo/margin_std": 0.3682215213775635, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.03320121765136719, + "KL/mean": -0.0465509295463562, + "KL/rejected_KL_mean": -0.059902191162109375, + "KL/std": 0.23382139205932617, + "epoch": 0.022026431718061675, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.026701807975769043, + "fcm_dpo/q_t": 0.4993324279785156, + "grad_norm": 64.0005874633789, + "learning_rate": 1.0144927536231885e-07, + "logits/chosen": -0.5041216015815735, + "logits/rejected": -0.48607927560806274, + "logps/chosen": -57.0001106262207, + "logps/ref_chosen": -56.96691131591797, + "logps/ref_rejected": -80.80863952636719, + "logps/rejected": -80.86854553222656, + "loss": 1.3839, + "margin_dpo/margin_mean": 0.026701688766479492, + "margin_dpo/margin_std": 0.33812767267227173, + "step": 15 + }, + { + "KL/chosen_KL_mean": -0.0052700042724609375, + "KL/mean": -0.040506258606910706, + "KL/rejected_KL_mean": -0.07574081420898438, + "KL/std": 0.2736942172050476, + "epoch": 0.023494860499265784, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.07047191262245178, + "fcm_dpo/q_t": 0.4982394576072693, + "grad_norm": 83.81981658935547, + "learning_rate": 1.0869565217391303e-07, + "logits/chosen": -0.5340527296066284, + "logits/rejected": -0.4940548539161682, + "logps/chosen": -61.74516296386719, + "logps/ref_chosen": -61.739891052246094, + "logps/ref_rejected": -84.36947631835938, + "logps/rejected": -84.4452133178711, + "loss": 1.3796, + "margin_dpo/margin_mean": 0.07047182321548462, + "margin_dpo/margin_std": 0.3839455842971802, + "step": 16 + }, + { + "KL/chosen_KL_mean": -0.008800506591796875, + "KL/mean": -0.020151451230049133, + "KL/rejected_KL_mean": -0.031505584716796875, + "KL/std": 0.2639004588127136, + "epoch": 0.024963289280469897, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.02270624041557312, + "fcm_dpo/q_t": 0.49943220615386963, + "grad_norm": 78.86737060546875, + "learning_rate": 1.1594202898550725e-07, + "logits/chosen": -0.504452109336853, + "logits/rejected": -0.46770912408828735, + "logps/chosen": -67.71913146972656, + "logps/ref_chosen": -67.71033477783203, + "logps/ref_rejected": -85.37865447998047, + "logps/rejected": -85.41015625, + "loss": 1.3843, + "margin_dpo/margin_mean": 0.022706151008605957, + "margin_dpo/margin_std": 0.3559093475341797, + "step": 17 + }, + { + "KL/chosen_KL_mean": -0.010873794555664062, + "KL/mean": -0.05961640179157257, + "KL/rejected_KL_mean": -0.10836029052734375, + "KL/std": 0.30446290969848633, + "epoch": 0.02643171806167401, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.09748539328575134, + "fcm_dpo/q_t": 0.497562050819397, + "grad_norm": 81.79119110107422, + "learning_rate": 1.2318840579710146e-07, + "logits/chosen": -0.5100164413452148, + "logits/rejected": -0.4564880430698395, + "logps/chosen": -47.7503662109375, + "logps/ref_chosen": -47.7394905090332, + "logps/ref_rejected": -75.4722900390625, + "logps/rejected": -75.58064270019531, + "loss": 1.377, + "margin_dpo/margin_mean": 0.09748479723930359, + "margin_dpo/margin_std": 0.4179475009441376, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.0426177978515625, + "KL/mean": -0.00010053813457489014, + "KL/rejected_KL_mean": -0.042816162109375, + "KL/std": 0.2286176085472107, + "epoch": 0.027900146842878122, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08543136715888977, + "fcm_dpo/q_t": 0.497865229845047, + "grad_norm": 72.6661376953125, + "learning_rate": 1.3043478260869563e-07, + "logits/chosen": -0.49207890033721924, + "logits/rejected": -0.44229188561439514, + "logps/chosen": -70.1627426147461, + "logps/ref_chosen": -70.20536041259766, + "logps/ref_rejected": -89.7575912475586, + "logps/rejected": -89.80040740966797, + "loss": 1.378, + "margin_dpo/margin_mean": 0.08543169498443604, + "margin_dpo/margin_std": 0.3262811303138733, + "step": 19 + }, + { + "KL/chosen_KL_mean": 0.0006504058837890625, + "KL/mean": -0.034426525235176086, + "KL/rejected_KL_mean": -0.06949996948242188, + "KL/std": 0.27714747190475464, + "epoch": 0.02936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.07015708088874817, + "fcm_dpo/q_t": 0.4982466399669647, + "grad_norm": 73.6260986328125, + "learning_rate": 1.3768115942028986e-07, + "logits/chosen": -0.5626459121704102, + "logits/rejected": -0.5075259804725647, + "logps/chosen": -50.802589416503906, + "logps/ref_chosen": -50.80324172973633, + "logps/ref_rejected": -78.82334899902344, + "logps/rejected": -78.89285278320312, + "loss": 1.3797, + "margin_dpo/margin_mean": 0.07015728950500488, + "margin_dpo/margin_std": 0.38508284091949463, + "step": 20 + }, + { + "KL/chosen_KL_mean": 0.022371292114257812, + "KL/mean": -0.05759835243225098, + "KL/rejected_KL_mean": -0.13756561279296875, + "KL/std": 0.3261793851852417, + "epoch": 0.030837004405286344, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.15993809700012207, + "fcm_dpo/q_t": 0.4960044026374817, + "grad_norm": 77.39993286132812, + "learning_rate": 1.4492753623188405e-07, + "logits/chosen": -0.5166388750076294, + "logits/rejected": -0.4944336712360382, + "logps/chosen": -50.040645599365234, + "logps/ref_chosen": -50.063018798828125, + "logps/ref_rejected": -77.86878967285156, + "logps/rejected": -78.00636291503906, + "loss": 1.3708, + "margin_dpo/margin_mean": 0.15993818640708923, + "margin_dpo/margin_std": 0.40345868468284607, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.03532218933105469, + "KL/mean": -0.06052166223526001, + "KL/rejected_KL_mean": -0.15636444091796875, + "KL/std": 0.314807265996933, + "epoch": 0.032305433186490456, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.191686749458313, + "fcm_dpo/q_t": 0.49521100521087646, + "grad_norm": 84.29412841796875, + "learning_rate": 1.5217391304347825e-07, + "logits/chosen": -0.4891554117202759, + "logits/rejected": -0.44577914476394653, + "logps/chosen": -59.022315979003906, + "logps/ref_chosen": -59.05763626098633, + "logps/ref_rejected": -97.50466918945312, + "logps/rejected": -97.66104125976562, + "loss": 1.3678, + "margin_dpo/margin_mean": 0.1916864514350891, + "margin_dpo/margin_std": 0.4764353930950165, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.03306007385253906, + "KL/mean": -0.06440095603466034, + "KL/rejected_KL_mean": -0.16185760498046875, + "KL/std": 0.31806907057762146, + "epoch": 0.033773861967694566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.19491460919380188, + "fcm_dpo/q_t": 0.49513041973114014, + "grad_norm": 80.66082000732422, + "learning_rate": 1.5942028985507245e-07, + "logits/chosen": -0.4961227774620056, + "logits/rejected": -0.47408023476600647, + "logps/chosen": -60.04463577270508, + "logps/ref_chosen": -60.07769775390625, + "logps/ref_rejected": -81.13955688476562, + "logps/rejected": -81.30140686035156, + "loss": 1.3674, + "margin_dpo/margin_mean": 0.19491499662399292, + "margin_dpo/margin_std": 0.43538808822631836, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.03735542297363281, + "KL/mean": -0.08538687229156494, + "KL/rejected_KL_mean": -0.2081298828125, + "KL/std": 0.27013736963272095, + "epoch": 0.03524229074889868, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2454851269721985, + "fcm_dpo/q_t": 0.49386560916900635, + "grad_norm": 80.96056365966797, + "learning_rate": 1.6666666666666665e-07, + "logits/chosen": -0.5057855844497681, + "logits/rejected": -0.4892767071723938, + "logps/chosen": -44.25367736816406, + "logps/ref_chosen": -44.29103469848633, + "logps/ref_rejected": -99.12521362304688, + "logps/rejected": -99.33334350585938, + "loss": 1.3622, + "margin_dpo/margin_mean": 0.2454843521118164, + "margin_dpo/margin_std": 0.3252241015434265, + "step": 24 + }, + { + "KL/chosen_KL_mean": 0.0063114166259765625, + "KL/mean": -0.09504103660583496, + "KL/rejected_KL_mean": -0.19639205932617188, + "KL/std": 0.31363558769226074, + "epoch": 0.03671071953010279, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.20270639657974243, + "fcm_dpo/q_t": 0.4949347674846649, + "grad_norm": 73.89991760253906, + "learning_rate": 1.7391304347826085e-07, + "logits/chosen": -0.5002730488777161, + "logits/rejected": -0.47041672468185425, + "logps/chosen": -52.530738830566406, + "logps/ref_chosen": -52.537052154541016, + "logps/ref_rejected": -89.34219360351562, + "logps/rejected": -89.53858184814453, + "loss": 1.3665, + "margin_dpo/margin_mean": 0.20270657539367676, + "margin_dpo/margin_std": 0.37986764311790466, + "step": 25 + }, + { + "KL/chosen_KL_mean": 0.09214401245117188, + "KL/mean": -0.10877400636672974, + "KL/rejected_KL_mean": -0.3096961975097656, + "KL/std": 0.4387274384498596, + "epoch": 0.0381791483113069, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4018453359603882, + "fcm_dpo/q_t": 0.4899684488773346, + "grad_norm": 87.35763549804688, + "learning_rate": 1.8115942028985507e-07, + "logits/chosen": -0.5315044522285461, + "logits/rejected": -0.4998011589050293, + "logps/chosen": -53.830665588378906, + "logps/ref_chosen": -53.92280578613281, + "logps/ref_rejected": -103.35971069335938, + "logps/rejected": -103.6694107055664, + "loss": 1.3474, + "margin_dpo/margin_mean": 0.40184569358825684, + "margin_dpo/margin_std": 0.5766628980636597, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.14463043212890625, + "KL/mean": -0.12463589012622833, + "KL/rejected_KL_mean": -0.3939056396484375, + "KL/std": 0.4663229286670685, + "epoch": 0.039647577092511016, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5385316610336304, + "fcm_dpo/q_t": 0.48655465245246887, + "grad_norm": 93.67314147949219, + "learning_rate": 1.8840579710144927e-07, + "logits/chosen": -0.5397769212722778, + "logits/rejected": -0.503648579120636, + "logps/chosen": -42.75389862060547, + "logps/ref_chosen": -42.898529052734375, + "logps/ref_rejected": -98.72419738769531, + "logps/rejected": -99.11810302734375, + "loss": 1.334, + "margin_dpo/margin_mean": 0.5385308265686035, + "margin_dpo/margin_std": 0.5469115972518921, + "step": 27 + }, + { + "KL/chosen_KL_mean": -0.03975486755371094, + "KL/mean": -0.16874754428863525, + "KL/rejected_KL_mean": -0.29773712158203125, + "KL/std": 0.41560396552085876, + "epoch": 0.041116005873715125, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2579805254936218, + "fcm_dpo/q_t": 0.49355655908584595, + "grad_norm": 75.45986938476562, + "learning_rate": 1.9565217391304347e-07, + "logits/chosen": -0.5163806080818176, + "logits/rejected": -0.4621371328830719, + "logps/chosen": -60.596256256103516, + "logps/ref_chosen": -60.55650329589844, + "logps/ref_rejected": -91.40111541748047, + "logps/rejected": -91.6988525390625, + "loss": 1.3616, + "margin_dpo/margin_mean": 0.25798097252845764, + "margin_dpo/margin_std": 0.6161943674087524, + "step": 28 + }, + { + "KL/chosen_KL_mean": 0.11548995971679688, + "KL/mean": -0.14322738349437714, + "KL/rejected_KL_mean": -0.4019432067871094, + "KL/std": 0.47610336542129517, + "epoch": 0.042584434654919234, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5174299478530884, + "fcm_dpo/q_t": 0.48708075284957886, + "grad_norm": 90.54450225830078, + "learning_rate": 2.028985507246377e-07, + "logits/chosen": -0.5383598208427429, + "logits/rejected": -0.49088796973228455, + "logps/chosen": -57.69229507446289, + "logps/ref_chosen": -57.80778503417969, + "logps/ref_rejected": -97.39434814453125, + "logps/rejected": -97.79629516601562, + "loss": 1.3361, + "margin_dpo/margin_mean": 0.5174306631088257, + "margin_dpo/margin_std": 0.5744448900222778, + "step": 29 + }, + { + "KL/chosen_KL_mean": 0.1258563995361328, + "KL/mean": -0.19550739228725433, + "KL/rejected_KL_mean": -0.5168724060058594, + "KL/std": 0.5710160136222839, + "epoch": 0.04405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6427308320999146, + "fcm_dpo/q_t": 0.48395419120788574, + "grad_norm": 87.73825073242188, + "learning_rate": 2.1014492753623187e-07, + "logits/chosen": -0.5053558349609375, + "logits/rejected": -0.4750453233718872, + "logps/chosen": -52.451515197753906, + "logps/ref_chosen": -52.577369689941406, + "logps/ref_rejected": -98.48920440673828, + "logps/rejected": -99.00607299804688, + "loss": 1.324, + "margin_dpo/margin_mean": 0.6427305340766907, + "margin_dpo/margin_std": 0.6174843311309814, + "step": 30 + }, + { + "KL/chosen_KL_mean": 0.0767364501953125, + "KL/mean": -0.1584608256816864, + "KL/rejected_KL_mean": -0.3936576843261719, + "KL/std": 0.5441341400146484, + "epoch": 0.04552129221732746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.47039252519607544, + "fcm_dpo/q_t": 0.48826301097869873, + "grad_norm": 68.57320404052734, + "learning_rate": 2.1739130434782607e-07, + "logits/chosen": -0.5012216567993164, + "logits/rejected": -0.456045925617218, + "logps/chosen": -63.730186462402344, + "logps/ref_chosen": -63.806922912597656, + "logps/ref_rejected": -72.89400482177734, + "logps/rejected": -73.28765869140625, + "loss": 1.341, + "margin_dpo/margin_mean": 0.4703920781612396, + "margin_dpo/margin_std": 0.7006645202636719, + "step": 31 + }, + { + "KL/chosen_KL_mean": 0.20220184326171875, + "KL/mean": -0.18391788005828857, + "KL/rejected_KL_mean": -0.570037841796875, + "KL/std": 0.7176538705825806, + "epoch": 0.04698972099853157, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7722392082214355, + "fcm_dpo/q_t": 0.4807533621788025, + "grad_norm": 83.21366119384766, + "learning_rate": 2.2463768115942027e-07, + "logits/chosen": -0.5237495303153992, + "logits/rejected": -0.48302900791168213, + "logps/chosen": -62.537322998046875, + "logps/ref_chosen": -62.739524841308594, + "logps/ref_rejected": -89.3175048828125, + "logps/rejected": -89.88754272460938, + "loss": 1.3123, + "margin_dpo/margin_mean": 0.7722393274307251, + "margin_dpo/margin_std": 0.842420220375061, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.12135887145996094, + "KL/mean": -0.17980101704597473, + "KL/rejected_KL_mean": -0.4809608459472656, + "KL/std": 0.5358744859695435, + "epoch": 0.048458149779735685, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6023146510124207, + "fcm_dpo/q_t": 0.48496127128601074, + "grad_norm": 72.2737045288086, + "learning_rate": 2.318840579710145e-07, + "logits/chosen": -0.5086983442306519, + "logits/rejected": -0.4830739498138428, + "logps/chosen": -53.13961410522461, + "logps/ref_chosen": -53.26097106933594, + "logps/ref_rejected": -87.8851318359375, + "logps/rejected": -88.36609649658203, + "loss": 1.3279, + "margin_dpo/margin_mean": 0.6023147106170654, + "margin_dpo/margin_std": 0.5953254699707031, + "step": 33 + }, + { + "KL/chosen_KL_mean": 0.08457756042480469, + "KL/mean": -0.30423083901405334, + "KL/rejected_KL_mean": -0.6930427551269531, + "KL/std": 0.7175294160842896, + "epoch": 0.049926578560939794, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.77761310338974, + "fcm_dpo/q_t": 0.4806172847747803, + "grad_norm": 77.8739013671875, + "learning_rate": 2.391304347826087e-07, + "logits/chosen": -0.5117413401603699, + "logits/rejected": -0.4942744970321655, + "logps/chosen": -50.732749938964844, + "logps/ref_chosen": -50.81732940673828, + "logps/ref_rejected": -101.92184448242188, + "logps/rejected": -102.61488342285156, + "loss": 1.312, + "margin_dpo/margin_mean": 0.7776126861572266, + "margin_dpo/margin_std": 0.8780879974365234, + "step": 34 + }, + { + "KL/chosen_KL_mean": 0.09637641906738281, + "KL/mean": -0.5095885992050171, + "KL/rejected_KL_mean": -1.1155509948730469, + "KL/std": 0.9993947744369507, + "epoch": 0.0513950073421439, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2119297981262207, + "fcm_dpo/q_t": 0.46987059712409973, + "grad_norm": 82.90363311767578, + "learning_rate": 2.463768115942029e-07, + "logits/chosen": -0.5279215574264526, + "logits/rejected": -0.49103760719299316, + "logps/chosen": -50.92811584472656, + "logps/ref_chosen": -51.02449035644531, + "logps/ref_rejected": -106.82443237304688, + "logps/rejected": -107.93997955322266, + "loss": 1.2717, + "margin_dpo/margin_mean": 1.2119297981262207, + "margin_dpo/margin_std": 1.0788286924362183, + "step": 35 + }, + { + "KL/chosen_KL_mean": 0.040618896484375, + "KL/mean": -0.5388714671134949, + "KL/rejected_KL_mean": -1.1183624267578125, + "KL/std": 1.0968472957611084, + "epoch": 0.05286343612334802, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1589829921722412, + "fcm_dpo/q_t": 0.4711868166923523, + "grad_norm": 72.5723648071289, + "learning_rate": 2.536231884057971e-07, + "logits/chosen": -0.5531824827194214, + "logits/rejected": -0.5167204737663269, + "logps/chosen": -51.95087432861328, + "logps/ref_chosen": -51.991493225097656, + "logps/ref_rejected": -86.0406265258789, + "logps/rejected": -87.15898895263672, + "loss": 1.2777, + "margin_dpo/margin_mean": 1.158982753753662, + "margin_dpo/margin_std": 1.2281889915466309, + "step": 36 + }, + { + "KL/chosen_KL_mean": -0.025007247924804688, + "KL/mean": -0.5262736082077026, + "KL/rejected_KL_mean": -1.0275382995605469, + "KL/std": 1.0382901430130005, + "epoch": 0.05433186490455213, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.002532720565796, + "fcm_dpo/q_t": 0.4751211404800415, + "grad_norm": 61.644954681396484, + "learning_rate": 2.6086956521739126e-07, + "logits/chosen": -0.5020361542701721, + "logits/rejected": -0.45762014389038086, + "logps/chosen": -62.832115173339844, + "logps/ref_chosen": -62.807106018066406, + "logps/ref_rejected": -77.89507293701172, + "logps/rejected": -78.922607421875, + "loss": 1.2928, + "margin_dpo/margin_mean": 1.0025322437286377, + "margin_dpo/margin_std": 1.2644333839416504, + "step": 37 + }, + { + "KL/chosen_KL_mean": 0.15131187438964844, + "KL/mean": -0.5383007526397705, + "KL/rejected_KL_mean": -1.2279129028320312, + "KL/std": 1.3978111743927002, + "epoch": 0.055800293685756244, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3792251348495483, + "fcm_dpo/q_t": 0.4659920334815979, + "grad_norm": 69.25381469726562, + "learning_rate": 2.681159420289855e-07, + "logits/chosen": -0.5107109546661377, + "logits/rejected": -0.4779038727283478, + "logps/chosen": -48.23920822143555, + "logps/ref_chosen": -48.39051818847656, + "logps/ref_rejected": -97.91244506835938, + "logps/rejected": -99.14035034179688, + "loss": 1.2602, + "margin_dpo/margin_mean": 1.379225730895996, + "margin_dpo/margin_std": 1.6747769117355347, + "step": 38 + }, + { + "KL/chosen_KL_mean": 0.03895378112792969, + "KL/mean": -0.7721768617630005, + "KL/rejected_KL_mean": -1.5833053588867188, + "KL/std": 1.307983636856079, + "epoch": 0.05726872246696035, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.6222527027130127, + "fcm_dpo/q_t": 0.45977842807769775, + "grad_norm": 74.82653045654297, + "learning_rate": 2.753623188405797e-07, + "logits/chosen": -0.5557675361633301, + "logits/rejected": -0.5158565044403076, + "logps/chosen": -50.711517333984375, + "logps/ref_chosen": -50.75047302246094, + "logps/ref_rejected": -78.56951141357422, + "logps/rejected": -80.15281677246094, + "loss": 1.2361, + "margin_dpo/margin_mean": 1.6222525835037231, + "margin_dpo/margin_std": 1.4611896276474, + "step": 39 + }, + { + "KL/chosen_KL_mean": 0.18808746337890625, + "KL/mean": -0.5939724445343018, + "KL/rejected_KL_mean": -1.37603759765625, + "KL/std": 1.4154611825942993, + "epoch": 0.05873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5641274452209473, + "fcm_dpo/q_t": 0.46141234040260315, + "grad_norm": 60.164329528808594, + "learning_rate": 2.8260869565217386e-07, + "logits/chosen": -0.5338292121887207, + "logits/rejected": -0.50484299659729, + "logps/chosen": -57.79698181152344, + "logps/ref_chosen": -57.985069274902344, + "logps/ref_rejected": -74.3000717163086, + "logps/rejected": -75.67610931396484, + "loss": 1.2436, + "margin_dpo/margin_mean": 1.5641283988952637, + "margin_dpo/margin_std": 1.7055679559707642, + "step": 40 + }, + { + "KL/chosen_KL_mean": 0.035152435302734375, + "KL/mean": -0.9116830825805664, + "KL/rejected_KL_mean": -1.8585128784179688, + "KL/std": 1.8412256240844727, + "epoch": 0.06020558002936858, + "fcm_dpo/beta": 0.2878764867782593, + "fcm_dpo/delta": 0.09748303890228271, + "fcm_dpo/margin": 1.893665075302124, + "fcm_dpo/q_t": 0.4534370005130768, + "grad_norm": 148.9955596923828, + "learning_rate": 2.898550724637681e-07, + "logits/chosen": -0.5401488542556763, + "logits/rejected": -0.5035785436630249, + "logps/chosen": -62.66066360473633, + "logps/ref_chosen": -62.69581604003906, + "logps/ref_rejected": -97.02352905273438, + "logps/rejected": -98.88204193115234, + "loss": 1.0448, + "margin_dpo/margin_mean": 1.8936657905578613, + "margin_dpo/margin_std": 2.0037331581115723, + "step": 41 + }, + { + "KL/chosen_KL_mean": 0.2072906494140625, + "KL/mean": -1.0888673067092896, + "KL/rejected_KL_mean": -2.3850173950195312, + "KL/std": 2.1535332202911377, + "epoch": 0.06167400881057269, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": -0.38896632194519043, + "fcm_dpo/margin": 2.592313289642334, + "fcm_dpo/q_t": 0.3977593779563904, + "grad_norm": 0.964080810546875, + "learning_rate": 2.971014492753623e-07, + "logits/chosen": -0.527120053768158, + "logits/rejected": -0.4803665280342102, + "logps/chosen": -58.75913619995117, + "logps/ref_chosen": -58.966426849365234, + "logps/ref_rejected": -109.90837097167969, + "logps/rejected": -112.29339599609375, + "loss": 1.3839, + "margin_dpo/margin_mean": 2.592312812805176, + "margin_dpo/margin_std": 2.3912642002105713, + "step": 42 + }, + { + "KL/chosen_KL_mean": 0.5225181579589844, + "KL/mean": -0.7178996205329895, + "KL/rejected_KL_mean": -1.9583206176757812, + "KL/std": 1.8281306028366089, + "epoch": 0.0631424375917768, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.480837345123291, + "fcm_dpo/q_t": 0.4994150698184967, + "grad_norm": 0.8368467688560486, + "learning_rate": 3.043478260869565e-07, + "logits/chosen": -0.5465719699859619, + "logits/rejected": -0.5219491720199585, + "logps/chosen": -53.63347625732422, + "logps/ref_chosen": -54.15599822998047, + "logps/ref_rejected": -96.48019409179688, + "logps/rejected": -98.43850708007812, + "loss": 1.384, + "margin_dpo/margin_mean": 2.480837345123291, + "margin_dpo/margin_std": 1.950972557067871, + "step": 43 + }, + { + "KL/chosen_KL_mean": 0.2370586395263672, + "KL/mean": -1.170802116394043, + "KL/rejected_KL_mean": -2.578662872314453, + "KL/std": 2.2450320720672607, + "epoch": 0.06461086637298091, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.815722703933716, + "fcm_dpo/q_t": 0.4993360936641693, + "grad_norm": 0.9514136910438538, + "learning_rate": 3.115942028985507e-07, + "logits/chosen": -0.4892653822898865, + "logits/rejected": -0.468932181596756, + "logps/chosen": -49.84143829345703, + "logps/ref_chosen": -50.07849884033203, + "logps/ref_rejected": -108.78376007080078, + "logps/rejected": -111.3624267578125, + "loss": 1.3836, + "margin_dpo/margin_mean": 2.8157224655151367, + "margin_dpo/margin_std": 2.159979820251465, + "step": 44 + }, + { + "KL/chosen_KL_mean": 0.1477680206298828, + "KL/mean": -0.9873858690261841, + "KL/rejected_KL_mean": -2.122539520263672, + "KL/std": 2.0420613288879395, + "epoch": 0.06607929515418502, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.2703118324279785, + "fcm_dpo/q_t": 0.4994646906852722, + "grad_norm": 0.7598158717155457, + "learning_rate": 3.188405797101449e-07, + "logits/chosen": -0.5002434253692627, + "logits/rejected": -0.4881095290184021, + "logps/chosen": -48.26715850830078, + "logps/ref_chosen": -48.4149284362793, + "logps/ref_rejected": -77.93643188476562, + "logps/rejected": -80.05896759033203, + "loss": 1.3842, + "margin_dpo/margin_mean": 2.2703113555908203, + "margin_dpo/margin_std": 2.5159013271331787, + "step": 45 + }, + { + "KL/chosen_KL_mean": 0.1900959014892578, + "KL/mean": -1.2448352575302124, + "KL/rejected_KL_mean": -2.679767608642578, + "KL/std": 2.5515077114105225, + "epoch": 0.06754772393538913, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.869863510131836, + "fcm_dpo/q_t": 0.4993233382701874, + "grad_norm": 0.9118256568908691, + "learning_rate": 3.260869565217391e-07, + "logits/chosen": -0.5423866510391235, + "logits/rejected": -0.4921818673610687, + "logps/chosen": -55.809329986572266, + "logps/ref_chosen": -55.999427795410156, + "logps/ref_rejected": -95.652587890625, + "logps/rejected": -98.33235168457031, + "loss": 1.3836, + "margin_dpo/margin_mean": 2.869863510131836, + "margin_dpo/margin_std": 3.1087698936462402, + "step": 46 + }, + { + "KL/chosen_KL_mean": 0.38346290588378906, + "KL/mean": -1.0860923528671265, + "KL/rejected_KL_mean": -2.5556488037109375, + "KL/std": 2.5621590614318848, + "epoch": 0.06901615271659324, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.9391136169433594, + "fcm_dpo/q_t": 0.4993070363998413, + "grad_norm": 0.8042023777961731, + "learning_rate": 3.333333333333333e-07, + "logits/chosen": -0.5681043267250061, + "logits/rejected": -0.515215277671814, + "logps/chosen": -57.54261779785156, + "logps/ref_chosen": -57.92607879638672, + "logps/ref_rejected": -94.67920684814453, + "logps/rejected": -97.23485565185547, + "loss": 1.3835, + "margin_dpo/margin_mean": 2.9391140937805176, + "margin_dpo/margin_std": 2.639211654663086, + "step": 47 + }, + { + "KL/chosen_KL_mean": 0.08052635192871094, + "KL/mean": -1.415499210357666, + "KL/rejected_KL_mean": -2.9115257263183594, + "KL/std": 2.53715181350708, + "epoch": 0.07048458149779736, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.992050886154175, + "fcm_dpo/q_t": 0.49929454922676086, + "grad_norm": 0.9025480151176453, + "learning_rate": 3.4057971014492755e-07, + "logits/chosen": -0.5785880088806152, + "logits/rejected": -0.5198041200637817, + "logps/chosen": -57.107547760009766, + "logps/ref_chosen": -57.188072204589844, + "logps/ref_rejected": -88.0166015625, + "logps/rejected": -90.92813110351562, + "loss": 1.3835, + "margin_dpo/margin_mean": 2.992050886154175, + "margin_dpo/margin_std": 2.7764430046081543, + "step": 48 + }, + { + "KL/chosen_KL_mean": 0.3786048889160156, + "KL/mean": -1.4605863094329834, + "KL/rejected_KL_mean": -3.299774169921875, + "KL/std": 3.252890110015869, + "epoch": 0.07195301027900147, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.6783740520477295, + "fcm_dpo/q_t": 0.4991326928138733, + "grad_norm": 0.8797588348388672, + "learning_rate": 3.478260869565217e-07, + "logits/chosen": -0.5455284118652344, + "logits/rejected": -0.48684394359588623, + "logps/chosen": -61.306663513183594, + "logps/ref_chosen": -61.685272216796875, + "logps/ref_rejected": -83.76747131347656, + "logps/rejected": -87.06724548339844, + "loss": 1.3828, + "margin_dpo/margin_mean": 3.6783740520477295, + "margin_dpo/margin_std": 3.7025225162506104, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.12292289733886719, + "KL/mean": -2.0747485160827637, + "KL/rejected_KL_mean": -4.026569366455078, + "KL/std": 3.336162567138672, + "epoch": 0.07342143906020558, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.9036431312561035, + "fcm_dpo/q_t": 0.4990795850753784, + "grad_norm": 0.8429669141769409, + "learning_rate": 3.5507246376811595e-07, + "logits/chosen": -0.5466989278793335, + "logits/rejected": -0.5109710693359375, + "logps/chosen": -58.84706115722656, + "logps/ref_chosen": -58.72413635253906, + "logps/ref_rejected": -96.35814666748047, + "logps/rejected": -100.38471984863281, + "loss": 1.3826, + "margin_dpo/margin_mean": 3.9036436080932617, + "margin_dpo/margin_std": 3.8791186809539795, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.27999114990234375, + "KL/mean": -2.2037620544433594, + "KL/rejected_KL_mean": -4.127536773681641, + "KL/std": 3.985077142715454, + "epoch": 0.07488986784140969, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.847546100616455, + "fcm_dpo/q_t": 0.49909284710884094, + "grad_norm": 0.7773189544677734, + "learning_rate": 3.6231884057971015e-07, + "logits/chosen": -0.5368514060974121, + "logits/rejected": -0.5046820044517517, + "logps/chosen": -61.65365982055664, + "logps/ref_chosen": -61.3736686706543, + "logps/ref_rejected": -76.00199890136719, + "logps/rejected": -80.1295394897461, + "loss": 1.3827, + "margin_dpo/margin_mean": 3.847546100616455, + "margin_dpo/margin_std": 4.900827407836914, + "step": 51 + }, + { + "KL/chosen_KL_mean": 0.37728309631347656, + "KL/mean": -2.584096908569336, + "KL/rejected_KL_mean": -5.545478820800781, + "KL/std": 4.599925994873047, + "epoch": 0.0763582966226138, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 5.922760963439941, + "fcm_dpo/q_t": 0.4986035227775574, + "grad_norm": 0.9805387258529663, + "learning_rate": 3.695652173913043e-07, + "logits/chosen": -0.5656962394714355, + "logits/rejected": -0.5110952258110046, + "logps/chosen": -51.96007537841797, + "logps/ref_chosen": -52.33735656738281, + "logps/ref_rejected": -79.97391510009766, + "logps/rejected": -85.51939392089844, + "loss": 1.3807, + "margin_dpo/margin_mean": 5.922760963439941, + "margin_dpo/margin_std": 4.9100751876831055, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.1439056396484375, + "KL/mean": -3.184206485748291, + "KL/rejected_KL_mean": -6.224506378173828, + "KL/std": 5.271965980529785, + "epoch": 0.07782672540381791, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.08060884475708, + "fcm_dpo/q_t": 0.49856632947921753, + "grad_norm": 1.0281976461410522, + "learning_rate": 3.7681159420289855e-07, + "logits/chosen": -0.6287667155265808, + "logits/rejected": -0.6076550483703613, + "logps/chosen": -53.45855712890625, + "logps/ref_chosen": -53.31465148925781, + "logps/ref_rejected": -91.78359985351562, + "logps/rejected": -98.00810241699219, + "loss": 1.3806, + "margin_dpo/margin_mean": 6.080608367919922, + "margin_dpo/margin_std": 6.02375602722168, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.37529754638671875, + "KL/mean": -3.0028505325317383, + "KL/rejected_KL_mean": -5.630397796630859, + "KL/std": 4.981309413909912, + "epoch": 0.07929515418502203, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 5.255102157592773, + "fcm_dpo/q_t": 0.4987609386444092, + "grad_norm": 0.8609330654144287, + "learning_rate": 3.8405797101449274e-07, + "logits/chosen": -0.5812989473342896, + "logits/rejected": -0.5273776054382324, + "logps/chosen": -51.06395721435547, + "logps/ref_chosen": -50.68865966796875, + "logps/ref_rejected": -91.71539306640625, + "logps/rejected": -97.34579467773438, + "loss": 1.3814, + "margin_dpo/margin_mean": 5.255101203918457, + "margin_dpo/margin_std": 5.078667163848877, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.8801307678222656, + "KL/mean": -4.046672344207764, + "KL/rejected_KL_mean": -7.213207244873047, + "KL/std": 6.092576026916504, + "epoch": 0.08076358296622614, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.333078384399414, + "fcm_dpo/q_t": 0.4985068142414093, + "grad_norm": 0.9243965744972229, + "learning_rate": 3.9130434782608694e-07, + "logits/chosen": -0.6232470273971558, + "logits/rejected": -0.5590590238571167, + "logps/chosen": -63.495365142822266, + "logps/ref_chosen": -62.615234375, + "logps/ref_rejected": -88.99349975585938, + "logps/rejected": -96.20671081542969, + "loss": 1.3803, + "margin_dpo/margin_mean": 6.333077907562256, + "margin_dpo/margin_std": 7.390386581420898, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.6633682250976562, + "KL/mean": -3.713230609893799, + "KL/rejected_KL_mean": -6.763092041015625, + "KL/std": 5.995309829711914, + "epoch": 0.08223201174743025, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.099724769592285, + "fcm_dpo/q_t": 0.4985618591308594, + "grad_norm": 0.8911805152893066, + "learning_rate": 3.9855072463768114e-07, + "logits/chosen": -0.6286439895629883, + "logits/rejected": -0.5872344970703125, + "logps/chosen": -58.59609603881836, + "logps/ref_chosen": -57.9327278137207, + "logps/ref_rejected": -94.1744384765625, + "logps/rejected": -100.93753051757812, + "loss": 1.3806, + "margin_dpo/margin_mean": 6.099723815917969, + "margin_dpo/margin_std": 7.246601104736328, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.7514553070068359, + "KL/mean": -4.159761428833008, + "KL/rejected_KL_mean": -7.568061828613281, + "KL/std": 5.89918327331543, + "epoch": 0.08370044052863436, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.816606521606445, + "fcm_dpo/q_t": 0.498392790555954, + "grad_norm": 0.9736577868461609, + "learning_rate": 4.057971014492754e-07, + "logits/chosen": -0.5877238512039185, + "logits/rejected": -0.5602452754974365, + "logps/chosen": -71.24674224853516, + "logps/ref_chosen": -70.49528503417969, + "logps/ref_rejected": -95.56546020507812, + "logps/rejected": -103.13351440429688, + "loss": 1.3799, + "margin_dpo/margin_mean": 6.816606521606445, + "margin_dpo/margin_std": 6.500908851623535, + "step": 57 + }, + { + "KL/chosen_KL_mean": -1.0264396667480469, + "KL/mean": -4.852134704589844, + "KL/rejected_KL_mean": -8.67782974243164, + "KL/std": 6.786162376403809, + "epoch": 0.08516886930983847, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.651383399963379, + "fcm_dpo/q_t": 0.498196005821228, + "grad_norm": 0.9956030249595642, + "learning_rate": 4.1304347826086954e-07, + "logits/chosen": -0.6043895483016968, + "logits/rejected": -0.5297507047653198, + "logps/chosen": -63.15937805175781, + "logps/ref_chosen": -62.13294219970703, + "logps/ref_rejected": -84.61729431152344, + "logps/rejected": -93.29512023925781, + "loss": 1.3791, + "margin_dpo/margin_mean": 7.651384353637695, + "margin_dpo/margin_std": 8.097295761108398, + "step": 58 + }, + { + "KL/chosen_KL_mean": -1.4949493408203125, + "KL/mean": -5.701706886291504, + "KL/rejected_KL_mean": -9.908466339111328, + "KL/std": 7.460696697235107, + "epoch": 0.08663729809104258, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 8.413518905639648, + "fcm_dpo/q_t": 0.4980163276195526, + "grad_norm": 1.1153010129928589, + "learning_rate": 4.2028985507246374e-07, + "logits/chosen": -0.6216074228286743, + "logits/rejected": -0.5813175439834595, + "logps/chosen": -53.42747497558594, + "logps/ref_chosen": -51.932525634765625, + "logps/ref_rejected": -88.88520050048828, + "logps/rejected": -98.79367065429688, + "loss": 1.3784, + "margin_dpo/margin_mean": 8.413517951965332, + "margin_dpo/margin_std": 8.59628677368164, + "step": 59 + }, + { + "KL/chosen_KL_mean": -2.799711227416992, + "KL/mean": -6.168231964111328, + "KL/rejected_KL_mean": -9.536746978759766, + "KL/std": 6.918834686279297, + "epoch": 0.0881057268722467, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.737033843994141, + "fcm_dpo/q_t": 0.4984115660190582, + "grad_norm": 0.9285561442375183, + "learning_rate": 4.2753623188405794e-07, + "logits/chosen": -0.606009840965271, + "logits/rejected": -0.5456796884536743, + "logps/chosen": -63.74189758300781, + "logps/ref_chosen": -60.94218826293945, + "logps/ref_rejected": -85.39340209960938, + "logps/rejected": -94.93014526367188, + "loss": 1.38, + "margin_dpo/margin_mean": 6.737033367156982, + "margin_dpo/margin_std": 7.816346645355225, + "step": 60 + }, + { + "KL/chosen_KL_mean": -1.6795711517333984, + "KL/mean": -5.893689155578613, + "KL/rejected_KL_mean": -10.107810974121094, + "KL/std": 9.398534774780273, + "epoch": 0.08957415565345081, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 8.428237915039062, + "fcm_dpo/q_t": 0.498012900352478, + "grad_norm": 1.0661332607269287, + "learning_rate": 4.3478260869565214e-07, + "logits/chosen": -0.6031610369682312, + "logits/rejected": -0.5691741704940796, + "logps/chosen": -62.31309509277344, + "logps/ref_chosen": -60.633522033691406, + "logps/ref_rejected": -89.85249328613281, + "logps/rejected": -99.96031188964844, + "loss": 1.3784, + "margin_dpo/margin_mean": 8.428237915039062, + "margin_dpo/margin_std": 11.791951179504395, + "step": 61 + }, + { + "KL/chosen_KL_mean": -1.8402385711669922, + "KL/mean": -5.0417022705078125, + "KL/rejected_KL_mean": -8.243167877197266, + "KL/std": 7.013890266418457, + "epoch": 0.09104258443465492, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.402925491333008, + "fcm_dpo/q_t": 0.4984903633594513, + "grad_norm": 0.8285877704620361, + "learning_rate": 4.420289855072464e-07, + "logits/chosen": -0.6119799613952637, + "logits/rejected": -0.577934980392456, + "logps/chosen": -57.99101257324219, + "logps/ref_chosen": -56.15077209472656, + "logps/ref_rejected": -75.56619262695312, + "logps/rejected": -83.80936431884766, + "loss": 1.3803, + "margin_dpo/margin_mean": 6.402926445007324, + "margin_dpo/margin_std": 8.621729850769043, + "step": 62 + }, + { + "KL/chosen_KL_mean": -3.1350955963134766, + "KL/mean": -7.4803924560546875, + "KL/rejected_KL_mean": -11.825679779052734, + "KL/std": 8.981889724731445, + "epoch": 0.09251101321585903, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 8.690584182739258, + "fcm_dpo/q_t": 0.4979510009288788, + "grad_norm": 1.0361840724945068, + "learning_rate": 4.4927536231884053e-07, + "logits/chosen": -0.6032196283340454, + "logits/rejected": -0.5593255758285522, + "logps/chosen": -76.28248596191406, + "logps/ref_chosen": -73.14739227294922, + "logps/ref_rejected": -97.61006164550781, + "logps/rejected": -109.43574523925781, + "loss": 1.3781, + "margin_dpo/margin_mean": 8.690584182739258, + "margin_dpo/margin_std": 9.405391693115234, + "step": 63 + }, + { + "KL/chosen_KL_mean": -1.344888687133789, + "KL/mean": -6.571680068969727, + "KL/rejected_KL_mean": -11.798473358154297, + "KL/std": 9.811971664428711, + "epoch": 0.09397944199706314, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 10.45357894897461, + "fcm_dpo/q_t": 0.49753537774086, + "grad_norm": 1.1841435432434082, + "learning_rate": 4.5652173913043473e-07, + "logits/chosen": -0.5892411470413208, + "logits/rejected": -0.5572826862335205, + "logps/chosen": -55.34349060058594, + "logps/ref_chosen": -53.998600006103516, + "logps/ref_rejected": -93.53019714355469, + "logps/rejected": -105.32867431640625, + "loss": 1.3765, + "margin_dpo/margin_mean": 10.45357894897461, + "margin_dpo/margin_std": 12.130199432373047, + "step": 64 + }, + { + "KL/chosen_KL_mean": -3.9116477966308594, + "KL/mean": -9.302034378051758, + "KL/rejected_KL_mean": -14.692424774169922, + "KL/std": 11.291559219360352, + "epoch": 0.09544787077826726, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 10.780777931213379, + "fcm_dpo/q_t": 0.49745821952819824, + "grad_norm": 1.2158167362213135, + "learning_rate": 4.63768115942029e-07, + "logits/chosen": -0.6837905049324036, + "logits/rejected": -0.6725139021873474, + "logps/chosen": -68.74765014648438, + "logps/ref_chosen": -64.83599853515625, + "logps/ref_rejected": -109.94645690917969, + "logps/rejected": -124.63888549804688, + "loss": 1.3762, + "margin_dpo/margin_mean": 10.780776977539062, + "margin_dpo/margin_std": 12.423912048339844, + "step": 65 + }, + { + "KL/chosen_KL_mean": -3.748075485229492, + "KL/mean": -8.604604721069336, + "KL/rejected_KL_mean": -13.461128234863281, + "KL/std": 10.147451400756836, + "epoch": 0.09691629955947137, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 9.713046073913574, + "fcm_dpo/q_t": 0.4977099895477295, + "grad_norm": 1.069097638130188, + "learning_rate": 4.7101449275362313e-07, + "logits/chosen": -0.6540935039520264, + "logits/rejected": -0.6227428913116455, + "logps/chosen": -55.19160079956055, + "logps/ref_chosen": -51.44352722167969, + "logps/ref_rejected": -75.63629913330078, + "logps/rejected": -89.09742736816406, + "loss": 1.3772, + "margin_dpo/margin_mean": 9.713046073913574, + "margin_dpo/margin_std": 12.457484245300293, + "step": 66 + }, + { + "KL/chosen_KL_mean": -3.5582504272460938, + "KL/mean": -8.677906036376953, + "KL/rejected_KL_mean": -13.797565460205078, + "KL/std": 10.710579872131348, + "epoch": 0.09838472834067548, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 10.239309310913086, + "fcm_dpo/q_t": 0.4975859522819519, + "grad_norm": 1.0920296907424927, + "learning_rate": 4.782608695652174e-07, + "logits/chosen": -0.6097283363342285, + "logits/rejected": -0.5669122934341431, + "logps/chosen": -62.89905548095703, + "logps/ref_chosen": -59.34080505371094, + "logps/ref_rejected": -72.78728485107422, + "logps/rejected": -86.58485412597656, + "loss": 1.3767, + "margin_dpo/margin_mean": 10.239309310913086, + "margin_dpo/margin_std": 13.077437400817871, + "step": 67 + }, + { + "KL/chosen_KL_mean": -4.973361968994141, + "KL/mean": -9.859228134155273, + "KL/rejected_KL_mean": -14.745094299316406, + "KL/std": 9.751331329345703, + "epoch": 0.09985315712187959, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 9.771727561950684, + "fcm_dpo/q_t": 0.4976961016654968, + "grad_norm": 0.9640390276908875, + "learning_rate": 4.855072463768116e-07, + "logits/chosen": -0.6477208137512207, + "logits/rejected": -0.5843826532363892, + "logps/chosen": -70.17919921875, + "logps/ref_chosen": -65.2058334350586, + "logps/ref_rejected": -77.20724487304688, + "logps/rejected": -91.95234680175781, + "loss": 1.3771, + "margin_dpo/margin_mean": 9.771727561950684, + "margin_dpo/margin_std": 11.175668716430664, + "step": 68 + }, + { + "KL/chosen_KL_mean": -6.378795623779297, + "KL/mean": -13.339784622192383, + "KL/rejected_KL_mean": -20.300777435302734, + "KL/std": 13.335014343261719, + "epoch": 0.1013215859030837, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 13.921974182128906, + "fcm_dpo/q_t": 0.49671778082847595, + "grad_norm": 1.3528519868850708, + "learning_rate": 4.927536231884058e-07, + "logits/chosen": -0.6241370439529419, + "logits/rejected": -0.6030097007751465, + "logps/chosen": -66.19803619384766, + "logps/ref_chosen": -59.81924057006836, + "logps/ref_rejected": -103.38886260986328, + "logps/rejected": -123.68963623046875, + "loss": 1.3733, + "margin_dpo/margin_mean": 13.921974182128906, + "margin_dpo/margin_std": 15.130611419677734, + "step": 69 + }, + { + "KL/chosen_KL_mean": -9.457656860351562, + "KL/mean": -16.863452911376953, + "KL/rejected_KL_mean": -24.269248962402344, + "KL/std": 17.1834716796875, + "epoch": 0.1027900146842878, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 14.811593055725098, + "fcm_dpo/q_t": 0.4965081214904785, + "grad_norm": 1.351995825767517, + "learning_rate": 5e-07, + "logits/chosen": -0.6134965419769287, + "logits/rejected": -0.5804483890533447, + "logps/chosen": -71.38829803466797, + "logps/ref_chosen": -61.930641174316406, + "logps/ref_rejected": -91.06078338623047, + "logps/rejected": -115.33003234863281, + "loss": 1.3724, + "margin_dpo/margin_mean": 14.811592102050781, + "margin_dpo/margin_std": 18.324480056762695, + "step": 70 + }, + { + "KL/chosen_KL_mean": -10.057798385620117, + "KL/mean": -19.228286743164062, + "KL/rejected_KL_mean": -28.398780822753906, + "KL/std": 19.04326820373535, + "epoch": 0.10425844346549193, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 18.340984344482422, + "fcm_dpo/q_t": 0.4956763982772827, + "grad_norm": 1.5885552167892456, + "learning_rate": 4.999967061337492e-07, + "logits/chosen": -0.6829918622970581, + "logits/rejected": -0.650745153427124, + "logps/chosen": -71.80813598632812, + "logps/ref_chosen": -61.750335693359375, + "logps/ref_rejected": -97.33662414550781, + "logps/rejected": -125.73539733886719, + "loss": 1.3692, + "margin_dpo/margin_mean": 18.340984344482422, + "margin_dpo/margin_std": 21.016489028930664, + "step": 71 + }, + { + "KL/chosen_KL_mean": -11.907201766967773, + "KL/mean": -21.156211853027344, + "KL/rejected_KL_mean": -30.405223846435547, + "KL/std": 20.45846939086914, + "epoch": 0.10572687224669604, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 18.498023986816406, + "fcm_dpo/q_t": 0.4956393539905548, + "grad_norm": 1.4867459535598755, + "learning_rate": 4.999868246217933e-07, + "logits/chosen": -0.6599299907684326, + "logits/rejected": -0.6240644454956055, + "logps/chosen": -77.96061706542969, + "logps/ref_chosen": -66.05341339111328, + "logps/ref_rejected": -95.2869873046875, + "logps/rejected": -125.69221496582031, + "loss": 1.3691, + "margin_dpo/margin_mean": 18.498022079467773, + "margin_dpo/margin_std": 25.244348526000977, + "step": 72 + }, + { + "KL/chosen_KL_mean": -14.035341262817383, + "KL/mean": -23.60753059387207, + "KL/rejected_KL_mean": -33.17972183227539, + "KL/std": 26.00308609008789, + "epoch": 0.10719530102790015, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 19.144380569458008, + "fcm_dpo/q_t": 0.4954878091812134, + "grad_norm": 1.645006775856018, + "learning_rate": 4.999703557245192e-07, + "logits/chosen": -0.7174981832504272, + "logits/rejected": -0.681929349899292, + "logps/chosen": -80.29161071777344, + "logps/ref_chosen": -66.25627136230469, + "logps/ref_rejected": -90.45613098144531, + "logps/rejected": -123.63585662841797, + "loss": 1.3686, + "margin_dpo/margin_mean": 19.144378662109375, + "margin_dpo/margin_std": 33.420066833496094, + "step": 73 + }, + { + "KL/chosen_KL_mean": -14.948610305786133, + "KL/mean": -26.0163631439209, + "KL/rejected_KL_mean": -37.0841178894043, + "KL/std": 29.37421989440918, + "epoch": 0.10866372980910426, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 22.13550567626953, + "fcm_dpo/q_t": 0.49478527903556824, + "grad_norm": 1.8224822282791138, + "learning_rate": 4.999472998758977e-07, + "logits/chosen": -0.6496589183807373, + "logits/rejected": -0.6474369764328003, + "logps/chosen": -68.37348937988281, + "logps/ref_chosen": -53.42488098144531, + "logps/ref_rejected": -95.94693756103516, + "logps/rejected": -133.0310516357422, + "loss": 1.3659, + "margin_dpo/margin_mean": 22.13550567626953, + "margin_dpo/margin_std": 37.34225845336914, + "step": 74 + }, + { + "KL/chosen_KL_mean": -15.307416915893555, + "KL/mean": -31.02456283569336, + "KL/rejected_KL_mean": -46.74169158935547, + "KL/std": 30.637062072753906, + "epoch": 0.11013215859030837, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 31.434282302856445, + "fcm_dpo/q_t": 0.4925920069217682, + "grad_norm": 2.1851608753204346, + "learning_rate": 4.999176576834721e-07, + "logits/chosen": -0.6600038409233093, + "logits/rejected": -0.6581864953041077, + "logps/chosen": -67.16908264160156, + "logps/ref_chosen": -51.861663818359375, + "logps/ref_rejected": -111.25398254394531, + "logps/rejected": -157.99566650390625, + "loss": 1.3572, + "margin_dpo/margin_mean": 31.434280395507812, + "margin_dpo/margin_std": 35.90174102783203, + "step": 75 + }, + { + "KL/chosen_KL_mean": -18.540132522583008, + "KL/mean": -28.030113220214844, + "KL/rejected_KL_mean": -37.52008819580078, + "KL/std": 25.205242156982422, + "epoch": 0.11160058737151249, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 18.979961395263672, + "fcm_dpo/q_t": 0.49552568793296814, + "grad_norm": 1.614672064781189, + "learning_rate": 4.998814299283415e-07, + "logits/chosen": -0.6978548765182495, + "logits/rejected": -0.6618174910545349, + "logps/chosen": -71.80616760253906, + "logps/ref_chosen": -53.26603698730469, + "logps/ref_rejected": -78.21662902832031, + "logps/rejected": -115.7367172241211, + "loss": 1.3687, + "margin_dpo/margin_mean": 18.979961395263672, + "margin_dpo/margin_std": 28.169784545898438, + "step": 76 + }, + { + "KL/chosen_KL_mean": -17.505142211914062, + "KL/mean": -33.80626678466797, + "KL/rejected_KL_mean": -50.107398986816406, + "KL/std": 34.56831359863281, + "epoch": 0.1130690161527166, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 32.602256774902344, + "fcm_dpo/q_t": 0.4923163056373596, + "grad_norm": 2.1774914264678955, + "learning_rate": 4.998386175651409e-07, + "logits/chosen": -0.6647493839263916, + "logits/rejected": -0.631820797920227, + "logps/chosen": -75.60182189941406, + "logps/ref_chosen": -58.0966796875, + "logps/ref_rejected": -93.77361297607422, + "logps/rejected": -143.88101196289062, + "loss": 1.3561, + "margin_dpo/margin_mean": 32.602256774902344, + "margin_dpo/margin_std": 40.01980972290039, + "step": 77 + }, + { + "KL/chosen_KL_mean": -18.282596588134766, + "KL/mean": -30.055492401123047, + "KL/rejected_KL_mean": -41.82838821411133, + "KL/std": 29.950658798217773, + "epoch": 0.1145374449339207, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 23.545791625976562, + "fcm_dpo/q_t": 0.4944510757923126, + "grad_norm": 1.7334191799163818, + "learning_rate": 4.997892217220159e-07, + "logits/chosen": -0.631340503692627, + "logits/rejected": -0.6118667721748352, + "logps/chosen": -73.89637756347656, + "logps/ref_chosen": -55.61378479003906, + "logps/ref_rejected": -84.93436431884766, + "logps/rejected": -126.76275634765625, + "loss": 1.3645, + "margin_dpo/margin_mean": 23.545791625976562, + "margin_dpo/margin_std": 35.435997009277344, + "step": 78 + }, + { + "KL/chosen_KL_mean": -19.754838943481445, + "KL/mean": -32.51957702636719, + "KL/rejected_KL_mean": -45.28432083129883, + "KL/std": 32.64012908935547, + "epoch": 0.11600587371512482, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 25.52947998046875, + "fcm_dpo/q_t": 0.4939836859703064, + "grad_norm": 1.8259940147399902, + "learning_rate": 4.997332437005931e-07, + "logits/chosen": -0.6151328682899475, + "logits/rejected": -0.5880134105682373, + "logps/chosen": -75.205322265625, + "logps/ref_chosen": -55.45048522949219, + "logps/ref_rejected": -87.64756774902344, + "logps/rejected": -132.931884765625, + "loss": 1.3627, + "margin_dpo/margin_mean": 25.529478073120117, + "margin_dpo/margin_std": 39.56281280517578, + "step": 79 + }, + { + "KL/chosen_KL_mean": -24.592361450195312, + "KL/mean": -38.224281311035156, + "KL/rejected_KL_mean": -51.856201171875, + "KL/std": 38.630069732666016, + "epoch": 0.11747430249632893, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 27.263832092285156, + "fcm_dpo/q_t": 0.49357831478118896, + "grad_norm": 2.117938995361328, + "learning_rate": 4.996706849759452e-07, + "logits/chosen": -0.7042999267578125, + "logits/rejected": -0.672730565071106, + "logps/chosen": -83.11164855957031, + "logps/ref_chosen": -58.519290924072266, + "logps/ref_rejected": -87.54750061035156, + "logps/rejected": -139.40370178222656, + "loss": 1.3613, + "margin_dpo/margin_mean": 27.263832092285156, + "margin_dpo/margin_std": 45.90215301513672, + "step": 80 + }, + { + "KL/chosen_KL_mean": -25.093727111816406, + "KL/mean": -43.66114044189453, + "KL/rejected_KL_mean": -62.228553771972656, + "KL/std": 46.15410232543945, + "epoch": 0.11894273127753303, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 37.13482666015625, + "fcm_dpo/q_t": 0.4912540912628174, + "grad_norm": 2.5071187019348145, + "learning_rate": 4.996015471965529e-07, + "logits/chosen": -0.6825768947601318, + "logits/rejected": -0.6628377437591553, + "logps/chosen": -91.54259490966797, + "logps/ref_chosen": -66.44886779785156, + "logps/ref_rejected": -129.66270446777344, + "logps/rejected": -191.89126586914062, + "loss": 1.3523, + "margin_dpo/margin_mean": 37.13482666015625, + "margin_dpo/margin_std": 55.59607696533203, + "step": 81 + }, + { + "KL/chosen_KL_mean": -28.598669052124023, + "KL/mean": -43.321502685546875, + "KL/rejected_KL_mean": -58.04433822631836, + "KL/std": 42.634342193603516, + "epoch": 0.12041116005873716, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 29.445667266845703, + "fcm_dpo/q_t": 0.4930664896965027, + "grad_norm": 2.0818185806274414, + "learning_rate": 4.995258321842611e-07, + "logits/chosen": -0.5801957249641418, + "logits/rejected": -0.5773532390594482, + "logps/chosen": -80.8310546875, + "logps/ref_chosen": -52.232383728027344, + "logps/ref_rejected": -90.74325561523438, + "logps/rejected": -148.78759765625, + "loss": 1.3596, + "margin_dpo/margin_mean": 29.445667266845703, + "margin_dpo/margin_std": 59.75962829589844, + "step": 82 + }, + { + "KL/chosen_KL_mean": -30.05629539489746, + "KL/mean": -49.223304748535156, + "KL/rejected_KL_mean": -68.39031982421875, + "KL/std": 45.316627502441406, + "epoch": 0.12187958883994127, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 38.33401870727539, + "fcm_dpo/q_t": 0.4909701645374298, + "grad_norm": 2.6023781299591064, + "learning_rate": 4.994435419342304e-07, + "logits/chosen": -0.614409327507019, + "logits/rejected": -0.5958919525146484, + "logps/chosen": -85.88368225097656, + "logps/ref_chosen": -55.82738494873047, + "logps/ref_rejected": -103.71589660644531, + "logps/rejected": -172.10623168945312, + "loss": 1.3511, + "margin_dpo/margin_mean": 38.334014892578125, + "margin_dpo/margin_std": 54.87122344970703, + "step": 83 + }, + { + "KL/chosen_KL_mean": -32.64569091796875, + "KL/mean": -46.82188415527344, + "KL/rejected_KL_mean": -60.99808120727539, + "KL/std": 39.93206787109375, + "epoch": 0.12334801762114538, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 28.352384567260742, + "fcm_dpo/q_t": 0.49331820011138916, + "grad_norm": 2.06538462638855, + "learning_rate": 4.993546786148857e-07, + "logits/chosen": -0.645265519618988, + "logits/rejected": -0.6129859685897827, + "logps/chosen": -99.82185363769531, + "logps/ref_chosen": -67.1761703491211, + "logps/ref_rejected": -87.29859924316406, + "logps/rejected": -148.29669189453125, + "loss": 1.3601, + "margin_dpo/margin_mean": 28.352384567260742, + "margin_dpo/margin_std": 39.42462158203125, + "step": 84 + }, + { + "KL/chosen_KL_mean": -32.759803771972656, + "KL/mean": -47.751686096191406, + "KL/rejected_KL_mean": -62.743568420410156, + "KL/std": 39.7208251953125, + "epoch": 0.12481644640234948, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 29.9837646484375, + "fcm_dpo/q_t": 0.4929351806640625, + "grad_norm": 2.0294606685638428, + "learning_rate": 4.992592445678582e-07, + "logits/chosen": -0.5920377969741821, + "logits/rejected": -0.5608456134796143, + "logps/chosen": -91.16642761230469, + "logps/ref_chosen": -58.4066162109375, + "logps/ref_rejected": -78.63880157470703, + "logps/rejected": -141.3823699951172, + "loss": 1.3588, + "margin_dpo/margin_mean": 29.9837646484375, + "margin_dpo/margin_std": 49.01397705078125, + "step": 85 + }, + { + "KL/chosen_KL_mean": -44.688865661621094, + "KL/mean": -60.836769104003906, + "KL/rejected_KL_mean": -76.98466491699219, + "KL/std": 61.91584396362305, + "epoch": 0.1262848751835536, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 32.29580307006836, + "fcm_dpo/q_t": 0.492409884929657, + "grad_norm": 2.3101718425750732, + "learning_rate": 4.991572423079235e-07, + "logits/chosen": -0.6010197401046753, + "logits/rejected": -0.5972357392311096, + "logps/chosen": -100.82633209228516, + "logps/ref_chosen": -56.13746643066406, + "logps/ref_rejected": -88.12165069580078, + "logps/rejected": -165.1063232421875, + "loss": 1.3578, + "margin_dpo/margin_mean": 32.29580307006836, + "margin_dpo/margin_std": 87.33891296386719, + "step": 86 + }, + { + "KL/chosen_KL_mean": -44.94416046142578, + "KL/mean": -66.77362823486328, + "KL/rejected_KL_mean": -88.60308837890625, + "KL/std": 63.363311767578125, + "epoch": 0.1277533039647577, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 43.65892028808594, + "fcm_dpo/q_t": 0.48971858620643616, + "grad_norm": 2.6735432147979736, + "learning_rate": 4.990486745229364e-07, + "logits/chosen": -0.6408615112304688, + "logits/rejected": -0.6341418027877808, + "logps/chosen": -100.58026123046875, + "logps/ref_chosen": -55.63609313964844, + "logps/ref_rejected": -95.46757507324219, + "logps/rejected": -184.0706787109375, + "loss": 1.3471, + "margin_dpo/margin_mean": 43.6589241027832, + "margin_dpo/margin_std": 80.54689025878906, + "step": 87 + }, + { + "KL/chosen_KL_mean": -58.59263229370117, + "KL/mean": -75.60669708251953, + "KL/rejected_KL_mean": -92.62075805664062, + "KL/std": 71.91948699951172, + "epoch": 0.12922173274596183, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 34.02812957763672, + "fcm_dpo/q_t": 0.49200400710105896, + "grad_norm": 2.6304991245269775, + "learning_rate": 4.989335440737586e-07, + "logits/chosen": -0.5753469467163086, + "logits/rejected": -0.57887864112854, + "logps/chosen": -132.26377868652344, + "logps/ref_chosen": -73.67115020751953, + "logps/ref_rejected": -106.70849609375, + "logps/rejected": -199.32925415039062, + "loss": 1.3562, + "margin_dpo/margin_mean": 34.02812576293945, + "margin_dpo/margin_std": 82.46296691894531, + "step": 88 + }, + { + "KL/chosen_KL_mean": -44.48945617675781, + "KL/mean": -63.188812255859375, + "KL/rejected_KL_mean": -81.88816833496094, + "KL/std": 67.53860473632812, + "epoch": 0.13069016152716592, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 37.39871597290039, + "fcm_dpo/q_t": 0.4911901354789734, + "grad_norm": 2.4624993801116943, + "learning_rate": 4.988118539941847e-07, + "logits/chosen": -0.6243025660514832, + "logits/rejected": -0.6062852144241333, + "logps/chosen": -105.11437225341797, + "logps/ref_chosen": -60.624916076660156, + "logps/ref_rejected": -82.08354949951172, + "logps/rejected": -163.97171020507812, + "loss": 1.3528, + "margin_dpo/margin_mean": 37.398719787597656, + "margin_dpo/margin_std": 81.66424560546875, + "step": 89 + }, + { + "KL/chosen_KL_mean": -48.97517395019531, + "KL/mean": -81.48391723632812, + "KL/rejected_KL_mean": -113.99266052246094, + "KL/std": 89.38653564453125, + "epoch": 0.13215859030837004, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 65.01748657226562, + "fcm_dpo/q_t": 0.48473870754241943, + "grad_norm": 3.8177247047424316, + "learning_rate": 4.986836074908615e-07, + "logits/chosen": -0.5124300122261047, + "logits/rejected": -0.5386440753936768, + "logps/chosen": -102.26048278808594, + "logps/ref_chosen": -53.285308837890625, + "logps/ref_rejected": -111.54470825195312, + "logps/rejected": -225.53738403320312, + "loss": 1.3289, + "margin_dpo/margin_mean": 65.01748657226562, + "margin_dpo/margin_std": 113.12422943115234, + "step": 90 + }, + { + "KL/chosen_KL_mean": -57.929931640625, + "KL/mean": -77.27233123779297, + "KL/rejected_KL_mean": -96.6147232055664, + "KL/std": 69.36103057861328, + "epoch": 0.13362701908957417, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 38.684791564941406, + "fcm_dpo/q_t": 0.4909099340438843, + "grad_norm": 2.578143358230591, + "learning_rate": 4.985488079432037e-07, + "logits/chosen": -0.5383755564689636, + "logits/rejected": -0.5206432342529297, + "logps/chosen": -119.73289489746094, + "logps/ref_chosen": -61.802955627441406, + "logps/ref_rejected": -87.87395477294922, + "logps/rejected": -184.48867797851562, + "loss": 1.3522, + "margin_dpo/margin_mean": 38.68478775024414, + "margin_dpo/margin_std": 95.96929931640625, + "step": 91 + }, + { + "KL/chosen_KL_mean": -53.14446258544922, + "KL/mean": -75.70481872558594, + "KL/rejected_KL_mean": -98.26518249511719, + "KL/std": 75.28404235839844, + "epoch": 0.13509544787077826, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 45.120723724365234, + "fcm_dpo/q_t": 0.4893913269042969, + "grad_norm": 3.023728370666504, + "learning_rate": 4.984074589033043e-07, + "logits/chosen": -0.5423229932785034, + "logits/rejected": -0.5309740304946899, + "logps/chosen": -104.78523254394531, + "logps/ref_chosen": -51.640769958496094, + "logps/ref_rejected": -77.88117980957031, + "logps/rejected": -176.1463623046875, + "loss": 1.3461, + "margin_dpo/margin_mean": 45.1207275390625, + "margin_dpo/margin_std": 90.38905334472656, + "step": 92 + }, + { + "KL/chosen_KL_mean": -59.912742614746094, + "KL/mean": -81.39234924316406, + "KL/rejected_KL_mean": -102.8719482421875, + "KL/std": 67.60503387451172, + "epoch": 0.13656387665198239, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 42.959205627441406, + "fcm_dpo/q_t": 0.4898951053619385, + "grad_norm": 2.951615810394287, + "learning_rate": 4.982595640958425e-07, + "logits/chosen": -0.5452234745025635, + "logits/rejected": -0.524033784866333, + "logps/chosen": -112.44198608398438, + "logps/ref_chosen": -52.529239654541016, + "logps/ref_rejected": -77.16075134277344, + "logps/rejected": -180.03269958496094, + "loss": 1.3476, + "margin_dpo/margin_mean": 42.959205627441406, + "margin_dpo/margin_std": 79.7041015625, + "step": 93 + }, + { + "KL/chosen_KL_mean": -66.89620208740234, + "KL/mean": -95.25006103515625, + "KL/rejected_KL_mean": -123.60391235351562, + "KL/std": 84.54957580566406, + "epoch": 0.13803230543318648, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 56.70771408081055, + "fcm_dpo/q_t": 0.4866675138473511, + "grad_norm": 3.3346071243286133, + "learning_rate": 4.98105127417984e-07, + "logits/chosen": -0.5058275461196899, + "logits/rejected": -0.5132472515106201, + "logps/chosen": -128.11880493164062, + "logps/ref_chosen": -61.22261047363281, + "logps/ref_rejected": -99.59902954101562, + "logps/rejected": -223.20294189453125, + "loss": 1.3356, + "margin_dpo/margin_mean": 56.70771026611328, + "margin_dpo/margin_std": 94.90916442871094, + "step": 94 + }, + { + "KL/chosen_KL_mean": -66.25115203857422, + "KL/mean": -89.31552124023438, + "KL/rejected_KL_mean": -112.37989044189453, + "KL/std": 95.1351089477539, + "epoch": 0.1395007342143906, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 46.12873840332031, + "fcm_dpo/q_t": 0.4891487956047058, + "grad_norm": 3.2070772647857666, + "learning_rate": 4.979441529392784e-07, + "logits/chosen": -0.47669315338134766, + "logits/rejected": -0.4623689651489258, + "logps/chosen": -118.77479553222656, + "logps/ref_chosen": -52.523643493652344, + "logps/ref_rejected": -75.8803482055664, + "logps/rejected": -188.26023864746094, + "loss": 1.3457, + "margin_dpo/margin_mean": 46.12873840332031, + "margin_dpo/margin_std": 102.57861328125, + "step": 95 + }, + { + "KL/chosen_KL_mean": -75.35250854492188, + "KL/mean": -108.11980438232422, + "KL/rejected_KL_mean": -140.88711547851562, + "KL/std": 99.36506652832031, + "epoch": 0.14096916299559473, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 65.53459930419922, + "fcm_dpo/q_t": 0.4846518039703369, + "grad_norm": 3.4605512619018555, + "learning_rate": 4.977766449015534e-07, + "logits/chosen": -0.4755854606628418, + "logits/rejected": -0.46957653760910034, + "logps/chosen": -137.50949096679688, + "logps/ref_chosen": -62.15697479248047, + "logps/ref_rejected": -96.59601593017578, + "logps/rejected": -237.48312377929688, + "loss": 1.3288, + "margin_dpo/margin_mean": 65.53459167480469, + "margin_dpo/margin_std": 118.84884643554688, + "step": 96 + }, + { + "KL/chosen_KL_mean": -70.3343734741211, + "KL/mean": -93.92042541503906, + "KL/rejected_KL_mean": -117.50647735595703, + "KL/std": 90.63258361816406, + "epoch": 0.14243759177679882, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 47.1721076965332, + "fcm_dpo/q_t": 0.4889163076877594, + "grad_norm": 3.436333417892456, + "learning_rate": 4.976026077188012e-07, + "logits/chosen": -0.4467729330062866, + "logits/rejected": -0.43124696612358093, + "logps/chosen": -124.98074340820312, + "logps/ref_chosen": -54.646366119384766, + "logps/ref_rejected": -76.96475219726562, + "logps/rejected": -194.47122192382812, + "loss": 1.3443, + "margin_dpo/margin_mean": 47.17211151123047, + "margin_dpo/margin_std": 92.33546447753906, + "step": 97 + }, + { + "KL/chosen_KL_mean": -95.86998748779297, + "KL/mean": -124.71914672851562, + "KL/rejected_KL_mean": -153.5682830810547, + "KL/std": 112.18405151367188, + "epoch": 0.14390602055800295, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.69831848144531, + "fcm_dpo/q_t": 0.486421674489975, + "grad_norm": 4.174478530883789, + "learning_rate": 4.974220459770639e-07, + "logits/chosen": -0.4279845356941223, + "logits/rejected": -0.43450891971588135, + "logps/chosen": -161.1286163330078, + "logps/ref_chosen": -65.25862884521484, + "logps/ref_rejected": -96.5274887084961, + "logps/rejected": -250.0957794189453, + "loss": 1.3368, + "margin_dpo/margin_mean": 57.69831085205078, + "margin_dpo/margin_std": 136.08026123046875, + "step": 98 + }, + { + "KL/chosen_KL_mean": -70.87637329101562, + "KL/mean": -108.80039978027344, + "KL/rejected_KL_mean": -146.72442626953125, + "KL/std": 105.301513671875, + "epoch": 0.14537444933920704, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 75.84803771972656, + "fcm_dpo/q_t": 0.48224809765815735, + "grad_norm": 4.009462833404541, + "learning_rate": 4.972349644343108e-07, + "logits/chosen": -0.3845648467540741, + "logits/rejected": -0.39073482155799866, + "logps/chosen": -116.51486206054688, + "logps/ref_chosen": -45.638484954833984, + "logps/ref_rejected": -86.43793487548828, + "logps/rejected": -233.162353515625, + "loss": 1.3202, + "margin_dpo/margin_mean": 75.84803771972656, + "margin_dpo/margin_std": 136.87684631347656, + "step": 99 + }, + { + "KL/chosen_KL_mean": -88.81651306152344, + "KL/mean": -101.03938293457031, + "KL/rejected_KL_mean": -113.26225280761719, + "KL/std": 101.6782455444336, + "epoch": 0.14684287812041116, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 24.445741653442383, + "fcm_dpo/q_t": 0.49424445629119873, + "grad_norm": 3.8610727787017822, + "learning_rate": 4.970413680203148e-07, + "logits/chosen": -0.3569236397743225, + "logits/rejected": -0.35417813062667847, + "logps/chosen": -146.41049194335938, + "logps/ref_chosen": -57.59397888183594, + "logps/ref_rejected": -74.06021118164062, + "logps/rejected": -187.3224639892578, + "loss": 1.3669, + "margin_dpo/margin_mean": 24.44573974609375, + "margin_dpo/margin_std": 121.69652557373047, + "step": 100 + }, + { + "KL/chosen_KL_mean": -103.06278991699219, + "KL/mean": -123.61421203613281, + "KL/rejected_KL_mean": -144.16561889648438, + "KL/std": 117.26971435546875, + "epoch": 0.14831130690161526, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 41.10283660888672, + "fcm_dpo/q_t": 0.49035578966140747, + "grad_norm": 3.403747081756592, + "learning_rate": 4.968412618365215e-07, + "logits/chosen": -0.3375687599182129, + "logits/rejected": -0.3361296057701111, + "logps/chosen": -164.71163940429688, + "logps/ref_chosen": -61.64885330200195, + "logps/ref_rejected": -83.18968200683594, + "logps/rejected": -227.35531616210938, + "loss": 1.352, + "margin_dpo/margin_mean": 41.10283660888672, + "margin_dpo/margin_std": 134.45703125, + "step": 101 + }, + { + "KL/chosen_KL_mean": -104.34712982177734, + "KL/mean": -118.05978393554688, + "KL/rejected_KL_mean": -131.77243041992188, + "KL/std": 111.49862670898438, + "epoch": 0.14977973568281938, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 27.42530059814453, + "fcm_dpo/q_t": 0.4935542345046997, + "grad_norm": 3.6488194465637207, + "learning_rate": 4.966346511559149e-07, + "logits/chosen": -0.36390089988708496, + "logits/rejected": -0.3599966764450073, + "logps/chosen": -168.42601013183594, + "logps/ref_chosen": -64.0788803100586, + "logps/ref_rejected": -68.18707275390625, + "logps/rejected": -199.95950317382812, + "loss": 1.3651, + "margin_dpo/margin_mean": 27.425302505493164, + "margin_dpo/margin_std": 138.6975555419922, + "step": 102 + }, + { + "KL/chosen_KL_mean": -99.00223541259766, + "KL/mean": -133.520263671875, + "KL/rejected_KL_mean": -168.03829956054688, + "KL/std": 128.6646728515625, + "epoch": 0.1512481644640235, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 69.03605651855469, + "fcm_dpo/q_t": 0.4838695824146271, + "grad_norm": 3.918245553970337, + "learning_rate": 4.964215414228785e-07, + "logits/chosen": -0.3378918170928955, + "logits/rejected": -0.33644479513168335, + "logps/chosen": -160.301513671875, + "logps/ref_chosen": -61.299278259277344, + "logps/ref_rejected": -93.57270812988281, + "logps/rejected": -261.61102294921875, + "loss": 1.3284, + "margin_dpo/margin_mean": 69.03605651855469, + "margin_dpo/margin_std": 166.41140747070312, + "step": 103 + }, + { + "KL/chosen_KL_mean": -91.2227783203125, + "KL/mean": -124.46497344970703, + "KL/rejected_KL_mean": -157.70718383789062, + "KL/std": 121.44420623779297, + "epoch": 0.1527165932452276, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.48440551757812, + "fcm_dpo/q_t": 0.4844798445701599, + "grad_norm": 4.455353736877441, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": -0.3243046998977661, + "logits/rejected": -0.33145880699157715, + "logps/chosen": -145.59555053710938, + "logps/ref_chosen": -54.372772216796875, + "logps/ref_rejected": -89.5647201538086, + "logps/rejected": -247.2718963623047, + "loss": 1.33, + "margin_dpo/margin_mean": 66.4843978881836, + "margin_dpo/margin_std": 155.1224365234375, + "step": 104 + }, + { + "KL/chosen_KL_mean": -89.49755859375, + "KL/mean": -140.19944763183594, + "KL/rejected_KL_mean": -190.9013214111328, + "KL/std": 112.19564819335938, + "epoch": 0.15418502202643172, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 101.40377807617188, + "fcm_dpo/q_t": 0.47620758414268494, + "grad_norm": 4.004859924316406, + "learning_rate": 4.959758474331832e-07, + "logits/chosen": -0.2886583209037781, + "logits/rejected": -0.2872416079044342, + "logps/chosen": -144.13650512695312, + "logps/ref_chosen": -54.638946533203125, + "logps/ref_rejected": -97.97351837158203, + "logps/rejected": -288.8748474121094, + "loss": 1.297, + "margin_dpo/margin_mean": 101.40377807617188, + "margin_dpo/margin_std": 133.38304138183594, + "step": 105 + }, + { + "KL/chosen_KL_mean": -103.17796325683594, + "KL/mean": -136.15133666992188, + "KL/rejected_KL_mean": -169.12469482421875, + "KL/std": 121.09173583984375, + "epoch": 0.15565345080763582, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 65.94671630859375, + "fcm_dpo/q_t": 0.4845297336578369, + "grad_norm": 3.938218593597412, + "learning_rate": 4.957432749209755e-07, + "logits/chosen": -0.2749488949775696, + "logits/rejected": -0.29554685950279236, + "logps/chosen": -158.0108642578125, + "logps/ref_chosen": -54.83289337158203, + "logps/ref_rejected": -85.22461700439453, + "logps/rejected": -254.34930419921875, + "loss": 1.3289, + "margin_dpo/margin_mean": 65.94671630859375, + "margin_dpo/margin_std": 128.7685546875, + "step": 106 + }, + { + "KL/chosen_KL_mean": -125.75239562988281, + "KL/mean": -159.4122314453125, + "KL/rejected_KL_mean": -193.07208251953125, + "KL/std": 142.40440368652344, + "epoch": 0.15712187958883994, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 67.31967163085938, + "fcm_dpo/q_t": 0.48424452543258667, + "grad_norm": 4.5661797523498535, + "learning_rate": 4.955042268449307e-07, + "logits/chosen": -0.2691690921783447, + "logits/rejected": -0.27037227153778076, + "logps/chosen": -195.460205078125, + "logps/ref_chosen": -69.70780944824219, + "logps/ref_rejected": -94.73950958251953, + "logps/rejected": -287.81158447265625, + "loss": 1.3291, + "margin_dpo/margin_mean": 67.31967163085938, + "margin_dpo/margin_std": 150.08763122558594, + "step": 107 + }, + { + "KL/chosen_KL_mean": -99.49694061279297, + "KL/mean": -149.0161895751953, + "KL/rejected_KL_mean": -198.53543090820312, + "KL/std": 152.70819091796875, + "epoch": 0.15859030837004406, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 99.03849029541016, + "fcm_dpo/q_t": 0.4770011305809021, + "grad_norm": 4.919802665710449, + "learning_rate": 4.952587095041881e-07, + "logits/chosen": -0.2482558786869049, + "logits/rejected": -0.2509337365627289, + "logps/chosen": -155.5068359375, + "logps/ref_chosen": -56.0098876953125, + "logps/ref_rejected": -95.79601287841797, + "logps/rejected": -294.3314514160156, + "loss": 1.3035, + "margin_dpo/margin_mean": 99.03849029541016, + "margin_dpo/margin_std": 194.90606689453125, + "step": 108 + }, + { + "KL/chosen_KL_mean": -109.02256774902344, + "KL/mean": -155.30252075195312, + "KL/rejected_KL_mean": -201.5824737548828, + "KL/std": 154.06240844726562, + "epoch": 0.16005873715124816, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 92.55990600585938, + "fcm_dpo/q_t": 0.4782617688179016, + "grad_norm": 4.247311592102051, + "learning_rate": 4.95006729368358e-07, + "logits/chosen": -0.23186370730400085, + "logits/rejected": -0.22922776639461517, + "logps/chosen": -171.90805053710938, + "logps/ref_chosen": -62.88549041748047, + "logps/ref_rejected": -98.68573760986328, + "logps/rejected": -300.2681884765625, + "loss": 1.3106, + "margin_dpo/margin_mean": 92.55990600585938, + "margin_dpo/margin_std": 207.37530517578125, + "step": 109 + }, + { + "KL/chosen_KL_mean": -112.6722640991211, + "KL/mean": -150.91159057617188, + "KL/rejected_KL_mean": -189.1509552001953, + "KL/std": 145.97552490234375, + "epoch": 0.16152716593245228, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.47869873046875, + "fcm_dpo/q_t": 0.48217493295669556, + "grad_norm": 4.57998514175415, + "learning_rate": 4.947482930773511e-07, + "logits/chosen": -0.19486288726329803, + "logits/rejected": -0.20499581098556519, + "logps/chosen": -171.4259490966797, + "logps/ref_chosen": -58.753684997558594, + "logps/ref_rejected": -79.75001525878906, + "logps/rejected": -268.9009704589844, + "loss": 1.3238, + "margin_dpo/margin_mean": 76.47869873046875, + "margin_dpo/margin_std": 185.61337280273438, + "step": 110 + }, + { + "KL/chosen_KL_mean": -127.63728332519531, + "KL/mean": -176.09266662597656, + "KL/rejected_KL_mean": -224.5480499267578, + "KL/std": 172.54864501953125, + "epoch": 0.16299559471365638, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 96.91078186035156, + "fcm_dpo/q_t": 0.4774002432823181, + "grad_norm": 4.575905799865723, + "learning_rate": 4.944834074412042e-07, + "logits/chosen": -0.2067311704158783, + "logits/rejected": -0.20800232887268066, + "logps/chosen": -196.26138305664062, + "logps/ref_chosen": -68.62410736083984, + "logps/ref_rejected": -98.42886352539062, + "logps/rejected": -322.9769287109375, + "loss": 1.3096, + "margin_dpo/margin_mean": 96.91078186035156, + "margin_dpo/margin_std": 237.3184356689453, + "step": 111 + }, + { + "KL/chosen_KL_mean": -111.8941650390625, + "KL/mean": -135.28634643554688, + "KL/rejected_KL_mean": -158.6785430908203, + "KL/std": 107.61951446533203, + "epoch": 0.1644640234948605, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 46.78435516357422, + "fcm_dpo/q_t": 0.48899713158607483, + "grad_norm": 5.0745391845703125, + "learning_rate": 4.942120794399002e-07, + "logits/chosen": -0.13949084281921387, + "logits/rejected": -0.1618117392063141, + "logps/chosen": -162.143798828125, + "logps/ref_chosen": -50.24964141845703, + "logps/ref_rejected": -64.77442932128906, + "logps/rejected": -223.45297241210938, + "loss": 1.3476, + "margin_dpo/margin_mean": 46.78435516357422, + "margin_dpo/margin_std": 145.43516540527344, + "step": 112 + }, + { + "KL/chosen_KL_mean": -146.45416259765625, + "KL/mean": -168.53370666503906, + "KL/rejected_KL_mean": -190.61326599121094, + "KL/std": 156.10946655273438, + "epoch": 0.16593245227606462, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 44.15910339355469, + "fcm_dpo/q_t": 0.48973506689071655, + "grad_norm": 4.978002071380615, + "learning_rate": 4.939343162231841e-07, + "logits/chosen": -0.18881914019584656, + "logits/rejected": -0.1964729130268097, + "logps/chosen": -213.1671142578125, + "logps/ref_chosen": -66.71295166015625, + "logps/ref_rejected": -77.96870422363281, + "logps/rejected": -268.58197021484375, + "loss": 1.3508, + "margin_dpo/margin_mean": 44.15910339355469, + "margin_dpo/margin_std": 160.76222229003906, + "step": 113 + }, + { + "KL/chosen_KL_mean": -121.00819396972656, + "KL/mean": -166.08595275878906, + "KL/rejected_KL_mean": -211.16371154785156, + "KL/std": 159.60922241210938, + "epoch": 0.16740088105726872, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 90.155517578125, + "fcm_dpo/q_t": 0.47918766736984253, + "grad_norm": 4.788021087646484, + "learning_rate": 4.936501251103751e-07, + "logits/chosen": -0.1772385984659195, + "logits/rejected": -0.18084228038787842, + "logps/chosen": -178.79327392578125, + "logps/ref_chosen": -57.78507995605469, + "logps/ref_rejected": -87.10966491699219, + "logps/rejected": -298.27337646484375, + "loss": 1.3122, + "margin_dpo/margin_mean": 90.155517578125, + "margin_dpo/margin_std": 198.69287109375, + "step": 114 + }, + { + "KL/chosen_KL_mean": -145.8248748779297, + "KL/mean": -179.474365234375, + "KL/rejected_KL_mean": -213.12384033203125, + "KL/std": 172.7808837890625, + "epoch": 0.16886930983847284, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 67.29896545410156, + "fcm_dpo/q_t": 0.48439520597457886, + "grad_norm": 4.369140148162842, + "learning_rate": 4.933595135901732e-07, + "logits/chosen": -0.23590882122516632, + "logits/rejected": -0.24995499849319458, + "logps/chosen": -211.40750122070312, + "logps/ref_chosen": -65.5826416015625, + "logps/ref_rejected": -98.56552124023438, + "logps/rejected": -311.6893615722656, + "loss": 1.3361, + "margin_dpo/margin_mean": 67.29896545410156, + "margin_dpo/margin_std": 233.44024658203125, + "step": 115 + }, + { + "KL/chosen_KL_mean": -99.08972930908203, + "KL/mean": -135.49847412109375, + "KL/rejected_KL_mean": -171.9072265625, + "KL/std": 123.32942199707031, + "epoch": 0.17033773861967694, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 72.8175048828125, + "fcm_dpo/q_t": 0.48297327756881714, + "grad_norm": 5.752792835235596, + "learning_rate": 4.930624893204624e-07, + "logits/chosen": -0.23723204433918, + "logits/rejected": -0.25562626123428345, + "logps/chosen": -150.49005126953125, + "logps/ref_chosen": -51.40031433105469, + "logps/ref_rejected": -80.5218505859375, + "logps/rejected": -252.4290771484375, + "loss": 1.3235, + "margin_dpo/margin_mean": 72.81749725341797, + "margin_dpo/margin_std": 141.2083282470703, + "step": 116 + }, + { + "KL/chosen_KL_mean": -142.8017120361328, + "KL/mean": -156.0880126953125, + "KL/rejected_KL_mean": -169.37429809570312, + "KL/std": 135.4499053955078, + "epoch": 0.17180616740088106, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 26.572589874267578, + "fcm_dpo/q_t": 0.49375978112220764, + "grad_norm": 4.810681343078613, + "learning_rate": 4.927590601281083e-07, + "logits/chosen": -0.19404463469982147, + "logits/rejected": -0.19508536159992218, + "logps/chosen": -212.10012817382812, + "logps/ref_chosen": -69.29840850830078, + "logps/ref_rejected": -66.583984375, + "logps/rejected": -235.95828247070312, + "loss": 1.3678, + "margin_dpo/margin_mean": 26.572589874267578, + "margin_dpo/margin_std": 167.6946258544922, + "step": 117 + }, + { + "KL/chosen_KL_mean": -102.44888305664062, + "KL/mean": -137.05758666992188, + "KL/rejected_KL_mean": -171.66629028320312, + "KL/std": 128.6864776611328, + "epoch": 0.17327459618208516, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 69.2174072265625, + "fcm_dpo/q_t": 0.48380762338638306, + "grad_norm": 5.621657371520996, + "learning_rate": 4.924492340087524e-07, + "logits/chosen": -0.3070230484008789, + "logits/rejected": -0.3028327226638794, + "logps/chosen": -158.08985900878906, + "logps/ref_chosen": -55.6409797668457, + "logps/ref_rejected": -75.66905975341797, + "logps/rejected": -247.33534240722656, + "loss": 1.327, + "margin_dpo/margin_mean": 69.2174072265625, + "margin_dpo/margin_std": 148.856201171875, + "step": 118 + }, + { + "KL/chosen_KL_mean": -149.1088104248047, + "KL/mean": -178.7309112548828, + "KL/rejected_KL_mean": -208.35299682617188, + "KL/std": 138.42144775390625, + "epoch": 0.17474302496328928, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 59.24419403076172, + "fcm_dpo/q_t": 0.4861356019973755, + "grad_norm": 4.433512210845947, + "learning_rate": 4.92133019126601e-07, + "logits/chosen": -0.297086238861084, + "logits/rejected": -0.3139578104019165, + "logps/chosen": -222.61900329589844, + "logps/ref_chosen": -73.51019287109375, + "logps/ref_rejected": -102.977294921875, + "logps/rejected": -311.3302917480469, + "loss": 1.3383, + "margin_dpo/margin_mean": 59.24420166015625, + "margin_dpo/margin_std": 175.93429565429688, + "step": 119 + }, + { + "KL/chosen_KL_mean": -147.74859619140625, + "KL/mean": -195.46881103515625, + "KL/rejected_KL_mean": -243.18905639648438, + "KL/std": 166.3782958984375, + "epoch": 0.1762114537444934, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 95.4404525756836, + "fcm_dpo/q_t": 0.4778479337692261, + "grad_norm": 8.700891494750977, + "learning_rate": 4.918104238142103e-07, + "logits/chosen": -0.293715238571167, + "logits/rejected": -0.30091625452041626, + "logps/chosen": -224.5294189453125, + "logps/ref_chosen": -76.78083801269531, + "logps/ref_rejected": -108.02374267578125, + "logps/rejected": -351.2127685546875, + "loss": 1.3067, + "margin_dpo/margin_mean": 95.4404525756836, + "margin_dpo/margin_std": 195.20413208007812, + "step": 120 + }, + { + "KL/chosen_KL_mean": -136.00462341308594, + "KL/mean": -193.11111450195312, + "KL/rejected_KL_mean": -250.2176055908203, + "KL/std": 168.69393920898438, + "epoch": 0.1776798825256975, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 114.21296691894531, + "fcm_dpo/q_t": 0.473272442817688, + "grad_norm": 6.882490634918213, + "learning_rate": 4.91481456572267e-07, + "logits/chosen": -0.2620672583580017, + "logits/rejected": -0.2657386064529419, + "logps/chosen": -197.79452514648438, + "logps/ref_chosen": -61.789894104003906, + "logps/ref_rejected": -109.99456787109375, + "logps/rejected": -360.212158203125, + "loss": 1.29, + "margin_dpo/margin_mean": 114.21297454833984, + "margin_dpo/margin_std": 193.61260986328125, + "step": 121 + }, + { + "KL/chosen_KL_mean": -107.97517395019531, + "KL/mean": -183.01693725585938, + "KL/rejected_KL_mean": -258.0587158203125, + "KL/std": 169.63442993164062, + "epoch": 0.17914831130690162, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 150.0835418701172, + "fcm_dpo/q_t": 0.46513625979423523, + "grad_norm": 10.18962287902832, + "learning_rate": 4.911461260693638e-07, + "logits/chosen": -0.19436302781105042, + "logits/rejected": -0.20805394649505615, + "logps/chosen": -154.87738037109375, + "logps/ref_chosen": -46.9022102355957, + "logps/ref_rejected": -106.71418762207031, + "logps/rejected": -364.7729187011719, + "loss": 1.2581, + "margin_dpo/margin_mean": 150.0835418701172, + "margin_dpo/margin_std": 193.8829345703125, + "step": 122 + }, + { + "KL/chosen_KL_mean": -145.32444763183594, + "KL/mean": -189.00643920898438, + "KL/rejected_KL_mean": -232.6884307861328, + "KL/std": 162.26889038085938, + "epoch": 0.18061674008810572, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 87.36398315429688, + "fcm_dpo/q_t": 0.47969871759414673, + "grad_norm": 4.467272758483887, + "learning_rate": 4.908044411417711e-07, + "logits/chosen": -0.15873804688453674, + "logits/rejected": -0.1691892147064209, + "logps/chosen": -206.6630859375, + "logps/ref_chosen": -61.33863830566406, + "logps/ref_rejected": -87.775390625, + "logps/rejected": -320.46380615234375, + "loss": 1.3167, + "margin_dpo/margin_mean": 87.36398315429688, + "margin_dpo/margin_std": 223.19760131835938, + "step": 123 + }, + { + "KL/chosen_KL_mean": -179.08816528320312, + "KL/mean": -245.78233337402344, + "KL/rejected_KL_mean": -312.47650146484375, + "KL/std": 236.51095581054688, + "epoch": 0.18208516886930984, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 133.3883056640625, + "fcm_dpo/q_t": 0.4694249927997589, + "grad_norm": 5.300126552581787, + "learning_rate": 4.904564107932048e-07, + "logits/chosen": -0.1333819478750229, + "logits/rejected": -0.14498867094516754, + "logps/chosen": -250.5364990234375, + "logps/ref_chosen": -71.44833374023438, + "logps/ref_rejected": -117.58056640625, + "logps/rejected": -430.05706787109375, + "loss": 1.2852, + "margin_dpo/margin_mean": 133.38832092285156, + "margin_dpo/margin_std": 305.14990234375, + "step": 124 + }, + { + "KL/chosen_KL_mean": -135.3007049560547, + "KL/mean": -188.19273376464844, + "KL/rejected_KL_mean": -241.08474731445312, + "KL/std": 185.8060302734375, + "epoch": 0.18355359765051396, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 105.78404998779297, + "fcm_dpo/q_t": 0.47543567419052124, + "grad_norm": 5.876065254211426, + "learning_rate": 4.90102044194588e-07, + "logits/chosen": -0.11746616661548615, + "logits/rejected": -0.1338275969028473, + "logps/chosen": -185.43765258789062, + "logps/ref_chosen": -50.136940002441406, + "logps/ref_rejected": -83.98861694335938, + "logps/rejected": -325.0733642578125, + "loss": 1.3011, + "margin_dpo/margin_mean": 105.7840576171875, + "margin_dpo/margin_std": 230.14541625976562, + "step": 125 + }, + { + "KL/chosen_KL_mean": -139.29977416992188, + "KL/mean": -200.52383422851562, + "KL/rejected_KL_mean": -261.7478942871094, + "KL/std": 185.92526245117188, + "epoch": 0.18502202643171806, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 122.4481201171875, + "fcm_dpo/q_t": 0.47186100482940674, + "grad_norm": 4.977867126464844, + "learning_rate": 4.897413506838102e-07, + "logits/chosen": -0.1627553403377533, + "logits/rejected": -0.17375735938549042, + "logps/chosen": -194.96685791015625, + "logps/ref_chosen": -55.66706848144531, + "logps/ref_rejected": -98.1297607421875, + "logps/rejected": -359.8776550292969, + "loss": 1.2868, + "margin_dpo/margin_mean": 122.4481201171875, + "margin_dpo/margin_std": 236.15780639648438, + "step": 126 + }, + { + "KL/chosen_KL_mean": -138.76910400390625, + "KL/mean": -169.734130859375, + "KL/rejected_KL_mean": -200.6991729736328, + "KL/std": 156.78909301757812, + "epoch": 0.18649045521292218, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.930057525634766, + "fcm_dpo/q_t": 0.4853692650794983, + "grad_norm": 5.053523540496826, + "learning_rate": 4.89374339765481e-07, + "logits/chosen": -0.20304161310195923, + "logits/rejected": -0.21361683309078217, + "logps/chosen": -195.3237762451172, + "logps/ref_chosen": -56.55467987060547, + "logps/ref_rejected": -76.7957763671875, + "logps/rejected": -277.49493408203125, + "loss": 1.3373, + "margin_dpo/margin_mean": 61.9300537109375, + "margin_dpo/margin_std": 196.28656005859375, + "step": 127 + }, + { + "KL/chosen_KL_mean": -137.2354736328125, + "KL/mean": -177.0436553955078, + "KL/rejected_KL_mean": -216.85183715820312, + "KL/std": 186.32858276367188, + "epoch": 0.18795888399412627, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 79.61640167236328, + "fcm_dpo/q_t": 0.4815317690372467, + "grad_norm": 4.554657936096191, + "learning_rate": 4.890010211106795e-07, + "logits/chosen": -0.18499845266342163, + "logits/rejected": -0.2032622992992401, + "logps/chosen": -195.35641479492188, + "logps/ref_chosen": -58.12095642089844, + "logps/ref_rejected": -76.43896484375, + "logps/rejected": -293.2908020019531, + "loss": 1.3258, + "margin_dpo/margin_mean": 79.61640167236328, + "margin_dpo/margin_std": 241.3561248779297, + "step": 128 + }, + { + "KL/chosen_KL_mean": -175.23907470703125, + "KL/mean": -220.06051635742188, + "KL/rejected_KL_mean": -264.8819580078125, + "KL/std": 237.86366271972656, + "epoch": 0.1894273127753304, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.64288330078125, + "fcm_dpo/q_t": 0.4801744818687439, + "grad_norm": 4.864995002746582, + "learning_rate": 4.88621404556699e-07, + "logits/chosen": -0.187529057264328, + "logits/rejected": -0.1925836205482483, + "logps/chosen": -242.1554412841797, + "logps/ref_chosen": -66.91637420654297, + "logps/ref_rejected": -96.6422119140625, + "logps/rejected": -361.524169921875, + "loss": 1.3266, + "margin_dpo/margin_mean": 89.64288330078125, + "margin_dpo/margin_std": 326.5260009765625, + "step": 129 + }, + { + "KL/chosen_KL_mean": -115.89335632324219, + "KL/mean": -175.7540283203125, + "KL/rejected_KL_mean": -235.61468505859375, + "KL/std": 157.26568603515625, + "epoch": 0.19089574155653452, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 119.72134399414062, + "fcm_dpo/q_t": 0.47216594219207764, + "grad_norm": 4.814020156860352, + "learning_rate": 4.882355001067891e-07, + "logits/chosen": -0.17016103863716125, + "logits/rejected": -0.17204934358596802, + "logps/chosen": -160.56021118164062, + "logps/ref_chosen": -44.66685104370117, + "logps/ref_rejected": -82.78165435791016, + "logps/rejected": -318.3963623046875, + "loss": 1.2868, + "margin_dpo/margin_mean": 119.72134399414062, + "margin_dpo/margin_std": 210.9378204345703, + "step": 130 + }, + { + "KL/chosen_KL_mean": -119.94183349609375, + "KL/mean": -184.79937744140625, + "KL/rejected_KL_mean": -249.65692138671875, + "KL/std": 192.86123657226562, + "epoch": 0.19236417033773862, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 129.715087890625, + "fcm_dpo/q_t": 0.4696594178676605, + "grad_norm": 7.062121868133545, + "learning_rate": 4.878433179298909e-07, + "logits/chosen": -0.15699008107185364, + "logits/rejected": -0.16018380224704742, + "logps/chosen": -164.86642456054688, + "logps/ref_chosen": -44.924591064453125, + "logps/ref_rejected": -88.44401550292969, + "logps/rejected": -338.1009521484375, + "loss": 1.2773, + "margin_dpo/margin_mean": 129.715087890625, + "margin_dpo/margin_std": 207.52944946289062, + "step": 131 + }, + { + "KL/chosen_KL_mean": -152.28724670410156, + "KL/mean": -203.29061889648438, + "KL/rejected_KL_mean": -254.29400634765625, + "KL/std": 203.4266357421875, + "epoch": 0.19383259911894274, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 102.00675964355469, + "fcm_dpo/q_t": 0.47675275802612305, + "grad_norm": 6.158812522888184, + "learning_rate": 4.874448683603694e-07, + "logits/chosen": -0.16013693809509277, + "logits/rejected": -0.18753768503665924, + "logps/chosen": -211.288330078125, + "logps/ref_chosen": -59.00108337402344, + "logps/ref_rejected": -87.89215087890625, + "logps/rejected": -342.1861572265625, + "loss": 1.3077, + "margin_dpo/margin_mean": 102.00675964355469, + "margin_dpo/margin_std": 263.32513427734375, + "step": 132 + }, + { + "KL/chosen_KL_mean": -176.15794372558594, + "KL/mean": -220.2735137939453, + "KL/rejected_KL_mean": -264.3890686035156, + "KL/std": 171.11268615722656, + "epoch": 0.19530102790014683, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 88.23113250732422, + "fcm_dpo/q_t": 0.47959333658218384, + "grad_norm": 5.555715084075928, + "learning_rate": 4.870401618977415e-07, + "logits/chosen": -0.16391275823116302, + "logits/rejected": -0.17706236243247986, + "logps/chosen": -242.762451171875, + "logps/ref_chosen": -66.60449981689453, + "logps/ref_rejected": -96.33355712890625, + "logps/rejected": -360.7226257324219, + "loss": 1.3163, + "margin_dpo/margin_mean": 88.23113250732422, + "margin_dpo/margin_std": 228.1553955078125, + "step": 133 + }, + { + "KL/chosen_KL_mean": -138.08108520507812, + "KL/mean": -183.7069091796875, + "KL/rejected_KL_mean": -229.33270263671875, + "KL/std": 146.15443420410156, + "epoch": 0.19676945668135096, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.2516098022461, + "fcm_dpo/q_t": 0.4787331819534302, + "grad_norm": 5.492349624633789, + "learning_rate": 4.866292092063986e-07, + "logits/chosen": -0.14529156684875488, + "logits/rejected": -0.16231930255889893, + "logps/chosen": -190.15036010742188, + "logps/ref_chosen": -52.06925582885742, + "logps/ref_rejected": -87.6545181274414, + "logps/rejected": -316.9872131347656, + "loss": 1.3092, + "margin_dpo/margin_mean": 91.2516098022461, + "margin_dpo/margin_std": 175.44699096679688, + "step": 134 + }, + { + "KL/chosen_KL_mean": -141.79519653320312, + "KL/mean": -230.31271362304688, + "KL/rejected_KL_mean": -318.8302001953125, + "KL/std": 236.4542694091797, + "epoch": 0.19823788546255505, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 177.03501892089844, + "fcm_dpo/q_t": 0.45972955226898193, + "grad_norm": 10.549337387084961, + "learning_rate": 4.862120211153265e-07, + "logits/chosen": -0.12863895297050476, + "logits/rejected": -0.1378084421157837, + "logps/chosen": -192.14907836914062, + "logps/ref_chosen": -50.353858947753906, + "logps/ref_rejected": -115.97975158691406, + "logps/rejected": -434.8099670410156, + "loss": 1.2474, + "margin_dpo/margin_mean": 177.03500366210938, + "margin_dpo/margin_std": 301.78912353515625, + "step": 135 + }, + { + "KL/chosen_KL_mean": -178.12527465820312, + "KL/mean": -235.43304443359375, + "KL/rejected_KL_mean": -292.7408142089844, + "KL/std": 245.4122772216797, + "epoch": 0.19970631424375918, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 114.61553192138672, + "fcm_dpo/q_t": 0.47485947608947754, + "grad_norm": 5.174078941345215, + "learning_rate": 4.857886086178193e-07, + "logits/chosen": -0.10895825922489166, + "logits/rejected": -0.12058396637439728, + "logps/chosen": -243.1977996826172, + "logps/ref_chosen": -65.072509765625, + "logps/ref_rejected": -96.32122802734375, + "logps/rejected": -389.06207275390625, + "loss": 1.3052, + "margin_dpo/margin_mean": 114.61553192138672, + "margin_dpo/margin_std": 335.3597106933594, + "step": 136 + }, + { + "KL/chosen_KL_mean": -150.92208862304688, + "KL/mean": -239.39349365234375, + "KL/rejected_KL_mean": -327.86492919921875, + "KL/std": 264.5162353515625, + "epoch": 0.2011747430249633, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 176.94281005859375, + "fcm_dpo/q_t": 0.46023792028427124, + "grad_norm": 6.715754508972168, + "learning_rate": 4.853589828711902e-07, + "logits/chosen": -0.08718404918909073, + "logits/rejected": -0.09329766035079956, + "logps/chosen": -199.68121337890625, + "logps/ref_chosen": -48.759117126464844, + "logps/ref_rejected": -113.86376953125, + "logps/rejected": -441.72869873046875, + "loss": 1.2512, + "margin_dpo/margin_mean": 176.94281005859375, + "margin_dpo/margin_std": 338.74969482421875, + "step": 137 + }, + { + "KL/chosen_KL_mean": -174.56582641601562, + "KL/mean": -233.0918426513672, + "KL/rejected_KL_mean": -291.61785888671875, + "KL/std": 202.8188934326172, + "epoch": 0.2026431718061674, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 117.0520248413086, + "fcm_dpo/q_t": 0.47279053926467896, + "grad_norm": 6.762569904327393, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": -0.09231037646532059, + "logits/rejected": -0.10022014379501343, + "logps/chosen": -235.08547973632812, + "logps/ref_chosen": -60.519649505615234, + "logps/ref_rejected": -93.19694519042969, + "logps/rejected": -384.8148193359375, + "loss": 1.2889, + "margin_dpo/margin_mean": 117.0520248413086, + "margin_dpo/margin_std": 211.6190185546875, + "step": 138 + }, + { + "KL/chosen_KL_mean": -153.19482421875, + "KL/mean": -214.05612182617188, + "KL/rejected_KL_mean": -274.9173889160156, + "KL/std": 174.04698181152344, + "epoch": 0.20411160058737152, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 121.72257232666016, + "fcm_dpo/q_t": 0.4718576669692993, + "grad_norm": 5.579147815704346, + "learning_rate": 4.844811370781446e-07, + "logits/chosen": -0.053176525980234146, + "logits/rejected": -0.06008898839354515, + "logps/chosen": -200.08621215820312, + "logps/ref_chosen": -46.89138412475586, + "logps/ref_rejected": -79.72798156738281, + "logps/rejected": -354.6453857421875, + "loss": 1.2875, + "margin_dpo/margin_mean": 121.72257232666016, + "margin_dpo/margin_std": 241.1826171875, + "step": 139 + }, + { + "KL/chosen_KL_mean": -195.38796997070312, + "KL/mean": -242.62347412109375, + "KL/rejected_KL_mean": -289.8590087890625, + "KL/std": 201.94166564941406, + "epoch": 0.2055800293685756, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.47102355957031, + "fcm_dpo/q_t": 0.47789740562438965, + "grad_norm": 7.205073833465576, + "learning_rate": 4.840329401637809e-07, + "logits/chosen": -0.008332394063472748, + "logits/rejected": -0.023142114281654358, + "logps/chosen": -254.36268615722656, + "logps/ref_chosen": -58.97471618652344, + "logps/ref_rejected": -83.28410339355469, + "logps/rejected": -373.14312744140625, + "loss": 1.3137, + "margin_dpo/margin_mean": 94.47102355957031, + "margin_dpo/margin_std": 255.87286376953125, + "step": 140 + }, + { + "KL/chosen_KL_mean": -251.51254272460938, + "KL/mean": -293.2801513671875, + "KL/rejected_KL_mean": -335.04779052734375, + "KL/std": 324.080078125, + "epoch": 0.20704845814977973, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 83.5352554321289, + "fcm_dpo/q_t": 0.4800110161304474, + "grad_norm": 14.238146781921387, + "learning_rate": 4.83578576263792e-07, + "logits/chosen": -0.020058374851942062, + "logits/rejected": -0.05175706744194031, + "logps/chosen": -326.58819580078125, + "logps/ref_chosen": -75.07566833496094, + "logps/ref_rejected": -98.1922607421875, + "logps/rejected": -433.24005126953125, + "loss": 1.3458, + "margin_dpo/margin_mean": 83.53524780273438, + "margin_dpo/margin_std": 406.5972900390625, + "step": 141 + }, + { + "KL/chosen_KL_mean": -205.25633239746094, + "KL/mean": -271.9860534667969, + "KL/rejected_KL_mean": -338.71575927734375, + "KL/std": 258.38970947265625, + "epoch": 0.20851688693098386, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 133.45944213867188, + "fcm_dpo/q_t": 0.4691423177719116, + "grad_norm": 6.6939826011657715, + "learning_rate": 4.83118057351089e-07, + "logits/chosen": -0.04343932867050171, + "logits/rejected": -0.06279127299785614, + "logps/chosen": -263.28424072265625, + "logps/ref_chosen": -58.027931213378906, + "logps/ref_rejected": -94.58222961425781, + "logps/rejected": -433.2979736328125, + "loss": 1.2852, + "margin_dpo/margin_mean": 133.45944213867188, + "margin_dpo/margin_std": 297.07684326171875, + "step": 142 + }, + { + "KL/chosen_KL_mean": -197.44586181640625, + "KL/mean": -235.8623046875, + "KL/rejected_KL_mean": -274.2787780761719, + "KL/std": 196.98611450195312, + "epoch": 0.20998531571218795, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.83290100097656, + "fcm_dpo/q_t": 0.4824400544166565, + "grad_norm": 6.687804222106934, + "learning_rate": 4.826513955607734e-07, + "logits/chosen": -0.008544637821614742, + "logits/rejected": -0.0213165283203125, + "logps/chosen": -255.0423126220703, + "logps/ref_chosen": -57.59645080566406, + "logps/ref_rejected": -78.99957275390625, + "logps/rejected": -353.2783203125, + "loss": 1.3288, + "margin_dpo/margin_mean": 76.83290100097656, + "margin_dpo/margin_std": 242.87838745117188, + "step": 143 + }, + { + "KL/chosen_KL_mean": -199.11557006835938, + "KL/mean": -241.8614959716797, + "KL/rejected_KL_mean": -284.607421875, + "KL/std": 204.11404418945312, + "epoch": 0.21145374449339208, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 85.49183654785156, + "fcm_dpo/q_t": 0.48005104064941406, + "grad_norm": 6.8599467277526855, + "learning_rate": 4.821786031898176e-07, + "logits/chosen": -0.04758778586983681, + "logits/rejected": -0.08662058413028717, + "logps/chosen": -259.0219421386719, + "logps/ref_chosen": -59.90636444091797, + "logps/ref_rejected": -82.00025939941406, + "logps/rejected": -366.607666015625, + "loss": 1.319, + "margin_dpo/margin_mean": 85.49183654785156, + "margin_dpo/margin_std": 223.2879638671875, + "step": 144 + }, + { + "KL/chosen_KL_mean": -174.992919921875, + "KL/mean": -220.36129760742188, + "KL/rejected_KL_mean": -265.72967529296875, + "KL/std": 170.8227996826172, + "epoch": 0.21292217327459617, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 90.73674774169922, + "fcm_dpo/q_t": 0.47885861992836, + "grad_norm": 8.647239685058594, + "learning_rate": 4.816996926967401e-07, + "logits/chosen": -0.11733024567365646, + "logits/rejected": -0.14884145557880402, + "logps/chosen": -231.59359741210938, + "logps/ref_chosen": -56.60066604614258, + "logps/ref_rejected": -77.86631774902344, + "logps/rejected": -343.59600830078125, + "loss": 1.3128, + "margin_dpo/margin_mean": 90.73674011230469, + "margin_dpo/margin_std": 214.95761108398438, + "step": 145 + }, + { + "KL/chosen_KL_mean": -224.58987426757812, + "KL/mean": -252.029541015625, + "KL/rejected_KL_mean": -279.46923828125, + "KL/std": 185.42648315429688, + "epoch": 0.2143906020558003, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 54.87935256958008, + "fcm_dpo/q_t": 0.48712092638015747, + "grad_norm": 10.154541969299316, + "learning_rate": 4.812146767012779e-07, + "logits/chosen": -0.047742657363414764, + "logits/rejected": -0.07091985642910004, + "logps/chosen": -290.59033203125, + "logps/ref_chosen": -66.00045013427734, + "logps/ref_rejected": -81.70278930664062, + "logps/rejected": -361.1719970703125, + "loss": 1.3475, + "margin_dpo/margin_mean": 54.87934875488281, + "margin_dpo/margin_std": 230.03529357910156, + "step": 146 + }, + { + "KL/chosen_KL_mean": -175.193359375, + "KL/mean": -218.81103515625, + "KL/rejected_KL_mean": -262.4287109375, + "KL/std": 185.93280029296875, + "epoch": 0.21585903083700442, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 87.23533630371094, + "fcm_dpo/q_t": 0.4797496497631073, + "grad_norm": 5.8443684577941895, + "learning_rate": 4.807235679840536e-07, + "logits/chosen": -0.08670485019683838, + "logits/rejected": -0.11605459451675415, + "logps/chosen": -228.59884643554688, + "logps/ref_chosen": -53.405487060546875, + "logps/ref_rejected": -71.39060974121094, + "logps/rejected": -333.8193359375, + "loss": 1.3179, + "margin_dpo/margin_mean": 87.23533630371094, + "margin_dpo/margin_std": 234.81607055664062, + "step": 147 + }, + { + "KL/chosen_KL_mean": -186.26583862304688, + "KL/mean": -237.65237426757812, + "KL/rejected_KL_mean": -289.0389404296875, + "KL/std": 223.15664672851562, + "epoch": 0.2173274596182085, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 102.77307891845703, + "fcm_dpo/q_t": 0.476764976978302, + "grad_norm": 6.361453056335449, + "learning_rate": 4.802263794862384e-07, + "logits/chosen": -0.14481475949287415, + "logits/rejected": -0.15246915817260742, + "logps/chosen": -251.20291137695312, + "logps/ref_chosen": -64.93708038330078, + "logps/ref_rejected": -103.09384155273438, + "logps/rejected": -392.13275146484375, + "loss": 1.3073, + "margin_dpo/margin_mean": 102.77307891845703, + "margin_dpo/margin_std": 248.232666015625, + "step": 148 + }, + { + "KL/chosen_KL_mean": -163.42788696289062, + "KL/mean": -227.01951599121094, + "KL/rejected_KL_mean": -290.61114501953125, + "KL/std": 197.52664184570312, + "epoch": 0.21879588839941264, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 127.1832275390625, + "fcm_dpo/q_t": 0.4706050753593445, + "grad_norm": 9.70356273651123, + "learning_rate": 4.797231243092118e-07, + "logits/chosen": -0.1462225615978241, + "logits/rejected": -0.16033855080604553, + "logps/chosen": -221.90164184570312, + "logps/ref_chosen": -58.47376251220703, + "logps/ref_rejected": -99.31474304199219, + "logps/rejected": -389.9258728027344, + "loss": 1.2842, + "margin_dpo/margin_mean": 127.1832275390625, + "margin_dpo/margin_std": 244.41700744628906, + "step": 149 + }, + { + "KL/chosen_KL_mean": -142.97955322265625, + "KL/mean": -210.34365844726562, + "KL/rejected_KL_mean": -277.707763671875, + "KL/std": 207.92286682128906, + "epoch": 0.22026431718061673, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 134.72821044921875, + "fcm_dpo/q_t": 0.469161719083786, + "grad_norm": 6.090360641479492, + "learning_rate": 4.792138157142157e-07, + "logits/chosen": -0.12846021354198456, + "logits/rejected": -0.1382494419813156, + "logps/chosen": -188.68536376953125, + "logps/ref_chosen": -45.705810546875, + "logps/ref_rejected": -83.34759521484375, + "logps/rejected": -361.05535888671875, + "loss": 1.2781, + "margin_dpo/margin_mean": 134.72821044921875, + "margin_dpo/margin_std": 255.6205596923828, + "step": 150 + }, + { + "KL/chosen_KL_mean": -200.44656372070312, + "KL/mean": -258.9799499511719, + "KL/rejected_KL_mean": -317.5133056640625, + "KL/std": 248.0785369873047, + "epoch": 0.22173274596182085, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 117.06673431396484, + "fcm_dpo/q_t": 0.47344115376472473, + "grad_norm": 6.703395843505859, + "learning_rate": 4.786984671220053e-07, + "logits/chosen": -0.14405345916748047, + "logits/rejected": -0.16287846863269806, + "logps/chosen": -271.01739501953125, + "logps/ref_chosen": -70.57083129882812, + "logps/ref_rejected": -100.46382141113281, + "logps/rejected": -417.97711181640625, + "loss": 1.2977, + "margin_dpo/margin_mean": 117.06673431396484, + "margin_dpo/margin_std": 292.07391357421875, + "step": 151 + }, + { + "KL/chosen_KL_mean": -173.80703735351562, + "KL/mean": -253.65969848632812, + "KL/rejected_KL_mean": -333.5123596191406, + "KL/std": 224.35980224609375, + "epoch": 0.22320117474302498, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 159.705322265625, + "fcm_dpo/q_t": 0.4632638990879059, + "grad_norm": 9.056612968444824, + "learning_rate": 4.78177092112495e-07, + "logits/chosen": -0.09410399198532104, + "logits/rejected": -0.10684510320425034, + "logps/chosen": -233.97142028808594, + "logps/ref_chosen": -60.16438674926758, + "logps/ref_rejected": -106.14045715332031, + "logps/rejected": -439.65283203125, + "loss": 1.2584, + "margin_dpo/margin_mean": 159.705322265625, + "margin_dpo/margin_std": 275.8470458984375, + "step": 152 + }, + { + "KL/chosen_KL_mean": -167.72113037109375, + "KL/mean": -232.05372619628906, + "KL/rejected_KL_mean": -296.38629150390625, + "KL/std": 238.9656219482422, + "epoch": 0.22466960352422907, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 128.6651611328125, + "fcm_dpo/q_t": 0.47082751989364624, + "grad_norm": 7.3872175216674805, + "learning_rate": 4.776497044244016e-07, + "logits/chosen": -0.09062840044498444, + "logits/rejected": -0.11342403292655945, + "logps/chosen": -224.03640747070312, + "logps/ref_chosen": -56.315277099609375, + "logps/ref_rejected": -85.65583801269531, + "logps/rejected": -382.0421142578125, + "loss": 1.29, + "margin_dpo/margin_mean": 128.6651611328125, + "margin_dpo/margin_std": 311.77520751953125, + "step": 153 + }, + { + "KL/chosen_KL_mean": -209.06495666503906, + "KL/mean": -261.43896484375, + "KL/rejected_KL_mean": -313.81298828125, + "KL/std": 230.873291015625, + "epoch": 0.2261380323054332, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.7480239868164, + "fcm_dpo/q_t": 0.47572967410087585, + "grad_norm": 6.2794013023376465, + "learning_rate": 4.771163179548808e-07, + "logits/chosen": -0.12751531600952148, + "logits/rejected": -0.14344094693660736, + "logps/chosen": -271.8075256347656, + "logps/ref_chosen": -62.74256896972656, + "logps/ref_rejected": -104.24420166015625, + "logps/rejected": -418.05718994140625, + "loss": 1.3126, + "margin_dpo/margin_mean": 104.74801635742188, + "margin_dpo/margin_std": 321.2839660644531, + "step": 154 + }, + { + "KL/chosen_KL_mean": -190.59161376953125, + "KL/mean": -236.80764770507812, + "KL/rejected_KL_mean": -283.023681640625, + "KL/std": 196.6082763671875, + "epoch": 0.2276064610866373, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 92.43206787109375, + "fcm_dpo/q_t": 0.4784696102142334, + "grad_norm": 6.7807159423828125, + "learning_rate": 4.7657694675916247e-07, + "logits/chosen": -0.12184707820415497, + "logits/rejected": -0.1378769725561142, + "logps/chosen": -251.2447967529297, + "logps/ref_chosen": -60.65318298339844, + "logps/ref_rejected": -77.49220275878906, + "logps/rejected": -360.515869140625, + "loss": 1.3159, + "margin_dpo/margin_mean": 92.43206787109375, + "margin_dpo/margin_std": 260.202880859375, + "step": 155 + }, + { + "KL/chosen_KL_mean": -253.88046264648438, + "KL/mean": -270.5169677734375, + "KL/rejected_KL_mean": -287.15350341796875, + "KL/std": 232.72618103027344, + "epoch": 0.2290748898678414, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 33.27302551269531, + "fcm_dpo/q_t": 0.49217426776885986, + "grad_norm": 8.957015037536621, + "learning_rate": 4.7603160505017893e-07, + "logits/chosen": -0.08439283818006516, + "logits/rejected": -0.12552709877490997, + "logps/chosen": -323.3723449707031, + "logps/ref_chosen": -69.49188232421875, + "logps/ref_rejected": -77.16929626464844, + "logps/rejected": -364.3227844238281, + "loss": 1.3753, + "margin_dpo/margin_mean": 33.27302551269531, + "margin_dpo/margin_std": 293.46331787109375, + "step": 156 + }, + { + "KL/chosen_KL_mean": -210.72842407226562, + "KL/mean": -293.7404479980469, + "KL/rejected_KL_mean": -376.75250244140625, + "KL/std": 239.9976043701172, + "epoch": 0.2305433186490455, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 166.02407836914062, + "fcm_dpo/q_t": 0.46170127391815186, + "grad_norm": 8.381214141845703, + "learning_rate": 4.7548030719819154e-07, + "logits/chosen": -0.029853641986846924, + "logits/rejected": -0.06525542587041855, + "logps/chosen": -272.09686279296875, + "logps/ref_chosen": -61.368438720703125, + "logps/ref_rejected": -107.64636993408203, + "logps/rejected": -484.39886474609375, + "loss": 1.2578, + "margin_dpo/margin_mean": 166.02407836914062, + "margin_dpo/margin_std": 295.1898498535156, + "step": 157 + }, + { + "KL/chosen_KL_mean": -221.72073364257812, + "KL/mean": -324.26605224609375, + "KL/rejected_KL_mean": -426.8114013671875, + "KL/std": 309.63800048828125, + "epoch": 0.23201174743024963, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 205.0906524658203, + "fcm_dpo/q_t": 0.4540192782878876, + "grad_norm": 9.694887161254883, + "learning_rate": 4.7492306773041136e-07, + "logits/chosen": -0.0386735275387764, + "logits/rejected": -0.06256888806819916, + "logps/chosen": -279.3336486816406, + "logps/ref_chosen": -57.612918853759766, + "logps/ref_rejected": -113.6946792602539, + "logps/rejected": -540.506103515625, + "loss": 1.2334, + "margin_dpo/margin_mean": 205.09066772460938, + "margin_dpo/margin_std": 380.8212890625, + "step": 158 + }, + { + "KL/chosen_KL_mean": -299.9361267089844, + "KL/mean": -334.1776123046875, + "KL/rejected_KL_mean": -368.41912841796875, + "KL/std": 297.918212890625, + "epoch": 0.23348017621145375, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 68.48296356201172, + "fcm_dpo/q_t": 0.48391008377075195, + "grad_norm": 11.454606056213379, + "learning_rate": 4.743599013306165e-07, + "logits/chosen": 0.013804029673337936, + "logits/rejected": -0.0027273818850517273, + "logps/chosen": -381.4964599609375, + "logps/ref_chosen": -81.56034851074219, + "logps/ref_rejected": -88.89871215820312, + "logps/rejected": -457.31781005859375, + "loss": 1.3469, + "margin_dpo/margin_mean": 68.48297119140625, + "margin_dpo/margin_std": 327.9689636230469, + "step": 159 + }, + { + "KL/chosen_KL_mean": -264.8038330078125, + "KL/mean": -340.09210205078125, + "KL/rejected_KL_mean": -415.38037109375, + "KL/std": 267.2978515625, + "epoch": 0.23494860499265785, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 150.57656860351562, + "fcm_dpo/q_t": 0.4657573103904724, + "grad_norm": 8.678204536437988, + "learning_rate": 4.737908228387656e-07, + "logits/chosen": -0.02935023419559002, + "logits/rejected": -0.05578979104757309, + "logps/chosen": -330.53472900390625, + "logps/ref_chosen": -65.73088073730469, + "logps/ref_rejected": -97.21781921386719, + "logps/rejected": -512.5982055664062, + "loss": 1.275, + "margin_dpo/margin_mean": 150.57656860351562, + "margin_dpo/margin_std": 334.5924072265625, + "step": 160 + }, + { + "KL/chosen_KL_mean": -215.1328125, + "KL/mean": -277.35003662109375, + "KL/rejected_KL_mean": -339.5672607421875, + "KL/std": 214.84197998046875, + "epoch": 0.23641703377386197, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 124.43446350097656, + "fcm_dpo/q_t": 0.47127845883369446, + "grad_norm": 8.156302452087402, + "learning_rate": 4.7321584725060594e-07, + "logits/chosen": -0.02828654646873474, + "logits/rejected": -0.06187158077955246, + "logps/chosen": -267.56927490234375, + "logps/ref_chosen": -52.43647003173828, + "logps/ref_rejected": -83.43095397949219, + "logps/rejected": -422.99822998046875, + "loss": 1.2874, + "margin_dpo/margin_mean": 124.43446350097656, + "margin_dpo/margin_std": 254.58370971679688, + "step": 161 + }, + { + "KL/chosen_KL_mean": -217.1569366455078, + "KL/mean": -270.75323486328125, + "KL/rejected_KL_mean": -324.34954833984375, + "KL/std": 228.0889129638672, + "epoch": 0.23788546255506607, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 107.19261169433594, + "fcm_dpo/q_t": 0.47551965713500977, + "grad_norm": 9.41181755065918, + "learning_rate": 4.7263498971727905e-07, + "logits/chosen": -0.13572916388511658, + "logits/rejected": -0.15741822123527527, + "logps/chosen": -279.76751708984375, + "logps/ref_chosen": -62.6105842590332, + "logps/ref_rejected": -89.39057922363281, + "logps/rejected": -413.7401428222656, + "loss": 1.3073, + "margin_dpo/margin_mean": 107.1926040649414, + "margin_dpo/margin_std": 286.09466552734375, + "step": 162 + }, + { + "KL/chosen_KL_mean": -229.8784637451172, + "KL/mean": -275.2326354980469, + "KL/rejected_KL_mean": -320.5867919921875, + "KL/std": 224.8272705078125, + "epoch": 0.2393538913362702, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 90.70832824707031, + "fcm_dpo/q_t": 0.47908735275268555, + "grad_norm": 7.177928447723389, + "learning_rate": 4.720482655449212e-07, + "logits/chosen": -0.07068107277154922, + "logits/rejected": -0.08296166360378265, + "logps/chosen": -284.90008544921875, + "logps/ref_chosen": -55.021629333496094, + "logps/ref_rejected": -75.418212890625, + "logps/rejected": -396.0050048828125, + "loss": 1.3193, + "margin_dpo/margin_mean": 90.70832824707031, + "margin_dpo/margin_std": 275.54254150390625, + "step": 163 + }, + { + "KL/chosen_KL_mean": -202.0023956298828, + "KL/mean": -269.85015869140625, + "KL/rejected_KL_mean": -337.6978759765625, + "KL/std": 206.05075073242188, + "epoch": 0.24082232011747431, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 135.6955108642578, + "fcm_dpo/q_t": 0.4684979319572449, + "grad_norm": 8.586596488952637, + "learning_rate": 4.714556901942599e-07, + "logits/chosen": -0.0649288222193718, + "logits/rejected": -0.08924803882837296, + "logps/chosen": -257.64306640625, + "logps/ref_chosen": -55.64066696166992, + "logps/ref_rejected": -79.66463470458984, + "logps/rejected": -417.362548828125, + "loss": 1.2751, + "margin_dpo/margin_mean": 135.6955108642578, + "margin_dpo/margin_std": 236.37542724609375, + "step": 164 + }, + { + "KL/chosen_KL_mean": -217.8095703125, + "KL/mean": -258.1341552734375, + "KL/rejected_KL_mean": -298.45867919921875, + "KL/std": 203.58636474609375, + "epoch": 0.2422907488986784, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.64910888671875, + "fcm_dpo/q_t": 0.4814724326133728, + "grad_norm": 6.596712112426758, + "learning_rate": 4.708572792802069e-07, + "logits/chosen": -0.09720340371131897, + "logits/rejected": -0.09911538660526276, + "logps/chosen": -279.1202697753906, + "logps/ref_chosen": -61.310691833496094, + "logps/ref_rejected": -73.67060852050781, + "logps/rejected": -372.1292724609375, + "loss": 1.3307, + "margin_dpo/margin_mean": 80.64909362792969, + "margin_dpo/margin_std": 288.8643798828125, + "step": 165 + }, + { + "KL/chosen_KL_mean": -188.13812255859375, + "KL/mean": -287.0330505371094, + "KL/rejected_KL_mean": -385.927978515625, + "KL/std": 297.14111328125, + "epoch": 0.24375917767988253, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 197.7898406982422, + "fcm_dpo/q_t": 0.45629557967185974, + "grad_norm": 8.050973892211914, + "learning_rate": 4.702530485714461e-07, + "logits/chosen": -0.1186860129237175, + "logits/rejected": -0.127618670463562, + "logps/chosen": -239.12173461914062, + "logps/ref_chosen": -50.98360061645508, + "logps/ref_rejected": -98.09512329101562, + "logps/rejected": -484.0231018066406, + "loss": 1.2411, + "margin_dpo/margin_mean": 197.78982543945312, + "margin_dpo/margin_std": 393.486572265625, + "step": 166 + }, + { + "KL/chosen_KL_mean": -192.49603271484375, + "KL/mean": -283.8441162109375, + "KL/rejected_KL_mean": -375.19219970703125, + "KL/std": 246.08155822753906, + "epoch": 0.24522760646108663, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 182.6961669921875, + "fcm_dpo/q_t": 0.4583081901073456, + "grad_norm": 11.262971878051758, + "learning_rate": 4.6964301399001877e-07, + "logits/chosen": -0.13433954119682312, + "logits/rejected": -0.15051549673080444, + "logps/chosen": -242.92013549804688, + "logps/ref_chosen": -50.424095153808594, + "logps/ref_rejected": -96.03042602539062, + "logps/rejected": -471.22259521484375, + "loss": 1.2432, + "margin_dpo/margin_mean": 182.69615173339844, + "margin_dpo/margin_std": 304.00164794921875, + "step": 167 + }, + { + "KL/chosen_KL_mean": -209.07247924804688, + "KL/mean": -267.29833984375, + "KL/rejected_KL_mean": -325.524169921875, + "KL/std": 204.67034912109375, + "epoch": 0.24669603524229075, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 116.45173645019531, + "fcm_dpo/q_t": 0.4730868935585022, + "grad_norm": 8.38344669342041, + "learning_rate": 4.690271916109034e-07, + "logits/chosen": -0.07969611883163452, + "logits/rejected": -0.08738286793231964, + "logps/chosen": -258.5353088378906, + "logps/ref_chosen": -49.462825775146484, + "logps/ref_rejected": -75.30855560302734, + "logps/rejected": -400.832763671875, + "loss": 1.2933, + "margin_dpo/margin_mean": 116.45173645019531, + "margin_dpo/margin_std": 250.1861572265625, + "step": 168 + }, + { + "KL/chosen_KL_mean": -235.8634033203125, + "KL/mean": -288.0467834472656, + "KL/rejected_KL_mean": -340.2301330566406, + "KL/std": 220.77529907226562, + "epoch": 0.24816446402349487, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.36674499511719, + "fcm_dpo/q_t": 0.4762327969074249, + "grad_norm": 8.089078903198242, + "learning_rate": 4.6840559766159235e-07, + "logits/chosen": -0.09106568992137909, + "logits/rejected": -0.091008760035038, + "logps/chosen": -295.6668395996094, + "logps/ref_chosen": -59.803443908691406, + "logps/ref_rejected": -83.34574890136719, + "logps/rejected": -423.5758972167969, + "loss": 1.3126, + "margin_dpo/margin_mean": 104.36674499511719, + "margin_dpo/margin_std": 308.4825439453125, + "step": 169 + }, + { + "KL/chosen_KL_mean": -209.93124389648438, + "KL/mean": -269.19219970703125, + "KL/rejected_KL_mean": -328.453125, + "KL/std": 228.42990112304688, + "epoch": 0.24963289280469897, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 118.52188110351562, + "fcm_dpo/q_t": 0.4724269509315491, + "grad_norm": 9.027761459350586, + "learning_rate": 4.6777824852166437e-07, + "logits/chosen": -0.14047937095165253, + "logits/rejected": -0.1535281240940094, + "logps/chosen": -259.40301513671875, + "logps/ref_chosen": -49.471771240234375, + "logps/ref_rejected": -75.91734313964844, + "logps/rejected": -404.3704833984375, + "loss": 1.2953, + "margin_dpo/margin_mean": 118.52188110351562, + "margin_dpo/margin_std": 277.45550537109375, + "step": 170 + }, + { + "KL/chosen_KL_mean": -336.32196044921875, + "KL/mean": -386.96771240234375, + "KL/rejected_KL_mean": -437.61346435546875, + "KL/std": 330.1646728515625, + "epoch": 0.2511013215859031, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 101.29150390625, + "fcm_dpo/q_t": 0.47753575444221497, + "grad_norm": 14.035201072692871, + "learning_rate": 4.6714516072235273e-07, + "logits/chosen": -0.09937100857496262, + "logits/rejected": -0.11626763641834259, + "logps/chosen": -420.8212890625, + "logps/ref_chosen": -84.49931335449219, + "logps/ref_rejected": -109.38209533691406, + "logps/rejected": -546.99560546875, + "loss": 1.3297, + "margin_dpo/margin_mean": 101.29150390625, + "margin_dpo/margin_std": 415.2909851074219, + "step": 171 + }, + { + "KL/chosen_KL_mean": -284.5125427246094, + "KL/mean": -329.1851806640625, + "KL/rejected_KL_mean": -373.85784912109375, + "KL/std": 302.93798828125, + "epoch": 0.2525697503671072, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.34530639648438, + "fcm_dpo/q_t": 0.478992223739624, + "grad_norm": 11.372665405273438, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": -0.11791045218706131, + "logits/rejected": -0.13465595245361328, + "logps/chosen": -353.16644287109375, + "logps/ref_chosen": -68.65391540527344, + "logps/ref_rejected": -85.43667602539062, + "logps/rejected": -459.29449462890625, + "loss": 1.3256, + "margin_dpo/margin_mean": 89.34529876708984, + "margin_dpo/margin_std": 311.55474853515625, + "step": 172 + }, + { + "KL/chosen_KL_mean": -248.29794311523438, + "KL/mean": -298.60186767578125, + "KL/rejected_KL_mean": -348.9057922363281, + "KL/std": 261.31658935546875, + "epoch": 0.2540381791483113, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 100.60782623291016, + "fcm_dpo/q_t": 0.4770353138446808, + "grad_norm": 8.505526542663574, + "learning_rate": 4.6586183602616687e-07, + "logits/chosen": -0.14444559812545776, + "logits/rejected": -0.15536616742610931, + "logps/chosen": -311.34881591796875, + "logps/ref_chosen": -63.050880432128906, + "logps/ref_rejected": -78.68392181396484, + "logps/rejected": -427.5897216796875, + "loss": 1.3094, + "margin_dpo/margin_mean": 100.60781860351562, + "margin_dpo/margin_std": 267.5063171386719, + "step": 173 + }, + { + "KL/chosen_KL_mean": -225.7945556640625, + "KL/mean": -311.4512023925781, + "KL/rejected_KL_mean": -397.10784912109375, + "KL/std": 295.2328186035156, + "epoch": 0.2555066079295154, + "fcm_dpo/beta": 0.0009431193466298282, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 171.3133087158203, + "fcm_dpo/q_t": 0.46146026253700256, + "grad_norm": 9.906888008117676, + "learning_rate": 4.652116329460919e-07, + "logits/chosen": -0.1199919730424881, + "logits/rejected": -0.13182440400123596, + "logps/chosen": -279.15753173828125, + "logps/ref_chosen": -53.36296844482422, + "logps/ref_rejected": -101.91120910644531, + "logps/rejected": -499.0190734863281, + "loss": 1.2638, + "margin_dpo/margin_mean": 171.31329345703125, + "margin_dpo/margin_std": 387.0136413574219, + "step": 174 + }, + { + "KL/chosen_KL_mean": -183.84222412109375, + "KL/mean": -282.45843505859375, + "KL/rejected_KL_mean": -381.07464599609375, + "KL/std": 243.96095275878906, + "epoch": 0.25697503671071953, + "fcm_dpo/beta": 0.002304242691025138, + "fcm_dpo/delta": -0.08028450608253479, + "fcm_dpo/margin": 197.232421875, + "fcm_dpo/q_t": 0.40342098474502563, + "grad_norm": 22.565650939941406, + "learning_rate": 4.645557588393406e-07, + "logits/chosen": -0.1468554139137268, + "logits/rejected": -0.1525323987007141, + "logps/chosen": -229.260009765625, + "logps/ref_chosen": -45.417762756347656, + "logps/ref_rejected": -89.50579833984375, + "logps/rejected": -470.5804443359375, + "loss": 1.1443, + "margin_dpo/margin_mean": 197.232421875, + "margin_dpo/margin_std": 280.858642578125, + "step": 175 + }, + { + "KL/chosen_KL_mean": -198.94644165039062, + "KL/mean": -293.33734130859375, + "KL/rejected_KL_mean": -387.728271484375, + "KL/std": 279.7598876953125, + "epoch": 0.25844346549192365, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 188.78184509277344, + "fcm_dpo/q_t": 0.4877324104309082, + "grad_norm": 3.078137159347534, + "learning_rate": 4.638942309888058e-07, + "logits/chosen": -0.15219050645828247, + "logits/rejected": -0.16848905384540558, + "logps/chosen": -249.39926147460938, + "logps/ref_chosen": -50.452842712402344, + "logps/ref_rejected": -95.5589599609375, + "logps/rejected": -483.2872314453125, + "loss": 1.3396, + "margin_dpo/margin_mean": 188.7818603515625, + "margin_dpo/margin_std": 341.77484130859375, + "step": 176 + }, + { + "KL/chosen_KL_mean": -251.27587890625, + "KL/mean": -338.42828369140625, + "KL/rejected_KL_mean": -425.5806884765625, + "KL/std": 328.7501220703125, + "epoch": 0.2599118942731278, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 174.30484008789062, + "fcm_dpo/q_t": 0.4886671304702759, + "grad_norm": 3.2213590145111084, + "learning_rate": 4.6322706682636137e-07, + "logits/chosen": -0.15985408425331116, + "logits/rejected": -0.1774776577949524, + "logps/chosen": -312.4923400878906, + "logps/ref_chosen": -61.216468811035156, + "logps/ref_rejected": -95.89378356933594, + "logps/rejected": -521.4744873046875, + "loss": 1.3435, + "margin_dpo/margin_mean": 174.30482482910156, + "margin_dpo/margin_std": 355.32843017578125, + "step": 177 + }, + { + "KL/chosen_KL_mean": -255.82669067382812, + "KL/mean": -358.01220703125, + "KL/rejected_KL_mean": -460.19775390625, + "KL/std": 332.4327392578125, + "epoch": 0.26138032305433184, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 204.37107849121094, + "fcm_dpo/q_t": 0.4866999387741089, + "grad_norm": 4.488763332366943, + "learning_rate": 4.6255428393240354e-07, + "logits/chosen": -0.07843378186225891, + "logits/rejected": -0.09438927471637726, + "logps/chosen": -314.09149169921875, + "logps/ref_chosen": -58.26478958129883, + "logps/ref_rejected": -105.3653335571289, + "logps/rejected": -565.5631103515625, + "loss": 1.3357, + "margin_dpo/margin_mean": 204.37109375, + "margin_dpo/margin_std": 342.59222412109375, + "step": 178 + }, + { + "KL/chosen_KL_mean": -269.11553955078125, + "KL/mean": -328.9101867675781, + "KL/rejected_KL_mean": -388.704833984375, + "KL/std": 307.3699951171875, + "epoch": 0.26284875183553597, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 119.58927917480469, + "fcm_dpo/q_t": 0.4922333359718323, + "grad_norm": 2.9506919384002686, + "learning_rate": 4.6187590003538724e-07, + "logits/chosen": -0.13306602835655212, + "logits/rejected": -0.15629252791404724, + "logps/chosen": -330.17388916015625, + "logps/ref_chosen": -61.05832290649414, + "logps/ref_rejected": -90.52782440185547, + "logps/rejected": -479.232666015625, + "loss": 1.3581, + "margin_dpo/margin_mean": 119.58926391601562, + "margin_dpo/margin_std": 401.2672119140625, + "step": 179 + }, + { + "KL/chosen_KL_mean": -231.9340362548828, + "KL/mean": -326.68878173828125, + "KL/rejected_KL_mean": -421.44354248046875, + "KL/std": 240.43899536132812, + "epoch": 0.2643171806167401, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 189.5094757080078, + "fcm_dpo/q_t": 0.4876583218574524, + "grad_norm": 2.660843849182129, + "learning_rate": 4.611919330113591e-07, + "logits/chosen": -0.1463892161846161, + "logits/rejected": -0.16165244579315186, + "logps/chosen": -286.2767639160156, + "logps/ref_chosen": -54.34272003173828, + "logps/ref_rejected": -98.21183776855469, + "logps/rejected": -519.6553955078125, + "loss": 1.3391, + "margin_dpo/margin_mean": 189.50949096679688, + "margin_dpo/margin_std": 316.0293273925781, + "step": 180 + }, + { + "KL/chosen_KL_mean": -196.69769287109375, + "KL/mean": -247.92457580566406, + "KL/rejected_KL_mean": -299.15142822265625, + "KL/std": 217.79359436035156, + "epoch": 0.2657856093979442, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 102.4537353515625, + "fcm_dpo/q_t": 0.49332791566848755, + "grad_norm": 2.223689556121826, + "learning_rate": 4.605024008834863e-07, + "logits/chosen": -0.14982840418815613, + "logits/rejected": -0.1486806869506836, + "logps/chosen": -251.69815063476562, + "logps/ref_chosen": -55.000457763671875, + "logps/ref_rejected": -61.656166076660156, + "logps/rejected": -360.8076171875, + "loss": 1.3612, + "margin_dpo/margin_mean": 102.4537353515625, + "margin_dpo/margin_std": 290.69586181640625, + "step": 181 + }, + { + "KL/chosen_KL_mean": -208.18597412109375, + "KL/mean": -319.1571044921875, + "KL/rejected_KL_mean": -430.12823486328125, + "KL/std": 292.2494812011719, + "epoch": 0.26725403817914833, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 221.94229125976562, + "fcm_dpo/q_t": 0.4855644702911377, + "grad_norm": 3.2457919120788574, + "learning_rate": 4.598073218215817e-07, + "logits/chosen": -0.12511727213859558, + "logits/rejected": -0.1399608701467514, + "logps/chosen": -249.2938232421875, + "logps/ref_chosen": -41.107852935791016, + "logps/ref_rejected": -89.5215835571289, + "logps/rejected": -519.6498413085938, + "loss": 1.3315, + "margin_dpo/margin_mean": 221.94229125976562, + "margin_dpo/margin_std": 369.4161376953125, + "step": 182 + }, + { + "KL/chosen_KL_mean": -293.37567138671875, + "KL/mean": -343.2914123535156, + "KL/rejected_KL_mean": -393.2071533203125, + "KL/std": 252.21807861328125, + "epoch": 0.2687224669603524, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 99.83150482177734, + "fcm_dpo/q_t": 0.49350279569625854, + "grad_norm": 2.8942596912384033, + "learning_rate": 4.5910671414162484e-07, + "logits/chosen": -0.12328383326530457, + "logits/rejected": -0.1357804387807846, + "logps/chosen": -350.90020751953125, + "logps/ref_chosen": -57.52456283569336, + "logps/ref_rejected": -75.97572326660156, + "logps/rejected": -469.182861328125, + "loss": 1.3619, + "margin_dpo/margin_mean": 99.83151245117188, + "margin_dpo/margin_std": 281.327880859375, + "step": 183 + }, + { + "KL/chosen_KL_mean": -286.9473571777344, + "KL/mean": -329.740966796875, + "KL/rejected_KL_mean": -372.5345458984375, + "KL/std": 236.4288787841797, + "epoch": 0.2701908957415565, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 85.58721160888672, + "fcm_dpo/q_t": 0.4944261908531189, + "grad_norm": 2.9436886310577393, + "learning_rate": 4.5840059630527985e-07, + "logits/chosen": -0.15956711769104004, + "logits/rejected": -0.16407650709152222, + "logps/chosen": -345.4923095703125, + "logps/ref_chosen": -58.544952392578125, + "logps/ref_rejected": -76.63406372070312, + "logps/rejected": -449.1686096191406, + "loss": 1.3658, + "margin_dpo/margin_mean": 85.58720397949219, + "margin_dpo/margin_std": 308.574462890625, + "step": 184 + }, + { + "KL/chosen_KL_mean": -323.21502685546875, + "KL/mean": -346.2340393066406, + "KL/rejected_KL_mean": -369.2530212402344, + "KL/std": 302.51226806640625, + "epoch": 0.27165932452276065, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 46.0379753112793, + "fcm_dpo/q_t": 0.49701324105262756, + "grad_norm": 4.648723602294922, + "learning_rate": 4.5768898691940836e-07, + "logits/chosen": -0.11234143376350403, + "logits/rejected": -0.1124282032251358, + "logps/chosen": -385.24090576171875, + "logps/ref_chosen": -62.025848388671875, + "logps/ref_rejected": -73.7625961303711, + "logps/rejected": -443.015625, + "loss": 1.3767, + "margin_dpo/margin_mean": 46.03797912597656, + "margin_dpo/margin_std": 361.93035888671875, + "step": 185 + }, + { + "KL/chosen_KL_mean": -321.8814697265625, + "KL/mean": -380.7614440917969, + "KL/rejected_KL_mean": -439.64141845703125, + "KL/std": 288.25714111328125, + "epoch": 0.27312775330396477, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 117.75992584228516, + "fcm_dpo/q_t": 0.49232739210128784, + "grad_norm": 3.8786041736602783, + "learning_rate": 4.5697190473557947e-07, + "logits/chosen": -0.1721055507659912, + "logits/rejected": -0.1947544515132904, + "logps/chosen": -391.23492431640625, + "logps/ref_chosen": -69.35346984863281, + "logps/ref_rejected": -88.07244873046875, + "logps/rejected": -527.7138671875, + "loss": 1.3584, + "margin_dpo/margin_mean": 117.75991821289062, + "margin_dpo/margin_std": 379.52716064453125, + "step": 186 + }, + { + "KL/chosen_KL_mean": -243.9990234375, + "KL/mean": -316.26837158203125, + "KL/rejected_KL_mean": -388.53765869140625, + "KL/std": 256.2058410644531, + "epoch": 0.2745961820851689, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 144.5386199951172, + "fcm_dpo/q_t": 0.49058419466018677, + "grad_norm": 3.2976255416870117, + "learning_rate": 4.5624936864957555e-07, + "logits/chosen": -0.14343130588531494, + "logits/rejected": -0.15117818117141724, + "logps/chosen": -296.7554931640625, + "logps/ref_chosen": -52.7564582824707, + "logps/ref_rejected": -81.96910095214844, + "logps/rejected": -470.50677490234375, + "loss": 1.3503, + "margin_dpo/margin_mean": 144.53860473632812, + "margin_dpo/margin_std": 277.4365234375, + "step": 187 + }, + { + "KL/chosen_KL_mean": -212.0707550048828, + "KL/mean": -298.758544921875, + "KL/rejected_KL_mean": -385.44635009765625, + "KL/std": 242.26661682128906, + "epoch": 0.27606461086637296, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 173.37557983398438, + "fcm_dpo/q_t": 0.48871082067489624, + "grad_norm": 4.286935806274414, + "learning_rate": 4.5552139770089454e-07, + "logits/chosen": -0.17238366603851318, + "logits/rejected": -0.197291761636734, + "logps/chosen": -261.4862365722656, + "logps/ref_chosen": -49.415489196777344, + "logps/ref_rejected": -89.54043579101562, + "logps/rejected": -474.98675537109375, + "loss": 1.3429, + "margin_dpo/margin_mean": 173.37557983398438, + "margin_dpo/margin_std": 285.1446533203125, + "step": 188 + }, + { + "KL/chosen_KL_mean": -252.53201293945312, + "KL/mean": -311.17919921875, + "KL/rejected_KL_mean": -369.826416015625, + "KL/std": 267.0799560546875, + "epoch": 0.2775330396475771, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 117.29438781738281, + "fcm_dpo/q_t": 0.49236994981765747, + "grad_norm": 3.5573782920837402, + "learning_rate": 4.5478801107224794e-07, + "logits/chosen": -0.1831766963005066, + "logits/rejected": -0.19586911797523499, + "logps/chosen": -304.93096923828125, + "logps/ref_chosen": -52.39896011352539, + "logps/ref_rejected": -72.16735076904297, + "logps/rejected": -441.9937744140625, + "loss": 1.358, + "margin_dpo/margin_mean": 117.29438781738281, + "margin_dpo/margin_std": 346.103759765625, + "step": 189 + }, + { + "KL/chosen_KL_mean": -264.6123046875, + "KL/mean": -355.71697998046875, + "KL/rejected_KL_mean": -446.8216857910156, + "KL/std": 305.12103271484375, + "epoch": 0.2790014684287812, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 182.20941162109375, + "fcm_dpo/q_t": 0.48815953731536865, + "grad_norm": 3.5123867988586426, + "learning_rate": 4.5404922808905543e-07, + "logits/chosen": -0.2051401287317276, + "logits/rejected": -0.2246163785457611, + "logps/chosen": -329.29534912109375, + "logps/ref_chosen": -64.68305969238281, + "logps/ref_rejected": -102.55052185058594, + "logps/rejected": -549.3721923828125, + "loss": 1.3418, + "margin_dpo/margin_mean": 182.20941162109375, + "margin_dpo/margin_std": 370.34747314453125, + "step": 190 + }, + { + "KL/chosen_KL_mean": -261.7254333496094, + "KL/mean": -384.62042236328125, + "KL/rejected_KL_mean": -507.5153503417969, + "KL/std": 342.0569152832031, + "epoch": 0.28046989720998533, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 245.78993225097656, + "fcm_dpo/q_t": 0.48404163122177124, + "grad_norm": 5.01483678817749, + "learning_rate": 4.5330506821893565e-07, + "logits/chosen": -0.22717076539993286, + "logits/rejected": -0.2197323739528656, + "logps/chosen": -330.3843078613281, + "logps/ref_chosen": -68.65887451171875, + "logps/ref_rejected": -110.1396713256836, + "logps/rejected": -617.655029296875, + "loss": 1.3261, + "margin_dpo/margin_mean": 245.78994750976562, + "margin_dpo/margin_std": 414.4168701171875, + "step": 191 + }, + { + "KL/chosen_KL_mean": -326.44854736328125, + "KL/mean": -397.67572021484375, + "KL/rejected_KL_mean": -468.90289306640625, + "KL/std": 319.5667724609375, + "epoch": 0.28193832599118945, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 142.454345703125, + "fcm_dpo/q_t": 0.4907473027706146, + "grad_norm": 3.0714664459228516, + "learning_rate": 4.5255555107119336e-07, + "logits/chosen": -0.22967159748077393, + "logits/rejected": -0.23665109276771545, + "logps/chosen": -396.17547607421875, + "logps/ref_chosen": -69.72691345214844, + "logps/ref_rejected": -103.32135009765625, + "logps/rejected": -572.2242431640625, + "loss": 1.3524, + "margin_dpo/margin_mean": 142.454345703125, + "margin_dpo/margin_std": 413.8336181640625, + "step": 192 + }, + { + "KL/chosen_KL_mean": -321.8466796875, + "KL/mean": -365.2450256347656, + "KL/rejected_KL_mean": -408.64337158203125, + "KL/std": 342.6612854003906, + "epoch": 0.2834067547723935, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 86.79666900634766, + "fcm_dpo/q_t": 0.494370698928833, + "grad_norm": 3.745593786239624, + "learning_rate": 4.5180069639630236e-07, + "logits/chosen": -0.19167697429656982, + "logits/rejected": -0.2146417796611786, + "logps/chosen": -382.0372009277344, + "logps/ref_chosen": -60.19049835205078, + "logps/ref_rejected": -76.40755462646484, + "logps/rejected": -485.0509033203125, + "loss": 1.367, + "margin_dpo/margin_mean": 86.79667663574219, + "margin_dpo/margin_std": 428.4646911621094, + "step": 193 + }, + { + "KL/chosen_KL_mean": -199.5254364013672, + "KL/mean": -264.96240234375, + "KL/rejected_KL_mean": -330.3993835449219, + "KL/std": 220.21302795410156, + "epoch": 0.28487518355359764, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 130.8739471435547, + "fcm_dpo/q_t": 0.49146947264671326, + "grad_norm": 3.0979061126708984, + "learning_rate": 4.510405240853854e-07, + "logits/chosen": -0.12437999248504639, + "logits/rejected": -0.13489431142807007, + "logps/chosen": -237.36581420898438, + "logps/ref_chosen": -37.84037399291992, + "logps/ref_rejected": -60.684783935546875, + "logps/rejected": -391.08416748046875, + "loss": 1.3535, + "margin_dpo/margin_mean": 130.8739471435547, + "margin_dpo/margin_std": 246.00189208984375, + "step": 194 + }, + { + "KL/chosen_KL_mean": -305.24945068359375, + "KL/mean": -408.80322265625, + "KL/rejected_KL_mean": -512.35693359375, + "KL/std": 328.6646728515625, + "epoch": 0.28634361233480177, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 207.1074676513672, + "fcm_dpo/q_t": 0.4865289628505707, + "grad_norm": 5.246110439300537, + "learning_rate": 4.5027505416968985e-07, + "logits/chosen": -0.18198040127754211, + "logits/rejected": -0.18706868588924408, + "logps/chosen": -360.14105224609375, + "logps/ref_chosen": -54.891571044921875, + "logps/ref_rejected": -96.77095794677734, + "logps/rejected": -609.1279296875, + "loss": 1.335, + "margin_dpo/margin_mean": 207.1074676513672, + "margin_dpo/margin_std": 345.01220703125, + "step": 195 + }, + { + "KL/chosen_KL_mean": -284.6699523925781, + "KL/mean": -360.74468994140625, + "KL/rejected_KL_mean": -436.8194885253906, + "KL/std": 296.30322265625, + "epoch": 0.2878120411160059, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 152.1495361328125, + "fcm_dpo/q_t": 0.4900960326194763, + "grad_norm": 3.9334256649017334, + "learning_rate": 4.495043068200599e-07, + "logits/chosen": -0.18070095777511597, + "logits/rejected": -0.1948184370994568, + "logps/chosen": -337.9151916503906, + "logps/ref_chosen": -53.245243072509766, + "logps/ref_rejected": -76.05294799804688, + "logps/rejected": -512.8724365234375, + "loss": 1.3498, + "margin_dpo/margin_mean": 152.1495361328125, + "margin_dpo/margin_std": 388.2755126953125, + "step": 196 + }, + { + "KL/chosen_KL_mean": -307.69537353515625, + "KL/mean": -364.24560546875, + "KL/rejected_KL_mean": -420.79583740234375, + "KL/std": 271.6278076171875, + "epoch": 0.28928046989721, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 113.10043334960938, + "fcm_dpo/q_t": 0.49263206124305725, + "grad_norm": 3.3762285709381104, + "learning_rate": 4.4872830234640493e-07, + "logits/chosen": -0.1415313184261322, + "logits/rejected": -0.15090778470039368, + "logps/chosen": -368.11572265625, + "logps/ref_chosen": -60.42033386230469, + "logps/ref_rejected": -77.20890808105469, + "logps/rejected": -498.0047302246094, + "loss": 1.3587, + "margin_dpo/margin_mean": 113.10043334960938, + "margin_dpo/margin_std": 311.4438781738281, + "step": 197 + }, + { + "KL/chosen_KL_mean": -300.6440124511719, + "KL/mean": -404.669921875, + "KL/rejected_KL_mean": -508.69580078125, + "KL/std": 325.8094482421875, + "epoch": 0.2907488986784141, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 208.05178833007812, + "fcm_dpo/q_t": 0.48646122217178345, + "grad_norm": 3.8525943756103516, + "learning_rate": 4.479470611971645e-07, + "logits/chosen": -0.17303922772407532, + "logits/rejected": -0.1829444169998169, + "logps/chosen": -355.6802062988281, + "logps/ref_chosen": -55.03618621826172, + "logps/ref_rejected": -97.24325561523438, + "logps/rejected": -605.9390869140625, + "loss": 1.3347, + "margin_dpo/margin_mean": 208.05178833007812, + "margin_dpo/margin_std": 336.3865966796875, + "step": 198 + }, + { + "KL/chosen_KL_mean": -283.0042724609375, + "KL/mean": -371.278076171875, + "KL/rejected_KL_mean": -459.5518798828125, + "KL/std": 279.841064453125, + "epoch": 0.2922173274596182, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 176.547607421875, + "fcm_dpo/q_t": 0.48851385712623596, + "grad_norm": 4.46752405166626, + "learning_rate": 4.471606039587695e-07, + "logits/chosen": -0.19464930891990662, + "logits/rejected": -0.18601861596107483, + "logps/chosen": -339.8330993652344, + "logps/ref_chosen": -56.828826904296875, + "logps/ref_rejected": -84.64820861816406, + "logps/rejected": -544.2000732421875, + "loss": 1.3432, + "margin_dpo/margin_mean": 176.547607421875, + "margin_dpo/margin_std": 378.5025634765625, + "step": 199 + }, + { + "KL/chosen_KL_mean": -289.37652587890625, + "KL/mean": -378.7618408203125, + "KL/rejected_KL_mean": -468.1471862792969, + "KL/std": 326.5268249511719, + "epoch": 0.2936857562408223, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 178.77064514160156, + "fcm_dpo/q_t": 0.48838722705841064, + "grad_norm": 4.424515724182129, + "learning_rate": 4.4636895135509966e-07, + "logits/chosen": -0.16734561324119568, + "logits/rejected": -0.1917646825313568, + "logps/chosen": -342.4435729980469, + "logps/ref_chosen": -53.06706237792969, + "logps/ref_rejected": -80.60843658447266, + "logps/rejected": -548.755615234375, + "loss": 1.3438, + "margin_dpo/margin_mean": 178.77066040039062, + "margin_dpo/margin_std": 455.8370056152344, + "step": 200 + }, + { + "KL/chosen_KL_mean": -342.85003662109375, + "KL/mean": -442.3621520996094, + "KL/rejected_KL_mean": -541.874267578125, + "KL/std": 393.974609375, + "epoch": 0.29515418502202645, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 199.02426147460938, + "fcm_dpo/q_t": 0.48709234595298767, + "grad_norm": 4.1248321533203125, + "learning_rate": 4.455721242469372e-07, + "logits/chosen": -0.28452351689338684, + "logits/rejected": -0.2933052182197571, + "logps/chosen": -418.25225830078125, + "logps/ref_chosen": -75.4022216796875, + "logps/ref_rejected": -114.80821990966797, + "logps/rejected": -656.6824951171875, + "loss": 1.3392, + "margin_dpo/margin_mean": 199.02427673339844, + "margin_dpo/margin_std": 483.22381591796875, + "step": 201 + }, + { + "KL/chosen_KL_mean": -308.9986877441406, + "KL/mean": -377.71002197265625, + "KL/rejected_KL_mean": -446.42132568359375, + "KL/std": 325.0315246582031, + "epoch": 0.2966226138032305, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 137.42266845703125, + "fcm_dpo/q_t": 0.49106624722480774, + "grad_norm": 3.5341696739196777, + "learning_rate": 4.4477014363141755e-07, + "logits/chosen": -0.19032201170921326, + "logits/rejected": -0.20689848065376282, + "logps/chosen": -359.1000061035156, + "logps/ref_chosen": -50.101318359375, + "logps/ref_rejected": -86.98503112792969, + "logps/rejected": -533.4063720703125, + "loss": 1.3534, + "margin_dpo/margin_mean": 137.42266845703125, + "margin_dpo/margin_std": 391.82958984375, + "step": 202 + }, + { + "KL/chosen_KL_mean": -348.4794006347656, + "KL/mean": -433.90374755859375, + "KL/rejected_KL_mean": -519.3280639648438, + "KL/std": 378.5124816894531, + "epoch": 0.29809104258443464, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 170.84869384765625, + "fcm_dpo/q_t": 0.4889407753944397, + "grad_norm": 3.832104206085205, + "learning_rate": 4.439630306414758e-07, + "logits/chosen": -0.17535123229026794, + "logits/rejected": -0.18367072939872742, + "logps/chosen": -409.089111328125, + "logps/ref_chosen": -60.60969543457031, + "logps/ref_rejected": -85.89596557617188, + "logps/rejected": -605.2239990234375, + "loss": 1.3457, + "margin_dpo/margin_mean": 170.84869384765625, + "margin_dpo/margin_std": 453.1564636230469, + "step": 203 + }, + { + "KL/chosen_KL_mean": -439.98712158203125, + "KL/mean": -480.7139892578125, + "KL/rejected_KL_mean": -521.4407958984375, + "KL/std": 433.0479736328125, + "epoch": 0.29955947136563876, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 81.45366668701172, + "fcm_dpo/q_t": 0.4946914613246918, + "grad_norm": 7.0813751220703125, + "learning_rate": 4.431508065452897e-07, + "logits/chosen": -0.2240895926952362, + "logits/rejected": -0.23155343532562256, + "logps/chosen": -520.152099609375, + "logps/ref_chosen": -80.16496276855469, + "logps/ref_rejected": -87.69590759277344, + "logps/rejected": -609.13671875, + "loss": 1.37, + "margin_dpo/margin_mean": 81.45367431640625, + "margin_dpo/margin_std": 518.8585205078125, + "step": 204 + }, + { + "KL/chosen_KL_mean": -384.1351013183594, + "KL/mean": -467.84967041015625, + "KL/rejected_KL_mean": -551.564208984375, + "KL/std": 466.6665344238281, + "epoch": 0.3010279001468429, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 167.42913818359375, + "fcm_dpo/q_t": 0.48906105756759644, + "grad_norm": 4.989256381988525, + "learning_rate": 4.4233349274571974e-07, + "logits/chosen": -0.13244661688804626, + "logits/rejected": -0.1470334827899933, + "logps/chosen": -443.51983642578125, + "logps/ref_chosen": -59.384735107421875, + "logps/ref_rejected": -85.12505340576172, + "logps/rejected": -636.6893310546875, + "loss": 1.3482, + "margin_dpo/margin_mean": 167.42913818359375, + "margin_dpo/margin_std": 530.16162109375, + "step": 205 + }, + { + "KL/chosen_KL_mean": -307.87725830078125, + "KL/mean": -408.16168212890625, + "KL/rejected_KL_mean": -508.4461364746094, + "KL/std": 317.0555419921875, + "epoch": 0.302496328928047, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 200.56893920898438, + "fcm_dpo/q_t": 0.48691922426223755, + "grad_norm": 4.713033676147461, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": -0.13550423085689545, + "logits/rejected": -0.14084811508655548, + "logps/chosen": -354.84173583984375, + "logps/ref_chosen": -46.964500427246094, + "logps/ref_rejected": -98.9534912109375, + "logps/rejected": -607.399658203125, + "loss": 1.337, + "margin_dpo/margin_mean": 200.56893920898438, + "margin_dpo/margin_std": 370.0242919921875, + "step": 206 + }, + { + "KL/chosen_KL_mean": -296.52545166015625, + "KL/mean": -404.8011169433594, + "KL/rejected_KL_mean": -513.0767822265625, + "KL/std": 367.4708557128906, + "epoch": 0.3039647577092511, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 216.55137634277344, + "fcm_dpo/q_t": 0.485943078994751, + "grad_norm": 5.192399501800537, + "learning_rate": 4.4068368231789365e-07, + "logits/chosen": -0.17912110686302185, + "logits/rejected": -0.18409845232963562, + "logps/chosen": -352.5816955566406, + "logps/ref_chosen": -56.05625915527344, + "logps/ref_rejected": -84.44779968261719, + "logps/rejected": -597.5245971679688, + "loss": 1.3342, + "margin_dpo/margin_mean": 216.55136108398438, + "margin_dpo/margin_std": 459.0970458984375, + "step": 207 + }, + { + "KL/chosen_KL_mean": -403.71490478515625, + "KL/mean": -488.7073974609375, + "KL/rejected_KL_mean": -573.699951171875, + "KL/std": 352.8353271484375, + "epoch": 0.3054331864904552, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 169.98507690429688, + "fcm_dpo/q_t": 0.48894062638282776, + "grad_norm": 3.8758647441864014, + "learning_rate": 4.398512291636768e-07, + "logits/chosen": -0.16348713636398315, + "logits/rejected": -0.17789456248283386, + "logps/chosen": -470.78253173828125, + "logps/ref_chosen": -67.06761169433594, + "logps/ref_rejected": -94.28689575195312, + "logps/rejected": -667.98681640625, + "loss": 1.3459, + "margin_dpo/margin_mean": 169.98507690429688, + "margin_dpo/margin_std": 450.72607421875, + "step": 208 + }, + { + "KL/chosen_KL_mean": -328.24884033203125, + "KL/mean": -394.6558532714844, + "KL/rejected_KL_mean": -461.0628662109375, + "KL/std": 306.83245849609375, + "epoch": 0.3069016152716593, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 132.81405639648438, + "fcm_dpo/q_t": 0.49136117100715637, + "grad_norm": 4.020294666290283, + "learning_rate": 4.3901377325300857e-07, + "logits/chosen": -0.1070629358291626, + "logits/rejected": -0.12986576557159424, + "logps/chosen": -384.4305114746094, + "logps/ref_chosen": -56.18169403076172, + "logps/ref_rejected": -80.94152069091797, + "logps/rejected": -542.00439453125, + "loss": 1.3545, + "margin_dpo/margin_mean": 132.81405639648438, + "margin_dpo/margin_std": 382.304443359375, + "step": 209 + }, + { + "KL/chosen_KL_mean": -257.1470031738281, + "KL/mean": -342.8197021484375, + "KL/rejected_KL_mean": -428.4924011230469, + "KL/std": 300.5461730957031, + "epoch": 0.30837004405286345, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 171.34536743164062, + "fcm_dpo/q_t": 0.48883914947509766, + "grad_norm": 4.435647487640381, + "learning_rate": 4.381713366536311e-07, + "logits/chosen": -0.15906672179698944, + "logits/rejected": -0.18111161887645721, + "logps/chosen": -303.5188293457031, + "logps/ref_chosen": -46.371822357177734, + "logps/ref_rejected": -76.68162536621094, + "logps/rejected": -505.17401123046875, + "loss": 1.3446, + "margin_dpo/margin_mean": 171.34536743164062, + "margin_dpo/margin_std": 377.39703369140625, + "step": 210 + }, + { + "KL/chosen_KL_mean": -396.564208984375, + "KL/mean": -462.0057373046875, + "KL/rejected_KL_mean": -527.4472045898438, + "KL/std": 364.4835510253906, + "epoch": 0.30983847283406757, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 130.88299560546875, + "fcm_dpo/q_t": 0.4915020167827606, + "grad_norm": 4.021511554718018, + "learning_rate": 4.373239415645323e-07, + "logits/chosen": -0.13742633163928986, + "logits/rejected": -0.1542784869670868, + "logps/chosen": -475.49658203125, + "logps/ref_chosen": -78.93235778808594, + "logps/ref_rejected": -86.82098388671875, + "logps/rejected": -614.2681884765625, + "loss": 1.356, + "margin_dpo/margin_mean": 130.88299560546875, + "margin_dpo/margin_std": 444.1982116699219, + "step": 211 + }, + { + "KL/chosen_KL_mean": -325.6275634765625, + "KL/mean": -471.6920471191406, + "KL/rejected_KL_mean": -617.7564697265625, + "KL/std": 407.026123046875, + "epoch": 0.31130690161527164, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 292.12890625, + "fcm_dpo/q_t": 0.4810423254966736, + "grad_norm": 6.869906902313232, + "learning_rate": 4.3647161031536086e-07, + "logits/chosen": -0.14784768223762512, + "logits/rejected": -0.159469336271286, + "logps/chosen": -383.8245849609375, + "logps/ref_chosen": -58.19701385498047, + "logps/ref_rejected": -103.05785369873047, + "logps/rejected": -720.8143310546875, + "loss": 1.3156, + "margin_dpo/margin_mean": 292.12890625, + "margin_dpo/margin_std": 470.7772216796875, + "step": 212 + }, + { + "KL/chosen_KL_mean": -360.6119384765625, + "KL/mean": -459.34661865234375, + "KL/rejected_KL_mean": -558.0812377929688, + "KL/std": 398.5190124511719, + "epoch": 0.31277533039647576, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 197.4693603515625, + "fcm_dpo/q_t": 0.48718905448913574, + "grad_norm": 4.646012783050537, + "learning_rate": 4.3561436536583774e-07, + "logits/chosen": -0.11058177053928375, + "logits/rejected": -0.1331164687871933, + "logps/chosen": -428.1246337890625, + "logps/ref_chosen": -67.51271057128906, + "logps/ref_rejected": -93.91471862792969, + "logps/rejected": -651.9959716796875, + "loss": 1.3402, + "margin_dpo/margin_mean": 197.4693603515625, + "margin_dpo/margin_std": 528.8551635742188, + "step": 213 + }, + { + "KL/chosen_KL_mean": -279.425537109375, + "KL/mean": -385.142333984375, + "KL/rejected_KL_mean": -490.85919189453125, + "KL/std": 298.0137939453125, + "epoch": 0.3142437591776799, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 211.43362426757812, + "fcm_dpo/q_t": 0.4862461984157562, + "grad_norm": 4.712491512298584, + "learning_rate": 4.3475222930516473e-07, + "logits/chosen": -0.09634272009134293, + "logits/rejected": -0.10510338842868805, + "logps/chosen": -321.0304260253906, + "logps/ref_chosen": -41.604888916015625, + "logps/ref_rejected": -77.51741027832031, + "logps/rejected": -568.3765869140625, + "loss": 1.3342, + "margin_dpo/margin_mean": 211.43362426757812, + "margin_dpo/margin_std": 372.36749267578125, + "step": 214 + }, + { + "KL/chosen_KL_mean": -357.0162658691406, + "KL/mean": -462.54559326171875, + "KL/rejected_KL_mean": -568.074951171875, + "KL/std": 371.6539306640625, + "epoch": 0.315712187958884, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 211.05868530273438, + "fcm_dpo/q_t": 0.48629534244537354, + "grad_norm": 7.907057762145996, + "learning_rate": 4.3388522485142885e-07, + "logits/chosen": -0.1348808854818344, + "logits/rejected": -0.14514675736427307, + "logps/chosen": -410.2955322265625, + "logps/ref_chosen": -53.279266357421875, + "logps/ref_rejected": -89.96464538574219, + "logps/rejected": -658.03955078125, + "loss": 1.3349, + "margin_dpo/margin_mean": 211.05868530273438, + "margin_dpo/margin_std": 410.6441650390625, + "step": 215 + }, + { + "KL/chosen_KL_mean": -329.777587890625, + "KL/mean": -422.9111328125, + "KL/rejected_KL_mean": -516.0447387695312, + "KL/std": 320.753662109375, + "epoch": 0.31718061674008813, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 186.26715087890625, + "fcm_dpo/q_t": 0.4878776967525482, + "grad_norm": 4.5853495597839355, + "learning_rate": 4.330133748510036e-07, + "logits/chosen": -0.1529572308063507, + "logits/rejected": -0.1603568196296692, + "logps/chosen": -378.6653747558594, + "logps/ref_chosen": -48.887794494628906, + "logps/ref_rejected": -77.19892883300781, + "logps/rejected": -593.24365234375, + "loss": 1.3412, + "margin_dpo/margin_mean": 186.26715087890625, + "margin_dpo/margin_std": 412.505126953125, + "step": 216 + }, + { + "KL/chosen_KL_mean": -363.7659912109375, + "KL/mean": -485.22747802734375, + "KL/rejected_KL_mean": -606.68896484375, + "KL/std": 382.6268005371094, + "epoch": 0.3186490455212922, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 242.92294311523438, + "fcm_dpo/q_t": 0.48422378301620483, + "grad_norm": 4.863481044769287, + "learning_rate": 4.3213670227794757e-07, + "logits/chosen": -0.0929611325263977, + "logits/rejected": -0.10462786257266998, + "logps/chosen": -413.6112976074219, + "logps/ref_chosen": -49.845306396484375, + "logps/ref_rejected": -100.07832336425781, + "logps/rejected": -706.7672729492188, + "loss": 1.3279, + "margin_dpo/margin_mean": 242.92294311523438, + "margin_dpo/margin_std": 480.1194152832031, + "step": 217 + }, + { + "KL/chosen_KL_mean": -377.3120422363281, + "KL/mean": -467.94586181640625, + "KL/rejected_KL_mean": -558.5796508789062, + "KL/std": 384.8089904785156, + "epoch": 0.3201174743024963, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 181.26760864257812, + "fcm_dpo/q_t": 0.4882817268371582, + "grad_norm": 5.610445022583008, + "learning_rate": 4.3125523023339815e-07, + "logits/chosen": -0.1369292289018631, + "logits/rejected": -0.1515660285949707, + "logps/chosen": -435.88873291015625, + "logps/ref_chosen": -58.576683044433594, + "logps/ref_rejected": -87.84639739990234, + "logps/rejected": -646.426025390625, + "loss": 1.344, + "margin_dpo/margin_mean": 181.26760864257812, + "margin_dpo/margin_std": 512.280517578125, + "step": 218 + }, + { + "KL/chosen_KL_mean": -393.2535400390625, + "KL/mean": -450.13006591796875, + "KL/rejected_KL_mean": -507.00653076171875, + "KL/std": 405.0634765625, + "epoch": 0.32158590308370044, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 113.75296020507812, + "fcm_dpo/q_t": 0.49264246225357056, + "grad_norm": 5.182664394378662, + "learning_rate": 4.303689819449636e-07, + "logits/chosen": -0.15762826800346375, + "logits/rejected": -0.18174490332603455, + "logps/chosen": -454.33740234375, + "logps/ref_chosen": -61.083858489990234, + "logps/ref_rejected": -85.83042907714844, + "logps/rejected": -592.8369750976562, + "loss": 1.3619, + "margin_dpo/margin_mean": 113.75296020507812, + "margin_dpo/margin_std": 537.340087890625, + "step": 219 + }, + { + "KL/chosen_KL_mean": -386.52679443359375, + "KL/mean": -445.1171875, + "KL/rejected_KL_mean": -503.7076110839844, + "KL/std": 314.7594909667969, + "epoch": 0.32305433186490456, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 117.18080139160156, + "fcm_dpo/q_t": 0.4923757314682007, + "grad_norm": 4.579725742340088, + "learning_rate": 4.2947798076611047e-07, + "logits/chosen": -0.16421407461166382, + "logits/rejected": -0.16229598224163055, + "logps/chosen": -456.55810546875, + "logps/ref_chosen": -70.03128051757812, + "logps/ref_rejected": -87.68551635742188, + "logps/rejected": -591.3931274414062, + "loss": 1.3582, + "margin_dpo/margin_mean": 117.18080139160156, + "margin_dpo/margin_std": 367.78851318359375, + "step": 220 + }, + { + "KL/chosen_KL_mean": -283.6891784667969, + "KL/mean": -439.31463623046875, + "KL/rejected_KL_mean": -594.9400634765625, + "KL/std": 372.5516357421875, + "epoch": 0.3245227606461087, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 311.25091552734375, + "fcm_dpo/q_t": 0.47978127002716064, + "grad_norm": 6.784043312072754, + "learning_rate": 4.285822501755485e-07, + "logits/chosen": -0.17648911476135254, + "logits/rejected": -0.1816864311695099, + "logps/chosen": -335.8438720703125, + "logps/ref_chosen": -52.15470886230469, + "logps/ref_rejected": -106.46768188476562, + "logps/rejected": -701.40771484375, + "loss": 1.3099, + "margin_dpo/margin_mean": 311.25091552734375, + "margin_dpo/margin_std": 432.03814697265625, + "step": 221 + }, + { + "KL/chosen_KL_mean": -308.0732421875, + "KL/mean": -414.90313720703125, + "KL/rejected_KL_mean": -521.7329711914062, + "KL/std": 339.31243896484375, + "epoch": 0.32599118942731276, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 213.6597442626953, + "fcm_dpo/q_t": 0.48610588908195496, + "grad_norm": 4.452853202819824, + "learning_rate": 4.276818137766118e-07, + "logits/chosen": -0.1906522810459137, + "logits/rejected": -0.22085875272750854, + "logps/chosen": -369.04437255859375, + "logps/ref_chosen": -60.971099853515625, + "logps/ref_rejected": -100.00115203857422, + "logps/rejected": -621.734130859375, + "loss": 1.3344, + "margin_dpo/margin_mean": 213.6597442626953, + "margin_dpo/margin_std": 419.52435302734375, + "step": 222 + }, + { + "KL/chosen_KL_mean": -299.1932373046875, + "KL/mean": -382.9361572265625, + "KL/rejected_KL_mean": -466.6790771484375, + "KL/std": 313.127685546875, + "epoch": 0.3274596182085169, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 167.48585510253906, + "fcm_dpo/q_t": 0.4891514778137207, + "grad_norm": 4.128637313842773, + "learning_rate": 4.2677669529663686e-07, + "logits/chosen": -0.1469314992427826, + "logits/rejected": -0.16430073976516724, + "logps/chosen": -351.83380126953125, + "logps/ref_chosen": -52.64057540893555, + "logps/ref_rejected": -82.82502746582031, + "logps/rejected": -549.504150390625, + "loss": 1.3463, + "margin_dpo/margin_mean": 167.48585510253906, + "margin_dpo/margin_std": 419.4111328125, + "step": 223 + }, + { + "KL/chosen_KL_mean": -290.7779541015625, + "KL/mean": -380.89788818359375, + "KL/rejected_KL_mean": -471.0178527832031, + "KL/std": 377.8497314453125, + "epoch": 0.328928046989721, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 180.23985290527344, + "fcm_dpo/q_t": 0.48835575580596924, + "grad_norm": 4.23874568939209, + "learning_rate": 4.2586691858633747e-07, + "logits/chosen": -0.17558273673057556, + "logits/rejected": -0.1842508316040039, + "logps/chosen": -339.3733825683594, + "logps/ref_chosen": -48.59541320800781, + "logps/ref_rejected": -77.11648559570312, + "logps/rejected": -548.1343383789062, + "loss": 1.3437, + "margin_dpo/margin_mean": 180.23985290527344, + "margin_dpo/margin_std": 466.7406005859375, + "step": 224 + }, + { + "KL/chosen_KL_mean": -341.2305603027344, + "KL/mean": -462.2286376953125, + "KL/rejected_KL_mean": -583.2266845703125, + "KL/std": 376.18865966796875, + "epoch": 0.3303964757709251, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 241.99610900878906, + "fcm_dpo/q_t": 0.4843394160270691, + "grad_norm": 5.250749111175537, + "learning_rate": 4.249525076191759e-07, + "logits/chosen": -0.13447608053684235, + "logits/rejected": -0.1467527151107788, + "logps/chosen": -399.23101806640625, + "logps/ref_chosen": -58.000465393066406, + "logps/ref_rejected": -99.90291595458984, + "logps/rejected": -683.129638671875, + "loss": 1.3286, + "margin_dpo/margin_mean": 241.99610900878906, + "margin_dpo/margin_std": 503.9619140625, + "step": 225 + }, + { + "KL/chosen_KL_mean": -325.04632568359375, + "KL/mean": -414.34527587890625, + "KL/rejected_KL_mean": -503.6441650390625, + "KL/std": 336.3865966796875, + "epoch": 0.33186490455212925, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 178.5978240966797, + "fcm_dpo/q_t": 0.48840048909187317, + "grad_norm": 3.9235055446624756, + "learning_rate": 4.2403348649073167e-07, + "logits/chosen": -0.23346635699272156, + "logits/rejected": -0.2388550341129303, + "logps/chosen": -383.94512939453125, + "logps/ref_chosen": -58.898799896240234, + "logps/ref_rejected": -78.68775939941406, + "logps/rejected": -582.3319091796875, + "loss": 1.3435, + "margin_dpo/margin_mean": 178.5978240966797, + "margin_dpo/margin_std": 436.91937255859375, + "step": 226 + }, + { + "KL/chosen_KL_mean": -367.533447265625, + "KL/mean": -498.5723876953125, + "KL/rejected_KL_mean": -629.611328125, + "KL/std": 458.18707275390625, + "epoch": 0.3333333333333333, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 262.07794189453125, + "fcm_dpo/q_t": 0.48310425877571106, + "grad_norm": 4.806700229644775, + "learning_rate": 4.2310987941806615e-07, + "logits/chosen": -0.21147508919239044, + "logits/rejected": -0.2103334367275238, + "logps/chosen": -426.6055908203125, + "logps/ref_chosen": -59.072181701660156, + "logps/ref_rejected": -99.41236877441406, + "logps/rejected": -729.023681640625, + "loss": 1.3237, + "margin_dpo/margin_mean": 262.07794189453125, + "margin_dpo/margin_std": 509.9268493652344, + "step": 227 + }, + { + "KL/chosen_KL_mean": -403.41619873046875, + "KL/mean": -481.063720703125, + "KL/rejected_KL_mean": -558.7113037109375, + "KL/std": 388.300537109375, + "epoch": 0.33480176211453744, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 155.29510498046875, + "fcm_dpo/q_t": 0.48991623520851135, + "grad_norm": 3.973229169845581, + "learning_rate": 4.2218171073908463e-07, + "logits/chosen": -0.2081584632396698, + "logits/rejected": -0.20904359221458435, + "logps/chosen": -469.3074645996094, + "logps/ref_chosen": -65.89128875732422, + "logps/ref_rejected": -91.04875183105469, + "logps/rejected": -649.760009765625, + "loss": 1.3509, + "margin_dpo/margin_mean": 155.29510498046875, + "margin_dpo/margin_std": 524.5467529296875, + "step": 228 + }, + { + "KL/chosen_KL_mean": -462.6470642089844, + "KL/mean": -514.1942138671875, + "KL/rejected_KL_mean": -565.7413330078125, + "KL/std": 442.085205078125, + "epoch": 0.33627019089574156, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 103.0942611694336, + "fcm_dpo/q_t": 0.4933173358440399, + "grad_norm": 6.374516010284424, + "learning_rate": 4.212490049118951e-07, + "logits/chosen": -0.28295138478279114, + "logits/rejected": -0.2788426876068115, + "logps/chosen": -533.3534545898438, + "logps/ref_chosen": -70.70637512207031, + "logps/ref_rejected": -84.52741241455078, + "logps/rejected": -650.2687377929688, + "loss": 1.3638, + "margin_dpo/margin_mean": 103.09426879882812, + "margin_dpo/margin_std": 492.0677795410156, + "step": 229 + }, + { + "KL/chosen_KL_mean": -286.46820068359375, + "KL/mean": -433.4774169921875, + "KL/rejected_KL_mean": -580.486572265625, + "KL/std": 330.7721862792969, + "epoch": 0.3377386196769457, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 294.01837158203125, + "fcm_dpo/q_t": 0.48087483644485474, + "grad_norm": 5.674592018127441, + "learning_rate": 4.203117865141635e-07, + "logits/chosen": -0.21354076266288757, + "logits/rejected": -0.2245483547449112, + "logps/chosen": -325.75018310546875, + "logps/ref_chosen": -39.282005310058594, + "logps/ref_rejected": -85.62191009521484, + "logps/rejected": -666.1085205078125, + "loss": 1.3135, + "margin_dpo/margin_mean": 294.0184020996094, + "margin_dpo/margin_std": 378.5648498535156, + "step": 230 + }, + { + "KL/chosen_KL_mean": -383.29278564453125, + "KL/mean": -453.44512939453125, + "KL/rejected_KL_mean": -523.5975341796875, + "KL/std": 306.9707336425781, + "epoch": 0.3392070484581498, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 140.30477905273438, + "fcm_dpo/q_t": 0.4908882975578308, + "grad_norm": 5.705263137817383, + "learning_rate": 4.1937008024246625e-07, + "logits/chosen": -0.2526136338710785, + "logits/rejected": -0.24818205833435059, + "logps/chosen": -446.5692138671875, + "logps/ref_chosen": -63.27644348144531, + "logps/ref_rejected": -74.1239013671875, + "logps/rejected": -597.721435546875, + "loss": 1.3527, + "margin_dpo/margin_mean": 140.30477905273438, + "margin_dpo/margin_std": 391.14666748046875, + "step": 231 + }, + { + "KL/chosen_KL_mean": -460.69354248046875, + "KL/mean": -498.51318359375, + "KL/rejected_KL_mean": -536.3327026367188, + "KL/std": 363.728515625, + "epoch": 0.3406754772393539, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 75.63915252685547, + "fcm_dpo/q_t": 0.49512791633605957, + "grad_norm": 7.27602481842041, + "learning_rate": 4.1842391091163933e-07, + "logits/chosen": -0.24794329702854156, + "logits/rejected": -0.25373679399490356, + "logps/chosen": -531.4423217773438, + "logps/ref_chosen": -70.74876403808594, + "logps/ref_rejected": -83.97706604003906, + "logps/rejected": -620.309814453125, + "loss": 1.3698, + "margin_dpo/margin_mean": 75.63914489746094, + "margin_dpo/margin_std": 431.7769775390625, + "step": 232 + }, + { + "KL/chosen_KL_mean": -341.1106262207031, + "KL/mean": -483.42645263671875, + "KL/rejected_KL_mean": -625.7423095703125, + "KL/std": 399.1839599609375, + "epoch": 0.342143906020558, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 284.6317138671875, + "fcm_dpo/q_t": 0.48159539699554443, + "grad_norm": 4.810127258300781, + "learning_rate": 4.174733034541245e-07, + "logits/chosen": -0.2372874766588211, + "logits/rejected": -0.254278302192688, + "logps/chosen": -395.9935607910156, + "logps/ref_chosen": -54.8829345703125, + "logps/ref_rejected": -107.4800796508789, + "logps/rejected": -733.222412109375, + "loss": 1.319, + "margin_dpo/margin_mean": 284.6316833496094, + "margin_dpo/margin_std": 552.460205078125, + "step": 233 + }, + { + "KL/chosen_KL_mean": -317.89398193359375, + "KL/mean": -468.377685546875, + "KL/rejected_KL_mean": -618.861328125, + "KL/std": 384.4058837890625, + "epoch": 0.3436123348017621, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 300.9674072265625, + "fcm_dpo/q_t": 0.4804891347885132, + "grad_norm": 4.5735554695129395, + "learning_rate": 4.165182829193126e-07, + "logits/chosen": -0.192276269197464, + "logits/rejected": -0.20555615425109863, + "logps/chosen": -361.98846435546875, + "logps/ref_chosen": -44.094520568847656, + "logps/ref_rejected": -100.00663757324219, + "logps/rejected": -718.8680419921875, + "loss": 1.3133, + "margin_dpo/margin_mean": 300.9674072265625, + "margin_dpo/margin_std": 458.0181884765625, + "step": 234 + }, + { + "KL/chosen_KL_mean": -396.1971740722656, + "KL/mean": -479.3715515136719, + "KL/rejected_KL_mean": -562.5459594726562, + "KL/std": 388.67431640625, + "epoch": 0.34508076358296624, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 166.3487548828125, + "fcm_dpo/q_t": 0.489266961812973, + "grad_norm": 4.237292289733887, + "learning_rate": 4.1555887447288255e-07, + "logits/chosen": -0.1855693757534027, + "logits/rejected": -0.20113056898117065, + "logps/chosen": -458.4350891113281, + "logps/ref_chosen": -62.237911224365234, + "logps/ref_rejected": -90.39506530761719, + "logps/rejected": -652.9410400390625, + "loss": 1.3478, + "margin_dpo/margin_mean": 166.34877014160156, + "margin_dpo/margin_std": 514.4866333007812, + "step": 235 + }, + { + "KL/chosen_KL_mean": -272.7074279785156, + "KL/mean": -432.64501953125, + "KL/rejected_KL_mean": -592.5825805664062, + "KL/std": 387.78814697265625, + "epoch": 0.3465491923641703, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 319.8751220703125, + "fcm_dpo/q_t": 0.4792480170726776, + "grad_norm": 7.933678150177002, + "learning_rate": 4.1459510339613946e-07, + "logits/chosen": -0.18187743425369263, + "logits/rejected": -0.18875324726104736, + "logps/chosen": -322.0487976074219, + "logps/ref_chosen": -49.34136199951172, + "logps/ref_rejected": -103.51162719726562, + "logps/rejected": -696.09423828125, + "loss": 1.3079, + "margin_dpo/margin_mean": 319.8751220703125, + "margin_dpo/margin_std": 438.8853759765625, + "step": 236 + }, + { + "KL/chosen_KL_mean": -399.15289306640625, + "KL/mean": -506.1591491699219, + "KL/rejected_KL_mean": -613.1654052734375, + "KL/std": 423.4207763671875, + "epoch": 0.34801762114537443, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 214.01246643066406, + "fcm_dpo/q_t": 0.4862016439437866, + "grad_norm": 5.225279808044434, + "learning_rate": 4.136269950853473e-07, + "logits/chosen": -0.1944710910320282, + "logits/rejected": -0.20811229944229126, + "logps/chosen": -453.321044921875, + "logps/ref_chosen": -54.168121337890625, + "logps/ref_rejected": -94.78036499023438, + "logps/rejected": -707.94580078125, + "loss": 1.3365, + "margin_dpo/margin_mean": 214.01246643066406, + "margin_dpo/margin_std": 556.74853515625, + "step": 237 + }, + { + "KL/chosen_KL_mean": -385.951171875, + "KL/mean": -494.75103759765625, + "KL/rejected_KL_mean": -603.5508422851562, + "KL/std": 423.44287109375, + "epoch": 0.34948604992657856, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 217.59967041015625, + "fcm_dpo/q_t": 0.48592108488082886, + "grad_norm": 4.655148029327393, + "learning_rate": 4.126545750510605e-07, + "logits/chosen": -0.18144190311431885, + "logits/rejected": -0.1798359453678131, + "logps/chosen": -439.92431640625, + "logps/ref_chosen": -53.973121643066406, + "logps/ref_rejected": -89.41795349121094, + "logps/rejected": -692.9688110351562, + "loss": 1.3345, + "margin_dpo/margin_mean": 217.59967041015625, + "margin_dpo/margin_std": 496.64794921875, + "step": 238 + }, + { + "KL/chosen_KL_mean": -443.1206359863281, + "KL/mean": -553.6134643554688, + "KL/rejected_KL_mean": -664.106201171875, + "KL/std": 398.3800048828125, + "epoch": 0.3509544787077827, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 220.98562622070312, + "fcm_dpo/q_t": 0.4856555461883545, + "grad_norm": 4.865357875823975, + "learning_rate": 4.116778689174514e-07, + "logits/chosen": -0.17183178663253784, + "logits/rejected": -0.1842924803495407, + "logps/chosen": -501.21844482421875, + "logps/ref_chosen": -58.09782409667969, + "logps/ref_rejected": -93.59294128417969, + "logps/rejected": -757.69921875, + "loss": 1.3339, + "margin_dpo/margin_mean": 220.98565673828125, + "margin_dpo/margin_std": 507.60162353515625, + "step": 239 + }, + { + "KL/chosen_KL_mean": -512.9976806640625, + "KL/mean": -566.2974243164062, + "KL/rejected_KL_mean": -619.5970458984375, + "KL/std": 433.83050537109375, + "epoch": 0.3524229074889868, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 106.59938049316406, + "fcm_dpo/q_t": 0.4930400848388672, + "grad_norm": 7.718948841094971, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": -0.23433905839920044, + "logits/rejected": -0.23268157243728638, + "logps/chosen": -573.6121826171875, + "logps/ref_chosen": -60.6144905090332, + "logps/ref_rejected": -74.1185302734375, + "logps/rejected": -693.715576171875, + "loss": 1.3637, + "margin_dpo/margin_mean": 106.5993881225586, + "margin_dpo/margin_std": 538.6660766601562, + "step": 240 + }, + { + "KL/chosen_KL_mean": -482.7232360839844, + "KL/mean": -591.3538208007812, + "KL/rejected_KL_mean": -699.9844360351562, + "KL/std": 479.83416748046875, + "epoch": 0.35389133627019087, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 217.26119995117188, + "fcm_dpo/q_t": 0.4859463572502136, + "grad_norm": 7.044655799865723, + "learning_rate": 4.097117014129903e-07, + "logits/chosen": -0.22257652878761292, + "logits/rejected": -0.23118522763252258, + "logps/chosen": -548.8143310546875, + "logps/ref_chosen": -66.091064453125, + "logps/ref_rejected": -88.06088256835938, + "logps/rejected": -788.0452880859375, + "loss": 1.3367, + "margin_dpo/margin_mean": 217.2611846923828, + "margin_dpo/margin_std": 595.4739990234375, + "step": 241 + }, + { + "KL/chosen_KL_mean": -509.2297058105469, + "KL/mean": -566.0556640625, + "KL/rejected_KL_mean": -622.8816528320312, + "KL/std": 476.64068603515625, + "epoch": 0.355359765051395, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 113.65194702148438, + "fcm_dpo/q_t": 0.49247777462005615, + "grad_norm": 7.708585262298584, + "learning_rate": 4.087222918524807e-07, + "logits/chosen": -0.1928231418132782, + "logits/rejected": -0.19980955123901367, + "logps/chosen": -577.0936279296875, + "logps/ref_chosen": -67.86392974853516, + "logps/ref_rejected": -83.36033630371094, + "logps/rejected": -706.241943359375, + "loss": 1.3645, + "margin_dpo/margin_mean": 113.65194702148438, + "margin_dpo/margin_std": 659.6571655273438, + "step": 242 + }, + { + "KL/chosen_KL_mean": -418.51556396484375, + "KL/mean": -507.8591613769531, + "KL/rejected_KL_mean": -597.2027587890625, + "KL/std": 383.8472900390625, + "epoch": 0.3568281938325991, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 178.68716430664062, + "fcm_dpo/q_t": 0.4883681833744049, + "grad_norm": 6.65237283706665, + "learning_rate": 4.07728699811968e-07, + "logits/chosen": -0.1828111708164215, + "logits/rejected": -0.2104099690914154, + "logps/chosen": -481.5998229980469, + "logps/ref_chosen": -63.0842399597168, + "logps/ref_rejected": -76.33563232421875, + "logps/rejected": -673.538330078125, + "loss": 1.3436, + "margin_dpo/margin_mean": 178.68716430664062, + "margin_dpo/margin_std": 438.80706787109375, + "step": 243 + }, + { + "KL/chosen_KL_mean": -362.7891845703125, + "KL/mean": -491.2398681640625, + "KL/rejected_KL_mean": -619.6905517578125, + "KL/std": 432.4399719238281, + "epoch": 0.35829662261380324, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 256.90142822265625, + "fcm_dpo/q_t": 0.48334479331970215, + "grad_norm": 4.84958028793335, + "learning_rate": 4.067309514735267e-07, + "logits/chosen": -0.19160521030426025, + "logits/rejected": -0.21015608310699463, + "logps/chosen": -423.92987060546875, + "logps/ref_chosen": -61.140689849853516, + "logps/ref_rejected": -94.89193725585938, + "logps/rejected": -714.58251953125, + "loss": 1.3242, + "margin_dpo/margin_mean": 256.90142822265625, + "margin_dpo/margin_std": 459.79437255859375, + "step": 244 + }, + { + "KL/chosen_KL_mean": -390.1641540527344, + "KL/mean": -448.14697265625, + "KL/rejected_KL_mean": -506.1298522949219, + "KL/std": 339.2786865234375, + "epoch": 0.35976505139500736, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 115.9656753540039, + "fcm_dpo/q_t": 0.4924522638320923, + "grad_norm": 4.89885139465332, + "learning_rate": 4.057290731287531e-07, + "logits/chosen": -0.19130942225456238, + "logits/rejected": -0.19781740009784698, + "logps/chosen": -457.42645263671875, + "logps/ref_chosen": -67.26228332519531, + "logps/ref_rejected": -87.64010620117188, + "logps/rejected": -593.7699584960938, + "loss": 1.3592, + "margin_dpo/margin_mean": 115.9656753540039, + "margin_dpo/margin_std": 399.8912353515625, + "step": 245 + }, + { + "KL/chosen_KL_mean": -332.69659423828125, + "KL/mean": -417.5989685058594, + "KL/rejected_KL_mean": -502.5013427734375, + "KL/std": 386.60040283203125, + "epoch": 0.36123348017621143, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 169.80477905273438, + "fcm_dpo/q_t": 0.48896580934524536, + "grad_norm": 4.2765350341796875, + "learning_rate": 4.047230911780736e-07, + "logits/chosen": -0.20524047315120697, + "logits/rejected": -0.21078921854496002, + "logps/chosen": -399.3935546875, + "logps/ref_chosen": -66.69696807861328, + "logps/ref_rejected": -84.34634399414062, + "logps/rejected": -586.8477172851562, + "loss": 1.3457, + "margin_dpo/margin_mean": 169.80477905273438, + "margin_dpo/margin_std": 431.6588134765625, + "step": 246 + }, + { + "KL/chosen_KL_mean": -336.91021728515625, + "KL/mean": -498.5659484863281, + "KL/rejected_KL_mean": -660.2217407226562, + "KL/std": 401.452880859375, + "epoch": 0.36270190895741555, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 323.31146240234375, + "fcm_dpo/q_t": 0.4790780544281006, + "grad_norm": 6.638669490814209, + "learning_rate": 4.0371303213004814e-07, + "logits/chosen": -0.16730782389640808, + "logits/rejected": -0.17316174507141113, + "logps/chosen": -393.51556396484375, + "logps/ref_chosen": -56.6053466796875, + "logps/ref_rejected": -106.29326629638672, + "logps/rejected": -766.5150146484375, + "loss": 1.3086, + "margin_dpo/margin_mean": 323.31146240234375, + "margin_dpo/margin_std": 497.8956298828125, + "step": 247 + }, + { + "KL/chosen_KL_mean": -257.7315368652344, + "KL/mean": -389.46868896484375, + "KL/rejected_KL_mean": -521.205810546875, + "KL/std": 344.24053955078125, + "epoch": 0.3641703377386197, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 263.47430419921875, + "fcm_dpo/q_t": 0.48291152715682983, + "grad_norm": 7.276768684387207, + "learning_rate": 4.0269892260067197e-07, + "logits/chosen": -0.16964401304721832, + "logits/rejected": -0.1837081015110016, + "logps/chosen": -301.7747497558594, + "logps/ref_chosen": -44.043216705322266, + "logps/ref_rejected": -91.85687255859375, + "logps/rejected": -613.0626831054688, + "loss": 1.3216, + "margin_dpo/margin_mean": 263.47430419921875, + "margin_dpo/margin_std": 388.28094482421875, + "step": 248 + }, + { + "KL/chosen_KL_mean": -385.25128173828125, + "KL/mean": -434.3623046875, + "KL/rejected_KL_mean": -483.4732666015625, + "KL/std": 371.12042236328125, + "epoch": 0.3656387665198238, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 98.22201538085938, + "fcm_dpo/q_t": 0.49368637800216675, + "grad_norm": 3.791370391845703, + "learning_rate": 4.0168078931267426e-07, + "logits/chosen": -0.1448793113231659, + "logits/rejected": -0.15905806422233582, + "logps/chosen": -447.693603515625, + "logps/ref_chosen": -62.442352294921875, + "logps/ref_rejected": -80.46806335449219, + "logps/rejected": -563.9413452148438, + "loss": 1.3647, + "margin_dpo/margin_mean": 98.2220230102539, + "margin_dpo/margin_std": 475.65869140625, + "step": 249 + }, + { + "KL/chosen_KL_mean": -364.223388671875, + "KL/mean": -443.6612548828125, + "KL/rejected_KL_mean": -523.0990600585938, + "KL/std": 361.03973388671875, + "epoch": 0.3671071953010279, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 158.87567138671875, + "fcm_dpo/q_t": 0.4896411597728729, + "grad_norm": 4.5081467628479, + "learning_rate": 4.006586590948141e-07, + "logits/chosen": -0.1620238721370697, + "logits/rejected": -0.17770114541053772, + "logps/chosen": -429.860107421875, + "logps/ref_chosen": -65.63668823242188, + "logps/ref_rejected": -73.87184143066406, + "logps/rejected": -596.970947265625, + "loss": 1.3485, + "margin_dpo/margin_mean": 158.87567138671875, + "margin_dpo/margin_std": 429.6935729980469, + "step": 250 + }, + { + "KL/chosen_KL_mean": -367.9019470214844, + "KL/mean": -433.2972412109375, + "KL/rejected_KL_mean": -498.6925048828125, + "KL/std": 377.6723327636719, + "epoch": 0.368575624082232, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 130.79058837890625, + "fcm_dpo/q_t": 0.4915042221546173, + "grad_norm": 5.190096855163574, + "learning_rate": 3.9963255888117325e-07, + "logits/chosen": -0.17455437779426575, + "logits/rejected": -0.18709741532802582, + "logps/chosen": -425.08465576171875, + "logps/ref_chosen": -57.182716369628906, + "logps/ref_rejected": -77.66343688964844, + "logps/rejected": -576.35595703125, + "loss": 1.356, + "margin_dpo/margin_mean": 130.79058837890625, + "margin_dpo/margin_std": 441.9692077636719, + "step": 251 + }, + { + "KL/chosen_KL_mean": -409.4916687011719, + "KL/mean": -503.2325439453125, + "KL/rejected_KL_mean": -596.973388671875, + "KL/std": 341.1922302246094, + "epoch": 0.3700440528634361, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 187.481689453125, + "fcm_dpo/q_t": 0.48779380321502686, + "grad_norm": 5.0120744705200195, + "learning_rate": 3.9860251571044666e-07, + "logits/chosen": -0.15551039576530457, + "logits/rejected": -0.14695081114768982, + "logps/chosen": -481.17730712890625, + "logps/ref_chosen": -71.68563842773438, + "logps/ref_rejected": -84.75799560546875, + "logps/rejected": -681.7313232421875, + "loss": 1.3404, + "margin_dpo/margin_mean": 187.481689453125, + "margin_dpo/margin_std": 378.55426025390625, + "step": 252 + }, + { + "KL/chosen_KL_mean": -384.7890625, + "KL/mean": -510.7060546875, + "KL/rejected_KL_mean": -636.6231079101562, + "KL/std": 366.9384765625, + "epoch": 0.37151248164464024, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 251.8341064453125, + "fcm_dpo/q_t": 0.4836481809616089, + "grad_norm": 5.239863872528076, + "learning_rate": 3.9756855672522986e-07, + "logits/chosen": -0.24072858691215515, + "logits/rejected": -0.24128055572509766, + "logps/chosen": -453.9229736328125, + "logps/ref_chosen": -69.1339340209961, + "logps/ref_rejected": -98.70252990722656, + "logps/rejected": -735.32568359375, + "loss": 1.3252, + "margin_dpo/margin_mean": 251.8341064453125, + "margin_dpo/margin_std": 442.5596923828125, + "step": 253 + }, + { + "KL/chosen_KL_mean": -414.69989013671875, + "KL/mean": -491.3453369140625, + "KL/rejected_KL_mean": -567.9908447265625, + "KL/std": 399.92486572265625, + "epoch": 0.37298091042584436, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 153.29092407226562, + "fcm_dpo/q_t": 0.49005135893821716, + "grad_norm": 6.106077194213867, + "learning_rate": 3.965307091713037e-07, + "logits/chosen": -0.17732489109039307, + "logits/rejected": -0.17891092598438263, + "logps/chosen": -468.8548889160156, + "logps/ref_chosen": -54.154998779296875, + "logps/ref_rejected": -90.30764770507812, + "logps/rejected": -658.2984619140625, + "loss": 1.3504, + "margin_dpo/margin_mean": 153.29092407226562, + "margin_dpo/margin_std": 465.193115234375, + "step": 254 + }, + { + "KL/chosen_KL_mean": -445.1566162109375, + "KL/mean": -541.7379150390625, + "KL/rejected_KL_mean": -638.3192138671875, + "KL/std": 423.0499572753906, + "epoch": 0.3744493392070485, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 193.16268920898438, + "fcm_dpo/q_t": 0.4874666929244995, + "grad_norm": 7.234618663787842, + "learning_rate": 3.954890003969163e-07, + "logits/chosen": -0.20868191123008728, + "logits/rejected": -0.20996426045894623, + "logps/chosen": -502.29827880859375, + "logps/ref_chosen": -57.14167022705078, + "logps/ref_rejected": -90.2085952758789, + "logps/rejected": -728.52783203125, + "loss": 1.3419, + "margin_dpo/margin_mean": 193.16268920898438, + "margin_dpo/margin_std": 552.7227783203125, + "step": 255 + }, + { + "KL/chosen_KL_mean": -393.31829833984375, + "KL/mean": -517.4535522460938, + "KL/rejected_KL_mean": -641.5887451171875, + "KL/std": 461.53656005859375, + "epoch": 0.37591776798825255, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 248.27044677734375, + "fcm_dpo/q_t": 0.48388510942459106, + "grad_norm": 5.720200061798096, + "learning_rate": 3.944434578520628e-07, + "logits/chosen": -0.2003256380558014, + "logits/rejected": -0.22089025378227234, + "logps/chosen": -448.4818115234375, + "logps/ref_chosen": -55.163490295410156, + "logps/ref_rejected": -92.56291961669922, + "logps/rejected": -734.1517333984375, + "loss": 1.327, + "margin_dpo/margin_mean": 248.2704315185547, + "margin_dpo/margin_std": 509.9310607910156, + "step": 256 + }, + { + "KL/chosen_KL_mean": -340.8580322265625, + "KL/mean": -473.32391357421875, + "KL/rejected_KL_mean": -605.7897338867188, + "KL/std": 385.75244140625, + "epoch": 0.37738619676945667, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 264.93170166015625, + "fcm_dpo/q_t": 0.48283612728118896, + "grad_norm": 6.3806633949279785, + "learning_rate": 3.933941090877615e-07, + "logits/chosen": -0.18380647897720337, + "logits/rejected": -0.1893559843301773, + "logps/chosen": -390.28173828125, + "logps/ref_chosen": -49.42369842529297, + "logps/ref_rejected": -79.53791809082031, + "logps/rejected": -685.32763671875, + "loss": 1.3228, + "margin_dpo/margin_mean": 264.9317321777344, + "margin_dpo/margin_std": 493.8131103515625, + "step": 257 + }, + { + "KL/chosen_KL_mean": -456.58740234375, + "KL/mean": -573.7291259765625, + "KL/rejected_KL_mean": -690.870849609375, + "KL/std": 422.5630187988281, + "epoch": 0.3788546255506608, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 234.2834014892578, + "fcm_dpo/q_t": 0.4847896099090576, + "grad_norm": 6.556681156158447, + "learning_rate": 3.923409817553284e-07, + "logits/chosen": -0.16496992111206055, + "logits/rejected": -0.16959482431411743, + "logps/chosen": -515.9715576171875, + "logps/ref_chosen": -59.384124755859375, + "logps/ref_rejected": -95.99010467529297, + "logps/rejected": -786.8609619140625, + "loss": 1.331, + "margin_dpo/margin_mean": 234.2834014892578, + "margin_dpo/margin_std": 536.2554321289062, + "step": 258 + }, + { + "KL/chosen_KL_mean": -437.74139404296875, + "KL/mean": -527.7792358398438, + "KL/rejected_KL_mean": -617.8170776367188, + "KL/std": 397.44317626953125, + "epoch": 0.3803230543318649, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 180.0756378173828, + "fcm_dpo/q_t": 0.4883081018924713, + "grad_norm": 5.156373500823975, + "learning_rate": 3.9128410360564793e-07, + "logits/chosen": -0.2657771706581116, + "logits/rejected": -0.2764046788215637, + "logps/chosen": -490.56976318359375, + "logps/ref_chosen": -52.828346252441406, + "logps/ref_rejected": -89.191650390625, + "logps/rejected": -707.0087280273438, + "loss": 1.3441, + "margin_dpo/margin_mean": 180.0756378173828, + "margin_dpo/margin_std": 496.28350830078125, + "step": 259 + }, + { + "KL/chosen_KL_mean": -408.2018737792969, + "KL/mean": -567.6690673828125, + "KL/rejected_KL_mean": -727.13623046875, + "KL/std": 518.0980224609375, + "epoch": 0.38179148311306904, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 318.9344177246094, + "fcm_dpo/q_t": 0.47985440492630005, + "grad_norm": 6.141451835632324, + "learning_rate": 3.9022350248844246e-07, + "logits/chosen": -0.21567201614379883, + "logits/rejected": -0.22696366906166077, + "logps/chosen": -455.61956787109375, + "logps/ref_chosen": -47.41767501831055, + "logps/ref_rejected": -95.08978271484375, + "logps/rejected": -822.22607421875, + "loss": 1.3125, + "margin_dpo/margin_mean": 318.9344177246094, + "margin_dpo/margin_std": 634.315673828125, + "step": 260 + }, + { + "KL/chosen_KL_mean": -453.39117431640625, + "KL/mean": -568.876953125, + "KL/rejected_KL_mean": -684.36279296875, + "KL/std": 498.5777587890625, + "epoch": 0.3832599118942731, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 230.97158813476562, + "fcm_dpo/q_t": 0.48512423038482666, + "grad_norm": 6.820346832275391, + "learning_rate": 3.891592063515376e-07, + "logits/chosen": -0.19904325902462006, + "logits/rejected": -0.212869793176651, + "logps/chosen": -506.4225769042969, + "logps/ref_chosen": -53.03137969970703, + "logps/ref_rejected": -88.51494598388672, + "logps/rejected": -772.877685546875, + "loss": 1.3336, + "margin_dpo/margin_mean": 230.9715576171875, + "margin_dpo/margin_std": 618.2874755859375, + "step": 261 + }, + { + "KL/chosen_KL_mean": -557.084228515625, + "KL/mean": -635.599853515625, + "KL/rejected_KL_mean": -714.1155395507812, + "KL/std": 484.9010314941406, + "epoch": 0.38472834067547723, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 157.03131103515625, + "fcm_dpo/q_t": 0.48986271023750305, + "grad_norm": 11.662672996520996, + "learning_rate": 3.880912432401264e-07, + "logits/chosen": -0.2308690845966339, + "logits/rejected": -0.2301132082939148, + "logps/chosen": -616.704345703125, + "logps/ref_chosen": -59.620140075683594, + "logps/ref_rejected": -86.41853332519531, + "logps/rejected": -800.5340576171875, + "loss": 1.3504, + "margin_dpo/margin_mean": 157.03131103515625, + "margin_dpo/margin_std": 523.501953125, + "step": 262 + }, + { + "KL/chosen_KL_mean": -488.0692138671875, + "KL/mean": -623.00439453125, + "KL/rejected_KL_mean": -757.9396362304688, + "KL/std": 583.6214599609375, + "epoch": 0.38619676945668135, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 269.87042236328125, + "fcm_dpo/q_t": 0.4825875461101532, + "grad_norm": 8.312024116516113, + "learning_rate": 3.870196412960302e-07, + "logits/chosen": -0.22481516003608704, + "logits/rejected": -0.23364132642745972, + "logps/chosen": -547.4901733398438, + "logps/ref_chosen": -59.42094421386719, + "logps/ref_rejected": -96.85720825195312, + "logps/rejected": -854.796875, + "loss": 1.3262, + "margin_dpo/margin_mean": 269.87042236328125, + "margin_dpo/margin_std": 712.5626831054688, + "step": 263 + }, + { + "KL/chosen_KL_mean": -497.3571472167969, + "KL/mean": -621.4713134765625, + "KL/rejected_KL_mean": -745.5855102539062, + "KL/std": 472.39825439453125, + "epoch": 0.3876651982378855, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 248.22833251953125, + "fcm_dpo/q_t": 0.48396652936935425, + "grad_norm": 6.751438617706299, + "learning_rate": 3.8594442875695665e-07, + "logits/chosen": -0.23678645491600037, + "logits/rejected": -0.2437746822834015, + "logps/chosen": -560.0792236328125, + "logps/ref_chosen": -62.722084045410156, + "logps/ref_rejected": -93.85620880126953, + "logps/rejected": -839.4417114257812, + "loss": 1.3291, + "margin_dpo/margin_mean": 248.22830200195312, + "margin_dpo/margin_std": 611.2507934570312, + "step": 264 + }, + { + "KL/chosen_KL_mean": -507.474365234375, + "KL/mean": -589.9117431640625, + "KL/rejected_KL_mean": -672.34912109375, + "KL/std": 430.08966064453125, + "epoch": 0.3891336270190896, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 164.87478637695312, + "fcm_dpo/q_t": 0.48930346965789795, + "grad_norm": 7.545551300048828, + "learning_rate": 3.848656339557562e-07, + "logits/chosen": -0.15824304521083832, + "logits/rejected": -0.16281482577323914, + "logps/chosen": -569.44580078125, + "logps/ref_chosen": -61.971466064453125, + "logps/ref_rejected": -88.02059936523438, + "logps/rejected": -760.3697509765625, + "loss": 1.3489, + "margin_dpo/margin_mean": 164.8748016357422, + "margin_dpo/margin_std": 550.4019165039062, + "step": 265 + }, + { + "KL/chosen_KL_mean": -501.43597412109375, + "KL/mean": -529.8870849609375, + "KL/rejected_KL_mean": -558.3381958007812, + "KL/std": 390.973388671875, + "epoch": 0.39060205580029367, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 56.9022102355957, + "fcm_dpo/q_t": 0.4962887465953827, + "grad_norm": 11.123693466186523, + "learning_rate": 3.8378328531967507e-07, + "logits/chosen": -0.17855733633041382, + "logits/rejected": -0.19599322974681854, + "logps/chosen": -568.53564453125, + "logps/ref_chosen": -67.09967041015625, + "logps/ref_rejected": -67.97122192382812, + "logps/rejected": -626.3094482421875, + "loss": 1.3757, + "margin_dpo/margin_mean": 56.90220260620117, + "margin_dpo/margin_std": 493.3840026855469, + "step": 266 + }, + { + "KL/chosen_KL_mean": -404.4932556152344, + "KL/mean": -489.49261474609375, + "KL/rejected_KL_mean": -574.491943359375, + "KL/std": 411.52874755859375, + "epoch": 0.3920704845814978, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 169.99867248535156, + "fcm_dpo/q_t": 0.4889971613883972, + "grad_norm": 6.4354047775268555, + "learning_rate": 3.8269741136960646e-07, + "logits/chosen": -0.15721634030342102, + "logits/rejected": -0.1611420214176178, + "logps/chosen": -473.4639892578125, + "logps/ref_chosen": -68.97075653076172, + "logps/ref_rejected": -90.16844940185547, + "logps/rejected": -664.660400390625, + "loss": 1.3474, + "margin_dpo/margin_mean": 169.99867248535156, + "margin_dpo/margin_std": 537.1679077148438, + "step": 267 + }, + { + "KL/chosen_KL_mean": -367.1216125488281, + "KL/mean": -490.97198486328125, + "KL/rejected_KL_mean": -614.822265625, + "KL/std": 461.61407470703125, + "epoch": 0.3935389133627019, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 247.70066833496094, + "fcm_dpo/q_t": 0.483919620513916, + "grad_norm": 8.824798583984375, + "learning_rate": 3.8160804071933894e-07, + "logits/chosen": -0.13857878744602203, + "logits/rejected": -0.136179119348526, + "logps/chosen": -423.02191162109375, + "logps/ref_chosen": -55.90031051635742, + "logps/ref_rejected": -101.64763641357422, + "logps/rejected": -716.469970703125, + "loss": 1.3277, + "margin_dpo/margin_mean": 247.70066833496094, + "margin_dpo/margin_std": 543.2017211914062, + "step": 268 + }, + { + "KL/chosen_KL_mean": -424.86187744140625, + "KL/mean": -545.9503173828125, + "KL/rejected_KL_mean": -667.038818359375, + "KL/std": 403.55706787109375, + "epoch": 0.39500734214390604, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 242.17694091796875, + "fcm_dpo/q_t": 0.4843018651008606, + "grad_norm": 8.456543922424316, + "learning_rate": 3.8051520207480204e-07, + "logits/chosen": -0.16604793071746826, + "logits/rejected": -0.18448489904403687, + "logps/chosen": -494.90142822265625, + "logps/ref_chosen": -70.03955841064453, + "logps/ref_rejected": -107.34937286376953, + "logps/rejected": -774.38818359375, + "loss": 1.3295, + "margin_dpo/margin_mean": 242.1769256591797, + "margin_dpo/margin_std": 559.4677124023438, + "step": 269 + }, + { + "KL/chosen_KL_mean": -406.35626220703125, + "KL/mean": -518.3023071289062, + "KL/rejected_KL_mean": -630.248291015625, + "KL/std": 403.0386962890625, + "epoch": 0.3964757709251101, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 223.89202880859375, + "fcm_dpo/q_t": 0.4855157136917114, + "grad_norm": 6.902976036071777, + "learning_rate": 3.794189242333106e-07, + "logits/chosen": -0.18907925486564636, + "logits/rejected": -0.20055195689201355, + "logps/chosen": -475.8897705078125, + "logps/ref_chosen": -69.53347778320312, + "logps/ref_rejected": -109.92864990234375, + "logps/rejected": -740.1769409179688, + "loss": 1.3342, + "margin_dpo/margin_mean": 223.89202880859375, + "margin_dpo/margin_std": 561.119384765625, + "step": 270 + }, + { + "KL/chosen_KL_mean": -332.3415832519531, + "KL/mean": -452.5819091796875, + "KL/rejected_KL_mean": -572.822265625, + "KL/std": 376.9450378417969, + "epoch": 0.39794419970631423, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 240.48062133789062, + "fcm_dpo/q_t": 0.484383761882782, + "grad_norm": 7.943375110626221, + "learning_rate": 3.7831923608280514e-07, + "logits/chosen": -0.2072502076625824, + "logits/rejected": -0.2298785150051117, + "logps/chosen": -389.10614013671875, + "logps/ref_chosen": -56.76456832885742, + "logps/ref_rejected": -92.51383972167969, + "logps/rejected": -665.3360595703125, + "loss": 1.3284, + "margin_dpo/margin_mean": 240.48062133789062, + "margin_dpo/margin_std": 473.6417236328125, + "step": 271 + }, + { + "KL/chosen_KL_mean": -324.92218017578125, + "KL/mean": -495.7729187011719, + "KL/rejected_KL_mean": -666.6236572265625, + "KL/std": 421.35009765625, + "epoch": 0.39941262848751835, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 341.70147705078125, + "fcm_dpo/q_t": 0.4778040647506714, + "grad_norm": 15.071242332458496, + "learning_rate": 3.772161666010912e-07, + "logits/chosen": -0.19679825007915497, + "logits/rejected": -0.20816272497177124, + "logps/chosen": -374.41937255859375, + "logps/ref_chosen": -49.497154235839844, + "logps/ref_rejected": -105.54279327392578, + "logps/rejected": -772.16650390625, + "loss": 1.3028, + "margin_dpo/margin_mean": 341.70147705078125, + "margin_dpo/margin_std": 460.557373046875, + "step": 272 + }, + { + "KL/chosen_KL_mean": -434.1469421386719, + "KL/mean": -575.2939453125, + "KL/rejected_KL_mean": -716.4408569335938, + "KL/std": 456.7916259765625, + "epoch": 0.4008810572687225, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 282.29388427734375, + "fcm_dpo/q_t": 0.4816928505897522, + "grad_norm": 7.3902764320373535, + "learning_rate": 3.761097448550755e-07, + "logits/chosen": -0.26582562923431396, + "logits/rejected": -0.2689433991909027, + "logps/chosen": -497.1223449707031, + "logps/ref_chosen": -62.97539520263672, + "logps/ref_rejected": -92.49858093261719, + "logps/rejected": -808.939453125, + "loss": 1.3187, + "margin_dpo/margin_mean": 282.2939147949219, + "margin_dpo/margin_std": 524.8466796875, + "step": 273 + }, + { + "KL/chosen_KL_mean": -461.13970947265625, + "KL/mean": -559.404052734375, + "KL/rejected_KL_mean": -657.66845703125, + "KL/std": 382.537841796875, + "epoch": 0.4023494860499266, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 196.5286865234375, + "fcm_dpo/q_t": 0.48726481199264526, + "grad_norm": 8.061701774597168, + "learning_rate": 3.75e-07, + "logits/chosen": -0.2275162637233734, + "logits/rejected": -0.22361652553081512, + "logps/chosen": -516.8074340820312, + "logps/ref_chosen": -55.66770935058594, + "logps/ref_rejected": -77.33308410644531, + "logps/rejected": -735.00146484375, + "loss": 1.3397, + "margin_dpo/margin_mean": 196.52871704101562, + "margin_dpo/margin_std": 475.701171875, + "step": 274 + }, + { + "KL/chosen_KL_mean": -403.9293518066406, + "KL/mean": -539.42724609375, + "KL/rejected_KL_mean": -674.9251708984375, + "KL/std": 428.0262145996094, + "epoch": 0.40381791483113066, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 270.99578857421875, + "fcm_dpo/q_t": 0.4824160635471344, + "grad_norm": 9.460725784301758, + "learning_rate": 3.738869612786737e-07, + "logits/chosen": -0.27084943652153015, + "logits/rejected": -0.2707024812698364, + "logps/chosen": -452.5240478515625, + "logps/ref_chosen": -48.594703674316406, + "logps/ref_rejected": -93.30369567871094, + "logps/rejected": -768.2288818359375, + "loss": 1.3215, + "margin_dpo/margin_mean": 270.99578857421875, + "margin_dpo/margin_std": 521.1408081054688, + "step": 275 + }, + { + "KL/chosen_KL_mean": -461.385986328125, + "KL/mean": -551.3458862304688, + "KL/rejected_KL_mean": -641.3057861328125, + "KL/std": 443.9175109863281, + "epoch": 0.4052863436123348, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 179.91978454589844, + "fcm_dpo/q_t": 0.48834025859832764, + "grad_norm": 5.902976036071777, + "learning_rate": 3.7277065802070204e-07, + "logits/chosen": -0.275311142206192, + "logits/rejected": -0.2855873107910156, + "logps/chosen": -517.96337890625, + "logps/ref_chosen": -56.57740783691406, + "logps/ref_rejected": -70.36566925048828, + "logps/rejected": -711.6714477539062, + "loss": 1.3466, + "margin_dpo/margin_mean": 179.9197998046875, + "margin_dpo/margin_std": 613.080810546875, + "step": 276 + }, + { + "KL/chosen_KL_mean": -490.01446533203125, + "KL/mean": -628.3453979492188, + "KL/rejected_KL_mean": -766.67626953125, + "KL/std": 460.1982421875, + "epoch": 0.4067547723935389, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 276.66180419921875, + "fcm_dpo/q_t": 0.4820600152015686, + "grad_norm": 5.049780368804932, + "learning_rate": 3.71651119641714e-07, + "logits/chosen": -0.2941455841064453, + "logits/rejected": -0.2855493426322937, + "logps/chosen": -546.2860107421875, + "logps/ref_chosen": -56.27156066894531, + "logps/ref_rejected": -92.88127136230469, + "logps/rejected": -859.5575561523438, + "loss": 1.3207, + "margin_dpo/margin_mean": 276.66180419921875, + "margin_dpo/margin_std": 558.4758911132812, + "step": 277 + }, + { + "KL/chosen_KL_mean": -490.9912109375, + "KL/mean": -647.95458984375, + "KL/rejected_KL_mean": -804.91796875, + "KL/std": 546.8567504882812, + "epoch": 0.40822320117474303, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 313.92669677734375, + "fcm_dpo/q_t": 0.4797661304473877, + "grad_norm": 7.039979934692383, + "learning_rate": 3.705283756425872e-07, + "logits/chosen": -0.26116132736206055, + "logits/rejected": -0.2724123001098633, + "logps/chosen": -543.93310546875, + "logps/ref_chosen": -52.94194030761719, + "logps/ref_rejected": -91.25357818603516, + "logps/rejected": -896.1715087890625, + "loss": 1.3148, + "margin_dpo/margin_mean": 313.92669677734375, + "margin_dpo/margin_std": 714.0025634765625, + "step": 278 + }, + { + "KL/chosen_KL_mean": -528.8704833984375, + "KL/mean": -699.5042724609375, + "KL/rejected_KL_mean": -870.1380615234375, + "KL/std": 579.2550659179688, + "epoch": 0.40969162995594716, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 341.2675476074219, + "fcm_dpo/q_t": 0.47832390666007996, + "grad_norm": 8.836512565612793, + "learning_rate": 3.6940245560867e-07, + "logits/chosen": -0.2435876727104187, + "logits/rejected": -0.2540760934352875, + "logps/chosen": -577.5117797851562, + "logps/ref_chosen": -48.641319274902344, + "logps/ref_rejected": -87.8514404296875, + "logps/rejected": -957.989501953125, + "loss": 1.3106, + "margin_dpo/margin_mean": 341.26751708984375, + "margin_dpo/margin_std": 805.846435546875, + "step": 279 + }, + { + "KL/chosen_KL_mean": -519.3259887695312, + "KL/mean": -680.1437377929688, + "KL/rejected_KL_mean": -840.96142578125, + "KL/std": 494.4556884765625, + "epoch": 0.4111600587371512, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 321.635498046875, + "fcm_dpo/q_t": 0.4792141318321228, + "grad_norm": 7.04791784286499, + "learning_rate": 3.6827338920900253e-07, + "logits/chosen": -0.1967657059431076, + "logits/rejected": -0.21048101782798767, + "logps/chosen": -578.1231079101562, + "logps/ref_chosen": -58.797122955322266, + "logps/ref_rejected": -98.61885070800781, + "logps/rejected": -939.580322265625, + "loss": 1.3113, + "margin_dpo/margin_mean": 321.635498046875, + "margin_dpo/margin_std": 650.3975830078125, + "step": 280 + }, + { + "KL/chosen_KL_mean": -519.2962646484375, + "KL/mean": -635.3290405273438, + "KL/rejected_KL_mean": -751.3618774414062, + "KL/std": 475.23504638671875, + "epoch": 0.41262848751835535, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 232.0655975341797, + "fcm_dpo/q_t": 0.48489832878112793, + "grad_norm": 7.778337478637695, + "learning_rate": 3.6714120619553435e-07, + "logits/chosen": -0.19264814257621765, + "logits/rejected": -0.20138809084892273, + "logps/chosen": -574.7847900390625, + "logps/ref_chosen": -55.488521575927734, + "logps/ref_rejected": -80.88258361816406, + "logps/rejected": -832.2444458007812, + "loss": 1.3332, + "margin_dpo/margin_mean": 232.0655975341797, + "margin_dpo/margin_std": 605.12109375, + "step": 281 + }, + { + "KL/chosen_KL_mean": -589.7445068359375, + "KL/mean": -695.1300048828125, + "KL/rejected_KL_mean": -800.515380859375, + "KL/std": 656.7650146484375, + "epoch": 0.41409691629955947, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 210.77078247070312, + "fcm_dpo/q_t": 0.4869770109653473, + "grad_norm": 6.628312110900879, + "learning_rate": 3.660059364023408e-07, + "logits/chosen": -0.22694750130176544, + "logits/rejected": -0.23224890232086182, + "logps/chosen": -662.814697265625, + "logps/ref_chosen": -73.07014465332031, + "logps/ref_rejected": -95.35098266601562, + "logps/rejected": -895.8663330078125, + "loss": 1.3427, + "margin_dpo/margin_mean": 210.7707977294922, + "margin_dpo/margin_std": 778.447998046875, + "step": 282 + }, + { + "KL/chosen_KL_mean": -562.0560913085938, + "KL/mean": -729.2489013671875, + "KL/rejected_KL_mean": -896.4417724609375, + "KL/std": 537.7421264648438, + "epoch": 0.4155653450807636, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 334.38568115234375, + "fcm_dpo/q_t": 0.47830215096473694, + "grad_norm": 8.030158996582031, + "learning_rate": 3.6486760974483685e-07, + "logits/chosen": -0.15689605474472046, + "logits/rejected": -0.16659200191497803, + "logps/chosen": -623.9545288085938, + "logps/ref_chosen": -61.89844512939453, + "logps/ref_rejected": -96.98655700683594, + "logps/rejected": -993.4283447265625, + "loss": 1.3066, + "margin_dpo/margin_mean": 334.38568115234375, + "margin_dpo/margin_std": 558.3077392578125, + "step": 283 + }, + { + "KL/chosen_KL_mean": -445.6246337890625, + "KL/mean": -596.072509765625, + "KL/rejected_KL_mean": -746.5203857421875, + "KL/std": 443.46435546875, + "epoch": 0.4170337738619677, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 300.89569091796875, + "fcm_dpo/q_t": 0.48052817583084106, + "grad_norm": 7.716520309448242, + "learning_rate": 3.6372625621898863e-07, + "logits/chosen": -0.22630912065505981, + "logits/rejected": -0.2253245860338211, + "logps/chosen": -504.0601806640625, + "logps/ref_chosen": -58.4355354309082, + "logps/ref_rejected": -93.46926879882812, + "logps/rejected": -839.9896240234375, + "loss": 1.3141, + "margin_dpo/margin_mean": 300.89569091796875, + "margin_dpo/margin_std": 518.4878540039062, + "step": 284 + }, + { + "KL/chosen_KL_mean": -528.9168090820312, + "KL/mean": -623.526611328125, + "KL/rejected_KL_mean": -718.1364135742188, + "KL/std": 397.197509765625, + "epoch": 0.4185022026431718, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 189.2196502685547, + "fcm_dpo/q_t": 0.4876604974269867, + "grad_norm": 5.944766044616699, + "learning_rate": 3.625819059005228e-07, + "logits/chosen": -0.1648595631122589, + "logits/rejected": -0.16167069971561432, + "logps/chosen": -595.1489868164062, + "logps/ref_chosen": -66.23219299316406, + "logps/ref_rejected": -99.1268310546875, + "logps/rejected": -817.2632446289062, + "loss": 1.3422, + "margin_dpo/margin_mean": 189.2196502685547, + "margin_dpo/margin_std": 515.746337890625, + "step": 285 + }, + { + "KL/chosen_KL_mean": -525.88720703125, + "KL/mean": -641.6607666015625, + "KL/rejected_KL_mean": -757.434326171875, + "KL/std": 519.7754516601562, + "epoch": 0.4199706314243759, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 231.547119140625, + "fcm_dpo/q_t": 0.4849644899368286, + "grad_norm": 6.175311088562012, + "learning_rate": 3.614345889441346e-07, + "logits/chosen": -0.17296646535396576, + "logits/rejected": -0.17703449726104736, + "logps/chosen": -598.8381958007812, + "logps/ref_chosen": -72.95100402832031, + "logps/ref_rejected": -88.58845520019531, + "logps/rejected": -846.022705078125, + "loss": 1.3339, + "margin_dpo/margin_mean": 231.54708862304688, + "margin_dpo/margin_std": 646.5220336914062, + "step": 286 + }, + { + "KL/chosen_KL_mean": -439.7398986816406, + "KL/mean": -516.02294921875, + "KL/rejected_KL_mean": -592.3060302734375, + "KL/std": 409.7009582519531, + "epoch": 0.42143906020558003, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 152.56610107421875, + "fcm_dpo/q_t": 0.49012070894241333, + "grad_norm": 5.386925220489502, + "learning_rate": 3.6028433558269275e-07, + "logits/chosen": -0.18158751726150513, + "logits/rejected": -0.1914394050836563, + "logps/chosen": -501.28106689453125, + "logps/ref_chosen": -61.54115295410156, + "logps/ref_rejected": -77.69607543945312, + "logps/rejected": -670.0020751953125, + "loss": 1.3512, + "margin_dpo/margin_mean": 152.56613159179688, + "margin_dpo/margin_std": 495.0728759765625, + "step": 287 + }, + { + "KL/chosen_KL_mean": -347.9739074707031, + "KL/mean": -471.96112060546875, + "KL/rejected_KL_mean": -595.9483642578125, + "KL/std": 354.6069030761719, + "epoch": 0.42290748898678415, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 247.97445678710938, + "fcm_dpo/q_t": 0.48386895656585693, + "grad_norm": 5.900707244873047, + "learning_rate": 3.5913117612644327e-07, + "logits/chosen": -0.20927947759628296, + "logits/rejected": -0.21391412615776062, + "logps/chosen": -404.6351318359375, + "logps/ref_chosen": -56.661224365234375, + "logps/ref_rejected": -87.33570098876953, + "logps/rejected": -683.2840576171875, + "loss": 1.3261, + "margin_dpo/margin_mean": 247.97445678710938, + "margin_dpo/margin_std": 437.38543701171875, + "step": 288 + }, + { + "KL/chosen_KL_mean": -309.76373291015625, + "KL/mean": -466.19610595703125, + "KL/rejected_KL_mean": -622.6285400390625, + "KL/std": 418.3388977050781, + "epoch": 0.4243759177679883, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 312.86480712890625, + "fcm_dpo/q_t": 0.4797741770744324, + "grad_norm": 7.995468616485596, + "learning_rate": 3.5797514096221024e-07, + "logits/chosen": -0.15004077553749084, + "logits/rejected": -0.16116581857204437, + "logps/chosen": -354.9941101074219, + "logps/ref_chosen": -45.23039245605469, + "logps/ref_rejected": -87.64266967773438, + "logps/rejected": -710.2711791992188, + "loss": 1.3106, + "margin_dpo/margin_mean": 312.86480712890625, + "margin_dpo/margin_std": 488.91461181640625, + "step": 289 + }, + { + "KL/chosen_KL_mean": -354.0469055175781, + "KL/mean": -538.64892578125, + "KL/rejected_KL_mean": -723.2508544921875, + "KL/std": 475.892822265625, + "epoch": 0.42584434654919234, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 369.20404052734375, + "fcm_dpo/q_t": 0.4761792719364166, + "grad_norm": 10.0774564743042, + "learning_rate": 3.568162605525952e-07, + "logits/chosen": -0.1508667916059494, + "logits/rejected": -0.17154675722122192, + "logps/chosen": -409.5184020996094, + "logps/ref_chosen": -55.47149658203125, + "logps/ref_rejected": -116.70857238769531, + "logps/rejected": -839.95947265625, + "loss": 1.2979, + "margin_dpo/margin_mean": 369.2040100097656, + "margin_dpo/margin_std": 568.3728637695312, + "step": 290 + }, + { + "KL/chosen_KL_mean": -348.29998779296875, + "KL/mean": -480.74700927734375, + "KL/rejected_KL_mean": -613.194091796875, + "KL/std": 381.81988525390625, + "epoch": 0.42731277533039647, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 264.89410400390625, + "fcm_dpo/q_t": 0.48280757665634155, + "grad_norm": 7.444272994995117, + "learning_rate": 3.5565456543517485e-07, + "logits/chosen": -0.1743829846382141, + "logits/rejected": -0.18439102172851562, + "logps/chosen": -411.56036376953125, + "logps/ref_chosen": -63.26036834716797, + "logps/ref_rejected": -89.29708862304688, + "logps/rejected": -702.4911499023438, + "loss": 1.3228, + "margin_dpo/margin_mean": 264.89410400390625, + "margin_dpo/margin_std": 509.3777160644531, + "step": 291 + }, + { + "KL/chosen_KL_mean": -375.5555419921875, + "KL/mean": -526.6381225585938, + "KL/rejected_KL_mean": -677.7206420898438, + "KL/std": 419.32891845703125, + "epoch": 0.4287812041116006, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 302.1651306152344, + "fcm_dpo/q_t": 0.480354905128479, + "grad_norm": 6.041011333465576, + "learning_rate": 3.5449008622169583e-07, + "logits/chosen": -0.2032952606678009, + "logits/rejected": -0.21287089586257935, + "logps/chosen": -429.47406005859375, + "logps/ref_chosen": -53.91852951049805, + "logps/ref_rejected": -89.96138000488281, + "logps/rejected": -767.6820068359375, + "loss": 1.3134, + "margin_dpo/margin_mean": 302.1651306152344, + "margin_dpo/margin_std": 505.92315673828125, + "step": 292 + }, + { + "KL/chosen_KL_mean": -508.30621337890625, + "KL/mean": -616.5218505859375, + "KL/rejected_KL_mean": -724.7373657226562, + "KL/std": 533.0728759765625, + "epoch": 0.4302496328928047, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 216.43118286132812, + "fcm_dpo/q_t": 0.48600873351097107, + "grad_norm": 6.5855913162231445, + "learning_rate": 3.5332285359726846e-07, + "logits/chosen": -0.21148748695850372, + "logits/rejected": -0.22140976786613464, + "logps/chosen": -568.6822509765625, + "logps/ref_chosen": -60.376033782958984, + "logps/ref_rejected": -77.85244750976562, + "logps/rejected": -802.58984375, + "loss": 1.3359, + "margin_dpo/margin_mean": 216.43116760253906, + "margin_dpo/margin_std": 558.06982421875, + "step": 293 + }, + { + "KL/chosen_KL_mean": -426.6471862792969, + "KL/mean": -560.2008666992188, + "KL/rejected_KL_mean": -693.7545776367188, + "KL/std": 453.28826904296875, + "epoch": 0.43171806167400884, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 267.1073303222656, + "fcm_dpo/q_t": 0.48279574513435364, + "grad_norm": 7.555781841278076, + "learning_rate": 3.5215289831955786e-07, + "logits/chosen": -0.24115553498268127, + "logits/rejected": -0.25332289934158325, + "logps/chosen": -474.7347106933594, + "logps/ref_chosen": -48.0875358581543, + "logps/ref_rejected": -81.89698791503906, + "logps/rejected": -775.6515502929688, + "loss": 1.3246, + "margin_dpo/margin_mean": 267.1073303222656, + "margin_dpo/margin_std": 627.17822265625, + "step": 294 + }, + { + "KL/chosen_KL_mean": -553.238037109375, + "KL/mean": -707.3406982421875, + "KL/rejected_KL_mean": -861.443359375, + "KL/std": 558.95654296875, + "epoch": 0.4331864904552129, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 308.205322265625, + "fcm_dpo/q_t": 0.4802568554878235, + "grad_norm": 6.732223987579346, + "learning_rate": 3.509802512179737e-07, + "logits/chosen": -0.2826892137527466, + "logits/rejected": -0.28627097606658936, + "logps/chosen": -603.1626586914062, + "logps/ref_chosen": -49.92467498779297, + "logps/ref_rejected": -87.45632934570312, + "logps/rejected": -948.899658203125, + "loss": 1.3164, + "margin_dpo/margin_mean": 308.205322265625, + "margin_dpo/margin_std": 724.8931884765625, + "step": 295 + }, + { + "KL/chosen_KL_mean": -672.35009765625, + "KL/mean": -775.7928466796875, + "KL/rejected_KL_mean": -879.235595703125, + "KL/std": 503.78070068359375, + "epoch": 0.434654919236417, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 206.88555908203125, + "fcm_dpo/q_t": 0.486669659614563, + "grad_norm": 8.33739185333252, + "learning_rate": 3.498049431928577e-07, + "logits/chosen": -0.2496371865272522, + "logits/rejected": -0.24974963068962097, + "logps/chosen": -737.84130859375, + "logps/ref_chosen": -65.49124145507812, + "logps/ref_rejected": -93.08908081054688, + "logps/rejected": -972.32470703125, + "loss": 1.3406, + "margin_dpo/margin_mean": 206.88555908203125, + "margin_dpo/margin_std": 652.9727783203125, + "step": 296 + }, + { + "KL/chosen_KL_mean": -614.4139404296875, + "KL/mean": -749.4168090820312, + "KL/rejected_KL_mean": -884.419677734375, + "KL/std": 595.4171752929688, + "epoch": 0.43612334801762115, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 270.00567626953125, + "fcm_dpo/q_t": 0.4827141761779785, + "grad_norm": 8.87056827545166, + "learning_rate": 3.486270052146694e-07, + "logits/chosen": -0.2674667239189148, + "logits/rejected": -0.27443575859069824, + "logps/chosen": -670.890869140625, + "logps/ref_chosen": -56.476951599121094, + "logps/ref_rejected": -95.1385498046875, + "logps/rejected": -979.5582275390625, + "loss": 1.3246, + "margin_dpo/margin_mean": 270.00567626953125, + "margin_dpo/margin_std": 666.51513671875, + "step": 297 + }, + { + "KL/chosen_KL_mean": -678.1602783203125, + "KL/mean": -886.756591796875, + "KL/rejected_KL_mean": -1095.3529052734375, + "KL/std": 748.703857421875, + "epoch": 0.43759177679882527, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 417.192626953125, + "fcm_dpo/q_t": 0.4735215902328491, + "grad_norm": 8.397970199584961, + "learning_rate": 3.474464683231698e-07, + "logits/chosen": -0.2270020991563797, + "logits/rejected": -0.23736727237701416, + "logps/chosen": -745.4854736328125, + "logps/ref_chosen": -67.32516479492188, + "logps/ref_rejected": -116.66217041015625, + "logps/rejected": -1212.01513671875, + "loss": 1.2938, + "margin_dpo/margin_mean": 417.192626953125, + "margin_dpo/margin_std": 878.2392578125, + "step": 298 + }, + { + "KL/chosen_KL_mean": -549.8248291015625, + "KL/mean": -667.109130859375, + "KL/rejected_KL_mean": -784.3934936523438, + "KL/std": 495.10968017578125, + "epoch": 0.4390602055800294, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 234.56871032714844, + "fcm_dpo/q_t": 0.4848034977912903, + "grad_norm": 8.802879333496094, + "learning_rate": 3.462633636266041e-07, + "logits/chosen": -0.25002071261405945, + "logits/rejected": -0.2646872401237488, + "logps/chosen": -598.7869262695312, + "logps/ref_chosen": -48.96209716796875, + "logps/ref_rejected": -84.32823944091797, + "logps/rejected": -868.7217407226562, + "loss": 1.3337, + "margin_dpo/margin_mean": 234.5686798095703, + "margin_dpo/margin_std": 640.656982421875, + "step": 299 + }, + { + "KL/chosen_KL_mean": -645.182861328125, + "KL/mean": -834.207763671875, + "KL/rejected_KL_mean": -1023.232666015625, + "KL/std": 587.900390625, + "epoch": 0.44052863436123346, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 378.0498352050781, + "fcm_dpo/q_t": 0.4755672812461853, + "grad_norm": 7.947546005249023, + "learning_rate": 3.4507772230088147e-07, + "logits/chosen": -0.1696600764989853, + "logits/rejected": -0.18944713473320007, + "logps/chosen": -704.256591796875, + "logps/ref_chosen": -59.07371139526367, + "logps/ref_rejected": -95.9664535522461, + "logps/rejected": -1119.1990966796875, + "loss": 1.2997, + "margin_dpo/margin_mean": 378.0498046875, + "margin_dpo/margin_std": 751.8477783203125, + "step": 300 + }, + { + "KL/chosen_KL_mean": -525.6502685546875, + "KL/mean": -696.746337890625, + "KL/rejected_KL_mean": -867.8424072265625, + "KL/std": 537.3768310546875, + "epoch": 0.4419970631424376, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 342.19207763671875, + "fcm_dpo/q_t": 0.47806602716445923, + "grad_norm": 7.660505771636963, + "learning_rate": 3.4388957558875316e-07, + "logits/chosen": -0.19091829657554626, + "logits/rejected": -0.20372003316879272, + "logps/chosen": -582.899658203125, + "logps/ref_chosen": -57.249366760253906, + "logps/ref_rejected": -92.35354614257812, + "logps/rejected": -960.1959228515625, + "loss": 1.3081, + "margin_dpo/margin_mean": 342.1921081542969, + "margin_dpo/margin_std": 729.956298828125, + "step": 301 + }, + { + "KL/chosen_KL_mean": -453.0435791015625, + "KL/mean": -614.121337890625, + "KL/rejected_KL_mean": -775.19921875, + "KL/std": 527.9852294921875, + "epoch": 0.4434654919236417, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 322.1556396484375, + "fcm_dpo/q_t": 0.479292631149292, + "grad_norm": 8.750789642333984, + "learning_rate": 3.426989547989902e-07, + "logits/chosen": -0.15997496247291565, + "logits/rejected": -0.15943099558353424, + "logps/chosen": -504.2415771484375, + "logps/ref_chosen": -51.197994232177734, + "logps/ref_rejected": -97.22636413574219, + "logps/rejected": -872.425537109375, + "loss": 1.3124, + "margin_dpo/margin_mean": 322.1556396484375, + "margin_dpo/margin_std": 693.7476806640625, + "step": 302 + }, + { + "KL/chosen_KL_mean": -511.125244140625, + "KL/mean": -612.9938354492188, + "KL/rejected_KL_mean": -714.8624877929688, + "KL/std": 489.7009582519531, + "epoch": 0.44493392070484583, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 203.7372589111328, + "fcm_dpo/q_t": 0.48675549030303955, + "grad_norm": 6.2934136390686035, + "learning_rate": 3.4150589130555773e-07, + "logits/chosen": -0.16826504468917847, + "logits/rejected": -0.1783979833126068, + "logps/chosen": -577.8391723632812, + "logps/ref_chosen": -66.71394348144531, + "logps/ref_rejected": -86.94542694091797, + "logps/rejected": -801.8079223632812, + "loss": 1.3402, + "margin_dpo/margin_mean": 203.73727416992188, + "margin_dpo/margin_std": 608.4408569335938, + "step": 303 + }, + { + "KL/chosen_KL_mean": -474.92987060546875, + "KL/mean": -585.3536376953125, + "KL/rejected_KL_mean": -695.7774658203125, + "KL/std": 415.7464599609375, + "epoch": 0.44640234948604995, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 220.84759521484375, + "fcm_dpo/q_t": 0.4856563210487366, + "grad_norm": 7.61824893951416, + "learning_rate": 3.403104165467883e-07, + "logits/chosen": -0.1859014928340912, + "logits/rejected": -0.1942559778690338, + "logps/chosen": -546.8805541992188, + "logps/ref_chosen": -71.95069885253906, + "logps/ref_rejected": -90.47203063964844, + "logps/rejected": -786.24951171875, + "loss": 1.3334, + "margin_dpo/margin_mean": 220.84759521484375, + "margin_dpo/margin_std": 473.25732421875, + "step": 304 + }, + { + "KL/chosen_KL_mean": -428.038330078125, + "KL/mean": -549.772705078125, + "KL/rejected_KL_mean": -671.5070190429688, + "KL/std": 450.9216003417969, + "epoch": 0.447870778267254, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 243.46868896484375, + "fcm_dpo/q_t": 0.4842411279678345, + "grad_norm": 5.930902004241943, + "learning_rate": 3.391125620245535e-07, + "logits/chosen": -0.22110234200954437, + "logits/rejected": -0.23830103874206543, + "logps/chosen": -494.8335876464844, + "logps/ref_chosen": -66.79523468017578, + "logps/ref_rejected": -92.75459289550781, + "logps/rejected": -764.2615966796875, + "loss": 1.3293, + "margin_dpo/margin_mean": 243.46868896484375, + "margin_dpo/margin_std": 573.030029296875, + "step": 305 + }, + { + "KL/chosen_KL_mean": -461.40081787109375, + "KL/mean": -561.10595703125, + "KL/rejected_KL_mean": -660.81103515625, + "KL/std": 412.7308654785156, + "epoch": 0.44933920704845814, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 199.41024780273438, + "fcm_dpo/q_t": 0.4870589077472687, + "grad_norm": 6.410506725311279, + "learning_rate": 3.3791235930343417e-07, + "logits/chosen": -0.1642266809940338, + "logits/rejected": -0.17837274074554443, + "logps/chosen": -531.084716796875, + "logps/ref_chosen": -69.68389892578125, + "logps/ref_rejected": -85.15919494628906, + "logps/rejected": -745.97021484375, + "loss": 1.3388, + "margin_dpo/margin_mean": 199.41024780273438, + "margin_dpo/margin_std": 468.4999084472656, + "step": 306 + }, + { + "KL/chosen_KL_mean": -415.28350830078125, + "KL/mean": -531.0717163085938, + "KL/rejected_KL_mean": -646.8599243164062, + "KL/std": 411.35687255859375, + "epoch": 0.45080763582966227, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 231.57640075683594, + "fcm_dpo/q_t": 0.4849886894226074, + "grad_norm": 8.488301277160645, + "learning_rate": 3.367098400098881e-07, + "logits/chosen": -0.18022067844867706, + "logits/rejected": -0.1865011602640152, + "logps/chosen": -485.44891357421875, + "logps/ref_chosen": -70.16542053222656, + "logps/ref_rejected": -86.97230529785156, + "logps/rejected": -733.832275390625, + "loss": 1.3317, + "margin_dpo/margin_mean": 231.576416015625, + "margin_dpo/margin_std": 537.4920654296875, + "step": 307 + }, + { + "KL/chosen_KL_mean": -447.8622741699219, + "KL/mean": -599.67041015625, + "KL/rejected_KL_mean": -751.478515625, + "KL/std": 453.7755432128906, + "epoch": 0.4522760646108664, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 303.61627197265625, + "fcm_dpo/q_t": 0.48028409481048584, + "grad_norm": 12.173463821411133, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": -0.20448637008666992, + "logits/rejected": -0.21775467693805695, + "logps/chosen": -503.1072692871094, + "logps/ref_chosen": -55.2449951171875, + "logps/ref_rejected": -79.37226104736328, + "logps/rejected": -830.8507690429688, + "loss": 1.3125, + "margin_dpo/margin_mean": 303.6162414550781, + "margin_dpo/margin_std": 472.8870849609375, + "step": 308 + }, + { + "KL/chosen_KL_mean": -451.45013427734375, + "KL/mean": -587.4209594726562, + "KL/rejected_KL_mean": -723.3917846679688, + "KL/std": 521.2839965820312, + "epoch": 0.45374449339207046, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 271.9416809082031, + "fcm_dpo/q_t": 0.4822486639022827, + "grad_norm": 8.317163467407227, + "learning_rate": 3.3429797851573183e-07, + "logits/chosen": -0.1586446464061737, + "logits/rejected": -0.17579534649848938, + "logps/chosen": -500.4092102050781, + "logps/ref_chosen": -48.959083557128906, + "logps/ref_rejected": -82.34072875976562, + "logps/rejected": -805.7325439453125, + "loss": 1.3225, + "margin_dpo/margin_mean": 271.941650390625, + "margin_dpo/margin_std": 569.9622802734375, + "step": 309 + }, + { + "KL/chosen_KL_mean": -649.622802734375, + "KL/mean": -782.8016357421875, + "KL/rejected_KL_mean": -915.9805297851562, + "KL/std": 569.54541015625, + "epoch": 0.4552129221732746, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 266.3577575683594, + "fcm_dpo/q_t": 0.48275303840637207, + "grad_norm": 11.456547737121582, + "learning_rate": 3.3308869986991487e-07, + "logits/chosen": -0.16804948449134827, + "logits/rejected": -0.19079285860061646, + "logps/chosen": -712.364501953125, + "logps/ref_chosen": -62.74177932739258, + "logps/ref_rejected": -79.9302978515625, + "logps/rejected": -995.9108276367188, + "loss": 1.3254, + "margin_dpo/margin_mean": 266.3577575683594, + "margin_dpo/margin_std": 641.886962890625, + "step": 310 + }, + { + "KL/chosen_KL_mean": -619.607421875, + "KL/mean": -746.11328125, + "KL/rejected_KL_mean": -872.619140625, + "KL/std": 541.6986694335938, + "epoch": 0.4566813509544787, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 253.01168823242188, + "fcm_dpo/q_t": 0.4836409389972687, + "grad_norm": 12.55339241027832, + "learning_rate": 3.3187723175958346e-07, + "logits/chosen": -0.2095467746257782, + "logits/rejected": -0.21634000539779663, + "logps/chosen": -672.6354370117188, + "logps/ref_chosen": -53.02798080444336, + "logps/ref_rejected": -77.43820190429688, + "logps/rejected": -950.057373046875, + "loss": 1.3278, + "margin_dpo/margin_mean": 253.0116729736328, + "margin_dpo/margin_std": 614.6041870117188, + "step": 311 + }, + { + "KL/chosen_KL_mean": -701.8001708984375, + "KL/mean": -825.3548583984375, + "KL/rejected_KL_mean": -948.909423828125, + "KL/std": 627.0281982421875, + "epoch": 0.4581497797356828, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 247.10928344726562, + "fcm_dpo/q_t": 0.4842231571674347, + "grad_norm": 8.390949249267578, + "learning_rate": 3.306636061080487e-07, + "logits/chosen": -0.19486571848392487, + "logits/rejected": -0.20282875001430511, + "logps/chosen": -751.1923828125, + "logps/ref_chosen": -49.39221954345703, + "logps/ref_rejected": -75.79280853271484, + "logps/rejected": -1024.7022705078125, + "loss": 1.3342, + "margin_dpo/margin_mean": 247.10926818847656, + "margin_dpo/margin_std": 803.0827026367188, + "step": 312 + }, + { + "KL/chosen_KL_mean": -695.1138916015625, + "KL/mean": -879.8554077148438, + "KL/rejected_KL_mean": -1064.596923828125, + "KL/std": 770.5494384765625, + "epoch": 0.45961820851688695, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 369.48297119140625, + "fcm_dpo/q_t": 0.47646960616111755, + "grad_norm": 7.17667818069458, + "learning_rate": 3.2944785489547537e-07, + "logits/chosen": -0.25826430320739746, + "logits/rejected": -0.27332425117492676, + "logps/chosen": -745.2666625976562, + "logps/ref_chosen": -50.152740478515625, + "logps/ref_rejected": -86.40620422363281, + "logps/rejected": -1151.003173828125, + "loss": 1.3055, + "margin_dpo/margin_mean": 369.4830017089844, + "margin_dpo/margin_std": 858.5765380859375, + "step": 313 + }, + { + "KL/chosen_KL_mean": -779.741455078125, + "KL/mean": -921.0244750976562, + "KL/rejected_KL_mean": -1062.307373046875, + "KL/std": 699.74462890625, + "epoch": 0.461086637298091, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 282.5658874511719, + "fcm_dpo/q_t": 0.4820650815963745, + "grad_norm": 11.305423736572266, + "learning_rate": 3.2823001015803857e-07, + "logits/chosen": -0.2745167315006256, + "logits/rejected": -0.2868790030479431, + "logps/chosen": -836.9790649414062, + "logps/ref_chosen": -57.237579345703125, + "logps/ref_rejected": -97.5965347290039, + "logps/rejected": -1159.904052734375, + "loss": 1.329, + "margin_dpo/margin_mean": 282.56591796875, + "margin_dpo/margin_std": 936.9970703125, + "step": 314 + }, + { + "KL/chosen_KL_mean": -648.9210815429688, + "KL/mean": -783.290283203125, + "KL/rejected_KL_mean": -917.6595458984375, + "KL/std": 683.492431640625, + "epoch": 0.46255506607929514, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 268.738525390625, + "fcm_dpo/q_t": 0.48275700211524963, + "grad_norm": 11.812366485595703, + "learning_rate": 3.270101039870797e-07, + "logits/chosen": -0.23237919807434082, + "logits/rejected": -0.23814760148525238, + "logps/chosen": -697.9906616210938, + "logps/ref_chosen": -49.06958770751953, + "logps/ref_rejected": -85.68087768554688, + "logps/rejected": -1003.3404541015625, + "loss": 1.3289, + "margin_dpo/margin_mean": 268.738525390625, + "margin_dpo/margin_std": 817.6375732421875, + "step": 315 + }, + { + "KL/chosen_KL_mean": -583.3734130859375, + "KL/mean": -823.87109375, + "KL/rejected_KL_mean": -1064.3687744140625, + "KL/std": 654.394775390625, + "epoch": 0.46402349486049926, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 480.99542236328125, + "fcm_dpo/q_t": 0.4692395031452179, + "grad_norm": 9.822944641113281, + "learning_rate": 3.2578816852826086e-07, + "logits/chosen": -0.27629613876342773, + "logits/rejected": -0.2879878878593445, + "logps/chosen": -637.6341552734375, + "logps/ref_chosen": -54.26074981689453, + "logps/ref_rejected": -101.2814712524414, + "logps/rejected": -1165.650146484375, + "loss": 1.2759, + "margin_dpo/margin_mean": 480.995361328125, + "margin_dpo/margin_std": 791.6807861328125, + "step": 316 + }, + { + "KL/chosen_KL_mean": -654.828369140625, + "KL/mean": -917.2320556640625, + "KL/rejected_KL_mean": -1179.6356201171875, + "KL/std": 796.4964599609375, + "epoch": 0.4654919236417034, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 524.8072509765625, + "fcm_dpo/q_t": 0.46749019622802734, + "grad_norm": 13.96440601348877, + "learning_rate": 3.2456423598071783e-07, + "logits/chosen": -0.2654808461666107, + "logits/rejected": -0.27527332305908203, + "logps/chosen": -710.922607421875, + "logps/ref_chosen": -56.094207763671875, + "logps/ref_rejected": -100.69905090332031, + "logps/rejected": -1280.334716796875, + "loss": 1.2729, + "margin_dpo/margin_mean": 524.8072509765625, + "margin_dpo/margin_std": 1009.199462890625, + "step": 317 + }, + { + "KL/chosen_KL_mean": -712.6109619140625, + "KL/mean": -797.7545166015625, + "KL/rejected_KL_mean": -882.8980712890625, + "KL/std": 595.2559814453125, + "epoch": 0.4669603524229075, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 170.2871551513672, + "fcm_dpo/q_t": 0.48896729946136475, + "grad_norm": 11.569266319274902, + "learning_rate": 3.233383385962115e-07, + "logits/chosen": -0.3017517626285553, + "logits/rejected": -0.29757431149482727, + "logps/chosen": -777.256591796875, + "logps/ref_chosen": -64.64569854736328, + "logps/ref_rejected": -82.76425170898438, + "logps/rejected": -965.662353515625, + "loss": 1.3506, + "margin_dpo/margin_mean": 170.28717041015625, + "margin_dpo/margin_std": 690.7733154296875, + "step": 318 + }, + { + "KL/chosen_KL_mean": -496.1173095703125, + "KL/mean": -747.5867309570312, + "KL/rejected_KL_mean": -999.0562744140625, + "KL/std": 743.3543701171875, + "epoch": 0.4684287812041116, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 502.9389343261719, + "fcm_dpo/q_t": 0.46803224086761475, + "grad_norm": 14.8353271484375, + "learning_rate": 3.2211050867827805e-07, + "logits/chosen": -0.2799922227859497, + "logits/rejected": -0.2898196578025818, + "logps/chosen": -545.5010375976562, + "logps/ref_chosen": -49.383758544921875, + "logps/ref_rejected": -113.90650939941406, + "logps/rejected": -1112.9627685546875, + "loss": 1.2732, + "margin_dpo/margin_mean": 502.9389343261719, + "margin_dpo/margin_std": 895.0174560546875, + "step": 319 + }, + { + "KL/chosen_KL_mean": -562.5033569335938, + "KL/mean": -736.6514282226562, + "KL/rejected_KL_mean": -910.7994384765625, + "KL/std": 562.046142578125, + "epoch": 0.4698972099853157, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 348.2961120605469, + "fcm_dpo/q_t": 0.47749677300453186, + "grad_norm": 7.6283860206604, + "learning_rate": 3.208807785813777e-07, + "logits/chosen": -0.2599332332611084, + "logits/rejected": -0.274741530418396, + "logps/chosen": -622.00830078125, + "logps/ref_chosen": -59.50489044189453, + "logps/ref_rejected": -97.66717529296875, + "logps/rejected": -1008.4666748046875, + "loss": 1.3054, + "margin_dpo/margin_mean": 348.2961120605469, + "margin_dpo/margin_std": 674.3231201171875, + "step": 320 + }, + { + "KL/chosen_KL_mean": -725.1861572265625, + "KL/mean": -895.68994140625, + "KL/rejected_KL_mean": -1066.193603515625, + "KL/std": 716.3073120117188, + "epoch": 0.4713656387665198, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 341.00750732421875, + "fcm_dpo/q_t": 0.4783346951007843, + "grad_norm": 9.341181755065918, + "learning_rate": 3.1964918071004217e-07, + "logits/chosen": -0.23902887105941772, + "logits/rejected": -0.2461167275905609, + "logps/chosen": -786.73486328125, + "logps/ref_chosen": -61.548683166503906, + "logps/ref_rejected": -91.64103698730469, + "logps/rejected": -1157.834716796875, + "loss": 1.3123, + "margin_dpo/margin_mean": 341.00750732421875, + "margin_dpo/margin_std": 881.1239013671875, + "step": 321 + }, + { + "KL/chosen_KL_mean": -587.3211669921875, + "KL/mean": -776.135986328125, + "KL/rejected_KL_mean": -964.9507446289062, + "KL/std": 558.770751953125, + "epoch": 0.47283406754772395, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 377.6296081542969, + "fcm_dpo/q_t": 0.4755861759185791, + "grad_norm": 13.473780632019043, + "learning_rate": 3.184157475180207e-07, + "logits/chosen": -0.26182982325553894, + "logits/rejected": -0.2680904269218445, + "logps/chosen": -644.6112060546875, + "logps/ref_chosen": -57.29003143310547, + "logps/ref_rejected": -95.74992370605469, + "logps/rejected": -1060.70068359375, + "loss": 1.2985, + "margin_dpo/margin_mean": 377.629638671875, + "margin_dpo/margin_std": 694.3011474609375, + "step": 322 + }, + { + "KL/chosen_KL_mean": -609.4126586914062, + "KL/mean": -739.869140625, + "KL/rejected_KL_mean": -870.32568359375, + "KL/std": 562.13623046875, + "epoch": 0.47430249632892807, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 260.9129943847656, + "fcm_dpo/q_t": 0.4830913543701172, + "grad_norm": 9.924291610717773, + "learning_rate": 3.171805115074251e-07, + "logits/chosen": -0.26495540142059326, + "logits/rejected": -0.27675527334213257, + "logps/chosen": -660.6466064453125, + "logps/ref_chosen": -51.23395919799805, + "logps/ref_rejected": -75.06192016601562, + "logps/rejected": -945.3875732421875, + "loss": 1.3277, + "margin_dpo/margin_mean": 260.9129638671875, + "margin_dpo/margin_std": 673.8228149414062, + "step": 323 + }, + { + "KL/chosen_KL_mean": -724.944091796875, + "KL/mean": -845.3299560546875, + "KL/rejected_KL_mean": -965.7159423828125, + "KL/std": 667.5174560546875, + "epoch": 0.47577092511013214, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 240.7718505859375, + "fcm_dpo/q_t": 0.4845079779624939, + "grad_norm": 11.485196113586426, + "learning_rate": 3.1594350522787295e-07, + "logits/chosen": -0.26390862464904785, + "logits/rejected": -0.26681196689605713, + "logps/chosen": -790.0792236328125, + "logps/ref_chosen": -65.13516998291016, + "logps/ref_rejected": -86.47750854492188, + "logps/rejected": -1052.193359375, + "loss": 1.3385, + "margin_dpo/margin_mean": 240.77186584472656, + "margin_dpo/margin_std": 892.4651489257812, + "step": 324 + }, + { + "KL/chosen_KL_mean": -495.9256896972656, + "KL/mean": -597.509033203125, + "KL/rejected_KL_mean": -699.09228515625, + "KL/std": 471.72412109375, + "epoch": 0.47723935389133626, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 203.16659545898438, + "fcm_dpo/q_t": 0.4868882894515991, + "grad_norm": 10.78111743927002, + "learning_rate": 3.147047612756302e-07, + "logits/chosen": -0.29811710119247437, + "logits/rejected": -0.3030798137187958, + "logps/chosen": -552.1412963867188, + "logps/ref_chosen": -56.215599060058594, + "logps/ref_rejected": -70.08592987060547, + "logps/rejected": -769.17822265625, + "loss": 1.3397, + "margin_dpo/margin_mean": 203.16659545898438, + "margin_dpo/margin_std": 564.2322998046875, + "step": 325 + }, + { + "KL/chosen_KL_mean": -589.589599609375, + "KL/mean": -696.318359375, + "KL/rejected_KL_mean": -803.0471801757812, + "KL/std": 521.7206420898438, + "epoch": 0.4787077826725404, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 213.4575653076172, + "fcm_dpo/q_t": 0.48614704608917236, + "grad_norm": 9.612821578979492, + "learning_rate": 3.134643122927519e-07, + "logits/chosen": -0.305173397064209, + "logits/rejected": -0.3111916780471802, + "logps/chosen": -662.3145751953125, + "logps/ref_chosen": -72.72496032714844, + "logps/ref_rejected": -79.8467788696289, + "logps/rejected": -882.8939208984375, + "loss": 1.3363, + "margin_dpo/margin_mean": 213.45758056640625, + "margin_dpo/margin_std": 538.012939453125, + "step": 326 + }, + { + "KL/chosen_KL_mean": -558.3499755859375, + "KL/mean": -724.33056640625, + "KL/rejected_KL_mean": -890.311279296875, + "KL/std": 517.6776123046875, + "epoch": 0.4801762114537445, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 331.9613037109375, + "fcm_dpo/q_t": 0.47847986221313477, + "grad_norm": 15.699017524719238, + "learning_rate": 3.1222219096622264e-07, + "logits/chosen": -0.2681811451911926, + "logits/rejected": -0.27183783054351807, + "logps/chosen": -627.484375, + "logps/ref_chosen": -69.13441467285156, + "logps/ref_rejected": -111.93377685546875, + "logps/rejected": -1002.2450561523438, + "loss": 1.3086, + "margin_dpo/margin_mean": 331.9612731933594, + "margin_dpo/margin_std": 640.1740112304688, + "step": 327 + }, + { + "KL/chosen_KL_mean": -558.3543701171875, + "KL/mean": -749.8038330078125, + "KL/rejected_KL_mean": -941.2532348632812, + "KL/std": 636.8343505859375, + "epoch": 0.48164464023494863, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 382.89892578125, + "fcm_dpo/q_t": 0.4753231406211853, + "grad_norm": 14.158612251281738, + "learning_rate": 3.1097843002709427e-07, + "logits/chosen": -0.2592814862728119, + "logits/rejected": -0.2679283022880554, + "logps/chosen": -618.0415649414062, + "logps/ref_chosen": -59.68719482421875, + "logps/ref_rejected": -90.85499572753906, + "logps/rejected": -1032.1082763671875, + "loss": 1.2986, + "margin_dpo/margin_mean": 382.8988952636719, + "margin_dpo/margin_std": 756.589599609375, + "step": 328 + }, + { + "KL/chosen_KL_mean": -674.1375732421875, + "KL/mean": -896.6131591796875, + "KL/rejected_KL_mean": -1119.0888671875, + "KL/std": 738.45849609375, + "epoch": 0.4831130690161527, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 444.9512023925781, + "fcm_dpo/q_t": 0.47157514095306396, + "grad_norm": 9.89807415008545, + "learning_rate": 3.0973306224962437e-07, + "logits/chosen": -0.25955867767333984, + "logits/rejected": -0.2711198329925537, + "logps/chosen": -739.3837890625, + "logps/ref_chosen": -65.2461929321289, + "logps/ref_rejected": -100.69770812988281, + "logps/rejected": -1219.7864990234375, + "loss": 1.287, + "margin_dpo/margin_mean": 444.951171875, + "margin_dpo/margin_std": 891.010986328125, + "step": 329 + }, + { + "KL/chosen_KL_mean": -574.3134765625, + "KL/mean": -772.6851196289062, + "KL/rejected_KL_mean": -971.0567626953125, + "KL/std": 623.42626953125, + "epoch": 0.4845814977973568, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 396.7432861328125, + "fcm_dpo/q_t": 0.4744586944580078, + "grad_norm": 11.012582778930664, + "learning_rate": 3.084861204504122e-07, + "logits/chosen": -0.2380240261554718, + "logits/rejected": -0.23784172534942627, + "logps/chosen": -621.3118286132812, + "logps/ref_chosen": -46.998348236083984, + "logps/ref_rejected": -86.87684631347656, + "logps/rejected": -1057.93359375, + "loss": 1.2955, + "margin_dpo/margin_mean": 396.7432861328125, + "margin_dpo/margin_std": 763.457763671875, + "step": 330 + }, + { + "KL/chosen_KL_mean": -734.039306640625, + "KL/mean": -967.9661865234375, + "KL/rejected_KL_mean": -1201.8931884765625, + "KL/std": 592.183837890625, + "epoch": 0.48604992657856094, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 467.8538818359375, + "fcm_dpo/q_t": 0.46986526250839233, + "grad_norm": 14.263129234313965, + "learning_rate": 3.072376374875335e-07, + "logits/chosen": -0.23857057094573975, + "logits/rejected": -0.23989224433898926, + "logps/chosen": -784.5635375976562, + "logps/ref_chosen": -50.52424621582031, + "logps/ref_rejected": -89.01544189453125, + "logps/rejected": -1290.90869140625, + "loss": 1.2768, + "margin_dpo/margin_mean": 467.85394287109375, + "margin_dpo/margin_std": 727.0281982421875, + "step": 331 + }, + { + "KL/chosen_KL_mean": -898.7767333984375, + "KL/mean": -1009.329345703125, + "KL/rejected_KL_mean": -1119.8819580078125, + "KL/std": 775.726318359375, + "epoch": 0.48751835535976507, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 221.10525512695312, + "fcm_dpo/q_t": 0.4862329959869385, + "grad_norm": 11.460166931152344, + "learning_rate": 3.059876462596758e-07, + "logits/chosen": -0.2585371732711792, + "logits/rejected": -0.2638899087905884, + "logps/chosen": -947.95703125, + "logps/ref_chosen": -49.18028259277344, + "logps/ref_rejected": -76.48515319824219, + "logps/rejected": -1196.3671875, + "loss": 1.3453, + "margin_dpo/margin_mean": 221.10525512695312, + "margin_dpo/margin_std": 970.4208374023438, + "step": 332 + }, + { + "KL/chosen_KL_mean": -1128.766845703125, + "KL/mean": -1313.67626953125, + "KL/rejected_KL_mean": -1498.585693359375, + "KL/std": 929.1944580078125, + "epoch": 0.4889867841409692, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 369.8189697265625, + "fcm_dpo/q_t": 0.4764955937862396, + "grad_norm": 28.53215980529785, + "learning_rate": 3.0473617970527015e-07, + "logits/chosen": -0.29855263233184814, + "logits/rejected": -0.3038429021835327, + "logps/chosen": -1192.5224609375, + "logps/ref_chosen": -63.75574493408203, + "logps/ref_rejected": -95.04411315917969, + "logps/rejected": -1593.6298828125, + "loss": 1.3193, + "margin_dpo/margin_mean": 369.8189697265625, + "margin_dpo/margin_std": 1217.3851318359375, + "step": 333 + }, + { + "KL/chosen_KL_mean": -1276.139892578125, + "KL/mean": -1387.209716796875, + "KL/rejected_KL_mean": -1498.279541015625, + "KL/std": 1128.24169921875, + "epoch": 0.49045521292217326, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 222.13943481445312, + "fcm_dpo/q_t": 0.4864605665206909, + "grad_norm": 47.288944244384766, + "learning_rate": 3.034832708016243e-07, + "logits/chosen": -0.2978268265724182, + "logits/rejected": -0.3007590174674988, + "logps/chosen": -1343.1197509765625, + "logps/ref_chosen": -66.97975158691406, + "logps/ref_rejected": -95.31692504882812, + "logps/rejected": -1593.596435546875, + "loss": 1.3669, + "margin_dpo/margin_mean": 222.139404296875, + "margin_dpo/margin_std": 1478.4951171875, + "step": 334 + }, + { + "KL/chosen_KL_mean": -1121.868408203125, + "KL/mean": -1232.154052734375, + "KL/rejected_KL_mean": -1342.439453125, + "KL/std": 981.7579345703125, + "epoch": 0.4919236417033774, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 220.57101440429688, + "fcm_dpo/q_t": 0.48608070611953735, + "grad_norm": 25.209972381591797, + "learning_rate": 3.022289525640531e-07, + "logits/chosen": -0.32386332750320435, + "logits/rejected": -0.3207986354827881, + "logps/chosen": -1184.410888671875, + "logps/ref_chosen": -62.54248046875, + "logps/ref_rejected": -87.61770629882812, + "logps/rejected": -1430.05712890625, + "loss": 1.3467, + "margin_dpo/margin_mean": 220.571044921875, + "margin_dpo/margin_std": 993.0822143554688, + "step": 335 + }, + { + "KL/chosen_KL_mean": -1050.422607421875, + "KL/mean": -1337.56591796875, + "KL/rejected_KL_mean": -1624.709228515625, + "KL/std": 1164.25244140625, + "epoch": 0.4933920704845815, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 574.28662109375, + "fcm_dpo/q_t": 0.46436643600463867, + "grad_norm": 14.840729713439941, + "learning_rate": 3.009732580450086e-07, + "logits/chosen": -0.30795538425445557, + "logits/rejected": -0.31845933198928833, + "logps/chosen": -1104.9537353515625, + "logps/ref_chosen": -54.53115463256836, + "logps/ref_rejected": -104.40424346923828, + "logps/rejected": -1729.113525390625, + "loss": 1.2781, + "margin_dpo/margin_mean": 574.28662109375, + "margin_dpo/margin_std": 1452.664306640625, + "step": 336 + }, + { + "KL/chosen_KL_mean": -855.2294921875, + "KL/mean": -1081.739501953125, + "KL/rejected_KL_mean": -1308.249755859375, + "KL/std": 739.3917236328125, + "epoch": 0.4948604992657856, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 453.0201721191406, + "fcm_dpo/q_t": 0.4711313247680664, + "grad_norm": 11.842601776123047, + "learning_rate": 2.9971622033320914e-07, + "logits/chosen": -0.3099353313446045, + "logits/rejected": -0.3142092227935791, + "logps/chosen": -920.3582153320312, + "logps/ref_chosen": -65.12869262695312, + "logps/ref_rejected": -101.72701263427734, + "logps/rejected": -1409.9766845703125, + "loss": 1.2874, + "margin_dpo/margin_mean": 453.02020263671875, + "margin_dpo/margin_std": 964.009765625, + "step": 337 + }, + { + "KL/chosen_KL_mean": -630.074951171875, + "KL/mean": -862.7835083007812, + "KL/rejected_KL_mean": -1095.491943359375, + "KL/std": 717.5847778320312, + "epoch": 0.49632892804698975, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 465.41705322265625, + "fcm_dpo/q_t": 0.4699110984802246, + "grad_norm": 10.382853507995605, + "learning_rate": 2.984578725527675e-07, + "logits/chosen": -0.272180438041687, + "logits/rejected": -0.28474897146224976, + "logps/chosen": -688.4976806640625, + "logps/ref_chosen": -58.422706604003906, + "logps/ref_rejected": -89.06854248046875, + "logps/rejected": -1184.560546875, + "loss": 1.2764, + "margin_dpo/margin_mean": 465.4170227050781, + "margin_dpo/margin_std": 681.0792846679688, + "step": 338 + }, + { + "KL/chosen_KL_mean": -607.092041015625, + "KL/mean": -813.3968505859375, + "KL/rejected_KL_mean": -1019.7017211914062, + "KL/std": 651.6005859375, + "epoch": 0.4977973568281938, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 412.6096496582031, + "fcm_dpo/q_t": 0.47347021102905273, + "grad_norm": 14.31478500366211, + "learning_rate": 2.9719824786231796e-07, + "logits/chosen": -0.25229567289352417, + "logits/rejected": -0.26170292496681213, + "logps/chosen": -667.0873413085938, + "logps/ref_chosen": -59.99531555175781, + "logps/ref_rejected": -103.9109115600586, + "logps/rejected": -1123.612548828125, + "loss": 1.2931, + "margin_dpo/margin_mean": 412.6096496582031, + "margin_dpo/margin_std": 765.8604736328125, + "step": 339 + }, + { + "KL/chosen_KL_mean": -682.767822265625, + "KL/mean": -883.4863891601562, + "KL/rejected_KL_mean": -1084.2049560546875, + "KL/std": 732.8419189453125, + "epoch": 0.49926578560939794, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 401.4371337890625, + "fcm_dpo/q_t": 0.4744035601615906, + "grad_norm": 15.460363388061523, + "learning_rate": 2.959373794541426e-07, + "logits/chosen": -0.2546389698982239, + "logits/rejected": -0.25613105297088623, + "logps/chosen": -735.5980224609375, + "logps/ref_chosen": -52.83022689819336, + "logps/ref_rejected": -73.10723114013672, + "logps/rejected": -1157.312255859375, + "loss": 1.299, + "margin_dpo/margin_mean": 401.4371337890625, + "margin_dpo/margin_std": 928.7999267578125, + "step": 340 + }, + { + "KL/chosen_KL_mean": -624.3984375, + "KL/mean": -853.274169921875, + "KL/rejected_KL_mean": -1082.1497802734375, + "KL/std": 722.1390380859375, + "epoch": 0.5007342143906021, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 457.75128173828125, + "fcm_dpo/q_t": 0.4708910584449768, + "grad_norm": 21.002342224121094, + "learning_rate": 2.946753005532965e-07, + "logits/chosen": -0.23577266931533813, + "logits/rejected": -0.24404844641685486, + "logps/chosen": -672.2982788085938, + "logps/ref_chosen": -47.899803161621094, + "logps/ref_rejected": -101.80987548828125, + "logps/rejected": -1183.959716796875, + "loss": 1.2852, + "margin_dpo/margin_mean": 457.7512512207031, + "margin_dpo/margin_std": 918.9498291015625, + "step": 341 + }, + { + "KL/chosen_KL_mean": -543.6002197265625, + "KL/mean": -688.5452880859375, + "KL/rejected_KL_mean": -833.4903564453125, + "KL/std": 547.8942260742188, + "epoch": 0.5022026431718062, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 289.8901672363281, + "fcm_dpo/q_t": 0.48126789927482605, + "grad_norm": 9.170316696166992, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": -0.30688920617103577, + "logits/rejected": -0.30778950452804565, + "logps/chosen": -615.5968017578125, + "logps/ref_chosen": -71.99664306640625, + "logps/ref_rejected": -92.58959197998047, + "logps/rejected": -926.0799560546875, + "loss": 1.3206, + "margin_dpo/margin_mean": 289.8901672363281, + "margin_dpo/margin_std": 702.553955078125, + "step": 342 + }, + { + "KL/chosen_KL_mean": -471.5529479980469, + "KL/mean": -678.2440185546875, + "KL/rejected_KL_mean": -884.93505859375, + "KL/std": 528.4884033203125, + "epoch": 0.5036710719530103, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 413.38214111328125, + "fcm_dpo/q_t": 0.47324374318122864, + "grad_norm": 11.689512252807617, + "learning_rate": 2.9214764433242476e-07, + "logits/chosen": -0.24627618491649628, + "logits/rejected": -0.25361427664756775, + "logps/chosen": -525.9585571289062, + "logps/ref_chosen": -54.405616760253906, + "logps/ref_rejected": -111.04142761230469, + "logps/rejected": -995.9764404296875, + "loss": 1.2863, + "margin_dpo/margin_mean": 413.38214111328125, + "margin_dpo/margin_std": 545.55517578125, + "step": 343 + }, + { + "KL/chosen_KL_mean": -607.8341064453125, + "KL/mean": -803.11181640625, + "KL/rejected_KL_mean": -998.3895263671875, + "KL/std": 760.2177734375, + "epoch": 0.5051395007342144, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 390.555419921875, + "fcm_dpo/q_t": 0.4753226637840271, + "grad_norm": 9.874456405639648, + "learning_rate": 2.9088213361849126e-07, + "logits/chosen": -0.2654206454753876, + "logits/rejected": -0.2721824645996094, + "logps/chosen": -661.798828125, + "logps/ref_chosen": -53.96466827392578, + "logps/ref_rejected": -90.62336730957031, + "logps/rejected": -1089.012939453125, + "loss": 1.3019, + "margin_dpo/margin_mean": 390.555419921875, + "margin_dpo/margin_std": 907.13916015625, + "step": 344 + }, + { + "KL/chosen_KL_mean": -735.56591796875, + "KL/mean": -957.1832275390625, + "KL/rejected_KL_mean": -1178.8006591796875, + "KL/std": 641.4892578125, + "epoch": 0.5066079295154186, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 443.2347412109375, + "fcm_dpo/q_t": 0.4715315103530884, + "grad_norm": 11.198358535766602, + "learning_rate": 2.896155456223163e-07, + "logits/chosen": -0.27164191007614136, + "logits/rejected": -0.27693307399749756, + "logps/chosen": -797.2515869140625, + "logps/ref_chosen": -61.685699462890625, + "logps/ref_rejected": -99.49041748046875, + "logps/rejected": -1278.291015625, + "loss": 1.2858, + "margin_dpo/margin_mean": 443.2347412109375, + "margin_dpo/margin_std": 839.5887451171875, + "step": 345 + }, + { + "KL/chosen_KL_mean": -720.7941284179688, + "KL/mean": -924.3731079101562, + "KL/rejected_KL_mean": -1127.9521484375, + "KL/std": 670.3480224609375, + "epoch": 0.5080763582966226, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 407.1580505371094, + "fcm_dpo/q_t": 0.4737437069416046, + "grad_norm": 13.88206672668457, + "learning_rate": 2.883479137196714e-07, + "logits/chosen": -0.2596754729747772, + "logits/rejected": -0.26096630096435547, + "logps/chosen": -776.0504150390625, + "logps/ref_chosen": -55.256263732910156, + "logps/ref_rejected": -77.41532135009766, + "logps/rejected": -1205.367431640625, + "loss": 1.2905, + "margin_dpo/margin_mean": 407.15802001953125, + "margin_dpo/margin_std": 673.2942504882812, + "step": 346 + }, + { + "KL/chosen_KL_mean": -865.6432495117188, + "KL/mean": -1022.4780883789062, + "KL/rejected_KL_mean": -1179.31298828125, + "KL/std": 805.083251953125, + "epoch": 0.5095447870778267, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 313.669677734375, + "fcm_dpo/q_t": 0.47981417179107666, + "grad_norm": 10.583271026611328, + "learning_rate": 2.8707927131383614e-07, + "logits/chosen": -0.27997511625289917, + "logits/rejected": -0.28578126430511475, + "logps/chosen": -923.20947265625, + "logps/ref_chosen": -57.56623840332031, + "logps/ref_rejected": -92.35509490966797, + "logps/rejected": -1271.6680908203125, + "loss": 1.3201, + "margin_dpo/margin_mean": 313.669677734375, + "margin_dpo/margin_std": 908.4871826171875, + "step": 347 + }, + { + "KL/chosen_KL_mean": -720.2841796875, + "KL/mean": -897.8193359375, + "KL/rejected_KL_mean": -1075.3543701171875, + "KL/std": 628.7544555664062, + "epoch": 0.5110132158590308, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 355.0701904296875, + "fcm_dpo/q_t": 0.4771634340286255, + "grad_norm": 10.66627311706543, + "learning_rate": 2.858096518347179e-07, + "logits/chosen": -0.2865196466445923, + "logits/rejected": -0.2877465486526489, + "logps/chosen": -776.6019287109375, + "logps/ref_chosen": -56.31770324707031, + "logps/ref_rejected": -89.13836669921875, + "logps/rejected": -1164.4927978515625, + "loss": 1.307, + "margin_dpo/margin_mean": 355.0701904296875, + "margin_dpo/margin_std": 798.5880126953125, + "step": 348 + }, + { + "KL/chosen_KL_mean": -805.5579223632812, + "KL/mean": -1064.7337646484375, + "KL/rejected_KL_mean": -1323.90966796875, + "KL/std": 1003.7889404296875, + "epoch": 0.5124816446402349, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 518.3516845703125, + "fcm_dpo/q_t": 0.46881359815597534, + "grad_norm": 15.495862007141113, + "learning_rate": 2.845390887379706e-07, + "logits/chosen": -0.27927365899086, + "logits/rejected": -0.28703418374061584, + "logps/chosen": -863.5834350585938, + "logps/ref_chosen": -58.025516510009766, + "logps/ref_rejected": -97.50515747070312, + "logps/rejected": -1421.414794921875, + "loss": 1.2844, + "margin_dpo/margin_mean": 518.3516235351562, + "margin_dpo/margin_std": 1315.3780517578125, + "step": 349 + }, + { + "KL/chosen_KL_mean": -923.5478515625, + "KL/mean": -1110.17333984375, + "KL/rejected_KL_mean": -1296.798828125, + "KL/std": 911.0496826171875, + "epoch": 0.5139500734214391, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 373.2510070800781, + "fcm_dpo/q_t": 0.475941002368927, + "grad_norm": 21.998889923095703, + "learning_rate": 2.8326761550411346e-07, + "logits/chosen": -0.3065475821495056, + "logits/rejected": -0.31878340244293213, + "logps/chosen": -987.8783569335938, + "logps/ref_chosen": -64.33049011230469, + "logps/ref_rejected": -89.87164306640625, + "logps/rejected": -1386.67041015625, + "loss": 1.3163, + "margin_dpo/margin_mean": 373.2510070800781, + "margin_dpo/margin_std": 1196.018798828125, + "step": 350 + }, + { + "KL/chosen_KL_mean": -735.504638671875, + "KL/mean": -1000.615234375, + "KL/rejected_KL_mean": -1265.725830078125, + "KL/std": 860.2415771484375, + "epoch": 0.5154185022026432, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 530.22119140625, + "fcm_dpo/q_t": 0.46662163734436035, + "grad_norm": 10.427867889404297, + "learning_rate": 2.819952656376487e-07, + "logits/chosen": -0.3097544312477112, + "logits/rejected": -0.32192495465278625, + "logps/chosen": -796.1767578125, + "logps/ref_chosen": -60.6721305847168, + "logps/ref_rejected": -101.5654296875, + "logps/rejected": -1367.291259765625, + "loss": 1.2742, + "margin_dpo/margin_mean": 530.22119140625, + "margin_dpo/margin_std": 1098.137939453125, + "step": 351 + }, + { + "KL/chosen_KL_mean": -911.3470458984375, + "KL/mean": -1003.693359375, + "KL/rejected_KL_mean": -1096.0396728515625, + "KL/std": 876.0338134765625, + "epoch": 0.5168869309838473, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 184.69261169433594, + "fcm_dpo/q_t": 0.4880898594856262, + "grad_norm": 19.379594802856445, + "learning_rate": 2.8072207266617854e-07, + "logits/chosen": -0.30126988887786865, + "logits/rejected": -0.2974497675895691, + "logps/chosen": -982.29052734375, + "logps/ref_chosen": -70.9434585571289, + "logps/ref_rejected": -76.6419677734375, + "logps/rejected": -1172.681640625, + "loss": 1.3548, + "margin_dpo/margin_mean": 184.69261169433594, + "margin_dpo/margin_std": 969.9511108398438, + "step": 352 + }, + { + "KL/chosen_KL_mean": -786.5258178710938, + "KL/mean": -942.9754028320312, + "KL/rejected_KL_mean": -1099.425048828125, + "KL/std": 793.8702392578125, + "epoch": 0.5183553597650514, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 312.899169921875, + "fcm_dpo/q_t": 0.4803735613822937, + "grad_norm": 14.741339683532715, + "learning_rate": 2.794480701395219e-07, + "logits/chosen": -0.2722886800765991, + "logits/rejected": -0.2665886878967285, + "logps/chosen": -844.921142578125, + "logps/ref_chosen": -58.39533996582031, + "logps/ref_rejected": -80.33553314208984, + "logps/rejected": -1179.760498046875, + "loss": 1.3249, + "margin_dpo/margin_mean": 312.899169921875, + "margin_dpo/margin_std": 1029.1500244140625, + "step": 353 + }, + { + "KL/chosen_KL_mean": -602.9953002929688, + "KL/mean": -845.4747924804688, + "KL/rejected_KL_mean": -1087.9542236328125, + "KL/std": 870.7578125, + "epoch": 0.5198237885462555, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 484.9588928222656, + "fcm_dpo/q_t": 0.4696187674999237, + "grad_norm": 23.143972396850586, + "learning_rate": 2.781732916288303e-07, + "logits/chosen": -0.2781677842140198, + "logits/rejected": -0.2879522442817688, + "logps/chosen": -662.79833984375, + "logps/ref_chosen": -59.80299377441406, + "logps/ref_rejected": -88.75750732421875, + "logps/rejected": -1176.711669921875, + "loss": 1.2783, + "margin_dpo/margin_mean": 484.9588623046875, + "margin_dpo/margin_std": 908.888916015625, + "step": 354 + }, + { + "KL/chosen_KL_mean": -570.4051513671875, + "KL/mean": -812.153564453125, + "KL/rejected_KL_mean": -1053.9019775390625, + "KL/std": 624.933349609375, + "epoch": 0.5212922173274597, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 483.4968566894531, + "fcm_dpo/q_t": 0.468807578086853, + "grad_norm": 22.532615661621094, + "learning_rate": 2.7689777072570284e-07, + "logits/chosen": -0.26445209980010986, + "logits/rejected": -0.27031999826431274, + "logps/chosen": -624.5336303710938, + "logps/ref_chosen": -54.12849807739258, + "logps/ref_rejected": -82.40606689453125, + "logps/rejected": -1136.30810546875, + "loss": 1.2727, + "margin_dpo/margin_mean": 483.496826171875, + "margin_dpo/margin_std": 713.3695068359375, + "step": 355 + }, + { + "KL/chosen_KL_mean": -677.2150268554688, + "KL/mean": -766.095703125, + "KL/rejected_KL_mean": -854.9764404296875, + "KL/std": 658.3951416015625, + "epoch": 0.5227606461086637, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 177.76144409179688, + "fcm_dpo/q_t": 0.4887309670448303, + "grad_norm": 10.51893138885498, + "learning_rate": 2.7562154104130176e-07, + "logits/chosen": -0.2572939395904541, + "logits/rejected": -0.2499064952135086, + "logps/chosen": -741.8887939453125, + "logps/ref_chosen": -64.6738052368164, + "logps/ref_rejected": -75.89926147460938, + "logps/rejected": -930.875732421875, + "loss": 1.3528, + "margin_dpo/margin_mean": 177.76144409179688, + "margin_dpo/margin_std": 852.14697265625, + "step": 356 + }, + { + "KL/chosen_KL_mean": -588.591552734375, + "KL/mean": -709.5107421875, + "KL/rejected_KL_mean": -830.4298095703125, + "KL/std": 630.9422607421875, + "epoch": 0.5242290748898678, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 241.83822631835938, + "fcm_dpo/q_t": 0.4842875003814697, + "grad_norm": 20.39065170288086, + "learning_rate": 2.7434463620546594e-07, + "logits/chosen": -0.3094186782836914, + "logits/rejected": -0.310663104057312, + "logps/chosen": -641.3173828125, + "logps/ref_chosen": -52.725799560546875, + "logps/ref_rejected": -86.84115600585938, + "logps/rejected": -917.27099609375, + "loss": 1.3353, + "margin_dpo/margin_mean": 241.83822631835938, + "margin_dpo/margin_std": 793.8090209960938, + "step": 357 + }, + { + "KL/chosen_KL_mean": -518.7498779296875, + "KL/mean": -689.32568359375, + "KL/rejected_KL_mean": -859.9015502929688, + "KL/std": 604.7923583984375, + "epoch": 0.5256975036710719, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 341.15167236328125, + "fcm_dpo/q_t": 0.4782078266143799, + "grad_norm": 12.143132209777832, + "learning_rate": 2.730670898658255e-07, + "logits/chosen": -0.2932926118373871, + "logits/rejected": -0.29479849338531494, + "logps/chosen": -581.955322265625, + "logps/ref_chosen": -63.20543670654297, + "logps/ref_rejected": -88.373291015625, + "logps/rejected": -948.2748413085938, + "loss": 1.3097, + "margin_dpo/margin_mean": 341.15167236328125, + "margin_dpo/margin_std": 780.6080322265625, + "step": 358 + }, + { + "KL/chosen_KL_mean": -671.3048706054688, + "KL/mean": -872.405029296875, + "KL/rejected_KL_mean": -1073.505126953125, + "KL/std": 705.5187377929688, + "epoch": 0.527165932452276, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 402.20025634765625, + "fcm_dpo/q_t": 0.4743681848049164, + "grad_norm": 13.933921813964844, + "learning_rate": 2.717889356869146e-07, + "logits/chosen": -0.25826162099838257, + "logits/rejected": -0.2619970738887787, + "logps/chosen": -727.6751098632812, + "logps/ref_chosen": -56.370216369628906, + "logps/ref_rejected": -82.17375183105469, + "logps/rejected": -1155.678955078125, + "loss": 1.3002, + "margin_dpo/margin_mean": 402.2002258300781, + "margin_dpo/margin_std": 950.1470947265625, + "step": 359 + }, + { + "KL/chosen_KL_mean": -685.37353515625, + "KL/mean": -844.9564208984375, + "KL/rejected_KL_mean": -1004.539306640625, + "KL/std": 776.1425170898438, + "epoch": 0.5286343612334802, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 319.16571044921875, + "fcm_dpo/q_t": 0.47939565777778625, + "grad_norm": 11.378792762756348, + "learning_rate": 2.7051020734928443e-07, + "logits/chosen": -0.2582198679447174, + "logits/rejected": -0.257494181394577, + "logps/chosen": -736.8339233398438, + "logps/ref_chosen": -51.460384368896484, + "logps/ref_rejected": -69.83892059326172, + "logps/rejected": -1074.378173828125, + "loss": 1.3129, + "margin_dpo/margin_mean": 319.1656494140625, + "margin_dpo/margin_std": 692.6895751953125, + "step": 360 + }, + { + "KL/chosen_KL_mean": -665.7442626953125, + "KL/mean": -860.533935546875, + "KL/rejected_KL_mean": -1055.323486328125, + "KL/std": 791.1251220703125, + "epoch": 0.5301027900146843, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 389.5793151855469, + "fcm_dpo/q_t": 0.47519195079803467, + "grad_norm": 16.64234733581543, + "learning_rate": 2.6923093854861593e-07, + "logits/chosen": -0.26100149750709534, + "logits/rejected": -0.2740827798843384, + "logps/chosen": -719.61376953125, + "logps/ref_chosen": -53.86951446533203, + "logps/ref_rejected": -90.7692642211914, + "logps/rejected": -1146.0927734375, + "loss": 1.302, + "margin_dpo/margin_mean": 389.5793151855469, + "margin_dpo/margin_std": 934.74609375, + "step": 361 + }, + { + "KL/chosen_KL_mean": -607.8038330078125, + "KL/mean": -897.657958984375, + "KL/rejected_KL_mean": -1187.5120849609375, + "KL/std": 833.4835815429688, + "epoch": 0.5315712187958884, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 579.708251953125, + "fcm_dpo/q_t": 0.46335670351982117, + "grad_norm": 23.430381774902344, + "learning_rate": 2.679511629948319e-07, + "logits/chosen": -0.28775614500045776, + "logits/rejected": -0.2993890643119812, + "logps/chosen": -666.44287109375, + "logps/ref_chosen": -58.639060974121094, + "logps/ref_rejected": -105.58195495605469, + "logps/rejected": -1293.093994140625, + "loss": 1.2568, + "margin_dpo/margin_mean": 579.708251953125, + "margin_dpo/margin_std": 957.5960693359375, + "step": 362 + }, + { + "KL/chosen_KL_mean": -668.0380859375, + "KL/mean": -925.823974609375, + "KL/rejected_KL_mean": -1183.6099853515625, + "KL/std": 786.2000732421875, + "epoch": 0.5330396475770925, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 515.5718994140625, + "fcm_dpo/q_t": 0.4670901298522949, + "grad_norm": 13.35830020904541, + "learning_rate": 2.6667091441120816e-07, + "logits/chosen": -0.2682288885116577, + "logits/rejected": -0.2802722454071045, + "logps/chosen": -712.596435546875, + "logps/ref_chosen": -44.558380126953125, + "logps/ref_rejected": -74.69496154785156, + "logps/rejected": -1258.304931640625, + "loss": 1.2705, + "margin_dpo/margin_mean": 515.5718994140625, + "margin_dpo/margin_std": 912.45263671875, + "step": 363 + }, + { + "KL/chosen_KL_mean": -679.378662109375, + "KL/mean": -908.24169921875, + "KL/rejected_KL_mean": -1137.104736328125, + "KL/std": 799.5364379882812, + "epoch": 0.5345080763582967, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 457.7259826660156, + "fcm_dpo/q_t": 0.4708824157714844, + "grad_norm": 17.259096145629883, + "learning_rate": 2.6539022653348575e-07, + "logits/chosen": -0.30234140157699585, + "logits/rejected": -0.31423497200012207, + "logps/chosen": -728.2733154296875, + "logps/ref_chosen": -48.894622802734375, + "logps/ref_rejected": -91.395751953125, + "logps/rejected": -1228.50048828125, + "loss": 1.2858, + "margin_dpo/margin_mean": 457.7259826660156, + "margin_dpo/margin_std": 911.04443359375, + "step": 364 + }, + { + "KL/chosen_KL_mean": -783.33154296875, + "KL/mean": -1068.940185546875, + "KL/rejected_KL_mean": -1354.548828125, + "KL/std": 1015.5641479492188, + "epoch": 0.5359765051395007, + "fcm_dpo/beta": 0.00026103141135536134, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 571.2171630859375, + "fcm_dpo/q_t": 0.46466195583343506, + "grad_norm": 16.475479125976562, + "learning_rate": 2.641091331089811e-07, + "logits/chosen": -0.2803174555301666, + "logits/rejected": -0.30174314975738525, + "logps/chosen": -834.8242797851562, + "logps/ref_chosen": -51.49274444580078, + "logps/ref_rejected": -92.70166778564453, + "logps/rejected": -1447.25048828125, + "loss": 1.2675, + "margin_dpo/margin_mean": 571.2171630859375, + "margin_dpo/margin_std": 1223.385498046875, + "step": 365 + }, + { + "KL/chosen_KL_mean": -831.8895263671875, + "KL/mean": -1130.086669921875, + "KL/rejected_KL_mean": -1428.2838134765625, + "KL/std": 1027.395263671875, + "epoch": 0.5374449339207048, + "fcm_dpo/beta": 0.0014974254881963134, + "fcm_dpo/delta": 0.12657544016838074, + "fcm_dpo/margin": 596.3943481445312, + "fcm_dpo/q_t": 0.446040540933609, + "grad_norm": 130.77503967285156, + "learning_rate": 2.6282766789569736e-07, + "logits/chosen": -0.324015349149704, + "logits/rejected": -0.34143155813217163, + "logps/chosen": -876.610107421875, + "logps/ref_chosen": -44.7205696105957, + "logps/ref_rejected": -83.31040954589844, + "logps/rejected": -1511.59423828125, + "loss": 1.4374, + "margin_dpo/margin_mean": 596.3943481445312, + "margin_dpo/margin_std": 1373.01025390625, + "step": 366 + }, + { + "KL/chosen_KL_mean": -953.1829223632812, + "KL/mean": -1134.24951171875, + "KL/rejected_KL_mean": -1315.31591796875, + "KL/std": 867.8139038085938, + "epoch": 0.5389133627019089, + "fcm_dpo/beta": 0.0010564139811322093, + "fcm_dpo/delta": -0.019750438630580902, + "fcm_dpo/margin": 362.13287353515625, + "fcm_dpo/q_t": 0.42206183075904846, + "grad_norm": 113.0547103881836, + "learning_rate": 2.615458646614349e-07, + "logits/chosen": -0.31291621923446655, + "logits/rejected": -0.31141310930252075, + "logps/chosen": -1011.58837890625, + "logps/ref_chosen": -58.405418395996094, + "logps/ref_rejected": -76.75132751464844, + "logps/rejected": -1392.067138671875, + "loss": 1.322, + "margin_dpo/margin_mean": 362.1329345703125, + "margin_dpo/margin_std": 1070.106201171875, + "step": 367 + }, + { + "KL/chosen_KL_mean": -768.363037109375, + "KL/mean": -1200.409912109375, + "KL/rejected_KL_mean": -1632.4569091796875, + "KL/std": 953.82568359375, + "epoch": 0.540381791483113, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": -0.44886526465415955, + "fcm_dpo/margin": 864.0938720703125, + "fcm_dpo/q_t": 0.39556068181991577, + "grad_norm": 0.13906408846378326, + "learning_rate": 2.6026375718290083e-07, + "logits/chosen": -0.3350134491920471, + "logits/rejected": -0.3492249548435211, + "logps/chosen": -812.8155517578125, + "logps/ref_chosen": -44.452518463134766, + "logps/ref_rejected": -98.55526733398438, + "logps/rejected": -1731.01220703125, + "loss": 1.3853, + "margin_dpo/margin_mean": 864.0938720703125, + "margin_dpo/margin_std": 1078.280029296875, + "step": 368 + }, + { + "KL/chosen_KL_mean": -998.696533203125, + "KL/mean": -1232.2938232421875, + "KL/rejected_KL_mean": -1465.89111328125, + "KL/std": 959.7288208007812, + "epoch": 0.5418502202643172, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 467.19464111328125, + "fcm_dpo/q_t": 0.4998719394207001, + "grad_norm": 0.0773184671998024, + "learning_rate": 2.589813792448196e-07, + "logits/chosen": -0.3440535366535187, + "logits/rejected": -0.3447309732437134, + "logps/chosen": -1070.0780029296875, + "logps/ref_chosen": -71.38150024414062, + "logps/ref_rejected": -91.29582214355469, + "logps/rejected": -1557.18701171875, + "loss": 1.3858, + "margin_dpo/margin_mean": 467.19464111328125, + "margin_dpo/margin_std": 1237.226806640625, + "step": 369 + }, + { + "KL/chosen_KL_mean": -1075.063720703125, + "KL/mean": -1165.5103759765625, + "KL/rejected_KL_mean": -1255.95703125, + "KL/std": 1011.834228515625, + "epoch": 0.5433186490455213, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 180.89346313476562, + "fcm_dpo/q_t": 0.4999504089355469, + "grad_norm": 0.07674546539783478, + "learning_rate": 2.5769876463904263e-07, + "logits/chosen": -0.33612939715385437, + "logits/rejected": -0.3382912278175354, + "logps/chosen": -1146.671142578125, + "logps/ref_chosen": -71.60749816894531, + "logps/ref_rejected": -97.25978088378906, + "logps/rejected": -1353.216796875, + "loss": 1.3861, + "margin_dpo/margin_mean": 180.89344787597656, + "margin_dpo/margin_std": 1252.171630859375, + "step": 370 + }, + { + "KL/chosen_KL_mean": -1031.171142578125, + "KL/mean": -1296.2216796875, + "KL/rejected_KL_mean": -1561.272216796875, + "KL/std": 1119.242919921875, + "epoch": 0.5447870778267254, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 530.1011352539062, + "fcm_dpo/q_t": 0.4998546838760376, + "grad_norm": 0.06114376336336136, + "learning_rate": 2.5641594716365744e-07, + "logits/chosen": -0.342197060585022, + "logits/rejected": -0.3496757447719574, + "logps/chosen": -1100.5855712890625, + "logps/ref_chosen": -69.41448974609375, + "logps/ref_rejected": -99.17217254638672, + "logps/rejected": -1660.4443359375, + "loss": 1.3857, + "margin_dpo/margin_mean": 530.1011962890625, + "margin_dpo/margin_std": 1297.156494140625, + "step": 371 + }, + { + "KL/chosen_KL_mean": -913.1492919921875, + "KL/mean": -1198.9420166015625, + "KL/rejected_KL_mean": -1484.73486328125, + "KL/std": 1118.00634765625, + "epoch": 0.5462555066079295, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 571.58544921875, + "fcm_dpo/q_t": 0.4998433291912079, + "grad_norm": 0.058624256402254105, + "learning_rate": 2.551329606220976e-07, + "logits/chosen": -0.2874826192855835, + "logits/rejected": -0.3007184565067291, + "logps/chosen": -974.9673461914062, + "logps/ref_chosen": -61.8179931640625, + "logps/ref_rejected": -78.53948974609375, + "logps/rejected": -1563.2742919921875, + "loss": 1.3857, + "margin_dpo/margin_mean": 571.58544921875, + "margin_dpo/margin_std": 1257.53955078125, + "step": 372 + }, + { + "KL/chosen_KL_mean": -990.4146118164062, + "KL/mean": -1182.953857421875, + "KL/rejected_KL_mean": -1375.4930419921875, + "KL/std": 983.2168579101562, + "epoch": 0.5477239353891337, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 385.07843017578125, + "fcm_dpo/q_t": 0.49989446997642517, + "grad_norm": 0.07196224480867386, + "learning_rate": 2.538498388222517e-07, + "logits/chosen": -0.27686309814453125, + "logits/rejected": -0.27830153703689575, + "logps/chosen": -1054.6317138671875, + "logps/ref_chosen": -64.21713256835938, + "logps/ref_rejected": -85.95960998535156, + "logps/rejected": -1461.45263671875, + "loss": 1.3859, + "margin_dpo/margin_mean": 385.07843017578125, + "margin_dpo/margin_std": 1091.12890625, + "step": 373 + }, + { + "KL/chosen_KL_mean": -760.64453125, + "KL/mean": -1015.76416015625, + "KL/rejected_KL_mean": -1270.8837890625, + "KL/std": 982.1947021484375, + "epoch": 0.5491923641703378, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 510.2392883300781, + "fcm_dpo/q_t": 0.49986016750335693, + "grad_norm": 0.0749947726726532, + "learning_rate": 2.525666155755725e-07, + "logits/chosen": -0.3357512354850769, + "logits/rejected": -0.33956992626190186, + "logps/chosen": -831.2947387695312, + "logps/ref_chosen": -70.65018463134766, + "logps/ref_rejected": -93.64016723632812, + "logps/rejected": -1364.52392578125, + "loss": 1.3857, + "margin_dpo/margin_mean": 510.23931884765625, + "margin_dpo/margin_std": 1257.8927001953125, + "step": 374 + }, + { + "KL/chosen_KL_mean": -773.878662109375, + "KL/mean": -1017.3250732421875, + "KL/rejected_KL_mean": -1260.771484375, + "KL/std": 752.9087524414062, + "epoch": 0.5506607929515418, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 486.89276123046875, + "fcm_dpo/q_t": 0.4998665452003479, + "grad_norm": 0.10702349245548248, + "learning_rate": 2.512833246961859e-07, + "logits/chosen": -0.2944202721118927, + "logits/rejected": -0.3069683909416199, + "logps/chosen": -833.9588623046875, + "logps/ref_chosen": -60.080223083496094, + "logps/ref_rejected": -88.93830871582031, + "logps/rejected": -1349.709716796875, + "loss": 1.3858, + "margin_dpo/margin_mean": 486.8927917480469, + "margin_dpo/margin_std": 893.77978515625, + "step": 375 + }, + { + "KL/chosen_KL_mean": -906.502685546875, + "KL/mean": -1175.470703125, + "KL/rejected_KL_mean": -1444.438720703125, + "KL/std": 964.0004272460938, + "epoch": 0.5521292217327459, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 537.9361572265625, + "fcm_dpo/q_t": 0.49985256791114807, + "grad_norm": 0.08315837383270264, + "learning_rate": 2.5e-07, + "logits/chosen": -0.30217331647872925, + "logits/rejected": -0.30075645446777344, + "logps/chosen": -969.1629638671875, + "logps/ref_chosen": -62.660308837890625, + "logps/ref_rejected": -105.52660369873047, + "logps/rejected": -1549.96533203125, + "loss": 1.3857, + "margin_dpo/margin_mean": 537.9361572265625, + "margin_dpo/margin_std": 1077.5111083984375, + "step": 376 + }, + { + "KL/chosen_KL_mean": -898.3602905273438, + "KL/mean": -1259.601318359375, + "KL/rejected_KL_mean": -1620.84228515625, + "KL/std": 1318.417724609375, + "epoch": 0.55359765051395, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 722.4820556640625, + "fcm_dpo/q_t": 0.49980199337005615, + "grad_norm": 0.2360827922821045, + "learning_rate": 2.487166753038141e-07, + "logits/chosen": -0.2579605281352997, + "logits/rejected": -0.27427393198013306, + "logps/chosen": -952.8390502929688, + "logps/ref_chosen": -54.478736877441406, + "logps/ref_rejected": -98.70335388183594, + "logps/rejected": -1719.545654296875, + "loss": 1.3855, + "margin_dpo/margin_mean": 722.4820556640625, + "margin_dpo/margin_std": 1614.525634765625, + "step": 377 + }, + { + "KL/chosen_KL_mean": -706.89794921875, + "KL/mean": -1019.0738525390625, + "KL/rejected_KL_mean": -1331.2496337890625, + "KL/std": 896.5899047851562, + "epoch": 0.5550660792951542, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 624.3516845703125, + "fcm_dpo/q_t": 0.49982887506484985, + "grad_norm": 0.07591956853866577, + "learning_rate": 2.4743338442442754e-07, + "logits/chosen": -0.2470046877861023, + "logits/rejected": -0.26460781693458557, + "logps/chosen": -751.9185180664062, + "logps/ref_chosen": -45.02053451538086, + "logps/ref_rejected": -88.0469741821289, + "logps/rejected": -1419.296630859375, + "loss": 1.3856, + "margin_dpo/margin_mean": 624.3516235351562, + "margin_dpo/margin_std": 1124.9613037109375, + "step": 378 + }, + { + "KL/chosen_KL_mean": -1031.705078125, + "KL/mean": -1375.2432861328125, + "KL/rejected_KL_mean": -1718.7816162109375, + "KL/std": 1386.5126953125, + "epoch": 0.5565345080763583, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 687.07666015625, + "fcm_dpo/q_t": 0.49981167912483215, + "grad_norm": 0.19187086820602417, + "learning_rate": 2.461501611777483e-07, + "logits/chosen": -0.2563135623931885, + "logits/rejected": -0.2795560956001282, + "logps/chosen": -1084.88720703125, + "logps/ref_chosen": -53.182098388671875, + "logps/ref_rejected": -114.3001708984375, + "logps/rejected": -1833.081787109375, + "loss": 1.3855, + "margin_dpo/margin_mean": 687.0765991210938, + "margin_dpo/margin_std": 1713.9254150390625, + "step": 379 + }, + { + "KL/chosen_KL_mean": -902.8639526367188, + "KL/mean": -1237.943359375, + "KL/rejected_KL_mean": -1573.0228271484375, + "KL/std": 1124.9130859375, + "epoch": 0.5580029368575624, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 670.158935546875, + "fcm_dpo/q_t": 0.49981629848480225, + "grad_norm": 0.0999961718916893, + "learning_rate": 2.4486703937790243e-07, + "logits/chosen": -0.2708591818809509, + "logits/rejected": -0.29440176486968994, + "logps/chosen": -954.2169799804688, + "logps/ref_chosen": -51.3530387878418, + "logps/ref_rejected": -104.19169616699219, + "logps/rejected": -1677.214599609375, + "loss": 1.3856, + "margin_dpo/margin_mean": 670.158935546875, + "margin_dpo/margin_std": 1411.98486328125, + "step": 380 + }, + { + "KL/chosen_KL_mean": -949.5911865234375, + "KL/mean": -1092.4285888671875, + "KL/rejected_KL_mean": -1235.26611328125, + "KL/std": 1007.7601318359375, + "epoch": 0.5594713656387665, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 285.6749267578125, + "fcm_dpo/q_t": 0.49992167949676514, + "grad_norm": 0.06599840521812439, + "learning_rate": 2.435840528363426e-07, + "logits/chosen": -0.28272417187690735, + "logits/rejected": -0.2873557209968567, + "logps/chosen": -1007.394287109375, + "logps/ref_chosen": -57.80306625366211, + "logps/ref_rejected": -79.21940612792969, + "logps/rejected": -1314.4854736328125, + "loss": 1.386, + "margin_dpo/margin_mean": 285.6749267578125, + "margin_dpo/margin_std": 1382.336181640625, + "step": 381 + }, + { + "KL/chosen_KL_mean": -899.02978515625, + "KL/mean": -1149.078369140625, + "KL/rejected_KL_mean": -1399.1270751953125, + "KL/std": 840.8511962890625, + "epoch": 0.5609397944199707, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 500.0972900390625, + "fcm_dpo/q_t": 0.4998629093170166, + "grad_norm": 0.14141029119491577, + "learning_rate": 2.4230123536095745e-07, + "logits/chosen": -0.3087759017944336, + "logits/rejected": -0.3224329948425293, + "logps/chosen": -965.050048828125, + "logps/ref_chosen": -66.02030181884766, + "logps/ref_rejected": -110.71016693115234, + "logps/rejected": -1509.837158203125, + "loss": 1.3857, + "margin_dpo/margin_mean": 500.0972595214844, + "margin_dpo/margin_std": 881.119384765625, + "step": 382 + }, + { + "KL/chosen_KL_mean": -956.48486328125, + "KL/mean": -1221.72021484375, + "KL/rejected_KL_mean": -1486.9556884765625, + "KL/std": 1089.745849609375, + "epoch": 0.5624082232011748, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 530.4708862304688, + "fcm_dpo/q_t": 0.49985459446907043, + "grad_norm": 0.18441635370254517, + "learning_rate": 2.4101862075518037e-07, + "logits/chosen": -0.2525390684604645, + "logits/rejected": -0.2726495563983917, + "logps/chosen": -1006.8762817382812, + "logps/ref_chosen": -50.39148712158203, + "logps/ref_rejected": -93.71589660644531, + "logps/rejected": -1580.671630859375, + "loss": 1.3857, + "margin_dpo/margin_mean": 530.4708862304688, + "margin_dpo/margin_std": 1537.28515625, + "step": 383 + }, + { + "KL/chosen_KL_mean": -968.1121826171875, + "KL/mean": -1152.972900390625, + "KL/rejected_KL_mean": -1337.8336181640625, + "KL/std": 965.8948364257812, + "epoch": 0.5638766519823789, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 369.7212829589844, + "fcm_dpo/q_t": 0.49989867210388184, + "grad_norm": 0.07994517683982849, + "learning_rate": 2.397362428170992e-07, + "logits/chosen": -0.31392794847488403, + "logits/rejected": -0.31402724981307983, + "logps/chosen": -1020.1583251953125, + "logps/ref_chosen": -52.046104431152344, + "logps/ref_rejected": -85.76089477539062, + "logps/rejected": -1423.594482421875, + "loss": 1.3859, + "margin_dpo/margin_mean": 369.7212829589844, + "margin_dpo/margin_std": 968.5846557617188, + "step": 384 + }, + { + "KL/chosen_KL_mean": -954.828369140625, + "KL/mean": -1170.8798828125, + "KL/rejected_KL_mean": -1386.931396484375, + "KL/std": 830.7667236328125, + "epoch": 0.5653450807635829, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 432.1029357910156, + "fcm_dpo/q_t": 0.4998815655708313, + "grad_norm": 0.0972379669547081, + "learning_rate": 2.3845413533856514e-07, + "logits/chosen": -0.3255078196525574, + "logits/rejected": -0.3330804109573364, + "logps/chosen": -1020.3804931640625, + "logps/ref_chosen": -65.55215454101562, + "logps/ref_rejected": -77.82792663574219, + "logps/rejected": -1464.75927734375, + "loss": 1.3858, + "margin_dpo/margin_mean": 432.1029357910156, + "margin_dpo/margin_std": 1054.419921875, + "step": 385 + }, + { + "KL/chosen_KL_mean": -900.58447265625, + "KL/mean": -1217.9410400390625, + "KL/rejected_KL_mean": -1535.2978515625, + "KL/std": 988.79345703125, + "epoch": 0.566813509544787, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 634.7132568359375, + "fcm_dpo/q_t": 0.499826043844223, + "grad_norm": 0.1786276251077652, + "learning_rate": 2.3717233210430254e-07, + "logits/chosen": -0.30368974804878235, + "logits/rejected": -0.3101522922515869, + "logps/chosen": -958.8062744140625, + "logps/ref_chosen": -58.22185516357422, + "logps/ref_rejected": -92.32742309570312, + "logps/rejected": -1627.625244140625, + "loss": 1.3856, + "margin_dpo/margin_mean": 634.7132568359375, + "margin_dpo/margin_std": 1239.7891845703125, + "step": 386 + }, + { + "KL/chosen_KL_mean": -1167.38916015625, + "KL/mean": -1465.2535400390625, + "KL/rejected_KL_mean": -1763.11767578125, + "KL/std": 1312.04248046875, + "epoch": 0.5682819383259912, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 595.728515625, + "fcm_dpo/q_t": 0.4998367428779602, + "grad_norm": 0.07909517735242844, + "learning_rate": 2.3589086689101889e-07, + "logits/chosen": -0.3058868646621704, + "logits/rejected": -0.30179405212402344, + "logps/chosen": -1233.8087158203125, + "logps/ref_chosen": -66.41944885253906, + "logps/ref_rejected": -92.16915893554688, + "logps/rejected": -1855.286865234375, + "loss": 1.3856, + "margin_dpo/margin_mean": 595.728515625, + "margin_dpo/margin_std": 1582.2099609375, + "step": 387 + }, + { + "KL/chosen_KL_mean": -1178.15576171875, + "KL/mean": -1469.792236328125, + "KL/rejected_KL_mean": -1761.428466796875, + "KL/std": 1194.9736328125, + "epoch": 0.5697503671071953, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 583.2728271484375, + "fcm_dpo/q_t": 0.49984011054039, + "grad_norm": 0.13137727975845337, + "learning_rate": 2.3460977346651428e-07, + "logits/chosen": -0.277870774269104, + "logits/rejected": -0.29402369260787964, + "logps/chosen": -1228.28515625, + "logps/ref_chosen": -50.129459381103516, + "logps/ref_rejected": -104.43305969238281, + "logps/rejected": -1865.861572265625, + "loss": 1.3857, + "margin_dpo/margin_mean": 583.2728271484375, + "margin_dpo/margin_std": 1514.8427734375, + "step": 388 + }, + { + "KL/chosen_KL_mean": -1263.1337890625, + "KL/mean": -1510.9075927734375, + "KL/rejected_KL_mean": -1758.681396484375, + "KL/std": 1231.6842041015625, + "epoch": 0.5712187958883994, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 495.54779052734375, + "fcm_dpo/q_t": 0.4998641610145569, + "grad_norm": 0.13468711078166962, + "learning_rate": 2.3332908558879177e-07, + "logits/chosen": -0.316806823015213, + "logits/rejected": -0.3238714635372162, + "logps/chosen": -1321.040283203125, + "logps/ref_chosen": -57.906593322753906, + "logps/ref_rejected": -77.91454315185547, + "logps/rejected": -1836.595947265625, + "loss": 1.3858, + "margin_dpo/margin_mean": 495.5478210449219, + "margin_dpo/margin_std": 1563.379638671875, + "step": 389 + }, + { + "KL/chosen_KL_mean": -1154.65478515625, + "KL/mean": -1450.0166015625, + "KL/rejected_KL_mean": -1745.378173828125, + "KL/std": 1158.6177978515625, + "epoch": 0.5726872246696035, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 590.7235107421875, + "fcm_dpo/q_t": 0.49983808398246765, + "grad_norm": 0.0688537061214447, + "learning_rate": 2.320488370051681e-07, + "logits/chosen": -0.29213494062423706, + "logits/rejected": -0.30260345339775085, + "logps/chosen": -1203.880615234375, + "logps/ref_chosen": -49.22591781616211, + "logps/ref_rejected": -85.5281982421875, + "logps/rejected": -1830.906494140625, + "loss": 1.3856, + "margin_dpo/margin_mean": 590.7235107421875, + "margin_dpo/margin_std": 1621.9512939453125, + "step": 390 + }, + { + "KL/chosen_KL_mean": -1452.38134765625, + "KL/mean": -1540.049072265625, + "KL/rejected_KL_mean": -1627.7166748046875, + "KL/std": 1353.0072021484375, + "epoch": 0.5741556534508077, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 175.33538818359375, + "fcm_dpo/q_t": 0.49995192885398865, + "grad_norm": 0.1264980137348175, + "learning_rate": 2.3076906145138405e-07, + "logits/chosen": -0.311432421207428, + "logits/rejected": -0.30648520588874817, + "logps/chosen": -1516.7109375, + "logps/ref_chosen": -64.32965087890625, + "logps/ref_rejected": -86.73820495605469, + "logps/rejected": -1714.454833984375, + "loss": 1.3861, + "margin_dpo/margin_mean": 175.3353729248047, + "margin_dpo/margin_std": 1589.905517578125, + "step": 391 + }, + { + "KL/chosen_KL_mean": -981.5897216796875, + "KL/mean": -1465.202880859375, + "KL/rejected_KL_mean": -1948.816162109375, + "KL/std": 1405.0225830078125, + "epoch": 0.5756240822320118, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 967.2265625, + "fcm_dpo/q_t": 0.49973487854003906, + "grad_norm": 0.13413800299167633, + "learning_rate": 2.294897926507156e-07, + "logits/chosen": -0.3217155933380127, + "logits/rejected": -0.33385881781578064, + "logps/chosen": -1035.09375, + "logps/ref_chosen": -53.50397872924805, + "logps/ref_rejected": -102.34584045410156, + "logps/rejected": -2051.162109375, + "loss": 1.3852, + "margin_dpo/margin_mean": 967.2265014648438, + "margin_dpo/margin_std": 1548.9447021484375, + "step": 392 + }, + { + "KL/chosen_KL_mean": -1129.197998046875, + "KL/mean": -1415.0509033203125, + "KL/rejected_KL_mean": -1700.9039306640625, + "KL/std": 1278.572998046875, + "epoch": 0.5770925110132159, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 571.7060546875, + "fcm_dpo/q_t": 0.4998432993888855, + "grad_norm": 0.09915830194950104, + "learning_rate": 2.2821106431308543e-07, + "logits/chosen": -0.30509477853775024, + "logits/rejected": -0.3083847165107727, + "logps/chosen": -1175.671875, + "logps/ref_chosen": -46.473915100097656, + "logps/ref_rejected": -71.96885681152344, + "logps/rejected": -1772.872802734375, + "loss": 1.3857, + "margin_dpo/margin_mean": 571.7060546875, + "margin_dpo/margin_std": 1579.58154296875, + "step": 393 + }, + { + "KL/chosen_KL_mean": -1186.830322265625, + "KL/mean": -1445.2498779296875, + "KL/rejected_KL_mean": -1703.66943359375, + "KL/std": 1274.763671875, + "epoch": 0.57856093979442, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 516.8390502929688, + "fcm_dpo/q_t": 0.4998583197593689, + "grad_norm": 0.06946764141321182, + "learning_rate": 2.2693291013417452e-07, + "logits/chosen": -0.3197329044342041, + "logits/rejected": -0.3295214772224426, + "logps/chosen": -1239.741943359375, + "logps/ref_chosen": -52.91154861450195, + "logps/ref_rejected": -90.8226318359375, + "logps/rejected": -1794.4920654296875, + "loss": 1.3857, + "margin_dpo/margin_mean": 516.839111328125, + "margin_dpo/margin_std": 1375.14990234375, + "step": 394 + }, + { + "KL/chosen_KL_mean": -1227.3748779296875, + "KL/mean": -1495.52783203125, + "KL/rejected_KL_mean": -1763.6806640625, + "KL/std": 1176.624267578125, + "epoch": 0.580029368575624, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 536.3056640625, + "fcm_dpo/q_t": 0.4998530149459839, + "grad_norm": 0.11004586517810822, + "learning_rate": 2.2565536379453404e-07, + "logits/chosen": -0.29644620418548584, + "logits/rejected": -0.2991763949394226, + "logps/chosen": -1289.9210205078125, + "logps/ref_chosen": -62.546112060546875, + "logps/ref_rejected": -83.78262329101562, + "logps/rejected": -1847.4632568359375, + "loss": 1.3857, + "margin_dpo/margin_mean": 536.3056640625, + "margin_dpo/margin_std": 1487.039794921875, + "step": 395 + }, + { + "KL/chosen_KL_mean": -1308.0919189453125, + "KL/mean": -1489.021240234375, + "KL/rejected_KL_mean": -1669.950439453125, + "KL/std": 1317.3895263671875, + "epoch": 0.5814977973568282, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 361.8585205078125, + "fcm_dpo/q_t": 0.49990081787109375, + "grad_norm": 0.08168221265077591, + "learning_rate": 2.2437845895869825e-07, + "logits/chosen": -0.3633381128311157, + "logits/rejected": -0.36133861541748047, + "logps/chosen": -1377.087890625, + "logps/ref_chosen": -68.99594116210938, + "logps/ref_rejected": -88.64665985107422, + "logps/rejected": -1758.59716796875, + "loss": 1.3859, + "margin_dpo/margin_mean": 361.8585205078125, + "margin_dpo/margin_std": 1191.16259765625, + "step": 396 + }, + { + "KL/chosen_KL_mean": -1209.757568359375, + "KL/mean": -1688.31787109375, + "KL/rejected_KL_mean": -2166.8779296875, + "KL/std": 1485.7529296875, + "epoch": 0.5829662261380323, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 957.1202392578125, + "fcm_dpo/q_t": 0.4997376799583435, + "grad_norm": 0.16516095399856567, + "learning_rate": 2.2310222927429716e-07, + "logits/chosen": -0.28989318013191223, + "logits/rejected": -0.30872002243995667, + "logps/chosen": -1271.0347900390625, + "logps/ref_chosen": -61.27716827392578, + "logps/ref_rejected": -103.11612701416016, + "logps/rejected": -2269.994140625, + "loss": 1.3852, + "margin_dpo/margin_mean": 957.1203002929688, + "margin_dpo/margin_std": 1786.31591796875, + "step": 397 + }, + { + "KL/chosen_KL_mean": -1300.635986328125, + "KL/mean": -1687.136962890625, + "KL/rejected_KL_mean": -2073.637451171875, + "KL/std": 1495.1741943359375, + "epoch": 0.5844346549192364, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 773.0015258789062, + "fcm_dpo/q_t": 0.4997881352901459, + "grad_norm": 0.11998479068279266, + "learning_rate": 2.2182670837116972e-07, + "logits/chosen": -0.3601279854774475, + "logits/rejected": -0.37861526012420654, + "logps/chosen": -1368.78759765625, + "logps/ref_chosen": -68.15155029296875, + "logps/ref_rejected": -108.52360534667969, + "logps/rejected": -2182.1611328125, + "loss": 1.3854, + "margin_dpo/margin_mean": 773.0015258789062, + "margin_dpo/margin_std": 1943.517822265625, + "step": 398 + }, + { + "KL/chosen_KL_mean": -1110.9375, + "KL/mean": -1328.4151611328125, + "KL/rejected_KL_mean": -1545.89306640625, + "KL/std": 1136.963134765625, + "epoch": 0.5859030837004405, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 434.9555358886719, + "fcm_dpo/q_t": 0.4998807907104492, + "grad_norm": 0.17861014604568481, + "learning_rate": 2.2055192986047804e-07, + "logits/chosen": -0.3131040334701538, + "logits/rejected": -0.3180689215660095, + "logps/chosen": -1171.8271484375, + "logps/ref_chosen": -60.889801025390625, + "logps/ref_rejected": -77.965576171875, + "logps/rejected": -1623.858642578125, + "loss": 1.3858, + "margin_dpo/margin_mean": 434.95556640625, + "margin_dpo/margin_std": 1538.548583984375, + "step": 399 + }, + { + "KL/chosen_KL_mean": -1076.3807373046875, + "KL/mean": -1457.690673828125, + "KL/rejected_KL_mean": -1839.00048828125, + "KL/std": 1168.469482421875, + "epoch": 0.5873715124816447, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 762.619873046875, + "fcm_dpo/q_t": 0.4997909665107727, + "grad_norm": 0.11573357880115509, + "learning_rate": 2.192779273338215e-07, + "logits/chosen": -0.2832724153995514, + "logits/rejected": -0.2879381775856018, + "logps/chosen": -1140.0242919921875, + "logps/ref_chosen": -63.64359664916992, + "logps/ref_rejected": -105.252685546875, + "logps/rejected": -1944.253173828125, + "loss": 1.3855, + "margin_dpo/margin_mean": 762.619873046875, + "margin_dpo/margin_std": 1511.848388671875, + "step": 400 + }, + { + "KL/chosen_KL_mean": -1283.578369140625, + "KL/mean": -1460.8790283203125, + "KL/rejected_KL_mean": -1638.1796875, + "KL/std": 1255.499267578125, + "epoch": 0.5888399412628488, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 354.60137939453125, + "fcm_dpo/q_t": 0.4999028146266937, + "grad_norm": 0.0686601996421814, + "learning_rate": 2.1800473436235136e-07, + "logits/chosen": -0.29005157947540283, + "logits/rejected": -0.28908443450927734, + "logps/chosen": -1340.7413330078125, + "logps/ref_chosen": -57.16303253173828, + "logps/ref_rejected": -83.79249572753906, + "logps/rejected": -1721.97216796875, + "loss": 1.3859, + "margin_dpo/margin_mean": 354.60137939453125, + "margin_dpo/margin_std": 1711.057861328125, + "step": 401 + }, + { + "KL/chosen_KL_mean": -1143.9306640625, + "KL/mean": -1476.869384765625, + "KL/rejected_KL_mean": -1809.80810546875, + "KL/std": 1272.91845703125, + "epoch": 0.5903083700440529, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 665.8773193359375, + "fcm_dpo/q_t": 0.49981752038002014, + "grad_norm": 0.14180243015289307, + "learning_rate": 2.1673238449588665e-07, + "logits/chosen": -0.2916412651538849, + "logits/rejected": -0.29614734649658203, + "logps/chosen": -1194.671142578125, + "logps/ref_chosen": -50.74037170410156, + "logps/ref_rejected": -81.0460433959961, + "logps/rejected": -1890.8541259765625, + "loss": 1.3856, + "margin_dpo/margin_mean": 665.8773193359375, + "margin_dpo/margin_std": 1277.2041015625, + "step": 402 + }, + { + "KL/chosen_KL_mean": -1045.534912109375, + "KL/mean": -1374.931396484375, + "KL/rejected_KL_mean": -1704.3280029296875, + "KL/std": 1424.170654296875, + "epoch": 0.591776798825257, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 658.793212890625, + "fcm_dpo/q_t": 0.49981942772865295, + "grad_norm": 0.09196604043245316, + "learning_rate": 2.154609112620295e-07, + "logits/chosen": -0.2864852547645569, + "logits/rejected": -0.3000401258468628, + "logps/chosen": -1092.68212890625, + "logps/ref_chosen": -47.14731216430664, + "logps/ref_rejected": -77.2666015625, + "logps/rejected": -1781.5946044921875, + "loss": 1.3856, + "margin_dpo/margin_mean": 658.7931518554688, + "margin_dpo/margin_std": 1556.16015625, + "step": 403 + }, + { + "KL/chosen_KL_mean": -1062.3935546875, + "KL/mean": -1348.3885498046875, + "KL/rejected_KL_mean": -1634.3836669921875, + "KL/std": 1081.5550537109375, + "epoch": 0.593245227606461, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 571.990234375, + "fcm_dpo/q_t": 0.49984320998191833, + "grad_norm": 0.08485666662454605, + "learning_rate": 2.1419034816528218e-07, + "logits/chosen": -0.27411210536956787, + "logits/rejected": -0.2838093042373657, + "logps/chosen": -1110.268798828125, + "logps/ref_chosen": -47.875274658203125, + "logps/ref_rejected": -77.15499877929688, + "logps/rejected": -1711.53857421875, + "loss": 1.3857, + "margin_dpo/margin_mean": 571.990234375, + "margin_dpo/margin_std": 1484.571533203125, + "step": 404 + }, + { + "KL/chosen_KL_mean": -1302.394775390625, + "KL/mean": -1553.5927734375, + "KL/rejected_KL_mean": -1804.7906494140625, + "KL/std": 1502.25146484375, + "epoch": 0.5947136563876652, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 502.3959655761719, + "fcm_dpo/q_t": 0.49986231327056885, + "grad_norm": 0.18242168426513672, + "learning_rate": 2.129207286861638e-07, + "logits/chosen": -0.30928295850753784, + "logits/rejected": -0.319208562374115, + "logps/chosen": -1367.5576171875, + "logps/ref_chosen": -65.16290283203125, + "logps/ref_rejected": -87.18678283691406, + "logps/rejected": -1891.9774169921875, + "loss": 1.3857, + "margin_dpo/margin_mean": 502.3959655761719, + "margin_dpo/margin_std": 1869.696044921875, + "step": 405 + }, + { + "KL/chosen_KL_mean": -1168.0423583984375, + "KL/mean": -1538.92822265625, + "KL/rejected_KL_mean": -1909.814208984375, + "KL/std": 1370.6793212890625, + "epoch": 0.5961820851688693, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 741.771728515625, + "fcm_dpo/q_t": 0.49979668855667114, + "grad_norm": 0.15367038547992706, + "learning_rate": 2.1165208628032861e-07, + "logits/chosen": -0.2888413965702057, + "logits/rejected": -0.3083066940307617, + "logps/chosen": -1217.783203125, + "logps/ref_chosen": -49.740814208984375, + "logps/ref_rejected": -92.07862854003906, + "logps/rejected": -2001.892822265625, + "loss": 1.3855, + "margin_dpo/margin_mean": 741.771728515625, + "margin_dpo/margin_std": 1697.0975341796875, + "step": 406 + }, + { + "KL/chosen_KL_mean": -1067.0177001953125, + "KL/mean": -1228.9609375, + "KL/rejected_KL_mean": -1390.904296875, + "KL/std": 1061.7740478515625, + "epoch": 0.5976505139500734, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 323.8866271972656, + "fcm_dpo/q_t": 0.49991121888160706, + "grad_norm": 0.0722283273935318, + "learning_rate": 2.1038445437768375e-07, + "logits/chosen": -0.26621389389038086, + "logits/rejected": -0.267297625541687, + "logps/chosen": -1123.348388671875, + "logps/ref_chosen": -56.33069610595703, + "logps/ref_rejected": -77.51209259033203, + "logps/rejected": -1468.4163818359375, + "loss": 1.3859, + "margin_dpo/margin_mean": 323.88665771484375, + "margin_dpo/margin_std": 1483.5982666015625, + "step": 407 + }, + { + "KL/chosen_KL_mean": -1197.566162109375, + "KL/mean": -1406.6339111328125, + "KL/rejected_KL_mean": -1615.701416015625, + "KL/std": 1249.1373291015625, + "epoch": 0.5991189427312775, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 418.13525390625, + "fcm_dpo/q_t": 0.4998853802680969, + "grad_norm": 0.08033311367034912, + "learning_rate": 2.0911786638150872e-07, + "logits/chosen": -0.2984810471534729, + "logits/rejected": -0.3063961863517761, + "logps/chosen": -1267.35546875, + "logps/ref_chosen": -69.789306640625, + "logps/ref_rejected": -90.09693908691406, + "logps/rejected": -1705.7984619140625, + "loss": 1.3858, + "margin_dpo/margin_mean": 418.13525390625, + "margin_dpo/margin_std": 1451.70849609375, + "step": 408 + }, + { + "KL/chosen_KL_mean": -1052.406982421875, + "KL/mean": -1284.6824951171875, + "KL/rejected_KL_mean": -1516.9578857421875, + "KL/std": 1181.7725830078125, + "epoch": 0.6005873715124816, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 464.55096435546875, + "fcm_dpo/q_t": 0.49987268447875977, + "grad_norm": 0.09263404458761215, + "learning_rate": 2.0785235566757517e-07, + "logits/chosen": -0.3008713126182556, + "logits/rejected": -0.3142474293708801, + "logps/chosen": -1119.724365234375, + "logps/ref_chosen": -67.31744384765625, + "logps/ref_rejected": -84.904296875, + "logps/rejected": -1601.8621826171875, + "loss": 1.3858, + "margin_dpo/margin_mean": 464.5509033203125, + "margin_dpo/margin_std": 1583.107177734375, + "step": 409 + }, + { + "KL/chosen_KL_mean": -1056.844482421875, + "KL/mean": -1352.3720703125, + "KL/rejected_KL_mean": -1647.899658203125, + "KL/std": 1171.762451171875, + "epoch": 0.6020558002936858, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 591.05517578125, + "fcm_dpo/q_t": 0.4998379945755005, + "grad_norm": 0.08020097762346268, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": -0.27830201387405396, + "logits/rejected": -0.28551220893859863, + "logps/chosen": -1108.309814453125, + "logps/ref_chosen": -51.465354919433594, + "logps/ref_rejected": -83.198974609375, + "logps/rejected": -1731.0986328125, + "loss": 1.3856, + "margin_dpo/margin_mean": 591.05517578125, + "margin_dpo/margin_std": 1348.9642333984375, + "step": 410 + }, + { + "KL/chosen_KL_mean": -1087.2542724609375, + "KL/mean": -1336.627685546875, + "KL/rejected_KL_mean": -1586.0010986328125, + "KL/std": 1284.406494140625, + "epoch": 0.6035242290748899, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 498.746826171875, + "fcm_dpo/q_t": 0.49986329674720764, + "grad_norm": 0.06663346290588379, + "learning_rate": 2.0532469944670343e-07, + "logits/chosen": -0.24104216694831848, + "logits/rejected": -0.24600106477737427, + "logps/chosen": -1139.5615234375, + "logps/ref_chosen": -52.30727005004883, + "logps/ref_rejected": -80.69495391845703, + "logps/rejected": -1666.696044921875, + "loss": 1.3857, + "margin_dpo/margin_mean": 498.7467956542969, + "margin_dpo/margin_std": 1673.5802001953125, + "step": 411 + }, + { + "KL/chosen_KL_mean": -1194.093994140625, + "KL/mean": -1541.434326171875, + "KL/rejected_KL_mean": -1888.7745361328125, + "KL/std": 1356.6710205078125, + "epoch": 0.604992657856094, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 694.6804809570312, + "fcm_dpo/q_t": 0.499809592962265, + "grad_norm": 0.16892312467098236, + "learning_rate": 2.0406262054585738e-07, + "logits/chosen": -0.2868436574935913, + "logits/rejected": -0.3128548562526703, + "logps/chosen": -1247.2381591796875, + "logps/ref_chosen": -53.144126892089844, + "logps/ref_rejected": -100.0608139038086, + "logps/rejected": -1988.8353271484375, + "loss": 1.3855, + "margin_dpo/margin_mean": 694.6804809570312, + "margin_dpo/margin_std": 1838.8818359375, + "step": 412 + }, + { + "KL/chosen_KL_mean": -1412.858642578125, + "KL/mean": -1703.552734375, + "KL/rejected_KL_mean": -1994.246826171875, + "KL/std": 1468.816162109375, + "epoch": 0.6064610866372981, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 581.3883056640625, + "fcm_dpo/q_t": 0.499840646982193, + "grad_norm": 0.09038924425840378, + "learning_rate": 2.0280175213768205e-07, + "logits/chosen": -0.27624720335006714, + "logits/rejected": -0.2850080728530884, + "logps/chosen": -1474.440673828125, + "logps/ref_chosen": -61.58196258544922, + "logps/ref_rejected": -99.47340393066406, + "logps/rejected": -2093.72021484375, + "loss": 1.3857, + "margin_dpo/margin_mean": 581.3883056640625, + "margin_dpo/margin_std": 1843.23681640625, + "step": 413 + }, + { + "KL/chosen_KL_mean": -1188.63134765625, + "KL/mean": -1421.198974609375, + "KL/rejected_KL_mean": -1653.7666015625, + "KL/std": 1262.495849609375, + "epoch": 0.6079295154185022, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 465.1351318359375, + "fcm_dpo/q_t": 0.49987250566482544, + "grad_norm": 0.09474116563796997, + "learning_rate": 2.0154212744723247e-07, + "logits/chosen": -0.255864679813385, + "logits/rejected": -0.2618110179901123, + "logps/chosen": -1235.262939453125, + "logps/ref_chosen": -46.63148498535156, + "logps/ref_rejected": -87.64653015136719, + "logps/rejected": -1741.4130859375, + "loss": 1.3858, + "margin_dpo/margin_mean": 465.13507080078125, + "margin_dpo/margin_std": 1664.598876953125, + "step": 414 + }, + { + "KL/chosen_KL_mean": -1491.892578125, + "KL/mean": -1723.411865234375, + "KL/rejected_KL_mean": -1954.93115234375, + "KL/std": 1511.8848876953125, + "epoch": 0.6093979441997063, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 463.03875732421875, + "fcm_dpo/q_t": 0.4998731017112732, + "grad_norm": 0.09545289725065231, + "learning_rate": 2.002837796667909e-07, + "logits/chosen": -0.3036535382270813, + "logits/rejected": -0.3018915057182312, + "logps/chosen": -1570.5107421875, + "logps/ref_chosen": -78.6182861328125, + "logps/ref_rejected": -100.47752380371094, + "logps/rejected": -2055.40869140625, + "loss": 1.3858, + "margin_dpo/margin_mean": 463.03875732421875, + "margin_dpo/margin_std": 1996.5362548828125, + "step": 415 + }, + { + "KL/chosen_KL_mean": -1218.62744140625, + "KL/mean": -1573.49609375, + "KL/rejected_KL_mean": -1928.3646240234375, + "KL/std": 1414.0084228515625, + "epoch": 0.6108663729809104, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 709.7371215820312, + "fcm_dpo/q_t": 0.4998054504394531, + "grad_norm": 0.09039551764726639, + "learning_rate": 1.990267419549914e-07, + "logits/chosen": -0.2990315556526184, + "logits/rejected": -0.3177641034126282, + "logps/chosen": -1276.9066162109375, + "logps/ref_chosen": -58.27912521362305, + "logps/ref_rejected": -90.56871795654297, + "logps/rejected": -2018.933349609375, + "loss": 1.3855, + "margin_dpo/margin_mean": 709.7371215820312, + "margin_dpo/margin_std": 1649.296630859375, + "step": 416 + }, + { + "KL/chosen_KL_mean": -1047.98291015625, + "KL/mean": -1337.9010009765625, + "KL/rejected_KL_mean": -1627.819091796875, + "KL/std": 1125.5438232421875, + "epoch": 0.6123348017621145, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 579.8360595703125, + "fcm_dpo/q_t": 0.4998410642147064, + "grad_norm": 0.07524900883436203, + "learning_rate": 1.9777104743594686e-07, + "logits/chosen": -0.26090145111083984, + "logits/rejected": -0.26464271545410156, + "logps/chosen": -1098.181640625, + "logps/ref_chosen": -50.1987190246582, + "logps/ref_rejected": -68.15184020996094, + "logps/rejected": -1695.970947265625, + "loss": 1.3857, + "margin_dpo/margin_mean": 579.8360595703125, + "margin_dpo/margin_std": 1336.595458984375, + "step": 417 + }, + { + "KL/chosen_KL_mean": -1321.910400390625, + "KL/mean": -1603.6380615234375, + "KL/rejected_KL_mean": -1885.36572265625, + "KL/std": 1282.54248046875, + "epoch": 0.6138032305433186, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 563.455322265625, + "fcm_dpo/q_t": 0.49984556436538696, + "grad_norm": 0.10205204784870148, + "learning_rate": 1.965167291983757e-07, + "logits/chosen": -0.3112577199935913, + "logits/rejected": -0.31804031133651733, + "logps/chosen": -1403.888916015625, + "logps/ref_chosen": -81.97846984863281, + "logps/ref_rejected": -104.69148254394531, + "logps/rejected": -1990.05712890625, + "loss": 1.3857, + "margin_dpo/margin_mean": 563.4553833007812, + "margin_dpo/margin_std": 1835.57958984375, + "step": 418 + }, + { + "KL/chosen_KL_mean": -1055.2227783203125, + "KL/mean": -1440.5726318359375, + "KL/rejected_KL_mean": -1825.9224853515625, + "KL/std": 1226.0582275390625, + "epoch": 0.6152716593245228, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 770.6995849609375, + "fcm_dpo/q_t": 0.49978873133659363, + "grad_norm": 0.13206477463245392, + "learning_rate": 1.9526382029472988e-07, + "logits/chosen": -0.32074233889579773, + "logits/rejected": -0.3380177319049835, + "logps/chosen": -1108.17138671875, + "logps/ref_chosen": -52.948646545410156, + "logps/ref_rejected": -91.58309936523438, + "logps/rejected": -1917.505615234375, + "loss": 1.3855, + "margin_dpo/margin_mean": 770.6995849609375, + "margin_dpo/margin_std": 1524.392333984375, + "step": 419 + }, + { + "KL/chosen_KL_mean": -1762.050537109375, + "KL/mean": -1727.145263671875, + "KL/rejected_KL_mean": -1692.239990234375, + "KL/std": 1848.707763671875, + "epoch": 0.6167400881057269, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -69.8105697631836, + "fcm_dpo/q_t": 0.5000191330909729, + "grad_norm": 0.40303945541381836, + "learning_rate": 1.9401235374032425e-07, + "logits/chosen": -0.30101293325424194, + "logits/rejected": -0.30338841676712036, + "logps/chosen": -1839.8204345703125, + "logps/ref_chosen": -77.7699203491211, + "logps/ref_rejected": -69.31985473632812, + "logps/rejected": -1761.559814453125, + "loss": 1.3864, + "margin_dpo/margin_mean": -69.81057739257812, + "margin_dpo/margin_std": 2532.6513671875, + "step": 420 + }, + { + "KL/chosen_KL_mean": -1044.2967529296875, + "KL/mean": -1356.22119140625, + "KL/rejected_KL_mean": -1668.145751953125, + "KL/std": 1289.6182861328125, + "epoch": 0.618208516886931, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 623.848876953125, + "fcm_dpo/q_t": 0.4998289942741394, + "grad_norm": 0.21130794286727905, + "learning_rate": 1.9276236251246653e-07, + "logits/chosen": -0.30749380588531494, + "logits/rejected": -0.3160657286643982, + "logps/chosen": -1098.062744140625, + "logps/ref_chosen": -53.765865325927734, + "logps/ref_rejected": -89.28144836425781, + "logps/rejected": -1757.4271240234375, + "loss": 1.3856, + "margin_dpo/margin_mean": 623.848876953125, + "margin_dpo/margin_std": 1501.941162109375, + "step": 421 + }, + { + "KL/chosen_KL_mean": -1388.9171142578125, + "KL/mean": -1671.041015625, + "KL/rejected_KL_mean": -1953.1650390625, + "KL/std": 1397.2171630859375, + "epoch": 0.6196769456681351, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 564.2479248046875, + "fcm_dpo/q_t": 0.49984532594680786, + "grad_norm": 0.08393198996782303, + "learning_rate": 1.9151387954958792e-07, + "logits/chosen": -0.3038041591644287, + "logits/rejected": -0.3160828649997711, + "logps/chosen": -1457.5509033203125, + "logps/ref_chosen": -68.6337661743164, + "logps/ref_rejected": -87.86351013183594, + "logps/rejected": -2041.0284423828125, + "loss": 1.3857, + "margin_dpo/margin_mean": 564.2479248046875, + "margin_dpo/margin_std": 1596.736572265625, + "step": 422 + }, + { + "KL/chosen_KL_mean": -1274.769287109375, + "KL/mean": -1504.78369140625, + "KL/rejected_KL_mean": -1734.798095703125, + "KL/std": 1181.0693359375, + "epoch": 0.6211453744493393, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 460.02880859375, + "fcm_dpo/q_t": 0.49987390637397766, + "grad_norm": 0.11960334330797195, + "learning_rate": 1.902669377503756e-07, + "logits/chosen": -0.2938622534275055, + "logits/rejected": -0.3022230267524719, + "logps/chosen": -1329.7596435546875, + "logps/ref_chosen": -54.99030303955078, + "logps/ref_rejected": -86.30654907226562, + "logps/rejected": -1821.104736328125, + "loss": 1.3858, + "margin_dpo/margin_mean": 460.0288391113281, + "margin_dpo/margin_std": 1325.856201171875, + "step": 423 + }, + { + "KL/chosen_KL_mean": -1139.594970703125, + "KL/mean": -1338.92041015625, + "KL/rejected_KL_mean": -1538.2457275390625, + "KL/std": 1100.13720703125, + "epoch": 0.6226138032305433, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 398.6508483886719, + "fcm_dpo/q_t": 0.4998907446861267, + "grad_norm": 0.21664263308048248, + "learning_rate": 1.890215699729057e-07, + "logits/chosen": -0.24696838855743408, + "logits/rejected": -0.24573785066604614, + "logps/chosen": -1195.6068115234375, + "logps/ref_chosen": -56.01192092895508, + "logps/ref_rejected": -66.47896575927734, + "logps/rejected": -1604.7247314453125, + "loss": 1.3859, + "margin_dpo/margin_mean": 398.6508483886719, + "margin_dpo/margin_std": 1355.67431640625, + "step": 424 + }, + { + "KL/chosen_KL_mean": -1245.33984375, + "KL/mean": -1545.072998046875, + "KL/rejected_KL_mean": -1844.806396484375, + "KL/std": 1275.583251953125, + "epoch": 0.6240822320117474, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 599.4664306640625, + "fcm_dpo/q_t": 0.49983566999435425, + "grad_norm": 0.0939221978187561, + "learning_rate": 1.8777780903377732e-07, + "logits/chosen": -0.2669602334499359, + "logits/rejected": -0.2846640348434448, + "logps/chosen": -1292.2088623046875, + "logps/ref_chosen": -46.86899948120117, + "logps/ref_rejected": -95.92545318603516, + "logps/rejected": -1940.7318115234375, + "loss": 1.3856, + "margin_dpo/margin_mean": 599.4664306640625, + "margin_dpo/margin_std": 1722.2542724609375, + "step": 425 + }, + { + "KL/chosen_KL_mean": -1096.74462890625, + "KL/mean": -1299.535888671875, + "KL/rejected_KL_mean": -1502.327392578125, + "KL/std": 1008.5482177734375, + "epoch": 0.6255506607929515, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 405.5826416015625, + "fcm_dpo/q_t": 0.4998888373374939, + "grad_norm": 0.09973110258579254, + "learning_rate": 1.8653568770724803e-07, + "logits/chosen": -0.27449798583984375, + "logits/rejected": -0.27245032787323, + "logps/chosen": -1173.328125, + "logps/ref_chosen": -76.58354187011719, + "logps/ref_rejected": -81.26658630371094, + "logps/rejected": -1583.5938720703125, + "loss": 1.3859, + "margin_dpo/margin_mean": 405.5826721191406, + "margin_dpo/margin_std": 1211.4677734375, + "step": 426 + }, + { + "KL/chosen_KL_mean": -1049.704345703125, + "KL/mean": -1220.3751220703125, + "KL/rejected_KL_mean": -1391.0460205078125, + "KL/std": 1035.2784423828125, + "epoch": 0.6270190895741556, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 341.3416748046875, + "fcm_dpo/q_t": 0.49990642070770264, + "grad_norm": 0.07983623445034027, + "learning_rate": 1.8529523872436977e-07, + "logits/chosen": -0.2558949291706085, + "logits/rejected": -0.25835371017456055, + "logps/chosen": -1114.55810546875, + "logps/ref_chosen": -64.8538818359375, + "logps/ref_rejected": -78.5660171508789, + "logps/rejected": -1469.612060546875, + "loss": 1.3859, + "margin_dpo/margin_mean": 341.34161376953125, + "margin_dpo/margin_std": 1104.6048583984375, + "step": 427 + }, + { + "KL/chosen_KL_mean": -1136.0379638671875, + "KL/mean": -1533.8326416015625, + "KL/rejected_KL_mean": -1931.62744140625, + "KL/std": 1306.2242431640625, + "epoch": 0.6284875183553598, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 795.5894775390625, + "fcm_dpo/q_t": 0.49978193640708923, + "grad_norm": 0.18045121431350708, + "learning_rate": 1.8405649477212697e-07, + "logits/chosen": -0.2967488467693329, + "logits/rejected": -0.3061639666557312, + "logps/chosen": -1198.674560546875, + "logps/ref_chosen": -62.63666534423828, + "logps/ref_rejected": -103.28181457519531, + "logps/rejected": -2034.9091796875, + "loss": 1.3854, + "margin_dpo/margin_mean": 795.5894775390625, + "margin_dpo/margin_std": 1710.707275390625, + "step": 428 + }, + { + "KL/chosen_KL_mean": -1484.685302734375, + "KL/mean": -1713.23291015625, + "KL/rejected_KL_mean": -1941.780517578125, + "KL/std": 1404.762939453125, + "epoch": 0.6299559471365639, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 457.09521484375, + "fcm_dpo/q_t": 0.49987471103668213, + "grad_norm": 0.10288428515195847, + "learning_rate": 1.828194884925749e-07, + "logits/chosen": -0.2776823043823242, + "logits/rejected": -0.27903062105178833, + "logps/chosen": -1565.919189453125, + "logps/ref_chosen": -81.23401641845703, + "logps/ref_rejected": -91.79493713378906, + "logps/rejected": -2033.575439453125, + "loss": 1.3858, + "margin_dpo/margin_mean": 457.09527587890625, + "margin_dpo/margin_std": 1604.7532958984375, + "step": 429 + }, + { + "KL/chosen_KL_mean": -1077.0499267578125, + "KL/mean": -1380.4365234375, + "KL/rejected_KL_mean": -1683.822998046875, + "KL/std": 1355.672119140625, + "epoch": 0.631424375917768, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 606.7730712890625, + "fcm_dpo/q_t": 0.4998336732387543, + "grad_norm": 0.13458018004894257, + "learning_rate": 1.8158425248197928e-07, + "logits/chosen": -0.3125556707382202, + "logits/rejected": -0.3251245617866516, + "logps/chosen": -1137.97021484375, + "logps/ref_chosen": -60.920326232910156, + "logps/ref_rejected": -104.42280578613281, + "logps/rejected": -1788.2457275390625, + "loss": 1.3856, + "margin_dpo/margin_mean": 606.7731323242188, + "margin_dpo/margin_std": 1768.553955078125, + "step": 430 + }, + { + "KL/chosen_KL_mean": -989.0621337890625, + "KL/mean": -1313.78857421875, + "KL/rejected_KL_mean": -1638.514892578125, + "KL/std": 1129.193115234375, + "epoch": 0.6328928046989721, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 649.452880859375, + "fcm_dpo/q_t": 0.4998219907283783, + "grad_norm": 0.10206551104784012, + "learning_rate": 1.8035081928995788e-07, + "logits/chosen": -0.2263118028640747, + "logits/rejected": -0.23984399437904358, + "logps/chosen": -1046.410888671875, + "logps/ref_chosen": -57.34874725341797, + "logps/ref_rejected": -92.84022521972656, + "logps/rejected": -1731.355224609375, + "loss": 1.3856, + "margin_dpo/margin_mean": 649.452880859375, + "margin_dpo/margin_std": 1453.598388671875, + "step": 431 + }, + { + "KL/chosen_KL_mean": -953.16552734375, + "KL/mean": -1365.68798828125, + "KL/rejected_KL_mean": -1778.210693359375, + "KL/std": 1248.4228515625, + "epoch": 0.6343612334801763, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 825.0450439453125, + "fcm_dpo/q_t": 0.49977385997772217, + "grad_norm": 0.11800537258386612, + "learning_rate": 1.791192214186223e-07, + "logits/chosen": -0.2888324558734894, + "logits/rejected": -0.300900399684906, + "logps/chosen": -1024.240234375, + "logps/ref_chosen": -71.07479095458984, + "logps/ref_rejected": -98.57952880859375, + "logps/rejected": -1876.7901611328125, + "loss": 1.3854, + "margin_dpo/margin_mean": 825.0451049804688, + "margin_dpo/margin_std": 1362.866943359375, + "step": 432 + }, + { + "KL/chosen_KL_mean": -1171.21923828125, + "KL/mean": -1499.420166015625, + "KL/rejected_KL_mean": -1827.62109375, + "KL/std": 1200.9256591796875, + "epoch": 0.6358296622613803, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 656.4017944335938, + "fcm_dpo/q_t": 0.49982011318206787, + "grad_norm": 0.0767941102385521, + "learning_rate": 1.7788949132172193e-07, + "logits/chosen": -0.28949469327926636, + "logits/rejected": -0.30554211139678955, + "logps/chosen": -1229.492431640625, + "logps/ref_chosen": -58.273193359375, + "logps/ref_rejected": -95.95089721679688, + "logps/rejected": -1923.572021484375, + "loss": 1.3856, + "margin_dpo/margin_mean": 656.4017944335938, + "margin_dpo/margin_std": 1420.9886474609375, + "step": 433 + }, + { + "KL/chosen_KL_mean": -1108.557861328125, + "KL/mean": -1367.704345703125, + "KL/rejected_KL_mean": -1626.8509521484375, + "KL/std": 1420.2242431640625, + "epoch": 0.6372980910425844, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 518.2930908203125, + "fcm_dpo/q_t": 0.49985793232917786, + "grad_norm": 0.12584568560123444, + "learning_rate": 1.7666166140378853e-07, + "logits/chosen": -0.2219351828098297, + "logits/rejected": -0.23550885915756226, + "logps/chosen": -1170.531494140625, + "logps/ref_chosen": -61.97370147705078, + "logps/ref_rejected": -78.49861145019531, + "logps/rejected": -1705.349609375, + "loss": 1.3857, + "margin_dpo/margin_mean": 518.2930908203125, + "margin_dpo/margin_std": 2014.797607421875, + "step": 434 + }, + { + "KL/chosen_KL_mean": -905.6617431640625, + "KL/mean": -1253.97802734375, + "KL/rejected_KL_mean": -1602.2943115234375, + "KL/std": 1168.88720703125, + "epoch": 0.6387665198237885, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 696.6324462890625, + "fcm_dpo/q_t": 0.49980905652046204, + "grad_norm": 0.11213953047990799, + "learning_rate": 1.7543576401928218e-07, + "logits/chosen": -0.22496365010738373, + "logits/rejected": -0.22140967845916748, + "logps/chosen": -957.163818359375, + "logps/ref_chosen": -51.502052307128906, + "logps/ref_rejected": -87.56689453125, + "logps/rejected": -1689.861083984375, + "loss": 1.3855, + "margin_dpo/margin_mean": 696.6324462890625, + "margin_dpo/margin_std": 1404.775390625, + "step": 435 + }, + { + "KL/chosen_KL_mean": -951.4571533203125, + "KL/mean": -1099.720947265625, + "KL/rejected_KL_mean": -1247.9847412109375, + "KL/std": 928.330810546875, + "epoch": 0.6402349486049926, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 296.52752685546875, + "fcm_dpo/q_t": 0.49991875886917114, + "grad_norm": 0.06883503496646881, + "learning_rate": 1.742118314717391e-07, + "logits/chosen": -0.27546530961990356, + "logits/rejected": -0.2770288288593292, + "logps/chosen": -1022.86083984375, + "logps/ref_chosen": -71.40371704101562, + "logps/ref_rejected": -82.72775268554688, + "logps/rejected": -1330.71240234375, + "loss": 1.386, + "margin_dpo/margin_mean": 296.5275573730469, + "margin_dpo/margin_std": 1227.0777587890625, + "step": 436 + }, + { + "KL/chosen_KL_mean": -1031.511962890625, + "KL/mean": -1204.827392578125, + "KL/rejected_KL_mean": -1378.1427001953125, + "KL/std": 1022.2294311523438, + "epoch": 0.6417033773861968, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 346.63067626953125, + "fcm_dpo/q_t": 0.499904990196228, + "grad_norm": 0.06253945827484131, + "learning_rate": 1.7298989601292036e-07, + "logits/chosen": -0.2463058978319168, + "logits/rejected": -0.2471463531255722, + "logps/chosen": -1096.2562255859375, + "logps/ref_chosen": -64.7442626953125, + "logps/ref_rejected": -82.04356384277344, + "logps/rejected": -1460.186279296875, + "loss": 1.3859, + "margin_dpo/margin_mean": 346.63067626953125, + "margin_dpo/margin_std": 1298.53076171875, + "step": 437 + }, + { + "KL/chosen_KL_mean": -917.509765625, + "KL/mean": -1201.0556640625, + "KL/rejected_KL_mean": -1484.6015625, + "KL/std": 1046.44921875, + "epoch": 0.6431718061674009, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 567.091796875, + "fcm_dpo/q_t": 0.4998445510864258, + "grad_norm": 0.06463596969842911, + "learning_rate": 1.7176998984196144e-07, + "logits/chosen": -0.23667128384113312, + "logits/rejected": -0.24160590767860413, + "logps/chosen": -976.5284423828125, + "logps/ref_chosen": -59.0186653137207, + "logps/ref_rejected": -83.07682800292969, + "logps/rejected": -1567.678466796875, + "loss": 1.3857, + "margin_dpo/margin_mean": 567.0918579101562, + "margin_dpo/margin_std": 1066.5599365234375, + "step": 438 + }, + { + "KL/chosen_KL_mean": -925.7264404296875, + "KL/mean": -1227.3575439453125, + "KL/rejected_KL_mean": -1528.988525390625, + "KL/std": 1210.546630859375, + "epoch": 0.644640234948605, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 603.2620849609375, + "fcm_dpo/q_t": 0.49983465671539307, + "grad_norm": 0.1023441031575203, + "learning_rate": 1.7055214510452458e-07, + "logits/chosen": -0.24012020230293274, + "logits/rejected": -0.25954288244247437, + "logps/chosen": -979.510498046875, + "logps/ref_chosen": -53.78407669067383, + "logps/ref_rejected": -83.98545837402344, + "logps/rejected": -1612.9739990234375, + "loss": 1.3856, + "margin_dpo/margin_mean": 603.2620849609375, + "margin_dpo/margin_std": 1559.177490234375, + "step": 439 + }, + { + "KL/chosen_KL_mean": -1179.697998046875, + "KL/mean": -1430.30322265625, + "KL/rejected_KL_mean": -1680.908447265625, + "KL/std": 1239.806884765625, + "epoch": 0.6461086637298091, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 501.2104797363281, + "fcm_dpo/q_t": 0.4998626410961151, + "grad_norm": 0.1351250261068344, + "learning_rate": 1.6933639389195134e-07, + "logits/chosen": -0.2789965271949768, + "logits/rejected": -0.28014230728149414, + "logps/chosen": -1258.2646484375, + "logps/ref_chosen": -78.56671905517578, + "logps/ref_rejected": -96.49775695800781, + "logps/rejected": -1777.40625, + "loss": 1.3857, + "margin_dpo/margin_mean": 501.21051025390625, + "margin_dpo/margin_std": 1619.5087890625, + "step": 440 + }, + { + "KL/chosen_KL_mean": -1438.1356201171875, + "KL/mean": -1626.278076171875, + "KL/rejected_KL_mean": -1814.420654296875, + "KL/std": 1308.76953125, + "epoch": 0.6475770925110133, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 376.28509521484375, + "fcm_dpo/q_t": 0.4998968541622162, + "grad_norm": 0.09744305908679962, + "learning_rate": 1.681227682404166e-07, + "logits/chosen": -0.303938090801239, + "logits/rejected": -0.30175545811653137, + "logps/chosen": -1498.9600830078125, + "logps/ref_chosen": -60.824440002441406, + "logps/ref_rejected": -96.47080993652344, + "logps/rejected": -1910.8914794921875, + "loss": 1.3859, + "margin_dpo/margin_mean": 376.28509521484375, + "margin_dpo/margin_std": 1590.92333984375, + "step": 441 + }, + { + "KL/chosen_KL_mean": -1014.8897094726562, + "KL/mean": -1272.22607421875, + "KL/rejected_KL_mean": -1529.562255859375, + "KL/std": 1172.866943359375, + "epoch": 0.6490455212922174, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 514.672607421875, + "fcm_dpo/q_t": 0.49985894560813904, + "grad_norm": 0.0797480046749115, + "learning_rate": 1.669113001300851e-07, + "logits/chosen": -0.29978764057159424, + "logits/rejected": -0.3030720353126526, + "logps/chosen": -1061.90087890625, + "logps/ref_chosen": -47.01121520996094, + "logps/ref_rejected": -76.53926086425781, + "logps/rejected": -1606.1015625, + "loss": 1.3857, + "margin_dpo/margin_mean": 514.672607421875, + "margin_dpo/margin_std": 1585.113525390625, + "step": 442 + }, + { + "KL/chosen_KL_mean": -1156.9581298828125, + "KL/mean": -1253.765869140625, + "KL/rejected_KL_mean": -1350.57373046875, + "KL/std": 1065.82373046875, + "epoch": 0.6505139500734214, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 193.615478515625, + "fcm_dpo/q_t": 0.4999469518661499, + "grad_norm": 0.1120293065905571, + "learning_rate": 1.6570202148426815e-07, + "logits/chosen": -0.246562659740448, + "logits/rejected": -0.25994592905044556, + "logps/chosen": -1228.231201171875, + "logps/ref_chosen": -71.27301788330078, + "logps/ref_rejected": -86.679931640625, + "logps/rejected": -1437.253662109375, + "loss": 1.3861, + "margin_dpo/margin_mean": 193.615478515625, + "margin_dpo/margin_std": 1421.083251953125, + "step": 443 + }, + { + "KL/chosen_KL_mean": -1464.67724609375, + "KL/mean": -1763.21875, + "KL/rejected_KL_mean": -2061.76025390625, + "KL/std": 1581.70458984375, + "epoch": 0.6519823788546255, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 597.0831909179688, + "fcm_dpo/q_t": 0.4998363256454468, + "grad_norm": 0.13168448209762573, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": -0.26163217425346375, + "logits/rejected": -0.26668477058410645, + "logps/chosen": -1521.890869140625, + "logps/ref_chosen": -57.213706970214844, + "logps/ref_rejected": -97.25489807128906, + "logps/rejected": -2159.015380859375, + "loss": 1.3856, + "margin_dpo/margin_mean": 597.0831909179688, + "margin_dpo/margin_std": 1862.0196533203125, + "step": 444 + }, + { + "KL/chosen_KL_mean": -1140.0430908203125, + "KL/mean": -1497.06689453125, + "KL/rejected_KL_mean": -1854.090576171875, + "KL/std": 1608.1549072265625, + "epoch": 0.6534508076358296, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 714.047607421875, + "fcm_dpo/q_t": 0.4998042583465576, + "grad_norm": 0.12488368898630142, + "learning_rate": 1.6329015999011182e-07, + "logits/chosen": -0.28203797340393066, + "logits/rejected": -0.284241646528244, + "logps/chosen": -1207.3427734375, + "logps/ref_chosen": -67.29979705810547, + "logps/ref_rejected": -92.68267059326172, + "logps/rejected": -1946.7733154296875, + "loss": 1.3855, + "margin_dpo/margin_mean": 714.0476684570312, + "margin_dpo/margin_std": 2080.40625, + "step": 445 + }, + { + "KL/chosen_KL_mean": -1288.3699951171875, + "KL/mean": -1554.406494140625, + "KL/rejected_KL_mean": -1820.443115234375, + "KL/std": 1258.46044921875, + "epoch": 0.6549192364170338, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 532.0732421875, + "fcm_dpo/q_t": 0.4998541474342346, + "grad_norm": 0.0856269970536232, + "learning_rate": 1.6208764069656578e-07, + "logits/chosen": -0.2942993938922882, + "logits/rejected": -0.30123913288116455, + "logps/chosen": -1347.468505859375, + "logps/ref_chosen": -59.098487854003906, + "logps/ref_rejected": -101.26419067382812, + "logps/rejected": -1921.707275390625, + "loss": 1.3857, + "margin_dpo/margin_mean": 532.0732421875, + "margin_dpo/margin_std": 1500.825927734375, + "step": 446 + }, + { + "KL/chosen_KL_mean": -1074.880126953125, + "KL/mean": -1467.91357421875, + "KL/rejected_KL_mean": -1860.9471435546875, + "KL/std": 1457.51513671875, + "epoch": 0.6563876651982379, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 786.0670776367188, + "fcm_dpo/q_t": 0.49978452920913696, + "grad_norm": 0.19844543933868408, + "learning_rate": 1.608874379754465e-07, + "logits/chosen": -0.2915193438529968, + "logits/rejected": -0.2992427349090576, + "logps/chosen": -1130.955322265625, + "logps/ref_chosen": -56.07533264160156, + "logps/ref_rejected": -98.69475555419922, + "logps/rejected": -1959.641845703125, + "loss": 1.3854, + "margin_dpo/margin_mean": 786.0670776367188, + "margin_dpo/margin_std": 1644.028564453125, + "step": 447 + }, + { + "KL/chosen_KL_mean": -1095.27685546875, + "KL/mean": -1506.1376953125, + "KL/rejected_KL_mean": -1916.998291015625, + "KL/std": 1311.316650390625, + "epoch": 0.657856093979442, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 821.721435546875, + "fcm_dpo/q_t": 0.4997747540473938, + "grad_norm": 0.13404960930347443, + "learning_rate": 1.5968958345321177e-07, + "logits/chosen": -0.25953444838523865, + "logits/rejected": -0.26477596163749695, + "logps/chosen": -1155.28076171875, + "logps/ref_chosen": -60.00384521484375, + "logps/ref_rejected": -102.26465606689453, + "logps/rejected": -2019.262939453125, + "loss": 1.3854, + "margin_dpo/margin_mean": 821.721435546875, + "margin_dpo/margin_std": 1332.540771484375, + "step": 448 + }, + { + "KL/chosen_KL_mean": -1298.99755859375, + "KL/mean": -1579.174560546875, + "KL/rejected_KL_mean": -1859.3515625, + "KL/std": 1209.3814697265625, + "epoch": 0.6593245227606461, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 560.3539428710938, + "fcm_dpo/q_t": 0.4998464286327362, + "grad_norm": 0.09452203661203384, + "learning_rate": 1.584941086944423e-07, + "logits/chosen": -0.3198021352291107, + "logits/rejected": -0.3200905919075012, + "logps/chosen": -1366.524169921875, + "logps/ref_chosen": -67.52661895751953, + "logps/ref_rejected": -88.59690856933594, + "logps/rejected": -1947.948486328125, + "loss": 1.3857, + "margin_dpo/margin_mean": 560.3539428710938, + "margin_dpo/margin_std": 1519.3251953125, + "step": 449 + }, + { + "KL/chosen_KL_mean": -882.3565673828125, + "KL/mean": -1320.7303466796875, + "KL/rejected_KL_mean": -1759.1041259765625, + "KL/std": 1190.139892578125, + "epoch": 0.6607929515418502, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 876.7476806640625, + "fcm_dpo/q_t": 0.4997596740722656, + "grad_norm": 0.14239948987960815, + "learning_rate": 1.573010452010098e-07, + "logits/chosen": -0.2664763331413269, + "logits/rejected": -0.2864212989807129, + "logps/chosen": -939.4646606445312, + "logps/ref_chosen": -57.10811996459961, + "logps/ref_rejected": -102.75494384765625, + "logps/rejected": -1861.859130859375, + "loss": 1.3853, + "margin_dpo/margin_mean": 876.7476806640625, + "margin_dpo/margin_std": 1359.66943359375, + "step": 450 + }, + { + "KL/chosen_KL_mean": -1219.8216552734375, + "KL/mean": -1379.736328125, + "KL/rejected_KL_mean": -1539.651123046875, + "KL/std": 1262.7548828125, + "epoch": 0.6622613803230544, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 319.829345703125, + "fcm_dpo/q_t": 0.4999123513698578, + "grad_norm": 0.06787317246198654, + "learning_rate": 1.5611042441124687e-07, + "logits/chosen": -0.235478937625885, + "logits/rejected": -0.23634299635887146, + "logps/chosen": -1278.29052734375, + "logps/ref_chosen": -58.46883010864258, + "logps/ref_rejected": -72.92941284179688, + "logps/rejected": -1612.58056640625, + "loss": 1.3859, + "margin_dpo/margin_mean": 319.829345703125, + "margin_dpo/margin_std": 1746.6220703125, + "step": 451 + }, + { + "KL/chosen_KL_mean": -1223.572265625, + "KL/mean": -1464.432861328125, + "KL/rejected_KL_mean": -1705.29345703125, + "KL/std": 1323.29638671875, + "epoch": 0.6637298091042585, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 481.72125244140625, + "fcm_dpo/q_t": 0.4998679757118225, + "grad_norm": 0.1068187803030014, + "learning_rate": 1.549222776991186e-07, + "logits/chosen": -0.24899950623512268, + "logits/rejected": -0.26239603757858276, + "logps/chosen": -1273.9627685546875, + "logps/ref_chosen": -50.39055252075195, + "logps/ref_rejected": -97.77142333984375, + "logps/rejected": -1803.06494140625, + "loss": 1.3858, + "margin_dpo/margin_mean": 481.7213134765625, + "margin_dpo/margin_std": 1598.2059326171875, + "step": 452 + }, + { + "KL/chosen_KL_mean": -1116.9976806640625, + "KL/mean": -1405.4814453125, + "KL/rejected_KL_mean": -1693.96533203125, + "KL/std": 1186.461669921875, + "epoch": 0.6651982378854625, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 576.9676513671875, + "fcm_dpo/q_t": 0.4998418688774109, + "grad_norm": 0.11661150306463242, + "learning_rate": 1.5373663637339584e-07, + "logits/chosen": -0.29134508967399597, + "logits/rejected": -0.28827986121177673, + "logps/chosen": -1174.7125244140625, + "logps/ref_chosen": -57.71485137939453, + "logps/ref_rejected": -82.20741271972656, + "logps/rejected": -1776.1727294921875, + "loss": 1.3857, + "margin_dpo/margin_mean": 576.9676513671875, + "margin_dpo/margin_std": 1536.704833984375, + "step": 453 + }, + { + "KL/chosen_KL_mean": -1351.572021484375, + "KL/mean": -1602.408447265625, + "KL/rejected_KL_mean": -1853.244873046875, + "KL/std": 1257.90771484375, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 501.67279052734375, + "fcm_dpo/q_t": 0.4998624920845032, + "grad_norm": 0.10205873847007751, + "learning_rate": 1.5255353167683017e-07, + "logits/chosen": -0.2497563660144806, + "logits/rejected": -0.25889086723327637, + "logps/chosen": -1412.517822265625, + "logps/ref_chosen": -60.945648193359375, + "logps/ref_rejected": -84.95079040527344, + "logps/rejected": -1938.19580078125, + "loss": 1.3857, + "margin_dpo/margin_mean": 501.67279052734375, + "margin_dpo/margin_std": 1519.4886474609375, + "step": 454 + }, + { + "KL/chosen_KL_mean": -1146.478271484375, + "KL/mean": -1673.6832275390625, + "KL/rejected_KL_mean": -2200.88818359375, + "KL/std": 1628.58154296875, + "epoch": 0.6681350954478708, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1054.409912109375, + "fcm_dpo/q_t": 0.49971097707748413, + "grad_norm": 0.21247386932373047, + "learning_rate": 1.5137299478533064e-07, + "logits/chosen": -0.31010985374450684, + "logits/rejected": -0.32810282707214355, + "logps/chosen": -1191.364990234375, + "logps/ref_chosen": -44.88671112060547, + "logps/ref_rejected": -115.30147552490234, + "logps/rejected": -2316.189697265625, + "loss": 1.3851, + "margin_dpo/margin_mean": 1054.4097900390625, + "margin_dpo/margin_std": 2063.35009765625, + "step": 455 + }, + { + "KL/chosen_KL_mean": -1452.318603515625, + "KL/mean": -1761.364501953125, + "KL/rejected_KL_mean": -2070.41064453125, + "KL/std": 1583.523193359375, + "epoch": 0.6696035242290749, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 618.0919799804688, + "fcm_dpo/q_t": 0.49983057379722595, + "grad_norm": 0.1313888132572174, + "learning_rate": 1.5019505680714232e-07, + "logits/chosen": -0.28538087010383606, + "logits/rejected": -0.29785096645355225, + "logps/chosen": -1509.35546875, + "logps/ref_chosen": -57.036781311035156, + "logps/ref_rejected": -105.21784210205078, + "logps/rejected": -2175.62841796875, + "loss": 1.3856, + "margin_dpo/margin_mean": 618.0919189453125, + "margin_dpo/margin_std": 1710.4136962890625, + "step": 456 + }, + { + "KL/chosen_KL_mean": -1115.773193359375, + "KL/mean": -1518.5028076171875, + "KL/rejected_KL_mean": -1921.232421875, + "KL/std": 1319.357177734375, + "epoch": 0.671071953010279, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 805.4591674804688, + "fcm_dpo/q_t": 0.49977922439575195, + "grad_norm": 0.09444835782051086, + "learning_rate": 1.4901974878202627e-07, + "logits/chosen": -0.3091793656349182, + "logits/rejected": -0.3152327537536621, + "logps/chosen": -1170.0157470703125, + "logps/ref_chosen": -54.24253845214844, + "logps/ref_rejected": -85.10956573486328, + "logps/rejected": -2006.342041015625, + "loss": 1.3854, + "margin_dpo/margin_mean": 805.4591064453125, + "margin_dpo/margin_std": 1653.950439453125, + "step": 457 + }, + { + "KL/chosen_KL_mean": -1173.20263671875, + "KL/mean": -1522.257568359375, + "KL/rejected_KL_mean": -1871.312255859375, + "KL/std": 1152.4215087890625, + "epoch": 0.6725403817914831, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 698.1096801757812, + "fcm_dpo/q_t": 0.499808669090271, + "grad_norm": 0.09302345663309097, + "learning_rate": 1.4784710168044212e-07, + "logits/chosen": -0.27563565969467163, + "logits/rejected": -0.2783600091934204, + "logps/chosen": -1228.611572265625, + "logps/ref_chosen": -55.40888214111328, + "logps/ref_rejected": -97.68325805664062, + "logps/rejected": -1968.99560546875, + "loss": 1.3855, + "margin_dpo/margin_mean": 698.1096801757812, + "margin_dpo/margin_std": 1354.467041015625, + "step": 458 + }, + { + "KL/chosen_KL_mean": -1204.60009765625, + "KL/mean": -1605.72021484375, + "KL/rejected_KL_mean": -2006.840576171875, + "KL/std": 1491.602783203125, + "epoch": 0.6740088105726872, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 802.2403564453125, + "fcm_dpo/q_t": 0.4997801184654236, + "grad_norm": 0.07887598872184753, + "learning_rate": 1.466771464027316e-07, + "logits/chosen": -0.29390937089920044, + "logits/rejected": -0.3003736734390259, + "logps/chosen": -1251.1575927734375, + "logps/ref_chosen": -46.55748748779297, + "logps/ref_rejected": -86.16854095458984, + "logps/rejected": -2093.009033203125, + "loss": 1.3854, + "margin_dpo/margin_mean": 802.2403564453125, + "margin_dpo/margin_std": 1910.6488037109375, + "step": 459 + }, + { + "KL/chosen_KL_mean": -1455.231689453125, + "KL/mean": -1830.2767333984375, + "KL/rejected_KL_mean": -2205.32177734375, + "KL/std": 1589.1705322265625, + "epoch": 0.6754772393538914, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 750.090087890625, + "fcm_dpo/q_t": 0.4997944235801697, + "grad_norm": 0.14750739932060242, + "learning_rate": 1.4550991377830423e-07, + "logits/chosen": -0.2926778197288513, + "logits/rejected": -0.31306982040405273, + "logps/chosen": -1506.8665771484375, + "logps/ref_chosen": -51.63489532470703, + "logps/ref_rejected": -104.11935424804688, + "logps/rejected": -2309.441162109375, + "loss": 1.3855, + "margin_dpo/margin_mean": 750.090087890625, + "margin_dpo/margin_std": 1895.516845703125, + "step": 460 + }, + { + "KL/chosen_KL_mean": -1283.6640625, + "KL/mean": -1520.5830078125, + "KL/rejected_KL_mean": -1757.5018310546875, + "KL/std": 1185.034423828125, + "epoch": 0.6769456681350955, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 473.8377685546875, + "fcm_dpo/q_t": 0.4998701214790344, + "grad_norm": 0.07519460469484329, + "learning_rate": 1.4434543456482518e-07, + "logits/chosen": -0.3480767011642456, + "logits/rejected": -0.35095617175102234, + "logps/chosen": -1338.8460693359375, + "logps/ref_chosen": -55.18195724487305, + "logps/ref_rejected": -86.47689819335938, + "logps/rejected": -1843.978759765625, + "loss": 1.3858, + "margin_dpo/margin_mean": 473.83770751953125, + "margin_dpo/margin_std": 1498.89306640625, + "step": 461 + }, + { + "KL/chosen_KL_mean": -1435.103759765625, + "KL/mean": -1612.481689453125, + "KL/rejected_KL_mean": -1789.859375, + "KL/std": 1408.7646484375, + "epoch": 0.6784140969162996, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 354.7555847167969, + "fcm_dpo/q_t": 0.49990275502204895, + "grad_norm": 0.25926288962364197, + "learning_rate": 1.4318373944740484e-07, + "logits/chosen": -0.29589688777923584, + "logits/rejected": -0.2953804135322571, + "logps/chosen": -1505.0318603515625, + "logps/ref_chosen": -69.92803192138672, + "logps/ref_rejected": -78.84111022949219, + "logps/rejected": -1868.700439453125, + "loss": 1.3859, + "margin_dpo/margin_mean": 354.75555419921875, + "margin_dpo/margin_std": 1666.0538330078125, + "step": 462 + }, + { + "KL/chosen_KL_mean": -1326.2193603515625, + "KL/mean": -1661.249267578125, + "KL/rejected_KL_mean": -1996.279052734375, + "KL/std": 1530.80078125, + "epoch": 0.6798825256975036, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 670.059814453125, + "fcm_dpo/q_t": 0.499816358089447, + "grad_norm": 0.15044397115707397, + "learning_rate": 1.4202485903778976e-07, + "logits/chosen": -0.3050798773765564, + "logits/rejected": -0.31533971428871155, + "logps/chosen": -1381.49365234375, + "logps/ref_chosen": -55.27437210083008, + "logps/ref_rejected": -89.02497863769531, + "logps/rejected": -2085.303955078125, + "loss": 1.3856, + "margin_dpo/margin_mean": 670.0597534179688, + "margin_dpo/margin_std": 2028.2119140625, + "step": 463 + }, + { + "KL/chosen_KL_mean": -1342.7457275390625, + "KL/mean": -1907.53564453125, + "KL/rejected_KL_mean": -2472.32568359375, + "KL/std": 1601.189697265625, + "epoch": 0.6813509544787077, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1129.5799560546875, + "fcm_dpo/q_t": 0.4996904134750366, + "grad_norm": 0.16032731533050537, + "learning_rate": 1.4086882387355658e-07, + "logits/chosen": -0.263306200504303, + "logits/rejected": -0.27996712923049927, + "logps/chosen": -1393.657958984375, + "logps/ref_chosen": -50.91230010986328, + "logps/ref_rejected": -102.4893798828125, + "logps/rejected": -2574.81494140625, + "loss": 1.3851, + "margin_dpo/margin_mean": 1129.5799560546875, + "margin_dpo/margin_std": 1819.3665771484375, + "step": 464 + }, + { + "KL/chosen_KL_mean": -1383.0517578125, + "KL/mean": -1747.37158203125, + "KL/rejected_KL_mean": -2111.691650390625, + "KL/std": 1785.874755859375, + "epoch": 0.6828193832599119, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 728.6397705078125, + "fcm_dpo/q_t": 0.49980029463768005, + "grad_norm": 0.10014206171035767, + "learning_rate": 1.3971566441730714e-07, + "logits/chosen": -0.2919054627418518, + "logits/rejected": -0.30143067240715027, + "logps/chosen": -1443.1685791015625, + "logps/ref_chosen": -60.116851806640625, + "logps/ref_rejected": -113.94602966308594, + "logps/rejected": -2225.6376953125, + "loss": 1.3855, + "margin_dpo/margin_mean": 728.6398315429688, + "margin_dpo/margin_std": 2277.84765625, + "step": 465 + }, + { + "KL/chosen_KL_mean": -1208.7822265625, + "KL/mean": -1561.8572998046875, + "KL/rejected_KL_mean": -1914.9324951171875, + "KL/std": 1355.089599609375, + "epoch": 0.684287812041116, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 706.1502685546875, + "fcm_dpo/q_t": 0.4998064637184143, + "grad_norm": 0.12803106009960175, + "learning_rate": 1.3856541105586545e-07, + "logits/chosen": -0.272348016500473, + "logits/rejected": -0.28248733282089233, + "logps/chosen": -1261.703125, + "logps/ref_chosen": -52.920921325683594, + "logps/ref_rejected": -90.3154296875, + "logps/rejected": -2005.2479248046875, + "loss": 1.3855, + "margin_dpo/margin_mean": 706.1502685546875, + "margin_dpo/margin_std": 1762.6644287109375, + "step": 466 + }, + { + "KL/chosen_KL_mean": -1984.1240234375, + "KL/mean": -2191.9873046875, + "KL/rejected_KL_mean": -2399.850341796875, + "KL/std": 1883.2333984375, + "epoch": 0.6857562408223201, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 415.726318359375, + "fcm_dpo/q_t": 0.49988603591918945, + "grad_norm": 0.10527905076742172, + "learning_rate": 1.3741809409947729e-07, + "logits/chosen": -0.28988873958587646, + "logits/rejected": -0.29053300619125366, + "logps/chosen": -2062.83984375, + "logps/ref_chosen": -78.7158203125, + "logps/ref_rejected": -102.86019897460938, + "logps/rejected": -2502.71044921875, + "loss": 1.3858, + "margin_dpo/margin_mean": 415.7263488769531, + "margin_dpo/margin_std": 2690.570556640625, + "step": 467 + }, + { + "KL/chosen_KL_mean": -1457.8660888671875, + "KL/mean": -1854.851318359375, + "KL/rejected_KL_mean": -2251.83642578125, + "KL/std": 1709.1533203125, + "epoch": 0.6872246696035242, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 793.97021484375, + "fcm_dpo/q_t": 0.49978235363960266, + "grad_norm": 0.15743249654769897, + "learning_rate": 1.362737437810114e-07, + "logits/chosen": -0.32493388652801514, + "logits/rejected": -0.32417386770248413, + "logps/chosen": -1527.801513671875, + "logps/ref_chosen": -69.93536376953125, + "logps/ref_rejected": -101.02880859375, + "logps/rejected": -2352.865234375, + "loss": 1.3854, + "margin_dpo/margin_mean": 793.97021484375, + "margin_dpo/margin_std": 2327.3564453125, + "step": 468 + }, + { + "KL/chosen_KL_mean": -1288.8515625, + "KL/mean": -1759.3974609375, + "KL/rejected_KL_mean": -2229.943359375, + "KL/std": 1474.0218505859375, + "epoch": 0.6886930983847284, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 941.0918579101562, + "fcm_dpo/q_t": 0.4997420310974121, + "grad_norm": 0.0797450840473175, + "learning_rate": 1.351323902551631e-07, + "logits/chosen": -0.3195040225982666, + "logits/rejected": -0.3280268907546997, + "logps/chosen": -1356.976318359375, + "logps/ref_chosen": -68.12469482421875, + "logps/ref_rejected": -104.78640747070312, + "logps/rejected": -2334.729736328125, + "loss": 1.3853, + "margin_dpo/margin_mean": 941.091796875, + "margin_dpo/margin_std": 1737.1143798828125, + "step": 469 + }, + { + "KL/chosen_KL_mean": -1046.739501953125, + "KL/mean": -1347.437744140625, + "KL/rejected_KL_mean": -1648.1357421875, + "KL/std": 1214.6531982421875, + "epoch": 0.6901615271659325, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 601.3963623046875, + "fcm_dpo/q_t": 0.49983516335487366, + "grad_norm": 0.08385954052209854, + "learning_rate": 1.339940635976592e-07, + "logits/chosen": -0.262192964553833, + "logits/rejected": -0.2826134264469147, + "logps/chosen": -1090.531494140625, + "logps/ref_chosen": -43.791927337646484, + "logps/ref_rejected": -82.70285034179688, + "logps/rejected": -1730.838623046875, + "loss": 1.3856, + "margin_dpo/margin_mean": 601.3963012695312, + "margin_dpo/margin_std": 1419.621826171875, + "step": 470 + }, + { + "KL/chosen_KL_mean": -1479.0435791015625, + "KL/mean": -1652.86083984375, + "KL/rejected_KL_mean": -1826.677978515625, + "KL/std": 1403.427734375, + "epoch": 0.6916299559471366, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 347.63446044921875, + "fcm_dpo/q_t": 0.49990469217300415, + "grad_norm": 0.2278512865304947, + "learning_rate": 1.3285879380446563e-07, + "logits/chosen": -0.3011685609817505, + "logits/rejected": -0.30319029092788696, + "logps/chosen": -1542.383056640625, + "logps/ref_chosen": -63.33952331542969, + "logps/ref_rejected": -83.61048126220703, + "logps/rejected": -1910.2884521484375, + "loss": 1.3859, + "margin_dpo/margin_mean": 347.6344909667969, + "margin_dpo/margin_std": 1685.4962158203125, + "step": 471 + }, + { + "KL/chosen_KL_mean": -1486.688232421875, + "KL/mean": -1817.849853515625, + "KL/rejected_KL_mean": -2149.01123046875, + "KL/std": 1643.24609375, + "epoch": 0.6930983847283406, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 662.3231811523438, + "fcm_dpo/q_t": 0.49981844425201416, + "grad_norm": 0.12714260816574097, + "learning_rate": 1.317266107909975e-07, + "logits/chosen": -0.36696937680244446, + "logits/rejected": -0.37076157331466675, + "logps/chosen": -1570.3543701171875, + "logps/ref_chosen": -83.66610717773438, + "logps/ref_rejected": -117.20919799804688, + "logps/rejected": -2266.220703125, + "loss": 1.3856, + "margin_dpo/margin_mean": 662.3231201171875, + "margin_dpo/margin_std": 2021.212646484375, + "step": 472 + }, + { + "KL/chosen_KL_mean": -1896.511474609375, + "KL/mean": -1903.3677978515625, + "KL/rejected_KL_mean": -1910.22412109375, + "KL/std": 1944.942626953125, + "epoch": 0.6945668135095447, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 13.71258544921875, + "fcm_dpo/q_t": 0.49999624490737915, + "grad_norm": 0.13963881134986877, + "learning_rate": 1.3059754439133002e-07, + "logits/chosen": -0.24187920987606049, + "logits/rejected": -0.24330055713653564, + "logps/chosen": -1960.008544921875, + "logps/ref_chosen": -63.49696731567383, + "logps/ref_rejected": -81.14657592773438, + "logps/rejected": -1991.3707275390625, + "loss": 1.3863, + "margin_dpo/margin_mean": 13.712547302246094, + "margin_dpo/margin_std": 2561.2255859375, + "step": 473 + }, + { + "KL/chosen_KL_mean": -1129.59130859375, + "KL/mean": -1574.693359375, + "KL/rejected_KL_mean": -2019.795654296875, + "KL/std": 1632.31298828125, + "epoch": 0.6960352422907489, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 890.2041625976562, + "fcm_dpo/q_t": 0.49975600838661194, + "grad_norm": 0.13947923481464386, + "learning_rate": 1.2947162435741277e-07, + "logits/chosen": -0.2791895866394043, + "logits/rejected": -0.2850819528102875, + "logps/chosen": -1182.2032470703125, + "logps/ref_chosen": -52.6119384765625, + "logps/ref_rejected": -90.08041381835938, + "logps/rejected": -2109.8759765625, + "loss": 1.3853, + "margin_dpo/margin_mean": 890.2042236328125, + "margin_dpo/margin_std": 1976.387939453125, + "step": 474 + }, + { + "KL/chosen_KL_mean": -839.9306030273438, + "KL/mean": -1260.02001953125, + "KL/rejected_KL_mean": -1680.109375, + "KL/std": 1373.2816162109375, + "epoch": 0.697503671071953, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 840.1787109375, + "fcm_dpo/q_t": 0.4997697174549103, + "grad_norm": 0.15250550210475922, + "learning_rate": 1.2834888035828596e-07, + "logits/chosen": -0.23655390739440918, + "logits/rejected": -0.24804091453552246, + "logps/chosen": -882.42578125, + "logps/ref_chosen": -42.49519348144531, + "logps/ref_rejected": -90.06294250488281, + "logps/rejected": -1770.17236328125, + "loss": 1.3854, + "margin_dpo/margin_mean": 840.1787109375, + "margin_dpo/margin_std": 1737.7989501953125, + "step": 475 + }, + { + "KL/chosen_KL_mean": -1057.037353515625, + "KL/mean": -1375.7066650390625, + "KL/rejected_KL_mean": -1694.3759765625, + "KL/std": 1179.803466796875, + "epoch": 0.6989720998531571, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 637.3386840820312, + "fcm_dpo/q_t": 0.49982529878616333, + "grad_norm": 0.06534449756145477, + "learning_rate": 1.2722934197929802e-07, + "logits/chosen": -0.25173643231391907, + "logits/rejected": -0.26171934604644775, + "logps/chosen": -1099.9866943359375, + "logps/ref_chosen": -42.94938278198242, + "logps/ref_rejected": -73.71023559570312, + "logps/rejected": -1768.086181640625, + "loss": 1.3856, + "margin_dpo/margin_mean": 637.3387451171875, + "margin_dpo/margin_std": 1320.2890625, + "step": 476 + }, + { + "KL/chosen_KL_mean": -1168.0440673828125, + "KL/mean": -1340.4217529296875, + "KL/rejected_KL_mean": -1512.799560546875, + "KL/std": 1134.163818359375, + "epoch": 0.7004405286343612, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 344.75555419921875, + "fcm_dpo/q_t": 0.4999054968357086, + "grad_norm": 0.10434012115001678, + "learning_rate": 1.2611303872132631e-07, + "logits/chosen": -0.27185824513435364, + "logits/rejected": -0.2656621038913727, + "logps/chosen": -1238.816650390625, + "logps/ref_chosen": -70.77261352539062, + "logps/ref_rejected": -76.13737487792969, + "logps/rejected": -1588.93701171875, + "loss": 1.3859, + "margin_dpo/margin_mean": 344.75555419921875, + "margin_dpo/margin_std": 1227.234130859375, + "step": 477 + }, + { + "KL/chosen_KL_mean": -786.3189697265625, + "KL/mean": -1181.30322265625, + "KL/rejected_KL_mean": -1576.28759765625, + "KL/std": 1160.535400390625, + "epoch": 0.7019089574155654, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 789.96875, + "fcm_dpo/q_t": 0.4997834861278534, + "grad_norm": 0.17800100147724152, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": -0.24197494983673096, + "logits/rejected": -0.24932987987995148, + "logps/chosen": -827.7594604492188, + "logps/ref_chosen": -41.440513610839844, + "logps/ref_rejected": -85.36196899414062, + "logps/rejected": -1661.649658203125, + "loss": 1.3854, + "margin_dpo/margin_mean": 789.96875, + "margin_dpo/margin_std": 1413.09228515625, + "step": 478 + }, + { + "KL/chosen_KL_mean": -1186.480712890625, + "KL/mean": -1565.7657470703125, + "KL/rejected_KL_mean": -1945.05078125, + "KL/std": 1548.585693359375, + "epoch": 0.7033773861967695, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 758.5699462890625, + "fcm_dpo/q_t": 0.49979206919670105, + "grad_norm": 0.1228935718536377, + "learning_rate": 1.2389025514492456e-07, + "logits/chosen": -0.2601107954978943, + "logits/rejected": -0.2672087550163269, + "logps/chosen": -1240.388671875, + "logps/ref_chosen": -53.907920837402344, + "logps/ref_rejected": -95.1163330078125, + "logps/rejected": -2040.1671142578125, + "loss": 1.3855, + "margin_dpo/margin_mean": 758.5700073242188, + "margin_dpo/margin_std": 1593.4921875, + "step": 479 + }, + { + "KL/chosen_KL_mean": -1442.3372802734375, + "KL/mean": -1559.3134765625, + "KL/rejected_KL_mean": -1676.2896728515625, + "KL/std": 1411.744873046875, + "epoch": 0.7048458149779736, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 233.95236206054688, + "fcm_dpo/q_t": 0.4999358654022217, + "grad_norm": 0.17090091109275818, + "learning_rate": 1.227838333989088e-07, + "logits/chosen": -0.22087758779525757, + "logits/rejected": -0.2172781080007553, + "logps/chosen": -1501.02001953125, + "logps/ref_chosen": -58.682701110839844, + "logps/ref_rejected": -82.93248748779297, + "logps/rejected": -1759.22216796875, + "loss": 1.386, + "margin_dpo/margin_mean": 233.95237731933594, + "margin_dpo/margin_std": 1820.455810546875, + "step": 480 + }, + { + "KL/chosen_KL_mean": -1078.3800048828125, + "KL/mean": -1499.417236328125, + "KL/rejected_KL_mean": -1920.454833984375, + "KL/std": 1449.1815185546875, + "epoch": 0.7063142437591777, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 842.0747680664062, + "fcm_dpo/q_t": 0.4997692108154297, + "grad_norm": 0.13816991448402405, + "learning_rate": 1.2168076391719489e-07, + "logits/chosen": -0.2306959182024002, + "logits/rejected": -0.24468478560447693, + "logps/chosen": -1133.34423828125, + "logps/ref_chosen": -54.964271545410156, + "logps/ref_rejected": -92.42044067382812, + "logps/rejected": -2012.875244140625, + "loss": 1.3854, + "margin_dpo/margin_mean": 842.0747680664062, + "margin_dpo/margin_std": 1748.59130859375, + "step": 481 + }, + { + "KL/chosen_KL_mean": -1496.91259765625, + "KL/mean": -1609.91015625, + "KL/rejected_KL_mean": -1722.907958984375, + "KL/std": 1626.68359375, + "epoch": 0.7077826725403817, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 225.99520874023438, + "fcm_dpo/q_t": 0.49993807077407837, + "grad_norm": 0.11034970730543137, + "learning_rate": 1.2058107576668938e-07, + "logits/chosen": -0.2795412540435791, + "logits/rejected": -0.2840754985809326, + "logps/chosen": -1564.466064453125, + "logps/ref_chosen": -67.553466796875, + "logps/ref_rejected": -87.58953857421875, + "logps/rejected": -1810.4974365234375, + "loss": 1.386, + "margin_dpo/margin_mean": 225.99520874023438, + "margin_dpo/margin_std": 2063.758544921875, + "step": 482 + }, + { + "KL/chosen_KL_mean": -1195.4193115234375, + "KL/mean": -1492.416259765625, + "KL/rejected_KL_mean": -1789.4130859375, + "KL/std": 1201.998779296875, + "epoch": 0.7092511013215859, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 593.9937133789062, + "fcm_dpo/q_t": 0.499837189912796, + "grad_norm": 0.0831841230392456, + "learning_rate": 1.194847979251979e-07, + "logits/chosen": -0.28483182191848755, + "logits/rejected": -0.29176396131515503, + "logps/chosen": -1258.7491455078125, + "logps/ref_chosen": -63.32981872558594, + "logps/ref_rejected": -95.78697204589844, + "logps/rejected": -1885.2000732421875, + "loss": 1.3856, + "margin_dpo/margin_mean": 593.9937133789062, + "margin_dpo/margin_std": 1516.528564453125, + "step": 483 + }, + { + "KL/chosen_KL_mean": -1180.9385986328125, + "KL/mean": -1519.308349609375, + "KL/rejected_KL_mean": -1857.677978515625, + "KL/std": 1603.910888671875, + "epoch": 0.71071953010279, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 676.7393798828125, + "fcm_dpo/q_t": 0.499814510345459, + "grad_norm": 0.10013484954833984, + "learning_rate": 1.1839195928066101e-07, + "logits/chosen": -0.25293561816215515, + "logits/rejected": -0.2621699571609497, + "logps/chosen": -1240.07666015625, + "logps/ref_chosen": -59.13812255859375, + "logps/ref_rejected": -84.37144470214844, + "logps/rejected": -1942.0494384765625, + "loss": 1.3856, + "margin_dpo/margin_mean": 676.7393798828125, + "margin_dpo/margin_std": 2009.601318359375, + "step": 484 + }, + { + "KL/chosen_KL_mean": -1249.12060546875, + "KL/mean": -1599.3984375, + "KL/rejected_KL_mean": -1949.676513671875, + "KL/std": 1546.91796875, + "epoch": 0.7121879588839941, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 700.555908203125, + "fcm_dpo/q_t": 0.4998079836368561, + "grad_norm": 0.10648392140865326, + "learning_rate": 1.1730258863039347e-07, + "logits/chosen": -0.27771198749542236, + "logits/rejected": -0.29167696833610535, + "logps/chosen": -1307.97021484375, + "logps/ref_chosen": -58.849571228027344, + "logps/ref_rejected": -103.36408233642578, + "logps/rejected": -2053.04052734375, + "loss": 1.3855, + "margin_dpo/margin_mean": 700.555908203125, + "margin_dpo/margin_std": 2090.70458984375, + "step": 485 + }, + { + "KL/chosen_KL_mean": -1269.178466796875, + "KL/mean": -1632.4638671875, + "KL/rejected_KL_mean": -1995.749267578125, + "KL/std": 1473.224853515625, + "epoch": 0.7136563876651982, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 726.5706787109375, + "fcm_dpo/q_t": 0.4998008608818054, + "grad_norm": 0.10227925330400467, + "learning_rate": 1.1621671468032493e-07, + "logits/chosen": -0.2663297653198242, + "logits/rejected": -0.29365283250808716, + "logps/chosen": -1324.43798828125, + "logps/ref_chosen": -55.25966262817383, + "logps/ref_rejected": -92.13936614990234, + "logps/rejected": -2087.888671875, + "loss": 1.3855, + "margin_dpo/margin_mean": 726.5706787109375, + "margin_dpo/margin_std": 1893.989501953125, + "step": 486 + }, + { + "KL/chosen_KL_mean": -1264.476806640625, + "KL/mean": -1589.610107421875, + "KL/rejected_KL_mean": -1914.743408203125, + "KL/std": 1264.77880859375, + "epoch": 0.7151248164464024, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 650.2666015625, + "fcm_dpo/q_t": 0.4998217821121216, + "grad_norm": 0.11868096888065338, + "learning_rate": 1.1513436604424378e-07, + "logits/chosen": -0.266316294670105, + "logits/rejected": -0.2759333848953247, + "logps/chosen": -1317.5400390625, + "logps/ref_chosen": -53.06330871582031, + "logps/ref_rejected": -92.41883087158203, + "logps/rejected": -2007.162109375, + "loss": 1.3856, + "margin_dpo/margin_mean": 650.2665405273438, + "margin_dpo/margin_std": 1406.77490234375, + "step": 487 + }, + { + "KL/chosen_KL_mean": -947.0422973632812, + "KL/mean": -1332.60205078125, + "KL/rejected_KL_mean": -1718.16162109375, + "KL/std": 1295.9998779296875, + "epoch": 0.7165932452276065, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 771.1192626953125, + "fcm_dpo/q_t": 0.49978864192962646, + "grad_norm": 0.08489668369293213, + "learning_rate": 1.1405557124304335e-07, + "logits/chosen": -0.2604157328605652, + "logits/rejected": -0.2639409303665161, + "logps/chosen": -999.2705078125, + "logps/ref_chosen": -52.22815704345703, + "logps/ref_rejected": -84.00656127929688, + "logps/rejected": -1802.168212890625, + "loss": 1.3854, + "margin_dpo/margin_mean": 771.1192626953125, + "margin_dpo/margin_std": 1547.508056640625, + "step": 488 + }, + { + "KL/chosen_KL_mean": -1105.8419189453125, + "KL/mean": -1353.78662109375, + "KL/rejected_KL_mean": -1601.7314453125, + "KL/std": 1445.2835693359375, + "epoch": 0.7180616740088106, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 495.88958740234375, + "fcm_dpo/q_t": 0.4998641014099121, + "grad_norm": 0.14254043996334076, + "learning_rate": 1.1298035870396985e-07, + "logits/chosen": -0.3298376500606537, + "logits/rejected": -0.333371639251709, + "logps/chosen": -1161.83154296875, + "logps/ref_chosen": -55.989627838134766, + "logps/ref_rejected": -79.39812469482422, + "logps/rejected": -1681.129638671875, + "loss": 1.3858, + "margin_dpo/margin_mean": 495.8895568847656, + "margin_dpo/margin_std": 1753.814453125, + "step": 489 + }, + { + "KL/chosen_KL_mean": -1167.674072265625, + "KL/mean": -1577.68701171875, + "KL/rejected_KL_mean": -1987.699951171875, + "KL/std": 1686.5123291015625, + "epoch": 0.7195301027900147, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 820.026123046875, + "fcm_dpo/q_t": 0.499775230884552, + "grad_norm": 0.14442802965641022, + "learning_rate": 1.1190875675987355e-07, + "logits/chosen": -0.3567619323730469, + "logits/rejected": -0.3724161982536316, + "logps/chosen": -1220.0404052734375, + "logps/ref_chosen": -52.36639404296875, + "logps/ref_rejected": -110.4090576171875, + "logps/rejected": -2098.109130859375, + "loss": 1.3854, + "margin_dpo/margin_mean": 820.026123046875, + "margin_dpo/margin_std": 2067.93212890625, + "step": 490 + }, + { + "KL/chosen_KL_mean": -1103.107177734375, + "KL/mean": -1224.5498046875, + "KL/rejected_KL_mean": -1345.99267578125, + "KL/std": 1070.229736328125, + "epoch": 0.7209985315712188, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 242.8854217529297, + "fcm_dpo/q_t": 0.4999334216117859, + "grad_norm": 0.10099928826093674, + "learning_rate": 1.1084079364846241e-07, + "logits/chosen": -0.24923652410507202, + "logits/rejected": -0.2605559825897217, + "logps/chosen": -1163.223388671875, + "logps/ref_chosen": -60.11626434326172, + "logps/ref_rejected": -73.27278900146484, + "logps/rejected": -1419.265380859375, + "loss": 1.386, + "margin_dpo/margin_mean": 242.88543701171875, + "margin_dpo/margin_std": 1138.2882080078125, + "step": 491 + }, + { + "KL/chosen_KL_mean": -1155.7711181640625, + "KL/mean": -1256.9951171875, + "KL/rejected_KL_mean": -1358.218994140625, + "KL/std": 1135.314208984375, + "epoch": 0.7224669603524229, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 202.44784545898438, + "fcm_dpo/q_t": 0.4999445080757141, + "grad_norm": 0.10892591625452042, + "learning_rate": 1.097764975115576e-07, + "logits/chosen": -0.211615651845932, + "logits/rejected": -0.21193912625312805, + "logps/chosen": -1209.765380859375, + "logps/ref_chosen": -53.994178771972656, + "logps/ref_rejected": -72.65962219238281, + "logps/rejected": -1430.878662109375, + "loss": 1.3861, + "margin_dpo/margin_mean": 202.44786071777344, + "margin_dpo/margin_std": 1114.23291015625, + "step": 492 + }, + { + "KL/chosen_KL_mean": -1369.488525390625, + "KL/mean": -1456.96533203125, + "KL/rejected_KL_mean": -1544.44189453125, + "KL/std": 1230.5186767578125, + "epoch": 0.723935389133627, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 174.95330810546875, + "fcm_dpo/q_t": 0.4999520182609558, + "grad_norm": 0.23652927577495575, + "learning_rate": 1.0871589639435203e-07, + "logits/chosen": -0.269443154335022, + "logits/rejected": -0.26523542404174805, + "logps/chosen": -1444.98583984375, + "logps/ref_chosen": -75.49723815917969, + "logps/ref_rejected": -87.32301330566406, + "logps/rejected": -1631.764892578125, + "loss": 1.3861, + "margin_dpo/margin_mean": 174.95333862304688, + "margin_dpo/margin_std": 1543.530029296875, + "step": 493 + }, + { + "KL/chosen_KL_mean": -821.7962036132812, + "KL/mean": -1201.152587890625, + "KL/rejected_KL_mean": -1580.509033203125, + "KL/std": 1113.7763671875, + "epoch": 0.7254038179148311, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 758.7128295898438, + "fcm_dpo/q_t": 0.49979203939437866, + "grad_norm": 0.18678583204746246, + "learning_rate": 1.0765901824467166e-07, + "logits/chosen": -0.27484673261642456, + "logits/rejected": -0.29044800996780396, + "logps/chosen": -863.1554565429688, + "logps/ref_chosen": -41.35926818847656, + "logps/ref_rejected": -86.09136962890625, + "logps/rejected": -1666.600341796875, + "loss": 1.3855, + "margin_dpo/margin_mean": 758.7128295898438, + "margin_dpo/margin_std": 1463.646484375, + "step": 494 + }, + { + "KL/chosen_KL_mean": -1247.958740234375, + "KL/mean": -1488.2684326171875, + "KL/rejected_KL_mean": -1728.578125, + "KL/std": 1216.287109375, + "epoch": 0.7268722466960352, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 480.6193542480469, + "fcm_dpo/q_t": 0.499868243932724, + "grad_norm": 0.10272727906703949, + "learning_rate": 1.0660589091223854e-07, + "logits/chosen": -0.2950224280357361, + "logits/rejected": -0.3044213652610779, + "logps/chosen": -1311.493896484375, + "logps/ref_chosen": -63.53507995605469, + "logps/ref_rejected": -91.42443084716797, + "logps/rejected": -1820.0025634765625, + "loss": 1.3858, + "margin_dpo/margin_mean": 480.619384765625, + "margin_dpo/margin_std": 1789.05224609375, + "step": 495 + }, + { + "KL/chosen_KL_mean": -1368.56884765625, + "KL/mean": -1406.943115234375, + "KL/rejected_KL_mean": -1445.317626953125, + "KL/std": 1098.4658203125, + "epoch": 0.7283406754772394, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.74879455566406, + "fcm_dpo/q_t": 0.4999789595603943, + "grad_norm": 0.19332145154476166, + "learning_rate": 1.0555654214793722e-07, + "logits/chosen": -0.22733855247497559, + "logits/rejected": -0.23675402998924255, + "logps/chosen": -1441.1607666015625, + "logps/ref_chosen": -72.5919189453125, + "logps/ref_rejected": -84.32933807373047, + "logps/rejected": -1529.64697265625, + "loss": 1.3862, + "margin_dpo/margin_mean": 76.74880981445312, + "margin_dpo/margin_std": 1383.187255859375, + "step": 496 + }, + { + "KL/chosen_KL_mean": -1017.3779296875, + "KL/mean": -1118.634765625, + "KL/rejected_KL_mean": -1219.8914794921875, + "KL/std": 858.4376831054688, + "epoch": 0.7298091042584435, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 202.51368713378906, + "fcm_dpo/q_t": 0.49994444847106934, + "grad_norm": 0.11383572220802307, + "learning_rate": 1.0451099960308374e-07, + "logits/chosen": -0.23537415266036987, + "logits/rejected": -0.23635084927082062, + "logps/chosen": -1075.971923828125, + "logps/ref_chosen": -58.59397506713867, + "logps/ref_rejected": -76.28836822509766, + "logps/rejected": -1296.179931640625, + "loss": 1.3861, + "margin_dpo/margin_mean": 202.51368713378906, + "margin_dpo/margin_std": 849.7100830078125, + "step": 497 + }, + { + "KL/chosen_KL_mean": -1325.987060546875, + "KL/mean": -1531.426025390625, + "KL/rejected_KL_mean": -1736.864990234375, + "KL/std": 1285.070068359375, + "epoch": 0.7312775330396476, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 410.8779602050781, + "fcm_dpo/q_t": 0.4998873770236969, + "grad_norm": 0.14165054261684418, + "learning_rate": 1.0346929082869641e-07, + "logits/chosen": -0.26703372597694397, + "logits/rejected": -0.2662290334701538, + "logps/chosen": -1397.192626953125, + "logps/ref_chosen": -71.20565795898438, + "logps/ref_rejected": -83.95803833007812, + "logps/rejected": -1820.8231201171875, + "loss": 1.3858, + "margin_dpo/margin_mean": 410.8779602050781, + "margin_dpo/margin_std": 1588.934814453125, + "step": 498 + }, + { + "KL/chosen_KL_mean": -987.0281982421875, + "KL/mean": -1476.96337890625, + "KL/rejected_KL_mean": -1966.8984375, + "KL/std": 1366.307373046875, + "epoch": 0.7327459618208517, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 979.8701171875, + "fcm_dpo/q_t": 0.4997314214706421, + "grad_norm": 0.11465054005384445, + "learning_rate": 1.0243144327477013e-07, + "logits/chosen": -0.2148178517818451, + "logits/rejected": -0.22484102845191956, + "logps/chosen": -1038.283447265625, + "logps/ref_chosen": -51.25519561767578, + "logps/ref_rejected": -101.07870483398438, + "logps/rejected": -2067.97705078125, + "loss": 1.3852, + "margin_dpo/margin_mean": 979.8701171875, + "margin_dpo/margin_std": 1764.344970703125, + "step": 499 + }, + { + "KL/chosen_KL_mean": -1238.251708984375, + "KL/mean": -1586.513671875, + "KL/rejected_KL_mean": -1934.7757568359375, + "KL/std": 1321.060791015625, + "epoch": 0.7342143906020558, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 696.5239868164062, + "fcm_dpo/q_t": 0.4998090863227844, + "grad_norm": 0.0977897122502327, + "learning_rate": 1.0139748428955333e-07, + "logits/chosen": -0.2564852237701416, + "logits/rejected": -0.2703487277030945, + "logps/chosen": -1295.279296875, + "logps/ref_chosen": -57.027442932128906, + "logps/ref_rejected": -93.93421173095703, + "logps/rejected": -2028.7099609375, + "loss": 1.3855, + "margin_dpo/margin_mean": 696.5239868164062, + "margin_dpo/margin_std": 1756.97802734375, + "step": 500 + }, + { + "KL/chosen_KL_mean": -1067.537109375, + "KL/mean": -1295.0264892578125, + "KL/rejected_KL_mean": -1522.515869140625, + "KL/std": 1159.325927734375, + "epoch": 0.73568281938326, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 454.978759765625, + "fcm_dpo/q_t": 0.4998753070831299, + "grad_norm": 0.0951857939362526, + "learning_rate": 1.0036744111882672e-07, + "logits/chosen": -0.25967469811439514, + "logits/rejected": -0.25465235114097595, + "logps/chosen": -1121.8966064453125, + "logps/ref_chosen": -54.359527587890625, + "logps/ref_rejected": -80.15670013427734, + "logps/rejected": -1602.672607421875, + "loss": 1.3858, + "margin_dpo/margin_mean": 454.978759765625, + "margin_dpo/margin_std": 1693.336669921875, + "step": 501 + }, + { + "KL/chosen_KL_mean": -1118.3916015625, + "KL/mean": -1272.55224609375, + "KL/rejected_KL_mean": -1426.713134765625, + "KL/std": 1068.8515625, + "epoch": 0.737151248164464, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 308.3215026855469, + "fcm_dpo/q_t": 0.4999154806137085, + "grad_norm": 0.20658744871616364, + "learning_rate": 9.934134090518592e-08, + "logits/chosen": -0.2730405032634735, + "logits/rejected": -0.2703663110733032, + "logps/chosen": -1185.9921875, + "logps/ref_chosen": -67.60050964355469, + "logps/ref_rejected": -82.94876098632812, + "logps/rejected": -1509.661865234375, + "loss": 1.386, + "margin_dpo/margin_mean": 308.32147216796875, + "margin_dpo/margin_std": 1074.383544921875, + "step": 502 + }, + { + "KL/chosen_KL_mean": -966.2994384765625, + "KL/mean": -1260.075439453125, + "KL/rejected_KL_mean": -1553.8515625, + "KL/std": 1145.9617919921875, + "epoch": 0.7386196769456681, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 587.552001953125, + "fcm_dpo/q_t": 0.4998389482498169, + "grad_norm": 0.11802306771278381, + "learning_rate": 9.831921068732571e-08, + "logits/chosen": -0.23120424151420593, + "logits/rejected": -0.2398468554019928, + "logps/chosen": -1021.3778686523438, + "logps/ref_chosen": -55.078407287597656, + "logps/ref_rejected": -82.50544738769531, + "logps/rejected": -1636.35693359375, + "loss": 1.3857, + "margin_dpo/margin_mean": 587.552001953125, + "margin_dpo/margin_std": 1356.695068359375, + "step": 503 + }, + { + "KL/chosen_KL_mean": -1249.80419921875, + "KL/mean": -1683.2874755859375, + "KL/rejected_KL_mean": -2116.770751953125, + "KL/std": 1505.3199462890625, + "epoch": 0.7400881057268722, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 866.9666137695312, + "fcm_dpo/q_t": 0.4997623562812805, + "grad_norm": 0.16962231695652008, + "learning_rate": 9.730107739932805e-08, + "logits/chosen": -0.28023457527160645, + "logits/rejected": -0.29524755477905273, + "logps/chosen": -1309.7698974609375, + "logps/ref_chosen": -59.96575164794922, + "logps/ref_rejected": -103.76212310791016, + "logps/rejected": -2220.532958984375, + "loss": 1.3853, + "margin_dpo/margin_mean": 866.9666137695312, + "margin_dpo/margin_std": 1842.38818359375, + "step": 504 + }, + { + "KL/chosen_KL_mean": -1363.998779296875, + "KL/mean": -1380.1290283203125, + "KL/rejected_KL_mean": -1396.25927734375, + "KL/std": 1373.6171875, + "epoch": 0.7415565345080763, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 32.26036071777344, + "fcm_dpo/q_t": 0.49999117851257324, + "grad_norm": 0.16270355880260468, + "learning_rate": 9.628696786995188e-08, + "logits/chosen": -0.3044007420539856, + "logits/rejected": -0.3048001229763031, + "logps/chosen": -1440.15380859375, + "logps/ref_chosen": -76.1549072265625, + "logps/ref_rejected": -88.58537292480469, + "logps/rejected": -1484.8446044921875, + "loss": 1.3863, + "margin_dpo/margin_mean": 32.26038360595703, + "margin_dpo/margin_std": 1598.21826171875, + "step": 505 + }, + { + "KL/chosen_KL_mean": -901.4547729492188, + "KL/mean": -1252.69677734375, + "KL/rejected_KL_mean": -1603.938720703125, + "KL/std": 1069.479248046875, + "epoch": 0.7430249632892805, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 702.4840087890625, + "fcm_dpo/q_t": 0.4998074471950531, + "grad_norm": 0.07248572260141373, + "learning_rate": 9.527690882192635e-08, + "logits/chosen": -0.2846633791923523, + "logits/rejected": -0.29492151737213135, + "logps/chosen": -950.415283203125, + "logps/ref_chosen": -48.96050262451172, + "logps/ref_rejected": -78.41505432128906, + "logps/rejected": -1682.353759765625, + "loss": 1.3855, + "margin_dpo/margin_mean": 702.4840087890625, + "margin_dpo/margin_std": 1198.2059326171875, + "step": 506 + }, + { + "KL/chosen_KL_mean": -1269.53466796875, + "KL/mean": -1508.1639404296875, + "KL/rejected_KL_mean": -1746.793212890625, + "KL/std": 1457.87353515625, + "epoch": 0.7444933920704846, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 477.2585754394531, + "fcm_dpo/q_t": 0.4998691976070404, + "grad_norm": 0.07682963460683823, + "learning_rate": 9.427092687124691e-08, + "logits/chosen": -0.26889654994010925, + "logits/rejected": -0.2736053466796875, + "logps/chosen": -1336.336181640625, + "logps/ref_chosen": -66.80149841308594, + "logps/ref_rejected": -95.37289428710938, + "logps/rejected": -1842.1661376953125, + "loss": 1.3858, + "margin_dpo/margin_mean": 477.258544921875, + "margin_dpo/margin_std": 1696.332763671875, + "step": 507 + }, + { + "KL/chosen_KL_mean": -1198.90625, + "KL/mean": -1422.8087158203125, + "KL/rejected_KL_mean": -1646.7113037109375, + "KL/std": 1172.433837890625, + "epoch": 0.7459618208516887, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 447.80511474609375, + "fcm_dpo/q_t": 0.49987727403640747, + "grad_norm": 0.11619852483272552, + "learning_rate": 9.326904852647344e-08, + "logits/chosen": -0.27682003378868103, + "logits/rejected": -0.29388418793678284, + "logps/chosen": -1270.209716796875, + "logps/ref_chosen": -71.303466796875, + "logps/ref_rejected": -95.6275405883789, + "logps/rejected": -1742.3388671875, + "loss": 1.3858, + "margin_dpo/margin_mean": 447.80511474609375, + "margin_dpo/margin_std": 1444.593994140625, + "step": 508 + }, + { + "KL/chosen_KL_mean": -972.7108764648438, + "KL/mean": -1096.7337646484375, + "KL/rejected_KL_mean": -1220.7567138671875, + "KL/std": 1112.60400390625, + "epoch": 0.7474302496328928, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 248.04592895507812, + "fcm_dpo/q_t": 0.4999319911003113, + "grad_norm": 0.1427529752254486, + "learning_rate": 9.227130018803195e-08, + "logits/chosen": -0.2824665904045105, + "logits/rejected": -0.2912002205848694, + "logps/chosen": -1036.52978515625, + "logps/ref_chosen": -63.81895065307617, + "logps/ref_rejected": -83.25643920898438, + "logps/rejected": -1304.01318359375, + "loss": 1.386, + "margin_dpo/margin_mean": 248.04592895507812, + "margin_dpo/margin_std": 1467.73828125, + "step": 509 + }, + { + "KL/chosen_KL_mean": -1051.2216796875, + "KL/mean": -1357.23681640625, + "KL/rejected_KL_mean": -1663.2518310546875, + "KL/std": 1066.6669921875, + "epoch": 0.748898678414097, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 612.0302734375, + "fcm_dpo/q_t": 0.4998322129249573, + "grad_norm": 0.13177689909934998, + "learning_rate": 9.127770814751932e-08, + "logits/chosen": -0.24248462915420532, + "logits/rejected": -0.2578527629375458, + "logps/chosen": -1103.10009765625, + "logps/ref_chosen": -51.878448486328125, + "logps/ref_rejected": -102.7651596069336, + "logps/rejected": -1766.01708984375, + "loss": 1.3856, + "margin_dpo/margin_mean": 612.0302734375, + "margin_dpo/margin_std": 1055.28466796875, + "step": 510 + }, + { + "KL/chosen_KL_mean": -1168.831298828125, + "KL/mean": -1416.7269287109375, + "KL/rejected_KL_mean": -1664.62255859375, + "KL/std": 1328.281982421875, + "epoch": 0.750367107195301, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 495.7911376953125, + "fcm_dpo/q_t": 0.4998641014099121, + "grad_norm": 0.08437525480985641, + "learning_rate": 9.028829858700973e-08, + "logits/chosen": -0.2679316997528076, + "logits/rejected": -0.2680022120475769, + "logps/chosen": -1229.0694580078125, + "logps/ref_chosen": -60.23811721801758, + "logps/ref_rejected": -92.85676574707031, + "logps/rejected": -1757.479248046875, + "loss": 1.3858, + "margin_dpo/margin_mean": 495.79119873046875, + "margin_dpo/margin_std": 1695.87744140625, + "step": 511 + }, + { + "KL/chosen_KL_mean": -957.021484375, + "KL/mean": -1312.630126953125, + "KL/rejected_KL_mean": -1668.23876953125, + "KL/std": 1370.5361328125, + "epoch": 0.7518355359765051, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 711.2174072265625, + "fcm_dpo/q_t": 0.4998050630092621, + "grad_norm": 0.11381080746650696, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": -0.2461620569229126, + "logits/rejected": -0.2682782709598541, + "logps/chosen": -1011.9268798828125, + "logps/ref_chosen": -54.905494689941406, + "logps/ref_rejected": -81.87586975097656, + "logps/rejected": -1750.11474609375, + "loss": 1.3855, + "margin_dpo/margin_mean": 711.2174682617188, + "margin_dpo/margin_std": 1575.836181640625, + "step": 512 + }, + { + "KL/chosen_KL_mean": -1210.6767578125, + "KL/mean": -1353.77685546875, + "KL/rejected_KL_mean": -1496.876953125, + "KL/std": 1319.700927734375, + "epoch": 0.7533039647577092, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 286.2001953125, + "fcm_dpo/q_t": 0.4999215602874756, + "grad_norm": 0.08147355914115906, + "learning_rate": 8.832213108254863e-08, + "logits/chosen": -0.22810330986976624, + "logits/rejected": -0.23092329502105713, + "logps/chosen": -1275.59326171875, + "logps/ref_chosen": -64.91644287109375, + "logps/ref_rejected": -76.06245422363281, + "logps/rejected": -1572.9393310546875, + "loss": 1.386, + "margin_dpo/margin_mean": 286.2001647949219, + "margin_dpo/margin_std": 1764.772216796875, + "step": 513 + }, + { + "KL/chosen_KL_mean": -1492.766845703125, + "KL/mean": -1600.861083984375, + "KL/rejected_KL_mean": -1708.955322265625, + "KL/std": 1616.942626953125, + "epoch": 0.7547723935389133, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 216.18853759765625, + "fcm_dpo/q_t": 0.49994075298309326, + "grad_norm": 0.1525028944015503, + "learning_rate": 8.734542494893954e-08, + "logits/chosen": -0.2767585515975952, + "logits/rejected": -0.27932238578796387, + "logps/chosen": -1566.996337890625, + "logps/ref_chosen": -74.22957611083984, + "logps/ref_rejected": -78.945556640625, + "logps/rejected": -1787.90087890625, + "loss": 1.3861, + "margin_dpo/margin_mean": 216.18850708007812, + "margin_dpo/margin_std": 1974.9534912109375, + "step": 514 + }, + { + "KL/chosen_KL_mean": -965.72412109375, + "KL/mean": -1177.3330078125, + "KL/rejected_KL_mean": -1388.9420166015625, + "KL/std": 1055.767578125, + "epoch": 0.7562408223201175, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 423.2179260253906, + "fcm_dpo/q_t": 0.4998840093612671, + "grad_norm": 0.07925697416067123, + "learning_rate": 8.637300491465272e-08, + "logits/chosen": -0.2743576169013977, + "logits/rejected": -0.28066229820251465, + "logps/chosen": -1016.1256713867188, + "logps/ref_chosen": -50.40156555175781, + "logps/ref_rejected": -87.09774780273438, + "logps/rejected": -1476.039794921875, + "loss": 1.3858, + "margin_dpo/margin_mean": 423.2179260253906, + "margin_dpo/margin_std": 1284.13916015625, + "step": 515 + }, + { + "KL/chosen_KL_mean": -1195.279541015625, + "KL/mean": -1572.9501953125, + "KL/rejected_KL_mean": -1950.620849609375, + "KL/std": 1871.8084716796875, + "epoch": 0.7577092511013216, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 755.3414306640625, + "fcm_dpo/q_t": 0.4997929632663727, + "grad_norm": 0.1906149983406067, + "learning_rate": 8.540489660386064e-08, + "logits/chosen": -0.2554052472114563, + "logits/rejected": -0.2716732621192932, + "logps/chosen": -1259.9290771484375, + "logps/ref_chosen": -64.64956665039062, + "logps/ref_rejected": -111.72237396240234, + "logps/rejected": -2062.34326171875, + "loss": 1.3855, + "margin_dpo/margin_mean": 755.3414306640625, + "margin_dpo/margin_std": 2260.271484375, + "step": 516 + }, + { + "KL/chosen_KL_mean": -1145.0179443359375, + "KL/mean": -1422.536865234375, + "KL/rejected_KL_mean": -1700.0556640625, + "KL/std": 1352.418212890625, + "epoch": 0.7591776798825257, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 555.037841796875, + "fcm_dpo/q_t": 0.4998478889465332, + "grad_norm": 0.09951838850975037, + "learning_rate": 8.444112552711752e-08, + "logits/chosen": -0.28132858872413635, + "logits/rejected": -0.2950979471206665, + "logps/chosen": -1205.9315185546875, + "logps/ref_chosen": -60.913551330566406, + "logps/ref_rejected": -89.08308410644531, + "logps/rejected": -1789.1387939453125, + "loss": 1.3857, + "margin_dpo/margin_mean": 555.037841796875, + "margin_dpo/margin_std": 1767.117431640625, + "step": 517 + }, + { + "KL/chosen_KL_mean": -1181.3603515625, + "KL/mean": -1440.18896484375, + "KL/rejected_KL_mean": -1699.017578125, + "KL/std": 1185.616943359375, + "epoch": 0.7606461086637298, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 517.6571655273438, + "fcm_dpo/q_t": 0.4998581111431122, + "grad_norm": 0.09385734796524048, + "learning_rate": 8.348171708068747e-08, + "logits/chosen": -0.27268654108047485, + "logits/rejected": -0.28850501775741577, + "logps/chosen": -1238.816162109375, + "logps/ref_chosen": -57.45589065551758, + "logps/ref_rejected": -85.31269836425781, + "logps/rejected": -1784.3302001953125, + "loss": 1.3857, + "margin_dpo/margin_mean": 517.6571655273438, + "margin_dpo/margin_std": 1383.952880859375, + "step": 518 + }, + { + "KL/chosen_KL_mean": -1131.969970703125, + "KL/mean": -1446.619873046875, + "KL/rejected_KL_mean": -1761.27001953125, + "KL/std": 1455.6708984375, + "epoch": 0.762114537444934, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 629.3001708984375, + "fcm_dpo/q_t": 0.49982750415802, + "grad_norm": 0.12801778316497803, + "learning_rate": 8.25266965458755e-08, + "logits/chosen": -0.3059653043746948, + "logits/rejected": -0.3031119108200073, + "logps/chosen": -1206.033203125, + "logps/ref_chosen": -74.06331634521484, + "logps/ref_rejected": -104.44416809082031, + "logps/rejected": -1865.7142333984375, + "loss": 1.3856, + "margin_dpo/margin_mean": 629.3001708984375, + "margin_dpo/margin_std": 1972.010986328125, + "step": 519 + }, + { + "KL/chosen_KL_mean": -1056.5577392578125, + "KL/mean": -1340.5191650390625, + "KL/rejected_KL_mean": -1624.4805908203125, + "KL/std": 1151.4697265625, + "epoch": 0.7635829662261381, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 567.9228515625, + "fcm_dpo/q_t": 0.49984434247016907, + "grad_norm": 0.11603305488824844, + "learning_rate": 8.15760890883607e-08, + "logits/chosen": -0.23705577850341797, + "logits/rejected": -0.24325022101402283, + "logps/chosen": -1126.857666015625, + "logps/ref_chosen": -70.2998275756836, + "logps/ref_rejected": -99.98133850097656, + "logps/rejected": -1724.4619140625, + "loss": 1.3857, + "margin_dpo/margin_mean": 567.9227905273438, + "margin_dpo/margin_std": 1425.7099609375, + "step": 520 + }, + { + "KL/chosen_KL_mean": -1010.5167846679688, + "KL/mean": -1221.34521484375, + "KL/rejected_KL_mean": -1432.173583984375, + "KL/std": 1044.7950439453125, + "epoch": 0.7650513950073421, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 421.65692138671875, + "fcm_dpo/q_t": 0.4998844265937805, + "grad_norm": 0.0845324769616127, + "learning_rate": 8.062991975753378e-08, + "logits/chosen": -0.26425081491470337, + "logits/rejected": -0.2640856206417084, + "logps/chosen": -1068.65966796875, + "logps/ref_chosen": -58.14292526245117, + "logps/ref_rejected": -83.28060913085938, + "logps/rejected": -1515.454345703125, + "loss": 1.3858, + "margin_dpo/margin_mean": 421.65692138671875, + "margin_dpo/margin_std": 1294.17919921875, + "step": 521 + }, + { + "KL/chosen_KL_mean": -1076.2818603515625, + "KL/mean": -1360.576416015625, + "KL/rejected_KL_mean": -1644.87109375, + "KL/std": 1606.474853515625, + "epoch": 0.7665198237885462, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 568.589111328125, + "fcm_dpo/q_t": 0.49984413385391235, + "grad_norm": 0.07642640173435211, + "learning_rate": 7.968821348583643e-08, + "logits/chosen": -0.2520676851272583, + "logits/rejected": -0.2678247392177582, + "logps/chosen": -1122.82958984375, + "logps/ref_chosen": -46.54766845703125, + "logps/ref_rejected": -66.01388549804688, + "logps/rejected": -1710.885009765625, + "loss": 1.3857, + "margin_dpo/margin_mean": 568.589111328125, + "margin_dpo/margin_std": 2015.031982421875, + "step": 522 + }, + { + "KL/chosen_KL_mean": -1033.318359375, + "KL/mean": -1335.800537109375, + "KL/rejected_KL_mean": -1638.28271484375, + "KL/std": 1357.097412109375, + "epoch": 0.7679882525697503, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 604.9642944335938, + "fcm_dpo/q_t": 0.49983417987823486, + "grad_norm": 0.10172777622938156, + "learning_rate": 7.875099508810484e-08, + "logits/chosen": -0.2722175717353821, + "logits/rejected": -0.2761549651622772, + "logps/chosen": -1095.087890625, + "logps/ref_chosen": -61.76960372924805, + "logps/ref_rejected": -83.76141357421875, + "logps/rejected": -1722.0440673828125, + "loss": 1.3856, + "margin_dpo/margin_mean": 604.96435546875, + "margin_dpo/margin_std": 1610.768798828125, + "step": 523 + }, + { + "KL/chosen_KL_mean": -1311.5902099609375, + "KL/mean": -1617.090576171875, + "KL/rejected_KL_mean": -1922.5909423828125, + "KL/std": 1613.0634765625, + "epoch": 0.7694566813509545, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 611.0007934570312, + "fcm_dpo/q_t": 0.49983254075050354, + "grad_norm": 0.1079510748386383, + "learning_rate": 7.781828926091535e-08, + "logits/chosen": -0.32919204235076904, + "logits/rejected": -0.31630319356918335, + "logps/chosen": -1389.6622314453125, + "logps/ref_chosen": -78.0720443725586, + "logps/ref_rejected": -81.30198669433594, + "logps/rejected": -2003.8929443359375, + "loss": 1.3856, + "margin_dpo/margin_mean": 611.000732421875, + "margin_dpo/margin_std": 1673.618896484375, + "step": 524 + }, + { + "KL/chosen_KL_mean": -979.3692626953125, + "KL/mean": -1385.7138671875, + "KL/rejected_KL_mean": -1792.058349609375, + "KL/std": 1283.711669921875, + "epoch": 0.7709251101321586, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 812.6890869140625, + "fcm_dpo/q_t": 0.49977725744247437, + "grad_norm": 0.18908537924289703, + "learning_rate": 7.689012058193384e-08, + "logits/chosen": -0.26024335622787476, + "logits/rejected": -0.2741534113883972, + "logps/chosen": -1030.1971435546875, + "logps/ref_chosen": -50.827857971191406, + "logps/ref_rejected": -100.05294036865234, + "logps/rejected": -1892.111328125, + "loss": 1.3854, + "margin_dpo/margin_mean": 812.6890869140625, + "margin_dpo/margin_std": 1620.133056640625, + "step": 525 + }, + { + "KL/chosen_KL_mean": -1037.4547119140625, + "KL/mean": -1374.19921875, + "KL/rejected_KL_mean": -1710.94384765625, + "KL/std": 1307.3369140625, + "epoch": 0.7723935389133627, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 673.4891357421875, + "fcm_dpo/q_t": 0.4998154044151306, + "grad_norm": 0.09858764708042145, + "learning_rate": 7.596651350926836e-08, + "logits/chosen": -0.28175538778305054, + "logits/rejected": -0.2941705584526062, + "logps/chosen": -1100.6219482421875, + "logps/ref_chosen": -63.167236328125, + "logps/ref_rejected": -86.30934143066406, + "logps/rejected": -1797.253173828125, + "loss": 1.3856, + "margin_dpo/margin_mean": 673.4891357421875, + "margin_dpo/margin_std": 1447.4334716796875, + "step": 526 + }, + { + "KL/chosen_KL_mean": -1263.2935791015625, + "KL/mean": -1372.807861328125, + "KL/rejected_KL_mean": -1482.322265625, + "KL/std": 1163.5068359375, + "epoch": 0.7738619676945668, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 219.02862548828125, + "fcm_dpo/q_t": 0.4999399781227112, + "grad_norm": 0.1761639416217804, + "learning_rate": 7.504749238082414e-08, + "logits/chosen": -0.23617637157440186, + "logits/rejected": -0.23162142932415009, + "logps/chosen": -1334.4222412109375, + "logps/ref_chosen": -71.12867736816406, + "logps/ref_rejected": -78.3425521850586, + "logps/rejected": -1560.664794921875, + "loss": 1.3861, + "margin_dpo/margin_mean": 219.02862548828125, + "margin_dpo/margin_std": 1113.69873046875, + "step": 527 + }, + { + "KL/chosen_KL_mean": -1018.8012084960938, + "KL/mean": -1362.28564453125, + "KL/rejected_KL_mean": -1705.7701416015625, + "KL/std": 1300.2275390625, + "epoch": 0.775330396475771, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 686.9689331054688, + "fcm_dpo/q_t": 0.49981170892715454, + "grad_norm": 0.08465294539928436, + "learning_rate": 7.413308141366254e-08, + "logits/chosen": -0.2715555429458618, + "logits/rejected": -0.263353168964386, + "logps/chosen": -1086.890625, + "logps/ref_chosen": -68.0894546508789, + "logps/ref_rejected": -93.91006469726562, + "logps/rejected": -1799.68017578125, + "loss": 1.3855, + "margin_dpo/margin_mean": 686.9688720703125, + "margin_dpo/margin_std": 1761.4932861328125, + "step": 528 + }, + { + "KL/chosen_KL_mean": -1227.3046875, + "KL/mean": -1537.86181640625, + "KL/rejected_KL_mean": -1848.4189453125, + "KL/std": 1296.86279296875, + "epoch": 0.7767988252569751, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 621.1142578125, + "fcm_dpo/q_t": 0.4998297691345215, + "grad_norm": 0.08333950489759445, + "learning_rate": 7.322330470336313e-08, + "logits/chosen": -0.2381981611251831, + "logits/rejected": -0.25011301040649414, + "logps/chosen": -1282.879638671875, + "logps/ref_chosen": -55.57495880126953, + "logps/ref_rejected": -89.20909118652344, + "logps/rejected": -1937.6279296875, + "loss": 1.3856, + "margin_dpo/margin_mean": 621.1142578125, + "margin_dpo/margin_std": 1707.134765625, + "step": 529 + }, + { + "KL/chosen_KL_mean": -909.7897338867188, + "KL/mean": -1290.269775390625, + "KL/rejected_KL_mean": -1670.75, + "KL/std": 1348.552734375, + "epoch": 0.7782672540381792, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 760.960205078125, + "fcm_dpo/q_t": 0.4997914433479309, + "grad_norm": 0.1400674730539322, + "learning_rate": 7.231818622338822e-08, + "logits/chosen": -0.2535117566585541, + "logits/rejected": -0.25747549533843994, + "logps/chosen": -957.39111328125, + "logps/ref_chosen": -47.601417541503906, + "logps/ref_rejected": -87.2845230102539, + "logps/rejected": -1758.034423828125, + "loss": 1.3855, + "margin_dpo/margin_mean": 760.960205078125, + "margin_dpo/margin_std": 1795.02294921875, + "step": 530 + }, + { + "KL/chosen_KL_mean": -895.466552734375, + "KL/mean": -1198.396484375, + "KL/rejected_KL_mean": -1501.32666015625, + "KL/std": 1112.87744140625, + "epoch": 0.7797356828193832, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 605.860107421875, + "fcm_dpo/q_t": 0.49983394145965576, + "grad_norm": 0.08130745589733124, + "learning_rate": 7.141774982445147e-08, + "logits/chosen": -0.28837865591049194, + "logits/rejected": -0.3007437586784363, + "logps/chosen": -950.712646484375, + "logps/ref_chosen": -55.246063232421875, + "logps/ref_rejected": -70.60598754882812, + "logps/rejected": -1571.9326171875, + "loss": 1.3856, + "margin_dpo/margin_mean": 605.860107421875, + "margin_dpo/margin_std": 1083.08837890625, + "step": 531 + }, + { + "KL/chosen_KL_mean": -1241.084228515625, + "KL/mean": -1509.7344970703125, + "KL/rejected_KL_mean": -1778.384765625, + "KL/std": 1285.7984619140625, + "epoch": 0.7812041116005873, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 537.300537109375, + "fcm_dpo/q_t": 0.49985271692276, + "grad_norm": 0.17090633511543274, + "learning_rate": 7.052201923388953e-08, + "logits/chosen": -0.2878720164299011, + "logits/rejected": -0.2896765470504761, + "logps/chosen": -1311.3702392578125, + "logps/ref_chosen": -70.28601837158203, + "logps/ref_rejected": -86.5913314819336, + "logps/rejected": -1864.97607421875, + "loss": 1.3857, + "margin_dpo/margin_mean": 537.300537109375, + "margin_dpo/margin_std": 1761.650390625, + "step": 532 + }, + { + "KL/chosen_KL_mean": -1169.46240234375, + "KL/mean": -1342.16552734375, + "KL/rejected_KL_mean": -1514.868408203125, + "KL/std": 1102.200439453125, + "epoch": 0.7826725403817915, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 345.4060363769531, + "fcm_dpo/q_t": 0.4999053180217743, + "grad_norm": 0.08745371550321579, + "learning_rate": 6.963101805503646e-08, + "logits/chosen": -0.23491737246513367, + "logits/rejected": -0.2443659007549286, + "logps/chosen": -1234.317626953125, + "logps/ref_chosen": -64.8551025390625, + "logps/ref_rejected": -76.58805847167969, + "logps/rejected": -1591.45654296875, + "loss": 1.3859, + "margin_dpo/margin_mean": 345.406005859375, + "margin_dpo/margin_std": 1247.744873046875, + "step": 533 + }, + { + "KL/chosen_KL_mean": -1153.016845703125, + "KL/mean": -1350.9849853515625, + "KL/rejected_KL_mean": -1548.953125, + "KL/std": 1091.08642578125, + "epoch": 0.7841409691629956, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 395.936279296875, + "fcm_dpo/q_t": 0.499891459941864, + "grad_norm": 0.06997805833816528, + "learning_rate": 6.874476976660184e-08, + "logits/chosen": -0.285087913274765, + "logits/rejected": -0.2840203046798706, + "logps/chosen": -1213.13623046875, + "logps/ref_chosen": -60.119388580322266, + "logps/ref_rejected": -78.54347229003906, + "logps/rejected": -1627.49658203125, + "loss": 1.3859, + "margin_dpo/margin_mean": 395.9362487792969, + "margin_dpo/margin_std": 1245.4730224609375, + "step": 534 + }, + { + "KL/chosen_KL_mean": -931.0291748046875, + "KL/mean": -1277.57470703125, + "KL/rejected_KL_mean": -1624.1201171875, + "KL/std": 1066.3153076171875, + "epoch": 0.7856093979441997, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 693.091064453125, + "fcm_dpo/q_t": 0.49981003999710083, + "grad_norm": 0.2384890913963318, + "learning_rate": 6.786329772205246e-08, + "logits/chosen": -0.25958138704299927, + "logits/rejected": -0.2646050751209259, + "logps/chosen": -985.359375, + "logps/ref_chosen": -54.330238342285156, + "logps/ref_rejected": -96.30763244628906, + "logps/rejected": -1720.4278564453125, + "loss": 1.3855, + "margin_dpo/margin_mean": 693.091064453125, + "margin_dpo/margin_std": 1384.616943359375, + "step": 535 + }, + { + "KL/chosen_KL_mean": -901.54736328125, + "KL/mean": -1342.12353515625, + "KL/rejected_KL_mean": -1782.69970703125, + "KL/std": 1309.0029296875, + "epoch": 0.7870778267254038, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 881.1522216796875, + "fcm_dpo/q_t": 0.4997584819793701, + "grad_norm": 0.08399941027164459, + "learning_rate": 6.698662514899638e-08, + "logits/chosen": -0.25612354278564453, + "logits/rejected": -0.26890262961387634, + "logps/chosen": -948.6279296875, + "logps/ref_chosen": -47.08053207397461, + "logps/ref_rejected": -89.09783935546875, + "logps/rejected": -1871.797607421875, + "loss": 1.3853, + "margin_dpo/margin_mean": 881.1522216796875, + "margin_dpo/margin_std": 1578.1298828125, + "step": 536 + }, + { + "KL/chosen_KL_mean": -964.5877685546875, + "KL/mean": -1165.6494140625, + "KL/rejected_KL_mean": -1366.7109375, + "KL/std": 1214.634033203125, + "epoch": 0.788546255506608, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 402.1231994628906, + "fcm_dpo/q_t": 0.4998897910118103, + "grad_norm": 0.12264511734247208, + "learning_rate": 6.611477514857114e-08, + "logits/chosen": -0.24898353219032288, + "logits/rejected": -0.2636551558971405, + "logps/chosen": -1022.335205078125, + "logps/ref_chosen": -57.747467041015625, + "logps/ref_rejected": -70.43838500976562, + "logps/rejected": -1437.1492919921875, + "loss": 1.3859, + "margin_dpo/margin_mean": 402.1231689453125, + "margin_dpo/margin_std": 1610.4888916015625, + "step": 537 + }, + { + "KL/chosen_KL_mean": -1419.4970703125, + "KL/mean": -1645.308349609375, + "KL/rejected_KL_mean": -1871.11962890625, + "KL/std": 1241.21337890625, + "epoch": 0.7900146842878121, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 451.6224670410156, + "fcm_dpo/q_t": 0.4998762011528015, + "grad_norm": 0.15423403680324554, + "learning_rate": 6.524777069483525e-08, + "logits/chosen": -0.27448660135269165, + "logits/rejected": -0.27430930733680725, + "logps/chosen": -1485.9130859375, + "logps/ref_chosen": -66.41594696044922, + "logps/ref_rejected": -84.22808837890625, + "logps/rejected": -1955.34765625, + "loss": 1.3858, + "margin_dpo/margin_mean": 451.6224670410156, + "margin_dpo/margin_std": 1483.188232421875, + "step": 538 + }, + { + "KL/chosen_KL_mean": -1247.627197265625, + "KL/mean": -1601.21533203125, + "KL/rejected_KL_mean": -1954.803466796875, + "KL/std": 1497.35498046875, + "epoch": 0.7914831130690162, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 707.17626953125, + "fcm_dpo/q_t": 0.49980616569519043, + "grad_norm": 0.08800466358661652, + "learning_rate": 6.438563463416221e-08, + "logits/chosen": -0.23933789134025574, + "logits/rejected": -0.23772892355918884, + "logps/chosen": -1306.1201171875, + "logps/ref_chosen": -58.492855072021484, + "logps/ref_rejected": -91.85395050048828, + "logps/rejected": -2046.657470703125, + "loss": 1.3855, + "margin_dpo/margin_mean": 707.17626953125, + "margin_dpo/margin_std": 1571.142822265625, + "step": 539 + }, + { + "KL/chosen_KL_mean": -1132.70263671875, + "KL/mean": -1627.818603515625, + "KL/rejected_KL_mean": -2122.934326171875, + "KL/std": 1450.533935546875, + "epoch": 0.7929515418502202, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 990.231689453125, + "fcm_dpo/q_t": 0.49972859025001526, + "grad_norm": 0.2339281588792801, + "learning_rate": 6.352838968463919e-08, + "logits/chosen": -0.2771497368812561, + "logits/rejected": -0.28398674726486206, + "logps/chosen": -1196.1851806640625, + "logps/ref_chosen": -63.482513427734375, + "logps/ref_rejected": -116.42999267578125, + "logps/rejected": -2239.3642578125, + "loss": 1.3852, + "margin_dpo/margin_mean": 990.2315673828125, + "margin_dpo/margin_std": 1892.115966796875, + "step": 540 + }, + { + "KL/chosen_KL_mean": -1705.358154296875, + "KL/mean": -1753.754638671875, + "KL/rejected_KL_mean": -1802.151123046875, + "KL/std": 1488.664794921875, + "epoch": 0.7944199706314243, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 96.79295349121094, + "fcm_dpo/q_t": 0.49997344613075256, + "grad_norm": 0.18836979568004608, + "learning_rate": 6.267605843546767e-08, + "logits/chosen": -0.27575767040252686, + "logits/rejected": -0.28435438871383667, + "logps/chosen": -1783.6385498046875, + "logps/ref_chosen": -78.28036499023438, + "logps/ref_rejected": -103.273681640625, + "logps/rejected": -1905.4248046875, + "loss": 1.3862, + "margin_dpo/margin_mean": 96.79295349121094, + "margin_dpo/margin_std": 1649.4097900390625, + "step": 541 + }, + { + "KL/chosen_KL_mean": -1150.528076171875, + "KL/mean": -1601.9080810546875, + "KL/rejected_KL_mean": -2053.2880859375, + "KL/std": 1511.0010986328125, + "epoch": 0.7958883994126285, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 902.7598876953125, + "fcm_dpo/q_t": 0.49975258111953735, + "grad_norm": 0.09656842052936554, + "learning_rate": 6.182866334636888e-08, + "logits/chosen": -0.2542586326599121, + "logits/rejected": -0.258800745010376, + "logps/chosen": -1208.01318359375, + "logps/ref_chosen": -57.48497009277344, + "logps/ref_rejected": -96.47506713867188, + "logps/rejected": -2149.76318359375, + "loss": 1.3853, + "margin_dpo/margin_mean": 902.7598876953125, + "margin_dpo/margin_std": 1906.48046875, + "step": 542 + }, + { + "KL/chosen_KL_mean": -1388.8468017578125, + "KL/mean": -1585.422607421875, + "KL/rejected_KL_mean": -1781.9981689453125, + "KL/std": 1386.4052734375, + "epoch": 0.7973568281938326, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 393.1514587402344, + "fcm_dpo/q_t": 0.4998922348022461, + "grad_norm": 0.08593297749757767, + "learning_rate": 6.098622674699147e-08, + "logits/chosen": -0.2811450660228729, + "logits/rejected": -0.2933238744735718, + "logps/chosen": -1449.46435546875, + "logps/ref_chosen": -60.61750793457031, + "logps/ref_rejected": -105.59896850585938, + "logps/rejected": -1887.59716796875, + "loss": 1.3859, + "margin_dpo/margin_mean": 393.15142822265625, + "margin_dpo/margin_std": 1973.71875, + "step": 543 + }, + { + "KL/chosen_KL_mean": -1308.500244140625, + "KL/mean": -1758.7490234375, + "KL/rejected_KL_mean": -2208.997802734375, + "KL/std": 1845.3236083984375, + "epoch": 0.7988252569750367, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 900.497802734375, + "fcm_dpo/q_t": 0.4997531771659851, + "grad_norm": 0.11313273012638092, + "learning_rate": 6.01487708363232e-08, + "logits/chosen": -0.2731946110725403, + "logits/rejected": -0.2808229923248291, + "logps/chosen": -1368.1424560546875, + "logps/ref_chosen": -59.642303466796875, + "logps/ref_rejected": -100.95469665527344, + "logps/rejected": -2309.95263671875, + "loss": 1.3853, + "margin_dpo/margin_mean": 900.497802734375, + "margin_dpo/margin_std": 2468.87841796875, + "step": 544 + }, + { + "KL/chosen_KL_mean": -1315.6982421875, + "KL/mean": -1654.6534423828125, + "KL/rejected_KL_mean": -1993.6083984375, + "KL/std": 1559.415771484375, + "epoch": 0.8002936857562408, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 677.9102783203125, + "fcm_dpo/q_t": 0.4998142123222351, + "grad_norm": 0.1363622099161148, + "learning_rate": 5.9316317682106294e-08, + "logits/chosen": -0.2993026077747345, + "logits/rejected": -0.3052732050418854, + "logps/chosen": -1383.346923828125, + "logps/ref_chosen": -67.64859771728516, + "logps/ref_rejected": -95.90800476074219, + "logps/rejected": -2089.5166015625, + "loss": 1.3856, + "margin_dpo/margin_mean": 677.9102783203125, + "margin_dpo/margin_std": 1948.7215576171875, + "step": 545 + }, + { + "KL/chosen_KL_mean": -1233.961669921875, + "KL/mean": -1423.4217529296875, + "KL/rejected_KL_mean": -1612.8819580078125, + "KL/std": 1190.663818359375, + "epoch": 0.801762114537445, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 378.920166015625, + "fcm_dpo/q_t": 0.4998961389064789, + "grad_norm": 0.1315159946680069, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": -0.2829248607158661, + "logits/rejected": -0.2874290645122528, + "logps/chosen": -1284.7059326171875, + "logps/ref_chosen": -50.744232177734375, + "logps/ref_rejected": -81.86622619628906, + "logps/rejected": -1694.7481689453125, + "loss": 1.3859, + "margin_dpo/margin_mean": 378.920166015625, + "margin_dpo/margin_std": 1452.426025390625, + "step": 546 + }, + { + "KL/chosen_KL_mean": -1516.5628662109375, + "KL/mean": -1795.2900390625, + "KL/rejected_KL_mean": -2074.01708984375, + "KL/std": 2049.671142578125, + "epoch": 0.8032305433186491, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 557.4541625976562, + "fcm_dpo/q_t": 0.4998472034931183, + "grad_norm": 0.10258148610591888, + "learning_rate": 5.7666507254280265e-08, + "logits/chosen": -0.30326375365257263, + "logits/rejected": -0.30700263381004333, + "logps/chosen": -1590.2506103515625, + "logps/ref_chosen": -73.6877212524414, + "logps/ref_rejected": -90.76136779785156, + "logps/rejected": -2164.778564453125, + "loss": 1.3857, + "margin_dpo/margin_mean": 557.4541625976562, + "margin_dpo/margin_std": 2793.952392578125, + "step": 547 + }, + { + "KL/chosen_KL_mean": -1279.252197265625, + "KL/mean": -1552.047607421875, + "KL/rejected_KL_mean": -1824.84326171875, + "KL/std": 1413.7266845703125, + "epoch": 0.8046989720998532, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 545.5911865234375, + "fcm_dpo/q_t": 0.49985045194625854, + "grad_norm": 0.08995510637760162, + "learning_rate": 5.684919345471029e-08, + "logits/chosen": -0.2966746985912323, + "logits/rejected": -0.2986268401145935, + "logps/chosen": -1344.49853515625, + "logps/ref_chosen": -65.24634552001953, + "logps/ref_rejected": -94.11807250976562, + "logps/rejected": -1918.9613037109375, + "loss": 1.3857, + "margin_dpo/margin_mean": 545.5911865234375, + "margin_dpo/margin_std": 1728.4644775390625, + "step": 548 + }, + { + "KL/chosen_KL_mean": -1257.45849609375, + "KL/mean": -1397.7392578125, + "KL/rejected_KL_mean": -1538.02001953125, + "KL/std": 1288.01708984375, + "epoch": 0.8061674008810573, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 280.5616455078125, + "fcm_dpo/q_t": 0.49992311000823975, + "grad_norm": 0.0936603769659996, + "learning_rate": 5.603696935852426e-08, + "logits/chosen": -0.23646308481693268, + "logits/rejected": -0.23927268385887146, + "logps/chosen": -1306.6708984375, + "logps/ref_chosen": -49.21235656738281, + "logps/ref_rejected": -73.91031646728516, + "logps/rejected": -1611.930419921875, + "loss": 1.386, + "margin_dpo/margin_mean": 280.5616455078125, + "margin_dpo/margin_std": 1482.34765625, + "step": 549 + }, + { + "KL/chosen_KL_mean": -1198.25732421875, + "KL/mean": -1502.1494140625, + "KL/rejected_KL_mean": -1806.04150390625, + "KL/std": 1327.75634765625, + "epoch": 0.8076358296622613, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 607.7839965820312, + "fcm_dpo/q_t": 0.4998334050178528, + "grad_norm": 0.12111429870128632, + "learning_rate": 5.5229856368582376e-08, + "logits/chosen": -0.2754247784614563, + "logits/rejected": -0.2830020785331726, + "logps/chosen": -1255.0643310546875, + "logps/ref_chosen": -56.80695343017578, + "logps/ref_rejected": -95.12580871582031, + "logps/rejected": -1901.167236328125, + "loss": 1.3856, + "margin_dpo/margin_mean": 607.783935546875, + "margin_dpo/margin_std": 1616.297119140625, + "step": 550 + }, + { + "KL/chosen_KL_mean": -1037.71484375, + "KL/mean": -1604.3447265625, + "KL/rejected_KL_mean": -2170.974609375, + "KL/std": 1566.04296875, + "epoch": 0.8091042584434655, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1133.2598876953125, + "fcm_dpo/q_t": 0.49968937039375305, + "grad_norm": 0.18915165960788727, + "learning_rate": 5.4427875753062734e-08, + "logits/chosen": -0.24009448289871216, + "logits/rejected": -0.24743011593818665, + "logps/chosen": -1096.821044921875, + "logps/ref_chosen": -59.10633087158203, + "logps/ref_rejected": -111.67280578613281, + "logps/rejected": -2282.6474609375, + "loss": 1.3851, + "margin_dpo/margin_mean": 1133.259765625, + "margin_dpo/margin_std": 1756.314453125, + "step": 551 + }, + { + "KL/chosen_KL_mean": -1126.0601806640625, + "KL/mean": -1655.011962890625, + "KL/rejected_KL_mean": -2183.963623046875, + "KL/std": 1489.4439697265625, + "epoch": 0.8105726872246696, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1057.9033203125, + "fcm_dpo/q_t": 0.4997100532054901, + "grad_norm": 0.170448437333107, + "learning_rate": 5.363104864490034e-08, + "logits/chosen": -0.27598607540130615, + "logits/rejected": -0.279513955116272, + "logps/chosen": -1188.414794921875, + "logps/ref_chosen": -62.35459899902344, + "logps/ref_rejected": -104.56210327148438, + "logps/rejected": -2288.52587890625, + "loss": 1.3851, + "margin_dpo/margin_mean": 1057.9033203125, + "margin_dpo/margin_std": 1938.4671630859375, + "step": 552 + }, + { + "KL/chosen_KL_mean": -1235.739990234375, + "KL/mean": -1500.985595703125, + "KL/rejected_KL_mean": -1766.2314453125, + "KL/std": 1410.8720703125, + "epoch": 0.8120411160058737, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 530.491455078125, + "fcm_dpo/q_t": 0.49985459446907043, + "grad_norm": 0.0851590484380722, + "learning_rate": 5.2839396041230415e-08, + "logits/chosen": -0.307264506816864, + "logits/rejected": -0.3080984950065613, + "logps/chosen": -1303.998779296875, + "logps/ref_chosen": -68.25881958007812, + "logps/ref_rejected": -98.0971450805664, + "logps/rejected": -1864.32861328125, + "loss": 1.3857, + "margin_dpo/margin_mean": 530.4915161132812, + "margin_dpo/margin_std": 1780.082763671875, + "step": 553 + }, + { + "KL/chosen_KL_mean": -1218.116455078125, + "KL/mean": -1475.4810791015625, + "KL/rejected_KL_mean": -1732.845703125, + "KL/std": 1494.07275390625, + "epoch": 0.8135095447870778, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 514.72900390625, + "fcm_dpo/q_t": 0.49985891580581665, + "grad_norm": 0.17466723918914795, + "learning_rate": 5.205293880283551e-08, + "logits/chosen": -0.29812121391296387, + "logits/rejected": -0.3056314289569855, + "logps/chosen": -1286.064208984375, + "logps/ref_chosen": -67.94767761230469, + "logps/ref_rejected": -89.78272247314453, + "logps/rejected": -1822.62841796875, + "loss": 1.3857, + "margin_dpo/margin_mean": 514.7290649414062, + "margin_dpo/margin_std": 2061.585205078125, + "step": 554 + }, + { + "KL/chosen_KL_mean": -1154.8209228515625, + "KL/mean": -1718.331298828125, + "KL/rejected_KL_mean": -2281.841796875, + "KL/std": 1818.7333984375, + "epoch": 0.8149779735682819, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1127.020751953125, + "fcm_dpo/q_t": 0.49969106912612915, + "grad_norm": 0.10861152410507202, + "learning_rate": 5.127169765359515e-08, + "logits/chosen": -0.28964054584503174, + "logits/rejected": -0.30473101139068604, + "logps/chosen": -1208.1513671875, + "logps/ref_chosen": -53.33049011230469, + "logps/ref_rejected": -108.47937774658203, + "logps/rejected": -2390.321044921875, + "loss": 1.3851, + "margin_dpo/margin_mean": 1127.020751953125, + "margin_dpo/margin_std": 2308.85205078125, + "step": 555 + }, + { + "KL/chosen_KL_mean": -1159.304443359375, + "KL/mean": -1518.688232421875, + "KL/rejected_KL_mean": -1878.072021484375, + "KL/std": 1266.625244140625, + "epoch": 0.8164464023494861, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 718.7675170898438, + "fcm_dpo/q_t": 0.49980297684669495, + "grad_norm": 0.2766326069831848, + "learning_rate": 5.049569317994012e-08, + "logits/chosen": -0.292389452457428, + "logits/rejected": -0.3039926290512085, + "logps/chosen": -1217.948974609375, + "logps/ref_chosen": -58.64447021484375, + "logps/ref_rejected": -101.34040832519531, + "logps/rejected": -1979.4124755859375, + "loss": 1.3855, + "margin_dpo/margin_mean": 718.767578125, + "margin_dpo/margin_std": 1265.7459716796875, + "step": 556 + }, + { + "KL/chosen_KL_mean": -1447.0789794921875, + "KL/mean": -1821.1226806640625, + "KL/rejected_KL_mean": -2195.16650390625, + "KL/std": 1652.57763671875, + "epoch": 0.8179148311306902, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 748.08740234375, + "fcm_dpo/q_t": 0.49979496002197266, + "grad_norm": 0.14727355539798737, + "learning_rate": 4.9724945830310144e-08, + "logits/chosen": -0.3024740219116211, + "logits/rejected": -0.3093288242816925, + "logps/chosen": -1514.919677734375, + "logps/ref_chosen": -67.84066009521484, + "logps/ref_rejected": -109.93965911865234, + "logps/rejected": -2305.10595703125, + "loss": 1.3855, + "margin_dpo/margin_mean": 748.0873413085938, + "margin_dpo/margin_std": 1915.6552734375, + "step": 557 + }, + { + "KL/chosen_KL_mean": -1248.1954345703125, + "KL/mean": -1703.244140625, + "KL/rejected_KL_mean": -2158.29296875, + "KL/std": 1753.9251708984375, + "epoch": 0.8193832599118943, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 910.097412109375, + "fcm_dpo/q_t": 0.499750554561615, + "grad_norm": 0.15762417018413544, + "learning_rate": 4.8959475914614554e-08, + "logits/chosen": -0.29016417264938354, + "logits/rejected": -0.310586541891098, + "logps/chosen": -1310.563720703125, + "logps/ref_chosen": -62.36824035644531, + "logps/ref_rejected": -102.16102600097656, + "logps/rejected": -2260.453857421875, + "loss": 1.3853, + "margin_dpo/margin_mean": 910.097412109375, + "margin_dpo/margin_std": 2112.92626953125, + "step": 558 + }, + { + "KL/chosen_KL_mean": -1379.48486328125, + "KL/mean": -1745.984619140625, + "KL/rejected_KL_mean": -2112.484375, + "KL/std": 1507.8521728515625, + "epoch": 0.8208516886930984, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 732.9996337890625, + "fcm_dpo/q_t": 0.49979907274246216, + "grad_norm": 0.19343577325344086, + "learning_rate": 4.8199303603697614e-08, + "logits/chosen": -0.2922292947769165, + "logits/rejected": -0.2874986529350281, + "logps/chosen": -1440.237060546875, + "logps/ref_chosen": -60.752323150634766, + "logps/ref_rejected": -93.44229125976562, + "logps/rejected": -2205.9267578125, + "loss": 1.3855, + "margin_dpo/margin_mean": 732.9995727539062, + "margin_dpo/margin_std": 2046.0732421875, + "step": 559 + }, + { + "KL/chosen_KL_mean": -1155.489990234375, + "KL/mean": -1382.8680419921875, + "KL/rejected_KL_mean": -1610.24609375, + "KL/std": 1237.19091796875, + "epoch": 0.8223201174743024, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 454.7560729980469, + "fcm_dpo/q_t": 0.49987536668777466, + "grad_norm": 0.12281012535095215, + "learning_rate": 4.7444448928806615e-08, + "logits/chosen": -0.3528594374656677, + "logits/rejected": -0.3549266755580902, + "logps/chosen": -1213.5938720703125, + "logps/ref_chosen": -58.10382080078125, + "logps/ref_rejected": -79.99122619628906, + "logps/rejected": -1690.2373046875, + "loss": 1.3858, + "margin_dpo/margin_mean": 454.7560729980469, + "margin_dpo/margin_std": 1460.5318603515625, + "step": 560 + }, + { + "KL/chosen_KL_mean": -1324.06201171875, + "KL/mean": -1605.061767578125, + "KL/rejected_KL_mean": -1886.061279296875, + "KL/std": 1419.917236328125, + "epoch": 0.8237885462555066, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 561.9992065429688, + "fcm_dpo/q_t": 0.499845951795578, + "grad_norm": 0.09632235765457153, + "learning_rate": 4.669493178106432e-08, + "logits/chosen": -0.30480241775512695, + "logits/rejected": -0.3091276288032532, + "logps/chosen": -1374.9749755859375, + "logps/ref_chosen": -50.912879943847656, + "logps/ref_rejected": -99.06856536865234, + "logps/rejected": -1985.1298828125, + "loss": 1.3857, + "margin_dpo/margin_mean": 561.999267578125, + "margin_dpo/margin_std": 1919.3912353515625, + "step": 561 + }, + { + "KL/chosen_KL_mean": -1076.00439453125, + "KL/mean": -1480.02001953125, + "KL/rejected_KL_mean": -1884.035400390625, + "KL/std": 1383.9652099609375, + "epoch": 0.8252569750367107, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 808.031005859375, + "fcm_dpo/q_t": 0.49977853894233704, + "grad_norm": 0.11567293107509613, + "learning_rate": 4.5950771910944596e-08, + "logits/chosen": -0.2903629541397095, + "logits/rejected": -0.3054039776325226, + "logps/chosen": -1135.4688720703125, + "logps/ref_chosen": -59.46440124511719, + "logps/ref_rejected": -96.54266357421875, + "logps/rejected": -1980.578125, + "loss": 1.3854, + "margin_dpo/margin_mean": 808.031005859375, + "margin_dpo/margin_std": 1623.262939453125, + "step": 562 + }, + { + "KL/chosen_KL_mean": -1435.2509765625, + "KL/mean": -1691.4266357421875, + "KL/rejected_KL_mean": -1947.60205078125, + "KL/std": 1680.6656494140625, + "epoch": 0.8267254038179148, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 512.3511352539062, + "fcm_dpo/q_t": 0.4998595714569092, + "grad_norm": 0.2434445023536682, + "learning_rate": 4.521198892775202e-08, + "logits/chosen": -0.2510477304458618, + "logits/rejected": -0.2518584430217743, + "logps/chosen": -1495.859130859375, + "logps/ref_chosen": -60.60819625854492, + "logps/ref_rejected": -94.56770324707031, + "logps/rejected": -2042.1697998046875, + "loss": 1.3857, + "margin_dpo/margin_mean": 512.3511352539062, + "margin_dpo/margin_std": 1910.9169921875, + "step": 563 + }, + { + "KL/chosen_KL_mean": -1244.1939697265625, + "KL/mean": -1480.053955078125, + "KL/rejected_KL_mean": -1715.9140625, + "KL/std": 1373.150634765625, + "epoch": 0.8281938325991189, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 471.72003173828125, + "fcm_dpo/q_t": 0.4998707175254822, + "grad_norm": 0.11106614768505096, + "learning_rate": 4.447860229910544e-08, + "logits/chosen": -0.28053054213523865, + "logits/rejected": -0.276623010635376, + "logps/chosen": -1318.46240234375, + "logps/ref_chosen": -74.26837921142578, + "logps/ref_rejected": -93.23818969726562, + "logps/rejected": -1809.1522216796875, + "loss": 1.3858, + "margin_dpo/margin_mean": 471.72003173828125, + "margin_dpo/margin_std": 1663.8760986328125, + "step": 564 + }, + { + "KL/chosen_KL_mean": -1467.5367431640625, + "KL/mean": -1689.719482421875, + "KL/rejected_KL_mean": -1911.90234375, + "KL/std": 1842.6632080078125, + "epoch": 0.8296622613803231, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 444.36553955078125, + "fcm_dpo/q_t": 0.4998781979084015, + "grad_norm": 0.12787476181983948, + "learning_rate": 4.375063135042445e-08, + "logits/chosen": -0.3117861747741699, + "logits/rejected": -0.3122204542160034, + "logps/chosen": -1536.556640625, + "logps/ref_chosen": -69.0199203491211, + "logps/ref_rejected": -85.7789306640625, + "logps/rejected": -1997.6812744140625, + "loss": 1.3858, + "margin_dpo/margin_mean": 444.36553955078125, + "margin_dpo/margin_std": 2245.18408203125, + "step": 565 + }, + { + "KL/chosen_KL_mean": -1446.294677734375, + "KL/mean": -1779.049072265625, + "KL/rejected_KL_mean": -2111.803466796875, + "KL/std": 1603.380126953125, + "epoch": 0.8311306901615272, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 665.5087890625, + "fcm_dpo/q_t": 0.4998175799846649, + "grad_norm": 0.12405520677566528, + "learning_rate": 4.3028095264420525e-08, + "logits/chosen": -0.30484163761138916, + "logits/rejected": -0.3086811304092407, + "logps/chosen": -1512.8399658203125, + "logps/ref_chosen": -66.5453109741211, + "logps/ref_rejected": -103.86932373046875, + "logps/rejected": -2215.6728515625, + "loss": 1.3856, + "margin_dpo/margin_mean": 665.5087890625, + "margin_dpo/margin_std": 2031.6650390625, + "step": 566 + }, + { + "KL/chosen_KL_mean": -1235.8284912109375, + "KL/mean": -1397.942626953125, + "KL/rejected_KL_mean": -1560.056884765625, + "KL/std": 1143.8087158203125, + "epoch": 0.8325991189427313, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 324.22845458984375, + "fcm_dpo/q_t": 0.4999111294746399, + "grad_norm": 0.15335722267627716, + "learning_rate": 4.231101308059165e-08, + "logits/chosen": -0.23657140135765076, + "logits/rejected": -0.24908195436000824, + "logps/chosen": -1288.686767578125, + "logps/ref_chosen": -52.85829544067383, + "logps/ref_rejected": -85.37095642089844, + "logps/rejected": -1645.427978515625, + "loss": 1.3859, + "margin_dpo/margin_mean": 324.22845458984375, + "margin_dpo/margin_std": 1360.3614501953125, + "step": 567 + }, + { + "KL/chosen_KL_mean": -988.34228515625, + "KL/mean": -1452.588623046875, + "KL/rejected_KL_mean": -1916.8349609375, + "KL/std": 1454.534423828125, + "epoch": 0.8340675477239354, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 928.4924926757812, + "fcm_dpo/q_t": 0.49974551796913147, + "grad_norm": 0.1366208791732788, + "learning_rate": 4.1599403694720145e-08, + "logits/chosen": -0.28976213932037354, + "logits/rejected": -0.30281949043273926, + "logps/chosen": -1033.53466796875, + "logps/ref_chosen": -45.1923828125, + "logps/ref_rejected": -89.09236907958984, + "logps/rejected": -2005.92724609375, + "loss": 1.3853, + "margin_dpo/margin_mean": 928.4924926757812, + "margin_dpo/margin_std": 1870.345947265625, + "step": 568 + }, + { + "KL/chosen_KL_mean": -1346.852294921875, + "KL/mean": -1611.91015625, + "KL/rejected_KL_mean": -1876.9678955078125, + "KL/std": 1544.3018798828125, + "epoch": 0.8355359765051396, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 530.1155395507812, + "fcm_dpo/q_t": 0.4998546838760376, + "grad_norm": 0.11353439837694168, + "learning_rate": 4.089328585837512e-08, + "logits/chosen": -0.282316118478775, + "logits/rejected": -0.28839677572250366, + "logps/chosen": -1410.5728759765625, + "logps/ref_chosen": -63.72056198120117, + "logps/ref_rejected": -79.10325622558594, + "logps/rejected": -1956.071044921875, + "loss": 1.3857, + "margin_dpo/margin_mean": 530.1155395507812, + "margin_dpo/margin_std": 1794.29150390625, + "step": 569 + }, + { + "KL/chosen_KL_mean": -1434.75341796875, + "KL/mean": -1581.2052001953125, + "KL/rejected_KL_mean": -1727.656982421875, + "KL/std": 1449.1363525390625, + "epoch": 0.8370044052863436, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 292.9036865234375, + "fcm_dpo/q_t": 0.49991971254348755, + "grad_norm": 0.19029207527637482, + "learning_rate": 4.019267817841834e-08, + "logits/chosen": -0.3203706443309784, + "logits/rejected": -0.32249945402145386, + "logps/chosen": -1496.367919921875, + "logps/ref_chosen": -61.61454391479492, + "logps/ref_rejected": -82.14186096191406, + "logps/rejected": -1809.7989501953125, + "loss": 1.386, + "margin_dpo/margin_mean": 292.9036865234375, + "margin_dpo/margin_std": 1891.564697265625, + "step": 570 + }, + { + "KL/chosen_KL_mean": -1393.2899169921875, + "KL/mean": -1646.490478515625, + "KL/rejected_KL_mean": -1899.691162109375, + "KL/std": 1575.805908203125, + "epoch": 0.8384728340675477, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 506.4012756347656, + "fcm_dpo/q_t": 0.4998612105846405, + "grad_norm": 0.10252400487661362, + "learning_rate": 3.9497599116513705e-08, + "logits/chosen": -0.31373995542526245, + "logits/rejected": -0.3259819746017456, + "logps/chosen": -1446.343994140625, + "logps/ref_chosen": -53.05406188964844, + "logps/ref_rejected": -91.33682250976562, + "logps/rejected": -1991.02783203125, + "loss": 1.3857, + "margin_dpo/margin_mean": 506.4012145996094, + "margin_dpo/margin_std": 1877.108154296875, + "step": 571 + }, + { + "KL/chosen_KL_mean": -1139.670166015625, + "KL/mean": -1465.0963134765625, + "KL/rejected_KL_mean": -1790.5224609375, + "KL/std": 1222.9151611328125, + "epoch": 0.8399412628487518, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 650.8524169921875, + "fcm_dpo/q_t": 0.49982160329818726, + "grad_norm": 0.06481507420539856, + "learning_rate": 3.880806698864086e-08, + "logits/chosen": -0.29127341508865356, + "logits/rejected": -0.2989095449447632, + "logps/chosen": -1188.12939453125, + "logps/ref_chosen": -48.45928955078125, + "logps/ref_rejected": -83.55703735351562, + "logps/rejected": -1874.07958984375, + "loss": 1.3856, + "margin_dpo/margin_mean": 650.8524169921875, + "margin_dpo/margin_std": 1491.64501953125, + "step": 572 + }, + { + "KL/chosen_KL_mean": -1286.8150634765625, + "KL/mean": -1555.65185546875, + "KL/rejected_KL_mean": -1824.4886474609375, + "KL/std": 1397.0576171875, + "epoch": 0.8414096916299559, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 537.6735229492188, + "fcm_dpo/q_t": 0.49985259771347046, + "grad_norm": 0.19167454540729523, + "learning_rate": 3.812409996461275e-08, + "logits/chosen": -0.2431725263595581, + "logits/rejected": -0.24603477120399475, + "logps/chosen": -1338.437744140625, + "logps/ref_chosen": -51.62262725830078, + "logps/ref_rejected": -85.32499694824219, + "logps/rejected": -1909.8135986328125, + "loss": 1.3857, + "margin_dpo/margin_mean": 537.6735229492188, + "margin_dpo/margin_std": 1513.1845703125, + "step": 573 + }, + { + "KL/chosen_KL_mean": -1268.6588134765625, + "KL/mean": -1637.5677490234375, + "KL/rejected_KL_mean": -2006.4765625, + "KL/std": 1344.135009765625, + "epoch": 0.8428781204111601, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 737.81787109375, + "fcm_dpo/q_t": 0.4997977614402771, + "grad_norm": 0.1747097373008728, + "learning_rate": 3.74457160675965e-08, + "logits/chosen": -0.3217272460460663, + "logits/rejected": -0.3388410806655884, + "logps/chosen": -1319.7032470703125, + "logps/ref_chosen": -51.04446029663086, + "logps/ref_rejected": -92.80640411376953, + "logps/rejected": -2099.283203125, + "loss": 1.3855, + "margin_dpo/margin_mean": 737.81787109375, + "margin_dpo/margin_std": 1608.3687744140625, + "step": 574 + }, + { + "KL/chosen_KL_mean": -1481.532470703125, + "KL/mean": -1745.35205078125, + "KL/rejected_KL_mean": -2009.171630859375, + "KL/std": 1655.3306884765625, + "epoch": 0.8443465491923642, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 527.63916015625, + "fcm_dpo/q_t": 0.4998553693294525, + "grad_norm": 0.14064446091651917, + "learning_rate": 3.677293317363864e-08, + "logits/chosen": -0.28930941224098206, + "logits/rejected": -0.30165034532546997, + "logps/chosen": -1553.322509765625, + "logps/ref_chosen": -71.7901382446289, + "logps/ref_rejected": -95.38619995117188, + "logps/rejected": -2104.557861328125, + "loss": 1.3857, + "margin_dpo/margin_mean": 527.63916015625, + "margin_dpo/margin_std": 1978.1063232421875, + "step": 575 + }, + { + "KL/chosen_KL_mean": -1153.835205078125, + "KL/mean": -1575.923095703125, + "KL/rejected_KL_mean": -1998.0111083984375, + "KL/std": 1582.68212890625, + "epoch": 0.8458149779735683, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 844.176025390625, + "fcm_dpo/q_t": 0.49976861476898193, + "grad_norm": 0.17970313131809235, + "learning_rate": 3.6105769011194224e-08, + "logits/chosen": -0.26422959566116333, + "logits/rejected": -0.2707493007183075, + "logps/chosen": -1208.09814453125, + "logps/ref_chosen": -54.262962341308594, + "logps/ref_rejected": -100.75428009033203, + "logps/rejected": -2098.765380859375, + "loss": 1.3854, + "margin_dpo/margin_mean": 844.176025390625, + "margin_dpo/margin_std": 1967.03369140625, + "step": 576 + }, + { + "KL/chosen_KL_mean": -1261.4288330078125, + "KL/mean": -1436.2958984375, + "KL/rejected_KL_mean": -1611.1629638671875, + "KL/std": 1171.484130859375, + "epoch": 0.8472834067547724, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 349.734130859375, + "fcm_dpo/q_t": 0.49990415573120117, + "grad_norm": 0.11516524851322174, + "learning_rate": 3.5444241160659304e-08, + "logits/chosen": -0.28592342138290405, + "logits/rejected": -0.2935359477996826, + "logps/chosen": -1323.338623046875, + "logps/ref_chosen": -61.909706115722656, + "logps/ref_rejected": -84.07069396972656, + "logps/rejected": -1695.233642578125, + "loss": 1.3859, + "margin_dpo/margin_mean": 349.7341003417969, + "margin_dpo/margin_std": 1406.8392333984375, + "step": 577 + }, + { + "KL/chosen_KL_mean": -1049.0771484375, + "KL/mean": -1431.76318359375, + "KL/rejected_KL_mean": -1814.4490966796875, + "KL/std": 1241.43701171875, + "epoch": 0.8487518355359766, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 765.3720703125, + "fcm_dpo/q_t": 0.4997902512550354, + "grad_norm": 0.06985733658075333, + "learning_rate": 3.478836705390808e-08, + "logits/chosen": -0.26539209485054016, + "logits/rejected": -0.2741260528564453, + "logps/chosen": -1098.3408203125, + "logps/ref_chosen": -49.26368713378906, + "logps/ref_rejected": -83.4362564086914, + "logps/rejected": -1897.8853759765625, + "loss": 1.3855, + "margin_dpo/margin_mean": 765.3720092773438, + "margin_dpo/margin_std": 1238.185546875, + "step": 578 + }, + { + "KL/chosen_KL_mean": -1310.3199462890625, + "KL/mean": -1541.328857421875, + "KL/rejected_KL_mean": -1772.337890625, + "KL/std": 1645.46044921875, + "epoch": 0.8502202643171806, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 462.01800537109375, + "fcm_dpo/q_t": 0.4998733401298523, + "grad_norm": 0.11409077048301697, + "learning_rate": 3.41381639738331e-08, + "logits/chosen": -0.2609277367591858, + "logits/rejected": -0.27032503485679626, + "logps/chosen": -1369.205810546875, + "logps/ref_chosen": -58.88581848144531, + "logps/ref_rejected": -94.78762817382812, + "logps/rejected": -1867.12548828125, + "loss": 1.3858, + "margin_dpo/margin_mean": 462.0180358886719, + "margin_dpo/margin_std": 2106.6123046875, + "step": 579 + }, + { + "KL/chosen_KL_mean": -962.358154296875, + "KL/mean": -1297.83984375, + "KL/rejected_KL_mean": -1633.3214111328125, + "KL/std": 1324.714599609375, + "epoch": 0.8516886930983847, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 670.9632568359375, + "fcm_dpo/q_t": 0.49981608986854553, + "grad_norm": 0.15588979423046112, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": -0.29815828800201416, + "logits/rejected": -0.31025731563568115, + "logps/chosen": -1011.06494140625, + "logps/ref_chosen": -48.70683670043945, + "logps/ref_rejected": -81.7583999633789, + "logps/rejected": -1715.079833984375, + "loss": 1.3856, + "margin_dpo/margin_mean": 670.9632568359375, + "margin_dpo/margin_std": 1515.060302734375, + "step": 580 + }, + { + "KL/chosen_KL_mean": -1320.83349609375, + "KL/mean": -1543.870361328125, + "KL/rejected_KL_mean": -1766.9071044921875, + "KL/std": 1370.167724609375, + "epoch": 0.8531571218795888, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 446.07366943359375, + "fcm_dpo/q_t": 0.4998777508735657, + "grad_norm": 0.11037159711122513, + "learning_rate": 3.285483927764726e-08, + "logits/chosen": -0.2854025065898895, + "logits/rejected": -0.29709920287132263, + "logps/chosen": -1383.055908203125, + "logps/ref_chosen": -62.22235107421875, + "logps/ref_rejected": -91.73568725585938, + "logps/rejected": -1858.642822265625, + "loss": 1.3858, + "margin_dpo/margin_mean": 446.0736999511719, + "margin_dpo/margin_std": 1553.092041015625, + "step": 581 + }, + { + "KL/chosen_KL_mean": -1259.357421875, + "KL/mean": -1655.957275390625, + "KL/rejected_KL_mean": -2052.55712890625, + "KL/std": 1327.726318359375, + "epoch": 0.8546255506607929, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 793.1998901367188, + "fcm_dpo/q_t": 0.4997825622558594, + "grad_norm": 0.182276651263237, + "learning_rate": 3.222175147833556e-08, + "logits/chosen": -0.29098963737487793, + "logits/rejected": -0.3058258593082428, + "logps/chosen": -1317.5860595703125, + "logps/ref_chosen": -58.228660583496094, + "logps/ref_rejected": -110.06959533691406, + "logps/rejected": -2162.626953125, + "loss": 1.3854, + "margin_dpo/margin_mean": 793.1998901367188, + "margin_dpo/margin_std": 1530.111083984375, + "step": 582 + }, + { + "KL/chosen_KL_mean": -1210.8515625, + "KL/mean": -1471.690185546875, + "KL/rejected_KL_mean": -1732.52880859375, + "KL/std": 1479.544677734375, + "epoch": 0.856093979441997, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 521.67724609375, + "fcm_dpo/q_t": 0.49985700845718384, + "grad_norm": 0.2394990772008896, + "learning_rate": 3.159440233840763e-08, + "logits/chosen": -0.32131147384643555, + "logits/rejected": -0.3250162601470947, + "logps/chosen": -1267.7144775390625, + "logps/ref_chosen": -56.86286163330078, + "logps/ref_rejected": -88.4039306640625, + "logps/rejected": -1820.932861328125, + "loss": 1.3857, + "margin_dpo/margin_mean": 521.67724609375, + "margin_dpo/margin_std": 1832.6014404296875, + "step": 583 + }, + { + "KL/chosen_KL_mean": -1345.5399169921875, + "KL/mean": -1634.996826171875, + "KL/rejected_KL_mean": -1924.45361328125, + "KL/std": 1520.7857666015625, + "epoch": 0.8575624082232012, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 578.91357421875, + "fcm_dpo/q_t": 0.4998413324356079, + "grad_norm": 0.2904651463031769, + "learning_rate": 3.0972808389096635e-08, + "logits/chosen": -0.3031461238861084, + "logits/rejected": -0.315969854593277, + "logps/chosen": -1402.440673828125, + "logps/ref_chosen": -56.90068054199219, + "logps/ref_rejected": -97.63606262207031, + "logps/rejected": -2022.089599609375, + "loss": 1.3857, + "margin_dpo/margin_mean": 578.91357421875, + "margin_dpo/margin_std": 1762.9036865234375, + "step": 584 + }, + { + "KL/chosen_KL_mean": -1292.40576171875, + "KL/mean": -1650.23583984375, + "KL/rejected_KL_mean": -2008.06591796875, + "KL/std": 1508.56494140625, + "epoch": 0.8590308370044053, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 715.66015625, + "fcm_dpo/q_t": 0.4998038411140442, + "grad_norm": 0.07096313685178757, + "learning_rate": 3.035698600998121e-08, + "logits/chosen": -0.3163740932941437, + "logits/rejected": -0.32364410161972046, + "logps/chosen": -1353.379638671875, + "logps/ref_chosen": -60.973968505859375, + "logps/ref_rejected": -84.16952514648438, + "logps/rejected": -2092.2353515625, + "loss": 1.3855, + "margin_dpo/margin_mean": 715.66015625, + "margin_dpo/margin_std": 2054.603515625, + "step": 585 + }, + { + "KL/chosen_KL_mean": -1451.108642578125, + "KL/mean": -1723.1895751953125, + "KL/rejected_KL_mean": -1995.2705078125, + "KL/std": 1733.5499267578125, + "epoch": 0.8604992657856094, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 544.1619262695312, + "fcm_dpo/q_t": 0.4998508393764496, + "grad_norm": 0.13893093168735504, + "learning_rate": 2.974695142855388e-08, + "logits/chosen": -0.26731252670288086, + "logits/rejected": -0.2813112139701843, + "logps/chosen": -1507.9642333984375, + "logps/ref_chosen": -56.85559844970703, + "logps/ref_rejected": -91.80261993408203, + "logps/rejected": -2087.0732421875, + "loss": 1.3857, + "margin_dpo/margin_mean": 544.1619262695312, + "margin_dpo/margin_std": 1499.9649658203125, + "step": 586 + }, + { + "KL/chosen_KL_mean": -989.1253662109375, + "KL/mean": -1301.080810546875, + "KL/rejected_KL_mean": -1613.0361328125, + "KL/std": 1182.3687744140625, + "epoch": 0.8619676945668135, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 623.9107666015625, + "fcm_dpo/q_t": 0.4998289942741394, + "grad_norm": 0.19780661165714264, + "learning_rate": 2.9142720719793122e-08, + "logits/chosen": -0.26986804604530334, + "logits/rejected": -0.2820490598678589, + "logps/chosen": -1033.81689453125, + "logps/ref_chosen": -44.69159698486328, + "logps/ref_rejected": -82.62385559082031, + "logps/rejected": -1695.659912109375, + "loss": 1.3856, + "margin_dpo/margin_mean": 623.9107666015625, + "margin_dpo/margin_std": 1396.97021484375, + "step": 587 + }, + { + "KL/chosen_KL_mean": -1144.2742919921875, + "KL/mean": -1511.224609375, + "KL/rejected_KL_mean": -1878.175048828125, + "KL/std": 1298.37890625, + "epoch": 0.8634361233480177, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 733.900634765625, + "fcm_dpo/q_t": 0.49979886412620544, + "grad_norm": 0.20571593940258026, + "learning_rate": 2.8544309805740018e-08, + "logits/chosen": -0.27409303188323975, + "logits/rejected": -0.28234797716140747, + "logps/chosen": -1194.5692138671875, + "logps/ref_chosen": -50.29494857788086, + "logps/ref_rejected": -107.36988067626953, + "logps/rejected": -1985.544921875, + "loss": 1.3855, + "margin_dpo/margin_mean": 733.900634765625, + "margin_dpo/margin_std": 1562.85205078125, + "step": 588 + }, + { + "KL/chosen_KL_mean": -1428.042724609375, + "KL/mean": -1843.700439453125, + "KL/rejected_KL_mean": -2259.358154296875, + "KL/std": 1713.4039306640625, + "epoch": 0.8649045521292217, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 831.3154296875, + "fcm_dpo/q_t": 0.4997721314430237, + "grad_norm": 0.11402349174022675, + "learning_rate": 2.7951734455078786e-08, + "logits/chosen": -0.3039804995059967, + "logits/rejected": -0.31366050243377686, + "logps/chosen": -1487.97265625, + "logps/ref_chosen": -59.929908752441406, + "logps/ref_rejected": -111.65534973144531, + "logps/rejected": -2371.013671875, + "loss": 1.3854, + "margin_dpo/margin_mean": 831.3154296875, + "margin_dpo/margin_std": 2030.774658203125, + "step": 589 + }, + { + "KL/chosen_KL_mean": -1006.583251953125, + "KL/mean": -1440.79833984375, + "KL/rejected_KL_mean": -1875.013427734375, + "KL/std": 1557.4278564453125, + "epoch": 0.8663729809104258, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 868.43017578125, + "fcm_dpo/q_t": 0.4997619688510895, + "grad_norm": 0.09915481507778168, + "learning_rate": 2.736501028272095e-08, + "logits/chosen": -0.29607412219047546, + "logits/rejected": -0.30719584226608276, + "logps/chosen": -1062.39306640625, + "logps/ref_chosen": -55.80979537963867, + "logps/ref_rejected": -106.06282043457031, + "logps/rejected": -1981.076171875, + "loss": 1.3853, + "margin_dpo/margin_mean": 868.4302368164062, + "margin_dpo/margin_std": 1991.411865234375, + "step": 590 + }, + { + "KL/chosen_KL_mean": -1081.7724609375, + "KL/mean": -1400.282470703125, + "KL/rejected_KL_mean": -1718.79248046875, + "KL/std": 1268.409912109375, + "epoch": 0.8678414096916299, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 637.02001953125, + "fcm_dpo/q_t": 0.4998254179954529, + "grad_norm": 0.149546280503273, + "learning_rate": 2.678415274939408e-08, + "logits/chosen": -0.2419218122959137, + "logits/rejected": -0.2442915141582489, + "logps/chosen": -1138.0130615234375, + "logps/ref_chosen": -56.24061965942383, + "logps/ref_rejected": -83.78629302978516, + "logps/rejected": -1802.5787353515625, + "loss": 1.3856, + "margin_dpo/margin_mean": 637.02001953125, + "margin_dpo/margin_std": 1643.7451171875, + "step": 591 + }, + { + "KL/chosen_KL_mean": -1236.419189453125, + "KL/mean": -1495.9189453125, + "KL/rejected_KL_mean": -1755.418701171875, + "KL/std": 1230.9473876953125, + "epoch": 0.869309838472834, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 518.9995727539062, + "fcm_dpo/q_t": 0.49985772371292114, + "grad_norm": 0.11462758481502533, + "learning_rate": 2.6209177161234442e-08, + "logits/chosen": -0.2386205792427063, + "logits/rejected": -0.24258291721343994, + "logps/chosen": -1284.359375, + "logps/ref_chosen": -47.94025421142578, + "logps/ref_rejected": -75.73287963867188, + "logps/rejected": -1831.151611328125, + "loss": 1.3857, + "margin_dpo/margin_mean": 518.9995727539062, + "margin_dpo/margin_std": 1722.8271484375, + "step": 592 + }, + { + "KL/chosen_KL_mean": -1129.862548828125, + "KL/mean": -1286.4898681640625, + "KL/rejected_KL_mean": -1443.1171875, + "KL/std": 1180.02734375, + "epoch": 0.8707782672540382, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 313.25439453125, + "fcm_dpo/q_t": 0.49991413950920105, + "grad_norm": 0.07443266361951828, + "learning_rate": 2.564009866938349e-08, + "logits/chosen": -0.296904981136322, + "logits/rejected": -0.302301287651062, + "logps/chosen": -1178.5533447265625, + "logps/ref_chosen": -48.690757751464844, + "logps/ref_rejected": -60.90800094604492, + "logps/rejected": -1504.025146484375, + "loss": 1.386, + "margin_dpo/margin_mean": 313.2544250488281, + "margin_dpo/margin_std": 1417.1861572265625, + "step": 593 + }, + { + "KL/chosen_KL_mean": -1156.621337890625, + "KL/mean": -1437.635986328125, + "KL/rejected_KL_mean": -1718.6505126953125, + "KL/std": 1289.333984375, + "epoch": 0.8722466960352423, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 562.0292358398438, + "fcm_dpo/q_t": 0.499845951795578, + "grad_norm": 0.08804459124803543, + "learning_rate": 2.5076932269588708e-08, + "logits/chosen": -0.29046431183815, + "logits/rejected": -0.291781485080719, + "logps/chosen": -1211.55615234375, + "logps/ref_chosen": -54.93488693237305, + "logps/ref_rejected": -86.09967803955078, + "logps/rejected": -1804.750244140625, + "loss": 1.3857, + "margin_dpo/margin_mean": 562.029296875, + "margin_dpo/margin_std": 1550.49609375, + "step": 594 + }, + { + "KL/chosen_KL_mean": -1090.362060546875, + "KL/mean": -1483.697021484375, + "KL/rejected_KL_mean": -1877.0322265625, + "KL/std": 1451.2752685546875, + "epoch": 0.8737151248164464, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 786.6702880859375, + "fcm_dpo/q_t": 0.49978435039520264, + "grad_norm": 0.18253269791603088, + "learning_rate": 2.451969280180849e-08, + "logits/chosen": -0.2718271017074585, + "logits/rejected": -0.2887038588523865, + "logps/chosen": -1139.782470703125, + "logps/ref_chosen": -49.4204216003418, + "logps/ref_rejected": -80.62731170654297, + "logps/rejected": -1957.65966796875, + "loss": 1.3854, + "margin_dpo/margin_mean": 786.6702880859375, + "margin_dpo/margin_std": 1885.255859375, + "step": 595 + }, + { + "KL/chosen_KL_mean": -1452.7657470703125, + "KL/mean": -1550.644287109375, + "KL/rejected_KL_mean": -1648.522705078125, + "KL/std": 1658.8240966796875, + "epoch": 0.8751835535976505, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 195.75689697265625, + "fcm_dpo/q_t": 0.4999462962150574, + "grad_norm": 0.1237877607345581, + "learning_rate": 2.396839494982103e-08, + "logits/chosen": -0.2950517535209656, + "logits/rejected": -0.30836156010627747, + "logps/chosen": -1512.557373046875, + "logps/ref_chosen": -59.791683197021484, + "logps/ref_rejected": -80.09111785888672, + "logps/rejected": -1728.61376953125, + "loss": 1.3861, + "margin_dpo/margin_mean": 195.75692749023438, + "margin_dpo/margin_std": 2088.86572265625, + "step": 596 + }, + { + "KL/chosen_KL_mean": -1269.8262939453125, + "KL/mean": -1677.495849609375, + "KL/rejected_KL_mean": -2085.16552734375, + "KL/std": 1558.5076904296875, + "epoch": 0.8766519823788547, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 815.3389892578125, + "fcm_dpo/q_t": 0.49977654218673706, + "grad_norm": 0.09684523195028305, + "learning_rate": 2.3423053240837514e-08, + "logits/chosen": -0.2993488013744354, + "logits/rejected": -0.3121323585510254, + "logps/chosen": -1327.087158203125, + "logps/ref_chosen": -57.26078796386719, + "logps/ref_rejected": -100.6937255859375, + "logps/rejected": -2185.859130859375, + "loss": 1.3854, + "margin_dpo/margin_mean": 815.3390502929688, + "margin_dpo/margin_std": 1795.6142578125, + "step": 597 + }, + { + "KL/chosen_KL_mean": -1115.9447021484375, + "KL/mean": -1398.647216796875, + "KL/rejected_KL_mean": -1681.349853515625, + "KL/std": 1223.042236328125, + "epoch": 0.8781204111600588, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 565.4051513671875, + "fcm_dpo/q_t": 0.499845027923584, + "grad_norm": 0.0884322002530098, + "learning_rate": 2.2883682045119062e-08, + "logits/chosen": -0.25602036714553833, + "logits/rejected": -0.26312920451164246, + "logps/chosen": -1168.463134765625, + "logps/ref_chosen": -52.51850509643555, + "logps/ref_rejected": -89.44385528564453, + "logps/rejected": -1770.793701171875, + "loss": 1.3857, + "margin_dpo/margin_mean": 565.4051513671875, + "margin_dpo/margin_std": 1462.629150390625, + "step": 598 + }, + { + "KL/chosen_KL_mean": -1233.416015625, + "KL/mean": -1432.468017578125, + "KL/rejected_KL_mean": -1631.52001953125, + "KL/std": 1322.410888671875, + "epoch": 0.8795888399412628, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 398.10406494140625, + "fcm_dpo/q_t": 0.49989089369773865, + "grad_norm": 0.11296947300434113, + "learning_rate": 2.2350295575598367e-08, + "logits/chosen": -0.30364251136779785, + "logits/rejected": -0.3106527626514435, + "logps/chosen": -1283.21875, + "logps/ref_chosen": -49.802677154541016, + "logps/ref_rejected": -82.978515625, + "logps/rejected": -1714.49853515625, + "loss": 1.3859, + "margin_dpo/margin_mean": 398.10406494140625, + "margin_dpo/margin_std": 1483.496337890625, + "step": 599 + }, + { + "KL/chosen_KL_mean": -1541.261962890625, + "KL/mean": -1724.763427734375, + "KL/rejected_KL_mean": -1908.2646484375, + "KL/std": 1566.6689453125, + "epoch": 0.8810572687224669, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 367.0028991699219, + "fcm_dpo/q_t": 0.49989941716194153, + "grad_norm": 0.14191482961177826, + "learning_rate": 2.1822907887504932e-08, + "logits/chosen": -0.28465044498443604, + "logits/rejected": -0.2864704430103302, + "logps/chosen": -1607.69677734375, + "logps/ref_chosen": -66.43487548828125, + "logps/ref_rejected": -85.45649719238281, + "logps/rejected": -1993.72119140625, + "loss": 1.3859, + "margin_dpo/margin_mean": 367.0029296875, + "margin_dpo/margin_std": 2167.3740234375, + "step": 600 + }, + { + "KL/chosen_KL_mean": -1384.5076904296875, + "KL/mean": -1658.9830322265625, + "KL/rejected_KL_mean": -1933.458251953125, + "KL/std": 1353.0380859375, + "epoch": 0.882525697503671, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 548.95068359375, + "fcm_dpo/q_t": 0.4998495280742645, + "grad_norm": 0.10512831807136536, + "learning_rate": 2.1301532877994742e-08, + "logits/chosen": -0.27232688665390015, + "logits/rejected": -0.28348982334136963, + "logps/chosen": -1443.641357421875, + "logps/ref_chosen": -59.13361358642578, + "logps/ref_rejected": -94.69093322753906, + "logps/rejected": -2028.1492919921875, + "loss": 1.3857, + "margin_dpo/margin_mean": 548.95068359375, + "margin_dpo/margin_std": 1670.189208984375, + "step": 601 + }, + { + "KL/chosen_KL_mean": -983.9776000976562, + "KL/mean": -1467.59375, + "KL/rejected_KL_mean": -1951.2099609375, + "KL/std": 1416.9248046875, + "epoch": 0.8839941262848752, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 967.2324829101562, + "fcm_dpo/q_t": 0.49973487854003906, + "grad_norm": 0.20267777144908905, + "learning_rate": 2.0786184285784298e-08, + "logits/chosen": -0.3050517141819, + "logits/rejected": -0.31041401624679565, + "logps/chosen": -1032.5711669921875, + "logps/ref_chosen": -48.59352111816406, + "logps/ref_rejected": -87.6685562133789, + "logps/rejected": -2038.878662109375, + "loss": 1.3852, + "margin_dpo/margin_mean": 967.2324829101562, + "margin_dpo/margin_std": 1628.935302734375, + "step": 602 + }, + { + "KL/chosen_KL_mean": -1451.09375, + "KL/mean": -1751.2628173828125, + "KL/rejected_KL_mean": -2051.431884765625, + "KL/std": 1832.94677734375, + "epoch": 0.8854625550660793, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 600.338134765625, + "fcm_dpo/q_t": 0.49983546137809753, + "grad_norm": 0.17601169645786285, + "learning_rate": 2.0276875690788204e-08, + "logits/chosen": -0.317105770111084, + "logits/rejected": -0.32393383979797363, + "logps/chosen": -1521.50830078125, + "logps/ref_chosen": -70.41461944580078, + "logps/ref_rejected": -100.32559967041016, + "logps/rejected": -2151.75732421875, + "loss": 1.3856, + "margin_dpo/margin_mean": 600.338134765625, + "margin_dpo/margin_std": 2273.0205078125, + "step": 603 + }, + { + "KL/chosen_KL_mean": -1229.3902587890625, + "KL/mean": -1712.8712158203125, + "KL/rejected_KL_mean": -2196.35205078125, + "KL/std": 1597.14794921875, + "epoch": 0.8869309838472834, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 966.9619140625, + "fcm_dpo/q_t": 0.49973493814468384, + "grad_norm": 0.16109979152679443, + "learning_rate": 1.977362051376158e-08, + "logits/chosen": -0.2505277395248413, + "logits/rejected": -0.2674684226512909, + "logps/chosen": -1275.848388671875, + "logps/ref_chosen": -46.45808029174805, + "logps/ref_rejected": -91.8544921875, + "logps/rejected": -2288.20654296875, + "loss": 1.3852, + "margin_dpo/margin_mean": 966.9619750976562, + "margin_dpo/margin_std": 2260.287109375, + "step": 604 + }, + { + "KL/chosen_KL_mean": -1268.0721435546875, + "KL/mean": -1561.545654296875, + "KL/rejected_KL_mean": -1855.019287109375, + "KL/std": 1494.844482421875, + "epoch": 0.8883994126284875, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 586.9471435546875, + "fcm_dpo/q_t": 0.4998391270637512, + "grad_norm": 0.11086463928222656, + "learning_rate": 1.9276432015946446e-08, + "logits/chosen": -0.32396382093429565, + "logits/rejected": -0.3408808708190918, + "logps/chosen": -1334.321533203125, + "logps/ref_chosen": -66.24933624267578, + "logps/ref_rejected": -102.30496978759766, + "logps/rejected": -1957.32421875, + "loss": 1.3857, + "margin_dpo/margin_mean": 586.9471435546875, + "margin_dpo/margin_std": 2043.394287109375, + "step": 605 + }, + { + "KL/chosen_KL_mean": -1332.7421875, + "KL/mean": -1664.57763671875, + "KL/rejected_KL_mean": -1996.4130859375, + "KL/std": 1885.167236328125, + "epoch": 0.8898678414096917, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 663.6708984375, + "fcm_dpo/q_t": 0.4998180866241455, + "grad_norm": 0.10756733268499374, + "learning_rate": 1.8785323298722093e-08, + "logits/chosen": -0.26073816418647766, + "logits/rejected": -0.2791990637779236, + "logps/chosen": -1387.561279296875, + "logps/ref_chosen": -54.819122314453125, + "logps/ref_rejected": -98.37146759033203, + "logps/rejected": -2094.784423828125, + "loss": 1.3856, + "margin_dpo/margin_mean": 663.6708984375, + "margin_dpo/margin_std": 2427.927734375, + "step": 606 + }, + { + "KL/chosen_KL_mean": -1345.4205322265625, + "KL/mean": -1563.27392578125, + "KL/rejected_KL_mean": -1781.12744140625, + "KL/std": 1508.8651123046875, + "epoch": 0.8913362701908958, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 435.70703125, + "fcm_dpo/q_t": 0.4998805820941925, + "grad_norm": 0.14810404181480408, + "learning_rate": 1.8300307303259904e-08, + "logits/chosen": -0.26724961400032043, + "logits/rejected": -0.26801979541778564, + "logps/chosen": -1403.5045166015625, + "logps/ref_chosen": -58.08403778076172, + "logps/ref_rejected": -79.777099609375, + "logps/rejected": -1860.904541015625, + "loss": 1.3858, + "margin_dpo/margin_mean": 435.70703125, + "margin_dpo/margin_std": 1512.35107421875, + "step": 607 + }, + { + "KL/chosen_KL_mean": -1220.277099609375, + "KL/mean": -1495.2237548828125, + "KL/rejected_KL_mean": -1770.17041015625, + "KL/std": 1245.552734375, + "epoch": 0.8928046989720999, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 549.8931274414062, + "fcm_dpo/q_t": 0.49984925985336304, + "grad_norm": 0.1457754224538803, + "learning_rate": 1.7821396810182437e-08, + "logits/chosen": -0.30721715092658997, + "logits/rejected": -0.30426025390625, + "logps/chosen": -1277.72802734375, + "logps/ref_chosen": -57.450836181640625, + "logps/ref_rejected": -94.77339172363281, + "logps/rejected": -1864.9437255859375, + "loss": 1.3857, + "margin_dpo/margin_mean": 549.8931274414062, + "margin_dpo/margin_std": 1515.70947265625, + "step": 608 + }, + { + "KL/chosen_KL_mean": -1087.919189453125, + "KL/mean": -1466.904052734375, + "KL/rejected_KL_mean": -1845.888671875, + "KL/std": 1240.9404296875, + "epoch": 0.8942731277533039, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 757.969482421875, + "fcm_dpo/q_t": 0.4997922480106354, + "grad_norm": 0.11522836238145828, + "learning_rate": 1.7348604439226617e-08, + "logits/chosen": -0.2894991636276245, + "logits/rejected": -0.2997610569000244, + "logps/chosen": -1146.724609375, + "logps/ref_chosen": -58.805355072021484, + "logps/ref_rejected": -88.81600952148438, + "logps/rejected": -1934.70458984375, + "loss": 1.3855, + "margin_dpo/margin_mean": 757.969482421875, + "margin_dpo/margin_std": 1586.349609375, + "step": 609 + }, + { + "KL/chosen_KL_mean": -1374.5, + "KL/mean": -1497.664794921875, + "KL/rejected_KL_mean": -1620.8294677734375, + "KL/std": 1488.536865234375, + "epoch": 0.895741556534508, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 246.32940673828125, + "fcm_dpo/q_t": 0.49993249773979187, + "grad_norm": 0.23700614273548126, + "learning_rate": 1.6881942648911074e-08, + "logits/chosen": -0.30130696296691895, + "logits/rejected": -0.30593734979629517, + "logps/chosen": -1440.195068359375, + "logps/ref_chosen": -65.69503784179688, + "logps/ref_rejected": -83.40538787841797, + "logps/rejected": -1704.23486328125, + "loss": 1.386, + "margin_dpo/margin_mean": 246.32940673828125, + "margin_dpo/margin_std": 2019.56494140625, + "step": 610 + }, + { + "KL/chosen_KL_mean": -1254.2606201171875, + "KL/mean": -1662.910888671875, + "KL/rejected_KL_mean": -2071.561279296875, + "KL/std": 1563.9205322265625, + "epoch": 0.8972099853157122, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 817.3005981445312, + "fcm_dpo/q_t": 0.4997760057449341, + "grad_norm": 0.11909603327512741, + "learning_rate": 1.6421423736208e-08, + "logits/chosen": -0.2602178752422333, + "logits/rejected": -0.27245327830314636, + "logps/chosen": -1306.860107421875, + "logps/ref_chosen": -52.59946823120117, + "logps/ref_rejected": -86.33099365234375, + "logps/rejected": -2157.89208984375, + "loss": 1.3854, + "margin_dpo/margin_mean": 817.3007202148438, + "margin_dpo/margin_std": 1820.92919921875, + "step": 611 + }, + { + "KL/chosen_KL_mean": -1383.036376953125, + "KL/mean": -1637.382568359375, + "KL/rejected_KL_mean": -1891.7288818359375, + "KL/std": 1444.4013671875, + "epoch": 0.8986784140969163, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 508.692626953125, + "fcm_dpo/q_t": 0.49986058473587036, + "grad_norm": 0.17006807029247284, + "learning_rate": 1.5967059836219042e-08, + "logits/chosen": -0.24827291071414948, + "logits/rejected": -0.2793204188346863, + "logps/chosen": -1442.3599853515625, + "logps/ref_chosen": -59.32372283935547, + "logps/ref_rejected": -88.31239318847656, + "logps/rejected": -1980.041259765625, + "loss": 1.3857, + "margin_dpo/margin_mean": 508.692626953125, + "margin_dpo/margin_std": 1928.337890625, + "step": 612 + }, + { + "KL/chosen_KL_mean": -1221.9931640625, + "KL/mean": -1589.656005859375, + "KL/rejected_KL_mean": -1957.319091796875, + "KL/std": 1439.966796875, + "epoch": 0.9001468428781204, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 735.325927734375, + "fcm_dpo/q_t": 0.4997984766960144, + "grad_norm": 0.09561502188444138, + "learning_rate": 1.551886292185553e-08, + "logits/chosen": -0.2727387547492981, + "logits/rejected": -0.2924768030643463, + "logps/chosen": -1281.72314453125, + "logps/ref_chosen": -59.72996520996094, + "logps/ref_rejected": -105.10752868652344, + "logps/rejected": -2062.4267578125, + "loss": 1.3855, + "margin_dpo/margin_mean": 735.325927734375, + "margin_dpo/margin_std": 1807.414306640625, + "step": 613 + }, + { + "KL/chosen_KL_mean": -1286.597900390625, + "KL/mean": -1722.857177734375, + "KL/rejected_KL_mean": -2159.116455078125, + "KL/std": 1653.336669921875, + "epoch": 0.9016152716593245, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 872.5186157226562, + "fcm_dpo/q_t": 0.49976086616516113, + "grad_norm": 0.130734384059906, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": -0.28130537271499634, + "logits/rejected": -0.2899243235588074, + "logps/chosen": -1339.536865234375, + "logps/ref_chosen": -52.93898010253906, + "logps/ref_rejected": -104.67938232421875, + "logps/rejected": -2263.7958984375, + "loss": 1.3853, + "margin_dpo/margin_mean": 872.5186157226562, + "margin_dpo/margin_std": 2277.9794921875, + "step": 614 + }, + { + "KL/chosen_KL_mean": -1210.6558837890625, + "KL/mean": -1517.5098876953125, + "KL/rejected_KL_mean": -1824.3638916015625, + "KL/std": 1424.183837890625, + "epoch": 0.9030837004405287, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 613.7081298828125, + "fcm_dpo/q_t": 0.49983179569244385, + "grad_norm": 0.09694825112819672, + "learning_rate": 1.4641017128809801e-08, + "logits/chosen": -0.306599497795105, + "logits/rejected": -0.31541186571121216, + "logps/chosen": -1276.47314453125, + "logps/ref_chosen": -65.81727600097656, + "logps/ref_rejected": -95.17749786376953, + "logps/rejected": -1919.5413818359375, + "loss": 1.3856, + "margin_dpo/margin_mean": 613.7081298828125, + "margin_dpo/margin_std": 1839.60302734375, + "step": 615 + }, + { + "KL/chosen_KL_mean": -1449.919189453125, + "KL/mean": -1705.6136474609375, + "KL/rejected_KL_mean": -1961.30810546875, + "KL/std": 1484.240966796875, + "epoch": 0.9045521292217328, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 511.3887939453125, + "fcm_dpo/q_t": 0.49985983967781067, + "grad_norm": 0.09458575397729874, + "learning_rate": 1.4211391382180637e-08, + "logits/chosen": -0.2769748270511627, + "logits/rejected": -0.2750539779663086, + "logps/chosen": -1515.0521240234375, + "logps/ref_chosen": -65.13285827636719, + "logps/ref_rejected": -74.70050048828125, + "logps/rejected": -2036.008544921875, + "loss": 1.3857, + "margin_dpo/margin_mean": 511.3887939453125, + "margin_dpo/margin_std": 1792.7286376953125, + "step": 616 + }, + { + "KL/chosen_KL_mean": -1687.5743408203125, + "KL/mean": -1679.172119140625, + "KL/rejected_KL_mean": -1670.7698974609375, + "KL/std": 1601.2689208984375, + "epoch": 0.9060205580029369, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -16.80441665649414, + "fcm_dpo/q_t": 0.5000046491622925, + "grad_norm": 0.2592397928237915, + "learning_rate": 1.378797888467345e-08, + "logits/chosen": -0.28151631355285645, + "logits/rejected": -0.28539106249809265, + "logps/chosen": -1750.579833984375, + "logps/ref_chosen": -63.005550384521484, + "logps/ref_rejected": -64.234130859375, + "logps/rejected": -1735.0040283203125, + "loss": 1.3863, + "margin_dpo/margin_mean": -16.804431915283203, + "margin_dpo/margin_std": 2027.412109375, + "step": 617 + }, + { + "KL/chosen_KL_mean": -1634.56396484375, + "KL/mean": -1991.8416748046875, + "KL/rejected_KL_mean": -2349.119140625, + "KL/std": 1661.743408203125, + "epoch": 0.9074889867841409, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 714.5552978515625, + "fcm_dpo/q_t": 0.49980413913726807, + "grad_norm": 0.1074250191450119, + "learning_rate": 1.3370790793601371e-08, + "logits/chosen": -0.29117417335510254, + "logits/rejected": -0.2950536608695984, + "logps/chosen": -1701.665283203125, + "logps/ref_chosen": -67.10134887695312, + "logps/ref_rejected": -92.15340423583984, + "logps/rejected": -2441.272705078125, + "loss": 1.3855, + "margin_dpo/margin_mean": 714.5552978515625, + "margin_dpo/margin_std": 1959.51904296875, + "step": 618 + }, + { + "KL/chosen_KL_mean": -1394.2530517578125, + "KL/mean": -1653.03515625, + "KL/rejected_KL_mean": -1911.817138671875, + "KL/std": 1647.781982421875, + "epoch": 0.908957415565345, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 517.564208984375, + "fcm_dpo/q_t": 0.49985814094543457, + "grad_norm": 0.1203947588801384, + "learning_rate": 1.2959838102258535e-08, + "logits/chosen": -0.29787400364875793, + "logits/rejected": -0.30747509002685547, + "logps/chosen": -1450.2313232421875, + "logps/ref_chosen": -55.978233337402344, + "logps/ref_rejected": -93.1854019165039, + "logps/rejected": -2005.002685546875, + "loss": 1.3857, + "margin_dpo/margin_mean": 517.5641479492188, + "margin_dpo/margin_std": 2044.474853515625, + "step": 619 + }, + { + "KL/chosen_KL_mean": -1305.902099609375, + "KL/mean": -1547.302734375, + "KL/rejected_KL_mean": -1788.7034912109375, + "KL/std": 1333.70654296875, + "epoch": 0.9104258443465492, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 482.80145263671875, + "fcm_dpo/q_t": 0.49986767768859863, + "grad_norm": 0.10311917960643768, + "learning_rate": 1.2555131639630567e-08, + "logits/chosen": -0.2799570858478546, + "logits/rejected": -0.2871643304824829, + "logps/chosen": -1365.6995849609375, + "logps/ref_chosen": -59.79750061035156, + "logps/ref_rejected": -78.41075134277344, + "logps/rejected": -1867.1142578125, + "loss": 1.3858, + "margin_dpo/margin_mean": 482.8013916015625, + "margin_dpo/margin_std": 1826.294189453125, + "step": 620 + }, + { + "KL/chosen_KL_mean": -1232.763427734375, + "KL/mean": -1668.678955078125, + "KL/rejected_KL_mean": -2104.59423828125, + "KL/std": 1500.2493896484375, + "epoch": 0.9118942731277533, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 871.8309936523438, + "fcm_dpo/q_t": 0.49976104497909546, + "grad_norm": 0.08543728291988373, + "learning_rate": 1.2156682070109086e-08, + "logits/chosen": -0.26303690671920776, + "logits/rejected": -0.27306389808654785, + "logps/chosen": -1286.697265625, + "logps/ref_chosen": -53.93375778198242, + "logps/ref_rejected": -88.36951446533203, + "logps/rejected": -2192.9638671875, + "loss": 1.3853, + "margin_dpo/margin_mean": 871.8309326171875, + "margin_dpo/margin_std": 1943.66357421875, + "step": 621 + }, + { + "KL/chosen_KL_mean": -1230.0145263671875, + "KL/mean": -1463.860107421875, + "KL/rejected_KL_mean": -1697.70556640625, + "KL/std": 1287.034423828125, + "epoch": 0.9133627019089574, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 467.6911926269531, + "fcm_dpo/q_t": 0.49987179040908813, + "grad_norm": 0.12527480721473694, + "learning_rate": 1.1764499893210878e-08, + "logits/chosen": -0.2892225980758667, + "logits/rejected": -0.29653114080429077, + "logps/chosen": -1290.30029296875, + "logps/ref_chosen": -60.28582000732422, + "logps/ref_rejected": -85.51873779296875, + "logps/rejected": -1783.224365234375, + "loss": 1.3858, + "margin_dpo/margin_mean": 467.6911926269531, + "margin_dpo/margin_std": 1662.3623046875, + "step": 622 + }, + { + "KL/chosen_KL_mean": -1547.281494140625, + "KL/mean": -1695.7080078125, + "KL/rejected_KL_mean": -1844.134765625, + "KL/std": 1741.5623779296875, + "epoch": 0.9148311306901615, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 296.853271484375, + "fcm_dpo/q_t": 0.4999186396598816, + "grad_norm": 0.21695291996002197, + "learning_rate": 1.1378595443300998e-08, + "logits/chosen": -0.2584553360939026, + "logits/rejected": -0.25353342294692993, + "logps/chosen": -1611.4383544921875, + "logps/ref_chosen": -64.1569595336914, + "logps/ref_rejected": -85.08304595947266, + "logps/rejected": -1929.2177734375, + "loss": 1.386, + "margin_dpo/margin_mean": 296.8532409667969, + "margin_dpo/margin_std": 2120.8935546875, + "step": 623 + }, + { + "KL/chosen_KL_mean": -1372.0699462890625, + "KL/mean": -1760.053955078125, + "KL/rejected_KL_mean": -2148.037841796875, + "KL/std": 1600.662353515625, + "epoch": 0.9162995594713657, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 775.9679565429688, + "fcm_dpo/q_t": 0.4997873306274414, + "grad_norm": 0.13158676028251648, + "learning_rate": 1.0998978889320582e-08, + "logits/chosen": -0.30938100814819336, + "logits/rejected": -0.3142179846763611, + "logps/chosen": -1443.988525390625, + "logps/ref_chosen": -71.91862487792969, + "logps/ref_rejected": -97.13203430175781, + "logps/rejected": -2245.169921875, + "loss": 1.3854, + "margin_dpo/margin_mean": 775.9679565429688, + "margin_dpo/margin_std": 2050.025390625, + "step": 624 + }, + { + "KL/chosen_KL_mean": -1321.643798828125, + "KL/mean": -1664.8505859375, + "KL/rejected_KL_mean": -2008.057373046875, + "KL/std": 1380.1881103515625, + "epoch": 0.9177679882525698, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 686.41357421875, + "fcm_dpo/q_t": 0.49981188774108887, + "grad_norm": 0.08273319900035858, + "learning_rate": 1.0625660234518913e-08, + "logits/chosen": -0.2660679519176483, + "logits/rejected": -0.26351404190063477, + "logps/chosen": -1379.98583984375, + "logps/ref_chosen": -58.342071533203125, + "logps/ref_rejected": -86.09038543701172, + "logps/rejected": -2094.147705078125, + "loss": 1.3855, + "margin_dpo/margin_mean": 686.41357421875, + "margin_dpo/margin_std": 1431.6141357421875, + "step": 625 + }, + { + "KL/chosen_KL_mean": -1492.2509765625, + "KL/mean": -1669.962890625, + "KL/rejected_KL_mean": -1847.6748046875, + "KL/std": 1588.1826171875, + "epoch": 0.9192364170337739, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 355.42376708984375, + "fcm_dpo/q_t": 0.49990254640579224, + "grad_norm": 0.11186686158180237, + "learning_rate": 1.0258649316189721e-08, + "logits/chosen": -0.298173189163208, + "logits/rejected": -0.3013492822647095, + "logps/chosen": -1567.363525390625, + "logps/ref_chosen": -75.11260986328125, + "logps/ref_rejected": -99.188720703125, + "logps/rejected": -1946.863525390625, + "loss": 1.3859, + "margin_dpo/margin_mean": 355.4237365722656, + "margin_dpo/margin_std": 1804.6630859375, + "step": 626 + }, + { + "KL/chosen_KL_mean": -888.7783203125, + "KL/mean": -1339.638916015625, + "KL/rejected_KL_mean": -1790.4996337890625, + "KL/std": 1510.39453125, + "epoch": 0.920704845814978, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 901.7212524414062, + "fcm_dpo/q_t": 0.49975284934043884, + "grad_norm": 0.09694371372461319, + "learning_rate": 9.897955805412e-09, + "logits/chosen": -0.27288323640823364, + "logits/rejected": -0.2847945988178253, + "logps/chosen": -936.5214233398438, + "logps/ref_chosen": -47.74314880371094, + "logps/ref_rejected": -106.75448608398438, + "logps/rejected": -1897.254150390625, + "loss": 1.3853, + "margin_dpo/margin_mean": 901.7213134765625, + "margin_dpo/margin_std": 1993.0135498046875, + "step": 627 + }, + { + "KL/chosen_KL_mean": -1294.9599609375, + "KL/mean": -1728.273193359375, + "KL/rejected_KL_mean": -2161.586181640625, + "KL/std": 1660.6864013671875, + "epoch": 0.922173274596182, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 866.6260986328125, + "fcm_dpo/q_t": 0.49976247549057007, + "grad_norm": 0.12284790724515915, + "learning_rate": 9.543589206795238e-09, + "logits/chosen": -0.2784745693206787, + "logits/rejected": -0.2820804715156555, + "logps/chosen": -1355.14306640625, + "logps/ref_chosen": -60.182945251464844, + "logps/ref_rejected": -101.55467224121094, + "logps/rejected": -2263.140869140625, + "loss": 1.3853, + "margin_dpo/margin_mean": 866.6260986328125, + "margin_dpo/margin_std": 2102.763671875, + "step": 628 + }, + { + "KL/chosen_KL_mean": -1483.812255859375, + "KL/mean": -1790.922607421875, + "KL/rejected_KL_mean": -2098.032958984375, + "KL/std": 1915.09033203125, + "epoch": 0.9236417033773862, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 614.2205810546875, + "fcm_dpo/q_t": 0.4998316466808319, + "grad_norm": 0.12040552496910095, + "learning_rate": 9.19555885822887e-09, + "logits/chosen": -0.23021405935287476, + "logits/rejected": -0.23377496004104614, + "logps/chosen": -1548.02587890625, + "logps/ref_chosen": -64.21354675292969, + "logps/ref_rejected": -91.65367126464844, + "logps/rejected": -2189.6865234375, + "loss": 1.3856, + "margin_dpo/margin_mean": 614.2206420898438, + "margin_dpo/margin_std": 2356.16259765625, + "step": 629 + }, + { + "KL/chosen_KL_mean": -1243.9722900390625, + "KL/mean": -1250.4580078125, + "KL/rejected_KL_mean": -1256.94384765625, + "KL/std": 1320.2613525390625, + "epoch": 0.9251101321585903, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 12.971527099609375, + "fcm_dpo/q_t": 0.4999964237213135, + "grad_norm": 0.1596960425376892, + "learning_rate": 8.85387393063622e-09, + "logits/chosen": -0.30787450075149536, + "logits/rejected": -0.30197858810424805, + "logps/chosen": -1303.2633056640625, + "logps/ref_chosen": -59.29100036621094, + "logps/ref_rejected": -83.59829711914062, + "logps/rejected": -1340.5421142578125, + "loss": 1.3863, + "margin_dpo/margin_mean": 12.971527099609375, + "margin_dpo/margin_std": 1790.94384765625, + "step": 630 + }, + { + "KL/chosen_KL_mean": -1294.0426025390625, + "KL/mean": -1551.5064697265625, + "KL/rejected_KL_mean": -1808.97021484375, + "KL/std": 1244.8095703125, + "epoch": 0.9265785609397944, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 514.9276733398438, + "fcm_dpo/q_t": 0.4998588562011719, + "grad_norm": 0.11925707757472992, + "learning_rate": 8.518543427732949e-09, + "logits/chosen": -0.24693992733955383, + "logits/rejected": -0.26080799102783203, + "logps/chosen": -1353.4962158203125, + "logps/ref_chosen": -59.45360565185547, + "logps/ref_rejected": -80.95156860351562, + "logps/rejected": -1889.921875, + "loss": 1.3857, + "margin_dpo/margin_mean": 514.927734375, + "margin_dpo/margin_std": 1695.441162109375, + "step": 631 + }, + { + "KL/chosen_KL_mean": -1142.801513671875, + "KL/mean": -1368.833984375, + "KL/rejected_KL_mean": -1594.8663330078125, + "KL/std": 1164.1339111328125, + "epoch": 0.9280469897209985, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 452.06488037109375, + "fcm_dpo/q_t": 0.49987608194351196, + "grad_norm": 0.08552023023366928, + "learning_rate": 8.189576185789637e-09, + "logits/chosen": -0.25456511974334717, + "logits/rejected": -0.2616008222103119, + "logps/chosen": -1204.153076171875, + "logps/ref_chosen": -61.35155487060547, + "logps/ref_rejected": -86.16017150878906, + "logps/rejected": -1681.0264892578125, + "loss": 1.3858, + "margin_dpo/margin_mean": 452.06488037109375, + "margin_dpo/margin_std": 1714.6295166015625, + "step": 632 + }, + { + "KL/chosen_KL_mean": -1356.769287109375, + "KL/mean": -1573.962158203125, + "KL/rejected_KL_mean": -1791.1549072265625, + "KL/std": 1136.525390625, + "epoch": 0.9295154185022027, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 434.38580322265625, + "fcm_dpo/q_t": 0.49988090991973877, + "grad_norm": 0.10118667781352997, + "learning_rate": 7.866980873399015e-09, + "logits/chosen": -0.26894307136535645, + "logits/rejected": -0.2839438021183014, + "logps/chosen": -1414.04736328125, + "logps/ref_chosen": -57.27816390991211, + "logps/ref_rejected": -91.58395385742188, + "logps/rejected": -1882.7388916015625, + "loss": 1.3858, + "margin_dpo/margin_mean": 434.38580322265625, + "margin_dpo/margin_std": 1371.88671875, + "step": 633 + }, + { + "KL/chosen_KL_mean": -1407.193359375, + "KL/mean": -1582.7567138671875, + "KL/rejected_KL_mean": -1758.320068359375, + "KL/std": 1496.943359375, + "epoch": 0.9309838472834068, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 351.12677001953125, + "fcm_dpo/q_t": 0.49990376830101013, + "grad_norm": 0.10061313956975937, + "learning_rate": 7.550765991247654e-09, + "logits/chosen": -0.312860369682312, + "logits/rejected": -0.32211965322494507, + "logps/chosen": -1473.812255859375, + "logps/ref_chosen": -66.61896514892578, + "logps/ref_rejected": -107.12564849853516, + "logps/rejected": -1865.44580078125, + "loss": 1.3859, + "margin_dpo/margin_mean": 351.1267395019531, + "margin_dpo/margin_std": 1739.1158447265625, + "step": 634 + }, + { + "KL/chosen_KL_mean": -1539.84619140625, + "KL/mean": -1678.4853515625, + "KL/rejected_KL_mean": -1817.12451171875, + "KL/std": 1806.3402099609375, + "epoch": 0.9324522760646109, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 277.2781982421875, + "fcm_dpo/q_t": 0.4999240040779114, + "grad_norm": 0.15843528509140015, + "learning_rate": 7.240939871891699e-09, + "logits/chosen": -0.3303324580192566, + "logits/rejected": -0.31194010376930237, + "logps/chosen": -1613.8017578125, + "logps/ref_chosen": -73.95551300048828, + "logps/ref_rejected": -82.50045776367188, + "logps/rejected": -1899.625, + "loss": 1.386, + "margin_dpo/margin_mean": 277.2781982421875, + "margin_dpo/margin_std": 1970.5750732421875, + "step": 635 + }, + { + "KL/chosen_KL_mean": -1336.476318359375, + "KL/mean": -1476.48828125, + "KL/rejected_KL_mean": -1616.5, + "KL/std": 1605.052734375, + "epoch": 0.933920704845815, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 280.0237731933594, + "fcm_dpo/q_t": 0.4999232590198517, + "grad_norm": 0.10239730030298233, + "learning_rate": 6.937510679537628e-09, + "logits/chosen": -0.3119809031486511, + "logits/rejected": -0.3075520992279053, + "logps/chosen": -1396.105224609375, + "logps/ref_chosen": -59.628910064697266, + "logps/ref_rejected": -81.97883605957031, + "logps/rejected": -1698.4788818359375, + "loss": 1.386, + "margin_dpo/margin_mean": 280.0237731933594, + "margin_dpo/margin_std": 2099.6357421875, + "step": 636 + }, + { + "KL/chosen_KL_mean": -1203.1727294921875, + "KL/mean": -1576.09716796875, + "KL/rejected_KL_mean": -1949.0216064453125, + "KL/std": 1357.664306640625, + "epoch": 0.9353891336270191, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 745.848876953125, + "fcm_dpo/q_t": 0.4997955560684204, + "grad_norm": 0.10817310959100723, + "learning_rate": 6.640486409826785e-09, + "logits/chosen": -0.2595457136631012, + "logits/rejected": -0.27550387382507324, + "logps/chosen": -1252.825439453125, + "logps/ref_chosen": -49.652687072753906, + "logps/ref_rejected": -98.40513610839844, + "logps/rejected": -2047.4267578125, + "loss": 1.3855, + "margin_dpo/margin_mean": 745.848876953125, + "margin_dpo/margin_std": 1527.508056640625, + "step": 637 + }, + { + "KL/chosen_KL_mean": -1121.654052734375, + "KL/mean": -1419.8997802734375, + "KL/rejected_KL_mean": -1718.1455078125, + "KL/std": 1313.08056640625, + "epoch": 0.9368575624082232, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 596.4912719726562, + "fcm_dpo/q_t": 0.4998365044593811, + "grad_norm": 0.0910566970705986, + "learning_rate": 6.349874889624962e-09, + "logits/chosen": -0.28878459334373474, + "logits/rejected": -0.29849740862846375, + "logps/chosen": -1179.810791015625, + "logps/ref_chosen": -58.156639099121094, + "logps/ref_rejected": -79.3014907836914, + "logps/rejected": -1797.4468994140625, + "loss": 1.3856, + "margin_dpo/margin_mean": 596.4913330078125, + "margin_dpo/margin_std": 1834.4677734375, + "step": 638 + }, + { + "KL/chosen_KL_mean": -1723.419189453125, + "KL/mean": -1606.199951171875, + "KL/rejected_KL_mean": -1488.980712890625, + "KL/std": 1386.6025390625, + "epoch": 0.9383259911894273, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -234.43856811523438, + "fcm_dpo/q_t": 0.5000642538070679, + "grad_norm": 0.28992533683776855, + "learning_rate": 6.065683776815933e-09, + "logits/chosen": -0.2773544192314148, + "logits/rejected": -0.27241623401641846, + "logps/chosen": -1795.742431640625, + "logps/ref_chosen": -72.32319641113281, + "logps/ref_rejected": -74.2749252319336, + "logps/rejected": -1563.255615234375, + "loss": 1.3866, + "margin_dpo/margin_mean": -234.4385223388672, + "margin_dpo/margin_std": 1711.879638671875, + "step": 639 + }, + { + "KL/chosen_KL_mean": -1145.3177490234375, + "KL/mean": -1647.8726806640625, + "KL/rejected_KL_mean": -2150.427490234375, + "KL/std": 1739.093505859375, + "epoch": 0.9397944199706314, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1005.10986328125, + "fcm_dpo/q_t": 0.49972450733184814, + "grad_norm": 0.08556817471981049, + "learning_rate": 5.7879205600998296e-09, + "logits/chosen": -0.2978939414024353, + "logits/rejected": -0.31190764904022217, + "logps/chosen": -1201.4521484375, + "logps/ref_chosen": -56.13436508178711, + "logps/ref_rejected": -108.60014343261719, + "logps/rejected": -2259.02783203125, + "loss": 1.3852, + "margin_dpo/margin_mean": 1005.10986328125, + "margin_dpo/margin_std": 2191.3310546875, + "step": 640 + }, + { + "KL/chosen_KL_mean": -1309.43408203125, + "KL/mean": -1710.717529296875, + "KL/rejected_KL_mean": -2112.0009765625, + "KL/std": 1404.828857421875, + "epoch": 0.9412628487518355, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 802.56689453125, + "fcm_dpo/q_t": 0.4997800290584564, + "grad_norm": 0.11998488008975983, + "learning_rate": 5.516592558795746e-09, + "logits/chosen": -0.23249101638793945, + "logits/rejected": -0.23721981048583984, + "logps/chosen": -1374.430908203125, + "logps/ref_chosen": -64.99689483642578, + "logps/ref_rejected": -86.99232482910156, + "logps/rejected": -2198.9931640625, + "loss": 1.3854, + "margin_dpo/margin_mean": 802.56689453125, + "margin_dpo/margin_std": 1926.6470947265625, + "step": 641 + }, + { + "KL/chosen_KL_mean": -1485.472900390625, + "KL/mean": -1839.117919921875, + "KL/rejected_KL_mean": -2192.7626953125, + "KL/std": 1906.6669921875, + "epoch": 0.9427312775330396, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 707.289794921875, + "fcm_dpo/q_t": 0.49980613589286804, + "grad_norm": 0.1159994825720787, + "learning_rate": 5.251706922648868e-09, + "logits/chosen": -0.2811075448989868, + "logits/rejected": -0.293199360370636, + "logps/chosen": -1551.1622314453125, + "logps/ref_chosen": -65.68924713134766, + "logps/ref_rejected": -110.24205017089844, + "logps/rejected": -2303.0048828125, + "loss": 1.3855, + "margin_dpo/margin_mean": 707.289794921875, + "margin_dpo/margin_std": 2484.1240234375, + "step": 642 + }, + { + "KL/chosen_KL_mean": -1354.5499267578125, + "KL/mean": -1558.93896484375, + "KL/rejected_KL_mean": -1763.3280029296875, + "KL/std": 1380.723876953125, + "epoch": 0.9441997063142438, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 408.7780456542969, + "fcm_dpo/q_t": 0.49988794326782227, + "grad_norm": 0.1577404886484146, + "learning_rate": 4.993270631642038e-09, + "logits/chosen": -0.24811697006225586, + "logits/rejected": -0.2620430588722229, + "logps/chosen": -1406.5, + "logps/ref_chosen": -51.94999694824219, + "logps/ref_rejected": -87.46833801269531, + "logps/rejected": -1850.79638671875, + "loss": 1.3858, + "margin_dpo/margin_mean": 408.778076171875, + "margin_dpo/margin_std": 1750.3544921875, + "step": 643 + }, + { + "KL/chosen_KL_mean": -1188.603759765625, + "KL/mean": -1472.237548828125, + "KL/rejected_KL_mean": -1755.87158203125, + "KL/std": 1589.1102294921875, + "epoch": 0.9456681350954479, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 567.2678833007812, + "fcm_dpo/q_t": 0.499844491481781, + "grad_norm": 0.11378132551908493, + "learning_rate": 4.741290495811873e-09, + "logits/chosen": -0.28449851274490356, + "logits/rejected": -0.28798413276672363, + "logps/chosen": -1247.621337890625, + "logps/ref_chosen": -59.017662048339844, + "logps/ref_rejected": -87.13668823242188, + "logps/rejected": -1843.00830078125, + "loss": 1.3857, + "margin_dpo/margin_mean": 567.2678833007812, + "margin_dpo/margin_std": 1886.039306640625, + "step": 644 + }, + { + "KL/chosen_KL_mean": -1202.0780029296875, + "KL/mean": -1344.8292236328125, + "KL/rejected_KL_mean": -1487.580322265625, + "KL/std": 1167.88427734375, + "epoch": 0.947136563876652, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 285.50225830078125, + "fcm_dpo/q_t": 0.4999217391014099, + "grad_norm": 0.09203989803791046, + "learning_rate": 4.495773155069299e-09, + "logits/chosen": -0.27387818694114685, + "logits/rejected": -0.28450918197631836, + "logps/chosen": -1257.9541015625, + "logps/ref_chosen": -55.87602233886719, + "logps/ref_rejected": -97.78080749511719, + "logps/rejected": -1585.361083984375, + "loss": 1.386, + "margin_dpo/margin_mean": 285.50225830078125, + "margin_dpo/margin_std": 1691.6541748046875, + "step": 645 + }, + { + "KL/chosen_KL_mean": -1182.0609130859375, + "KL/mean": -1340.3609619140625, + "KL/rejected_KL_mean": -1498.6610107421875, + "KL/std": 1275.49169921875, + "epoch": 0.9486049926578561, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 316.60015869140625, + "fcm_dpo/q_t": 0.49991321563720703, + "grad_norm": 0.1333022117614746, + "learning_rate": 4.256725079024553e-09, + "logits/chosen": -0.2691464424133301, + "logits/rejected": -0.2712402939796448, + "logps/chosen": -1243.336669921875, + "logps/ref_chosen": -61.275787353515625, + "logps/ref_rejected": -77.50580596923828, + "logps/rejected": -1576.1668701171875, + "loss": 1.3859, + "margin_dpo/margin_mean": 316.6001892089844, + "margin_dpo/margin_std": 1448.23486328125, + "step": 646 + }, + { + "KL/chosen_KL_mean": -1000.0733032226562, + "KL/mean": -1245.947998046875, + "KL/rejected_KL_mean": -1491.822509765625, + "KL/std": 1033.54150390625, + "epoch": 0.9500734214390602, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 491.749267578125, + "fcm_dpo/q_t": 0.49986523389816284, + "grad_norm": 0.15133433043956757, + "learning_rate": 4.024152566816791e-09, + "logits/chosen": -0.29532963037490845, + "logits/rejected": -0.3036816120147705, + "logps/chosen": -1054.92578125, + "logps/ref_chosen": -54.8524169921875, + "logps/ref_rejected": -93.5194091796875, + "logps/rejected": -1585.342041015625, + "loss": 1.3858, + "margin_dpo/margin_mean": 491.74920654296875, + "margin_dpo/margin_std": 1186.4923095703125, + "step": 647 + }, + { + "KL/chosen_KL_mean": -1204.0748291015625, + "KL/mean": -1623.40478515625, + "KL/rejected_KL_mean": -2042.7344970703125, + "KL/std": 1525.134033203125, + "epoch": 0.9515418502202643, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 838.65966796875, + "fcm_dpo/q_t": 0.4997701048851013, + "grad_norm": 0.16558484733104706, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": -0.26186037063598633, + "logits/rejected": -0.2704648971557617, + "logps/chosen": -1258.246337890625, + "logps/ref_chosen": -54.17146682739258, + "logps/ref_rejected": -98.7127914428711, + "logps/rejected": -2141.447265625, + "loss": 1.3854, + "margin_dpo/margin_mean": 838.65966796875, + "margin_dpo/margin_std": 1715.158203125, + "step": 648 + }, + { + "KL/chosen_KL_mean": -1204.949462890625, + "KL/mean": -1460.752685546875, + "KL/rejected_KL_mean": -1716.555908203125, + "KL/std": 1403.10546875, + "epoch": 0.9530102790014684, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 511.6065368652344, + "fcm_dpo/q_t": 0.4998597502708435, + "grad_norm": 0.08931561559438705, + "learning_rate": 3.5784585771215235e-09, + "logits/chosen": -0.28651025891304016, + "logits/rejected": -0.287070095539093, + "logps/chosen": -1267.4296875, + "logps/ref_chosen": -62.480350494384766, + "logps/ref_rejected": -80.07717895507812, + "logps/rejected": -1796.633056640625, + "loss": 1.3857, + "margin_dpo/margin_mean": 511.6065368652344, + "margin_dpo/margin_std": 1865.652099609375, + "step": 649 + }, + { + "KL/chosen_KL_mean": -1396.656494140625, + "KL/mean": -1628.199951171875, + "KL/rejected_KL_mean": -1859.7435302734375, + "KL/std": 1600.5380859375, + "epoch": 0.9544787077826725, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 463.08709716796875, + "fcm_dpo/q_t": 0.4998730719089508, + "grad_norm": 0.07497455924749374, + "learning_rate": 3.3653488440851253e-09, + "logits/chosen": -0.2688780128955841, + "logits/rejected": -0.2748976945877075, + "logps/chosen": -1452.749267578125, + "logps/ref_chosen": -56.09281921386719, + "logps/ref_rejected": -98.26483917236328, + "logps/rejected": -1958.0084228515625, + "loss": 1.3858, + "margin_dpo/margin_mean": 463.08709716796875, + "margin_dpo/margin_std": 2139.24658203125, + "step": 650 + }, + { + "KL/chosen_KL_mean": -969.076171875, + "KL/mean": -1413.8095703125, + "KL/rejected_KL_mean": -1858.5428466796875, + "KL/std": 1293.34228515625, + "epoch": 0.9559471365638766, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 889.4666748046875, + "fcm_dpo/q_t": 0.49975621700286865, + "grad_norm": 0.19173209369182587, + "learning_rate": 3.158738163478475e-09, + "logits/chosen": -0.24564704298973083, + "logits/rejected": -0.2613285779953003, + "logps/chosen": -1012.5015869140625, + "logps/ref_chosen": -43.42544937133789, + "logps/ref_rejected": -99.95791625976562, + "logps/rejected": -1958.500732421875, + "loss": 1.3853, + "margin_dpo/margin_mean": 889.4666748046875, + "margin_dpo/margin_std": 1415.29345703125, + "step": 651 + }, + { + "KL/chosen_KL_mean": -1144.052734375, + "KL/mean": -1469.6400146484375, + "KL/rejected_KL_mean": -1795.227294921875, + "KL/std": 1436.3759765625, + "epoch": 0.9574155653450808, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 651.1746826171875, + "fcm_dpo/q_t": 0.4998215436935425, + "grad_norm": 0.2549162805080414, + "learning_rate": 2.9586319796851555e-09, + "logits/chosen": -0.2820361852645874, + "logits/rejected": -0.2880634665489197, + "logps/chosen": -1206.62939453125, + "logps/ref_chosen": -62.57680892944336, + "logps/ref_rejected": -111.76779174804688, + "logps/rejected": -1906.9951171875, + "loss": 1.3856, + "margin_dpo/margin_mean": 651.1746215820312, + "margin_dpo/margin_std": 1813.627197265625, + "step": 652 + }, + { + "KL/chosen_KL_mean": -1301.207763671875, + "KL/mean": -1669.75732421875, + "KL/rejected_KL_mean": -2038.306884765625, + "KL/std": 1641.0732421875, + "epoch": 0.9588839941262849, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 737.0989990234375, + "fcm_dpo/q_t": 0.4997979998588562, + "grad_norm": 0.15215131640434265, + "learning_rate": 2.7650355656892166e-09, + "logits/chosen": -0.2845923900604248, + "logits/rejected": -0.295174241065979, + "logps/chosen": -1362.32080078125, + "logps/ref_chosen": -61.11295700073242, + "logps/ref_rejected": -103.24960327148438, + "logps/rejected": -2141.556396484375, + "loss": 1.3855, + "margin_dpo/margin_mean": 737.0989990234375, + "margin_dpo/margin_std": 1964.80078125, + "step": 653 + }, + { + "KL/chosen_KL_mean": -1242.6630859375, + "KL/mean": -1478.22998046875, + "KL/rejected_KL_mean": -1713.796875, + "KL/std": 1377.341796875, + "epoch": 0.960352422907489, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 471.13385009765625, + "fcm_dpo/q_t": 0.4998708665370941, + "grad_norm": 0.11739946901798248, + "learning_rate": 2.577954022936174e-09, + "logits/chosen": -0.2545585632324219, + "logits/rejected": -0.2595174312591553, + "logps/chosen": -1304.391357421875, + "logps/ref_chosen": -61.7281379699707, + "logps/ref_rejected": -98.7738037109375, + "logps/rejected": -1812.57080078125, + "loss": 1.3858, + "margin_dpo/margin_mean": 471.13385009765625, + "margin_dpo/margin_std": 1693.6844482421875, + "step": 654 + }, + { + "KL/chosen_KL_mean": -1132.216552734375, + "KL/mean": -1500.26123046875, + "KL/rejected_KL_mean": -1868.305908203125, + "KL/std": 1507.53271484375, + "epoch": 0.9618208516886931, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 736.0894165039062, + "fcm_dpo/q_t": 0.4997982382774353, + "grad_norm": 0.24736790359020233, + "learning_rate": 2.397392281198729e-09, + "logits/chosen": -0.26983144879341125, + "logits/rejected": -0.2772262692451477, + "logps/chosen": -1181.793212890625, + "logps/ref_chosen": -49.576812744140625, + "logps/ref_rejected": -98.29183197021484, + "logps/rejected": -1966.59765625, + "loss": 1.3855, + "margin_dpo/margin_mean": 736.0894165039062, + "margin_dpo/margin_std": 1864.055908203125, + "step": 655 + }, + { + "KL/chosen_KL_mean": -1159.69091796875, + "KL/mean": -1784.111572265625, + "KL/rejected_KL_mean": -2408.5322265625, + "KL/std": 1622.1591796875, + "epoch": 0.9632892804698973, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1248.8411865234375, + "fcm_dpo/q_t": 0.4996577203273773, + "grad_norm": 0.3061199188232422, + "learning_rate": 2.223355098446622e-09, + "logits/chosen": -0.2754252552986145, + "logits/rejected": -0.2866843342781067, + "logps/chosen": -1212.240478515625, + "logps/ref_chosen": -52.54943084716797, + "logps/ref_rejected": -113.67464447021484, + "logps/rejected": -2522.206787109375, + "loss": 1.3849, + "margin_dpo/margin_mean": 1248.8411865234375, + "margin_dpo/margin_std": 1730.600830078125, + "step": 656 + }, + { + "KL/chosen_KL_mean": -1232.510986328125, + "KL/mean": -1685.77783203125, + "KL/rejected_KL_mean": -2139.04443359375, + "KL/std": 1773.5999755859375, + "epoch": 0.9647577092511013, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 906.533203125, + "fcm_dpo/q_t": 0.4997515082359314, + "grad_norm": 0.09144622087478638, + "learning_rate": 2.055847060721566e-09, + "logits/chosen": -0.2707776427268982, + "logits/rejected": -0.29089421033859253, + "logps/chosen": -1279.211669921875, + "logps/ref_chosen": -46.700538635253906, + "logps/ref_rejected": -97.91487121582031, + "logps/rejected": -2236.959228515625, + "loss": 1.3853, + "margin_dpo/margin_mean": 906.533203125, + "margin_dpo/margin_std": 1992.08984375, + "step": 657 + }, + { + "KL/chosen_KL_mean": -1293.7935791015625, + "KL/mean": -1548.7041015625, + "KL/rejected_KL_mean": -1803.614501953125, + "KL/std": 1294.736572265625, + "epoch": 0.9662261380323054, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 509.8209228515625, + "fcm_dpo/q_t": 0.4998602569103241, + "grad_norm": 0.13024263083934784, + "learning_rate": 1.8948725820160662e-09, + "logits/chosen": -0.22683298587799072, + "logits/rejected": -0.23709672689437866, + "logps/chosen": -1354.7518310546875, + "logps/ref_chosen": -60.95820999145508, + "logps/ref_rejected": -95.93949127197266, + "logps/rejected": -1899.553955078125, + "loss": 1.3857, + "margin_dpo/margin_mean": 509.82086181640625, + "margin_dpo/margin_std": 1722.4686279296875, + "step": 658 + }, + { + "KL/chosen_KL_mean": -1185.252197265625, + "KL/mean": -1470.142578125, + "KL/rejected_KL_mean": -1755.032958984375, + "KL/std": 1306.4189453125, + "epoch": 0.9676945668135095, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 569.7805786132812, + "fcm_dpo/q_t": 0.4998438358306885, + "grad_norm": 0.08449683338403702, + "learning_rate": 1.7404359041573723e-09, + "logits/chosen": -0.30121439695358276, + "logits/rejected": -0.29801568388938904, + "logps/chosen": -1261.9952392578125, + "logps/ref_chosen": -76.74298095703125, + "logps/ref_rejected": -87.4709701538086, + "logps/rejected": -1842.50390625, + "loss": 1.3857, + "margin_dpo/margin_mean": 569.7805786132812, + "margin_dpo/margin_std": 1671.21728515625, + "step": 659 + }, + { + "KL/chosen_KL_mean": -1273.4119873046875, + "KL/mean": -1517.3409423828125, + "KL/rejected_KL_mean": -1761.269775390625, + "KL/std": 1257.9273681640625, + "epoch": 0.9691629955947136, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 487.85797119140625, + "fcm_dpo/q_t": 0.4998662769794464, + "grad_norm": 0.15911172330379486, + "learning_rate": 1.592541096695571e-09, + "logits/chosen": -0.246282696723938, + "logits/rejected": -0.2528143525123596, + "logps/chosen": -1332.4599609375, + "logps/ref_chosen": -59.04788589477539, + "logps/ref_rejected": -75.96005249023438, + "logps/rejected": -1837.22998046875, + "loss": 1.3858, + "margin_dpo/margin_mean": 487.8579406738281, + "margin_dpo/margin_std": 1523.2557373046875, + "step": 660 + }, + { + "KL/chosen_KL_mean": -1017.4349365234375, + "KL/mean": -1504.7825927734375, + "KL/rejected_KL_mean": -1992.1302490234375, + "KL/std": 2107.52685546875, + "epoch": 0.9706314243759178, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 974.6953735351562, + "fcm_dpo/q_t": 0.4997328221797943, + "grad_norm": 0.09211470186710358, + "learning_rate": 1.4511920567963908e-09, + "logits/chosen": -0.29294365644454956, + "logits/rejected": -0.3058052062988281, + "logps/chosen": -1068.10888671875, + "logps/ref_chosen": -50.673973083496094, + "logps/ref_rejected": -86.00569152832031, + "logps/rejected": -2078.135986328125, + "loss": 1.3852, + "margin_dpo/margin_mean": 974.6953735351562, + "margin_dpo/margin_std": 2413.55224609375, + "step": 661 + }, + { + "KL/chosen_KL_mean": -1166.93701171875, + "KL/mean": -1460.897705078125, + "KL/rejected_KL_mean": -1754.8583984375, + "KL/std": 1245.419189453125, + "epoch": 0.9720998531571219, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 587.92138671875, + "fcm_dpo/q_t": 0.49983882904052734, + "grad_norm": 0.09338005632162094, + "learning_rate": 1.3163925091384532e-09, + "logits/chosen": -0.301297128200531, + "logits/rejected": -0.2998049855232239, + "logps/chosen": -1236.1981201171875, + "logps/ref_chosen": -69.26106262207031, + "logps/ref_rejected": -89.05593872070312, + "logps/rejected": -1843.914306640625, + "loss": 1.3857, + "margin_dpo/margin_mean": 587.92138671875, + "margin_dpo/margin_std": 1546.5623779296875, + "step": 662 + }, + { + "KL/chosen_KL_mean": -1115.79345703125, + "KL/mean": -1571.531005859375, + "KL/rejected_KL_mean": -2027.2685546875, + "KL/std": 1618.50732421875, + "epoch": 0.973568281938326, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 911.47509765625, + "fcm_dpo/q_t": 0.49975016713142395, + "grad_norm": 0.11960398405790329, + "learning_rate": 1.1881460058152382e-09, + "logits/chosen": -0.33110612630844116, + "logits/rejected": -0.3475431799888611, + "logps/chosen": -1180.67236328125, + "logps/ref_chosen": -64.87890625, + "logps/ref_rejected": -113.92536926269531, + "logps/rejected": -2141.19384765625, + "loss": 1.3853, + "margin_dpo/margin_mean": 911.47509765625, + "margin_dpo/margin_std": 2104.358154296875, + "step": 663 + }, + { + "KL/chosen_KL_mean": -1458.552978515625, + "KL/mean": -1998.9429931640625, + "KL/rejected_KL_mean": -2539.3330078125, + "KL/std": 2229.985595703125, + "epoch": 0.9750367107195301, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1080.7802734375, + "fcm_dpo/q_t": 0.4997037649154663, + "grad_norm": 0.11196719855070114, + "learning_rate": 1.066455926241383e-09, + "logits/chosen": -0.2733624577522278, + "logits/rejected": -0.2783682346343994, + "logps/chosen": -1519.44140625, + "logps/ref_chosen": -60.88847351074219, + "logps/ref_rejected": -105.521728515625, + "logps/rejected": -2644.85498046875, + "loss": 1.3851, + "margin_dpo/margin_mean": 1080.7801513671875, + "margin_dpo/margin_std": 2739.387939453125, + "step": 664 + }, + { + "KL/chosen_KL_mean": -1209.31884765625, + "KL/mean": -1550.11328125, + "KL/rejected_KL_mean": -1890.90771484375, + "KL/std": 1507.995361328125, + "epoch": 0.9765051395007343, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 681.5887451171875, + "fcm_dpo/q_t": 0.4998131990432739, + "grad_norm": 0.12111763656139374, + "learning_rate": 9.513254770636137e-10, + "logits/chosen": -0.19828855991363525, + "logits/rejected": -0.2052626609802246, + "logps/chosen": -1269.883056640625, + "logps/ref_chosen": -60.56413269042969, + "logps/ref_rejected": -84.80882263183594, + "logps/rejected": -1975.716552734375, + "loss": 1.3855, + "margin_dpo/margin_mean": 681.5887451171875, + "margin_dpo/margin_std": 1712.932373046875, + "step": 665 + }, + { + "KL/chosen_KL_mean": -1191.26708984375, + "KL/mean": -1576.1512451171875, + "KL/rejected_KL_mean": -1961.035400390625, + "KL/std": 1501.64208984375, + "epoch": 0.9779735682819384, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 769.7685546875, + "fcm_dpo/q_t": 0.4997889995574951, + "grad_norm": 0.10389173030853271, + "learning_rate": 8.427576920763956e-10, + "logits/chosen": -0.2726587951183319, + "logits/rejected": -0.27047300338745117, + "logps/chosen": -1255.68701171875, + "logps/ref_chosen": -64.41996002197266, + "logps/ref_rejected": -95.8916244506836, + "logps/rejected": -2056.92724609375, + "loss": 1.3855, + "margin_dpo/margin_mean": 769.7685546875, + "margin_dpo/margin_std": 1734.6624755859375, + "step": 666 + }, + { + "KL/chosen_KL_mean": -1549.532958984375, + "KL/mean": -1783.92333984375, + "KL/rejected_KL_mean": -2018.3138427734375, + "KL/std": 1649.5316162109375, + "epoch": 0.9794419970631424, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 468.78094482421875, + "fcm_dpo/q_t": 0.49987149238586426, + "grad_norm": 0.2005223035812378, + "learning_rate": 7.407554321417764e-10, + "logits/chosen": -0.289089560508728, + "logits/rejected": -0.286052405834198, + "logps/chosen": -1618.81005859375, + "logps/ref_chosen": -69.27702331542969, + "logps/ref_rejected": -87.83549499511719, + "logps/rejected": -2106.1494140625, + "loss": 1.3858, + "margin_dpo/margin_mean": 468.78094482421875, + "margin_dpo/margin_std": 1956.940185546875, + "step": 667 + }, + { + "KL/chosen_KL_mean": -1474.34619140625, + "KL/mean": -1627.808349609375, + "KL/rejected_KL_mean": -1781.2705078125, + "KL/std": 1383.871826171875, + "epoch": 0.9809104258443465, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 306.92425537109375, + "fcm_dpo/q_t": 0.4999158978462219, + "grad_norm": 0.3075452744960785, + "learning_rate": 6.453213851142225e-10, + "logits/chosen": -0.29430443048477173, + "logits/rejected": -0.30596286058425903, + "logps/chosen": -1546.9501953125, + "logps/ref_chosen": -72.60400390625, + "logps/ref_rejected": -103.73905944824219, + "logps/rejected": -1885.009521484375, + "loss": 1.386, + "margin_dpo/margin_mean": 306.92425537109375, + "margin_dpo/margin_std": 1629.07177734375, + "step": 668 + }, + { + "KL/chosen_KL_mean": -1009.0479125976562, + "KL/mean": -1348.203857421875, + "KL/rejected_KL_mean": -1687.359619140625, + "KL/std": 1146.410888671875, + "epoch": 0.9823788546255506, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 678.311767578125, + "fcm_dpo/q_t": 0.49981409311294556, + "grad_norm": 0.06971733272075653, + "learning_rate": 5.564580657695939e-10, + "logits/chosen": -0.25400370359420776, + "logits/rejected": -0.2593185305595398, + "logps/chosen": -1055.164306640625, + "logps/ref_chosen": -46.116416931152344, + "logps/ref_rejected": -77.92434692382812, + "logps/rejected": -1765.283935546875, + "loss": 1.3856, + "margin_dpo/margin_mean": 678.311767578125, + "margin_dpo/margin_std": 1458.8255615234375, + "step": 669 + }, + { + "KL/chosen_KL_mean": -1203.6142578125, + "KL/mean": -1688.494384765625, + "KL/rejected_KL_mean": -2173.37451171875, + "KL/std": 1796.819091796875, + "epoch": 0.9838472834067548, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 969.760009765625, + "fcm_dpo/q_t": 0.49973419308662415, + "grad_norm": 0.1595345139503479, + "learning_rate": 4.741678157389739e-10, + "logits/chosen": -0.24518835544586182, + "logits/rejected": -0.2584257423877716, + "logps/chosen": -1265.9600830078125, + "logps/ref_chosen": -62.34575271606445, + "logps/ref_rejected": -96.9405517578125, + "logps/rejected": -2270.31494140625, + "loss": 1.3852, + "margin_dpo/margin_mean": 969.760009765625, + "margin_dpo/margin_std": 2220.41748046875, + "step": 670 + }, + { + "KL/chosen_KL_mean": -1205.3656005859375, + "KL/mean": -1510.864501953125, + "KL/rejected_KL_mean": -1816.3634033203125, + "KL/std": 1332.0640869140625, + "epoch": 0.9853157121879589, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 610.9979248046875, + "fcm_dpo/q_t": 0.49983251094818115, + "grad_norm": 0.06985840201377869, + "learning_rate": 3.9845280344705245e-10, + "logits/chosen": -0.2608600854873657, + "logits/rejected": -0.27443787455558777, + "logps/chosen": -1253.36572265625, + "logps/ref_chosen": -48.00010681152344, + "logps/ref_rejected": -83.81932067871094, + "logps/rejected": -1900.1827392578125, + "loss": 1.3856, + "margin_dpo/margin_mean": 610.9979248046875, + "margin_dpo/margin_std": 1832.479736328125, + "step": 671 + }, + { + "KL/chosen_KL_mean": -1534.4940185546875, + "KL/mean": -1729.3173828125, + "KL/rejected_KL_mean": -1924.140625, + "KL/std": 1462.22021484375, + "epoch": 0.986784140969163, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 389.6468505859375, + "fcm_dpo/q_t": 0.4998931884765625, + "grad_norm": 0.23390474915504456, + "learning_rate": 3.293150240547549e-10, + "logits/chosen": -0.2893441915512085, + "logits/rejected": -0.2950705885887146, + "logps/chosen": -1593.0772705078125, + "logps/ref_chosen": -58.58328628540039, + "logps/ref_rejected": -93.14015197753906, + "logps/rejected": -2017.2808837890625, + "loss": 1.3859, + "margin_dpo/margin_mean": 389.6468200683594, + "margin_dpo/margin_std": 1572.853515625, + "step": 672 + }, + { + "KL/chosen_KL_mean": -1143.24072265625, + "KL/mean": -1430.210693359375, + "KL/rejected_KL_mean": -1717.1806640625, + "KL/std": 1203.540771484375, + "epoch": 0.9882525697503671, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 573.93994140625, + "fcm_dpo/q_t": 0.49984270334243774, + "grad_norm": 0.09268736839294434, + "learning_rate": 2.6675629940689504e-10, + "logits/chosen": -0.2500056326389313, + "logits/rejected": -0.25664058327674866, + "logps/chosen": -1189.9638671875, + "logps/ref_chosen": -46.72320556640625, + "logps/ref_rejected": -85.29623413085938, + "logps/rejected": -1802.476806640625, + "loss": 1.3857, + "margin_dpo/margin_mean": 573.93994140625, + "margin_dpo/margin_std": 1410.111083984375, + "step": 673 + }, + { + "KL/chosen_KL_mean": -984.7044067382812, + "KL/mean": -1303.4468994140625, + "KL/rejected_KL_mean": -1622.189453125, + "KL/std": 1034.3787841796875, + "epoch": 0.9897209985315712, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 637.4850463867188, + "fcm_dpo/q_t": 0.49982526898384094, + "grad_norm": 0.0636168122291565, + "learning_rate": 2.1077827798404725e-10, + "logits/chosen": -0.24484899640083313, + "logits/rejected": -0.24942630529403687, + "logps/chosen": -1030.14990234375, + "logps/ref_chosen": -45.445526123046875, + "logps/ref_rejected": -70.04593658447266, + "logps/rejected": -1692.2353515625, + "loss": 1.3856, + "margin_dpo/margin_mean": 637.4850463867188, + "margin_dpo/margin_std": 1280.18359375, + "step": 674 + }, + { + "KL/chosen_KL_mean": -1180.5653076171875, + "KL/mean": -1411.843994140625, + "KL/rejected_KL_mean": -1643.12255859375, + "KL/std": 1438.281005859375, + "epoch": 0.9911894273127754, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 462.5572509765625, + "fcm_dpo/q_t": 0.49987322092056274, + "grad_norm": 0.12069758027791977, + "learning_rate": 1.6138243485910863e-10, + "logits/chosen": -0.2583056390285492, + "logits/rejected": -0.26913708448410034, + "logps/chosen": -1224.7415771484375, + "logps/ref_chosen": -44.17628479003906, + "logps/ref_rejected": -74.09197998046875, + "logps/rejected": -1717.214599609375, + "loss": 1.3858, + "margin_dpo/margin_mean": 462.5572509765625, + "margin_dpo/margin_std": 1866.17919921875, + "step": 675 + }, + { + "KL/chosen_KL_mean": -1252.801025390625, + "KL/mean": -1582.269287109375, + "KL/rejected_KL_mean": -1911.7374267578125, + "KL/std": 1282.24658203125, + "epoch": 0.9926578560939795, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 658.9365234375, + "fcm_dpo/q_t": 0.49981939792633057, + "grad_norm": 0.0914895310997963, + "learning_rate": 1.1857007165852472e-10, + "logits/chosen": -0.28977373242378235, + "logits/rejected": -0.3063029944896698, + "logps/chosen": -1324.199462890625, + "logps/ref_chosen": -71.39852905273438, + "logps/ref_rejected": -88.3587646484375, + "logps/rejected": -2000.09619140625, + "loss": 1.3856, + "margin_dpo/margin_mean": 658.9365234375, + "margin_dpo/margin_std": 1711.732666015625, + "step": 676 + }, + { + "KL/chosen_KL_mean": -1265.82275390625, + "KL/mean": -1535.177734375, + "KL/rejected_KL_mean": -1804.532958984375, + "KL/std": 1238.73095703125, + "epoch": 0.9941262848751835, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 538.710205078125, + "fcm_dpo/q_t": 0.49985232949256897, + "grad_norm": 0.1667097508907318, + "learning_rate": 8.23423165278725e-11, + "logits/chosen": -0.23336629569530487, + "logits/rejected": -0.24292391538619995, + "logps/chosen": -1322.35009765625, + "logps/ref_chosen": -56.527435302734375, + "logps/ref_rejected": -78.22654724121094, + "logps/rejected": -1882.7593994140625, + "loss": 1.3857, + "margin_dpo/margin_mean": 538.710205078125, + "margin_dpo/margin_std": 1765.0107421875, + "step": 677 + }, + { + "KL/chosen_KL_mean": -927.6842041015625, + "KL/mean": -1416.5367431640625, + "KL/rejected_KL_mean": -1905.38916015625, + "KL/std": 1505.28564453125, + "epoch": 0.9955947136563876, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 977.7049560546875, + "fcm_dpo/q_t": 0.49973201751708984, + "grad_norm": 0.19531825184822083, + "learning_rate": 5.270012410216185e-11, + "logits/chosen": -0.2596043646335602, + "logits/rejected": -0.2691473960876465, + "logps/chosen": -973.8187255859375, + "logps/ref_chosen": -46.13447570800781, + "logps/ref_rejected": -80.60462951660156, + "logps/rejected": -1985.9937744140625, + "loss": 1.3852, + "margin_dpo/margin_mean": 977.7049560546875, + "margin_dpo/margin_std": 1734.43798828125, + "step": 678 + }, + { + "KL/chosen_KL_mean": -1160.0855712890625, + "KL/mean": -1379.29541015625, + "KL/rejected_KL_mean": -1598.50537109375, + "KL/std": 1232.62939453125, + "epoch": 0.9970631424375918, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 438.4197998046875, + "fcm_dpo/q_t": 0.4998798370361328, + "grad_norm": 0.0700586810708046, + "learning_rate": 2.9644275480772416e-11, + "logits/chosen": -0.26852041482925415, + "logits/rejected": -0.2719077169895172, + "logps/chosen": -1210.3804931640625, + "logps/ref_chosen": -50.294921875, + "logps/ref_rejected": -76.59813690185547, + "logps/rejected": -1675.103515625, + "loss": 1.3858, + "margin_dpo/margin_mean": 438.4197998046875, + "margin_dpo/margin_std": 1638.0318603515625, + "step": 679 + }, + { + "KL/chosen_KL_mean": -1294.1790771484375, + "KL/mean": -1602.512939453125, + "KL/rejected_KL_mean": -1910.8466796875, + "KL/std": 1501.509033203125, + "epoch": 0.9985315712187959, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 616.6676025390625, + "fcm_dpo/q_t": 0.499830961227417, + "grad_norm": 0.09760192036628723, + "learning_rate": 1.31753782067201e-11, + "logits/chosen": -0.28134095668792725, + "logits/rejected": -0.28399893641471863, + "logps/chosen": -1371.0947265625, + "logps/ref_chosen": -76.91569519042969, + "logps/ref_rejected": -112.384765625, + "logps/rejected": -2023.2314453125, + "loss": 1.3856, + "margin_dpo/margin_mean": 616.667724609375, + "margin_dpo/margin_std": 1909.36962890625, + "step": 680 + }, + { + "KL/chosen_KL_mean": -1360.1488037109375, + "KL/mean": -1591.733642578125, + "KL/rejected_KL_mean": -1823.318603515625, + "KL/std": 1528.53564453125, + "epoch": 1.0, + "fcm_dpo/beta": 1.0963610748149222e-06, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 463.16986083984375, + "fcm_dpo/q_t": 0.4998730719089508, + "grad_norm": 0.14217820763587952, + "learning_rate": 3.2938662507808745e-12, + "logits/chosen": -0.27003633975982666, + "logits/rejected": -0.27188071608543396, + "logps/chosen": -1421.1060791015625, + "logps/ref_chosen": -60.957279205322266, + "logps/ref_rejected": -88.55797576904297, + "logps/rejected": -1911.8765869140625, + "loss": 1.3858, + "margin_dpo/margin_mean": 463.1697998046875, + "margin_dpo/margin_std": 2003.16552734375, + "step": 681 + }, + { + "epoch": 1.0, + "step": 681, + "total_flos": 0.0, + "train_loss": 1.356521034380763, + "train_runtime": 1742.8549, + "train_samples_per_second": 25.015, + "train_steps_per_second": 0.391 + } + ], + "logging_steps": 1, + "max_steps": 681, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}